diff --git a/include/crm/pengine/pe_types.h b/include/crm/pengine/pe_types.h
index 260bfbbac7..843cf869d3 100644
--- a/include/crm/pengine/pe_types.h
+++ b/include/crm/pengine/pe_types.h
@@ -1,528 +1,525 @@
 /*
  * Copyright 2004-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef PCMK__CRM_PENGINE_PE_TYPES__H
 #  define PCMK__CRM_PENGINE_PE_TYPES__H
 
 
 #  include <stdbool.h>              // bool
 #  include <sys/types.h>            // time_t
 #  include <libxml/tree.h>          // xmlNode
 #  include <glib.h>                 // gboolean, guint, GList, GHashTable
 #  include <crm/common/iso8601.h>
 #  include <crm/common/scheduler.h>
 #  include <crm/pengine/common.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /*!
  * \file
  * \brief Data types for cluster status
  * \ingroup pengine
  */
 
 typedef struct pe_node_s pe_node_t;
 typedef struct pe_action_s pe_action_t;
 typedef struct pe_resource_s pe_resource_t;
 typedef struct pe_working_set_s pe_working_set_t;
 
 typedef struct resource_object_functions_s {
     gboolean (*unpack) (pe_resource_t*, pe_working_set_t*);
     pe_resource_t *(*find_rsc) (pe_resource_t *parent, const char *search,
                                 const pe_node_t *node, int flags);
     /* parameter result must be free'd */
     char *(*parameter) (pe_resource_t*, pe_node_t*, gboolean, const char*,
                         pe_working_set_t*);
     //! \deprecated will be removed in a future release
     void (*print) (pe_resource_t*, const char*, long, void*);
     gboolean (*active) (pe_resource_t*, gboolean);
     enum rsc_role_e (*state) (const pe_resource_t*, gboolean);
     pe_node_t *(*location) (const pe_resource_t*, GList**, int);
     void (*free) (pe_resource_t*);
     void (*count) (pe_resource_t*);
     gboolean (*is_filtered) (const pe_resource_t*, GList *, gboolean);
 
     /*!
      * \brief Find a node (and optionally count all) where resource is active
      *
      * \param[in]  rsc          Resource to check
      * \param[out] count_all    If not NULL, set this to count of active nodes
      * \param[out] count_clean  If not NULL, set this to count of clean nodes
      *
      * \return A node where the resource is active, preferring the source node
      *         if the resource is involved in a partial migration or a clean,
      *         online node if the resource's "requires" is "quorum" or
      *         "nothing", or NULL if the resource is inactive.
      */
     pe_node_t *(*active_node)(const pe_resource_t *rsc, unsigned int *count_all,
                               unsigned int *count_clean);
 
     /*!
      * \brief Get maximum resource instances per node
      *
      * \param[in] rsc  Resource to check
      *
      * \return Maximum number of \p rsc instances that can be active on one node
      */
     unsigned int (*max_per_node)(const pe_resource_t *rsc);
 } resource_object_functions_t;
 
 typedef struct resource_alloc_functions_s resource_alloc_functions_t;
 
-// @TODO Make these an enum
-
-#  define pe_flag_have_quorum           0x00000001ULL
 #  define pe_flag_symmetric_cluster     0x00000002ULL
 #  define pe_flag_maintenance_mode      0x00000008ULL
 
 #  define pe_flag_stonith_enabled       0x00000010ULL
 #  define pe_flag_have_stonith_resource 0x00000020ULL
 #  define pe_flag_enable_unfencing      0x00000040ULL
 #  define pe_flag_concurrent_fencing    0x00000080ULL
 
 #  define pe_flag_stop_rsc_orphans      0x00000100ULL
 #  define pe_flag_stop_action_orphans   0x00000200ULL
 #  define pe_flag_stop_everything       0x00000400ULL
 
 #  define pe_flag_start_failure_fatal   0x00001000ULL
 
 //! \deprecated
 #  define pe_flag_remove_after_stop     0x00002000ULL
 
 #  define pe_flag_startup_fencing       0x00004000ULL
 #  define pe_flag_shutdown_lock         0x00008000ULL
 
 #  define pe_flag_startup_probes        0x00010000ULL
 #  define pe_flag_have_status           0x00020000ULL
 #  define pe_flag_have_remote_nodes     0x00040000ULL
 
 #  define pe_flag_quick_location        0x00100000ULL
 #  define pe_flag_sanitized             0x00200000ULL
 
 //! \deprecated
 #  define pe_flag_stdout                0x00400000ULL
 
 //! Don't count total, disabled and blocked resource instances
 #  define pe_flag_no_counts             0x00800000ULL
 
 /*! Skip deprecated code that is kept solely for backward API compatibility.
  * (Internal code should always set this.)
  */
 #  define pe_flag_no_compat             0x01000000ULL
 
 #  define pe_flag_show_scores           0x02000000ULL
 #  define pe_flag_show_utilization      0x04000000ULL
 
 /*!
  * When scheduling, only unpack the CIB (including constraints), calculate
  * as much cluster status as possible, and apply node health.
  */
 #  define pe_flag_check_config          0x08000000ULL
 
 struct pe_working_set_s {
     xmlNode *input;
     crm_time_t *now;
 
     /* options extracted from the input */
     char *dc_uuid;
     pe_node_t *dc_node;
     const char *stonith_action;
     const char *placement_strategy;
 
     unsigned long long flags;
 
     int stonith_timeout;
     enum pe_quorum_policy no_quorum_policy;
 
     GHashTable *config_hash;
     GHashTable *tickets;
 
     // Actions for which there can be only one (e.g. fence nodeX)
     GHashTable *singletons;
 
     GList *nodes;
     GList *resources;
     GList *placement_constraints;
     GList *ordering_constraints;
     GList *colocation_constraints;
     GList *ticket_constraints;
 
     GList *actions;
     xmlNode *failed;
     xmlNode *op_defaults;
     xmlNode *rsc_defaults;
 
     /* stats */
     int num_synapse;
     int max_valid_nodes;    //! Deprecated (will be removed in a future release)
     int order_id;
     int action_id;
 
     /* final output */
     xmlNode *graph;
 
     GHashTable *template_rsc_sets;
     const char *localhost;
     GHashTable *tags;
 
     int blocked_resources;
     int disabled_resources;
 
     GList *param_check; // History entries that need to be checked
     GList *stop_needed; // Containers that need stop actions
     time_t recheck_by;  // Hint to controller to re-run scheduler by this time
     int ninstances;     // Total number of resource instances
     guint shutdown_lock;// How long (seconds) to lock resources to shutdown node
     int priority_fencing_delay; // Priority fencing delay
 
     void *priv;
     guint node_pending_timeout; // Node pending timeout
 };
 
 struct pe_node_shared_s {
     const char *id;
     const char *uname;
     enum node_type type;
 
     /* @TODO convert these flags into a bitfield */
     gboolean online;
     gboolean standby;
     gboolean standby_onfail;
     gboolean pending;
     gboolean unclean;
     gboolean unseen;
     gboolean shutdown;
     gboolean expected_up;
     gboolean is_dc;
     gboolean maintenance;
     gboolean rsc_discovery_enabled;
     gboolean remote_requires_reset;
     gboolean remote_was_fenced;
     gboolean remote_maintenance; /* what the remote-rsc is thinking */
     gboolean unpacked;
 
     int num_resources;
     pe_resource_t *remote_rsc;
     GList *running_rsc;       /* pe_resource_t* */
     GList *allocated_rsc;     /* pe_resource_t* */
 
     GHashTable *attrs;          /* char* => char* */
     GHashTable *utilization;
     GHashTable *digest_cache;   //!< cache of calculated resource digests
     int priority; // calculated based on the priority of resources running on the node
     pe_working_set_t *data_set; //!< Cluster that this node is part of
 };
 
 struct pe_node_s {
     int weight;
     gboolean fixed; //!< \deprecated Will be removed in a future release
     int count;
     struct pe_node_shared_s *details;
     int rsc_discover_mode;
 };
 
 #  define pe_rsc_orphan                     0x00000001ULL
 #  define pe_rsc_managed                    0x00000002ULL
 #  define pe_rsc_block                      0x00000004ULL
 #  define pe_rsc_orphan_container_filler    0x00000008ULL
 
 #  define pe_rsc_notify                     0x00000010ULL
 #  define pe_rsc_unique                     0x00000020ULL
 #  define pe_rsc_fence_device               0x00000040ULL
 #  define pe_rsc_promotable                 0x00000080ULL
 
 #  define pe_rsc_provisional                0x00000100ULL
 #  define pe_rsc_allocating                 0x00000200ULL
 #  define pe_rsc_merging                    0x00000400ULL
 #  define pe_rsc_restarting                 0x00000800ULL
 
 #  define pe_rsc_stop                       0x00001000ULL
 #  define pe_rsc_reload                     0x00002000ULL
 #  define pe_rsc_allow_remote_remotes       0x00004000ULL
 #  define pe_rsc_critical                   0x00008000ULL
 
 #  define pe_rsc_failed                     0x00010000ULL
 #  define pe_rsc_detect_loop                0x00020000ULL
 #  define pe_rsc_runnable                   0x00040000ULL
 #  define pe_rsc_start_pending              0x00080000ULL
 
 //!< \deprecated Do not use
 #  define pe_rsc_starting                   0x00100000ULL
 
 //!< \deprecated Do not use
 #  define pe_rsc_stopping                   0x00200000ULL
 
 #  define pe_rsc_stop_unexpected            0x00400000ULL
 #  define pe_rsc_allow_migrate              0x00800000ULL
 
 #  define pe_rsc_failure_ignored            0x01000000ULL
 #  define pe_rsc_replica_container          0x02000000ULL
 #  define pe_rsc_maintenance                0x04000000ULL
 #  define pe_rsc_is_container               0x08000000ULL
 
 #  define pe_rsc_needs_quorum               0x10000000ULL
 #  define pe_rsc_needs_fencing              0x20000000ULL
 #  define pe_rsc_needs_unfencing            0x40000000ULL
 
 /* *INDENT-OFF* */
 enum pe_action_flags {
     pe_action_pseudo = 0x00001,
     pe_action_runnable = 0x00002,
     pe_action_optional = 0x00004,
     pe_action_print_always = 0x00008,
 
     pe_action_have_node_attrs = 0x00010,
     pe_action_implied_by_stonith = 0x00040,
     pe_action_migrate_runnable =   0x00080,
 
     pe_action_dumped = 0x00100,
     pe_action_processed = 0x00200,
 #if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
     pe_action_clear = 0x00400, //! \deprecated Unused
 #endif
     pe_action_dangle = 0x00800,
 
     /* This action requires one or more of its dependencies to be runnable.
      * We use this to clear the runnable flag before checking dependencies.
      */
     pe_action_requires_any = 0x01000,
 
     pe_action_reschedule = 0x02000,
     pe_action_tracking = 0x04000,
     pe_action_dedup = 0x08000, //! Internal state tracking when creating graph
 
     pe_action_dc = 0x10000,         //! Action may run on DC instead of target
 };
 /* *INDENT-ON* */
 
 struct pe_resource_s {
     char *id;
     char *clone_name;
     xmlNode *xml;
     xmlNode *orig_xml;
     xmlNode *ops_xml;
 
     pe_working_set_t *cluster;
     pe_resource_t *parent;
 
     enum pe_obj_types variant;
     void *variant_opaque;
     resource_object_functions_t *fns;
     resource_alloc_functions_t *cmds;
 
     enum rsc_recovery_type recovery_type;
 
     enum pe_restart restart_type; //!< \deprecated will be removed in future release
 
     int priority;
     int stickiness;
     int sort_index;
     int failure_timeout;
     int migration_threshold;
     guint remote_reconnect_ms;
     char *pending_task;
 
     unsigned long long flags;
 
     // @TODO merge these into flags
     gboolean is_remote_node;
     gboolean exclusive_discover;
 
     /* Pay special attention to whether you want to use rsc_cons_lhs and
      * rsc_cons directly, which include only colocations explicitly involving
      * this resource, or call libpacemaker's pcmk__with_this_colocations() and
      * pcmk__this_with_colocations() functions, which may return relevant
      * colocations involving the resource's ancestors as well.
      */
 
     //!@{
     //! This field should be treated as internal to Pacemaker
     GList *rsc_cons_lhs;      // List of pcmk__colocation_t*
     GList *rsc_cons;          // List of pcmk__colocation_t*
     GList *rsc_location;      // List of pe__location_t*
     GList *actions;           // List of pe_action_t*
     GList *rsc_tickets;       // List of rsc_ticket*
     //!@}
 
     pe_node_t *allocated_to;
     pe_node_t *partial_migration_target;
     pe_node_t *partial_migration_source;
     GList *running_on;        /* pe_node_t*   */
     GHashTable *known_on;       /* pe_node_t*   */
     GHashTable *allowed_nodes;  /* pe_node_t*   */
 
     enum rsc_role_e role;
     enum rsc_role_e next_role;
 
     GHashTable *meta;
     GHashTable *parameters; //! \deprecated Use pe_rsc_params() instead
     GHashTable *utilization;
 
     GList *children;          /* pe_resource_t*   */
     GList *dangling_migrations;       /* pe_node_t*       */
 
     pe_resource_t *container;
     GList *fillers;
 
     // @COMPAT These should be made const at next API compatibility break
     pe_node_t *pending_node;    // Node on which pending_task is happening
     pe_node_t *lock_node;       // Resource is shutdown-locked to this node
 
     time_t lock_time;           // When shutdown lock started
 
     /* Resource parameters may have node-attribute-based rules, which means the
      * values can vary by node. This table is a cache of parameter name/value
      * tables for each node (as needed). Use pe_rsc_params() to get the table
      * for a given node.
      */
     GHashTable *parameter_cache; // Key = node name, value = parameters table
 };
 
 struct pe_action_s {
     int id;
     int priority;
 
     pe_resource_t *rsc;
     pe_node_t *node;
     xmlNode *op_entry;
 
     char *task;
     char *uuid;
     char *cancel_task;
     char *reason;
 
     enum pe_action_flags flags;
     enum rsc_start_requirement needs;
     enum action_fail_response on_fail;
     enum rsc_role_e fail_role;
 
     GHashTable *meta;
     GHashTable *extra;
 
     /* 
      * These two varables are associated with the constraint logic
      * that involves first having one or more actions runnable before
      * then allowing this action to execute.
      *
      * These varables are used with features such as 'clone-min' which
      * requires at minimum X number of cloned instances to be running
      * before an order dependency can run. Another option that uses
      * this is 'require-all=false' in ordering constrants. This option
      * says "only require one instance of a resource to start before
      * allowing dependencies to start" -- basically, require-all=false is
      * the same as clone-min=1.
      */
 
     /* current number of known runnable actions in the before list. */
     int runnable_before;
     /* the number of "before" runnable actions required for this action
      * to be considered runnable */ 
     int required_runnable_before;
 
     GList *actions_before;    /* pe_action_wrapper_t* */
     GList *actions_after;     /* pe_action_wrapper_t* */
 
     /* Some of the above fields could be moved to the details,
      * except for API backward compatibility.
      */
     void *action_details; // varies by type of action
 };
 
 typedef struct pe_ticket_s {
     char *id;
     gboolean granted;
     time_t last_granted;
     gboolean standby;
     GHashTable *state;
 } pe_ticket_t;
 
 typedef struct pe_tag_s {
     char *id;
     GList *refs;
 } pe_tag_t;
 
 //! Internal tracking for transition graph creation
 enum pe_link_state {
     pe_link_not_dumped, //! Internal tracking for transition graph creation
     pe_link_dumped,     //! Internal tracking for transition graph creation
     pe_link_dup,        //! \deprecated No longer used by Pacemaker
 };
 
 enum pe_discover_e {
     pe_discover_always = 0,
     pe_discover_never,
     pe_discover_exclusive,
 };
 
 /* *INDENT-OFF* */
 enum pe_ordering {
     pe_order_none                  = 0x0,       /* deleted */
     pe_order_optional              = 0x1,       /* pure ordering, nothing implied */
     pe_order_apply_first_non_migratable = 0x2,  /* Only apply this constraint's ordering if first is not migratable. */
 
     pe_order_implies_first         = 0x10,      /* If 'then' is required, ensure 'first' is too */
     pe_order_implies_then          = 0x20,      /* If 'first' is required, ensure 'then' is too */
     pe_order_promoted_implies_first = 0x40,     /* If 'then' is required and then's rsc is promoted, ensure 'first' becomes required too */
 
     /* first requires then to be both runnable and migrate runnable. */
     pe_order_implies_first_migratable  = 0x80,
 
     pe_order_runnable_left         = 0x100,     /* 'then' requires 'first' to be runnable */
 
     pe_order_pseudo_left           = 0x200,     /* 'then' can only be pseudo if 'first' is runnable */
     pe_order_implies_then_on_node  = 0x400,     /* If 'first' is required on 'nodeX',
                                                  * ensure instances of 'then' on 'nodeX' are too.
                                                  * Only really useful if 'then' is a clone and 'first' is not
                                                  */
     pe_order_probe                 = 0x800,     /* If 'first->rsc' is
                                                  *  - running but about to stop, ignore the constraint
                                                  *  - otherwise, behave as runnable_left
                                                  */
 
     pe_order_restart               = 0x1000,    /* 'then' is runnable if 'first' is optional or runnable */
     pe_order_stonith_stop          = 0x2000,    //<! \deprecated Will be removed in future release
     pe_order_serialize_only        = 0x4000,    /* serialize */
     pe_order_same_node             = 0x8000,    /* applies only if 'first' and 'then' are on same node */
 
     pe_order_implies_first_printed = 0x10000,   /* Like ..implies_first but only ensures 'first' is printed, not mandatory */
     pe_order_implies_then_printed  = 0x20000,   /* Like ..implies_then but only ensures 'then' is printed, not mandatory */
 
     pe_order_asymmetrical          = 0x100000,  /* Indicates asymmetrical one way ordering constraint. */
     pe_order_load                  = 0x200000,  /* Only relevant if... */
     pe_order_one_or_more           = 0x400000,  /* 'then' is runnable only if one or more of its dependencies are too */
     pe_order_anti_colocation       = 0x800000,
 
     pe_order_preserve              = 0x1000000, /* Hack for breaking user ordering constraints with container resources */
     pe_order_then_cancels_first    = 0x2000000, // if 'then' becomes required, 'first' becomes optional
     pe_order_trace                 = 0x4000000, /* test marker */
 
 #if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
     // \deprecated Use pe_order_promoted_implies_first instead
     pe_order_implies_first_master  = pe_order_promoted_implies_first,
 #endif
 };
 /* *INDENT-ON* */
 
 typedef struct pe_action_wrapper_s {
     enum pe_ordering type;
     enum pe_link_state state;
     pe_action_t *action;
 } pe_action_wrapper_t;
 
 #if !defined(PCMK_ALLOW_DEPRECATED) || (PCMK_ALLOW_DEPRECATED == 1)
 #include <crm/pengine/pe_types_compat.h>
 #endif
 
 #ifdef __cplusplus
 }
 #endif
 
 #endif // PCMK__CRM_PENGINE_PE_TYPES__H
diff --git a/include/crm/pengine/pe_types_compat.h b/include/crm/pengine/pe_types_compat.h
index 8164fc7023..5f3f035f93 100644
--- a/include/crm/pengine/pe_types_compat.h
+++ b/include/crm/pengine/pe_types_compat.h
@@ -1,71 +1,74 @@
 /*
- * Copyright 2004-2022 the Pacemaker project contributors
+ * Copyright 2004-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef PCMK__CRM_PENGINE_PE_TYPES_COMPAT__H
 #  define PCMK__CRM_PENGINE_PE_TYPES_COMPAT__H
 
 #include <crm/pengine/pe_types.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /**
  * \file
  * \brief Deprecated Pacemaker scheduler API
  * \ingroup pengine
  * \deprecated Do not include this header directly. The scheduler APIs in this
  *             header, and the header itself, will be removed in a future
  *             release.
  */
 
+//! \deprecated Use pcmk_sched_quorate instead
+#define pe_flag_have_quorum             pcmk_sched_quorate
+
 //!@{
 //! \deprecated Do not use (unused by Pacemaker)
 enum pe_graph_flags {
     pe_graph_none = 0x00000,
     pe_graph_updated_first = 0x00001,
     pe_graph_updated_then = 0x00002,
     pe_graph_disable = 0x00004,
 };
 //!@}
 
 //!@{
 //! \deprecated Do not use
 enum pe_check_parameters {
     pe_check_last_failure,
     pe_check_active,
 };
 //!@}
 
 //!< \deprecated Use pe_action_t instead
 typedef struct pe_action_s action_t;
 
 //!< \deprecated Use pe_action_wrapper_t instead
 typedef struct pe_action_wrapper_s action_wrapper_t;
 
 //!< \deprecated Use pe_node_t instead
 typedef struct pe_node_s node_t;
 
 //!< \deprecated Use enum pe_quorum_policy instead
 typedef enum pe_quorum_policy no_quorum_policy_t;
 
 //!< \deprecated use pe_resource_t instead
 typedef struct pe_resource_s resource_t;
 
 //!< \deprecated Use pe_tag_t instead
 typedef struct pe_tag_s tag_t;
 
 //!< \deprecated Use pe_ticket_t instead
 typedef struct pe_ticket_s ticket_t;
 
 #ifdef __cplusplus
 }
 #endif
 
 #endif // PCMK__CRM_PENGINE_PE_TYPES_COMPAT__H
diff --git a/lib/pacemaker/pcmk_graph_producer.c b/lib/pacemaker/pcmk_graph_producer.c
index 0a7616e9f5..0d13f3066f 100644
--- a/lib/pacemaker/pcmk_graph_producer.c
+++ b/lib/pacemaker/pcmk_graph_producer.c
@@ -1,1094 +1,1094 @@
 /*
  * Copyright 2004-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <glib.h>
 
 #include <pacemaker-internal.h>
 
 #include "libpacemaker_private.h"
 
 // Convenience macros for logging action properties
 
 #define action_type_str(flags) \
     (pcmk_is_set((flags), pe_action_pseudo)? "pseudo-action" : "action")
 
 #define action_optional_str(flags) \
     (pcmk_is_set((flags), pe_action_optional)? "optional" : "required")
 
 #define action_runnable_str(flags) \
     (pcmk_is_set((flags), pe_action_runnable)? "runnable" : "unrunnable")
 
 #define action_node_str(a) \
     (((a)->node == NULL)? "no node" : (a)->node->details->uname)
 
 /*!
  * \internal
  * \brief Add an XML node tag for a specified ID
  *
  * \param[in]     id      Node UUID to add
  * \param[in,out] xml     Parent XML tag to add to
  */
 static xmlNode*
 add_node_to_xml_by_id(const char *id, xmlNode *xml)
 {
     xmlNode *node_xml;
 
     node_xml = create_xml_node(xml, XML_CIB_TAG_NODE);
     crm_xml_add(node_xml, XML_ATTR_ID, id);
 
     return node_xml;
 }
 
 /*!
  * \internal
  * \brief Add an XML node tag for a specified node
  *
  * \param[in]     node  Node to add
  * \param[in,out] xml   XML to add node to
  */
 static void
 add_node_to_xml(const pe_node_t *node, void *xml)
 {
     add_node_to_xml_by_id(node->details->id, (xmlNode *) xml);
 }
 
 /*!
  * \internal
  * \brief Count (optionally add to XML) nodes needing maintenance state update
  *
  * \param[in,out] xml       Parent XML tag to add to, if any
  * \param[in]     data_set  Working set for cluster
  *
  * \return Count of nodes added
  * \note Only Pacemaker Remote nodes are considered currently
  */
 static int
 add_maintenance_nodes(xmlNode *xml, const pe_working_set_t *data_set)
 {
     xmlNode *maintenance = NULL;
     int count = 0;
 
     if (xml != NULL) {
         maintenance = create_xml_node(xml, XML_GRAPH_TAG_MAINTENANCE);
     }
     for (const GList *iter = data_set->nodes; iter != NULL; iter = iter->next) {
         const pe_node_t *node = iter->data;
 
         if (pe__is_guest_or_remote_node(node) &&
             (node->details->maintenance != node->details->remote_maintenance)) {
 
             if (maintenance != NULL) {
                 crm_xml_add(add_node_to_xml_by_id(node->details->id,
                                                   maintenance),
                             XML_NODE_IS_MAINTENANCE,
                             (node->details->maintenance? "1" : "0"));
             }
             count++;
         }
     }
     crm_trace("%s %d nodes in need of maintenance mode update in state",
               ((maintenance == NULL)? "Counted" : "Added"), count);
     return count;
 }
 
 /*!
  * \internal
  * \brief Add pseudo action with nodes needing maintenance state update
  *
  * \param[in,out] data_set  Working set for cluster
  */
 static void
 add_maintenance_update(pe_working_set_t *data_set)
 {
     pe_action_t *action = NULL;
 
     if (add_maintenance_nodes(NULL, data_set) != 0) {
         action = get_pseudo_op(PCMK_ACTION_MAINTENANCE_NODES, data_set);
         pe__set_action_flags(action, pe_action_print_always);
     }
 }
 
 /*!
  * \internal
  * \brief Add XML with nodes that an action is expected to bring down
  *
  * If a specified action is expected to bring any nodes down, add an XML block
  * with their UUIDs. When a node is lost, this allows the controller to
  * determine whether it was expected.
  *
  * \param[in,out] xml       Parent XML tag to add to
  * \param[in]     action    Action to check for downed nodes
  */
 static void
 add_downed_nodes(xmlNode *xml, const pe_action_t *action)
 {
     CRM_CHECK((xml != NULL) && (action != NULL) && (action->node != NULL),
               return);
 
     if (pcmk__str_eq(action->task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {
 
         /* Shutdown makes the action's node down */
         xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
         add_node_to_xml_by_id(action->node->details->id, downed);
 
     } else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH,
                             pcmk__str_none)) {
 
         /* Fencing makes the action's node and any hosted guest nodes down */
         const char *fence = g_hash_table_lookup(action->meta, "stonith_action");
 
         if (pcmk__is_fencing_action(fence)) {
             xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
             add_node_to_xml_by_id(action->node->details->id, downed);
             pe_foreach_guest_node(action->node->details->data_set, action->node,
                                   add_node_to_xml, downed);
         }
 
     } else if (action->rsc && action->rsc->is_remote_node
                && pcmk__str_eq(action->task, PCMK_ACTION_STOP,
                                pcmk__str_none)) {
 
         /* Stopping a remote connection resource makes connected node down,
          * unless it's part of a migration
          */
         GList *iter;
         pe_action_t *input;
         bool migrating = false;
 
         for (iter = action->actions_before; iter != NULL; iter = iter->next) {
             input = ((pe_action_wrapper_t *) iter->data)->action;
             if ((input->rsc != NULL)
                 && pcmk__str_eq(action->rsc->id, input->rsc->id, pcmk__str_none)
                 && pcmk__str_eq(input->task, PCMK_ACTION_MIGRATE_FROM,
                                 pcmk__str_none)) {
                 migrating = true;
                 break;
             }
         }
         if (!migrating) {
             xmlNode *downed = create_xml_node(xml, XML_GRAPH_TAG_DOWNED);
             add_node_to_xml_by_id(action->rsc->id, downed);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Create a transition graph operation key for a clone action
  *
  * \param[in] action       Clone action
  * \param[in] interval_ms  Action interval in milliseconds
  *
  * \return Newly allocated string with transition graph operation key
  */
 static char *
 clone_op_key(const pe_action_t *action, guint interval_ms)
 {
     if (pcmk__str_eq(action->task, PCMK_ACTION_NOTIFY, pcmk__str_none)) {
         const char *n_type = g_hash_table_lookup(action->meta, "notify_type");
         const char *n_task = g_hash_table_lookup(action->meta,
                                                  "notify_operation");
 
         CRM_LOG_ASSERT((n_type != NULL) && (n_task != NULL));
         return pcmk__notify_key(action->rsc->clone_name, n_type, n_task);
 
     } else if (action->cancel_task != NULL) {
         return pcmk__op_key(action->rsc->clone_name, action->cancel_task,
                             interval_ms);
     } else {
         return pcmk__op_key(action->rsc->clone_name, action->task, interval_ms);
     }
 }
 
 /*!
  * \internal
  * \brief Add node details to transition graph action XML
  *
  * \param[in]     action  Scheduled action
  * \param[in,out] xml     Transition graph action XML for \p action
  */
 static void
 add_node_details(const pe_action_t *action, xmlNode *xml)
 {
     pe_node_t *router_node = pcmk__connection_host_for_action(action);
 
     crm_xml_add(xml, XML_LRM_ATTR_TARGET, action->node->details->uname);
     crm_xml_add(xml, XML_LRM_ATTR_TARGET_UUID, action->node->details->id);
     if (router_node != NULL) {
         crm_xml_add(xml, XML_LRM_ATTR_ROUTER_NODE, router_node->details->uname);
     }
 }
 
 /*!
  * \internal
  * \brief Add resource details to transition graph action XML
  *
  * \param[in]     action      Scheduled action
  * \param[in,out] action_xml  Transition graph action XML for \p action
  */
 static void
 add_resource_details(const pe_action_t *action, xmlNode *action_xml)
 {
     xmlNode *rsc_xml = NULL;
     const char *attr_list[] = {
         XML_AGENT_ATTR_CLASS,
         XML_AGENT_ATTR_PROVIDER,
         XML_ATTR_TYPE
     };
 
     /* If a resource is locked to a node via shutdown-lock, mark its actions
      * so the controller can preserve the lock when the action completes.
      */
     if (pcmk__action_locks_rsc_to_node(action)) {
         crm_xml_add_ll(action_xml, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
                        (long long) action->rsc->lock_time);
     }
 
     // List affected resource
 
     rsc_xml = create_xml_node(action_xml,
                               (const char *) action->rsc->xml->name);
     if (pcmk_is_set(action->rsc->flags, pe_rsc_orphan)
         && (action->rsc->clone_name != NULL)) {
         /* Use the numbered instance name here, because if there is more
          * than one instance on a node, we need to make sure the command
          * goes to the right one.
          *
          * This is important even for anonymous clones, because the clone's
          * unique meta-attribute might have just been toggled from on to
          * off.
          */
         crm_debug("Using orphan clone name %s instead of %s",
                   action->rsc->id, action->rsc->clone_name);
         crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->clone_name);
         crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);
 
     } else if (!pcmk_is_set(action->rsc->flags, pe_rsc_unique)) {
         const char *xml_id = ID(action->rsc->xml);
 
         crm_debug("Using anonymous clone name %s for %s (aka %s)",
                   xml_id, action->rsc->id, action->rsc->clone_name);
 
         /* ID is what we'd like client to use
          * ID_LONG is what they might know it as instead
          *
          * ID_LONG is only strictly needed /here/ during the
          * transition period until all nodes in the cluster
          * are running the new software /and/ have rebooted
          * once (meaning that they've only ever spoken to a DC
          * supporting this feature).
          *
          * If anyone toggles the unique flag to 'on', the
          * 'instance free' name will correspond to an orphan
          * and fall into the clause above instead
          */
         crm_xml_add(rsc_xml, XML_ATTR_ID, xml_id);
         if ((action->rsc->clone_name != NULL)
             && !pcmk__str_eq(xml_id, action->rsc->clone_name,
                              pcmk__str_none)) {
             crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->clone_name);
         } else {
             crm_xml_add(rsc_xml, XML_ATTR_ID_LONG, action->rsc->id);
         }
 
     } else {
         CRM_ASSERT(action->rsc->clone_name == NULL);
         crm_xml_add(rsc_xml, XML_ATTR_ID, action->rsc->id);
     }
 
     for (int lpc = 0; lpc < PCMK__NELEM(attr_list); lpc++) {
         crm_xml_add(rsc_xml, attr_list[lpc],
                     g_hash_table_lookup(action->rsc->meta, attr_list[lpc]));
     }
 }
 
 /*!
  * \internal
  * \brief Add action attributes to transition graph action XML
  *
  * \param[in,out] action      Scheduled action
  * \param[in,out] action_xml  Transition graph action XML for \p action
  */
 static void
 add_action_attributes(pe_action_t *action, xmlNode *action_xml)
 {
     xmlNode *args_xml = NULL;
 
     /* We create free-standing XML to start, so we can sort the attributes
      * before adding it to action_xml, which keeps the scheduler regression
      * test graphs comparable.
      */
     args_xml = create_xml_node(NULL, XML_TAG_ATTRS);
 
     crm_xml_add(args_xml, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
     g_hash_table_foreach(action->extra, hash2field, args_xml);
 
     if ((action->rsc != NULL) && (action->node != NULL)) {
         // Get the resource instance attributes, evaluated properly for node
         GHashTable *params = pe_rsc_params(action->rsc, action->node,
                                            action->rsc->cluster);
 
         pcmk__substitute_remote_addr(action->rsc, params);
 
         g_hash_table_foreach(params, hash2smartfield, args_xml);
 
     } else if ((action->rsc != NULL)
                && (action->rsc->variant <= pcmk_rsc_variant_primitive)) {
         GHashTable *params = pe_rsc_params(action->rsc, NULL,
                                            action->rsc->cluster);
 
         g_hash_table_foreach(params, hash2smartfield, args_xml);
     }
 
     g_hash_table_foreach(action->meta, hash2metafield, args_xml);
     if (action->rsc != NULL) {
         pe_resource_t *parent = action->rsc;
 
         while (parent != NULL) {
             parent->cmds->add_graph_meta(parent, args_xml);
             parent = parent->parent;
         }
 
         pcmk__add_bundle_meta_to_xml(args_xml, action);
 
     } else if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)
                && (action->node != NULL)) {
         /* Pass the node's attributes as meta-attributes.
          *
          * @TODO: Determine whether it is still necessary to do this. It was
          * added in 33d99707, probably for the libfence-based implementation in
          * c9a90bd, which is no longer used.
          */
         g_hash_table_foreach(action->node->details->attrs, hash2metafield,
                              args_xml);
     }
 
     sorted_xml(args_xml, action_xml, FALSE);
     free_xml(args_xml);
 }
 
 /*!
  * \internal
  * \brief Create the transition graph XML for a scheduled action
  *
  * \param[in,out] parent        Parent XML element to add action to
  * \param[in,out] action        Scheduled action
  * \param[in]     skip_details  If false, add action details as sub-elements
  * \param[in]     data_set      Cluster working set
  */
 static void
 create_graph_action(xmlNode *parent, pe_action_t *action, bool skip_details,
                     const pe_working_set_t *data_set)
 {
     bool needs_node_info = true;
     bool needs_maintenance_info = false;
     xmlNode *action_xml = NULL;
 
     if ((action == NULL) || (data_set == NULL)) {
         return;
     }
 
     // Create the top-level element based on task
 
     if (pcmk__str_eq(action->task, PCMK_ACTION_STONITH, pcmk__str_none)) {
         /* All fences need node info; guest node fences are pseudo-events */
         if (pcmk_is_set(action->flags, pe_action_pseudo)) {
             action_xml = create_xml_node(parent, XML_GRAPH_TAG_PSEUDO_EVENT);
         } else {
             action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
         }
 
     } else if (pcmk__str_any_of(action->task,
                                 PCMK_ACTION_DO_SHUTDOWN,
                                 PCMK_ACTION_CLEAR_FAILCOUNT, NULL)) {
         action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
 
     } else if (pcmk__str_eq(action->task, PCMK_ACTION_LRM_DELETE,
                             pcmk__str_none)) {
         // CIB-only clean-up for shutdown locks
         action_xml = create_xml_node(parent, XML_GRAPH_TAG_CRM_EVENT);
         crm_xml_add(action_xml, PCMK__XA_MODE, XML_TAG_CIB);
 
     } else if (pcmk_is_set(action->flags, pe_action_pseudo)) {
         if (pcmk__str_eq(action->task, PCMK_ACTION_MAINTENANCE_NODES,
                          pcmk__str_none)) {
             needs_maintenance_info = true;
         }
         action_xml = create_xml_node(parent, XML_GRAPH_TAG_PSEUDO_EVENT);
         needs_node_info = false;
 
     } else {
         action_xml = create_xml_node(parent, XML_GRAPH_TAG_RSC_OP);
     }
 
     crm_xml_add_int(action_xml, XML_ATTR_ID, action->id);
     crm_xml_add(action_xml, XML_LRM_ATTR_TASK, action->task);
 
     if ((action->rsc != NULL) && (action->rsc->clone_name != NULL)) {
         char *clone_key = NULL;
         guint interval_ms;
 
         if (pcmk__guint_from_hash(action->meta, XML_LRM_ATTR_INTERVAL_MS, 0,
                                   &interval_ms) != pcmk_rc_ok) {
             interval_ms = 0;
         }
         clone_key = clone_op_key(action, interval_ms);
         crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, clone_key);
         crm_xml_add(action_xml, "internal_" XML_LRM_ATTR_TASK_KEY,
                     action->uuid);
         free(clone_key);
     } else {
         crm_xml_add(action_xml, XML_LRM_ATTR_TASK_KEY, action->uuid);
     }
 
     if (needs_node_info && (action->node != NULL)) {
         add_node_details(action, action_xml);
         g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET),
                             strdup(action->node->details->uname));
         g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET_UUID),
                             strdup(action->node->details->id));
     }
 
     if (skip_details) {
         return;
     }
 
     if ((action->rsc != NULL)
         && !pcmk_is_set(action->flags, pe_action_pseudo)) {
 
         // This is a real resource action, so add resource details
         add_resource_details(action, action_xml);
     }
 
     /* List any attributes in effect */
     add_action_attributes(action, action_xml);
 
     /* List any nodes this action is expected to make down */
     if (needs_node_info && (action->node != NULL)) {
         add_downed_nodes(action_xml, action);
     }
 
     if (needs_maintenance_info) {
         add_maintenance_nodes(action_xml, data_set);
     }
 }
 
 /*!
  * \internal
  * \brief Check whether an action should be added to the transition graph
  *
  * \param[in] action  Action to check
  *
  * \return true if action should be added to graph, otherwise false
  */
 static bool
 should_add_action_to_graph(const pe_action_t *action)
 {
     if (!pcmk_is_set(action->flags, pe_action_runnable)) {
         crm_trace("Ignoring action %s (%d): unrunnable",
                   action->uuid, action->id);
         return false;
     }
 
     if (pcmk_is_set(action->flags, pe_action_optional)
         && !pcmk_is_set(action->flags, pe_action_print_always)) {
         crm_trace("Ignoring action %s (%d): optional",
                   action->uuid, action->id);
         return false;
     }
 
     /* Actions for unmanaged resources should be excluded from the graph,
      * with the exception of monitors and cancellation of recurring monitors.
      */
     if ((action->rsc != NULL)
         && !pcmk_is_set(action->rsc->flags, pe_rsc_managed)
         && !pcmk__str_eq(action->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
         const char *interval_ms_s;
 
         /* A cancellation of a recurring monitor will get here because the task
          * is cancel rather than monitor, but the interval can still be used to
          * recognize it. The interval has been normalized to milliseconds by
          * this point, so a string comparison is sufficient.
          */
         interval_ms_s = g_hash_table_lookup(action->meta,
                                             XML_LRM_ATTR_INTERVAL_MS);
         if (pcmk__str_eq(interval_ms_s, "0", pcmk__str_null_matches)) {
             crm_trace("Ignoring action %s (%d): for unmanaged resource (%s)",
                       action->uuid, action->id, action->rsc->id);
             return false;
         }
     }
 
     /* Always add pseudo-actions, fence actions, and shutdown actions (already
      * determined to be required and runnable by this point)
      */
     if (pcmk_is_set(action->flags, pe_action_pseudo)
         || pcmk__strcase_any_of(action->task, PCMK_ACTION_STONITH,
                                 PCMK_ACTION_DO_SHUTDOWN, NULL)) {
         return true;
     }
 
     if (action->node == NULL) {
         pe_err("Skipping action %s (%d) "
                "because it was not assigned to a node (bug?)",
                action->uuid, action->id);
         pcmk__log_action("Unassigned", action, false);
         return false;
     }
 
     if (pcmk_is_set(action->flags, pe_action_dc)) {
         crm_trace("Action %s (%d) should be dumped: "
                   "can run on DC instead of %s",
                   action->uuid, action->id, pe__node_name(action->node));
 
     } else if (pe__is_guest_node(action->node)
                && !action->node->details->remote_requires_reset) {
         crm_trace("Action %s (%d) should be dumped: "
                   "assuming will be runnable on guest %s",
                   action->uuid, action->id, pe__node_name(action->node));
 
     } else if (!action->node->details->online) {
         pe_err("Skipping action %s (%d) "
                "because it was scheduled for offline node (bug?)",
                action->uuid, action->id);
         pcmk__log_action("Offline node", action, false);
         return false;
 
     } else if (action->node->details->unclean) {
         pe_err("Skipping action %s (%d) "
                "because it was scheduled for unclean node (bug?)",
                action->uuid, action->id);
         pcmk__log_action("Unclean node", action, false);
         return false;
     }
     return true;
 }
 
 /*!
  * \internal
  * \brief Check whether an ordering's flags can change an action
  *
  * \param[in] ordering  Ordering to check
  *
  * \return true if ordering has flags that can change an action, false otherwise
  */
 static bool
 ordering_can_change_actions(const pe_action_wrapper_t *ordering)
 {
     return pcmk_any_flags_set(ordering->type, ~(pe_order_implies_first_printed
                                                 |pe_order_implies_then_printed
                                                 |pe_order_optional));
 }
 
 /*!
  * \internal
  * \brief Check whether an action input should be in the transition graph
  *
  * \param[in]     action  Action to check
  * \param[in,out] input   Action input to check
  *
  * \return true if input should be in graph, false otherwise
  * \note This function may not only check an input, but disable it under certian
  *       circumstances (load or anti-colocation orderings that are not needed).
  */
 static bool
 should_add_input_to_graph(const pe_action_t *action, pe_action_wrapper_t *input)
 {
     if (input->state == pe_link_dumped) {
         return true;
     }
 
     if (input->type == pe_order_none) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "ordering disabled",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (!pcmk_is_set(input->action->flags, pe_action_runnable)
                && !ordering_can_change_actions(input)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "optional and input unrunnable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (!pcmk_is_set(input->action->flags, pe_action_runnable)
                && pcmk_is_set(input->type, pe_order_one_or_more)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "one-or-more and input unrunnable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (pcmk_is_set(input->type, pe_order_implies_first_migratable)
                && !pcmk_is_set(input->action->flags, pe_action_runnable)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "implies input migratable but input unrunnable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (pcmk_is_set(input->type, pe_order_apply_first_non_migratable)
                && pcmk_is_set(input->action->flags,
                               pe_action_migrate_runnable)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "only if input unmigratable but input unrunnable",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if ((input->type == pe_order_optional)
                && pcmk_is_set(input->action->flags, pe_action_migrate_runnable)
                && pcmk__ends_with(input->action->uuid, "_stop_0")) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "optional but stop in migration",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
 
     } else if (input->type == pe_order_load) {
         pe_node_t *input_node = input->action->node;
 
         // load orderings are relevant only if actions are for same node
 
         if ((action->rsc != NULL)
             && pcmk__str_eq(action->task, PCMK_ACTION_MIGRATE_TO,
                             pcmk__str_none)) {
 
             pe_node_t *assigned = action->rsc->allocated_to;
 
             /* For load_stopped -> migrate_to orderings, we care about where it
              * has been assigned to, not where it will be executed.
              */
             if (!pe__same_node(input_node, assigned)) {
                 crm_trace("Ignoring %s (%d) input %s (%d): "
                           "load ordering node mismatch %s vs %s",
                           action->uuid, action->id,
                           input->action->uuid, input->action->id,
                           (assigned? assigned->details->uname : "<none>"),
                           (input_node? input_node->details->uname : "<none>"));
                 input->type = pe_order_none;
                 return false;
             }
 
         } else if (!pe__same_node(input_node, action->node)) {
             crm_trace("Ignoring %s (%d) input %s (%d): "
                       "load ordering node mismatch %s vs %s",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id,
                       (action->node? action->node->details->uname : "<none>"),
                       (input_node? input_node->details->uname : "<none>"));
             input->type = pe_order_none;
             return false;
 
         } else if (pcmk_is_set(input->action->flags, pe_action_optional)) {
             crm_trace("Ignoring %s (%d) input %s (%d): "
                       "load ordering input optional",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id);
             input->type = pe_order_none;
             return false;
         }
 
     } else if (input->type == pe_order_anti_colocation) {
         if (input->action->node && action->node
             && !pe__same_node(input->action->node, action->node)) {
             crm_trace("Ignoring %s (%d) input %s (%d): "
                       "anti-colocation node mismatch %s vs %s",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id,
                       pe__node_name(action->node),
                       pe__node_name(input->action->node));
             input->type = pe_order_none;
             return false;
 
         } else if (pcmk_is_set(input->action->flags, pe_action_optional)) {
             crm_trace("Ignoring %s (%d) input %s (%d): "
                       "anti-colocation input optional",
                       action->uuid, action->id,
                       input->action->uuid, input->action->id);
             input->type = pe_order_none;
             return false;
         }
 
     } else if (input->action->rsc
                && input->action->rsc != action->rsc
                && pcmk_is_set(input->action->rsc->flags, pe_rsc_failed)
                && !pcmk_is_set(input->action->rsc->flags, pe_rsc_managed)
                && pcmk__ends_with(input->action->uuid, "_stop_0")
                && action->rsc && pe_rsc_is_clone(action->rsc)) {
         crm_warn("Ignoring requirement that %s complete before %s:"
                  " unmanaged failed resources cannot prevent clone shutdown",
                  input->action->uuid, action->uuid);
         return false;
 
     } else if (pcmk_is_set(input->action->flags, pe_action_optional)
                && !pcmk_any_flags_set(input->action->flags,
                                       pe_action_print_always|pe_action_dumped)
                && !should_add_action_to_graph(input->action)) {
         crm_trace("Ignoring %s (%d) input %s (%d): "
                   "input optional",
                   action->uuid, action->id,
                   input->action->uuid, input->action->id);
         return false;
     }
 
     crm_trace("%s (%d) input %s %s (%d) on %s should be dumped: %s %s %#.6x",
               action->uuid, action->id, action_type_str(input->action->flags),
               input->action->uuid, input->action->id,
               action_node_str(input->action),
               action_runnable_str(input->action->flags),
               action_optional_str(input->action->flags), input->type);
     return true;
 }
 
 /*!
  * \internal
  * \brief Check whether an ordering creates an ordering loop
  *
  * \param[in]     init_action  "First" action in ordering
  * \param[in]     action       Callers should always set this the same as
  *                             \p init_action (this function may use a different
  *                             value for recursive calls)
  * \param[in,out] input        Action wrapper for "then" action in ordering
  *
  * \return true if the ordering creates a loop, otherwise false
  */
 bool
 pcmk__graph_has_loop(const pe_action_t *init_action, const pe_action_t *action,
                      pe_action_wrapper_t *input)
 {
     bool has_loop = false;
 
     if (pcmk_is_set(input->action->flags, pe_action_tracking)) {
         crm_trace("Breaking tracking loop: %s@%s -> %s@%s (%#.6x)",
                   input->action->uuid,
                   input->action->node? input->action->node->details->uname : "",
                   action->uuid,
                   action->node? action->node->details->uname : "",
                   input->type);
         return false;
     }
 
     // Don't need to check inputs that won't be used
     if (!should_add_input_to_graph(action, input)) {
         return false;
     }
 
     if (input->action == init_action) {
         crm_debug("Input loop found in %s@%s ->...-> %s@%s",
                   action->uuid,
                   action->node? action->node->details->uname : "",
                   init_action->uuid,
                   init_action->node? init_action->node->details->uname : "");
         return true;
     }
 
     pe__set_action_flags(input->action, pe_action_tracking);
 
     crm_trace("Checking inputs of action %s@%s input %s@%s (%#.6x)"
               "for graph loop with %s@%s ",
               action->uuid,
               action->node? action->node->details->uname : "",
               input->action->uuid,
               input->action->node? input->action->node->details->uname : "",
               input->type,
               init_action->uuid,
               init_action->node? init_action->node->details->uname : "");
 
     // Recursively check input itself for loops
     for (GList *iter = input->action->actions_before;
          iter != NULL; iter = iter->next) {
 
         if (pcmk__graph_has_loop(init_action, input->action,
                                  (pe_action_wrapper_t *) iter->data)) {
             // Recursive call already logged a debug message
             has_loop = true;
             break;
         }
     }
 
     pe__clear_action_flags(input->action, pe_action_tracking);
 
     if (!has_loop) {
         crm_trace("No input loop found in %s@%s -> %s@%s (%#.6x)",
                   input->action->uuid,
                   input->action->node? input->action->node->details->uname : "",
                   action->uuid,
                   action->node? action->node->details->uname : "",
                   input->type);
     }
     return has_loop;
 }
 
 /*!
  * \internal
  * \brief Create a synapse XML element for a transition graph
  *
  * \param[in]     action    Action that synapse is for
  * \param[in,out] data_set  Cluster working set containing graph
  *
  * \return Newly added XML element for new graph synapse
  */
 static xmlNode *
 create_graph_synapse(const pe_action_t *action, pe_working_set_t *data_set)
 {
     int synapse_priority = 0;
     xmlNode *syn = create_xml_node(data_set->graph, "synapse");
 
     crm_xml_add_int(syn, XML_ATTR_ID, data_set->num_synapse);
     data_set->num_synapse++;
 
     if (action->rsc != NULL) {
         synapse_priority = action->rsc->priority;
     }
     if (action->priority > synapse_priority) {
         synapse_priority = action->priority;
     }
     if (synapse_priority > 0) {
         crm_xml_add_int(syn, XML_CIB_ATTR_PRIORITY, synapse_priority);
     }
     return syn;
 }
 
 /*!
  * \internal
  * \brief Add an action to the transition graph XML if appropriate
  *
  * \param[in,out] data       Action to possibly add
  * \param[in,out] user_data  Cluster working set
  *
  * \note This will de-duplicate the action inputs, meaning that the
  *       pe_action_wrapper_t:type flags can no longer be relied on to retain
  *       their original settings. That means this MUST be called after
  *       pcmk__apply_orderings() is complete, and nothing after this should rely
  *       on those type flags. (For example, some code looks for type equal to
  *       some flag rather than whether the flag is set, and some code looks for
  *       particular combinations of flags -- such code must be done before
  *       pcmk__create_graph().)
  */
 static void
 add_action_to_graph(gpointer data, gpointer user_data)
 {
     pe_action_t *action = (pe_action_t *) data;
     pe_working_set_t *data_set = (pe_working_set_t *) user_data;
 
     xmlNode *syn = NULL;
     xmlNode *set = NULL;
     xmlNode *in = NULL;
 
     /* If we haven't already, de-duplicate inputs (even if we won't be adding
      * the action to the graph, so that crm_simulate's dot graphs don't have
      * duplicates).
      */
     if (!pcmk_is_set(action->flags, pe_action_dedup)) {
         pcmk__deduplicate_action_inputs(action);
         pe__set_action_flags(action, pe_action_dedup);
     }
 
     if (pcmk_is_set(action->flags, pe_action_dumped)    // Already added, or
         || !should_add_action_to_graph(action)) {       // shouldn't be added
         return;
     }
     pe__set_action_flags(action, pe_action_dumped);
 
     crm_trace("Adding action %d (%s%s%s) to graph",
               action->id, action->uuid,
               ((action->node == NULL)? "" : " on "),
               ((action->node == NULL)? "" : action->node->details->uname));
 
     syn = create_graph_synapse(action, data_set);
     set = create_xml_node(syn, "action_set");
     in = create_xml_node(syn, "inputs");
 
     create_graph_action(set, action, false, data_set);
 
     for (GList *lpc = action->actions_before; lpc != NULL; lpc = lpc->next) {
         pe_action_wrapper_t *input = (pe_action_wrapper_t *) lpc->data;
 
         if (should_add_input_to_graph(action, input)) {
             xmlNode *input_xml = create_xml_node(in, "trigger");
 
             input->state = pe_link_dumped;
             create_graph_action(input_xml, input->action, true, data_set);
         }
     }
 }
 
 static int transition_id = -1;
 
 /*!
  * \internal
  * \brief Log a message after calculating a transition
  *
  * \param[in] filename  Where transition input is stored
  */
 void
 pcmk__log_transition_summary(const char *filename)
 {
     if (was_processing_error) {
         crm_err("Calculated transition %d (with errors)%s%s",
                 transition_id,
                 (filename == NULL)? "" : ", saving inputs in ",
                 (filename == NULL)? "" : filename);
 
     } else if (was_processing_warning) {
         crm_warn("Calculated transition %d (with warnings)%s%s",
                  transition_id,
                  (filename == NULL)? "" : ", saving inputs in ",
                  (filename == NULL)? "" : filename);
 
     } else {
         crm_notice("Calculated transition %d%s%s",
                    transition_id,
                    (filename == NULL)? "" : ", saving inputs in ",
                    (filename == NULL)? "" : filename);
     }
     if (crm_config_error) {
         crm_notice("Configuration errors found during scheduler processing,"
                    "  please run \"crm_verify -L\" to identify issues");
     }
 }
 
 /*!
  * \internal
  * \brief Add a resource's actions to the transition graph
  *
  * \param[in,out] rsc  Resource whose actions should be added
  */
 void
 pcmk__add_rsc_actions_to_graph(pe_resource_t *rsc)
 {
     GList *iter = NULL;
 
     CRM_ASSERT(rsc != NULL);
     pe_rsc_trace(rsc, "Adding actions for %s to graph", rsc->id);
 
     // First add the resource's own actions
     g_list_foreach(rsc->actions, add_action_to_graph, rsc->cluster);
 
     // Then recursively add its children's actions (appropriate to variant)
     for (iter = rsc->children; iter != NULL; iter = iter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) iter->data;
 
         child_rsc->cmds->add_actions_to_graph(child_rsc);
     }
 }
 
 /*!
  * \internal
  * \brief Create a transition graph with all cluster actions needed
  *
  * \param[in,out] data_set  Cluster working set
  */
 void
 pcmk__create_graph(pe_working_set_t *data_set)
 {
     GList *iter = NULL;
     const char *value = NULL;
     long long limit = 0LL;
 
     transition_id++;
     crm_trace("Creating transition graph %d", transition_id);
 
     data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);
 
     value = pe_pref(data_set->config_hash, "cluster-delay");
     crm_xml_add(data_set->graph, "cluster-delay", value);
 
     value = pe_pref(data_set->config_hash, "stonith-timeout");
     crm_xml_add(data_set->graph, "stonith-timeout", value);
 
     crm_xml_add(data_set->graph, "failed-stop-offset", "INFINITY");
 
     if (pcmk_is_set(data_set->flags, pe_flag_start_failure_fatal)) {
         crm_xml_add(data_set->graph, "failed-start-offset", "INFINITY");
     } else {
         crm_xml_add(data_set->graph, "failed-start-offset", "1");
     }
 
     value = pe_pref(data_set->config_hash, "batch-limit");
     crm_xml_add(data_set->graph, "batch-limit", value);
 
     crm_xml_add_int(data_set->graph, "transition_id", transition_id);
 
     value = pe_pref(data_set->config_hash, "migration-limit");
     if ((pcmk__scan_ll(value, &limit, 0LL) == pcmk_rc_ok) && (limit > 0)) {
         crm_xml_add(data_set->graph, "migration-limit", value);
     }
 
     if (data_set->recheck_by > 0) {
         char *recheck_epoch = NULL;
 
         recheck_epoch = crm_strdup_printf("%llu",
                                           (long long) data_set->recheck_by);
         crm_xml_add(data_set->graph, "recheck-by", recheck_epoch);
         free(recheck_epoch);
     }
 
     /* The following code will de-duplicate action inputs, so nothing past this
      * should rely on the action input type flags retaining their original
      * values.
      */
 
     // Add resource actions to graph
     for (iter = data_set->resources; iter != NULL; iter = iter->next) {
         pe_resource_t *rsc = (pe_resource_t *) iter->data;
 
         pe_rsc_trace(rsc, "Processing actions for %s", rsc->id);
         rsc->cmds->add_actions_to_graph(rsc);
     }
 
     // Add pseudo-action for list of nodes with maintenance state update
     add_maintenance_update(data_set);
 
     // Add non-resource (node) actions
     for (iter = data_set->actions; iter != NULL; iter = iter->next) {
         pe_action_t *action = (pe_action_t *) iter->data;
 
         if ((action->rsc != NULL)
             && (action->node != NULL)
             && action->node->details->shutdown
             && !pcmk_is_set(action->rsc->flags, pe_rsc_maintenance)
             && !pcmk_any_flags_set(action->flags,
                                    pe_action_optional|pe_action_runnable)
             && pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_none)) {
             /* Eventually we should just ignore the 'fence' case, but for now
              * it's the best way to detect (in CTS) when CIB resource updates
              * are being lost.
              */
-            if (pcmk_is_set(data_set->flags, pe_flag_have_quorum)
+            if (pcmk_is_set(data_set->flags, pcmk_sched_quorate)
                 || (data_set->no_quorum_policy == pcmk_no_quorum_ignore)) {
                 const bool managed = pcmk_is_set(action->rsc->flags,
                                                  pe_rsc_managed);
                 const bool failed = pcmk_is_set(action->rsc->flags,
                                                 pe_rsc_failed);
 
                 crm_crit("Cannot %s %s because of %s:%s%s (%s)",
                          action->node->details->unclean? "fence" : "shut down",
                          pe__node_name(action->node), action->rsc->id,
                          (managed? " blocked" : " unmanaged"),
                          (failed? " failed" : ""), action->uuid);
             }
         }
 
         add_action_to_graph((gpointer) action, (gpointer) data_set);
     }
 
     crm_log_xml_trace(data_set->graph, "graph");
 }
diff --git a/lib/pacemaker/pcmk_sched_primitive.c b/lib/pacemaker/pcmk_sched_primitive.c
index 2c3f5f9fb4..9a6cd90520 100644
--- a/lib/pacemaker/pcmk_sched_primitive.c
+++ b/lib/pacemaker/pcmk_sched_primitive.c
@@ -1,1650 +1,1650 @@
 /*
  * Copyright 2004-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdbool.h>
 #include <stdint.h>                 // uint8_t, uint32_t
 
 #include <crm/msg_xml.h>
 #include <pacemaker-internal.h>
 
 #include "libpacemaker_private.h"
 
 static void stop_resource(pe_resource_t *rsc, pe_node_t *node, bool optional);
 static void start_resource(pe_resource_t *rsc, pe_node_t *node, bool optional);
 static void demote_resource(pe_resource_t *rsc, pe_node_t *node, bool optional);
 static void promote_resource(pe_resource_t *rsc, pe_node_t *node,
                              bool optional);
 static void assert_role_error(pe_resource_t *rsc, pe_node_t *node,
                               bool optional);
 
 #define RSC_ROLE_MAX    (pcmk_role_promoted + 1)
 
 static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
     /* This array lists the immediate next role when transitioning from one role
      * to a target role. For example, when going from Stopped to Promoted, the
      * next role is Unpromoted, because the resource must be started before it
      * can be promoted. The current state then becomes Started, which is fed
      * into this array again, giving a next role of Promoted.
      *
      * Current role       Immediate next role   Final target role
      * ------------       -------------------   -----------------
      */
     /* Unknown */       { pcmk_role_unknown,    /* Unknown */
                           pcmk_role_stopped,    /* Stopped */
                           pcmk_role_stopped,    /* Started */
                           pcmk_role_stopped,    /* Unpromoted */
                           pcmk_role_stopped,    /* Promoted */
                         },
     /* Stopped */       { pcmk_role_stopped,    /* Unknown */
                           pcmk_role_stopped,    /* Stopped */
                           pcmk_role_started,    /* Started */
                           pcmk_role_unpromoted, /* Unpromoted */
                           pcmk_role_unpromoted, /* Promoted */
                         },
     /* Started */       { pcmk_role_stopped,    /* Unknown */
                           pcmk_role_stopped,    /* Stopped */
                           pcmk_role_started,    /* Started */
                           pcmk_role_unpromoted, /* Unpromoted */
                           pcmk_role_promoted,   /* Promoted */
                         },
     /* Unpromoted */    { pcmk_role_stopped,    /* Unknown */
                           pcmk_role_stopped,    /* Stopped */
                           pcmk_role_stopped,    /* Started */
                           pcmk_role_unpromoted, /* Unpromoted */
                           pcmk_role_promoted,   /* Promoted */
                         },
     /* Promoted  */     { pcmk_role_stopped,    /* Unknown */
                           pcmk_role_unpromoted, /* Stopped */
                           pcmk_role_unpromoted, /* Started */
                           pcmk_role_unpromoted, /* Unpromoted */
                           pcmk_role_promoted,   /* Promoted */
                         },
 };
 
 /*!
  * \internal
  * \brief Function to schedule actions needed for a role change
  *
  * \param[in,out] rsc       Resource whose role is changing
  * \param[in,out] node      Node where resource will be in its next role
  * \param[in]     optional  Whether scheduled actions should be optional
  */
 typedef void (*rsc_transition_fn)(pe_resource_t *rsc, pe_node_t *node,
                                   bool optional);
 
 static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
     /* This array lists the function needed to transition directly from one role
      * to another. NULL indicates that nothing is needed.
      *
      * Current role         Transition function             Next role
      * ------------         -------------------             ----------
      */
     /* Unknown */       {   assert_role_error,              /* Unknown */
                             stop_resource,                  /* Stopped */
                             assert_role_error,              /* Started */
                             assert_role_error,              /* Unpromoted */
                             assert_role_error,              /* Promoted */
                         },
     /* Stopped */       {   assert_role_error,              /* Unknown */
                             NULL,                           /* Stopped */
                             start_resource,                 /* Started */
                             start_resource,                 /* Unpromoted */
                             assert_role_error,              /* Promoted */
                         },
     /* Started */       {   assert_role_error,              /* Unknown */
                             stop_resource,                  /* Stopped */
                             NULL,                           /* Started */
                             NULL,                           /* Unpromoted */
                             promote_resource,               /* Promoted */
                         },
     /* Unpromoted */    {   assert_role_error,              /* Unknown */
                             stop_resource,                  /* Stopped */
                             stop_resource,                  /* Started */
                             NULL,                           /* Unpromoted */
                             promote_resource,               /* Promoted */
                         },
     /* Promoted  */     {   assert_role_error,              /* Unknown */
                             demote_resource,                /* Stopped */
                             demote_resource,                /* Started */
                             demote_resource,                /* Unpromoted */
                             NULL,                           /* Promoted */
                         },
 };
 
 /*!
  * \internal
  * \brief Get a list of a resource's allowed nodes sorted by node score
  *
  * \param[in] rsc  Resource to check
  *
  * \return List of allowed nodes sorted by node score
  */
 static GList *
 sorted_allowed_nodes(const pe_resource_t *rsc)
 {
     if (rsc->allowed_nodes != NULL) {
         GList *nodes = g_hash_table_get_values(rsc->allowed_nodes);
 
         if (nodes != NULL) {
             return pcmk__sort_nodes(nodes, pe__current_node(rsc));
         }
     }
     return NULL;
 }
 
 /*!
  * \internal
  * \brief Assign a resource to its best allowed node, if possible
  *
  * \param[in,out] rsc           Resource to choose a node for
  * \param[in]     prefer        If not \c NULL, prefer this node when all else
  *                              equal
  * \param[in]     stop_if_fail  If \c true and \p rsc can't be assigned to a
  *                              node, set next role to stopped and update
  *                              existing actions
  *
  * \return true if \p rsc could be assigned to a node, otherwise false
  *
  * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
  *       completely undo the assignment. A successful assignment can be either
  *       undone or left alone as final. A failed assignment has the same effect
  *       as calling pcmk__unassign_resource(); there are no side effects on
  *       roles or actions.
  */
 static bool
 assign_best_node(pe_resource_t *rsc, const pe_node_t *prefer, bool stop_if_fail)
 {
     GList *nodes = NULL;
     pe_node_t *chosen = NULL;
     pe_node_t *best = NULL;
     const pe_node_t *most_free_node = pcmk__ban_insufficient_capacity(rsc);
 
     if (prefer == NULL) {
         prefer = most_free_node;
     }
 
     if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
         // We've already finished assignment of resources to nodes
         return rsc->allocated_to != NULL;
     }
 
     // Sort allowed nodes by score
     nodes = sorted_allowed_nodes(rsc);
     if (nodes != NULL) {
         best = (pe_node_t *) nodes->data; // First node has best score
     }
 
     if ((prefer != NULL) && (nodes != NULL)) {
         // Get the allowed node version of prefer
         chosen = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);
 
         if (chosen == NULL) {
             pe_rsc_trace(rsc, "Preferred node %s for %s was unknown",
                          pe__node_name(prefer), rsc->id);
 
         /* Favor the preferred node as long as its score is at least as good as
          * the best allowed node's.
          *
          * An alternative would be to favor the preferred node even if the best
          * node is better, when the best node's score is less than INFINITY.
          */
         } else if (chosen->weight < best->weight) {
             pe_rsc_trace(rsc, "Preferred node %s for %s was unsuitable",
                          pe__node_name(chosen), rsc->id);
             chosen = NULL;
 
         } else if (!pcmk__node_available(chosen, true, false)) {
             pe_rsc_trace(rsc, "Preferred node %s for %s was unavailable",
                          pe__node_name(chosen), rsc->id);
             chosen = NULL;
 
         } else {
             pe_rsc_trace(rsc,
                          "Chose preferred node %s for %s "
                          "(ignoring %d candidates)",
                          pe__node_name(chosen), rsc->id, g_list_length(nodes));
         }
     }
 
     if ((chosen == NULL) && (best != NULL)) {
         /* Either there is no preferred node, or the preferred node is not
          * suitable, but another node is allowed to run the resource.
          */
 
         chosen = best;
 
         if (!pe_rsc_is_unique_clone(rsc->parent)
             && (chosen->weight > 0) // Zero not acceptable
             && pcmk__node_available(chosen, false, false)) {
             /* If the resource is already running on a node, prefer that node if
              * it is just as good as the chosen node.
              *
              * We don't do this for unique clone instances, because
              * pcmk__assign_instances() has already assigned instances to their
              * running nodes when appropriate, and if we get here, we don't want
              * remaining unassigned instances to prefer a node that's already
              * running another instance.
              */
             pe_node_t *running = pe__current_node(rsc);
 
             if (running == NULL) {
                 // Nothing to do
 
             } else if (!pcmk__node_available(running, true, false)) {
                 pe_rsc_trace(rsc,
                              "Current node for %s (%s) can't run resources",
                              rsc->id, pe__node_name(running));
 
             } else {
                 int nodes_with_best_score = 1;
 
                 for (GList *iter = nodes->next; iter; iter = iter->next) {
                     pe_node_t *allowed = (pe_node_t *) iter->data;
 
                     if (allowed->weight != chosen->weight) {
                         // The nodes are sorted by score, so no more are equal
                         break;
                     }
                     if (pe__same_node(allowed, running)) {
                         // Scores are equal, so prefer the current node
                         chosen = allowed;
                     }
                     nodes_with_best_score++;
                 }
 
                 if (nodes_with_best_score > 1) {
                     uint8_t log_level = LOG_INFO;
 
                     if (chosen->weight >= INFINITY) {
                         log_level = LOG_WARNING;
                     }
                     do_crm_log(log_level,
                                "Chose %s for %s from %d nodes with score %s",
                                pe__node_name(chosen), rsc->id,
                                nodes_with_best_score,
                                pcmk_readable_score(chosen->weight));
                 }
             }
         }
 
         pe_rsc_trace(rsc, "Chose %s for %s from %d candidates",
                      pe__node_name(chosen), rsc->id, g_list_length(nodes));
     }
 
     pcmk__assign_resource(rsc, chosen, false, stop_if_fail);
     g_list_free(nodes);
     return rsc->allocated_to != NULL;
 }
 
 /*!
  * \internal
  * \brief Apply a "this with" colocation to a node's allowed node scores
  *
  * \param[in,out] colocation  Colocation to apply
  * \param[in,out] rsc         Resource being assigned
  */
 static void
 apply_this_with(pcmk__colocation_t *colocation, pe_resource_t *rsc)
 {
     GHashTable *archive = NULL;
     pe_resource_t *other = colocation->primary;
 
     // In certain cases, we will need to revert the node scores
     if ((colocation->dependent_role >= pcmk_role_promoted)
         || ((colocation->score < 0) && (colocation->score > -INFINITY))) {
         archive = pcmk__copy_node_table(rsc->allowed_nodes);
     }
 
     if (pcmk_is_set(other->flags, pe_rsc_provisional)) {
         pe_rsc_trace(rsc,
                      "%s: Assigning colocation %s primary %s first"
                      "(score=%d role=%s)",
                      rsc->id, colocation->id, other->id,
                      colocation->score, role2text(colocation->dependent_role));
         other->cmds->assign(other, NULL, true);
     }
 
     // Apply the colocation score to this resource's allowed node scores
     rsc->cmds->apply_coloc_score(rsc, other, colocation, true);
     if ((archive != NULL)
         && !pcmk__any_node_available(rsc->allowed_nodes)) {
         pe_rsc_info(rsc,
                     "%s: Reverting scores from colocation with %s "
                     "because no nodes allowed",
                     rsc->id, other->id);
         g_hash_table_destroy(rsc->allowed_nodes);
         rsc->allowed_nodes = archive;
         archive = NULL;
     }
     if (archive != NULL) {
         g_hash_table_destroy(archive);
     }
 }
 
 /*!
  * \internal
  * \brief Update a Pacemaker Remote node once its connection has been assigned
  *
  * \param[in] connection  Connection resource that has been assigned
  */
 static void
 remote_connection_assigned(const pe_resource_t *connection)
 {
     pe_node_t *remote_node = pe_find_node(connection->cluster->nodes,
                                           connection->id);
 
     CRM_CHECK(remote_node != NULL, return);
 
     if ((connection->allocated_to != NULL)
         && (connection->next_role != pcmk_role_stopped)) {
 
         crm_trace("Pacemaker Remote node %s will be online",
                   remote_node->details->id);
         remote_node->details->online = TRUE;
         if (remote_node->details->unseen) {
             // Avoid unnecessary fence, since we will attempt connection
             remote_node->details->unclean = FALSE;
         }
 
     } else {
         crm_trace("Pacemaker Remote node %s will be shut down "
                   "(%sassigned connection's next role is %s)",
                   remote_node->details->id,
                   ((connection->allocated_to == NULL)? "un" : ""),
                   role2text(connection->next_role));
         remote_node->details->shutdown = TRUE;
     }
 }
 
 /*!
  * \internal
  * \brief Assign a primitive resource to a node
  *
  * \param[in,out] rsc           Resource to assign to a node
  * \param[in]     prefer        Node to prefer, if all else is equal
  * \param[in]     stop_if_fail  If \c true and \p rsc can't be assigned to a
  *                              node, set next role to stopped and update
  *                              existing actions
  *
  * \return Node that \p rsc is assigned to, if assigned entirely to one node
  *
  * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
  *       completely undo the assignment. A successful assignment can be either
  *       undone or left alone as final. A failed assignment has the same effect
  *       as calling pcmk__unassign_resource(); there are no side effects on
  *       roles or actions.
  */
 pe_node_t *
 pcmk__primitive_assign(pe_resource_t *rsc, const pe_node_t *prefer,
                        bool stop_if_fail)
 {
     GList *this_with_colocations = NULL;
     GList *with_this_colocations = NULL;
     GList *iter = NULL;
     pcmk__colocation_t *colocation = NULL;
 
     CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));
 
     // Never assign a child without parent being assigned first
     if ((rsc->parent != NULL)
         && !pcmk_is_set(rsc->parent->flags, pe_rsc_allocating)) {
         pe_rsc_debug(rsc, "%s: Assigning parent %s first",
                      rsc->id, rsc->parent->id);
         rsc->parent->cmds->assign(rsc->parent, prefer, stop_if_fail);
     }
 
     if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
         // Assignment has already been done
         const char *node_name = "no node";
 
         if (rsc->allocated_to != NULL) {
             node_name = pe__node_name(rsc->allocated_to);
         }
         pe_rsc_debug(rsc, "%s: pre-assigned to %s", rsc->id, node_name);
         return rsc->allocated_to;
     }
 
     // Ensure we detect assignment loops
     if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
         pe_rsc_debug(rsc, "Breaking assignment loop involving %s", rsc->id);
         return NULL;
     }
     pe__set_resource_flags(rsc, pe_rsc_allocating);
 
     pe__show_node_scores(true, rsc, "Pre-assignment", rsc->allowed_nodes,
                          rsc->cluster);
 
     this_with_colocations = pcmk__this_with_colocations(rsc);
     with_this_colocations = pcmk__with_this_colocations(rsc);
 
     // Apply mandatory colocations first, to satisfy as many as possible
     for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
         colocation = iter->data;
 
         if ((colocation->score <= -CRM_SCORE_INFINITY)
             || (colocation->score >= CRM_SCORE_INFINITY)) {
             apply_this_with(colocation, rsc);
         }
     }
     for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
         colocation = iter->data;
 
         if ((colocation->score <= -CRM_SCORE_INFINITY)
             || (colocation->score >= CRM_SCORE_INFINITY)) {
             pcmk__add_dependent_scores(colocation, rsc);
         }
     }
 
     pe__show_node_scores(true, rsc, "Mandatory-colocations",
                          rsc->allowed_nodes, rsc->cluster);
 
     // Then apply optional colocations
     for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
         colocation = iter->data;
 
         if ((colocation->score > -CRM_SCORE_INFINITY)
             && (colocation->score < CRM_SCORE_INFINITY)) {
             apply_this_with(colocation, rsc);
         }
     }
     for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
         colocation = iter->data;
 
         if ((colocation->score > -CRM_SCORE_INFINITY)
             && (colocation->score < CRM_SCORE_INFINITY)) {
             pcmk__add_dependent_scores(colocation, rsc);
         }
     }
 
     g_list_free(this_with_colocations);
     g_list_free(with_this_colocations);
 
     if (rsc->next_role == pcmk_role_stopped) {
         pe_rsc_trace(rsc,
                      "Banning %s from all nodes because it will be stopped",
                      rsc->id);
         resource_location(rsc, NULL, -INFINITY, XML_RSC_ATTR_TARGET_ROLE,
                           rsc->cluster);
 
     } else if ((rsc->next_role > rsc->role)
-               && !pcmk_is_set(rsc->cluster->flags, pe_flag_have_quorum)
+               && !pcmk_is_set(rsc->cluster->flags, pcmk_sched_quorate)
                && (rsc->cluster->no_quorum_policy == pcmk_no_quorum_freeze)) {
         crm_notice("Resource %s cannot be elevated from %s to %s due to "
                    "no-quorum-policy=freeze",
                    rsc->id, role2text(rsc->role), role2text(rsc->next_role));
         pe__set_next_role(rsc, rsc->role, "no-quorum-policy=freeze");
     }
 
     pe__show_node_scores(!pcmk_is_set(rsc->cluster->flags, pe_flag_show_scores),
                          rsc, __func__, rsc->allowed_nodes, rsc->cluster);
 
     // Unmanage resource if fencing is enabled but no device is configured
     if (pcmk_is_set(rsc->cluster->flags, pe_flag_stonith_enabled)
         && !pcmk_is_set(rsc->cluster->flags, pe_flag_have_stonith_resource)) {
         pe__clear_resource_flags(rsc, pe_rsc_managed);
     }
 
     if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
         // Unmanaged resources stay on their current node
         const char *reason = NULL;
         pe_node_t *assign_to = NULL;
 
         pe__set_next_role(rsc, rsc->role, "unmanaged");
         assign_to = pe__current_node(rsc);
         if (assign_to == NULL) {
             reason = "inactive";
         } else if (rsc->role == pcmk_role_promoted) {
             reason = "promoted";
         } else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
             reason = "failed";
         } else {
             reason = "active";
         }
         pe_rsc_info(rsc, "Unmanaged resource %s assigned to %s: %s", rsc->id,
                     (assign_to? assign_to->details->uname : "no node"), reason);
         pcmk__assign_resource(rsc, assign_to, true, stop_if_fail);
 
     } else if (pcmk_is_set(rsc->cluster->flags, pe_flag_stop_everything)) {
         // Must stop at some point, but be consistent with stop_if_fail
         if (stop_if_fail) {
             pe_rsc_debug(rsc, "Forcing %s to stop: stop-all-resources",
                          rsc->id);
         }
         pcmk__assign_resource(rsc, NULL, true, stop_if_fail);
 
     } else if (!assign_best_node(rsc, prefer, stop_if_fail)) {
         // Assignment failed
         if (!pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
             pe_rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
         } else if ((rsc->running_on != NULL) && stop_if_fail) {
             pe_rsc_info(rsc, "Stopping orphan resource %s", rsc->id);
         }
     }
 
     pe__clear_resource_flags(rsc, pe_rsc_allocating);
 
     if (rsc->is_remote_node) {
         remote_connection_assigned(rsc);
     }
 
     return rsc->allocated_to;
 }
 
 /*!
  * \internal
  * \brief Schedule actions to bring resource down and back to current role
  *
  * \param[in,out] rsc           Resource to restart
  * \param[in,out] current       Node that resource should be brought down on
  * \param[in]     need_stop     Whether the resource must be stopped
  * \param[in]     need_promote  Whether the resource must be promoted
  *
  * \return Role that resource would have after scheduled actions are taken
  */
 static void
 schedule_restart_actions(pe_resource_t *rsc, pe_node_t *current,
                          bool need_stop, bool need_promote)
 {
     enum rsc_role_e role = rsc->role;
     enum rsc_role_e next_role;
     rsc_transition_fn fn = NULL;
 
     pe__set_resource_flags(rsc, pe_rsc_restarting);
 
     // Bring resource down to a stop on its current node
     while (role != pcmk_role_stopped) {
         next_role = rsc_state_matrix[role][pcmk_role_stopped];
         pe_rsc_trace(rsc, "Creating %s action to take %s down from %s to %s",
                      (need_stop? "required" : "optional"), rsc->id,
                      role2text(role), role2text(next_role));
         fn = rsc_action_matrix[role][next_role];
         if (fn == NULL) {
             break;
         }
         fn(rsc, current, !need_stop);
         role = next_role;
     }
 
     // Bring resource up to its next role on its next node
     while ((rsc->role <= rsc->next_role) && (role != rsc->role)
            && !pcmk_is_set(rsc->flags, pe_rsc_block)) {
         bool required = need_stop;
 
         next_role = rsc_state_matrix[role][rsc->role];
         if ((next_role == pcmk_role_promoted) && need_promote) {
             required = true;
         }
         pe_rsc_trace(rsc, "Creating %s action to take %s up from %s to %s",
                      (required? "required" : "optional"), rsc->id,
                      role2text(role), role2text(next_role));
         fn = rsc_action_matrix[role][next_role];
         if (fn == NULL) {
             break;
         }
         fn(rsc, rsc->allocated_to, !required);
         role = next_role;
     }
 
     pe__clear_resource_flags(rsc, pe_rsc_restarting);
 }
 
 /*!
  * \internal
  * \brief If a resource's next role is not explicitly specified, set a default
  *
  * \param[in,out] rsc  Resource to set next role for
  *
  * \return "explicit" if next role was explicitly set, otherwise "implicit"
  */
 static const char *
 set_default_next_role(pe_resource_t *rsc)
 {
     if (rsc->next_role != pcmk_role_unknown) {
         return "explicit";
     }
 
     if (rsc->allocated_to == NULL) {
         pe__set_next_role(rsc, pcmk_role_stopped, "assignment");
     } else {
         pe__set_next_role(rsc, pcmk_role_started, "assignment");
     }
     return "implicit";
 }
 
 /*!
  * \internal
  * \brief Create an action to represent an already pending start
  *
  * \param[in,out] rsc  Resource to create start action for
  */
 static void
 create_pending_start(pe_resource_t *rsc)
 {
     pe_action_t *start = NULL;
 
     pe_rsc_trace(rsc,
                  "Creating action for %s to represent already pending start",
                  rsc->id);
     start = start_action(rsc, rsc->allocated_to, TRUE);
     pe__set_action_flags(start, pe_action_print_always);
 }
 
 /*!
  * \internal
  * \brief Schedule actions needed to take a resource to its next role
  *
  * \param[in,out] rsc  Resource to schedule actions for
  */
 static void
 schedule_role_transition_actions(pe_resource_t *rsc)
 {
     enum rsc_role_e role = rsc->role;
 
     while (role != rsc->next_role) {
         enum rsc_role_e next_role = rsc_state_matrix[role][rsc->next_role];
         rsc_transition_fn fn = NULL;
 
         pe_rsc_trace(rsc,
                      "Creating action to take %s from %s to %s (ending at %s)",
                      rsc->id, role2text(role), role2text(next_role),
                      role2text(rsc->next_role));
         fn = rsc_action_matrix[role][next_role];
         if (fn == NULL) {
             break;
         }
         fn(rsc, rsc->allocated_to, false);
         role = next_role;
     }
 }
 
 /*!
  * \internal
  * \brief Create all actions needed for a given primitive resource
  *
  * \param[in,out] rsc  Primitive resource to create actions for
  */
 void
 pcmk__primitive_create_actions(pe_resource_t *rsc)
 {
     bool need_stop = false;
     bool need_promote = false;
     bool is_moving = false;
     bool allow_migrate = false;
     bool multiply_active = false;
 
     pe_node_t *current = NULL;
     unsigned int num_all_active = 0;
     unsigned int num_clean_active = 0;
     const char *next_role_source = NULL;
 
     CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));
 
     next_role_source = set_default_next_role(rsc);
     pe_rsc_trace(rsc,
                  "Creating all actions for %s transition from %s to %s "
                  "(%s) on %s",
                  rsc->id, role2text(rsc->role), role2text(rsc->next_role),
                  next_role_source, pe__node_name(rsc->allocated_to));
 
     current = rsc->fns->active_node(rsc, &num_all_active, &num_clean_active);
 
     g_list_foreach(rsc->dangling_migrations, pcmk__abort_dangling_migration,
                    rsc);
 
     if ((current != NULL) && (rsc->allocated_to != NULL)
         && !pe__same_node(current, rsc->allocated_to)
         && (rsc->next_role >= pcmk_role_started)) {
 
         pe_rsc_trace(rsc, "Moving %s from %s to %s",
                      rsc->id, pe__node_name(current),
                      pe__node_name(rsc->allocated_to));
         is_moving = true;
         allow_migrate = pcmk__rsc_can_migrate(rsc, current);
 
         // This is needed even if migrating (though I'm not sure why ...)
         need_stop = true;
     }
 
     // Check whether resource is partially migrated and/or multiply active
     if ((rsc->partial_migration_source != NULL)
         && (rsc->partial_migration_target != NULL)
         && allow_migrate && (num_all_active == 2)
         && pe__same_node(current, rsc->partial_migration_source)
         && pe__same_node(rsc->allocated_to, rsc->partial_migration_target)) {
         /* A partial migration is in progress, and the migration target remains
          * the same as when the migration began.
          */
         pe_rsc_trace(rsc, "Partial migration of %s from %s to %s will continue",
                      rsc->id, pe__node_name(rsc->partial_migration_source),
                      pe__node_name(rsc->partial_migration_target));
 
     } else if ((rsc->partial_migration_source != NULL)
                || (rsc->partial_migration_target != NULL)) {
         // A partial migration is in progress but can't be continued
 
         if (num_all_active > 2) {
             // The resource is migrating *and* multiply active!
             crm_notice("Forcing recovery of %s because it is migrating "
                        "from %s to %s and possibly active elsewhere",
                        rsc->id, pe__node_name(rsc->partial_migration_source),
                        pe__node_name(rsc->partial_migration_target));
         } else {
             // The migration source or target isn't available
             crm_notice("Forcing recovery of %s because it can no longer "
                        "migrate from %s to %s",
                        rsc->id, pe__node_name(rsc->partial_migration_source),
                        pe__node_name(rsc->partial_migration_target));
         }
         need_stop = true;
         rsc->partial_migration_source = rsc->partial_migration_target = NULL;
         allow_migrate = false;
 
     } else if (pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)) {
         multiply_active = (num_all_active > 1);
     } else {
         /* If a resource has "requires" set to nothing or quorum, don't consider
          * it active on unclean nodes (similar to how all resources behave when
          * stonith-enabled is false). We can start such resources elsewhere
          * before fencing completes, and if we considered the resource active on
          * the failed node, we would attempt recovery for being active on
          * multiple nodes.
          */
         multiply_active = (num_clean_active > 1);
     }
 
     if (multiply_active) {
         const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 
         // Resource was (possibly) incorrectly multiply active
         pe_proc_err("%s resource %s might be active on %u nodes (%s)",
                     pcmk__s(class, "Untyped"), rsc->id, num_all_active,
                     recovery2text(rsc->recovery_type));
         crm_notice("See https://wiki.clusterlabs.org/wiki/FAQ"
                    "#Resource_is_Too_Active for more information");
 
         switch (rsc->recovery_type) {
             case pcmk_multiply_active_restart:
                 need_stop = true;
                 break;
             case pcmk_multiply_active_unexpected:
                 need_stop = true; // stop_resource() will skip expected node
                 pe__set_resource_flags(rsc, pe_rsc_stop_unexpected);
                 break;
             default:
                 break;
         }
 
     } else {
         pe__clear_resource_flags(rsc, pe_rsc_stop_unexpected);
     }
 
     if (pcmk_is_set(rsc->flags, pe_rsc_start_pending)) {
         create_pending_start(rsc);
     }
 
     if (is_moving) {
         // Remaining tests are only for resources staying where they are
 
     } else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
         if (pcmk_is_set(rsc->flags, pe_rsc_stop)) {
             need_stop = true;
             pe_rsc_trace(rsc, "Recovering %s", rsc->id);
         } else {
             pe_rsc_trace(rsc, "Recovering %s by demotion", rsc->id);
             if (rsc->next_role == pcmk_role_promoted) {
                 need_promote = true;
             }
         }
 
     } else if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
         pe_rsc_trace(rsc, "Blocking further actions on %s", rsc->id);
         need_stop = true;
 
     } else if ((rsc->role > pcmk_role_started) && (current != NULL)
                && (rsc->allocated_to != NULL)) {
         pe_action_t *start = NULL;
 
         pe_rsc_trace(rsc, "Creating start action for promoted resource %s",
                      rsc->id);
         start = start_action(rsc, rsc->allocated_to, TRUE);
         if (!pcmk_is_set(start->flags, pe_action_optional)) {
             // Recovery of a promoted resource
             pe_rsc_trace(rsc, "%s restart is required for recovery", rsc->id);
             need_stop = true;
         }
     }
 
     // Create any actions needed to bring resource down and back up to same role
     schedule_restart_actions(rsc, current, need_stop, need_promote);
 
     // Create any actions needed to take resource from this role to the next
     schedule_role_transition_actions(rsc);
 
     pcmk__create_recurring_actions(rsc);
 
     if (allow_migrate) {
         pcmk__create_migration_actions(rsc, current);
     }
 }
 
 /*!
  * \internal
  * \brief Ban a resource from any allowed nodes that are Pacemaker Remote nodes
  *
  * \param[in] rsc  Resource to check
  */
 static void
 rsc_avoids_remote_nodes(const pe_resource_t *rsc)
 {
     GHashTableIter iter;
     pe_node_t *node = NULL;
 
     g_hash_table_iter_init(&iter, rsc->allowed_nodes);
     while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
         if (node->details->remote_rsc != NULL) {
             node->weight = -INFINITY;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Return allowed nodes as (possibly sorted) list
  *
  * Convert a resource's hash table of allowed nodes to a list. If printing to
  * stdout, sort the list, to keep action ID numbers consistent for regression
  * test output (while avoiding the performance hit on a live cluster).
  *
  * \param[in] rsc       Resource to check for allowed nodes
  *
  * \return List of resource's allowed nodes
  * \note Callers should take care not to rely on the list being sorted.
  */
 static GList *
 allowed_nodes_as_list(const pe_resource_t *rsc)
 {
     GList *allowed_nodes = NULL;
 
     if (rsc->allowed_nodes) {
         allowed_nodes = g_hash_table_get_values(rsc->allowed_nodes);
     }
 
     if (!pcmk__is_daemon) {
         allowed_nodes = g_list_sort(allowed_nodes, pe__cmp_node_name);
     }
 
     return allowed_nodes;
 }
 
 /*!
  * \internal
  * \brief Create implicit constraints needed for a primitive resource
  *
  * \param[in,out] rsc  Primitive resource to create implicit constraints for
  */
 void
 pcmk__primitive_internal_constraints(pe_resource_t *rsc)
 {
     GList *allowed_nodes = NULL;
     bool check_unfencing = false;
     bool check_utilization = false;
 
     CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));
 
     if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
         pe_rsc_trace(rsc,
                      "Skipping implicit constraints for unmanaged resource %s",
                      rsc->id);
         return;
     }
 
     // Whether resource requires unfencing
     check_unfencing = !pcmk_is_set(rsc->flags, pe_rsc_fence_device)
                       && pcmk_is_set(rsc->cluster->flags,
                                      pe_flag_enable_unfencing)
                       && pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing);
 
     // Whether a non-default placement strategy is used
     check_utilization = (g_hash_table_size(rsc->utilization) > 0)
                          && !pcmk__str_eq(rsc->cluster->placement_strategy,
                                           "default", pcmk__str_casei);
 
     // Order stops before starts (i.e. restart)
     pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0), NULL,
                        rsc, pcmk__op_key(rsc->id, PCMK_ACTION_START, 0), NULL,
                        pe_order_optional|pe_order_implies_then|pe_order_restart,
                        rsc->cluster);
 
     // Promotable ordering: demote before stop, start before promote
     if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
                     pe_rsc_promotable)
         || (rsc->role > pcmk_role_unpromoted)) {
 
         pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_DEMOTE, 0),
                            NULL,
                            rsc, pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0),
                            NULL,
                            pe_order_promoted_implies_first, rsc->cluster);
 
         pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_START, 0),
                            NULL,
                            rsc, pcmk__op_key(rsc->id, PCMK_ACTION_PROMOTE, 0),
                            NULL,
                            pe_order_runnable_left, rsc->cluster);
     }
 
     // Don't clear resource history if probing on same node
     pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0),
                        NULL, rsc,
                        pcmk__op_key(rsc->id, PCMK_ACTION_MONITOR, 0),
                        NULL, pe_order_same_node|pe_order_then_cancels_first,
                        rsc->cluster);
 
     // Certain checks need allowed nodes
     if (check_unfencing || check_utilization || (rsc->container != NULL)) {
         allowed_nodes = allowed_nodes_as_list(rsc);
     }
 
     if (check_unfencing) {
         g_list_foreach(allowed_nodes, pcmk__order_restart_vs_unfence, rsc);
     }
 
     if (check_utilization) {
         pcmk__create_utilization_constraints(rsc, allowed_nodes);
     }
 
     if (rsc->container != NULL) {
         pe_resource_t *remote_rsc = NULL;
 
         if (rsc->is_remote_node) {
             // rsc is the implicit remote connection for a guest or bundle node
 
             /* Guest resources are not allowed to run on Pacemaker Remote nodes,
              * to avoid nesting remotes. However, bundles are allowed.
              */
             if (!pcmk_is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
                 rsc_avoids_remote_nodes(rsc->container);
             }
 
             /* If someone cleans up a guest or bundle node's container, we will
              * likely schedule a (re-)probe of the container and recovery of the
              * connection. Order the connection stop after the container probe,
              * so that if we detect the container running, we will trigger a new
              * transition and avoid the unnecessary recovery.
              */
             pcmk__order_resource_actions(rsc->container, PCMK_ACTION_MONITOR,
                                          rsc, PCMK_ACTION_STOP,
                                          pe_order_optional);
 
         /* A user can specify that a resource must start on a Pacemaker Remote
          * node by explicitly configuring it with the container=NODENAME
          * meta-attribute. This is of questionable merit, since location
          * constraints can accomplish the same thing. But we support it, so here
          * we check whether a resource (that is not itself a remote connection)
          * has container set to a remote node or guest node resource.
          */
         } else if (rsc->container->is_remote_node) {
             remote_rsc = rsc->container;
         } else  {
             remote_rsc = pe__resource_contains_guest_node(rsc->cluster,
                                                           rsc->container);
         }
 
         if (remote_rsc != NULL) {
             /* Force the resource on the Pacemaker Remote node instead of
              * colocating the resource with the container resource.
              */
             for (GList *item = allowed_nodes; item; item = item->next) {
                 pe_node_t *node = item->data;
 
                 if (node->details->remote_rsc != remote_rsc) {
                     node->weight = -INFINITY;
                 }
             }
 
         } else {
             /* This resource is either a filler for a container that does NOT
              * represent a Pacemaker Remote node, or a Pacemaker Remote
              * connection resource for a guest node or bundle.
              */
             int score;
 
             crm_trace("Order and colocate %s relative to its container %s",
                       rsc->id, rsc->container->id);
 
             pcmk__new_ordering(rsc->container,
                                pcmk__op_key(rsc->container->id,
                                             PCMK_ACTION_START, 0),
                                NULL, rsc,
                                pcmk__op_key(rsc->id, PCMK_ACTION_START, 0),
                                NULL,
                                pe_order_implies_then|pe_order_runnable_left,
                                rsc->cluster);
 
             pcmk__new_ordering(rsc,
                                pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0),
                                NULL,
                                rsc->container,
                                pcmk__op_key(rsc->container->id,
                                             PCMK_ACTION_STOP, 0),
                                NULL, pe_order_implies_first, rsc->cluster);
 
             if (pcmk_is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
                 score = 10000;    /* Highly preferred but not essential */
             } else {
                 score = INFINITY; /* Force them to run on the same host */
             }
             pcmk__new_colocation("#resource-with-container", NULL, score, rsc,
                                  rsc->container, NULL, NULL,
                                  pcmk__coloc_influence);
         }
     }
 
     if (rsc->is_remote_node || pcmk_is_set(rsc->flags, pe_rsc_fence_device)) {
         /* Remote connections and fencing devices are not allowed to run on
          * Pacemaker Remote nodes
          */
         rsc_avoids_remote_nodes(rsc);
     }
     g_list_free(allowed_nodes);
 }
 
 /*!
  * \internal
  * \brief Apply a colocation's score to node scores or resource priority
  *
  * Given a colocation constraint, apply its score to the dependent's
  * allowed node scores (if we are still placing resources) or priority (if
  * we are choosing promotable clone instance roles).
  *
  * \param[in,out] dependent      Dependent resource in colocation
  * \param[in]     primary        Primary resource in colocation
  * \param[in]     colocation     Colocation constraint to apply
  * \param[in]     for_dependent  true if called on behalf of dependent
  */
 void
 pcmk__primitive_apply_coloc_score(pe_resource_t *dependent,
                                   const pe_resource_t *primary,
                                   const pcmk__colocation_t *colocation,
                                   bool for_dependent)
 {
     enum pcmk__coloc_affects filter_results;
 
     CRM_ASSERT((dependent != NULL) && (primary != NULL)
                && (colocation != NULL));
 
     if (for_dependent) {
         // Always process on behalf of primary resource
         primary->cmds->apply_coloc_score(dependent, primary, colocation, false);
         return;
     }
 
     filter_results = pcmk__colocation_affects(dependent, primary, colocation,
                                               false);
     pe_rsc_trace(dependent, "%s %s with %s (%s, score=%d, filter=%d)",
                  ((colocation->score > 0)? "Colocating" : "Anti-colocating"),
                  dependent->id, primary->id, colocation->id, colocation->score,
                  filter_results);
 
     switch (filter_results) {
         case pcmk__coloc_affects_role:
             pcmk__apply_coloc_to_priority(dependent, primary, colocation);
             break;
         case pcmk__coloc_affects_location:
             pcmk__apply_coloc_to_scores(dependent, primary, colocation);
             break;
         default: // pcmk__coloc_affects_nothing
             return;
     }
 }
 
 /* Primitive implementation of
  * resource_alloc_functions_t:with_this_colocations()
  */
 void
 pcmk__with_primitive_colocations(const pe_resource_t *rsc,
                                  const pe_resource_t *orig_rsc, GList **list)
 {
     CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
                && (list != NULL));
 
     if (rsc == orig_rsc) {
         /* For the resource itself, add all of its own colocations and relevant
          * colocations from its parent (if any).
          */
         pcmk__add_with_this_list(list, rsc->rsc_cons_lhs, orig_rsc);
         if (rsc->parent != NULL) {
             rsc->parent->cmds->with_this_colocations(rsc->parent, orig_rsc, list);
         }
     } else {
         // For an ancestor, add only explicitly configured constraints
         for (GList *iter = rsc->rsc_cons_lhs; iter != NULL; iter = iter->next) {
             pcmk__colocation_t *colocation = iter->data;
 
             if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
                 pcmk__add_with_this(list, colocation, orig_rsc);
             }
         }
     }
 }
 
 /* Primitive implementation of
  * resource_alloc_functions_t:this_with_colocations()
  */
 void
 pcmk__primitive_with_colocations(const pe_resource_t *rsc,
                                  const pe_resource_t *orig_rsc, GList **list)
 {
     CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
                && (list != NULL));
 
     if (rsc == orig_rsc) {
         /* For the resource itself, add all of its own colocations and relevant
          * colocations from its parent (if any).
          */
         pcmk__add_this_with_list(list, rsc->rsc_cons, orig_rsc);
         if (rsc->parent != NULL) {
             rsc->parent->cmds->this_with_colocations(rsc->parent, orig_rsc, list);
         }
     } else {
         // For an ancestor, add only explicitly configured constraints
         for (GList *iter = rsc->rsc_cons; iter != NULL; iter = iter->next) {
             pcmk__colocation_t *colocation = iter->data;
 
             if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
                 pcmk__add_this_with(list, colocation, orig_rsc);
             }
         }
     }
 }
 
 /*!
  * \internal
  * \brief Return action flags for a given primitive resource action
  *
  * \param[in,out] action  Action to get flags for
  * \param[in]     node    If not NULL, limit effects to this node (ignored)
  *
  * \return Flags appropriate to \p action on \p node
  */
 uint32_t
 pcmk__primitive_action_flags(pe_action_t *action, const pe_node_t *node)
 {
     CRM_ASSERT(action != NULL);
     return (uint32_t) action->flags;
 }
 
 /*!
  * \internal
  * \brief Check whether a node is a multiply active resource's expected node
  *
  * \param[in] rsc  Resource to check
  * \param[in] node  Node to check
  *
  * \return true if \p rsc is multiply active with multiple-active set to
  *         stop_unexpected, and \p node is the node where it will remain active
  * \note This assumes that the resource's next role cannot be changed to stopped
  *       after this is called, which should be reasonable if status has already
  *       been unpacked and resources have been assigned to nodes.
  */
 static bool
 is_expected_node(const pe_resource_t *rsc, const pe_node_t *node)
 {
     return pcmk_all_flags_set(rsc->flags,
                               pe_rsc_stop_unexpected|pe_rsc_restarting)
            && (rsc->next_role > pcmk_role_stopped)
            && pe__same_node(rsc->allocated_to, node);
 }
 
 /*!
  * \internal
  * \brief Schedule actions needed to stop a resource wherever it is active
  *
  * \param[in,out] rsc       Resource being stopped
  * \param[in]     node      Node where resource is being stopped (ignored)
  * \param[in]     optional  Whether actions should be optional
  */
 static void
 stop_resource(pe_resource_t *rsc, pe_node_t *node, bool optional)
 {
     for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
         pe_node_t *current = (pe_node_t *) iter->data;
         pe_action_t *stop = NULL;
 
         if (is_expected_node(rsc, current)) {
             /* We are scheduling restart actions for a multiply active resource
              * with multiple-active=stop_unexpected, and this is where it should
              * not be stopped.
              */
             pe_rsc_trace(rsc,
                          "Skipping stop of multiply active resource %s "
                          "on expected node %s",
                          rsc->id, pe__node_name(current));
             continue;
         }
 
         if (rsc->partial_migration_target != NULL) {
             // Continue migration if node originally was and remains target
             if (pe__same_node(current, rsc->partial_migration_target)
                 && pe__same_node(current, rsc->allocated_to)) {
                 pe_rsc_trace(rsc,
                              "Skipping stop of %s on %s "
                              "because partial migration there will continue",
                              rsc->id, pe__node_name(current));
                 continue;
             } else {
                 pe_rsc_trace(rsc,
                              "Forcing stop of %s on %s "
                              "because migration target changed",
                              rsc->id, pe__node_name(current));
                 optional = false;
             }
         }
 
         pe_rsc_trace(rsc, "Scheduling stop of %s on %s",
                      rsc->id, pe__node_name(current));
         stop = stop_action(rsc, current, optional);
 
         if (rsc->allocated_to == NULL) {
             pe_action_set_reason(stop, "node availability", true);
         } else if (pcmk_all_flags_set(rsc->flags, pe_rsc_restarting
                                                   |pe_rsc_stop_unexpected)) {
             /* We are stopping a multiply active resource on a node that is
              * not its expected node, and we are still scheduling restart
              * actions, so the stop is for being multiply active.
              */
             pe_action_set_reason(stop, "being multiply active", true);
         }
 
         if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
             pe__clear_action_flags(stop, pe_action_runnable);
         }
 
         if (pcmk_is_set(rsc->cluster->flags, pe_flag_remove_after_stop)) {
             pcmk__schedule_cleanup(rsc, current, optional);
         }
 
         if (pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing)) {
             pe_action_t *unfence = pe_fence_op(current, PCMK_ACTION_ON, true,
                                                NULL, false, rsc->cluster);
 
             order_actions(stop, unfence, pe_order_implies_first);
             if (!pcmk__node_unfenced(current)) {
                 pe_proc_err("Stopping %s until %s can be unfenced",
                             rsc->id, pe__node_name(current));
             }
         }
     }
 }
 
 /*!
  * \internal
  * \brief Schedule actions needed to start a resource on a node
  *
  * \param[in,out] rsc       Resource being started
  * \param[in,out] node      Node where resource should be started
  * \param[in]     optional  Whether actions should be optional
  */
 static void
 start_resource(pe_resource_t *rsc, pe_node_t *node, bool optional)
 {
     pe_action_t *start = NULL;
 
     CRM_ASSERT(node != NULL);
 
     pe_rsc_trace(rsc, "Scheduling %s start of %s on %s (score %d)",
                  (optional? "optional" : "required"), rsc->id,
                  pe__node_name(node), node->weight);
     start = start_action(rsc, node, TRUE);
 
     pcmk__order_vs_unfence(rsc, node, start, pe_order_implies_then);
 
     if (pcmk_is_set(start->flags, pe_action_runnable) && !optional) {
         pe__clear_action_flags(start, pe_action_optional);
     }
 
     if (is_expected_node(rsc, node)) {
         /* This could be a problem if the start becomes necessary for other
          * reasons later.
          */
         pe_rsc_trace(rsc,
                      "Start of multiply active resouce %s "
                      "on expected node %s will be a pseudo-action",
                      rsc->id, pe__node_name(node));
         pe__set_action_flags(start, pe_action_pseudo);
     }
 }
 
 /*!
  * \internal
  * \brief Schedule actions needed to promote a resource on a node
  *
  * \param[in,out] rsc       Resource being promoted
  * \param[in]     node      Node where resource should be promoted
  * \param[in]     optional  Whether actions should be optional
  */
 static void
 promote_resource(pe_resource_t *rsc, pe_node_t *node, bool optional)
 {
     GList *iter = NULL;
     GList *action_list = NULL;
     bool runnable = true;
 
     CRM_ASSERT(node != NULL);
 
     // Any start must be runnable for promotion to be runnable
     action_list = pe__resource_actions(rsc, node, PCMK_ACTION_START, true);
     for (iter = action_list; iter != NULL; iter = iter->next) {
         pe_action_t *start = (pe_action_t *) iter->data;
 
         if (!pcmk_is_set(start->flags, pe_action_runnable)) {
             runnable = false;
         }
     }
     g_list_free(action_list);
 
     if (runnable) {
         pe_action_t *promote = promote_action(rsc, node, optional);
 
         pe_rsc_trace(rsc, "Scheduling %s promotion of %s on %s",
                      (optional? "optional" : "required"), rsc->id,
                      pe__node_name(node));
 
         if (is_expected_node(rsc, node)) {
             /* This could be a problem if the promote becomes necessary for
              * other reasons later.
              */
             pe_rsc_trace(rsc,
                          "Promotion of multiply active resouce %s "
                          "on expected node %s will be a pseudo-action",
                          rsc->id, pe__node_name(node));
             pe__set_action_flags(promote, pe_action_pseudo);
         }
     } else {
         pe_rsc_trace(rsc, "Not promoting %s on %s: start unrunnable",
                      rsc->id, pe__node_name(node));
         action_list = pe__resource_actions(rsc, node, PCMK_ACTION_PROMOTE,
                                            true);
         for (iter = action_list; iter != NULL; iter = iter->next) {
             pe_action_t *promote = (pe_action_t *) iter->data;
 
             pe__clear_action_flags(promote, pe_action_runnable);
         }
         g_list_free(action_list);
     }
 }
 
 /*!
  * \internal
  * \brief Schedule actions needed to demote a resource wherever it is active
  *
  * \param[in,out] rsc       Resource being demoted
  * \param[in]     node      Node where resource should be demoted (ignored)
  * \param[in]     optional  Whether actions should be optional
  */
 static void
 demote_resource(pe_resource_t *rsc, pe_node_t *node, bool optional)
 {
     /* Since this will only be called for a primitive (possibly as an instance
      * of a collective resource), the resource is multiply active if it is
      * running on more than one node, so we want to demote on all of them as
      * part of recovery, regardless of which one is the desired node.
      */
     for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
         pe_node_t *current = (pe_node_t *) iter->data;
 
         if (is_expected_node(rsc, current)) {
             pe_rsc_trace(rsc,
                          "Skipping demote of multiply active resource %s "
                          "on expected node %s",
                          rsc->id, pe__node_name(current));
         } else {
             pe_rsc_trace(rsc, "Scheduling %s demotion of %s on %s",
                          (optional? "optional" : "required"), rsc->id,
                          pe__node_name(current));
             demote_action(rsc, current, optional);
         }
     }
 }
 
 static void
 assert_role_error(pe_resource_t *rsc, pe_node_t *node, bool optional)
 {
     CRM_ASSERT(false);
 }
 
 /*!
  * \internal
  * \brief Schedule cleanup of a resource
  *
  * \param[in,out] rsc       Resource to clean up
  * \param[in]     node      Node to clean up on
  * \param[in]     optional  Whether clean-up should be optional
  */
 void
 pcmk__schedule_cleanup(pe_resource_t *rsc, const pe_node_t *node, bool optional)
 {
     /* If the cleanup is required, its orderings are optional, because they're
      * relevant only if both actions are required. Conversely, if the cleanup is
      * optional, the orderings make the then action required if the first action
      * becomes required.
      */
     uint32_t flag = optional? pe_order_implies_then : pe_order_optional;
 
     CRM_CHECK((rsc != NULL) && (node != NULL), return);
 
     if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
         pe_rsc_trace(rsc, "Skipping clean-up of %s on %s: resource failed",
                      rsc->id, pe__node_name(node));
         return;
     }
 
     if (node->details->unclean || !node->details->online) {
         pe_rsc_trace(rsc, "Skipping clean-up of %s on %s: node unavailable",
                      rsc->id, pe__node_name(node));
         return;
     }
 
     crm_notice("Scheduling clean-up of %s on %s", rsc->id, pe__node_name(node));
     delete_action(rsc, node, optional);
 
     // stop -> clean-up -> start
     pcmk__order_resource_actions(rsc, PCMK_ACTION_STOP,
                                  rsc, PCMK_ACTION_DELETE, flag);
     pcmk__order_resource_actions(rsc, PCMK_ACTION_DELETE,
                                  rsc, PCMK_ACTION_START, flag);
 }
 
 /*!
  * \internal
  * \brief Add primitive meta-attributes relevant to graph actions to XML
  *
  * \param[in]     rsc  Primitive resource whose meta-attributes should be added
  * \param[in,out] xml  Transition graph action attributes XML to add to
  */
 void
 pcmk__primitive_add_graph_meta(const pe_resource_t *rsc, xmlNode *xml)
 {
     char *name = NULL;
     char *value = NULL;
     const pe_resource_t *parent = NULL;
 
     CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
                && (xml != NULL));
 
     /* Clone instance numbers get set internally as meta-attributes, and are
      * needed in the transition graph (for example, to tell unique clone
      * instances apart).
      */
     value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION);
     if (value != NULL) {
         name = crm_meta_name(XML_RSC_ATTR_INCARNATION);
         crm_xml_add(xml, name, value);
         free(name);
     }
 
     // Not sure if this one is really needed ...
     value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_REMOTE_NODE);
     if (value != NULL) {
         name = crm_meta_name(XML_RSC_ATTR_REMOTE_NODE);
         crm_xml_add(xml, name, value);
         free(name);
     }
 
     /* The container meta-attribute can be set on the primitive itself or one of
      * its parents (for example, a group inside a container resource), so check
      * them all, and keep the highest one found.
      */
     for (parent = rsc; parent != NULL; parent = parent->parent) {
         if (parent->container != NULL) {
             crm_xml_add(xml, CRM_META "_" XML_RSC_ATTR_CONTAINER,
                         parent->container->id);
         }
     }
 
     /* Bundle replica children will get their external-ip set internally as a
      * meta-attribute. The graph action needs it, but under a different naming
      * convention than other meta-attributes.
      */
     value = g_hash_table_lookup(rsc->meta, "external-ip");
     if (value != NULL) {
         crm_xml_add(xml, "pcmk_external_ip", value);
     }
 }
 
 // Primitive implementation of resource_alloc_functions_t:add_utilization()
 void
 pcmk__primitive_add_utilization(const pe_resource_t *rsc,
                                 const pe_resource_t *orig_rsc, GList *all_rscs,
                                 GHashTable *utilization)
 {
     CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)
                && (orig_rsc != NULL) && (utilization != NULL));
 
     if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
         return;
     }
 
     pe_rsc_trace(orig_rsc, "%s: Adding primitive %s as colocated utilization",
                  orig_rsc->id, rsc->id);
     pcmk__release_node_capacity(utilization, rsc);
 }
 
 /*!
  * \internal
  * \brief Get epoch time of node's shutdown attribute (or now if none)
  *
  * \param[in,out] node  Node to check
  *
  * \return Epoch time corresponding to shutdown attribute if set or now if not
  */
 static time_t
 shutdown_time(pe_node_t *node)
 {
     const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
     time_t result = 0;
 
     if (shutdown != NULL) {
         long long result_ll;
 
         if (pcmk__scan_ll(shutdown, &result_ll, 0LL) == pcmk_rc_ok) {
             result = (time_t) result_ll;
         }
     }
     return (result == 0)? get_effective_time(node->details->data_set) : result;
 }
 
 /*!
  * \internal
  * \brief Ban a resource from a node if it's not locked to the node
  *
  * \param[in]     data       Node to check
  * \param[in,out] user_data  Resource to check
  */
 static void
 ban_if_not_locked(gpointer data, gpointer user_data)
 {
     const pe_node_t *node = (const pe_node_t *) data;
     pe_resource_t *rsc = (pe_resource_t *) user_data;
 
     if (strcmp(node->details->uname, rsc->lock_node->details->uname) != 0) {
         resource_location(rsc, node, -CRM_SCORE_INFINITY,
                           XML_CONFIG_ATTR_SHUTDOWN_LOCK, rsc->cluster);
     }
 }
 
 // Primitive implementation of resource_alloc_functions_t:shutdown_lock()
 void
 pcmk__primitive_shutdown_lock(pe_resource_t *rsc)
 {
     const char *class = NULL;
 
     CRM_ASSERT((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive));
 
     class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 
     // Fence devices and remote connections can't be locked
     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_null_matches)
         || pe__resource_is_remote_conn(rsc)) {
         return;
     }
 
     if (rsc->lock_node != NULL) {
         // The lock was obtained from resource history
 
         if (rsc->running_on != NULL) {
             /* The resource was started elsewhere even though it is now
              * considered locked. This shouldn't be possible, but as a
              * failsafe, we don't want to disturb the resource now.
              */
             pe_rsc_info(rsc,
                         "Cancelling shutdown lock because %s is already active",
                         rsc->id);
             pe__clear_resource_history(rsc, rsc->lock_node, rsc->cluster);
             rsc->lock_node = NULL;
             rsc->lock_time = 0;
         }
 
     // Only a resource active on exactly one node can be locked
     } else if (pcmk__list_of_1(rsc->running_on)) {
         pe_node_t *node = rsc->running_on->data;
 
         if (node->details->shutdown) {
             if (node->details->unclean) {
                 pe_rsc_debug(rsc, "Not locking %s to unclean %s for shutdown",
                              rsc->id, pe__node_name(node));
             } else {
                 rsc->lock_node = node;
                 rsc->lock_time = shutdown_time(node);
             }
         }
     }
 
     if (rsc->lock_node == NULL) {
         // No lock needed
         return;
     }
 
     if (rsc->cluster->shutdown_lock > 0) {
         time_t lock_expiration = rsc->lock_time + rsc->cluster->shutdown_lock;
 
         pe_rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)",
                     rsc->id, pe__node_name(rsc->lock_node),
                     (long long) lock_expiration);
         pe__update_recheck_time(++lock_expiration, rsc->cluster);
     } else {
         pe_rsc_info(rsc, "Locking %s to %s due to shutdown",
                     rsc->id, pe__node_name(rsc->lock_node));
     }
 
     // If resource is locked to one node, ban it from all other nodes
     g_list_foreach(rsc->cluster->nodes, ban_if_not_locked, rsc);
 }
diff --git a/lib/pacemaker/pcmk_sched_promotable.c b/lib/pacemaker/pcmk_sched_promotable.c
index 76bf87ad01..2da1c2686c 100644
--- a/lib/pacemaker/pcmk_sched_promotable.c
+++ b/lib/pacemaker/pcmk_sched_promotable.c
@@ -1,1299 +1,1299 @@
 /*
  * Copyright 2004-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm/msg_xml.h>
 #include <pacemaker-internal.h>
 
 #include "libpacemaker_private.h"
 
 /*!
  * \internal
  * \brief Add implicit promotion ordering for a promotable instance
  *
  * \param[in,out] clone  Clone resource
  * \param[in,out] child  Instance of \p clone being ordered
  * \param[in,out] last   Previous instance ordered (NULL if \p child is first)
  */
 static void
 order_instance_promotion(pe_resource_t *clone, pe_resource_t *child,
                          pe_resource_t *last)
 {
     // "Promote clone" -> promote instance -> "clone promoted"
     pcmk__order_resource_actions(clone, PCMK_ACTION_PROMOTE,
                                  child, PCMK_ACTION_PROMOTE,
                                  pe_order_optional);
     pcmk__order_resource_actions(child, PCMK_ACTION_PROMOTE,
                                  clone, PCMK_ACTION_PROMOTED,
                                  pe_order_optional);
 
     // If clone is ordered, order this instance relative to last
     if ((last != NULL) && pe__clone_is_ordered(clone)) {
         pcmk__order_resource_actions(last, PCMK_ACTION_PROMOTE,
                                      child, PCMK_ACTION_PROMOTE,
                                      pe_order_optional);
     }
 }
 
 /*!
  * \internal
  * \brief Add implicit demotion ordering for a promotable instance
  *
  * \param[in,out] clone  Clone resource
  * \param[in,out] child  Instance of \p clone being ordered
  * \param[in]     last   Previous instance ordered (NULL if \p child is first)
  */
 static void
 order_instance_demotion(pe_resource_t *clone, pe_resource_t *child,
                         pe_resource_t *last)
 {
     // "Demote clone" -> demote instance -> "clone demoted"
     pcmk__order_resource_actions(clone, PCMK_ACTION_DEMOTE, child,
                                  PCMK_ACTION_DEMOTE,
                                  pe_order_implies_first_printed);
     pcmk__order_resource_actions(child, PCMK_ACTION_DEMOTE,
                                  clone, PCMK_ACTION_DEMOTED,
                                  pe_order_implies_then_printed);
 
     // If clone is ordered, order this instance relative to last
     if ((last != NULL) && pe__clone_is_ordered(clone)) {
         pcmk__order_resource_actions(child, PCMK_ACTION_DEMOTE, last,
                                      PCMK_ACTION_DEMOTE, pe_order_optional);
     }
 }
 
 /*!
  * \internal
  * \brief Check whether an instance will be promoted or demoted
  *
  * \param[in]  rsc        Instance to check
  * \param[out] demoting   If \p rsc will be demoted, this will be set to true
  * \param[out] promoting  If \p rsc will be promoted, this will be set to true
  */
 static void
 check_for_role_change(const pe_resource_t *rsc, bool *demoting, bool *promoting)
 {
     const GList *iter = NULL;
 
     // If this is a cloned group, check group members recursively
     if (rsc->children != NULL) {
         for (iter = rsc->children; iter != NULL; iter = iter->next) {
             check_for_role_change((const pe_resource_t *) iter->data,
                                   demoting, promoting);
         }
         return;
     }
 
     for (iter = rsc->actions; iter != NULL; iter = iter->next) {
         const pe_action_t *action = (const pe_action_t *) iter->data;
 
         if (*promoting && *demoting) {
             return;
 
         } else if (pcmk_is_set(action->flags, pe_action_optional)) {
             continue;
 
         } else if (pcmk__str_eq(PCMK_ACTION_DEMOTE, action->task,
                                 pcmk__str_none)) {
             *demoting = true;
 
         } else if (pcmk__str_eq(PCMK_ACTION_PROMOTE, action->task,
                                 pcmk__str_none)) {
             *promoting = true;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Add promoted-role location constraint scores to an instance's priority
  *
  * Adjust a promotable clone instance's promotion priority by the scores of any
  * location constraints in a list that are both limited to the promoted role and
  * for the node where the instance will be placed.
  *
  * \param[in,out] child                 Promotable clone instance
  * \param[in]     location_constraints  List of location constraints to apply
  * \param[in]     chosen                Node where \p child will be placed
  */
 static void
 apply_promoted_locations(pe_resource_t *child,
                          const GList *location_constraints,
                          const pe_node_t *chosen)
 {
     for (const GList *iter = location_constraints; iter; iter = iter->next) {
         const pe__location_t *location = iter->data;
         const pe_node_t *constraint_node = NULL;
 
         if (location->role_filter == pcmk_role_promoted) {
             constraint_node = pe_find_node_id(location->node_list_rh,
                                               chosen->details->id);
         }
         if (constraint_node != NULL) {
             int new_priority = pcmk__add_scores(child->priority,
                                                 constraint_node->weight);
 
             pe_rsc_trace(child,
                          "Applying location %s to %s promotion priority on %s: "
                          "%s + %s = %s",
                          location->id, child->id,
                          pe__node_name(constraint_node),
                          pcmk_readable_score(child->priority),
                          pcmk_readable_score(constraint_node->weight),
                          pcmk_readable_score(new_priority));
             child->priority = new_priority;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Get the node that an instance will be promoted on
  *
  * \param[in] rsc  Promotable clone instance to check
  *
  * \return Node that \p rsc will be promoted on, or NULL if none
  */
 static pe_node_t *
 node_to_be_promoted_on(const pe_resource_t *rsc)
 {
     pe_node_t *node = NULL;
     pe_node_t *local_node = NULL;
     const pe_resource_t *parent = NULL;
 
     // If this is a cloned group, bail if any group member can't be promoted
     for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
         pe_resource_t *child = (pe_resource_t *) iter->data;
 
         if (node_to_be_promoted_on(child) == NULL) {
             pe_rsc_trace(rsc,
                          "%s can't be promoted because member %s can't",
                          rsc->id, child->id);
             return NULL;
         }
     }
 
     node = rsc->fns->location(rsc, NULL, FALSE);
     if (node == NULL) {
         pe_rsc_trace(rsc, "%s can't be promoted because it won't be active",
                      rsc->id);
         return NULL;
 
     } else if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
         if (rsc->fns->state(rsc, TRUE) == pcmk_role_promoted) {
             crm_notice("Unmanaged instance %s will be left promoted on %s",
                        rsc->id, pe__node_name(node));
         } else {
             pe_rsc_trace(rsc, "%s can't be promoted because it is unmanaged",
                          rsc->id);
             return NULL;
         }
 
     } else if (rsc->priority < 0) {
         pe_rsc_trace(rsc,
                      "%s can't be promoted because its promotion priority %d "
                      "is negative",
                      rsc->id, rsc->priority);
         return NULL;
 
     } else if (!pcmk__node_available(node, false, true)) {
         pe_rsc_trace(rsc, "%s can't be promoted because %s can't run resources",
                      rsc->id, pe__node_name(node));
         return NULL;
     }
 
     parent = pe__const_top_resource(rsc, false);
     local_node = g_hash_table_lookup(parent->allowed_nodes, node->details->id);
 
     if (local_node == NULL) {
         /* It should not be possible for the scheduler to have assigned the
          * instance to a node where its parent is not allowed, but it's good to
          * have a fail-safe.
          */
         if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
             crm_warn("%s can't be promoted because %s is not allowed on %s "
                      "(scheduler bug?)",
                      rsc->id, parent->id, pe__node_name(node));
         } // else the instance is unmanaged and already promoted
         return NULL;
 
     } else if ((local_node->count >= pe__clone_promoted_node_max(parent))
                && pcmk_is_set(rsc->flags, pe_rsc_managed)) {
         pe_rsc_trace(rsc,
                      "%s can't be promoted because %s has "
                      "maximum promoted instances already",
                      rsc->id, pe__node_name(node));
         return NULL;
     }
 
     return local_node;
 }
 
 /*!
  * \internal
  * \brief Compare two promotable clone instances by promotion priority
  *
  * \param[in] a  First instance to compare
  * \param[in] b  Second instance to compare
  *
  * \return A negative number if \p a has higher promotion priority,
  *         a positive number if \p b has higher promotion priority,
  *         or 0 if promotion priorities are equal
  */
 static gint
 cmp_promotable_instance(gconstpointer a, gconstpointer b)
 {
     const pe_resource_t *rsc1 = (const pe_resource_t *) a;
     const pe_resource_t *rsc2 = (const pe_resource_t *) b;
 
     enum rsc_role_e role1 = pcmk_role_unknown;
     enum rsc_role_e role2 = pcmk_role_unknown;
 
     CRM_ASSERT((rsc1 != NULL) && (rsc2 != NULL));
 
     // Check sort index set by pcmk__set_instance_roles()
     if (rsc1->sort_index > rsc2->sort_index) {
         pe_rsc_trace(rsc1,
                      "%s has higher promotion priority than %s "
                      "(sort index %d > %d)",
                      rsc1->id, rsc2->id, rsc1->sort_index, rsc2->sort_index);
         return -1;
     } else if (rsc1->sort_index < rsc2->sort_index) {
         pe_rsc_trace(rsc1,
                      "%s has lower promotion priority than %s "
                      "(sort index %d < %d)",
                      rsc1->id, rsc2->id, rsc1->sort_index, rsc2->sort_index);
         return 1;
     }
 
     // If those are the same, prefer instance whose current role is higher
     role1 = rsc1->fns->state(rsc1, TRUE);
     role2 = rsc2->fns->state(rsc2, TRUE);
     if (role1 > role2) {
         pe_rsc_trace(rsc1,
                      "%s has higher promotion priority than %s "
                      "(higher current role)",
                      rsc1->id, rsc2->id);
         return -1;
     } else if (role1 < role2) {
         pe_rsc_trace(rsc1,
                      "%s has lower promotion priority than %s "
                      "(lower current role)",
                      rsc1->id, rsc2->id);
         return 1;
     }
 
     // Finally, do normal clone instance sorting
     return pcmk__cmp_instance(a, b);
 }
 
 /*!
  * \internal
  * \brief Add a promotable clone instance's sort index to its node's score
  *
  * Add a promotable clone instance's sort index (which sums its promotion
  * preferences and scores of relevant location constraints for the promoted
  * role) to the node score of the instance's assigned node.
  *
  * \param[in]     data       Promotable clone instance
  * \param[in,out] user_data  Clone parent of \p data
  */
 static void
 add_sort_index_to_node_score(gpointer data, gpointer user_data)
 {
     const pe_resource_t *child = (const pe_resource_t *) data;
     pe_resource_t *clone = (pe_resource_t *) user_data;
 
     pe_node_t *node = NULL;
     const pe_node_t *chosen = NULL;
 
     if (child->sort_index < 0) {
         pe_rsc_trace(clone, "Not adding sort index of %s: negative", child->id);
         return;
     }
 
     chosen = child->fns->location(child, NULL, FALSE);
     if (chosen == NULL) {
         pe_rsc_trace(clone, "Not adding sort index of %s: inactive", child->id);
         return;
     }
 
     node = g_hash_table_lookup(clone->allowed_nodes, chosen->details->id);
     CRM_ASSERT(node != NULL);
 
     node->weight = pcmk__add_scores(child->sort_index, node->weight);
     pe_rsc_trace(clone,
                  "Added cumulative priority of %s (%s) to score on %s (now %s)",
                  child->id, pcmk_readable_score(child->sort_index),
                  pe__node_name(node), pcmk_readable_score(node->weight));
 }
 
 /*!
  * \internal
  * \brief Apply colocation to dependent's node scores if for promoted role
  *
  * \param[in,out] data       Colocation constraint to apply
  * \param[in,out] user_data  Promotable clone that is constraint's dependent
  */
 static void
 apply_coloc_to_dependent(gpointer data, gpointer user_data)
 {
     pcmk__colocation_t *colocation = data;
     pe_resource_t *clone = user_data;
     pe_resource_t *primary = colocation->primary;
     uint32_t flags = pcmk__coloc_select_default;
     float factor = colocation->score / (float) INFINITY;
 
     if (colocation->dependent_role != pcmk_role_promoted) {
         return;
     }
     if (colocation->score < INFINITY) {
         flags = pcmk__coloc_select_active;
     }
     pe_rsc_trace(clone, "Applying colocation %s (promoted %s with %s) @%s",
                  colocation->id, colocation->dependent->id,
                  colocation->primary->id,
                  pcmk_readable_score(colocation->score));
     primary->cmds->add_colocated_node_scores(primary, clone, clone->id,
                                              &clone->allowed_nodes, colocation,
                                              factor, flags);
 }
 
 /*!
  * \internal
  * \brief Apply colocation to primary's node scores if for promoted role
  *
  * \param[in,out] data       Colocation constraint to apply
  * \param[in,out] user_data  Promotable clone that is constraint's primary
  */
 static void
 apply_coloc_to_primary(gpointer data, gpointer user_data)
 {
     pcmk__colocation_t *colocation = data;
     pe_resource_t *clone = user_data;
     pe_resource_t *dependent = colocation->dependent;
     const float factor = colocation->score / (float) INFINITY;
     const uint32_t flags = pcmk__coloc_select_active
                            |pcmk__coloc_select_nonnegative;
 
     if ((colocation->primary_role != pcmk_role_promoted)
          || !pcmk__colocation_has_influence(colocation, NULL)) {
         return;
     }
 
     pe_rsc_trace(clone, "Applying colocation %s (%s with promoted %s) @%s",
                  colocation->id, colocation->dependent->id,
                  colocation->primary->id,
                  pcmk_readable_score(colocation->score));
     dependent->cmds->add_colocated_node_scores(dependent, clone, clone->id,
                                                &clone->allowed_nodes,
                                                colocation, factor, flags);
 }
 
 /*!
  * \internal
  * \brief Set clone instance's sort index to its node's score
  *
  * \param[in,out] data       Promotable clone instance
  * \param[in]     user_data  Parent clone of \p data
  */
 static void
 set_sort_index_to_node_score(gpointer data, gpointer user_data)
 {
     pe_resource_t *child = (pe_resource_t *) data;
     const pe_resource_t *clone = (const pe_resource_t *) user_data;
 
     pe_node_t *chosen = child->fns->location(child, NULL, FALSE);
 
     if (!pcmk_is_set(child->flags, pe_rsc_managed)
         && (child->next_role == pcmk_role_promoted)) {
         child->sort_index = INFINITY;
         pe_rsc_trace(clone,
                      "Final sort index for %s is INFINITY (unmanaged promoted)",
                      child->id);
 
     } else if ((chosen == NULL) || (child->sort_index < 0)) {
         pe_rsc_trace(clone,
                      "Final sort index for %s is %d (ignoring node score)",
                      child->id, child->sort_index);
 
     } else {
         const pe_node_t *node = g_hash_table_lookup(clone->allowed_nodes,
                                                     chosen->details->id);
 
         CRM_ASSERT(node != NULL);
         child->sort_index = node->weight;
         pe_rsc_trace(clone,
                      "Adding scores for %s: final sort index for %s is %d",
                      clone->id, child->id, child->sort_index);
     }
 }
 
 /*!
  * \internal
  * \brief Sort a promotable clone's instances by descending promotion priority
  *
  * \param[in,out] clone  Promotable clone to sort
  */
 static void
 sort_promotable_instances(pe_resource_t *clone)
 {
     GList *colocations = NULL;
 
     if (pe__set_clone_flag(clone, pe__clone_promotion_constrained)
             == pcmk_rc_already) {
         return;
     }
     pe__set_resource_flags(clone, pe_rsc_merging);
 
     for (GList *iter = clone->children; iter != NULL; iter = iter->next) {
         pe_resource_t *child = (pe_resource_t *) iter->data;
 
         pe_rsc_trace(clone,
                      "Adding scores for %s: initial sort index for %s is %d",
                      clone->id, child->id, child->sort_index);
     }
     pe__show_node_scores(true, clone, "Before", clone->allowed_nodes,
                          clone->cluster);
 
     g_list_foreach(clone->children, add_sort_index_to_node_score, clone);
 
     colocations = pcmk__this_with_colocations(clone);
     g_list_foreach(colocations, apply_coloc_to_dependent, clone);
     g_list_free(colocations);
 
     colocations = pcmk__with_this_colocations(clone);
     g_list_foreach(colocations, apply_coloc_to_primary, clone);
     g_list_free(colocations);
 
     // Ban resource from all nodes if it needs a ticket but doesn't have it
     pcmk__require_promotion_tickets(clone);
 
     pe__show_node_scores(true, clone, "After", clone->allowed_nodes,
                          clone->cluster);
 
     // Reset sort indexes to final node scores
     g_list_foreach(clone->children, set_sort_index_to_node_score, clone);
 
     // Finally, sort instances in descending order of promotion priority
     clone->children = g_list_sort(clone->children, cmp_promotable_instance);
     pe__clear_resource_flags(clone, pe_rsc_merging);
 }
 
 /*!
  * \internal
  * \brief Find the active instance (if any) of an anonymous clone on a node
  *
  * \param[in] clone  Anonymous clone to check
  * \param[in] id     Instance ID (without instance number) to check
  * \param[in] node   Node to check
  *
  * \return
  */
 static pe_resource_t *
 find_active_anon_instance(const pe_resource_t *clone, const char *id,
                           const pe_node_t *node)
 {
     for (GList *iter = clone->children; iter; iter = iter->next) {
         pe_resource_t *child = iter->data;
         pe_resource_t *active = NULL;
 
         // Use ->find_rsc() in case this is a cloned group
         active = clone->fns->find_rsc(child, id, node,
                                       pcmk_rsc_match_clone_only
                                       |pcmk_rsc_match_current_node);
         if (active != NULL) {
             return active;
         }
     }
     return NULL;
 }
 
 /*
  * \brief Check whether an anonymous clone instance is known on a node
  *
  * \param[in] clone  Anonymous clone to check
  * \param[in] id     Instance ID (without instance number) to check
  * \param[in] node   Node to check
  *
  * \return true if \p id instance of \p clone is known on \p node,
  *         otherwise false
  */
 static bool
 anonymous_known_on(const pe_resource_t *clone, const char *id,
                    const pe_node_t *node)
 {
     for (GList *iter = clone->children; iter; iter = iter->next) {
         pe_resource_t *child = iter->data;
 
         /* Use ->find_rsc() because this might be a cloned group, and knowing
          * that other members of the group are known here implies nothing.
          */
         child = clone->fns->find_rsc(child, id, NULL,
                                      pcmk_rsc_match_clone_only);
         CRM_LOG_ASSERT(child != NULL);
         if (child != NULL) {
             if (g_hash_table_lookup(child->known_on, node->details->id)) {
                 return true;
             }
         }
     }
     return false;
 }
 
 /*!
  * \internal
  * \brief Check whether a node is allowed to run a resource
  *
  * \param[in] rsc   Resource to check
  * \param[in] node  Node to check
  *
  * \return true if \p node is allowed to run \p rsc, otherwise false
  */
 static bool
 is_allowed(const pe_resource_t *rsc, const pe_node_t *node)
 {
     pe_node_t *allowed = g_hash_table_lookup(rsc->allowed_nodes,
                                              node->details->id);
 
     return (allowed != NULL) && (allowed->weight >= 0);
 }
 
 /*!
  * \brief Check whether a clone instance's promotion score should be considered
  *
  * \param[in] rsc   Promotable clone instance to check
  * \param[in] node  Node where score would be applied
  *
  * \return true if \p rsc's promotion score should be considered on \p node,
  *         otherwise false
  */
 static bool
 promotion_score_applies(const pe_resource_t *rsc, const pe_node_t *node)
 {
     char *id = clone_strip(rsc->id);
     const pe_resource_t *parent = pe__const_top_resource(rsc, false);
     pe_resource_t *active = NULL;
     const char *reason = "allowed";
 
     // Some checks apply only to anonymous clone instances
     if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
 
         // If instance is active on the node, its score definitely applies
         active = find_active_anon_instance(parent, id, node);
         if (active == rsc) {
             reason = "active";
             goto check_allowed;
         }
 
         /* If *no* instance is active on this node, this instance's score will
          * count if it has been probed on this node.
          */
         if ((active == NULL) && anonymous_known_on(parent, id, node)) {
             reason = "probed";
             goto check_allowed;
         }
     }
 
     /* If this clone's status is unknown on *all* nodes (e.g. cluster startup),
      * take all instances' scores into account, to make sure we use any
      * permanent promotion scores.
      */
     if ((rsc->running_on == NULL) && (g_hash_table_size(rsc->known_on) == 0)) {
         reason = "none probed";
         goto check_allowed;
     }
 
     /* Otherwise, we've probed and/or started the resource *somewhere*, so
      * consider promotion scores on nodes where we know the status.
      */
     if ((g_hash_table_lookup(rsc->known_on, node->details->id) != NULL)
         || (pe_find_node_id(rsc->running_on, node->details->id) != NULL)) {
         reason = "known";
     } else {
         pe_rsc_trace(rsc,
                      "Ignoring %s promotion score (for %s) on %s: not probed",
                      rsc->id, id, pe__node_name(node));
         free(id);
         return false;
     }
 
 check_allowed:
     if (is_allowed(rsc, node)) {
         pe_rsc_trace(rsc, "Counting %s promotion score (for %s) on %s: %s",
                      rsc->id, id, pe__node_name(node), reason);
         free(id);
         return true;
     }
 
     pe_rsc_trace(rsc, "Ignoring %s promotion score (for %s) on %s: not allowed",
                  rsc->id, id, pe__node_name(node));
     free(id);
     return false;
 }
 
 /*!
  * \internal
  * \brief Get the value of a promotion score node attribute
  *
  * \param[in] rsc   Promotable clone instance to get promotion score for
  * \param[in] node  Node to get promotion score for
  * \param[in] name  Resource name to use in promotion score attribute name
  *
  * \return Value of promotion score node attribute for \p rsc on \p node
  */
 static const char *
 promotion_attr_value(const pe_resource_t *rsc, const pe_node_t *node,
                      const char *name)
 {
     char *attr_name = NULL;
     const char *attr_value = NULL;
     enum pe__rsc_node node_type = pe__rsc_node_assigned;
 
     if (pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
         // Not assigned yet
         node_type = pe__rsc_node_current;
     }
     attr_name = pcmk_promotion_score_name(name);
     attr_value = pe__node_attribute_calculated(node, attr_name, rsc, node_type,
                                                false);
     free(attr_name);
     return attr_value;
 }
 
 /*!
  * \internal
  * \brief Get the promotion score for a clone instance on a node
  *
  * \param[in]  rsc         Promotable clone instance to get score for
  * \param[in]  node        Node to get score for
  * \param[out] is_default  If non-NULL, will be set true if no score available
  *
  * \return Promotion score for \p rsc on \p node (or 0 if none)
  */
 static int
 promotion_score(const pe_resource_t *rsc, const pe_node_t *node,
                 bool *is_default)
 {
     char *name = NULL;
     const char *attr_value = NULL;
 
     if (is_default != NULL) {
         *is_default = true;
     }
 
     CRM_CHECK((rsc != NULL) && (node != NULL), return 0);
 
     /* If this is an instance of a cloned group, the promotion score is the sum
      * of all members' promotion scores.
      */
     if (rsc->children != NULL) {
         int score = 0;
 
         for (const GList *iter = rsc->children;
              iter != NULL; iter = iter->next) {
 
             const pe_resource_t *child = (const pe_resource_t *) iter->data;
             bool child_default = false;
             int child_score = promotion_score(child, node, &child_default);
 
             if (!child_default && (is_default != NULL)) {
                 *is_default = false;
             }
             score += child_score;
         }
         return score;
     }
 
     if (!promotion_score_applies(rsc, node)) {
         return 0;
     }
 
     /* For the promotion score attribute name, use the name the resource is
      * known as in resource history, since that's what crm_attribute --promotion
      * would have used.
      */
     name = (rsc->clone_name == NULL)? rsc->id : rsc->clone_name;
 
     attr_value = promotion_attr_value(rsc, node, name);
     if (attr_value != NULL) {
         pe_rsc_trace(rsc, "Promotion score for %s on %s = %s",
                      name, pe__node_name(node), pcmk__s(attr_value, "(unset)"));
     } else if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
         /* If we don't have any resource history yet, we won't have clone_name.
          * In that case, for anonymous clones, try the resource name without
          * any instance number.
          */
         name = clone_strip(rsc->id);
         if (strcmp(rsc->id, name) != 0) {
             attr_value = promotion_attr_value(rsc, node, name);
             pe_rsc_trace(rsc, "Promotion score for %s on %s (for %s) = %s",
                          name, pe__node_name(node), rsc->id,
                          pcmk__s(attr_value, "(unset)"));
         }
         free(name);
     }
 
     if (attr_value == NULL) {
         return 0;
     }
 
     if (is_default != NULL) {
         *is_default = false;
     }
     return char2score(attr_value);
 }
 
 /*!
  * \internal
  * \brief Include promotion scores in instances' node scores and priorities
  *
  * \param[in,out] rsc  Promotable clone resource to update
  */
 void
 pcmk__add_promotion_scores(pe_resource_t *rsc)
 {
     if (pe__set_clone_flag(rsc, pe__clone_promotion_added) == pcmk_rc_already) {
         return;
     }
 
     for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) iter->data;
 
         GHashTableIter iter;
         pe_node_t *node = NULL;
         int score, new_score;
 
         g_hash_table_iter_init(&iter, child_rsc->allowed_nodes);
         while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
             if (!pcmk__node_available(node, false, false)) {
                 /* This node will never be promoted, so don't apply the
                  * promotion score, as that may lead to clone shuffling.
                  */
                 continue;
             }
 
             score = promotion_score(child_rsc, node, NULL);
             if (score > 0) {
                 new_score = pcmk__add_scores(node->weight, score);
                 if (new_score != node->weight) { // Could remain INFINITY
                     node->weight = new_score;
                     pe_rsc_trace(rsc,
                                  "Added %s promotion priority (%s) to score "
                                  "on %s (now %s)",
                                  child_rsc->id, pcmk_readable_score(score),
                                  pe__node_name(node),
                                  pcmk_readable_score(new_score));
                 }
             }
 
             if (score > child_rsc->priority) {
                 pe_rsc_trace(rsc,
                              "Updating %s priority to promotion score (%d->%d)",
                              child_rsc->id, child_rsc->priority, score);
                 child_rsc->priority = score;
             }
         }
     }
 }
 
 /*!
  * \internal
  * \brief If a resource's current role is started, change it to unpromoted
  *
  * \param[in,out] data       Resource to update
  * \param[in]     user_data  Ignored
  */
 static void
 set_current_role_unpromoted(void *data, void *user_data)
 {
     pe_resource_t *rsc = (pe_resource_t *) data;
 
     if (rsc->role == pcmk_role_started) {
         // Promotable clones should use unpromoted role instead of started
         rsc->role = pcmk_role_unpromoted;
     }
     g_list_foreach(rsc->children, set_current_role_unpromoted, NULL);
 }
 
 /*!
  * \internal
  * \brief Set a resource's next role to unpromoted (or stopped if unassigned)
  *
  * \param[in,out] data       Resource to update
  * \param[in]     user_data  Ignored
  */
 static void
 set_next_role_unpromoted(void *data, void *user_data)
 {
     pe_resource_t *rsc = (pe_resource_t *) data;
     GList *assigned = NULL;
 
     rsc->fns->location(rsc, &assigned, FALSE);
     if (assigned == NULL) {
         pe__set_next_role(rsc, pcmk_role_stopped, "stopped instance");
     } else {
         pe__set_next_role(rsc, pcmk_role_unpromoted, "unpromoted instance");
         g_list_free(assigned);
     }
     g_list_foreach(rsc->children, set_next_role_unpromoted, NULL);
 }
 
 /*!
  * \internal
  * \brief Set a resource's next role to promoted if not already set
  *
  * \param[in,out] data       Resource to update
  * \param[in]     user_data  Ignored
  */
 static void
 set_next_role_promoted(void *data, gpointer user_data)
 {
     pe_resource_t *rsc = (pe_resource_t *) data;
 
     if (rsc->next_role == pcmk_role_unknown) {
         pe__set_next_role(rsc, pcmk_role_promoted, "promoted instance");
     }
     g_list_foreach(rsc->children, set_next_role_promoted, NULL);
 }
 
 /*!
  * \internal
  * \brief Show instance's promotion score on node where it will be active
  *
  * \param[in,out] instance  Promotable clone instance to show
  */
 static void
 show_promotion_score(pe_resource_t *instance)
 {
     pe_node_t *chosen = instance->fns->location(instance, NULL, FALSE);
 
     if (pcmk_is_set(instance->cluster->flags, pe_flag_show_scores)
         && !pcmk__is_daemon && (instance->cluster->priv != NULL)) {
 
         pcmk__output_t *out = instance->cluster->priv;
 
         out->message(out, "promotion-score", instance, chosen,
                      pcmk_readable_score(instance->sort_index));
     } else {
         pe_rsc_debug(pe__const_top_resource(instance, false),
                      "%s promotion score on %s: sort=%s priority=%s",
                      instance->id,
                      ((chosen == NULL)? "none" : pe__node_name(chosen)),
                      pcmk_readable_score(instance->sort_index),
                      pcmk_readable_score(instance->priority));
     }
 }
 
 /*!
  * \internal
  * \brief Set a clone instance's promotion priority
  *
  * \param[in,out] data       Promotable clone instance to update
  * \param[in]     user_data  Instance's parent clone
  */
 static void
 set_instance_priority(gpointer data, gpointer user_data)
 {
     pe_resource_t *instance = (pe_resource_t *) data;
     const pe_resource_t *clone = (const pe_resource_t *) user_data;
     const pe_node_t *chosen = NULL;
     enum rsc_role_e next_role = pcmk_role_unknown;
     GList *list = NULL;
 
     pe_rsc_trace(clone, "Assigning priority for %s: %s", instance->id,
                  role2text(instance->next_role));
 
     if (instance->fns->state(instance, TRUE) == pcmk_role_started) {
         set_current_role_unpromoted(instance, NULL);
     }
 
     // Only an instance that will be active can be promoted
     chosen = instance->fns->location(instance, &list, FALSE);
     if (pcmk__list_of_multiple(list)) {
         pcmk__config_err("Cannot promote non-colocated child %s",
                          instance->id);
     }
     g_list_free(list);
     if (chosen == NULL) {
         return;
     }
 
     next_role = instance->fns->state(instance, FALSE);
     switch (next_role) {
         case pcmk_role_started:
         case pcmk_role_unknown:
             // Set instance priority to its promotion score (or -1 if none)
             {
                 bool is_default = false;
 
                 instance->priority = promotion_score(instance, chosen,
                                                       &is_default);
                 if (is_default) {
                     /*
                      * Default to -1 if no value is set. This allows
                      * instances eligible for promotion to be specified
                      * based solely on rsc_location constraints, but
                      * prevents any instance from being promoted if neither
                      * a constraint nor a promotion score is present
                      */
                     instance->priority = -1;
                 }
             }
             break;
 
         case pcmk_role_unpromoted:
         case pcmk_role_stopped:
             // Instance can't be promoted
             instance->priority = -INFINITY;
             break;
 
         case pcmk_role_promoted:
             // Nothing needed (re-creating actions after scheduling fencing)
             break;
 
         default:
             CRM_CHECK(FALSE, crm_err("Unknown resource role %d for %s",
                                      next_role, instance->id));
     }
 
     // Add relevant location constraint scores for promoted role
     apply_promoted_locations(instance, instance->rsc_location, chosen);
     apply_promoted_locations(instance, clone->rsc_location, chosen);
 
     // Consider instance's role-based colocations with other resources
     list = pcmk__this_with_colocations(instance);
     for (GList *iter = list; iter != NULL; iter = iter->next) {
         pcmk__colocation_t *cons = (pcmk__colocation_t *) iter->data;
 
         instance->cmds->apply_coloc_score(instance, cons->primary, cons, true);
     }
     g_list_free(list);
 
     instance->sort_index = instance->priority;
     if (next_role == pcmk_role_promoted) {
         instance->sort_index = INFINITY;
     }
     pe_rsc_trace(clone, "Assigning %s priority = %d",
                  instance->id, instance->priority);
 }
 
 /*!
  * \internal
  * \brief Set a promotable clone instance's role
  *
  * \param[in,out] data       Promotable clone instance to update
  * \param[in,out] user_data  Pointer to count of instances chosen for promotion
  */
 static void
 set_instance_role(gpointer data, gpointer user_data)
 {
     pe_resource_t *instance = (pe_resource_t *) data;
     int *count = (int *) user_data;
 
     const pe_resource_t *clone = pe__const_top_resource(instance, false);
     pe_node_t *chosen = NULL;
 
     show_promotion_score(instance);
 
     if (instance->sort_index < 0) {
         pe_rsc_trace(clone, "Not supposed to promote instance %s",
                      instance->id);
 
     } else if ((*count < pe__clone_promoted_max(instance))
                || !pcmk_is_set(clone->flags, pe_rsc_managed)) {
         chosen = node_to_be_promoted_on(instance);
     }
 
     if (chosen == NULL) {
         set_next_role_unpromoted(instance, NULL);
         return;
     }
 
     if ((instance->role < pcmk_role_promoted)
-        && !pcmk_is_set(instance->cluster->flags, pe_flag_have_quorum)
+        && !pcmk_is_set(instance->cluster->flags, pcmk_sched_quorate)
         && (instance->cluster->no_quorum_policy == pcmk_no_quorum_freeze)) {
         crm_notice("Clone instance %s cannot be promoted without quorum",
                    instance->id);
         set_next_role_unpromoted(instance, NULL);
         return;
     }
 
     chosen->count++;
     pe_rsc_info(clone, "Choosing %s (%s) on %s for promotion",
                 instance->id, role2text(instance->role),
                 pe__node_name(chosen));
     set_next_role_promoted(instance, NULL);
     (*count)++;
 }
 
 /*!
  * \internal
  * \brief Set roles for all instances of a promotable clone
  *
  * \param[in,out] rsc  Promotable clone resource to update
  */
 void
 pcmk__set_instance_roles(pe_resource_t *rsc)
 {
     int promoted = 0;
     GHashTableIter iter;
     pe_node_t *node = NULL;
 
     // Repurpose count to track the number of promoted instances assigned
     g_hash_table_iter_init(&iter, rsc->allowed_nodes);
     while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
         node->count = 0;
     }
 
     // Set instances' promotion priorities and sort by highest priority first
     g_list_foreach(rsc->children, set_instance_priority, rsc);
     sort_promotable_instances(rsc);
 
     // Choose the first N eligible instances to be promoted
     g_list_foreach(rsc->children, set_instance_role, &promoted);
     pe_rsc_info(rsc, "%s: Promoted %d instances of a possible %d",
                 rsc->id, promoted, pe__clone_promoted_max(rsc));
 }
 
 /*!
  *
  * \internal
  * \brief Create actions for promotable clone instances
  *
  * \param[in,out] clone          Promotable clone to create actions for
  * \param[out]    any_promoting  Will be set true if any instance is promoting
  * \param[out]    any_demoting   Will be set true if any instance is demoting
  */
 static void
 create_promotable_instance_actions(pe_resource_t *clone,
                                    bool *any_promoting, bool *any_demoting)
 {
     for (GList *iter = clone->children; iter != NULL; iter = iter->next) {
         pe_resource_t *instance = (pe_resource_t *) iter->data;
 
         instance->cmds->create_actions(instance);
         check_for_role_change(instance, any_demoting, any_promoting);
     }
 }
 
 /*!
  * \internal
  * \brief Reset each promotable instance's resource priority
  *
  * Reset the priority of each instance of a promotable clone to the clone's
  * priority (after promotion actions are scheduled, when instance priorities
  * were repurposed as promotion scores).
  *
  * \param[in,out] clone  Promotable clone to reset
  */
 static void
 reset_instance_priorities(pe_resource_t *clone)
 {
     for (GList *iter = clone->children; iter != NULL; iter = iter->next) {
         pe_resource_t *instance = (pe_resource_t *) iter->data;
 
         instance->priority = clone->priority;
     }
 }
 
 /*!
  * \internal
  * \brief Create actions specific to promotable clones
  *
  * \param[in,out] clone  Promotable clone to create actions for
  */
 void
 pcmk__create_promotable_actions(pe_resource_t *clone)
 {
     bool any_promoting = false;
     bool any_demoting = false;
 
     // Create actions for each clone instance individually
     create_promotable_instance_actions(clone, &any_promoting, &any_demoting);
 
     // Create pseudo-actions for clone as a whole
     pe__create_promotable_pseudo_ops(clone, any_promoting, any_demoting);
 
     // Undo our temporary repurposing of resource priority for instances
     reset_instance_priorities(clone);
 }
 
 /*!
  * \internal
  * \brief Create internal orderings for a promotable clone's instances
  *
  * \param[in,out] clone  Promotable clone instance to order
  */
 void
 pcmk__order_promotable_instances(pe_resource_t *clone)
 {
     pe_resource_t *previous = NULL; // Needed for ordered clones
 
     pcmk__promotable_restart_ordering(clone);
 
     for (GList *iter = clone->children; iter != NULL; iter = iter->next) {
         pe_resource_t *instance = (pe_resource_t *) iter->data;
 
         // Demote before promote
         pcmk__order_resource_actions(instance, PCMK_ACTION_DEMOTE,
                                      instance, PCMK_ACTION_PROMOTE,
                                      pe_order_optional);
 
         order_instance_promotion(clone, instance, previous);
         order_instance_demotion(clone, instance, previous);
         previous = instance;
     }
 }
 
 /*!
  * \internal
  * \brief Update dependent's allowed nodes for colocation with promotable
  *
  * \param[in,out] dependent     Dependent resource to update
  * \param[in]     primary       Primary resource
  * \param[in]     primary_node  Node where an instance of the primary will be
  * \param[in]     colocation    Colocation constraint to apply
  */
 static void
 update_dependent_allowed_nodes(pe_resource_t *dependent,
                                const pe_resource_t *primary,
                                const pe_node_t *primary_node,
                                const pcmk__colocation_t *colocation)
 {
     GHashTableIter iter;
     pe_node_t *node = NULL;
     const char *primary_value = NULL;
     const char *attr = colocation->node_attribute;
 
     if (colocation->score >= INFINITY) {
         return; // Colocation is mandatory, so allowed node scores don't matter
     }
 
     primary_value = pcmk__colocation_node_attr(primary_node, attr, primary);
 
     pe_rsc_trace(colocation->primary,
                  "Applying %s (%s with %s on %s by %s @%d) to %s",
                  colocation->id, colocation->dependent->id,
                  colocation->primary->id, pe__node_name(primary_node), attr,
                  colocation->score, dependent->id);
 
     g_hash_table_iter_init(&iter, dependent->allowed_nodes);
     while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
         const char *dependent_value = pcmk__colocation_node_attr(node, attr,
                                                                  dependent);
 
         if (pcmk__str_eq(primary_value, dependent_value, pcmk__str_casei)) {
             node->weight = pcmk__add_scores(node->weight, colocation->score);
             pe_rsc_trace(colocation->primary,
                          "Added %s score (%s) to %s (now %s)",
                          colocation->id, pcmk_readable_score(colocation->score),
                          pe__node_name(node),
                          pcmk_readable_score(node->weight));
         }
     }
 }
 
 /*!
  * \brief Update dependent for a colocation with a promotable clone
  *
  * \param[in]     primary     Primary resource in the colocation
  * \param[in,out] dependent   Dependent resource in the colocation
  * \param[in]     colocation  Colocation constraint to apply
  */
 void
 pcmk__update_dependent_with_promotable(const pe_resource_t *primary,
                                        pe_resource_t *dependent,
                                        const pcmk__colocation_t *colocation)
 {
     GList *affected_nodes = NULL;
 
     /* Build a list of all nodes where an instance of the primary will be, and
      * (for optional colocations) update the dependent's allowed node scores for
      * each one.
      */
     for (GList *iter = primary->children; iter != NULL; iter = iter->next) {
         pe_resource_t *instance = (pe_resource_t *) iter->data;
         pe_node_t *node = instance->fns->location(instance, NULL, FALSE);
 
         if (node == NULL) {
             continue;
         }
         if (instance->fns->state(instance, FALSE) == colocation->primary_role) {
             update_dependent_allowed_nodes(dependent, primary, node,
                                            colocation);
             affected_nodes = g_list_prepend(affected_nodes, node);
         }
     }
 
     /* For mandatory colocations, add the primary's node score to the
      * dependent's node score for each affected node, and ban the dependent
      * from all other nodes.
      *
      * However, skip this for promoted-with-promoted colocations, otherwise
      * inactive dependent instances can't start (in the unpromoted role).
      */
     if ((colocation->score >= INFINITY)
         && ((colocation->dependent_role != pcmk_role_promoted)
             || (colocation->primary_role != pcmk_role_promoted))) {
 
         pe_rsc_trace(colocation->primary,
                      "Applying %s (mandatory %s with %s) to %s",
                      colocation->id, colocation->dependent->id,
                      colocation->primary->id, dependent->id);
         pcmk__colocation_intersect_nodes(dependent, primary, colocation,
                                          affected_nodes, true);
     }
     g_list_free(affected_nodes);
 }
 
 /*!
  * \internal
  * \brief Update dependent priority for colocation with promotable
  *
  * \param[in]     primary     Primary resource in the colocation
  * \param[in,out] dependent   Dependent resource in the colocation
  * \param[in]     colocation  Colocation constraint to apply
  */
 void
 pcmk__update_promotable_dependent_priority(const pe_resource_t *primary,
                                            pe_resource_t *dependent,
                                            const pcmk__colocation_t *colocation)
 {
     pe_resource_t *primary_instance = NULL;
 
     // Look for a primary instance where dependent will be
     primary_instance = pcmk__find_compatible_instance(dependent, primary,
                                                       colocation->primary_role,
                                                       false);
 
     if (primary_instance != NULL) {
         // Add primary instance's priority to dependent's
         int new_priority = pcmk__add_scores(dependent->priority,
                                             colocation->score);
 
         pe_rsc_trace(colocation->primary,
                      "Applying %s (%s with %s) to %s priority (%s + %s = %s)",
                      colocation->id, colocation->dependent->id,
                      colocation->primary->id, dependent->id,
                      pcmk_readable_score(dependent->priority),
                      pcmk_readable_score(colocation->score),
                      pcmk_readable_score(new_priority));
         dependent->priority = new_priority;
 
     } else if (colocation->score >= INFINITY) {
         // Mandatory colocation, but primary won't be here
         pe_rsc_trace(colocation->primary,
                      "Applying %s (%s with %s) to %s: can't be promoted",
                      colocation->id, colocation->dependent->id,
                      colocation->primary->id, dependent->id);
         dependent->priority = -INFINITY;
     }
 }
diff --git a/lib/pacemaker/pcmk_scheduler.c b/lib/pacemaker/pcmk_scheduler.c
index 9bc9bd6987..c1c57195c6 100644
--- a/lib/pacemaker/pcmk_scheduler.c
+++ b/lib/pacemaker/pcmk_scheduler.c
@@ -1,816 +1,816 @@
 /*
  * Copyright 2004-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/xml_internal.h>
 #include <crm/common/scheduler_internal.h>
 
 #include <glib.h>
 
 #include <crm/pengine/status.h>
 #include <pacemaker-internal.h>
 #include "libpacemaker_private.h"
 
 CRM_TRACE_INIT_DATA(pacemaker);
 
 /*!
  * \internal
  * \brief Do deferred action checks after assignment
  *
  * When unpacking the resource history, the scheduler checks for resource
  * configurations that have changed since an action was run. However, at that
  * time, bundles using the REMOTE_CONTAINER_HACK don't have their final
  * parameter information, so instead they add a deferred check to a list. This
  * function processes one entry in that list.
  *
  * \param[in,out] rsc     Resource that action history is for
  * \param[in,out] node    Node that action history is for
  * \param[in]     rsc_op  Action history entry
  * \param[in]     check   Type of deferred check to do
  */
 static void
 check_params(pe_resource_t *rsc, pe_node_t *node, const xmlNode *rsc_op,
              enum pcmk__check_parameters check)
 {
     const char *reason = NULL;
     op_digest_cache_t *digest_data = NULL;
 
     switch (check) {
         case pcmk__check_active:
             if (pcmk__check_action_config(rsc, node, rsc_op)
                 && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL)) {
                 reason = "action definition changed";
             }
             break;
 
         case pcmk__check_last_failure:
             digest_data = rsc_action_digest_cmp(rsc, rsc_op, node,
                                                 rsc->cluster);
             switch (digest_data->rc) {
                 case RSC_DIGEST_UNKNOWN:
                     crm_trace("Resource %s history entry %s on %s has "
                               "no digest to compare",
                               rsc->id, ID(rsc_op), node->details->id);
                     break;
                 case RSC_DIGEST_MATCH:
                     break;
                 default:
                     reason = "resource parameters have changed";
                     break;
             }
             break;
     }
     if (reason != NULL) {
         pe__clear_failcount(rsc, node, reason, rsc->cluster);
     }
 }
 
 /*!
  * \internal
  * \brief Check whether a resource has failcount clearing scheduled on a node
  *
  * \param[in] node  Node to check
  * \param[in] rsc   Resource to check
  *
  * \return true if \p rsc has failcount clearing scheduled on \p node,
  *         otherwise false
  */
 static bool
 failcount_clear_action_exists(const pe_node_t *node, const pe_resource_t *rsc)
 {
     GList *list = pe__resource_actions(rsc, node, PCMK_ACTION_CLEAR_FAILCOUNT,
                                        TRUE);
 
     if (list != NULL) {
         g_list_free(list);
         return true;
     }
     return false;
 }
 
 /*!
  * \internal
  * \brief Ban a resource from a node if it reached its failure threshold there
  *
  * \param[in,out] data       Resource to check failure threshold for
  * \param[in]     user_data  Node to check resource on
  */
 static void
 check_failure_threshold(gpointer data, gpointer user_data)
 {
     pe_resource_t *rsc = data;
     const pe_node_t *node = user_data;
 
     // If this is a collective resource, apply recursively to children instead
     if (rsc->children != NULL) {
         g_list_foreach(rsc->children, check_failure_threshold, user_data);
         return;
     }
 
     if (!failcount_clear_action_exists(node, rsc)) {
         /* Don't force the resource away from this node due to a failcount
          * that's going to be cleared.
          *
          * @TODO Failcount clearing can be scheduled in
          * pcmk__handle_rsc_config_changes() via process_rsc_history(), or in
          * schedule_resource_actions() via check_params(). This runs well before
          * then, so it cannot detect those, meaning we might check the migration
          * threshold when we shouldn't. Worst case, we stop or move the
          * resource, then move it back in the next transition.
          */
         pe_resource_t *failed = NULL;
 
         if (pcmk__threshold_reached(rsc, node, &failed)) {
             resource_location(failed, node, -INFINITY, "__fail_limit__",
                               rsc->cluster);
         }
     }
 }
 
 /*!
  * \internal
  * \brief If resource has exclusive discovery, ban node if not allowed
  *
  * Location constraints have a resource-discovery option that allows users to
  * specify where probes are done for the affected resource. If this is set to
  * exclusive, probes will only be done on nodes listed in exclusive constraints.
  * This function bans the resource from the node if the node is not listed.
  *
  * \param[in,out] data       Resource to check
  * \param[in]     user_data  Node to check resource on
  */
 static void
 apply_exclusive_discovery(gpointer data, gpointer user_data)
 {
     pe_resource_t *rsc = data;
     const pe_node_t *node = user_data;
 
     if (rsc->exclusive_discover
         || pe__const_top_resource(rsc, false)->exclusive_discover) {
         pe_node_t *match = NULL;
 
         // If this is a collective resource, apply recursively to children
         g_list_foreach(rsc->children, apply_exclusive_discovery, user_data);
 
         match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
         if ((match != NULL)
             && (match->rsc_discover_mode != pe_discover_exclusive)) {
             match->weight = -INFINITY;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Apply stickiness to a resource if appropriate
  *
  * \param[in,out] data       Resource to check for stickiness
  * \param[in]     user_data  Ignored
  */
 static void
 apply_stickiness(gpointer data, gpointer user_data)
 {
     pe_resource_t *rsc = data;
     pe_node_t *node = NULL;
 
     // If this is a collective resource, apply recursively to children instead
     if (rsc->children != NULL) {
         g_list_foreach(rsc->children, apply_stickiness, NULL);
         return;
     }
 
     /* A resource is sticky if it is managed, has stickiness configured, and is
      * active on a single node.
      */
     if (!pcmk_is_set(rsc->flags, pe_rsc_managed)
         || (rsc->stickiness < 1) || !pcmk__list_of_1(rsc->running_on)) {
         return;
     }
 
     node = rsc->running_on->data;
 
     /* In a symmetric cluster, stickiness can always be used. In an
      * asymmetric cluster, we have to check whether the resource is still
      * allowed on the node, so we don't keep the resource somewhere it is no
      * longer explicitly enabled.
      */
     if (!pcmk_is_set(rsc->cluster->flags, pe_flag_symmetric_cluster)
         && (g_hash_table_lookup(rsc->allowed_nodes,
                                 node->details->id) == NULL)) {
         pe_rsc_debug(rsc,
                      "Ignoring %s stickiness because the cluster is "
                      "asymmetric and %s is not explicitly allowed",
                      rsc->id, pe__node_name(node));
         return;
     }
 
     pe_rsc_debug(rsc, "Resource %s has %d stickiness on %s",
                  rsc->id, rsc->stickiness, pe__node_name(node));
     resource_location(rsc, node, rsc->stickiness, "stickiness", rsc->cluster);
 }
 
 /*!
  * \internal
  * \brief Apply shutdown locks for all resources as appropriate
  *
  * \param[in,out] data_set  Cluster working set
  */
 static void
 apply_shutdown_locks(pe_working_set_t *data_set)
 {
     if (!pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)) {
         return;
     }
     for (GList *iter = data_set->resources; iter != NULL; iter = iter->next) {
         pe_resource_t *rsc = (pe_resource_t *) iter->data;
 
         rsc->cmds->shutdown_lock(rsc);
     }
 }
 
 /*!
  * \internal
  * \brief Calculate the number of available nodes in the cluster
  *
  * \param[in,out] data_set  Cluster working set
  */
 static void
 count_available_nodes(pe_working_set_t *data_set)
 {
     if (pcmk_is_set(data_set->flags, pe_flag_no_compat)) {
         return;
     }
 
     // @COMPAT for API backward compatibility only (cluster does not use value)
     for (GList *iter = data_set->nodes; iter != NULL; iter = iter->next) {
         pe_node_t *node = (pe_node_t *) iter->data;
 
         if ((node != NULL) && (node->weight >= 0) && node->details->online
             && (node->details->type != node_ping)) {
             data_set->max_valid_nodes++;
         }
     }
     crm_trace("Online node count: %d", data_set->max_valid_nodes);
 }
 
 /*
  * \internal
  * \brief Apply node-specific scheduling criteria
  *
  * After the CIB has been unpacked, process node-specific scheduling criteria
  * including shutdown locks, location constraints, resource stickiness,
  * migration thresholds, and exclusive resource discovery.
  */
 static void
 apply_node_criteria(pe_working_set_t *data_set)
 {
     crm_trace("Applying node-specific scheduling criteria");
     apply_shutdown_locks(data_set);
     count_available_nodes(data_set);
     pcmk__apply_locations(data_set);
     g_list_foreach(data_set->resources, apply_stickiness, NULL);
 
     for (GList *node_iter = data_set->nodes; node_iter != NULL;
          node_iter = node_iter->next) {
         for (GList *rsc_iter = data_set->resources; rsc_iter != NULL;
              rsc_iter = rsc_iter->next) {
             check_failure_threshold(rsc_iter->data, node_iter->data);
             apply_exclusive_discovery(rsc_iter->data, node_iter->data);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Assign resources to nodes
  *
  * \param[in,out] data_set  Cluster working set
  */
 static void
 assign_resources(pe_working_set_t *data_set)
 {
     GList *iter = NULL;
 
     crm_trace("Assigning resources to nodes");
 
     if (!pcmk__str_eq(data_set->placement_strategy, "default",
                       pcmk__str_casei)) {
         pcmk__sort_resources(data_set);
     }
     pcmk__show_node_capacities("Original", data_set);
 
     if (pcmk_is_set(data_set->flags, pe_flag_have_remote_nodes)) {
         /* Assign remote connection resources first (which will also assign any
          * colocation dependencies). If the connection is migrating, always
          * prefer the partial migration target.
          */
         for (iter = data_set->resources; iter != NULL; iter = iter->next) {
             pe_resource_t *rsc = (pe_resource_t *) iter->data;
 
             if (rsc->is_remote_node) {
                 pe_rsc_trace(rsc, "Assigning remote connection resource '%s'",
                              rsc->id);
                 rsc->cmds->assign(rsc, rsc->partial_migration_target, true);
             }
         }
     }
 
     /* now do the rest of the resources */
     for (iter = data_set->resources; iter != NULL; iter = iter->next) {
         pe_resource_t *rsc = (pe_resource_t *) iter->data;
 
         if (!rsc->is_remote_node) {
             pe_rsc_trace(rsc, "Assigning %s resource '%s'",
                          rsc->xml->name, rsc->id);
             rsc->cmds->assign(rsc, NULL, true);
         }
     }
 
     pcmk__show_node_capacities("Remaining", data_set);
 }
 
 /*!
  * \internal
  * \brief Schedule fail count clearing on online nodes if resource is orphaned
  *
  * \param[in,out] data       Resource to check
  * \param[in]     user_data  Ignored
  */
 static void
 clear_failcounts_if_orphaned(gpointer data, gpointer user_data)
 {
     pe_resource_t *rsc = data;
 
     if (!pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
         return;
     }
     crm_trace("Clear fail counts for orphaned resource %s", rsc->id);
 
     /* There's no need to recurse into rsc->children because those
      * should just be unassigned clone instances.
      */
 
     for (GList *iter = rsc->cluster->nodes; iter != NULL; iter = iter->next) {
         pe_node_t *node = (pe_node_t *) iter->data;
         pe_action_t *clear_op = NULL;
 
         if (!node->details->online) {
             continue;
         }
         if (pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL) == 0) {
             continue;
         }
 
         clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
                                        rsc->cluster);
 
         /* We can't use order_action_then_stop() here because its
          * pe_order_preserve breaks things
          */
         pcmk__new_ordering(clear_op->rsc, NULL, clear_op, rsc, stop_key(rsc),
                            NULL, pe_order_optional, rsc->cluster);
     }
 }
 
 /*!
  * \internal
  * \brief Schedule any resource actions needed
  *
  * \param[in,out] data_set  Cluster working set
  */
 static void
 schedule_resource_actions(pe_working_set_t *data_set)
 {
     // Process deferred action checks
     pe__foreach_param_check(data_set, check_params);
     pe__free_param_checks(data_set);
 
     if (pcmk_is_set(data_set->flags, pe_flag_startup_probes)) {
         crm_trace("Scheduling probes");
         pcmk__schedule_probes(data_set);
     }
 
     if (pcmk_is_set(data_set->flags, pe_flag_stop_rsc_orphans)) {
         g_list_foreach(data_set->resources, clear_failcounts_if_orphaned, NULL);
     }
 
     crm_trace("Scheduling resource actions");
     for (GList *iter = data_set->resources; iter != NULL; iter = iter->next) {
         pe_resource_t *rsc = (pe_resource_t *) iter->data;
 
         rsc->cmds->create_actions(rsc);
     }
 }
 
 /*!
  * \internal
  * \brief Check whether a resource or any of its descendants are managed
  *
  * \param[in] rsc  Resource to check
  *
  * \return true if resource or any descendant is managed, otherwise false
  */
 static bool
 is_managed(const pe_resource_t *rsc)
 {
     if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
         return true;
     }
     for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
         if (is_managed((pe_resource_t *) iter->data)) {
             return true;
         }
     }
     return false;
 }
 
 /*!
  * \internal
  * \brief Check whether any resources in the cluster are managed
  *
  * \param[in] data_set  Cluster working set
  *
  * \return true if any resource is managed, otherwise false
  */
 static bool
 any_managed_resources(const pe_working_set_t *data_set)
 {
     for (const GList *iter = data_set->resources;
          iter != NULL; iter = iter->next) {
         if (is_managed((const pe_resource_t *) iter->data)) {
             return true;
         }
     }
     return false;
 }
 
 /*!
  * \internal
  * \brief Check whether a node requires fencing
  *
  * \param[in] node          Node to check
  * \param[in] have_managed  Whether any resource in cluster is managed
  *
  * \return true if \p node should be fenced, otherwise false
  */
 static bool
 needs_fencing(const pe_node_t *node, bool have_managed)
 {
     return have_managed && node->details->unclean
            && pe_can_fence(node->details->data_set, node);
 }
 
 /*!
  * \internal
  * \brief Check whether a node requires shutdown
  *
  * \param[in] node          Node to check
  *
  * \return true if \p node should be shut down, otherwise false
  */
 static bool
 needs_shutdown(const pe_node_t *node)
 {
     if (pe__is_guest_or_remote_node(node)) {
        /* Do not send shutdown actions for Pacemaker Remote nodes.
         * @TODO We might come up with a good use for this in the future.
         */
         return false;
     }
     return node->details->online && node->details->shutdown;
 }
 
 /*!
  * \internal
  * \brief Track and order non-DC fencing
  *
  * \param[in,out] list      List of existing non-DC fencing actions
  * \param[in,out] action    Fencing action to prepend to \p list
  * \param[in]     data_set  Cluster working set
  *
  * \return (Possibly new) head of \p list
  */
 static GList *
 add_nondc_fencing(GList *list, pe_action_t *action,
                   const pe_working_set_t *data_set)
 {
     if (!pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)
         && (list != NULL)) {
         /* Concurrent fencing is disabled, so order each non-DC
          * fencing in a chain. If there is any DC fencing or
          * shutdown, it will be ordered after the last action in the
          * chain later.
          */
         order_actions((pe_action_t *) list->data, action, pe_order_optional);
     }
     return g_list_prepend(list, action);
 }
 
 /*!
  * \internal
  * \brief Schedule a node for fencing
  *
  * \param[in,out] node      Node that requires fencing
  */
 static pe_action_t *
 schedule_fencing(pe_node_t *node)
 {
     pe_action_t *fencing = pe_fence_op(node, NULL, FALSE, "node is unclean",
                                        FALSE, node->details->data_set);
 
     pe_warn("Scheduling node %s for fencing", pe__node_name(node));
     pcmk__order_vs_fence(fencing, node->details->data_set);
     return fencing;
 }
 
 /*!
  * \internal
  * \brief Create and order node fencing and shutdown actions
  *
  * \param[in,out] data_set  Cluster working set
  */
 static void
 schedule_fencing_and_shutdowns(pe_working_set_t *data_set)
 {
     pe_action_t *dc_down = NULL;
     bool integrity_lost = false;
     bool have_managed = any_managed_resources(data_set);
     GList *fencing_ops = NULL;
     GList *shutdown_ops = NULL;
 
     crm_trace("Scheduling fencing and shutdowns as needed");
     if (!have_managed) {
         crm_notice("No fencing will be done until there are resources "
                    "to manage");
     }
 
     // Check each node for whether it needs fencing or shutdown
     for (GList *iter = data_set->nodes; iter != NULL; iter = iter->next) {
         pe_node_t *node = (pe_node_t *) iter->data;
         pe_action_t *fencing = NULL;
 
         /* Guest nodes are "fenced" by recovering their container resource,
          * so handle them separately.
          */
         if (pe__is_guest_node(node)) {
             if (node->details->remote_requires_reset && have_managed
                 && pe_can_fence(data_set, node)) {
                 pcmk__fence_guest(node);
             }
             continue;
         }
 
         if (needs_fencing(node, have_managed)) {
             fencing = schedule_fencing(node);
 
             // Track DC and non-DC fence actions separately
             if (node->details->is_dc) {
                 dc_down = fencing;
             } else {
                 fencing_ops = add_nondc_fencing(fencing_ops, fencing, data_set);
             }
 
         } else if (needs_shutdown(node)) {
             pe_action_t *down_op = pcmk__new_shutdown_action(node);
 
             // Track DC and non-DC shutdown actions separately
             if (node->details->is_dc) {
                 dc_down = down_op;
             } else {
                 shutdown_ops = g_list_prepend(shutdown_ops, down_op);
             }
         }
 
         if ((fencing == NULL) && node->details->unclean) {
             integrity_lost = true;
             pe_warn("Node %s is unclean but cannot be fenced",
                     pe__node_name(node));
         }
     }
 
     if (integrity_lost) {
         if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
             pe_warn("Resource functionality and data integrity cannot be "
                     "guaranteed (configure, enable, and test fencing to "
                     "correct this)");
 
-        } else if (!pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
+        } else if (!pcmk_is_set(data_set->flags, pcmk_sched_quorate)) {
             crm_notice("Unclean nodes will not be fenced until quorum is "
                        "attained or no-quorum-policy is set to ignore");
         }
     }
 
     if (dc_down != NULL) {
         /* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
          * DC elections. However, we don't want to order non-DC shutdowns before
          * a DC *fencing*, because even though we don't want a node that's
          * shutting down to become DC, the DC fencing could be ordered before a
          * clone stop that's also ordered before the shutdowns, thus leading to
          * a graph loop.
          */
         if (pcmk__str_eq(dc_down->task, PCMK_ACTION_DO_SHUTDOWN,
                          pcmk__str_none)) {
             pcmk__order_after_each(dc_down, shutdown_ops);
         }
 
         // Order any non-DC fencing before any DC fencing or shutdown
 
         if (pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)) {
             /* With concurrent fencing, order each non-DC fencing action
              * separately before any DC fencing or shutdown.
              */
             pcmk__order_after_each(dc_down, fencing_ops);
         } else if (fencing_ops != NULL) {
             /* Without concurrent fencing, the non-DC fencing actions are
              * already ordered relative to each other, so we just need to order
              * the DC fencing after the last action in the chain (which is the
              * first item in the list).
              */
             order_actions((pe_action_t *) fencing_ops->data, dc_down,
                           pe_order_optional);
         }
     }
     g_list_free(fencing_ops);
     g_list_free(shutdown_ops);
 }
 
 static void
 log_resource_details(pe_working_set_t *data_set)
 {
     pcmk__output_t *out = data_set->priv;
     GList *all = NULL;
 
     /* Due to the `crm_mon --node=` feature, out->message() for all the
      * resource-related messages expects a list of nodes that we are allowed to
      * output information for. Here, we create a wildcard to match all nodes.
      */
     all = g_list_prepend(all, (gpointer) "*");
 
     for (GList *item = data_set->resources; item != NULL; item = item->next) {
         pe_resource_t *rsc = (pe_resource_t *) item->data;
 
         // Log all resources except inactive orphans
         if (!pcmk_is_set(rsc->flags, pe_rsc_orphan)
             || (rsc->role != pcmk_role_stopped)) {
             out->message(out, crm_map_element_name(rsc->xml), 0, rsc, all, all);
         }
     }
 
     g_list_free(all);
 }
 
 static void
 log_all_actions(pe_working_set_t *data_set)
 {
     /* This only ever outputs to the log, so ignore whatever output object was
      * previously set and just log instead.
      */
     pcmk__output_t *prev_out = data_set->priv;
     pcmk__output_t *out = NULL;
 
     if (pcmk__log_output_new(&out) != pcmk_rc_ok) {
         return;
     }
 
     pe__register_messages(out);
     pcmk__register_lib_messages(out);
     pcmk__output_set_log_level(out, LOG_NOTICE);
     data_set->priv = out;
 
     out->begin_list(out, NULL, NULL, "Actions");
     pcmk__output_actions(data_set);
     out->end_list(out);
     out->finish(out, CRM_EX_OK, true, NULL);
     pcmk__output_free(out);
 
     data_set->priv = prev_out;
 }
 
 /*!
  * \internal
  * \brief Log all required but unrunnable actions at trace level
  *
  * \param[in] data_set  Cluster working set
  */
 static void
 log_unrunnable_actions(const pe_working_set_t *data_set)
 {
     const uint64_t flags = pe_action_optional
                            |pe_action_runnable
                            |pe_action_pseudo;
 
     crm_trace("Required but unrunnable actions:");
     for (const GList *iter = data_set->actions;
          iter != NULL; iter = iter->next) {
 
         const pe_action_t *action = (const pe_action_t *) iter->data;
 
         if (!pcmk_any_flags_set(action->flags, flags)) {
             pcmk__log_action("\t", action, true);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Unpack the CIB for scheduling
  *
  * \param[in,out] cib       CIB XML to unpack (may be NULL if already unpacked)
  * \param[in]     flags     Working set flags to set in addition to defaults
  * \param[in,out] data_set  Cluster working set
  */
 static void
 unpack_cib(xmlNode *cib, unsigned long long flags, pe_working_set_t *data_set)
 {
     const char* localhost_save = NULL;
 
     if (pcmk_is_set(data_set->flags, pe_flag_have_status)) {
         crm_trace("Reusing previously calculated cluster status");
         pe__set_working_set_flags(data_set, flags);
         return;
     }
 
     if (data_set->localhost) {
         localhost_save = data_set->localhost;
     }
 
     CRM_ASSERT(cib != NULL);
     crm_trace("Calculating cluster status");
 
     /* This will zero the entire struct without freeing anything first, so
      * callers should never call pcmk__schedule_actions() with a populated data
      * set unless pe_flag_have_status is set (i.e. cluster_status() was
      * previously called, whether directly or via pcmk__schedule_actions()).
      */
     set_working_set_defaults(data_set);
 
     if (localhost_save) {
         data_set->localhost = localhost_save;
     }
 
     pe__set_working_set_flags(data_set, flags);
     data_set->input = cib;
     cluster_status(data_set); // Sets pe_flag_have_status
 }
 
 /*!
  * \internal
  * \brief Run the scheduler for a given CIB
  *
  * \param[in,out] cib       CIB XML to use as scheduler input
  * \param[in]     flags     Working set flags to set in addition to defaults
  * \param[in,out] data_set  Cluster working set
  */
 void
 pcmk__schedule_actions(xmlNode *cib, unsigned long long flags,
                        pe_working_set_t *data_set)
 {
     unpack_cib(cib, flags, data_set);
     pcmk__set_assignment_methods(data_set);
     pcmk__apply_node_health(data_set);
     pcmk__unpack_constraints(data_set);
     if (pcmk_is_set(data_set->flags, pe_flag_check_config)) {
         return;
     }
 
     if (!pcmk_is_set(data_set->flags, pe_flag_quick_location) &&
          pcmk__is_daemon) {
         log_resource_details(data_set);
     }
 
     apply_node_criteria(data_set);
 
     if (pcmk_is_set(data_set->flags, pe_flag_quick_location)) {
         return;
     }
 
     pcmk__create_internal_constraints(data_set);
     pcmk__handle_rsc_config_changes(data_set);
     assign_resources(data_set);
     schedule_resource_actions(data_set);
 
     /* Remote ordering constraints need to happen prior to calculating fencing
      * because it is one more place we can mark nodes as needing fencing.
      */
     pcmk__order_remote_connection_actions(data_set);
 
     schedule_fencing_and_shutdowns(data_set);
     pcmk__apply_orderings(data_set);
     log_all_actions(data_set);
     pcmk__create_graph(data_set);
 
     if (get_crm_log_level() == LOG_TRACE) {
         log_unrunnable_actions(data_set);
     }
 }
diff --git a/lib/pengine/pe_actions.c b/lib/pengine/pe_actions.c
index 46605e1989..983c322813 100644
--- a/lib/pengine/pe_actions.c
+++ b/lib/pengine/pe_actions.c
@@ -1,1743 +1,1743 @@
 /*
  * Copyright 2004-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <glib.h>
 #include <stdbool.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/scheduler_internal.h>
 #include <crm/pengine/internal.h>
 #include <crm/common/xml_internal.h>
 #include "pe_status_private.h"
 
 static void unpack_operation(pe_action_t *action, const xmlNode *xml_obj,
                              const pe_resource_t *container, guint interval_ms);
 
 static void
 add_singleton(pe_working_set_t *data_set, pe_action_t *action)
 {
     if (data_set->singletons == NULL) {
         data_set->singletons = pcmk__strkey_table(NULL, NULL);
     }
     g_hash_table_insert(data_set->singletons, action->uuid, action);
 }
 
 static pe_action_t *
 lookup_singleton(pe_working_set_t *data_set, const char *action_uuid)
 {
     if (data_set->singletons == NULL) {
         return NULL;
     }
     return g_hash_table_lookup(data_set->singletons, action_uuid);
 }
 
 /*!
  * \internal
  * \brief Find an existing action that matches arguments
  *
  * \param[in] key        Action key to match
  * \param[in] rsc        Resource to match (if any)
  * \param[in] node       Node to match (if any)
  * \param[in] data_set   Cluster working set
  *
  * \return Existing action that matches arguments (or NULL if none)
  */
 static pe_action_t *
 find_existing_action(const char *key, const pe_resource_t *rsc,
                      const pe_node_t *node, const pe_working_set_t *data_set)
 {
     GList *matches = NULL;
     pe_action_t *action = NULL;
 
     /* When rsc is NULL, it would be quicker to check data_set->singletons,
      * but checking all data_set->actions takes the node into account.
      */
     matches = find_actions(((rsc == NULL)? data_set->actions : rsc->actions),
                            key, node);
     if (matches == NULL) {
         return NULL;
     }
     CRM_LOG_ASSERT(!pcmk__list_of_multiple(matches));
 
     action = matches->data;
     g_list_free(matches);
     return action;
 }
 
 static xmlNode *
 find_rsc_op_entry_helper(const pe_resource_t *rsc, const char *key,
                          gboolean include_disabled)
 {
     guint interval_ms = 0;
     gboolean do_retry = TRUE;
     char *local_key = NULL;
     const char *name = NULL;
     const char *interval_spec = NULL;
     char *match_key = NULL;
     xmlNode *op = NULL;
     xmlNode *operation = NULL;
 
   retry:
     for (operation = pcmk__xe_first_child(rsc->ops_xml); operation != NULL;
          operation = pcmk__xe_next(operation)) {
 
         if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
             bool enabled = false;
 
             name = crm_element_value(operation, "name");
             interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
             if (!include_disabled && pcmk__xe_get_bool_attr(operation, "enabled", &enabled) == pcmk_rc_ok &&
                 !enabled) {
                 continue;
             }
 
             interval_ms = crm_parse_interval_spec(interval_spec);
             match_key = pcmk__op_key(rsc->id, name, interval_ms);
             if (pcmk__str_eq(key, match_key, pcmk__str_casei)) {
                 op = operation;
             }
             free(match_key);
 
             if (rsc->clone_name) {
                 match_key = pcmk__op_key(rsc->clone_name, name, interval_ms);
                 if (pcmk__str_eq(key, match_key, pcmk__str_casei)) {
                     op = operation;
                 }
                 free(match_key);
             }
 
             if (op != NULL) {
                 free(local_key);
                 return op;
             }
         }
     }
 
     free(local_key);
     if (do_retry == FALSE) {
         return NULL;
     }
 
     do_retry = FALSE;
     if ((strstr(key, PCMK_ACTION_MIGRATE_TO) != NULL)
         || (strstr(key, PCMK_ACTION_MIGRATE_FROM) != NULL)) {
         local_key = pcmk__op_key(rsc->id, "migrate", 0);
         key = local_key;
         goto retry;
 
     } else if (strstr(key, "_notify_")) {
         local_key = pcmk__op_key(rsc->id, PCMK_ACTION_NOTIFY, 0);
         key = local_key;
         goto retry;
     }
 
     return NULL;
 }
 
 xmlNode *
 find_rsc_op_entry(const pe_resource_t *rsc, const char *key)
 {
     return find_rsc_op_entry_helper(rsc, key, FALSE);
 }
 
 /*!
  * \internal
  * \brief Create a new action object
  *
  * \param[in]     key        Action key
  * \param[in]     task       Action name
  * \param[in,out] rsc        Resource that action is for (if any)
  * \param[in]     node       Node that action is on (if any)
  * \param[in]     optional   Whether action should be considered optional
  * \param[in]     for_graph  Whether action should be recorded in transition graph
  * \param[in,out] data_set   Cluster working set
  *
  * \return Newly allocated action
  * \note This function takes ownership of \p key. It is the caller's
  *       responsibility to free the return value with pe_free_action().
  */
 static pe_action_t *
 new_action(char *key, const char *task, pe_resource_t *rsc,
            const pe_node_t *node, bool optional, bool for_graph,
            pe_working_set_t *data_set)
 {
     pe_action_t *action = calloc(1, sizeof(pe_action_t));
 
     CRM_ASSERT(action != NULL);
 
     action->rsc = rsc;
     action->task = strdup(task); CRM_ASSERT(action->task != NULL);
     action->uuid = key;
     action->extra = pcmk__strkey_table(free, free);
     action->meta = pcmk__strkey_table(free, free);
 
     if (node) {
         action->node = pe__copy_node(node);
     }
 
     if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_casei)) {
         // Resource history deletion for a node can be done on the DC
         pe__set_action_flags(action, pe_action_dc);
     }
 
     pe__set_action_flags(action, pe_action_runnable);
     if (optional) {
         pe__set_action_flags(action, pe_action_optional);
     } else {
         pe__clear_action_flags(action, pe_action_optional);
     }
 
     if (rsc != NULL) {
         guint interval_ms = 0;
 
         action->op_entry = find_rsc_op_entry_helper(rsc, key, TRUE);
         parse_op_key(key, NULL, NULL, &interval_ms);
         unpack_operation(action, action->op_entry, rsc->container, interval_ms);
     }
 
     if (for_graph) {
         pe_rsc_trace(rsc, "Created %s action %d (%s): %s for %s on %s",
                      (optional? "optional" : "required"),
                      data_set->action_id, key, task,
                      ((rsc == NULL)? "no resource" : rsc->id),
                      pe__node_name(node));
         action->id = data_set->action_id++;
 
         data_set->actions = g_list_prepend(data_set->actions, action);
         if (rsc == NULL) {
             add_singleton(data_set, action);
         } else {
             rsc->actions = g_list_prepend(rsc->actions, action);
         }
     }
     return action;
 }
 
 /*!
  * \internal
  * \brief Evaluate node attribute values for an action
  *
  * \param[in,out] action    Action to unpack attributes for
  * \param[in,out] data_set  Cluster working set
  */
 static void
 unpack_action_node_attributes(pe_action_t *action, pe_working_set_t *data_set)
 {
     if (!pcmk_is_set(action->flags, pe_action_have_node_attrs)
         && (action->op_entry != NULL)) {
 
         pe_rule_eval_data_t rule_data = {
             .node_hash = action->node->details->attrs,
             .role = pcmk_role_unknown,
             .now = data_set->now,
             .match_data = NULL,
             .rsc_data = NULL,
             .op_data = NULL
         };
 
         pe__set_action_flags(action, pe_action_have_node_attrs);
         pe__unpack_dataset_nvpairs(action->op_entry, XML_TAG_ATTR_SETS,
                                    &rule_data, action->extra, NULL,
                                    FALSE, data_set);
     }
 }
 
 /*!
  * \internal
  * \brief Update an action's optional flag
  *
  * \param[in,out] action    Action to update
  * \param[in]     optional  Requested optional status
  */
 static void
 update_action_optional(pe_action_t *action, gboolean optional)
 {
     // Force a non-recurring action to be optional if its resource is unmanaged
     if ((action->rsc != NULL) && (action->node != NULL)
         && !pcmk_is_set(action->flags, pe_action_pseudo)
         && !pcmk_is_set(action->rsc->flags, pe_rsc_managed)
         && (g_hash_table_lookup(action->meta,
                                 XML_LRM_ATTR_INTERVAL_MS) == NULL)) {
             pe_rsc_debug(action->rsc, "%s on %s is optional (%s is unmanaged)",
                          action->uuid, pe__node_name(action->node),
                          action->rsc->id);
             pe__set_action_flags(action, pe_action_optional);
             // We shouldn't clear runnable here because ... something
 
     // Otherwise require the action if requested
     } else if (!optional) {
         pe__clear_action_flags(action, pe_action_optional);
     }
 }
 
 static enum pe_quorum_policy
 effective_quorum_policy(pe_resource_t *rsc, pe_working_set_t *data_set)
 {
     enum pe_quorum_policy policy = data_set->no_quorum_policy;
 
-    if (pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
+    if (pcmk_is_set(data_set->flags, pcmk_sched_quorate)) {
         policy = pcmk_no_quorum_ignore;
 
     } else if (data_set->no_quorum_policy == pcmk_no_quorum_demote) {
         switch (rsc->role) {
             case pcmk_role_promoted:
             case pcmk_role_unpromoted:
                 if (rsc->next_role > pcmk_role_unpromoted) {
                     pe__set_next_role(rsc, pcmk_role_unpromoted,
                                       "no-quorum-policy=demote");
                 }
                 policy = pcmk_no_quorum_ignore;
                 break;
             default:
                 policy = pcmk_no_quorum_stop;
                 break;
         }
     }
     return policy;
 }
 
 /*!
  * \internal
  * \brief Update a resource action's runnable flag
  *
  * \param[in,out] action     Action to update
  * \param[in]     for_graph  Whether action should be recorded in transition graph
  * \param[in,out] data_set   Cluster working set
  *
  * \note This may also schedule fencing if a stop is unrunnable.
  */
 static void
 update_resource_action_runnable(pe_action_t *action, bool for_graph,
                                 pe_working_set_t *data_set)
 {
     if (pcmk_is_set(action->flags, pe_action_pseudo)) {
         return;
     }
 
     if (action->node == NULL) {
         pe_rsc_trace(action->rsc, "%s is unrunnable (unallocated)",
                      action->uuid);
         pe__clear_action_flags(action, pe_action_runnable);
 
     } else if (!pcmk_is_set(action->flags, pe_action_dc)
                && !(action->node->details->online)
                && (!pe__is_guest_node(action->node)
                    || action->node->details->remote_requires_reset)) {
         pe__clear_action_flags(action, pe_action_runnable);
         do_crm_log((for_graph? LOG_WARNING: LOG_TRACE),
                    "%s on %s is unrunnable (node is offline)",
                    action->uuid, pe__node_name(action->node));
         if (pcmk_is_set(action->rsc->flags, pe_rsc_managed)
             && for_graph
             && pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei)
             && !(action->node->details->unclean)) {
             pe_fence_node(data_set, action->node, "stop is unrunnable", false);
         }
 
     } else if (!pcmk_is_set(action->flags, pe_action_dc)
                && action->node->details->pending) {
         pe__clear_action_flags(action, pe_action_runnable);
         do_crm_log((for_graph? LOG_WARNING: LOG_TRACE),
                    "Action %s on %s is unrunnable (node is pending)",
                    action->uuid, pe__node_name(action->node));
 
     } else if (action->needs == pcmk_requires_nothing) {
         pe_action_set_reason(action, NULL, TRUE);
         if (pe__is_guest_node(action->node)
             && !pe_can_fence(data_set, action->node)) {
             /* An action that requires nothing usually does not require any
              * fencing in order to be runnable. However, there is an exception:
              * such an action cannot be completed if it is on a guest node whose
              * host is unclean and cannot be fenced.
              */
             pe_rsc_debug(action->rsc, "%s on %s is unrunnable "
                          "(node's host cannot be fenced)",
                          action->uuid, pe__node_name(action->node));
             pe__clear_action_flags(action, pe_action_runnable);
         } else {
             pe_rsc_trace(action->rsc,
                          "%s on %s does not require fencing or quorum",
                          action->uuid, pe__node_name(action->node));
             pe__set_action_flags(action, pe_action_runnable);
         }
 
     } else {
         switch (effective_quorum_policy(action->rsc, data_set)) {
             case pcmk_no_quorum_stop:
                 pe_rsc_debug(action->rsc, "%s on %s is unrunnable (no quorum)",
                              action->uuid, pe__node_name(action->node));
                 pe__clear_action_flags(action, pe_action_runnable);
                 pe_action_set_reason(action, "no quorum", true);
                 break;
 
             case pcmk_no_quorum_freeze:
                 if (!action->rsc->fns->active(action->rsc, TRUE)
                     || (action->rsc->next_role > action->rsc->role)) {
                     pe_rsc_debug(action->rsc,
                                  "%s on %s is unrunnable (no quorum)",
                                  action->uuid, pe__node_name(action->node));
                     pe__clear_action_flags(action, pe_action_runnable);
                     pe_action_set_reason(action, "quorum freeze", true);
                 }
                 break;
 
             default:
                 //pe_action_set_reason(action, NULL, TRUE);
                 pe__set_action_flags(action, pe_action_runnable);
                 break;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Update a resource object's flags for a new action on it
  *
  * \param[in,out] rsc     Resource that action is for (if any)
  * \param[in]     action  New action
  */
 static void
 update_resource_flags_for_action(pe_resource_t *rsc, const pe_action_t *action)
 {
     /* @COMPAT pe_rsc_starting and pe_rsc_stopping are not actually used
      * within Pacemaker, and should be deprecated and eventually removed
      */
     if (pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei)) {
         pe__set_resource_flags(rsc, pe_rsc_stopping);
 
     } else if (pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_casei)) {
         if (pcmk_is_set(action->flags, pe_action_runnable)) {
             pe__set_resource_flags(rsc, pe_rsc_starting);
         } else {
             pe__clear_resource_flags(rsc, pe_rsc_starting);
         }
     }
 }
 
 static bool
 valid_stop_on_fail(const char *value)
 {
     return !pcmk__strcase_any_of(value, "standby", "demote", "stop", NULL);
 }
 
 /*!
  * \internal
  * \brief Validate (and possibly reset) resource action's on_fail meta-attribute
  *
  * \param[in]     rsc            Resource that action is for
  * \param[in]     action_name    Action name
  * \param[in]     action_config  Action configuration XML from CIB (if any)
  * \param[in,out] meta           Table of action meta-attributes
  *
  * \return (Possibly new) value of on-fail meta-attribute
  */
 static const char *
 validate_on_fail(const pe_resource_t *rsc, const char *action_name,
                  const xmlNode *action_config, GHashTable *meta)
 {
     const char *name = NULL;
     const char *role = NULL;
     const char *interval_spec = NULL;
     const char *value = g_hash_table_lookup(meta, XML_OP_ATTR_ON_FAIL);
     char *key = NULL;
     char *new_value = NULL;
 
     // Stop actions can only use certain on-fail values
     if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)
         && !valid_stop_on_fail(value)) {
 
         pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s stop "
                          "action to default value because '%s' is not "
                          "allowed for stop", rsc->id, value);
         g_hash_table_remove(meta, XML_OP_ATTR_ON_FAIL);
         return NULL;
     }
 
     /* Demote actions default on-fail to the on-fail value for the first
      * recurring monitor for the promoted role (if any).
      */
     if (pcmk__str_eq(action_name, PCMK_ACTION_DEMOTE, pcmk__str_none)
         && (value == NULL)) {
 
         /* @TODO This does not consider promote options set in a meta-attribute
          * block (which may have rules that need to be evaluated) rather than
          * XML properties.
          */
         for (xmlNode *operation = first_named_child(rsc->ops_xml, XML_ATTR_OP);
              operation != NULL; operation = crm_next_same_xml(operation)) {
             bool enabled = false;
             const char *promote_on_fail = NULL;
 
             /* We only care about explicit on-fail (if promote uses default, so
              * can demote)
              */
             promote_on_fail = crm_element_value(operation, XML_OP_ATTR_ON_FAIL);
             if (promote_on_fail == NULL) {
                 continue;
             }
 
             // We only care about recurring monitors for the promoted role
             name = crm_element_value(operation, "name");
             role = crm_element_value(operation, "role");
             if (!pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
                 || !pcmk__strcase_any_of(role, PCMK__ROLE_PROMOTED,
                                          PCMK__ROLE_PROMOTED_LEGACY, NULL)) {
                 continue;
             }
             interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
             if (crm_parse_interval_spec(interval_spec) == 0) {
                 continue;
             }
 
             // We only care about enabled monitors
             if ((pcmk__xe_get_bool_attr(operation, "enabled",
                                         &enabled) == pcmk_rc_ok) && !enabled) {
                 continue;
             }
 
             // Demote actions can't default to on-fail="demote"
             if (pcmk__str_eq(promote_on_fail, "demote", pcmk__str_casei)) {
                 continue;
             }
 
             // Use value from first applicable promote action found
             key = strdup(XML_OP_ATTR_ON_FAIL);
             new_value = strdup(promote_on_fail);
             CRM_ASSERT((key != NULL) && (new_value != NULL));
             g_hash_table_insert(meta, key, new_value);
             return g_hash_table_lookup(meta, XML_OP_ATTR_ON_FAIL);
         }
         return NULL;
     }
 
     if (pcmk__str_eq(action_name, PCMK_ACTION_LRM_DELETE, pcmk__str_none)
         && !pcmk__str_eq(value, "ignore", pcmk__str_casei)) {
         key = strdup(XML_OP_ATTR_ON_FAIL);
         new_value = strdup("ignore");
         CRM_ASSERT((key != NULL) && (new_value != NULL));
         g_hash_table_insert(meta, key, new_value);
         return g_hash_table_lookup(meta, XML_OP_ATTR_ON_FAIL);
     }
 
     // on-fail="demote" is allowed only for certain actions
     if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
         name = crm_element_value(action_config, "name");
         role = crm_element_value(action_config, "role");
         interval_spec = crm_element_value(action_config,
                                           XML_LRM_ATTR_INTERVAL);
 
         if (!pcmk__str_eq(name, PCMK_ACTION_PROMOTE, pcmk__str_none)
             && (!pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
                 || !pcmk__strcase_any_of(role, PCMK__ROLE_PROMOTED,
                                          PCMK__ROLE_PROMOTED_LEGACY, NULL)
                 || (crm_parse_interval_spec(interval_spec) == 0))) {
             pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s %s "
                              "action to default value because 'demote' is not "
                              "allowed for it", rsc->id, name);
             g_hash_table_remove(meta, XML_OP_ATTR_ON_FAIL);
             return NULL;
         }
     }
 
     return value;
 }
 
 static int
 unpack_timeout(const char *value)
 {
     int timeout_ms = crm_get_msec(value);
 
     if (timeout_ms < 0) {
         timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
     }
     return timeout_ms;
 }
 
 // true if value contains valid, non-NULL interval origin for recurring op
 static bool
 unpack_interval_origin(const char *value, const xmlNode *xml_obj,
                        guint interval_ms, const crm_time_t *now,
                        long long *start_delay)
 {
     long long result = 0;
     guint interval_sec = interval_ms / 1000;
     crm_time_t *origin = NULL;
 
     // Ignore unspecified values and non-recurring operations
     if ((value == NULL) || (interval_ms == 0) || (now == NULL)) {
         return false;
     }
 
     // Parse interval origin from text
     origin = crm_time_new(value);
     if (origin == NULL) {
         pcmk__config_err("Ignoring '" XML_OP_ATTR_ORIGIN "' for operation "
                          "'%s' because '%s' is not valid",
                          (ID(xml_obj)? ID(xml_obj) : "(missing ID)"), value);
         return false;
     }
 
     // Get seconds since origin (negative if origin is in the future)
     result = crm_time_get_seconds(now) - crm_time_get_seconds(origin);
     crm_time_free(origin);
 
     // Calculate seconds from closest interval to now
     result = result % interval_sec;
 
     // Calculate seconds remaining until next interval
     result = ((result <= 0)? 0 : interval_sec) - result;
     crm_info("Calculated a start delay of %llds for operation '%s'",
              result,
              (ID(xml_obj)? ID(xml_obj) : "(unspecified)"));
 
     if (start_delay != NULL) {
         *start_delay = result * 1000; // milliseconds
     }
     return true;
 }
 
 static int
 unpack_start_delay(const char *value, GHashTable *meta)
 {
     int start_delay = 0;
 
     if (value != NULL) {
         start_delay = crm_get_msec(value);
 
         if (start_delay < 0) {
             start_delay = 0;
         }
 
         if (meta) {
             g_hash_table_replace(meta, strdup(XML_OP_ATTR_START_DELAY),
                                  pcmk__itoa(start_delay));
         }
     }
 
     return start_delay;
 }
 
 static xmlNode *
 find_min_interval_mon(pe_resource_t * rsc, gboolean include_disabled)
 {
     guint interval_ms = 0;
     guint min_interval_ms = G_MAXUINT;
     const char *name = NULL;
     const char *interval_spec = NULL;
     xmlNode *op = NULL;
     xmlNode *operation = NULL;
 
     for (operation = pcmk__xe_first_child(rsc->ops_xml);
          operation != NULL;
          operation = pcmk__xe_next(operation)) {
 
         if (pcmk__str_eq((const char *)operation->name, "op", pcmk__str_none)) {
             bool enabled = false;
 
             name = crm_element_value(operation, "name");
             interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
             if (!include_disabled && pcmk__xe_get_bool_attr(operation, "enabled", &enabled) == pcmk_rc_ok &&
                 !enabled) {
                 continue;
             }
 
             if (!pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_casei)) {
                 continue;
             }
 
             interval_ms = crm_parse_interval_spec(interval_spec);
 
             if (interval_ms && (interval_ms < min_interval_ms)) {
                 min_interval_ms = interval_ms;
                 op = operation;
             }
         }
     }
 
     return op;
 }
 
 /*!
  * \internal
  * \brief Unpack action configuration
  *
  * Unpack a resource action's meta-attributes (normalizing the interval,
  * timeout, and start delay values as integer milliseconds), requirements, and
  * failure policy from its CIB XML configuration (including defaults).
  *
  * \param[in,out] action       Resource action to unpack into
  * \param[in]     xml_obj      Action configuration XML (NULL for defaults only)
  * \param[in]     container    Resource that contains affected resource, if any
  * \param[in]     interval_ms  How frequently to perform the operation
  */
 static void
 unpack_operation(pe_action_t *action, const xmlNode *xml_obj,
                  const pe_resource_t *container, guint interval_ms)
 {
     int timeout_ms = 0;
     const char *value = NULL;
     bool is_probe = false;
 
     pe_rsc_eval_data_t rsc_rule_data = {
         .standard = crm_element_value(action->rsc->xml, XML_AGENT_ATTR_CLASS),
         .provider = crm_element_value(action->rsc->xml, XML_AGENT_ATTR_PROVIDER),
         .agent = crm_element_value(action->rsc->xml, XML_EXPR_ATTR_TYPE)
     };
 
     pe_op_eval_data_t op_rule_data = {
         .op_name = action->task,
         .interval = interval_ms
     };
 
     pe_rule_eval_data_t rule_data = {
         .node_hash = NULL,
         .role = pcmk_role_unknown,
         .now = action->rsc->cluster->now,
         .match_data = NULL,
         .rsc_data = &rsc_rule_data,
         .op_data = &op_rule_data
     };
 
     CRM_CHECK(action && action->rsc, return);
 
     is_probe = pcmk_is_probe(action->task, interval_ms);
 
     // Cluster-wide <op_defaults> <meta_attributes>
     pe__unpack_dataset_nvpairs(action->rsc->cluster->op_defaults,
                                XML_TAG_META_SETS, &rule_data, action->meta,
                                NULL, FALSE, action->rsc->cluster);
 
     // Determine probe default timeout differently
     if (is_probe) {
         xmlNode *min_interval_mon = find_min_interval_mon(action->rsc, FALSE);
 
         if (min_interval_mon) {
             value = crm_element_value(min_interval_mon, XML_ATTR_TIMEOUT);
             if (value) {
                 crm_trace("\t%s: Setting default timeout to minimum-interval "
                           "monitor's timeout '%s'", action->uuid, value);
                 g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT),
                                      strdup(value));
             }
         }
     }
 
     if (xml_obj) {
         xmlAttrPtr xIter = NULL;
 
         // <op> <meta_attributes> take precedence over defaults
         pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_META_SETS, &rule_data,
                                    action->meta, NULL, TRUE,
                                    action->rsc->cluster);
 
         /* Anything set as an <op> XML property has highest precedence.
          * This ensures we use the name and interval from the <op> tag.
          */
         for (xIter = xml_obj->properties; xIter; xIter = xIter->next) {
             const char *prop_name = (const char *)xIter->name;
             const char *prop_value = pcmk__xml_attr_value(xIter);
 
             g_hash_table_replace(action->meta, strdup(prop_name), strdup(prop_value));
         }
     }
 
     g_hash_table_remove(action->meta, "id");
 
     // Normalize interval to milliseconds
     if (interval_ms > 0) {
         g_hash_table_replace(action->meta, strdup(XML_LRM_ATTR_INTERVAL),
                              crm_strdup_printf("%u", interval_ms));
     } else {
         g_hash_table_remove(action->meta, XML_LRM_ATTR_INTERVAL);
     }
 
     /*
      * Timeout order of precedence:
      *   1. pcmk_monitor_timeout (if rsc has pcmk_ra_cap_fence_params
      *      and task is start or a probe; pcmk_monitor_timeout works
      *      by default for a recurring monitor)
      *   2. explicit op timeout on the primitive
      *   3. default op timeout
      *      a. if probe, then min-interval monitor's timeout
      *      b. else, in XML_CIB_TAG_OPCONFIG
      *   4. PCMK_DEFAULT_ACTION_TIMEOUT_MS
      *
      * #1 overrides general rule of <op> XML property having highest
      * precedence.
      */
     if (pcmk_is_set(pcmk_get_ra_caps(rsc_rule_data.standard),
                     pcmk_ra_cap_fence_params)
         && (pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_casei)
             || is_probe)) {
 
         GHashTable *params = pe_rsc_params(action->rsc, action->node,
                                            action->rsc->cluster);
 
         value = g_hash_table_lookup(params, "pcmk_monitor_timeout");
 
         if (value) {
             crm_trace("\t%s: Setting timeout to pcmk_monitor_timeout '%s', "
                       "overriding default", action->uuid, value);
             g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT),
                                  strdup(value));
         }
     }
 
     // Normalize timeout to positive milliseconds
     value = g_hash_table_lookup(action->meta, XML_ATTR_TIMEOUT);
     timeout_ms = unpack_timeout(value);
     g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT),
                          pcmk__itoa(timeout_ms));
 
     if (!pcmk__strcase_any_of(action->task, PCMK_ACTION_START,
                               PCMK_ACTION_PROMOTE, NULL)) {
         action->needs = pcmk_requires_nothing;
         value = "nothing (not start or promote)";
 
     } else if (pcmk_is_set(action->rsc->flags, pe_rsc_needs_fencing)) {
         action->needs = pcmk_requires_fencing;
         value = "fencing";
 
     } else if (pcmk_is_set(action->rsc->flags, pe_rsc_needs_quorum)) {
         action->needs = pcmk_requires_quorum;
         value = "quorum";
 
     } else {
         action->needs = pcmk_requires_nothing;
         value = "nothing";
     }
     pe_rsc_trace(action->rsc, "%s requires %s", action->uuid, value);
 
     value = validate_on_fail(action->rsc, action->task, xml_obj, action->meta);
 
     if (value == NULL) {
 
     } else if (pcmk__str_eq(value, "block", pcmk__str_casei)) {
         action->on_fail = pcmk_on_fail_block;
 
     } else if (pcmk__str_eq(value, "fence", pcmk__str_casei)) {
         action->on_fail = pcmk_on_fail_fence_node;
         value = "node fencing";
 
         if (!pcmk_is_set(action->rsc->cluster->flags,
                          pe_flag_stonith_enabled)) {
             pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for "
                              "operation '%s' to 'stop' because 'fence' is not "
                              "valid when fencing is disabled", action->uuid);
             action->on_fail = pcmk_on_fail_stop;
             action->fail_role = pcmk_role_stopped;
             value = "stop resource";
         }
 
     } else if (pcmk__str_eq(value, "standby", pcmk__str_casei)) {
         action->on_fail = pcmk_on_fail_standby_node;
         value = "node standby";
 
     } else if (pcmk__strcase_any_of(value, "ignore", PCMK__VALUE_NOTHING,
                                     NULL)) {
         action->on_fail = pcmk_on_fail_ignore;
         value = "ignore";
 
     } else if (pcmk__str_eq(value, "migrate", pcmk__str_casei)) {
         action->on_fail = pcmk_on_fail_ban;
         value = "force migration";
 
     } else if (pcmk__str_eq(value, "stop", pcmk__str_casei)) {
         action->on_fail = pcmk_on_fail_stop;
         action->fail_role = pcmk_role_stopped;
         value = "stop resource";
 
     } else if (pcmk__str_eq(value, "restart", pcmk__str_casei)) {
         action->on_fail = pcmk_on_fail_restart;
         value = "restart (and possibly migrate)";
 
     } else if (pcmk__str_eq(value, "restart-container", pcmk__str_casei)) {
         if (container) {
             action->on_fail = pcmk_on_fail_restart_container;
             value = "restart container (and possibly migrate)";
 
         } else {
             value = NULL;
         }
 
     } else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
         action->on_fail = pcmk_on_fail_demote;
         value = "demote instance";
 
     } else {
         pe_err("Resource %s: Unknown failure type (%s)", action->rsc->id, value);
         value = NULL;
     }
 
     /* defaults */
     if (value == NULL && container) {
         action->on_fail = pcmk_on_fail_restart_container;
         value = "restart container (and possibly migrate) (default)";
 
     /* For remote nodes, ensure that any failure that results in dropping an
      * active connection to the node results in fencing of the node.
      *
      * There are only two action failures that don't result in fencing.
      * 1. probes - probe failures are expected.
      * 2. start - a start failure indicates that an active connection does not already
      * exist. The user can set op on-fail=fence if they really want to fence start
      * failures. */
     } else if (((value == NULL) || !pcmk_is_set(action->rsc->flags, pe_rsc_managed))
                && pe__resource_is_remote_conn(action->rsc)
                && !(pcmk__str_eq(action->task, PCMK_ACTION_MONITOR,
                                  pcmk__str_casei)
                     && (interval_ms == 0))
                && !pcmk__str_eq(action->task, PCMK_ACTION_START, pcmk__str_casei)) {
 
         if (!pcmk_is_set(action->rsc->flags, pe_rsc_managed)) {
             action->on_fail = pcmk_on_fail_stop;
             action->fail_role = pcmk_role_stopped;
             value = "stop unmanaged remote node (enforcing default)";
 
         } else {
             if (pcmk_is_set(action->rsc->cluster->flags,
                             pe_flag_stonith_enabled)) {
                 value = "fence remote node (default)";
             } else {
                 value = "recover remote node connection (default)";
             }
 
             if (action->rsc->remote_reconnect_ms) {
                 action->fail_role = pcmk_role_stopped;
             }
             action->on_fail = pcmk_on_fail_reset_remote;
         }
 
     } else if ((value == NULL)
                && pcmk__str_eq(action->task, PCMK_ACTION_STOP,
                                pcmk__str_casei)) {
         if (pcmk_is_set(action->rsc->cluster->flags, pe_flag_stonith_enabled)) {
             action->on_fail = pcmk_on_fail_fence_node;
             value = "resource fence (default)";
 
         } else {
             action->on_fail = pcmk_on_fail_block;
             value = "resource block (default)";
         }
 
     } else if (value == NULL) {
         action->on_fail = pcmk_on_fail_restart;
         value = "restart (and possibly migrate) (default)";
     }
 
     pe_rsc_trace(action->rsc, "%s failure handling: %s",
                  action->uuid, value);
 
     value = NULL;
     if (xml_obj != NULL) {
         value = g_hash_table_lookup(action->meta, "role_after_failure");
         if (value) {
             pe_warn_once(pe_wo_role_after,
                         "Support for role_after_failure is deprecated and will be removed in a future release");
         }
     }
     if (value != NULL && action->fail_role == pcmk_role_unknown) {
         action->fail_role = text2role(value);
     }
     /* defaults */
     if (action->fail_role == pcmk_role_unknown) {
         if (pcmk__str_eq(action->task, PCMK_ACTION_PROMOTE, pcmk__str_casei)) {
             action->fail_role = pcmk_role_unpromoted;
         } else {
             action->fail_role = pcmk_role_started;
         }
     }
     pe_rsc_trace(action->rsc, "%s failure results in: %s",
                  action->uuid, role2text(action->fail_role));
 
     value = g_hash_table_lookup(action->meta, XML_OP_ATTR_START_DELAY);
     if (value) {
         unpack_start_delay(value, action->meta);
     } else {
         long long start_delay = 0;
 
         value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ORIGIN);
         if (unpack_interval_origin(value, xml_obj, interval_ms,
                                    action->rsc->cluster->now, &start_delay)) {
             g_hash_table_replace(action->meta, strdup(XML_OP_ATTR_START_DELAY),
                                  crm_strdup_printf("%lld", start_delay));
         }
     }
 }
 
 /*!
  * \brief Create or update an action object
  *
  * \param[in,out] rsc          Resource that action is for (if any)
  * \param[in,out] key          Action key (must be non-NULL)
  * \param[in]     task         Action name (must be non-NULL)
  * \param[in]     on_node      Node that action is on (if any)
  * \param[in]     optional     Whether action should be considered optional
  * \param[in]     save_action  Whether action should be recorded in transition graph
  * \param[in,out] data_set     Cluster working set
  *
  * \return Action object corresponding to arguments
  * \note This function takes ownership of (and might free) \p key. If
  *       \p save_action is true, \p data_set will own the returned action,
  *       otherwise it is the caller's responsibility to free the return value
  *       with pe_free_action().
  */
 pe_action_t *
 custom_action(pe_resource_t *rsc, char *key, const char *task,
               const pe_node_t *on_node, gboolean optional, gboolean save_action,
               pe_working_set_t *data_set)
 {
     pe_action_t *action = NULL;
 
     CRM_ASSERT((key != NULL) && (task != NULL) && (data_set != NULL));
 
     if (save_action) {
         action = find_existing_action(key, rsc, on_node, data_set);
     }
 
     if (action == NULL) {
         action = new_action(key, task, rsc, on_node, optional, save_action,
                             data_set);
     } else {
         free(key);
     }
 
     update_action_optional(action, optional);
 
     if (rsc != NULL) {
         if (action->node != NULL) {
             unpack_action_node_attributes(action, data_set);
         }
 
         update_resource_action_runnable(action, save_action, data_set);
 
         if (save_action) {
             update_resource_flags_for_action(rsc, action);
         }
     }
 
     return action;
 }
 
 pe_action_t *
 get_pseudo_op(const char *name, pe_working_set_t * data_set)
 {
     pe_action_t *op = lookup_singleton(data_set, name);
 
     if (op == NULL) {
         op = custom_action(NULL, strdup(name), name, NULL, TRUE, TRUE, data_set);
         pe__set_action_flags(op, pe_action_pseudo|pe_action_runnable);
     }
     return op;
 }
 
 static GList *
 find_unfencing_devices(GList *candidates, GList *matches) 
 {
     for (GList *gIter = candidates; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *candidate = gIter->data;
 
         if (candidate->children != NULL) {
             matches = find_unfencing_devices(candidate->children, matches);
 
         } else if (!pcmk_is_set(candidate->flags, pe_rsc_fence_device)) {
             continue;
 
         } else if (pcmk_is_set(candidate->flags, pe_rsc_needs_unfencing)) {
             matches = g_list_prepend(matches, candidate);
 
         } else if (pcmk__str_eq(g_hash_table_lookup(candidate->meta,
                                                     PCMK_STONITH_PROVIDES),
                                 PCMK__VALUE_UNFENCING,
                                 pcmk__str_casei)) {
             matches = g_list_prepend(matches, candidate);
         }
     }
     return matches;
 }
 
 static int
 node_priority_fencing_delay(const pe_node_t *node,
                             const pe_working_set_t *data_set)
 {
     int member_count = 0;
     int online_count = 0;
     int top_priority = 0;
     int lowest_priority = 0;
     GList *gIter = NULL;
 
     // `priority-fencing-delay` is disabled
     if (data_set->priority_fencing_delay <= 0) {
         return 0;
     }
 
     /* No need to request a delay if the fencing target is not a normal cluster
      * member, for example if it's a remote node or a guest node. */
     if (node->details->type != pcmk_node_variant_cluster) {
         return 0;
     }
 
     // No need to request a delay if the fencing target is in our partition
     if (node->details->online) {
         return 0;
     }
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         pe_node_t *n =  gIter->data;
 
         if (n->details->type != pcmk_node_variant_cluster) {
             continue;
         }
 
         member_count ++;
 
         if (n->details->online) {
             online_count++;
         }
 
         if (member_count == 1
             || n->details->priority > top_priority) {
             top_priority = n->details->priority;
         }
 
         if (member_count == 1
             || n->details->priority < lowest_priority) {
             lowest_priority = n->details->priority;
         }
     }
 
     // No need to delay if we have more than half of the cluster members
     if (online_count > member_count / 2) {
         return 0;
     }
 
     /* All the nodes have equal priority.
      * Any configured corresponding `pcmk_delay_base/max` will be applied. */
     if (lowest_priority == top_priority) {
         return 0;
     }
 
     if (node->details->priority < top_priority) {
         return 0;
     }
 
     return data_set->priority_fencing_delay;
 }
 
 pe_action_t *
 pe_fence_op(pe_node_t *node, const char *op, bool optional,
             const char *reason, bool priority_delay, pe_working_set_t *data_set)
 {
     char *op_key = NULL;
     pe_action_t *stonith_op = NULL;
 
     if(op == NULL) {
         op = data_set->stonith_action;
     }
 
     op_key = crm_strdup_printf("%s-%s-%s",
                                PCMK_ACTION_STONITH, node->details->uname, op);
 
     stonith_op = lookup_singleton(data_set, op_key);
     if(stonith_op == NULL) {
         stonith_op = custom_action(NULL, op_key, PCMK_ACTION_STONITH, node,
                                    TRUE, TRUE, data_set);
 
         add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname);
         add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id);
         add_hash_param(stonith_op->meta, "stonith_action", op);
 
         if (pcmk_is_set(data_set->flags, pe_flag_enable_unfencing)) {
             /* Extra work to detect device changes
              */
             GString *digests_all = g_string_sized_new(1024);
             GString *digests_secure = g_string_sized_new(1024);
 
             GList *matches = find_unfencing_devices(data_set->resources, NULL);
 
             char *key = NULL;
             char *value = NULL;
 
             for (GList *gIter = matches; gIter != NULL; gIter = gIter->next) {
                 pe_resource_t *match = gIter->data;
                 const char *agent = g_hash_table_lookup(match->meta,
                                                         XML_ATTR_TYPE);
                 op_digest_cache_t *data = NULL;
 
                 data = pe__compare_fencing_digest(match, agent, node, data_set);
                 if(data->rc == RSC_DIGEST_ALL) {
                     optional = FALSE;
                     crm_notice("Unfencing node %s because the definition of "
                                "%s changed", pe__node_name(node), match->id);
                     if (!pcmk__is_daemon && data_set->priv != NULL) {
                         pcmk__output_t *out = data_set->priv;
 
                         out->info(out,
                                   "notice: Unfencing node %s because the "
                                   "definition of %s changed",
                                   pe__node_name(node), match->id);
                     }
                 }
 
                 pcmk__g_strcat(digests_all,
                                match->id, ":", agent, ":",
                                data->digest_all_calc, ",", NULL);
                 pcmk__g_strcat(digests_secure,
                                match->id, ":", agent, ":",
                                data->digest_secure_calc, ",", NULL);
             }
             key = strdup(XML_OP_ATTR_DIGESTS_ALL);
             value = strdup((const char *) digests_all->str);
             CRM_ASSERT((key != NULL) && (value != NULL));
             g_hash_table_insert(stonith_op->meta, key, value);
             g_string_free(digests_all, TRUE);
 
             key = strdup(XML_OP_ATTR_DIGESTS_SECURE);
             value = strdup((const char *) digests_secure->str);
             CRM_ASSERT((key != NULL) && (value != NULL));
             g_hash_table_insert(stonith_op->meta, key, value);
             g_string_free(digests_secure, TRUE);
         }
 
     } else {
         free(op_key);
     }
 
     if (data_set->priority_fencing_delay > 0
 
             /* It's a suitable case where `priority-fencing-delay` applies.
              * At least add `priority-fencing-delay` field as an indicator. */
         && (priority_delay
 
             /* The priority delay needs to be recalculated if this function has
              * been called by schedule_fencing_and_shutdowns() after node
              * priority has already been calculated by native_add_running().
              */
             || g_hash_table_lookup(stonith_op->meta,
                                    XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY) != NULL)) {
 
             /* Add `priority-fencing-delay` to the fencing op even if it's 0 for
              * the targeting node. So that it takes precedence over any possible
              * `pcmk_delay_base/max`.
              */
             char *delay_s = pcmk__itoa(node_priority_fencing_delay(node, data_set));
 
             g_hash_table_insert(stonith_op->meta,
                                 strdup(XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY),
                                 delay_s);
     }
 
     if(optional == FALSE && pe_can_fence(data_set, node)) {
         pe__clear_action_flags(stonith_op, pe_action_optional);
         pe_action_set_reason(stonith_op, reason, false);
 
     } else if(reason && stonith_op->reason == NULL) {
         stonith_op->reason = strdup(reason);
     }
 
     return stonith_op;
 }
 
 void
 pe_free_action(pe_action_t * action)
 {
     if (action == NULL) {
         return;
     }
     g_list_free_full(action->actions_before, free);     /* pe_action_wrapper_t* */
     g_list_free_full(action->actions_after, free);      /* pe_action_wrapper_t* */
     if (action->extra) {
         g_hash_table_destroy(action->extra);
     }
     if (action->meta) {
         g_hash_table_destroy(action->meta);
     }
     free(action->cancel_task);
     free(action->reason);
     free(action->task);
     free(action->uuid);
     free(action->node);
     free(action);
 }
 
 int
 pe_get_configured_timeout(pe_resource_t *rsc, const char *action, pe_working_set_t *data_set)
 {
     xmlNode *child = NULL;
     GHashTable *action_meta = NULL;
     const char *timeout_spec = NULL;
     int timeout_ms = 0;
 
     pe_rule_eval_data_t rule_data = {
         .node_hash = NULL,
         .role = pcmk_role_unknown,
         .now = data_set->now,
         .match_data = NULL,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     for (child = first_named_child(rsc->ops_xml, XML_ATTR_OP);
          child != NULL; child = crm_next_same_xml(child)) {
         if (pcmk__str_eq(action, crm_element_value(child, XML_NVPAIR_ATTR_NAME),
                 pcmk__str_casei)) {
             timeout_spec = crm_element_value(child, XML_ATTR_TIMEOUT);
             break;
         }
     }
 
     if (timeout_spec == NULL && data_set->op_defaults) {
         action_meta = pcmk__strkey_table(free, free);
         pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS,
                                    &rule_data, action_meta, NULL, FALSE, data_set);
         timeout_spec = g_hash_table_lookup(action_meta, XML_ATTR_TIMEOUT);
     }
 
     // @TODO check meta-attributes
     // @TODO maybe use min-interval monitor timeout as default for monitors
 
     timeout_ms = crm_get_msec(timeout_spec);
     if (timeout_ms < 0) {
         timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
     }
 
     if (action_meta != NULL) {
         g_hash_table_destroy(action_meta);
     }
     return timeout_ms;
 }
 
 enum action_tasks
 get_complex_task(const pe_resource_t *rsc, const char *name)
 {
     enum action_tasks task = text2task(name);
 
     if ((rsc != NULL) && (rsc->variant == pcmk_rsc_variant_primitive)) {
         switch (task) {
             case pcmk_action_stopped:
             case pcmk_action_started:
             case pcmk_action_demoted:
             case pcmk_action_promoted:
                 crm_trace("Folding %s back into its atomic counterpart for %s",
                           name, rsc->id);
                 --task;
                 break;
             default:
                 break;
         }
     }
     return task;
 }
 
 /*!
  * \internal
  * \brief Find first matching action in a list
  *
  * \param[in] input    List of actions to search
  * \param[in] uuid     If not NULL, action must have this UUID
  * \param[in] task     If not NULL, action must have this action name
  * \param[in] on_node  If not NULL, action must be on this node
  *
  * \return First action in list that matches criteria, or NULL if none
  */
 pe_action_t *
 find_first_action(const GList *input, const char *uuid, const char *task,
                   const pe_node_t *on_node)
 {
     CRM_CHECK(uuid || task, return NULL);
 
     for (const GList *gIter = input; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if (uuid != NULL && !pcmk__str_eq(uuid, action->uuid, pcmk__str_casei)) {
             continue;
 
         } else if (task != NULL && !pcmk__str_eq(task, action->task, pcmk__str_casei)) {
             continue;
 
         } else if (on_node == NULL) {
             return action;
 
         } else if (action->node == NULL) {
             continue;
 
         } else if (on_node->details == action->node->details) {
             return action;
         }
     }
 
     return NULL;
 }
 
 GList *
 find_actions(GList *input, const char *key, const pe_node_t *on_node)
 {
     GList *gIter = input;
     GList *result = NULL;
 
     CRM_CHECK(key != NULL, return NULL);
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if (!pcmk__str_eq(key, action->uuid, pcmk__str_casei)) {
             continue;
 
         } else if (on_node == NULL) {
             crm_trace("Action %s matches (ignoring node)", key);
             result = g_list_prepend(result, action);
 
         } else if (action->node == NULL) {
             crm_trace("Action %s matches (unallocated, assigning to %s)",
                       key, pe__node_name(on_node));
 
             action->node = pe__copy_node(on_node);
             result = g_list_prepend(result, action);
 
         } else if (on_node->details == action->node->details) {
             crm_trace("Action %s on %s matches", key, pe__node_name(on_node));
             result = g_list_prepend(result, action);
         }
     }
 
     return result;
 }
 
 GList *
 find_actions_exact(GList *input, const char *key, const pe_node_t *on_node)
 {
     GList *result = NULL;
 
     CRM_CHECK(key != NULL, return NULL);
 
     if (on_node == NULL) {
         return NULL;
     }
 
     for (GList *gIter = input; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if ((action->node != NULL)
             && pcmk__str_eq(key, action->uuid, pcmk__str_casei)
             && pcmk__str_eq(on_node->details->id, action->node->details->id,
                             pcmk__str_casei)) {
 
             crm_trace("Action %s on %s matches", key, pe__node_name(on_node));
             result = g_list_prepend(result, action);
         }
     }
 
     return result;
 }
 
 /*!
  * \brief Find all actions of given type for a resource
  *
  * \param[in] rsc           Resource to search
  * \param[in] node          Find only actions scheduled on this node
  * \param[in] task          Action name to search for
  * \param[in] require_node  If TRUE, NULL node or action node will not match
  *
  * \return List of actions found (or NULL if none)
  * \note If node is not NULL and require_node is FALSE, matching actions
  *       without a node will be assigned to node.
  */
 GList *
 pe__resource_actions(const pe_resource_t *rsc, const pe_node_t *node,
                      const char *task, bool require_node)
 {
     GList *result = NULL;
     char *key = pcmk__op_key(rsc->id, task, 0);
 
     if (require_node) {
         result = find_actions_exact(rsc->actions, key, node);
     } else {
         result = find_actions(rsc->actions, key, node);
     }
     free(key);
     return result;
 }
 
 /*!
  * \internal
  * \brief Create an action reason string based on the action itself
  *
  * \param[in] action  Action to create reason string for
  * \param[in] flag    Action flag that was cleared
  *
  * \return Newly allocated string suitable for use as action reason
  * \note It is the caller's responsibility to free() the result.
  */
 char *
 pe__action2reason(const pe_action_t *action, enum pe_action_flags flag)
 {
     const char *change = NULL;
 
     switch (flag) {
         case pe_action_runnable:
             change = "unrunnable";
             break;
         case pe_action_migrate_runnable:
             change = "unmigrateable";
             break;
         case pe_action_optional:
             change = "required";
             break;
         default:
             // Bug: caller passed unsupported flag
             CRM_CHECK(change != NULL, change = "");
             break;
     }
     return crm_strdup_printf("%s%s%s %s", change,
                              (action->rsc == NULL)? "" : " ",
                              (action->rsc == NULL)? "" : action->rsc->id,
                              action->task);
 }
 
 void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrite) 
 {
     if (action->reason != NULL && overwrite) {
         pe_rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'",
                      action->uuid, action->reason, pcmk__s(reason, "(none)"));
     } else if (action->reason == NULL) {
         pe_rsc_trace(action->rsc, "Set %s reason to '%s'",
                      action->uuid, pcmk__s(reason, "(none)"));
     } else {
         // crm_assert(action->reason != NULL && !overwrite);
         return;
     }
 
     pcmk__str_update(&action->reason, reason);
 }
 
 /*!
  * \internal
  * \brief Create an action to clear a resource's history from CIB
  *
  * \param[in,out] rsc       Resource to clear
  * \param[in]     node      Node to clear history on
  * \param[in,out] data_set  Cluster working set
  *
  * \return New action to clear resource history
  */
 pe_action_t *
 pe__clear_resource_history(pe_resource_t *rsc, const pe_node_t *node,
                            pe_working_set_t *data_set)
 {
     char *key = NULL;
 
     CRM_ASSERT(rsc && node);
     key = pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0);
     return custom_action(rsc, key, PCMK_ACTION_LRM_DELETE, node, FALSE, TRUE,
                          data_set);
 }
 
 #define sort_return(an_int, why) do {					\
 	free(a_uuid);						\
 	free(b_uuid);						\
 	crm_trace("%s (%d) %c %s (%d) : %s",				\
 		  a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=',	\
 		  b_xml_id, b_call_id, why);				\
 	return an_int;							\
     } while(0)
 
 int
 pe__is_newer_op(const xmlNode *xml_a, const xmlNode *xml_b,
                 bool same_node_default)
 {
     int a_call_id = -1;
     int b_call_id = -1;
 
     char *a_uuid = NULL;
     char *b_uuid = NULL;
 
     const char *a_xml_id = crm_element_value(xml_a, XML_ATTR_ID);
     const char *b_xml_id = crm_element_value(xml_b, XML_ATTR_ID);
 
     const char *a_node = crm_element_value(xml_a, XML_LRM_ATTR_TARGET);
     const char *b_node = crm_element_value(xml_b, XML_LRM_ATTR_TARGET);
     bool same_node = true;
 
     /* @COMPAT The on_node attribute was added to last_failure as of 1.1.13 (via
      * 8b3ca1c) and the other entries as of 1.1.12 (via 0b07b5c).
      *
      * In case that any of the lrm_rsc_op entries doesn't have on_node
      * attribute, we need to explicitly tell whether the two operations are on
      * the same node.
      */
     if (a_node == NULL || b_node == NULL) {
         same_node = same_node_default;
 
     } else {
         same_node = pcmk__str_eq(a_node, b_node, pcmk__str_casei);
     }
 
     if (same_node && pcmk__str_eq(a_xml_id, b_xml_id, pcmk__str_none)) {
         /* We have duplicate lrm_rsc_op entries in the status
          * section which is unlikely to be a good thing
          *    - we can handle it easily enough, but we need to get
          *    to the bottom of why it's happening.
          */
         pe_err("Duplicate lrm_rsc_op entries named %s", a_xml_id);
         sort_return(0, "duplicate");
     }
 
     crm_element_value_int(xml_a, XML_LRM_ATTR_CALLID, &a_call_id);
     crm_element_value_int(xml_b, XML_LRM_ATTR_CALLID, &b_call_id);
 
     if (a_call_id == -1 && b_call_id == -1) {
         /* both are pending ops so it doesn't matter since
          *   stops are never pending
          */
         sort_return(0, "pending");
 
     } else if (same_node && a_call_id >= 0 && a_call_id < b_call_id) {
         sort_return(-1, "call id");
 
     } else if (same_node && b_call_id >= 0 && a_call_id > b_call_id) {
         sort_return(1, "call id");
 
     } else if (a_call_id >= 0 && b_call_id >= 0
                && (!same_node || a_call_id == b_call_id)) {
         /*
          * The op and last_failed_op are the same
          * Order on last-rc-change
          */
         time_t last_a = -1;
         time_t last_b = -1;
 
         crm_element_value_epoch(xml_a, XML_RSC_OP_LAST_CHANGE, &last_a);
         crm_element_value_epoch(xml_b, XML_RSC_OP_LAST_CHANGE, &last_b);
 
         crm_trace("rc-change: %lld vs %lld",
                   (long long) last_a, (long long) last_b);
         if (last_a >= 0 && last_a < last_b) {
             sort_return(-1, "rc-change");
 
         } else if (last_b >= 0 && last_a > last_b) {
             sort_return(1, "rc-change");
         }
         sort_return(0, "rc-change");
 
     } else {
         /* One of the inputs is a pending operation
          * Attempt to use XML_ATTR_TRANSITION_MAGIC to determine its age relative to the other
          */
 
         int a_id = -1;
         int b_id = -1;
 
         const char *a_magic = crm_element_value(xml_a, XML_ATTR_TRANSITION_MAGIC);
         const char *b_magic = crm_element_value(xml_b, XML_ATTR_TRANSITION_MAGIC);
 
         CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
         if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL,
                                      NULL)) {
             sort_return(0, "bad magic a");
         }
         if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL,
                                      NULL)) {
             sort_return(0, "bad magic b");
         }
         /* try to determine the relative age of the operation...
          * some pending operations (e.g. a start) may have been superseded
          *   by a subsequent stop
          *
          * [a|b]_id == -1 means it's a shutdown operation and _always_ comes last
          */
         if (!pcmk__str_eq(a_uuid, b_uuid, pcmk__str_casei) || a_id == b_id) {
             /*
              * some of the logic in here may be redundant...
              *
              * if the UUID from the TE doesn't match then one better
              *   be a pending operation.
              * pending operations don't survive between elections and joins
              *   because we query the LRM directly
              */
 
             if (b_call_id == -1) {
                 sort_return(-1, "transition + call");
 
             } else if (a_call_id == -1) {
                 sort_return(1, "transition + call");
             }
 
         } else if ((a_id >= 0 && a_id < b_id) || b_id == -1) {
             sort_return(-1, "transition");
 
         } else if ((b_id >= 0 && a_id > b_id) || a_id == -1) {
             sort_return(1, "transition");
         }
     }
 
     /* we should never end up here */
     CRM_CHECK(FALSE, sort_return(0, "default"));
 }
 
 gint
 sort_op_by_callid(gconstpointer a, gconstpointer b)
 {
     const xmlNode *xml_a = a;
     const xmlNode *xml_b = b;
 
     return pe__is_newer_op(xml_a, xml_b, true);
 }
 
 /*!
  * \internal
  * \brief Create a new pseudo-action for a resource
  *
  * \param[in,out] rsc       Resource to create action for
  * \param[in]     task      Action name
  * \param[in]     optional  Whether action should be considered optional
  * \param[in]     runnable  Whethe action should be considered runnable
  *
  * \return New action object corresponding to arguments
  */
 pe_action_t *
 pe__new_rsc_pseudo_action(pe_resource_t *rsc, const char *task, bool optional,
                           bool runnable)
 {
     pe_action_t *action = NULL;
 
     CRM_ASSERT((rsc != NULL) && (task != NULL));
 
     action = custom_action(rsc, pcmk__op_key(rsc->id, task, 0), task, NULL,
                            optional, TRUE, rsc->cluster);
     pe__set_action_flags(action, pe_action_pseudo);
     if (runnable) {
         pe__set_action_flags(action, pe_action_runnable);
     }
     return action;
 }
 
 /*!
  * \internal
  * \brief Add the expected result to an action
  *
  * \param[in,out] action           Action to add expected result to
  * \param[in]     expected_result  Expected result to add
  *
  * \note This is more efficient than calling add_hash_param().
  */
 void
 pe__add_action_expected_result(pe_action_t *action, int expected_result)
 {
     char *name = NULL;
 
     CRM_ASSERT((action != NULL) && (action->meta != NULL));
 
     name = strdup(XML_ATTR_TE_TARGET_RC);
     CRM_ASSERT (name != NULL);
 
     g_hash_table_insert(action->meta, name, pcmk__itoa(expected_result));
 }
diff --git a/lib/pengine/status.c b/lib/pengine/status.c
index 2d1577e7e2..bd5a9a7bbd 100644
--- a/lib/pengine/status.c
+++ b/lib/pengine/status.c
@@ -1,483 +1,483 @@
 /*
  * Copyright 2004-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <glib.h>
 
 #include <crm/pengine/internal.h>
 #include <pe_status_private.h>
 
 /*!
  * \brief Create a new working set
  *
  * \return New, initialized working set on success, else NULL (and set errno)
  * \note Only pe_working_set_t objects created with this function (as opposed
  *       to statically declared or directly allocated) should be used with the
  *       functions in this library, to allow for future extensions to the
  *       data type. The caller is responsible for freeing the memory with
  *       pe_free_working_set() when the instance is no longer needed.
  */
 pe_working_set_t *
 pe_new_working_set(void)
 {
     pe_working_set_t *data_set = calloc(1, sizeof(pe_working_set_t));
 
     if (data_set != NULL) {
         set_working_set_defaults(data_set);
     }
     return data_set;
 }
 
 /*!
  * \brief Free a working set
  *
  * \param[in,out] data_set  Working set to free
  */
 void
 pe_free_working_set(pe_working_set_t *data_set)
 {
     if (data_set != NULL) {
         pe_reset_working_set(data_set);
         data_set->priv = NULL;
         free(data_set);
     }
 }
 
 /*
  * Unpack everything
  * At the end you'll have:
  *  - A list of nodes
  *  - A list of resources (each with any dependencies on other resources)
  *  - A list of constraints between resources and nodes
  *  - A list of constraints between start/stop actions
  *  - A list of nodes that need to be stonith'd
  *  - A list of nodes that need to be shutdown
  *  - A list of the possible stop/start actions (without dependencies)
  */
 gboolean
 cluster_status(pe_working_set_t * data_set)
 {
     xmlNode *section = NULL;
 
     if ((data_set == NULL) || (data_set->input == NULL)) {
         return FALSE;
     }
 
     crm_trace("Beginning unpack");
 
     if (data_set->failed != NULL) {
         free_xml(data_set->failed);
     }
     data_set->failed = create_xml_node(NULL, "failed-ops");
 
     if (data_set->now == NULL) {
         data_set->now = crm_time_new(NULL);
     }
 
     if (data_set->dc_uuid == NULL) {
         data_set->dc_uuid = crm_element_value_copy(data_set->input,
                                                    XML_ATTR_DC_UUID);
     }
 
     if (pcmk__xe_attr_is_true(data_set->input, XML_ATTR_HAVE_QUORUM)) {
-        pe__set_working_set_flags(data_set, pe_flag_have_quorum);
+        pe__set_working_set_flags(data_set, pcmk_sched_quorate);
     } else {
-        pe__clear_working_set_flags(data_set, pe_flag_have_quorum);
+        pe__clear_working_set_flags(data_set, pcmk_sched_quorate);
     }
 
     data_set->op_defaults = get_xpath_object("//" XML_CIB_TAG_OPCONFIG,
                                              data_set->input, LOG_NEVER);
     data_set->rsc_defaults = get_xpath_object("//" XML_CIB_TAG_RSCCONFIG,
                                               data_set->input, LOG_NEVER);
 
     section = get_xpath_object("//" XML_CIB_TAG_CRMCONFIG, data_set->input,
                                LOG_TRACE);
     unpack_config(section, data_set);
 
    if (!pcmk_any_flags_set(data_set->flags,
-                           pe_flag_quick_location|pe_flag_have_quorum)
+                           pe_flag_quick_location|pcmk_sched_quorate)
        && (data_set->no_quorum_policy != pcmk_no_quorum_ignore)) {
         crm_warn("Fencing and resource management disabled due to lack of quorum");
     }
 
     section = get_xpath_object("//" XML_CIB_TAG_NODES, data_set->input,
                                LOG_TRACE);
     unpack_nodes(section, data_set);
 
     section = get_xpath_object("//" XML_CIB_TAG_RESOURCES, data_set->input,
                                LOG_TRACE);
     if (!pcmk_is_set(data_set->flags, pe_flag_quick_location)) {
         unpack_remote_nodes(section, data_set);
     }
     unpack_resources(section, data_set);
 
     section = get_xpath_object("//" XML_CIB_TAG_TAGS, data_set->input,
                                LOG_NEVER);
     unpack_tags(section, data_set);
 
     if (!pcmk_is_set(data_set->flags, pe_flag_quick_location)) {
         section = get_xpath_object("//"XML_CIB_TAG_STATUS, data_set->input,
                                    LOG_TRACE);
         unpack_status(section, data_set);
     }
 
     if (!pcmk_is_set(data_set->flags, pe_flag_no_counts)) {
         for (GList *item = data_set->resources; item != NULL;
              item = item->next) {
             ((pe_resource_t *) (item->data))->fns->count(item->data);
         }
         crm_trace("Cluster resource count: %d (%d disabled, %d blocked)",
                   data_set->ninstances, data_set->disabled_resources,
                   data_set->blocked_resources);
     }
 
     pe__set_working_set_flags(data_set, pe_flag_have_status);
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Free a list of pe_resource_t
  *
  * \param[in,out] resources  List to free
  *
  * \note When a working set's resource list is freed, that includes the original
  *       storage for the uname and id of any Pacemaker Remote nodes in the
  *       working set's node list, so take care not to use those afterward.
  * \todo Refactor pe_node_t to strdup() the node name.
  */
 static void
 pe_free_resources(GList *resources)
 {
     pe_resource_t *rsc = NULL;
     GList *iterator = resources;
 
     while (iterator != NULL) {
         rsc = (pe_resource_t *) iterator->data;
         iterator = iterator->next;
         rsc->fns->free(rsc);
     }
     if (resources != NULL) {
         g_list_free(resources);
     }
 }
 
 static void
 pe_free_actions(GList *actions)
 {
     GList *iterator = actions;
 
     while (iterator != NULL) {
         pe_free_action(iterator->data);
         iterator = iterator->next;
     }
     if (actions != NULL) {
         g_list_free(actions);
     }
 }
 
 static void
 pe_free_nodes(GList *nodes)
 {
     for (GList *iterator = nodes; iterator != NULL; iterator = iterator->next) {
         pe_node_t *node = (pe_node_t *) iterator->data;
 
         // Shouldn't be possible, but to be safe ...
         if (node == NULL) {
             continue;
         }
         if (node->details == NULL) {
             free(node);
             continue;
         }
 
         /* This is called after pe_free_resources(), which means that we can't
          * use node->details->uname for Pacemaker Remote nodes.
          */
         crm_trace("Freeing node %s", (pe__is_guest_or_remote_node(node)?
                   "(guest or remote)" : pe__node_name(node)));
 
         if (node->details->attrs != NULL) {
             g_hash_table_destroy(node->details->attrs);
         }
         if (node->details->utilization != NULL) {
             g_hash_table_destroy(node->details->utilization);
         }
         if (node->details->digest_cache != NULL) {
             g_hash_table_destroy(node->details->digest_cache);
         }
         g_list_free(node->details->running_rsc);
         g_list_free(node->details->allocated_rsc);
         free(node->details);
         free(node);
     }
     if (nodes != NULL) {
         g_list_free(nodes);
     }
 }
 
 static void
 pe__free_ordering(GList *constraints)
 {
     GList *iterator = constraints;
 
     while (iterator != NULL) {
         pe__ordering_t *order = iterator->data;
 
         iterator = iterator->next;
 
         free(order->lh_action_task);
         free(order->rh_action_task);
         free(order);
     }
     if (constraints != NULL) {
         g_list_free(constraints);
     }
 }
 
 static void
 pe__free_location(GList *constraints)
 {
     GList *iterator = constraints;
 
     while (iterator != NULL) {
         pe__location_t *cons = iterator->data;
 
         iterator = iterator->next;
 
         g_list_free_full(cons->node_list_rh, free);
         free(cons->id);
         free(cons);
     }
     if (constraints != NULL) {
         g_list_free(constraints);
     }
 }
 
 /*!
  * \brief Reset working set to default state without freeing it or constraints
  *
  * \param[in,out] data_set  Working set to reset
  *
  * \deprecated This function is deprecated as part of the API;
  *             pe_reset_working_set() should be used instead.
  */
 void
 cleanup_calculations(pe_working_set_t * data_set)
 {
     if (data_set == NULL) {
         return;
     }
 
     pe__clear_working_set_flags(data_set, pe_flag_have_status);
     if (data_set->config_hash != NULL) {
         g_hash_table_destroy(data_set->config_hash);
     }
 
     if (data_set->singletons != NULL) {
         g_hash_table_destroy(data_set->singletons);
     }
 
     if (data_set->tickets) {
         g_hash_table_destroy(data_set->tickets);
     }
 
     if (data_set->template_rsc_sets) {
         g_hash_table_destroy(data_set->template_rsc_sets);
     }
 
     if (data_set->tags) {
         g_hash_table_destroy(data_set->tags);
     }
 
     free(data_set->dc_uuid);
 
     crm_trace("deleting resources");
     pe_free_resources(data_set->resources);
 
     crm_trace("deleting actions");
     pe_free_actions(data_set->actions);
 
     crm_trace("deleting nodes");
     pe_free_nodes(data_set->nodes);
 
     pe__free_param_checks(data_set);
     g_list_free(data_set->stop_needed);
     free_xml(data_set->graph);
     crm_time_free(data_set->now);
     free_xml(data_set->input);
     free_xml(data_set->failed);
 
     set_working_set_defaults(data_set);
 
     CRM_CHECK(data_set->ordering_constraints == NULL,;
         );
     CRM_CHECK(data_set->placement_constraints == NULL,;
         );
 }
 
 /*!
  * \brief Reset a working set to default state without freeing it
  *
  * \param[in,out] data_set  Working set to reset
  */
 void
 pe_reset_working_set(pe_working_set_t *data_set)
 {
     if (data_set == NULL) {
         return;
     }
 
     crm_trace("Deleting %d ordering constraints",
               g_list_length(data_set->ordering_constraints));
     pe__free_ordering(data_set->ordering_constraints);
     data_set->ordering_constraints = NULL;
 
     crm_trace("Deleting %d location constraints",
               g_list_length(data_set->placement_constraints));
     pe__free_location(data_set->placement_constraints);
     data_set->placement_constraints = NULL;
 
     crm_trace("Deleting %d colocation constraints",
               g_list_length(data_set->colocation_constraints));
     g_list_free_full(data_set->colocation_constraints, free);
     data_set->colocation_constraints = NULL;
 
     crm_trace("Deleting %d ticket constraints",
               g_list_length(data_set->ticket_constraints));
     g_list_free_full(data_set->ticket_constraints, free);
     data_set->ticket_constraints = NULL;
 
     cleanup_calculations(data_set);
 }
 
 void
 set_working_set_defaults(pe_working_set_t * data_set)
 {
     void *priv = data_set->priv;
 
     memset(data_set, 0, sizeof(pe_working_set_t));
 
     data_set->priv = priv;
     data_set->order_id = 1;
     data_set->action_id = 1;
     data_set->no_quorum_policy = pcmk_no_quorum_stop;
 
     data_set->flags = 0x0ULL;
 
     pe__set_working_set_flags(data_set,
                               pe_flag_stop_rsc_orphans
                               |pe_flag_symmetric_cluster
                               |pe_flag_stop_action_orphans);
     if (!strcmp(PCMK__CONCURRENT_FENCING_DEFAULT, "true")) {
         pe__set_working_set_flags(data_set, pe_flag_concurrent_fencing);
     }
 }
 
 pe_resource_t *
 pe_find_resource(GList *rsc_list, const char *id)
 {
     return pe_find_resource_with_flags(rsc_list, id, pcmk_rsc_match_history);
 }
 
 pe_resource_t *
 pe_find_resource_with_flags(GList *rsc_list, const char *id, enum pe_find flags)
 {
     GList *rIter = NULL;
 
     for (rIter = rsc_list; id && rIter; rIter = rIter->next) {
         pe_resource_t *parent = rIter->data;
 
         pe_resource_t *match =
             parent->fns->find_rsc(parent, id, NULL, flags);
         if (match != NULL) {
             return match;
         }
     }
     crm_trace("No match for %s", id);
     return NULL;
 }
 
 /*!
  * \brief Find a node by name or ID in a list of nodes
  *
  * \param[in] nodes      List of nodes (as pe_node_t*)
  * \param[in] id         If not NULL, ID of node to find
  * \param[in] node_name  If not NULL, name of node to find
  *
  * \return Node from \p nodes that matches \p id if any,
  *         otherwise node from \p nodes that matches \p uname if any,
  *         otherwise NULL
  */
 pe_node_t *
 pe_find_node_any(const GList *nodes, const char *id, const char *uname)
 {
     pe_node_t *match = NULL;
 
     if (id != NULL) {
         match = pe_find_node_id(nodes, id);
     }
     if ((match == NULL) && (uname != NULL)) {
         match = pe_find_node(nodes, uname);
     }
     return match;
 }
 
 /*!
  * \brief Find a node by ID in a list of nodes
  *
  * \param[in] nodes  List of nodes (as pe_node_t*)
  * \param[in] id     ID of node to find
  *
  * \return Node from \p nodes that matches \p id if any, otherwise NULL
  */
 pe_node_t *
 pe_find_node_id(const GList *nodes, const char *id)
 {
     for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
         pe_node_t *node = (pe_node_t *) iter->data;
 
         /* @TODO Whether node IDs should be considered case-sensitive should
          * probably depend on the node type, so functionizing the comparison
          * would be worthwhile
          */
         if (pcmk__str_eq(node->details->id, id, pcmk__str_casei)) {
             return node;
         }
     }
     return NULL;
 }
 
 /*!
  * \brief Find a node by name in a list of nodes
  *
  * \param[in] nodes      List of nodes (as pe_node_t*)
  * \param[in] node_name  Name of node to find
  *
  * \return Node from \p nodes that matches \p node_name if any, otherwise NULL
  */
 pe_node_t *
 pe_find_node(const GList *nodes, const char *node_name)
 {
     for (const GList *iter = nodes; iter != NULL; iter = iter->next) {
         pe_node_t *node = (pe_node_t *) iter->data;
 
         if (pcmk__str_eq(node->details->uname, node_name, pcmk__str_casei)) {
             return node;
         }
     }
     return NULL;
 }
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index d0fefad910..2f6f07abf6 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -1,4923 +1,4923 @@
 /*
  * Copyright 2004-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <string.h>
 #include <glib.h>
 #include <time.h>
 
 #include <crm/crm.h>
 #include <crm/services.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/xml_internal.h>
 
 #include <crm/common/util.h>
 #include <crm/pengine/rules.h>
 #include <crm/pengine/internal.h>
 #include <pe_status_private.h>
 
 CRM_TRACE_INIT_DATA(pe_status);
 
 // A (parsed) resource action history entry
 struct action_history {
     pe_resource_t *rsc;       // Resource that history is for
     pe_node_t *node;          // Node that history is for
     xmlNode *xml;             // History entry XML
 
     // Parsed from entry XML
     const char *id;           // XML ID of history entry
     const char *key;          // Operation key of action
     const char *task;         // Action name
     const char *exit_reason;  // Exit reason given for result
     guint interval_ms;        // Action interval
     int call_id;              // Call ID of action
     int expected_exit_status; // Expected exit status of action
     int exit_status;          // Actual exit status of action
     int execution_status;     // Execution status of action
 };
 
 /* This uses pcmk__set_flags_as()/pcmk__clear_flags_as() directly rather than
  * use pe__set_working_set_flags()/pe__clear_working_set_flags() so that the
  * flag is stringified more readably in log messages.
  */
 #define set_config_flag(data_set, option, flag) do {                        \
         const char *scf_value = pe_pref((data_set)->config_hash, (option)); \
         if (scf_value != NULL) {                                            \
             if (crm_is_true(scf_value)) {                                   \
                 (data_set)->flags = pcmk__set_flags_as(__func__, __LINE__,  \
                                     LOG_TRACE, "Working set",               \
                                     crm_system_name, (data_set)->flags,     \
                                     (flag), #flag);                         \
             } else {                                                        \
                 (data_set)->flags = pcmk__clear_flags_as(__func__, __LINE__,\
                                     LOG_TRACE, "Working set",               \
                                     crm_system_name, (data_set)->flags,     \
                                     (flag), #flag);                         \
             }                                                               \
         }                                                                   \
     } while(0)
 
 static void unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
                           xmlNode **last_failure,
                           enum action_fail_response *failed);
 static void determine_remote_online_status(pe_working_set_t *data_set,
                                            pe_node_t *this_node);
 static void add_node_attrs(const xmlNode *xml_obj, pe_node_t *node,
                            bool overwrite, pe_working_set_t *data_set);
 static void determine_online_status(const xmlNode *node_state,
                                     pe_node_t *this_node,
                                     pe_working_set_t *data_set);
 
 static void unpack_node_lrm(pe_node_t *node, const xmlNode *xml,
                             pe_working_set_t *data_set);
 
 
 // Bitmask for warnings we only want to print once
 uint32_t pe_wo = 0;
 
 static gboolean
 is_dangling_guest_node(pe_node_t *node)
 {
     /* we are looking for a remote-node that was supposed to be mapped to a
      * container resource, but all traces of that container have disappeared 
      * from both the config and the status section. */
     if (pe__is_guest_or_remote_node(node) &&
         node->details->remote_rsc &&
         node->details->remote_rsc->container == NULL &&
         pcmk_is_set(node->details->remote_rsc->flags,
                     pe_rsc_orphan_container_filler)) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 /*!
  * \brief Schedule a fence action for a node
  *
  * \param[in,out] data_set  Current working set of cluster
  * \param[in,out] node      Node to fence
  * \param[in]     reason    Text description of why fencing is needed
  * \param[in]     priority_delay  Whether to consider `priority-fencing-delay`
  */
 void
 pe_fence_node(pe_working_set_t * data_set, pe_node_t * node,
               const char *reason, bool priority_delay)
 {
     CRM_CHECK(node, return);
 
     /* A guest node is fenced by marking its container as failed */
     if (pe__is_guest_node(node)) {
         pe_resource_t *rsc = node->details->remote_rsc->container;
 
         if (!pcmk_is_set(rsc->flags, pe_rsc_failed)) {
             if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
                 crm_notice("Not fencing guest node %s "
                            "(otherwise would because %s): "
                            "its guest resource %s is unmanaged",
                            pe__node_name(node), reason, rsc->id);
             } else {
                 crm_warn("Guest node %s will be fenced "
                          "(by recovering its guest resource %s): %s",
                          pe__node_name(node), rsc->id, reason);
 
                 /* We don't mark the node as unclean because that would prevent the
                  * node from running resources. We want to allow it to run resources
                  * in this transition if the recovery succeeds.
                  */
                 node->details->remote_requires_reset = TRUE;
                 pe__set_resource_flags(rsc, pe_rsc_failed|pe_rsc_stop);
             }
         }
 
     } else if (is_dangling_guest_node(node)) {
         crm_info("Cleaning up dangling connection for guest node %s: "
                  "fencing was already done because %s, "
                  "and guest resource no longer exists",
                  pe__node_name(node), reason);
         pe__set_resource_flags(node->details->remote_rsc,
                                pe_rsc_failed|pe_rsc_stop);
 
     } else if (pe__is_remote_node(node)) {
         pe_resource_t *rsc = node->details->remote_rsc;
 
         if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pe_rsc_managed)) {
             crm_notice("Not fencing remote node %s "
                        "(otherwise would because %s): connection is unmanaged",
                        pe__node_name(node), reason);
         } else if(node->details->remote_requires_reset == FALSE) {
             node->details->remote_requires_reset = TRUE;
             crm_warn("Remote node %s %s: %s",
                      pe__node_name(node),
                      pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
                      reason);
         }
         node->details->unclean = TRUE;
         // No need to apply `priority-fencing-delay` for remote nodes
         pe_fence_op(node, NULL, TRUE, reason, FALSE, data_set);
 
     } else if (node->details->unclean) {
         crm_trace("Cluster node %s %s because %s",
                   pe__node_name(node),
                   pe_can_fence(data_set, node)? "would also be fenced" : "also is unclean",
                   reason);
 
     } else {
         crm_warn("Cluster node %s %s: %s",
                  pe__node_name(node),
                  pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
                  reason);
         node->details->unclean = TRUE;
         pe_fence_op(node, NULL, TRUE, reason, priority_delay, data_set);
     }
 }
 
 // @TODO xpaths can't handle templates, rules, or id-refs
 
 // nvpair with provides or requires set to unfencing
 #define XPATH_UNFENCING_NVPAIR XML_CIB_TAG_NVPAIR                \
     "[(@" XML_NVPAIR_ATTR_NAME "='" PCMK_STONITH_PROVIDES "'"    \
     "or @" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_REQUIRES "') " \
     "and @" XML_NVPAIR_ATTR_VALUE "='" PCMK__VALUE_UNFENCING "']"
 
 // unfencing in rsc_defaults or any resource
 #define XPATH_ENABLE_UNFENCING \
     "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RESOURCES   \
     "//" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR                                               \
     "|/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RSCCONFIG  \
     "/" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR
 
 static void
 set_if_xpath(uint64_t flag, const char *xpath, pe_working_set_t *data_set)
 {
     xmlXPathObjectPtr result = NULL;
 
     if (!pcmk_is_set(data_set->flags, flag)) {
         result = xpath_search(data_set->input, xpath);
         if (result && (numXpathResults(result) > 0)) {
             pe__set_working_set_flags(data_set, flag);
         }
         freeXpathObject(result);
     }
 }
 
 gboolean
 unpack_config(xmlNode * config, pe_working_set_t * data_set)
 {
     const char *value = NULL;
     GHashTable *config_hash = pcmk__strkey_table(free, free);
 
     pe_rule_eval_data_t rule_data = {
         .node_hash = NULL,
         .role = pcmk_role_unknown,
         .now = data_set->now,
         .match_data = NULL,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     data_set->config_hash = config_hash;
 
     pe__unpack_dataset_nvpairs(config, XML_CIB_TAG_PROPSET, &rule_data, config_hash,
                                CIB_OPTIONS_FIRST, FALSE, data_set);
 
     verify_pe_options(data_set->config_hash);
 
     set_config_flag(data_set, "enable-startup-probes", pe_flag_startup_probes);
     if (!pcmk_is_set(data_set->flags, pe_flag_startup_probes)) {
         crm_info("Startup probes: disabled (dangerous)");
     }
 
     value = pe_pref(data_set->config_hash, XML_ATTR_HAVE_WATCHDOG);
     if (value && crm_is_true(value)) {
         crm_info("Watchdog-based self-fencing will be performed via SBD if "
                  "fencing is required and stonith-watchdog-timeout is nonzero");
         pe__set_working_set_flags(data_set, pe_flag_have_stonith_resource);
     }
 
     /* Set certain flags via xpath here, so they can be used before the relevant
      * configuration sections are unpacked.
      */
     set_if_xpath(pe_flag_enable_unfencing, XPATH_ENABLE_UNFENCING, data_set);
 
     value = pe_pref(data_set->config_hash, "stonith-timeout");
     data_set->stonith_timeout = (int) crm_parse_interval_spec(value);
     crm_debug("STONITH timeout: %d", data_set->stonith_timeout);
 
     set_config_flag(data_set, "stonith-enabled", pe_flag_stonith_enabled);
     crm_debug("STONITH of failed nodes is %s",
               pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)? "enabled" : "disabled");
 
     data_set->stonith_action = pe_pref(data_set->config_hash, "stonith-action");
     if (!strcmp(data_set->stonith_action, "poweroff")) {
         pe_warn_once(pe_wo_poweroff,
                      "Support for stonith-action of 'poweroff' is deprecated "
                      "and will be removed in a future release (use 'off' instead)");
         data_set->stonith_action = PCMK_ACTION_OFF;
     }
     crm_trace("STONITH will %s nodes", data_set->stonith_action);
 
     set_config_flag(data_set, "concurrent-fencing", pe_flag_concurrent_fencing);
     crm_debug("Concurrent fencing is %s",
               pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)? "enabled" : "disabled");
 
     value = pe_pref(data_set->config_hash,
                     XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY);
     if (value) {
         data_set->priority_fencing_delay = crm_parse_interval_spec(value) / 1000;
         crm_trace("Priority fencing delay is %ds", data_set->priority_fencing_delay);
     }
 
     set_config_flag(data_set, "stop-all-resources", pe_flag_stop_everything);
     crm_debug("Stop all active resources: %s",
               pcmk__btoa(pcmk_is_set(data_set->flags, pe_flag_stop_everything)));
 
     set_config_flag(data_set, "symmetric-cluster", pe_flag_symmetric_cluster);
     if (pcmk_is_set(data_set->flags, pe_flag_symmetric_cluster)) {
         crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
     }
 
     value = pe_pref(data_set->config_hash, "no-quorum-policy");
 
     if (pcmk__str_eq(value, "ignore", pcmk__str_casei)) {
         data_set->no_quorum_policy = pcmk_no_quorum_ignore;
 
     } else if (pcmk__str_eq(value, "freeze", pcmk__str_casei)) {
         data_set->no_quorum_policy = pcmk_no_quorum_freeze;
 
     } else if (pcmk__str_eq(value, "demote", pcmk__str_casei)) {
         data_set->no_quorum_policy = pcmk_no_quorum_demote;
 
     } else if (pcmk__str_eq(value, "suicide", pcmk__str_casei)) {
         if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
             int do_panic = 0;
 
             crm_element_value_int(data_set->input, XML_ATTR_QUORUM_PANIC,
                                   &do_panic);
-            if (do_panic || pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
+            if (do_panic || pcmk_is_set(data_set->flags, pcmk_sched_quorate)) {
                 data_set->no_quorum_policy = pcmk_no_quorum_fence;
             } else {
                 crm_notice("Resetting no-quorum-policy to 'stop': cluster has never had quorum");
                 data_set->no_quorum_policy = pcmk_no_quorum_stop;
             }
         } else {
             pcmk__config_err("Resetting no-quorum-policy to 'stop' because "
                              "fencing is disabled");
             data_set->no_quorum_policy = pcmk_no_quorum_stop;
         }
 
     } else {
         data_set->no_quorum_policy = pcmk_no_quorum_stop;
     }
 
     switch (data_set->no_quorum_policy) {
         case pcmk_no_quorum_freeze:
             crm_debug("On loss of quorum: Freeze resources");
             break;
         case pcmk_no_quorum_stop:
             crm_debug("On loss of quorum: Stop ALL resources");
             break;
         case pcmk_no_quorum_demote:
             crm_debug("On loss of quorum: "
                       "Demote promotable resources and stop other resources");
             break;
         case pcmk_no_quorum_fence:
             crm_notice("On loss of quorum: Fence all remaining nodes");
             break;
         case pcmk_no_quorum_ignore:
             crm_notice("On loss of quorum: Ignore");
             break;
     }
 
     set_config_flag(data_set, "stop-orphan-resources", pe_flag_stop_rsc_orphans);
     crm_trace("Orphan resources are %s",
               pcmk_is_set(data_set->flags, pe_flag_stop_rsc_orphans)? "stopped" : "ignored");
 
     set_config_flag(data_set, "stop-orphan-actions", pe_flag_stop_action_orphans);
     crm_trace("Orphan resource actions are %s",
               pcmk_is_set(data_set->flags, pe_flag_stop_action_orphans)? "stopped" : "ignored");
 
     value = pe_pref(data_set->config_hash, "remove-after-stop");
     if (value != NULL) {
         if (crm_is_true(value)) {
             pe__set_working_set_flags(data_set, pe_flag_remove_after_stop);
 #ifndef PCMK__COMPAT_2_0
             pe_warn_once(pe_wo_remove_after,
                          "Support for the remove-after-stop cluster property is"
                          " deprecated and will be removed in a future release");
 #endif
         } else {
             pe__clear_working_set_flags(data_set, pe_flag_remove_after_stop);
         }
     }
 
     set_config_flag(data_set, "maintenance-mode", pe_flag_maintenance_mode);
     crm_trace("Maintenance mode: %s",
               pcmk__btoa(pcmk_is_set(data_set->flags, pe_flag_maintenance_mode)));
 
     set_config_flag(data_set, "start-failure-is-fatal", pe_flag_start_failure_fatal);
     crm_trace("Start failures are %s",
               pcmk_is_set(data_set->flags, pe_flag_start_failure_fatal)? "always fatal" : "handled by failcount");
 
     if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
         set_config_flag(data_set, "startup-fencing", pe_flag_startup_fencing);
     }
     if (pcmk_is_set(data_set->flags, pe_flag_startup_fencing)) {
         crm_trace("Unseen nodes will be fenced");
     } else {
         pe_warn_once(pe_wo_blind, "Blind faith: not fencing unseen nodes");
     }
 
     pe__unpack_node_health_scores(data_set);
 
     data_set->placement_strategy = pe_pref(data_set->config_hash, "placement-strategy");
     crm_trace("Placement strategy: %s", data_set->placement_strategy);
 
     set_config_flag(data_set, "shutdown-lock", pe_flag_shutdown_lock);
     crm_trace("Resources will%s be locked to cleanly shut down nodes",
               (pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)? "" : " not"));
     if (pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)) {
         value = pe_pref(data_set->config_hash,
                         XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT);
         data_set->shutdown_lock = crm_parse_interval_spec(value) / 1000;
         crm_trace("Shutdown locks expire after %us", data_set->shutdown_lock);
     }
 
     value = pe_pref(data_set->config_hash,
                     XML_CONFIG_ATTR_NODE_PENDING_TIMEOUT);
     data_set->node_pending_timeout = crm_parse_interval_spec(value) / 1000;
     crm_trace("Node pending timeout is %us", data_set->node_pending_timeout);
 
     return TRUE;
 }
 
 pe_node_t *
 pe_create_node(const char *id, const char *uname, const char *type,
                const char *score, pe_working_set_t * data_set)
 {
     pe_node_t *new_node = NULL;
 
     if (pe_find_node(data_set->nodes, uname) != NULL) {
         pcmk__config_warn("More than one node entry has name '%s'", uname);
     }
 
     new_node = calloc(1, sizeof(pe_node_t));
     if (new_node == NULL) {
         return NULL;
     }
 
     new_node->weight = char2score(score);
     new_node->details = calloc(1, sizeof(struct pe_node_shared_s));
 
     if (new_node->details == NULL) {
         free(new_node);
         return NULL;
     }
 
     crm_trace("Creating node for entry %s/%s", uname, id);
     new_node->details->id = id;
     new_node->details->uname = uname;
     new_node->details->online = FALSE;
     new_node->details->shutdown = FALSE;
     new_node->details->rsc_discovery_enabled = TRUE;
     new_node->details->running_rsc = NULL;
     new_node->details->data_set = data_set;
 
     if (pcmk__str_eq(type, "member", pcmk__str_null_matches | pcmk__str_casei)) {
         new_node->details->type = pcmk_node_variant_cluster;
 
     } else if (pcmk__str_eq(type, "remote", pcmk__str_casei)) {
         new_node->details->type = pcmk_node_variant_remote;
         pe__set_working_set_flags(data_set, pe_flag_have_remote_nodes);
 
     } else {
         /* @COMPAT 'ping' is the default for backward compatibility, but it
          * should be changed to 'member' at a compatibility break
          */
         if (!pcmk__str_eq(type, "ping", pcmk__str_casei)) {
             pcmk__config_warn("Node %s has unrecognized type '%s', "
                               "assuming 'ping'", pcmk__s(uname, "without name"),
                               type);
         }
         pe_warn_once(pe_wo_ping_node,
                      "Support for nodes of type 'ping' (such as %s) is "
                      "deprecated and will be removed in a future release",
                      pcmk__s(uname, "unnamed node"));
         new_node->details->type = node_ping;
     }
 
     new_node->details->attrs = pcmk__strkey_table(free, free);
 
     if (pe__is_guest_or_remote_node(new_node)) {
         g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
                             strdup("remote"));
     } else {
         g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
                             strdup("cluster"));
     }
 
     new_node->details->utilization = pcmk__strkey_table(free, free);
     new_node->details->digest_cache = pcmk__strkey_table(free,
                                                           pe__free_digests);
 
     data_set->nodes = g_list_insert_sorted(data_set->nodes, new_node,
                                            pe__cmp_node_name);
     return new_node;
 }
 
 static const char *
 expand_remote_rsc_meta(xmlNode *xml_obj, xmlNode *parent, pe_working_set_t *data)
 {
     xmlNode *attr_set = NULL;
     xmlNode *attr = NULL;
 
     const char *container_id = ID(xml_obj);
     const char *remote_name = NULL;
     const char *remote_server = NULL;
     const char *remote_port = NULL;
     const char *connect_timeout = "60s";
     const char *remote_allow_migrate=NULL;
     const char *is_managed = NULL;
 
     for (attr_set = pcmk__xe_first_child(xml_obj); attr_set != NULL;
          attr_set = pcmk__xe_next(attr_set)) {
 
         if (!pcmk__str_eq((const char *)attr_set->name, XML_TAG_META_SETS,
                           pcmk__str_casei)) {
             continue;
         }
 
         for (attr = pcmk__xe_first_child(attr_set); attr != NULL;
              attr = pcmk__xe_next(attr)) {
             const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
             const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
 
             if (pcmk__str_eq(name, XML_RSC_ATTR_REMOTE_NODE, pcmk__str_casei)) {
                 remote_name = value;
             } else if (pcmk__str_eq(name, "remote-addr", pcmk__str_casei)) {
                 remote_server = value;
             } else if (pcmk__str_eq(name, "remote-port", pcmk__str_casei)) {
                 remote_port = value;
             } else if (pcmk__str_eq(name, "remote-connect-timeout", pcmk__str_casei)) {
                 connect_timeout = value;
             } else if (pcmk__str_eq(name, "remote-allow-migrate", pcmk__str_casei)) {
                 remote_allow_migrate=value;
             } else if (pcmk__str_eq(name, XML_RSC_ATTR_MANAGED, pcmk__str_casei)) {
                 is_managed = value;
             }
         }
     }
 
     if (remote_name == NULL) {
         return NULL;
     }
 
     if (pe_find_resource(data->resources, remote_name) != NULL) {
         return NULL;
     }
 
     pe_create_remote_xml(parent, remote_name, container_id,
                          remote_allow_migrate, is_managed,
                          connect_timeout, remote_server, remote_port);
     return remote_name;
 }
 
 static void
 handle_startup_fencing(pe_working_set_t *data_set, pe_node_t *new_node)
 {
     if ((new_node->details->type == pcmk_node_variant_remote)
         && (new_node->details->remote_rsc == NULL)) {
         /* Ignore fencing for remote nodes that don't have a connection resource
          * associated with them. This happens when remote node entries get left
          * in the nodes section after the connection resource is removed.
          */
         return;
     }
 
     if (pcmk_is_set(data_set->flags, pe_flag_startup_fencing)) {
         // All nodes are unclean until we've seen their status entry
         new_node->details->unclean = TRUE;
 
     } else {
         // Blind faith ...
         new_node->details->unclean = FALSE;
     }
 
     /* We need to be able to determine if a node's status section
      * exists or not separate from whether the node is unclean. */
     new_node->details->unseen = TRUE;
 }
 
 gboolean
 unpack_nodes(xmlNode * xml_nodes, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
     pe_node_t *new_node = NULL;
     const char *id = NULL;
     const char *uname = NULL;
     const char *type = NULL;
     const char *score = NULL;
 
     for (xml_obj = pcmk__xe_first_child(xml_nodes); xml_obj != NULL;
          xml_obj = pcmk__xe_next(xml_obj)) {
 
         if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, pcmk__str_none)) {
             new_node = NULL;
 
             id = crm_element_value(xml_obj, XML_ATTR_ID);
             uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
             type = crm_element_value(xml_obj, XML_ATTR_TYPE);
             score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
             crm_trace("Processing node %s/%s", uname, id);
 
             if (id == NULL) {
                 pcmk__config_err("Ignoring <" XML_CIB_TAG_NODE
                                  "> entry in configuration without id");
                 continue;
             }
             new_node = pe_create_node(id, uname, type, score, data_set);
 
             if (new_node == NULL) {
                 return FALSE;
             }
 
             handle_startup_fencing(data_set, new_node);
 
             add_node_attrs(xml_obj, new_node, FALSE, data_set);
 
             crm_trace("Done with node %s", crm_element_value(xml_obj, XML_ATTR_UNAME));
         }
     }
 
     if (data_set->localhost && pe_find_node(data_set->nodes, data_set->localhost) == NULL) {
         crm_info("Creating a fake local node");
         pe_create_node(data_set->localhost, data_set->localhost, NULL, 0,
                        data_set);
     }
 
     return TRUE;
 }
 
 static void
 setup_container(pe_resource_t * rsc, pe_working_set_t * data_set)
 {
     const char *container_id = NULL;
 
     if (rsc->children) {
         g_list_foreach(rsc->children, (GFunc) setup_container, data_set);
         return;
     }
 
     container_id = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_CONTAINER);
     if (container_id && !pcmk__str_eq(container_id, rsc->id, pcmk__str_casei)) {
         pe_resource_t *container = pe_find_resource(data_set->resources, container_id);
 
         if (container) {
             rsc->container = container;
             pe__set_resource_flags(container, pe_rsc_is_container);
             container->fillers = g_list_append(container->fillers, rsc);
             pe_rsc_trace(rsc, "Resource %s's container is %s", rsc->id, container_id);
         } else {
             pe_err("Resource %s: Unknown resource container (%s)", rsc->id, container_id);
         }
     }
 }
 
 gboolean
 unpack_remote_nodes(xmlNode * xml_resources, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
 
     /* Create remote nodes and guest nodes from the resource configuration
      * before unpacking resources.
      */
     for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
          xml_obj = pcmk__xe_next(xml_obj)) {
 
         const char *new_node_id = NULL;
 
         /* Check for remote nodes, which are defined by ocf:pacemaker:remote
          * primitives.
          */
         if (xml_contains_remote_node(xml_obj)) {
             new_node_id = ID(xml_obj);
             /* The "pe_find_node" check is here to make sure we don't iterate over
              * an expanded node that has already been added to the node list. */
             if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
                 crm_trace("Found remote node %s defined by resource %s",
                           new_node_id, ID(xml_obj));
                 pe_create_node(new_node_id, new_node_id, "remote", NULL,
                                data_set);
             }
             continue;
         }
 
         /* Check for guest nodes, which are defined by special meta-attributes
          * of a primitive of any type (for example, VirtualDomain or Xen).
          */
         if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_RESOURCE, pcmk__str_none)) {
             /* This will add an ocf:pacemaker:remote primitive to the
              * configuration for the guest node's connection, to be unpacked
              * later.
              */
             new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources, data_set);
             if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
                 crm_trace("Found guest node %s in resource %s",
                           new_node_id, ID(xml_obj));
                 pe_create_node(new_node_id, new_node_id, "remote", NULL,
                                data_set);
             }
             continue;
         }
 
         /* Check for guest nodes inside a group. Clones are currently not
          * supported as guest nodes.
          */
         if (pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, pcmk__str_none)) {
             xmlNode *xml_obj2 = NULL;
             for (xml_obj2 = pcmk__xe_first_child(xml_obj); xml_obj2 != NULL;
                  xml_obj2 = pcmk__xe_next(xml_obj2)) {
 
                 new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources, data_set);
 
                 if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
                     crm_trace("Found guest node %s in resource %s inside group %s",
                               new_node_id, ID(xml_obj2), ID(xml_obj));
                     pe_create_node(new_node_id, new_node_id, "remote", NULL,
                                    data_set);
                 }
             }
         }
     }
     return TRUE;
 }
 
 /* Call this after all the nodes and resources have been
  * unpacked, but before the status section is read.
  *
  * A remote node's online status is reflected by the state
  * of the remote node's connection resource. We need to link
  * the remote node to this connection resource so we can have
  * easy access to the connection resource during the scheduler calculations.
  */
 static void
 link_rsc2remotenode(pe_working_set_t *data_set, pe_resource_t *new_rsc)
 {
     pe_node_t *remote_node = NULL;
 
     if (new_rsc->is_remote_node == FALSE) {
         return;
     }
 
     if (pcmk_is_set(data_set->flags, pe_flag_quick_location)) {
         /* remote_nodes and remote_resources are not linked in quick location calculations */
         return;
     }
 
     remote_node = pe_find_node(data_set->nodes, new_rsc->id);
     CRM_CHECK(remote_node != NULL, return);
 
     pe_rsc_trace(new_rsc, "Linking remote connection resource %s to %s",
                  new_rsc->id, pe__node_name(remote_node));
     remote_node->details->remote_rsc = new_rsc;
 
     if (new_rsc->container == NULL) {
         /* Handle start-up fencing for remote nodes (as opposed to guest nodes)
          * the same as is done for cluster nodes.
          */
         handle_startup_fencing(data_set, remote_node);
 
     } else {
         /* pe_create_node() marks the new node as "remote" or "cluster"; now
          * that we know the node is a guest node, update it correctly.
          */
         g_hash_table_replace(remote_node->details->attrs, strdup(CRM_ATTR_KIND),
                              strdup("container"));
     }
 }
 
 static void
 destroy_tag(gpointer data)
 {
     pe_tag_t *tag = data;
 
     if (tag) {
         free(tag->id);
         g_list_free_full(tag->refs, free);
         free(tag);
     }
 }
 
 /*!
  * \internal
  * \brief Parse configuration XML for resource information
  *
  * \param[in]     xml_resources  Top of resource configuration XML
  * \param[in,out] data_set       Where to put resource information
  *
  * \return TRUE
  *
  * \note unpack_remote_nodes() MUST be called before this, so that the nodes can
  *       be used when pe__unpack_resource() calls resource_location()
  */
 gboolean
 unpack_resources(const xmlNode *xml_resources, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
     GList *gIter = NULL;
 
     data_set->template_rsc_sets = pcmk__strkey_table(free, destroy_tag);
 
     for (xml_obj = pcmk__xe_first_child(xml_resources); xml_obj != NULL;
          xml_obj = pcmk__xe_next(xml_obj)) {
 
         pe_resource_t *new_rsc = NULL;
         const char *id = ID(xml_obj);
 
         if (pcmk__str_empty(id)) {
             pcmk__config_err("Ignoring <%s> resource without ID",
                              xml_obj->name);
             continue;
         }
 
         if (pcmk__str_eq((const char *) xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE,
                          pcmk__str_none)) {
             if (g_hash_table_lookup_extended(data_set->template_rsc_sets, id,
                                              NULL, NULL) == FALSE) {
                 /* Record the template's ID for the knowledge of its existence anyway. */
                 g_hash_table_insert(data_set->template_rsc_sets, strdup(id), NULL);
             }
             continue;
         }
 
         crm_trace("Unpacking <%s " XML_ATTR_ID "='%s'>",
                   xml_obj->name, id);
         if (pe__unpack_resource(xml_obj, &new_rsc, NULL,
                                 data_set) == pcmk_rc_ok) {
             data_set->resources = g_list_append(data_set->resources, new_rsc);
             pe_rsc_trace(new_rsc, "Added resource %s", new_rsc->id);
 
         } else {
             pcmk__config_err("Ignoring <%s> resource '%s' "
                              "because configuration is invalid",
                              xml_obj->name, id);
         }
     }
 
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
         setup_container(rsc, data_set);
         link_rsc2remotenode(data_set, rsc);
     }
 
     data_set->resources = g_list_sort(data_set->resources,
                                       pe__cmp_rsc_priority);
     if (pcmk_is_set(data_set->flags, pe_flag_quick_location)) {
         /* Ignore */
 
     } else if (pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)
                && !pcmk_is_set(data_set->flags, pe_flag_have_stonith_resource)) {
 
         pcmk__config_err("Resource start-up disabled since no STONITH resources have been defined");
         pcmk__config_err("Either configure some or disable STONITH with the stonith-enabled option");
         pcmk__config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
     }
 
     return TRUE;
 }
 
 gboolean
 unpack_tags(xmlNode * xml_tags, pe_working_set_t * data_set)
 {
     xmlNode *xml_tag = NULL;
 
     data_set->tags = pcmk__strkey_table(free, destroy_tag);
 
     for (xml_tag = pcmk__xe_first_child(xml_tags); xml_tag != NULL;
          xml_tag = pcmk__xe_next(xml_tag)) {
 
         xmlNode *xml_obj_ref = NULL;
         const char *tag_id = ID(xml_tag);
 
         if (!pcmk__str_eq((const char *)xml_tag->name, XML_CIB_TAG_TAG, pcmk__str_none)) {
             continue;
         }
 
         if (tag_id == NULL) {
             pcmk__config_err("Ignoring <%s> without " XML_ATTR_ID,
                              (const char *) xml_tag->name);
             continue;
         }
 
         for (xml_obj_ref = pcmk__xe_first_child(xml_tag); xml_obj_ref != NULL;
              xml_obj_ref = pcmk__xe_next(xml_obj_ref)) {
 
             const char *obj_ref = ID(xml_obj_ref);
 
             if (!pcmk__str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, pcmk__str_none)) {
                 continue;
             }
 
             if (obj_ref == NULL) {
                 pcmk__config_err("Ignoring <%s> for tag '%s' without " XML_ATTR_ID,
                                  xml_obj_ref->name, tag_id);
                 continue;
             }
 
             if (add_tag_ref(data_set->tags, tag_id, obj_ref) == FALSE) {
                 return FALSE;
             }
         }
     }
 
     return TRUE;
 }
 
 /* The ticket state section:
  * "/cib/status/tickets/ticket_state" */
 static gboolean
 unpack_ticket_state(xmlNode * xml_ticket, pe_working_set_t * data_set)
 {
     const char *ticket_id = NULL;
     const char *granted = NULL;
     const char *last_granted = NULL;
     const char *standby = NULL;
     xmlAttrPtr xIter = NULL;
 
     pe_ticket_t *ticket = NULL;
 
     ticket_id = ID(xml_ticket);
     if (pcmk__str_empty(ticket_id)) {
         return FALSE;
     }
 
     crm_trace("Processing ticket state for %s", ticket_id);
 
     ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
     if (ticket == NULL) {
         ticket = ticket_new(ticket_id, data_set);
         if (ticket == NULL) {
             return FALSE;
         }
     }
 
     for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
         const char *prop_name = (const char *)xIter->name;
         const char *prop_value = pcmk__xml_attr_value(xIter);
 
         if (pcmk__str_eq(prop_name, XML_ATTR_ID, pcmk__str_none)) {
             continue;
         }
         g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
     }
 
     granted = g_hash_table_lookup(ticket->state, "granted");
     if (granted && crm_is_true(granted)) {
         ticket->granted = TRUE;
         crm_info("We have ticket '%s'", ticket->id);
     } else {
         ticket->granted = FALSE;
         crm_info("We do not have ticket '%s'", ticket->id);
     }
 
     last_granted = g_hash_table_lookup(ticket->state, "last-granted");
     if (last_granted) {
         long long last_granted_ll;
 
         pcmk__scan_ll(last_granted, &last_granted_ll, 0LL);
         ticket->last_granted = (time_t) last_granted_ll;
     }
 
     standby = g_hash_table_lookup(ticket->state, "standby");
     if (standby && crm_is_true(standby)) {
         ticket->standby = TRUE;
         if (ticket->granted) {
             crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
         }
     } else {
         ticket->standby = FALSE;
     }
 
     crm_trace("Done with ticket state for %s", ticket_id);
 
     return TRUE;
 }
 
 static gboolean
 unpack_tickets_state(xmlNode * xml_tickets, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
 
     for (xml_obj = pcmk__xe_first_child(xml_tickets); xml_obj != NULL;
          xml_obj = pcmk__xe_next(xml_obj)) {
 
         if (!pcmk__str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, pcmk__str_none)) {
             continue;
         }
         unpack_ticket_state(xml_obj, data_set);
     }
 
     return TRUE;
 }
 
 static void
 unpack_handle_remote_attrs(pe_node_t *this_node, const xmlNode *state,
                            pe_working_set_t *data_set)
 {
     const char *resource_discovery_enabled = NULL;
     const xmlNode *attrs = NULL;
     pe_resource_t *rsc = NULL;
 
     if (!pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
         return;
     }
 
     if ((this_node == NULL) || !pe__is_guest_or_remote_node(this_node)) {
         return;
     }
     crm_trace("Processing Pacemaker Remote node %s", pe__node_name(this_node));
 
     pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_MAINTENANCE),
                        &(this_node->details->remote_maintenance), 0);
 
     rsc = this_node->details->remote_rsc;
     if (this_node->details->remote_requires_reset == FALSE) {
         this_node->details->unclean = FALSE;
         this_node->details->unseen = FALSE;
     }
     attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
     add_node_attrs(attrs, this_node, TRUE, data_set);
 
     if (pe__shutdown_requested(this_node)) {
         crm_info("%s is shutting down", pe__node_name(this_node));
         this_node->details->shutdown = TRUE;
     }
  
     if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
         crm_info("%s is in standby mode", pe__node_name(this_node));
         this_node->details->standby = TRUE;
     }
 
     if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance")) ||
         ((rsc != NULL) && !pcmk_is_set(rsc->flags, pe_rsc_managed))) {
         crm_info("%s is in maintenance mode", pe__node_name(this_node));
         this_node->details->maintenance = TRUE;
     }
 
     resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
     if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
         if (pe__is_remote_node(this_node)
             && !pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
             crm_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY
                      " attribute on Pacemaker Remote node %s"
                      " because fencing is disabled",
                      pe__node_name(this_node));
         } else {
             /* This is either a remote node with fencing enabled, or a guest
              * node. We don't care whether fencing is enabled when fencing guest
              * nodes, because they are "fenced" by recovering their containing
              * resource.
              */
             crm_info("%s has resource discovery disabled",
                      pe__node_name(this_node));
             this_node->details->rsc_discovery_enabled = FALSE;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Unpack a cluster node's transient attributes
  *
  * \param[in]     state     CIB node state XML
  * \param[in,out] node      Cluster node whose attributes are being unpacked
  * \param[in,out] data_set  Cluster working set
  */
 static void
 unpack_transient_attributes(const xmlNode *state, pe_node_t *node,
                             pe_working_set_t *data_set)
 {
     const char *discovery = NULL;
     const xmlNode *attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS,
                                          FALSE);
 
     add_node_attrs(attrs, node, TRUE, data_set);
 
     if (crm_is_true(pe_node_attribute_raw(node, "standby"))) {
         crm_info("%s is in standby mode", pe__node_name(node));
         node->details->standby = TRUE;
     }
 
     if (crm_is_true(pe_node_attribute_raw(node, "maintenance"))) {
         crm_info("%s is in maintenance mode", pe__node_name(node));
         node->details->maintenance = TRUE;
     }
 
     discovery = pe_node_attribute_raw(node, XML_NODE_ATTR_RSC_DISCOVERY);
     if ((discovery != NULL) && !crm_is_true(discovery)) {
         crm_warn("Ignoring " XML_NODE_ATTR_RSC_DISCOVERY
                  " attribute for %s because disabling resource discovery "
                  "is not allowed for cluster nodes", pe__node_name(node));
     }
 }
 
 /*!
  * \internal
  * \brief Unpack a node state entry (first pass)
  *
  * Unpack one node state entry from status. This unpacks information from the
  * node_state element itself and node attributes inside it, but not the
  * resource history inside it. Multiple passes through the status are needed to
  * fully unpack everything.
  *
  * \param[in]     state     CIB node state XML
  * \param[in,out] data_set  Cluster working set
  */
 static void
 unpack_node_state(const xmlNode *state, pe_working_set_t *data_set)
 {
     const char *id = NULL;
     const char *uname = NULL;
     pe_node_t *this_node = NULL;
 
     id = crm_element_value(state, XML_ATTR_ID);
     if (id == NULL) {
         crm_warn("Ignoring malformed " XML_CIB_TAG_STATE " entry without "
                  XML_ATTR_ID);
         return;
     }
 
     uname = crm_element_value(state, XML_ATTR_UNAME);
     if (uname == NULL) {
         /* If a joining peer makes the cluster acquire the quorum from corosync
          * meanwhile it has not joined CPG membership of pacemaker-controld yet,
          * it's possible that the created node_state entry doesn't have an uname
          * yet. We should recognize the node as `pending` and wait for it to
          * join CPG.
          */
         crm_trace("Handling " XML_CIB_TAG_STATE " entry with id=\"%s\" without "
                   XML_ATTR_UNAME, id);
     }
 
     this_node = pe_find_node_any(data_set->nodes, id, uname);
     if (this_node == NULL) {
         pcmk__config_warn("Ignoring recorded node state for id=\"%s\" (%s) "
                           "because it is no longer in the configuration",
                           id, pcmk__s(uname, "uname unknown"));
         return;
     }
 
     if (pe__is_guest_or_remote_node(this_node)) {
         /* We can't determine the online status of Pacemaker Remote nodes until
          * after all resource history has been unpacked. In this first pass, we
          * do need to mark whether the node has been fenced, as this plays a
          * role during unpacking cluster node resource state.
          */
         pcmk__scan_min_int(crm_element_value(state, XML_NODE_IS_FENCED),
                            &(this_node->details->remote_was_fenced), 0);
         return;
     }
 
     unpack_transient_attributes(state, this_node, data_set);
 
     /* Provisionally mark this cluster node as clean. We have at least seen it
      * in the current cluster's lifetime.
      */
     this_node->details->unclean = FALSE;
     this_node->details->unseen = FALSE;
 
     crm_trace("Determining online status of cluster node %s (id %s)",
               pe__node_name(this_node), id);
     determine_online_status(state, this_node, data_set);
 
-    if (!pcmk_is_set(data_set->flags, pe_flag_have_quorum)
+    if (!pcmk_is_set(data_set->flags, pcmk_sched_quorate)
         && this_node->details->online
         && (data_set->no_quorum_policy == pcmk_no_quorum_fence)) {
         /* Everything else should flow from this automatically
          * (at least until the scheduler becomes able to migrate off
          * healthy resources)
          */
         pe_fence_node(data_set, this_node, "cluster does not have quorum",
                       FALSE);
     }
 }
 
 /*!
  * \internal
  * \brief Unpack nodes' resource history as much as possible
  *
  * Unpack as many nodes' resource history as possible in one pass through the
  * status. We need to process Pacemaker Remote nodes' connections/containers
  * before unpacking their history; the connection/container history will be
  * in another node's history, so it might take multiple passes to unpack
  * everything.
  *
  * \param[in]     status    CIB XML status section
  * \param[in]     fence     If true, treat any not-yet-unpacked nodes as unseen
  * \param[in,out] data_set  Cluster working set
  *
  * \return Standard Pacemaker return code (specifically pcmk_rc_ok if done,
  *         or EAGAIN if more unpacking remains to be done)
  */
 static int
 unpack_node_history(const xmlNode *status, bool fence,
                     pe_working_set_t *data_set)
 {
     int rc = pcmk_rc_ok;
 
     // Loop through all node_state entries in CIB status
     for (const xmlNode *state = first_named_child(status, XML_CIB_TAG_STATE);
          state != NULL; state = crm_next_same_xml(state)) {
 
         const char *id = ID(state);
         const char *uname = crm_element_value(state, XML_ATTR_UNAME);
         pe_node_t *this_node = NULL;
 
         if ((id == NULL) || (uname == NULL)) {
             // Warning already logged in first pass through status section
             crm_trace("Not unpacking resource history from malformed "
                       XML_CIB_TAG_STATE " without id and/or uname");
             continue;
         }
 
         this_node = pe_find_node_any(data_set->nodes, id, uname);
         if (this_node == NULL) {
             // Warning already logged in first pass through status section
             crm_trace("Not unpacking resource history for node %s because "
                       "no longer in configuration", id);
             continue;
         }
 
         if (this_node->details->unpacked) {
             crm_trace("Not unpacking resource history for node %s because "
                       "already unpacked", id);
             continue;
         }
 
         if (fence) {
             // We're processing all remaining nodes
 
         } else if (pe__is_guest_node(this_node)) {
             /* We can unpack a guest node's history only after we've unpacked
              * other resource history to the point that we know that the node's
              * connection and containing resource are both up.
              */
             pe_resource_t *rsc = this_node->details->remote_rsc;
 
             if ((rsc == NULL) || (rsc->role != pcmk_role_started)
                 || (rsc->container->role != pcmk_role_started)) {
                 crm_trace("Not unpacking resource history for guest node %s "
                           "because container and connection are not known to "
                           "be up", id);
                 continue;
             }
 
         } else if (pe__is_remote_node(this_node)) {
             /* We can unpack a remote node's history only after we've unpacked
              * other resource history to the point that we know that the node's
              * connection is up, with the exception of when shutdown locks are
              * in use.
              */
             pe_resource_t *rsc = this_node->details->remote_rsc;
 
             if ((rsc == NULL)
                 || (!pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)
                     && (rsc->role != pcmk_role_started))) {
                 crm_trace("Not unpacking resource history for remote node %s "
                           "because connection is not known to be up", id);
                 continue;
             }
 
         /* If fencing and shutdown locks are disabled and we're not processing
          * unseen nodes, then we don't want to unpack offline nodes until online
          * nodes have been unpacked. This allows us to number active clone
          * instances first.
          */
         } else if (!pcmk_any_flags_set(data_set->flags, pe_flag_stonith_enabled
                                                         |pe_flag_shutdown_lock)
                    && !this_node->details->online) {
             crm_trace("Not unpacking resource history for offline "
                       "cluster node %s", id);
             continue;
         }
 
         if (pe__is_guest_or_remote_node(this_node)) {
             determine_remote_online_status(data_set, this_node);
             unpack_handle_remote_attrs(this_node, state, data_set);
         }
 
         crm_trace("Unpacking resource history for %snode %s",
                   (fence? "unseen " : ""), id);
 
         this_node->details->unpacked = TRUE;
         unpack_node_lrm(this_node, state, data_set);
 
         rc = EAGAIN; // Other node histories might depend on this one
     }
     return rc;
 }
 
 /* remove nodes that are down, stopping */
 /* create positive rsc_to_node constraints between resources and the nodes they are running on */
 /* anything else? */
 gboolean
 unpack_status(xmlNode * status, pe_working_set_t * data_set)
 {
     xmlNode *state = NULL;
 
     crm_trace("Beginning unpack");
 
     if (data_set->tickets == NULL) {
         data_set->tickets = pcmk__strkey_table(free, destroy_ticket);
     }
 
     for (state = pcmk__xe_first_child(status); state != NULL;
          state = pcmk__xe_next(state)) {
 
         if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, pcmk__str_none)) {
             unpack_tickets_state((xmlNode *) state, data_set);
 
         } else if (pcmk__str_eq((const char *)state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
             unpack_node_state(state, data_set);
         }
     }
 
     while (unpack_node_history(status, FALSE, data_set) == EAGAIN) {
         crm_trace("Another pass through node resource histories is needed");
     }
 
     // Now catch any nodes we didn't see
     unpack_node_history(status,
                         pcmk_is_set(data_set->flags, pe_flag_stonith_enabled),
                         data_set);
 
     /* Now that we know where resources are, we can schedule stops of containers
      * with failed bundle connections
      */
     if (data_set->stop_needed != NULL) {
         for (GList *item = data_set->stop_needed; item; item = item->next) {
             pe_resource_t *container = item->data;
             pe_node_t *node = pe__current_node(container);
 
             if (node) {
                 stop_action(container, node, FALSE);
             }
         }
         g_list_free(data_set->stop_needed);
         data_set->stop_needed = NULL;
     }
 
     /* Now that we know status of all Pacemaker Remote connections and nodes,
      * we can stop connections for node shutdowns, and check the online status
      * of remote/guest nodes that didn't have any node history to unpack.
      */
     for (GList *gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         pe_node_t *this_node = gIter->data;
 
         if (!pe__is_guest_or_remote_node(this_node)) {
             continue;
         }
         if (this_node->details->shutdown
             && (this_node->details->remote_rsc != NULL)) {
             pe__set_next_role(this_node->details->remote_rsc, pcmk_role_stopped,
                               "remote shutdown");
         }
         if (!this_node->details->unpacked) {
             determine_remote_online_status(data_set, this_node);
         }
     }
 
     return TRUE;
 }
 
 static gboolean
 determine_online_status_no_fencing(pe_working_set_t *data_set,
                                    const xmlNode *node_state,
                                    pe_node_t *this_node)
 {
     gboolean online = FALSE;
     const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
     const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
     const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
     int member = false;
     bool crmd_online = false;
     long long when_member = 0;
     long long when_online = 0;
 
     // @COMPAT DCs < 2.1.7 use boolean instead of time for cluster membership
     if (crm_str_to_boolean(in_cluster, &member) != 1) {
         pcmk__scan_ll(in_cluster, &when_member, 0LL);
         member = (when_member > 0) ? true : false;
     }
 
     if (pcmk__str_eq(is_peer, ONLINESTATUS, pcmk__str_casei)) {
         crmd_online = true;
 
     } else if (pcmk__str_eq(is_peer, OFFLINESTATUS, pcmk__str_casei)) {
         crmd_online = false;
 
     } else {
         pcmk__scan_ll(is_peer, &when_online, 0LL);
         crmd_online = (when_online > 0) ? true : false;
     }
 
     if (!member) {
         crm_trace("Node is down: in_cluster=%s",
                   pcmk__s(in_cluster, "<null>"));
 
     } else if (crmd_online) {
         if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
             online = TRUE;
         } else {
             crm_debug("Node is not ready to run resources: %s", join);
         }
 
     } else if (this_node->details->expected_up == FALSE) {
         crm_trace("Controller is down: "
                   "in_cluster=%s is_peer=%s join=%s expected=%s",
                   pcmk__s(in_cluster, "<null>"), pcmk__s(is_peer, "<null>"),
                   pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));
 
     } else {
         /* mark it unclean */
         pe_fence_node(data_set, this_node, "peer is unexpectedly down", FALSE);
         crm_info("in_cluster=%s is_peer=%s join=%s expected=%s",
                  pcmk__s(in_cluster, "<null>"), pcmk__s(is_peer, "<null>"),
                  pcmk__s(join, "<null>"), pcmk__s(exp_state, "<null>"));
     }
     return online;
 }
 
 static gboolean
 determine_online_status_fencing(pe_working_set_t *data_set,
                                 const xmlNode *node_state, pe_node_t *this_node)
 {
     gboolean online = FALSE;
     gboolean do_terminate = FALSE;
     bool crmd_online = FALSE;
     const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
     const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
     const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
     const char *terminate = pe_node_attribute_raw(this_node, "terminate");
     int member = false;
     long long when_member = 0;
     long long when_online = 0;
 
 /*
   - XML_NODE_JOIN_STATE    ::= member|down|pending|banned
   - XML_NODE_EXPECTED      ::= member|down
 
   @COMPAT with entries recorded for DCs < 2.1.7
   - XML_NODE_IN_CLUSTER    ::= true|false
   - XML_NODE_IS_PEER       ::= online|offline
 
   Since crm_feature_set 3.18.0 (pacemaker-2.1.7):
   - XML_NODE_IN_CLUSTER    ::= <timestamp>|0
   Since when node has been a cluster member. A value 0 of means the node is not
   a cluster member.
 
   - XML_NODE_IS_PEER       ::= <timestamp>|0
   Since when peer has been online in CPG. A value 0 means the peer is offline
   in CPG.
 */
 
     if (crm_is_true(terminate)) {
         do_terminate = TRUE;
 
     } else if (terminate != NULL && strlen(terminate) > 0) {
         /* could be a time() value */
         char t = terminate[0];
 
         if (t != '0' && isdigit(t)) {
             do_terminate = TRUE;
         }
     }
 
     crm_trace("%s: in_cluster=%s is_peer=%s join=%s expected=%s term=%d",
               pe__node_name(this_node), pcmk__s(in_cluster, "<null>"),
               pcmk__s(is_peer, "<null>"), pcmk__s(join, "<null>"),
               pcmk__s(exp_state, "<null>"), do_terminate);
 
     /* @COMPAT with boolean values of XML_NODE_IN_CLUSTER recorded for
      * DCs < 2.1.7
      */
     if (crm_str_to_boolean(in_cluster, &member) != 1) {
         pcmk__scan_ll(in_cluster, &when_member, 0LL);
         member = (when_member > 0) ? true : false;
     }
 
     online = member;
 
     /* @COMPAT with "online"/"offline" values of XML_NODE_IS_PEER recorded for
      * DCs < 2.1.7
      */
     if (pcmk__str_eq(is_peer, ONLINESTATUS, pcmk__str_casei)) {
         crmd_online = true;
 
     } else if (pcmk__str_eq(is_peer, OFFLINESTATUS, pcmk__str_casei)) {
         crmd_online = false;
 
     } else {
         pcmk__scan_ll(is_peer, &when_online, 0LL);
         crmd_online = (when_online > 0) ? true : false;
     }
 
     if (exp_state == NULL) {
         exp_state = CRMD_JOINSTATE_DOWN;
     }
 
     if (this_node->details->shutdown) {
         crm_debug("%s is shutting down", pe__node_name(this_node));
 
         /* Slightly different criteria since we can't shut down a dead peer */
         online = crmd_online;
 
     } else if (in_cluster == NULL) {
         pe_fence_node(data_set, this_node, "peer has not been seen by the cluster", FALSE);
 
     } else if (pcmk__str_eq(join, CRMD_JOINSTATE_NACK, pcmk__str_casei)) {
         pe_fence_node(data_set, this_node,
                       "peer failed Pacemaker membership criteria", FALSE);
 
     } else if (do_terminate == FALSE && pcmk__str_eq(exp_state, CRMD_JOINSTATE_DOWN, pcmk__str_casei)) {
 
         if (when_member > 0
             && when_online == 0
             && (get_effective_time(data_set) - when_member
                 >= data_set->node_pending_timeout)) {
             pe_fence_node(data_set, this_node,
                           "peer pending timed out on joining the process group",
                           FALSE);
 
         } else if (member || crmd_online) {
             crm_info("- %s is not ready to run resources",
                      pe__node_name(this_node));
             this_node->details->standby = TRUE;
             this_node->details->pending = TRUE;
 
         } else {
             crm_trace("%s is down or still coming up",
                       pe__node_name(this_node));
         }
 
     } else if (do_terminate && pcmk__str_eq(join, CRMD_JOINSTATE_DOWN, pcmk__str_casei)
                && !member && !crmd_online) {
         crm_info("%s was just shot", pe__node_name(this_node));
         online = FALSE;
 
     } else if (!member) {
         // Consider `priority-fencing-delay` for lost nodes
         pe_fence_node(data_set, this_node, "peer is no longer part of the cluster", TRUE);
 
     } else if (!crmd_online) {
         pe_fence_node(data_set, this_node, "peer process is no longer available", FALSE);
 
         /* Everything is running at this point, now check join state */
     } else if (do_terminate) {
         pe_fence_node(data_set, this_node, "termination was requested", FALSE);
 
     } else if (pcmk__str_eq(join, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
         crm_info("%s is active", pe__node_name(this_node));
 
     } else if (pcmk__strcase_any_of(join, CRMD_JOINSTATE_PENDING, CRMD_JOINSTATE_DOWN, NULL)) {
         crm_info("%s is not ready to run resources", pe__node_name(this_node));
         this_node->details->standby = TRUE;
         this_node->details->pending = TRUE;
 
     } else {
         pe_fence_node(data_set, this_node, "peer was in an unknown state", FALSE);
         crm_warn("%s: in-cluster=%s is-peer=%s join=%s expected=%s term=%d shutdown=%d",
                  pe__node_name(this_node), pcmk__s(in_cluster, "<null>"),
                  pcmk__s(is_peer, "<null>"), pcmk__s(join, "<null>"),
                  pcmk__s(exp_state, "<null>"), do_terminate,
                  this_node->details->shutdown);
     }
 
     return online;
 }
 
 static void
 determine_remote_online_status(pe_working_set_t * data_set, pe_node_t * this_node)
 {
     pe_resource_t *rsc = this_node->details->remote_rsc;
     pe_resource_t *container = NULL;
     pe_node_t *host = NULL;
 
     /* If there is a node state entry for a (former) Pacemaker Remote node
      * but no resource creating that node, the node's connection resource will
      * be NULL. Consider it an offline remote node in that case.
      */
     if (rsc == NULL) {
         this_node->details->online = FALSE;
         goto remote_online_done;
     }
 
     container = rsc->container;
 
     if (container && pcmk__list_of_1(rsc->running_on)) {
         host = rsc->running_on->data;
     }
 
     /* If the resource is currently started, mark it online. */
     if (rsc->role == pcmk_role_started) {
         crm_trace("%s node %s presumed ONLINE because connection resource is started",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->online = TRUE;
     }
 
     /* consider this node shutting down if transitioning start->stop */
     if ((rsc->role == pcmk_role_started)
         && (rsc->next_role == pcmk_role_stopped)) {
 
         crm_trace("%s node %s shutting down because connection resource is stopping",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->shutdown = TRUE;
     }
 
     /* Now check all the failure conditions. */
     if(container && pcmk_is_set(container->flags, pe_rsc_failed)) {
         crm_trace("Guest node %s UNCLEAN because guest resource failed",
                   this_node->details->id);
         this_node->details->online = FALSE;
         this_node->details->remote_requires_reset = TRUE;
 
     } else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
         crm_trace("%s node %s OFFLINE because connection resource failed",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->online = FALSE;
 
     } else if ((rsc->role == pcmk_role_stopped)
                || ((container != NULL)
                    && (container->role == pcmk_role_stopped))) {
 
         crm_trace("%s node %s OFFLINE because its resource is stopped",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->online = FALSE;
         this_node->details->remote_requires_reset = FALSE;
 
     } else if (host && (host->details->online == FALSE)
                && host->details->unclean) {
         crm_trace("Guest node %s UNCLEAN because host is unclean",
                   this_node->details->id);
         this_node->details->online = FALSE;
         this_node->details->remote_requires_reset = TRUE;
     }
 
 remote_online_done:
     crm_trace("Remote node %s online=%s",
         this_node->details->id, this_node->details->online ? "TRUE" : "FALSE");
 }
 
 static void
 determine_online_status(const xmlNode *node_state, pe_node_t *this_node,
                         pe_working_set_t *data_set)
 {
     gboolean online = FALSE;
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
 
     CRM_CHECK(this_node != NULL, return);
 
     this_node->details->shutdown = FALSE;
     this_node->details->expected_up = FALSE;
 
     if (pe__shutdown_requested(this_node)) {
         this_node->details->shutdown = TRUE;
 
     } else if (pcmk__str_eq(exp_state, CRMD_JOINSTATE_MEMBER, pcmk__str_casei)) {
         this_node->details->expected_up = TRUE;
     }
 
     if (this_node->details->type == node_ping) {
         this_node->details->unclean = FALSE;
         online = FALSE;         /* As far as resource management is concerned,
                                  * the node is safely offline.
                                  * Anyone caught abusing this logic will be shot
                                  */
 
     } else if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
         online = determine_online_status_no_fencing(data_set, node_state, this_node);
 
     } else {
         online = determine_online_status_fencing(data_set, node_state, this_node);
     }
 
     if (online) {
         this_node->details->online = TRUE;
 
     } else {
         /* remove node from contention */
         this_node->fixed = TRUE; // @COMPAT deprecated and unused
         this_node->weight = -INFINITY;
     }
 
     if (online && this_node->details->shutdown) {
         /* don't run resources here */
         this_node->fixed = TRUE; // @COMPAT deprecated and unused
         this_node->weight = -INFINITY;
     }
 
     if (this_node->details->type == node_ping) {
         crm_info("%s is not a Pacemaker node", pe__node_name(this_node));
 
     } else if (this_node->details->unclean) {
         pe_proc_warn("%s is unclean", pe__node_name(this_node));
 
     } else if (this_node->details->online) {
         crm_info("%s is %s", pe__node_name(this_node),
                  this_node->details->shutdown ? "shutting down" :
                  this_node->details->pending ? "pending" :
                  this_node->details->standby ? "standby" :
                  this_node->details->maintenance ? "maintenance" : "online");
 
     } else {
         crm_trace("%s is offline", pe__node_name(this_node));
     }
 }
 
 /*!
  * \internal
  * \brief Find the end of a resource's name, excluding any clone suffix
  *
  * \param[in] id  Resource ID to check
  *
  * \return Pointer to last character of resource's base name
  */
 const char *
 pe_base_name_end(const char *id)
 {
     if (!pcmk__str_empty(id)) {
         const char *end = id + strlen(id) - 1;
 
         for (const char *s = end; s > id; --s) {
             switch (*s) {
                 case '0':
                 case '1':
                 case '2':
                 case '3':
                 case '4':
                 case '5':
                 case '6':
                 case '7':
                 case '8':
                 case '9':
                     break;
                 case ':':
                     return (s == end)? s : (s - 1);
                 default:
                     return end;
             }
         }
         return end;
     }
     return NULL;
 }
 
 /*!
  * \internal
  * \brief Get a resource name excluding any clone suffix
  *
  * \param[in] last_rsc_id  Resource ID to check
  *
  * \return Pointer to newly allocated string with resource's base name
  * \note It is the caller's responsibility to free() the result.
  *       This asserts on error, so callers can assume result is not NULL.
  */
 char *
 clone_strip(const char *last_rsc_id)
 {
     const char *end = pe_base_name_end(last_rsc_id);
     char *basename = NULL;
 
     CRM_ASSERT(end);
     basename = strndup(last_rsc_id, end - last_rsc_id + 1);
     CRM_ASSERT(basename);
     return basename;
 }
 
 /*!
  * \internal
  * \brief Get the name of the first instance of a cloned resource
  *
  * \param[in] last_rsc_id  Resource ID to check
  *
  * \return Pointer to newly allocated string with resource's base name plus :0
  * \note It is the caller's responsibility to free() the result.
  *       This asserts on error, so callers can assume result is not NULL.
  */
 char *
 clone_zero(const char *last_rsc_id)
 {
     const char *end = pe_base_name_end(last_rsc_id);
     size_t base_name_len = end - last_rsc_id + 1;
     char *zero = NULL;
 
     CRM_ASSERT(end);
     zero = calloc(base_name_len + 3, sizeof(char));
     CRM_ASSERT(zero);
     memcpy(zero, last_rsc_id, base_name_len);
     zero[base_name_len] = ':';
     zero[base_name_len + 1] = '0';
     return zero;
 }
 
 static pe_resource_t *
 create_fake_resource(const char *rsc_id, const xmlNode *rsc_entry,
                      pe_working_set_t *data_set)
 {
     pe_resource_t *rsc = NULL;
     xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
 
     copy_in_properties(xml_rsc, rsc_entry);
     crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
     crm_log_xml_debug(xml_rsc, "Orphan resource");
 
     if (pe__unpack_resource(xml_rsc, &rsc, NULL, data_set) != pcmk_rc_ok) {
         return NULL;
     }
 
     if (xml_contains_remote_node(xml_rsc)) {
         pe_node_t *node;
 
         crm_debug("Detected orphaned remote node %s", rsc_id);
         node = pe_find_node(data_set->nodes, rsc_id);
         if (node == NULL) {
 	        node = pe_create_node(rsc_id, rsc_id, "remote", NULL, data_set);
         }
         link_rsc2remotenode(data_set, rsc);
 
         if (node) {
             crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
             node->details->shutdown = TRUE;
         }
     }
 
     if (crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER)) {
         /* This orphaned rsc needs to be mapped to a container. */
         crm_trace("Detected orphaned container filler %s", rsc_id);
         pe__set_resource_flags(rsc, pe_rsc_orphan_container_filler);
     }
     pe__set_resource_flags(rsc, pe_rsc_orphan);
     data_set->resources = g_list_append(data_set->resources, rsc);
     return rsc;
 }
 
 /*!
  * \internal
  * \brief Create orphan instance for anonymous clone resource history
  *
  * \param[in,out] parent    Clone resource that orphan will be added to
  * \param[in]     rsc_id    Orphan's resource ID
  * \param[in]     node      Where orphan is active (for logging only)
  * \param[in,out] data_set  Cluster working set
  *
  * \return Newly added orphaned instance of \p parent
  */
 static pe_resource_t *
 create_anonymous_orphan(pe_resource_t *parent, const char *rsc_id,
                         const pe_node_t *node, pe_working_set_t *data_set)
 {
     pe_resource_t *top = pe__create_clone_child(parent, data_set);
 
     // find_rsc() because we might be a cloned group
     pe_resource_t *orphan = top->fns->find_rsc(top, rsc_id, NULL,
                                                pcmk_rsc_match_clone_only);
 
     pe_rsc_debug(parent, "Created orphan %s for %s: %s on %s",
                  top->id, parent->id, rsc_id, pe__node_name(node));
     return orphan;
 }
 
 /*!
  * \internal
  * \brief Check a node for an instance of an anonymous clone
  *
  * Return a child instance of the specified anonymous clone, in order of
  * preference: (1) the instance running on the specified node, if any;
  * (2) an inactive instance (i.e. within the total of clone-max instances);
  * (3) a newly created orphan (i.e. clone-max instances are already active).
  *
  * \param[in,out] data_set  Cluster information
  * \param[in]     node      Node on which to check for instance
  * \param[in,out] parent    Clone to check
  * \param[in]     rsc_id    Name of cloned resource in history (without instance)
  */
 static pe_resource_t *
 find_anonymous_clone(pe_working_set_t *data_set, const pe_node_t *node,
                      pe_resource_t *parent, const char *rsc_id)
 {
     GList *rIter = NULL;
     pe_resource_t *rsc = NULL;
     pe_resource_t *inactive_instance = NULL;
     gboolean skip_inactive = FALSE;
 
     CRM_ASSERT(parent != NULL);
     CRM_ASSERT(pe_rsc_is_clone(parent));
     CRM_ASSERT(!pcmk_is_set(parent->flags, pe_rsc_unique));
 
     // Check for active (or partially active, for cloned groups) instance
     pe_rsc_trace(parent, "Looking for %s on %s in %s",
                  rsc_id, pe__node_name(node), parent->id);
     for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
         GList *locations = NULL;
         pe_resource_t *child = rIter->data;
 
         /* Check whether this instance is already known to be active or pending
          * anywhere, at this stage of unpacking. Because this function is called
          * for a resource before the resource's individual operation history
          * entries are unpacked, locations will generally not contain the
          * desired node.
          *
          * However, there are three exceptions:
          * (1) when child is a cloned group and we have already unpacked the
          *     history of another member of the group on the same node;
          * (2) when we've already unpacked the history of another numbered
          *     instance on the same node (which can happen if globally-unique
          *     was flipped from true to false); and
          * (3) when we re-run calculations on the same data set as part of a
          *     simulation.
          */
         child->fns->location(child, &locations, 2);
         if (locations) {
             /* We should never associate the same numbered anonymous clone
              * instance with multiple nodes, and clone instances can't migrate,
              * so there must be only one location, regardless of history.
              */
             CRM_LOG_ASSERT(locations->next == NULL);
 
             if (((pe_node_t *)locations->data)->details == node->details) {
                 /* This child instance is active on the requested node, so check
                  * for a corresponding configured resource. We use find_rsc()
                  * instead of child because child may be a cloned group, and we
                  * need the particular member corresponding to rsc_id.
                  *
                  * If the history entry is orphaned, rsc will be NULL.
                  */
                 rsc = parent->fns->find_rsc(child, rsc_id, NULL,
                                             pcmk_rsc_match_clone_only);
                 if (rsc) {
                     /* If there are multiple instance history entries for an
                      * anonymous clone in a single node's history (which can
                      * happen if globally-unique is switched from true to
                      * false), we want to consider the instances beyond the
                      * first as orphans, even if there are inactive instance
                      * numbers available.
                      */
                     if (rsc->running_on) {
                         crm_notice("Active (now-)anonymous clone %s has "
                                    "multiple (orphan) instance histories on %s",
                                    parent->id, pe__node_name(node));
                         skip_inactive = TRUE;
                         rsc = NULL;
                     } else {
                         pe_rsc_trace(parent, "Resource %s, active", rsc->id);
                     }
                 }
             }
             g_list_free(locations);
 
         } else {
             pe_rsc_trace(parent, "Resource %s, skip inactive", child->id);
             if (!skip_inactive && !inactive_instance
                 && !pcmk_is_set(child->flags, pe_rsc_block)) {
                 // Remember one inactive instance in case we don't find active
                 inactive_instance = parent->fns->find_rsc(child, rsc_id, NULL,
                                                           pcmk_rsc_match_clone_only);
 
                 /* ... but don't use it if it was already associated with a
                  * pending action on another node
                  */
                 if (inactive_instance && inactive_instance->pending_node
                     && (inactive_instance->pending_node->details != node->details)) {
                     inactive_instance = NULL;
                 }
             }
         }
     }
 
     if ((rsc == NULL) && !skip_inactive && (inactive_instance != NULL)) {
         pe_rsc_trace(parent, "Resource %s, empty slot", inactive_instance->id);
         rsc = inactive_instance;
     }
 
     /* If the resource has "requires" set to "quorum" or "nothing", and we don't
      * have a clone instance for every node, we don't want to consume a valid
      * instance number for unclean nodes. Such instances may appear to be active
      * according to the history, but should be considered inactive, so we can
      * start an instance elsewhere. Treat such instances as orphans.
      *
      * An exception is instances running on guest nodes -- since guest node
      * "fencing" is actually just a resource stop, requires shouldn't apply.
      *
      * @TODO Ideally, we'd use an inactive instance number if it is not needed
      * for any clean instances. However, we don't know that at this point.
      */
     if ((rsc != NULL) && !pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)
         && (!node->details->online || node->details->unclean)
         && !pe__is_guest_node(node)
         && !pe__is_universal_clone(parent, data_set)) {
 
         rsc = NULL;
     }
 
     if (rsc == NULL) {
         rsc = create_anonymous_orphan(parent, rsc_id, node, data_set);
         pe_rsc_trace(parent, "Resource %s, orphan", rsc->id);
     }
     return rsc;
 }
 
 static pe_resource_t *
 unpack_find_resource(pe_working_set_t *data_set, const pe_node_t *node,
                      const char *rsc_id)
 {
     pe_resource_t *rsc = NULL;
     pe_resource_t *parent = NULL;
 
     crm_trace("looking for %s", rsc_id);
     rsc = pe_find_resource(data_set->resources, rsc_id);
 
     if (rsc == NULL) {
         /* If we didn't find the resource by its name in the operation history,
          * check it again as a clone instance. Even when clone-max=0, we create
          * a single :0 orphan to match against here.
          */
         char *clone0_id = clone_zero(rsc_id);
         pe_resource_t *clone0 = pe_find_resource(data_set->resources, clone0_id);
 
         if (clone0 && !pcmk_is_set(clone0->flags, pe_rsc_unique)) {
             rsc = clone0;
             parent = uber_parent(clone0);
             crm_trace("%s found as %s (%s)", rsc_id, clone0_id, parent->id);
         } else {
             crm_trace("%s is not known as %s either (orphan)",
                       rsc_id, clone0_id);
         }
         free(clone0_id);
 
     } else if (rsc->variant > pcmk_rsc_variant_primitive) {
         crm_trace("Resource history for %s is orphaned because it is no longer primitive",
                   rsc_id);
         return NULL;
 
     } else {
         parent = uber_parent(rsc);
     }
 
     if (pe_rsc_is_anon_clone(parent)) {
 
         if (pe_rsc_is_bundled(parent)) {
             rsc = pe__find_bundle_replica(parent->parent, node);
         } else {
             char *base = clone_strip(rsc_id);
 
             rsc = find_anonymous_clone(data_set, node, parent, base);
             free(base);
             CRM_ASSERT(rsc != NULL);
         }
     }
 
     if (rsc && !pcmk__str_eq(rsc_id, rsc->id, pcmk__str_casei)
         && !pcmk__str_eq(rsc_id, rsc->clone_name, pcmk__str_casei)) {
 
         pcmk__str_update(&rsc->clone_name, rsc_id);
         pe_rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
                      rsc_id, pe__node_name(node), rsc->id,
                      (pcmk_is_set(rsc->flags, pe_rsc_orphan)? " (ORPHAN)" : ""));
     }
     return rsc;
 }
 
 static pe_resource_t *
 process_orphan_resource(const xmlNode *rsc_entry, const pe_node_t *node,
                         pe_working_set_t *data_set)
 {
     pe_resource_t *rsc = NULL;
     const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
 
     crm_debug("Detected orphan resource %s on %s", rsc_id, pe__node_name(node));
     rsc = create_fake_resource(rsc_id, rsc_entry, data_set);
     if (rsc == NULL) {
         return NULL;
     }
 
     if (!pcmk_is_set(data_set->flags, pe_flag_stop_rsc_orphans)) {
         pe__clear_resource_flags(rsc, pe_rsc_managed);
 
     } else {
         CRM_CHECK(rsc != NULL, return NULL);
         pe_rsc_trace(rsc, "Added orphan %s", rsc->id);
         resource_location(rsc, NULL, -INFINITY, "__orphan_do_not_run__", data_set);
     }
     return rsc;
 }
 
 static void
 process_rsc_state(pe_resource_t * rsc, pe_node_t * node,
                   enum action_fail_response on_fail)
 {
     pe_node_t *tmpnode = NULL;
     char *reason = NULL;
     enum action_fail_response save_on_fail = pcmk_on_fail_ignore;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
                  rsc->id, role2text(rsc->role), pe__node_name(node),
                  fail2text(on_fail));
 
     /* process current state */
     if (rsc->role != pcmk_role_unknown) {
         pe_resource_t *iter = rsc;
 
         while (iter) {
             if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
                 pe_node_t *n = pe__copy_node(node);
 
                 pe_rsc_trace(rsc, "%s%s%s known on %s",
                              rsc->id,
                              ((rsc->clone_name == NULL)? "" : " also known as "),
                              ((rsc->clone_name == NULL)? "" : rsc->clone_name),
                              pe__node_name(n));
                 g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
             }
             if (pcmk_is_set(iter->flags, pe_rsc_unique)) {
                 break;
             }
             iter = iter->parent;
         }
     }
 
     /* If a managed resource is believed to be running, but node is down ... */
     if ((rsc->role > pcmk_role_stopped)
         && node->details->online == FALSE
         && node->details->maintenance == FALSE
         && pcmk_is_set(rsc->flags, pe_rsc_managed)) {
 
         gboolean should_fence = FALSE;
 
         /* If this is a guest node, fence it (regardless of whether fencing is
          * enabled, because guest node fencing is done by recovery of the
          * container resource rather than by the fencer). Mark the resource
          * we're processing as failed. When the guest comes back up, its
          * operation history in the CIB will be cleared, freeing the affected
          * resource to run again once we are sure we know its state.
          */
         if (pe__is_guest_node(node)) {
             pe__set_resource_flags(rsc, pe_rsc_failed|pe_rsc_stop);
             should_fence = TRUE;
 
         } else if (pcmk_is_set(rsc->cluster->flags, pe_flag_stonith_enabled)) {
             if (pe__is_remote_node(node) && node->details->remote_rsc
                 && !pcmk_is_set(node->details->remote_rsc->flags, pe_rsc_failed)) {
 
                 /* Setting unseen means that fencing of the remote node will
                  * occur only if the connection resource is not going to start
                  * somewhere. This allows connection resources on a failed
                  * cluster node to move to another node without requiring the
                  * remote nodes to be fenced as well.
                  */
                 node->details->unseen = TRUE;
                 reason = crm_strdup_printf("%s is active there (fencing will be"
                                            " revoked if remote connection can "
                                            "be re-established elsewhere)",
                                            rsc->id);
             }
             should_fence = TRUE;
         }
 
         if (should_fence) {
             if (reason == NULL) {
                reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
             }
             pe_fence_node(rsc->cluster, node, reason, FALSE);
         }
         free(reason);
     }
 
     /* In order to calculate priority_fencing_delay correctly, save the failure information and pass it to native_add_running(). */
     save_on_fail = on_fail;
 
     if (node->details->unclean) {
         /* No extra processing needed
          * Also allows resources to be started again after a node is shot
          */
         on_fail = pcmk_on_fail_ignore;
     }
 
     switch (on_fail) {
         case pcmk_on_fail_ignore:
             /* nothing to do */
             break;
 
         case pcmk_on_fail_demote:
             pe__set_resource_flags(rsc, pe_rsc_failed);
             demote_action(rsc, node, FALSE);
             break;
 
         case pcmk_on_fail_fence_node:
             /* treat it as if it is still running
              * but also mark the node as unclean
              */
             reason = crm_strdup_printf("%s failed there", rsc->id);
             pe_fence_node(rsc->cluster, node, reason, FALSE);
             free(reason);
             break;
 
         case pcmk_on_fail_standby_node:
             node->details->standby = TRUE;
             node->details->standby_onfail = TRUE;
             break;
 
         case pcmk_on_fail_block:
             /* is_managed == FALSE will prevent any
              * actions being sent for the resource
              */
             pe__clear_resource_flags(rsc, pe_rsc_managed);
             pe__set_resource_flags(rsc, pe_rsc_block);
             break;
 
         case pcmk_on_fail_ban:
             /* make sure it comes up somewhere else
              * or not at all
              */
             resource_location(rsc, node, -INFINITY, "__action_migration_auto__",
                               rsc->cluster);
             break;
 
         case pcmk_on_fail_stop:
             pe__set_next_role(rsc, pcmk_role_stopped, "on-fail=stop");
             break;
 
         case pcmk_on_fail_restart:
             if ((rsc->role != pcmk_role_stopped)
                 && (rsc->role != pcmk_role_unknown)) {
                 pe__set_resource_flags(rsc, pe_rsc_failed|pe_rsc_stop);
                 stop_action(rsc, node, FALSE);
             }
             break;
 
         case pcmk_on_fail_restart_container:
             pe__set_resource_flags(rsc, pe_rsc_failed|pe_rsc_stop);
             if (rsc->container && pe_rsc_is_bundled(rsc)) {
                 /* A bundle's remote connection can run on a different node than
                  * the bundle's container. We don't necessarily know where the
                  * container is running yet, so remember it and add a stop
                  * action for it later.
                  */
                 rsc->cluster->stop_needed =
                     g_list_prepend(rsc->cluster->stop_needed, rsc->container);
             } else if (rsc->container) {
                 stop_action(rsc->container, node, FALSE);
             } else if ((rsc->role != pcmk_role_stopped)
                        && (rsc->role != pcmk_role_unknown)) {
                 stop_action(rsc, node, FALSE);
             }
             break;
 
         case pcmk_on_fail_reset_remote:
             pe__set_resource_flags(rsc, pe_rsc_failed|pe_rsc_stop);
             if (pcmk_is_set(rsc->cluster->flags, pe_flag_stonith_enabled)) {
                 tmpnode = NULL;
                 if (rsc->is_remote_node) {
                     tmpnode = pe_find_node(rsc->cluster->nodes, rsc->id);
                 }
                 if (tmpnode &&
                     pe__is_remote_node(tmpnode) &&
                     tmpnode->details->remote_was_fenced == 0) {
 
                     /* The remote connection resource failed in a way that
                      * should result in fencing the remote node.
                      */
                     pe_fence_node(rsc->cluster, tmpnode,
                                   "remote connection is unrecoverable", FALSE);
                 }
             }
 
             /* require the stop action regardless if fencing is occurring or not. */
             if (rsc->role > pcmk_role_stopped) {
                 stop_action(rsc, node, FALSE);
             }
 
             /* if reconnect delay is in use, prevent the connection from exiting the
              * "STOPPED" role until the failure is cleared by the delay timeout. */
             if (rsc->remote_reconnect_ms) {
                 pe__set_next_role(rsc, pcmk_role_stopped, "remote reset");
             }
             break;
     }
 
     /* ensure a remote-node connection failure forces an unclean remote-node
      * to be fenced. By setting unseen = FALSE, the remote-node failure will
      * result in a fencing operation regardless if we're going to attempt to 
      * reconnect to the remote-node in this transition or not. */
     if (pcmk_is_set(rsc->flags, pe_rsc_failed) && rsc->is_remote_node) {
         tmpnode = pe_find_node(rsc->cluster->nodes, rsc->id);
         if (tmpnode && tmpnode->details->unclean) {
             tmpnode->details->unseen = FALSE;
         }
     }
 
     if ((rsc->role != pcmk_role_stopped)
         && (rsc->role != pcmk_role_unknown)) {
 
         if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
             if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
                 pcmk__config_warn("Detected active orphan %s running on %s",
                                   rsc->id, pe__node_name(node));
             } else {
                 pcmk__config_warn("Resource '%s' must be stopped manually on "
                                   "%s because cluster is configured not to "
                                   "stop active orphans",
                                   rsc->id, pe__node_name(node));
             }
         }
 
         native_add_running(rsc, node, rsc->cluster,
                            (save_on_fail != pcmk_on_fail_ignore));
         switch (on_fail) {
             case pcmk_on_fail_ignore:
                 break;
             case pcmk_on_fail_demote:
             case pcmk_on_fail_block:
                 pe__set_resource_flags(rsc, pe_rsc_failed);
                 break;
             default:
                 pe__set_resource_flags(rsc, pe_rsc_failed|pe_rsc_stop);
                 break;
         }
 
     } else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
         /* Only do this for older status sections that included instance numbers
          * Otherwise stopped instances will appear as orphans
          */
         pe_rsc_trace(rsc, "Resetting clone_name %s for %s (stopped)", rsc->clone_name, rsc->id);
         free(rsc->clone_name);
         rsc->clone_name = NULL;
 
     } else {
         GList *possible_matches = pe__resource_actions(rsc, node,
                                                        PCMK_ACTION_STOP, FALSE);
         GList *gIter = possible_matches;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_action_t *stop = (pe_action_t *) gIter->data;
 
             pe__set_action_flags(stop, pe_action_optional);
         }
 
         g_list_free(possible_matches);
     }
 
     /* A successful stop after migrate_to on the migration source doesn't make
      * the partially migrated resource stopped on the migration target.
      */
     if ((rsc->role == pcmk_role_stopped)
         && rsc->partial_migration_source
         && rsc->partial_migration_source->details == node->details
         && rsc->partial_migration_target
         && rsc->running_on) {
 
         rsc->role = pcmk_role_started;
     }
 }
 
 /* create active recurring operations as optional */
 static void
 process_recurring(pe_node_t * node, pe_resource_t * rsc,
                   int start_index, int stop_index,
                   GList *sorted_op_list, pe_working_set_t * data_set)
 {
     int counter = -1;
     const char *task = NULL;
     const char *status = NULL;
     GList *gIter = sorted_op_list;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "%s: Start index %d, stop index = %d", rsc->id, start_index, stop_index);
 
     for (; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         guint interval_ms = 0;
         char *key = NULL;
         const char *id = ID(rsc_op);
 
         counter++;
 
         if (node->details->online == FALSE) {
             pe_rsc_trace(rsc, "Skipping %s on %s: node is offline",
                          rsc->id, pe__node_name(node));
             break;
 
             /* Need to check if there's a monitor for role="Stopped" */
         } else if (start_index < stop_index && counter <= stop_index) {
             pe_rsc_trace(rsc, "Skipping %s on %s: resource is not active",
                          id, pe__node_name(node));
             continue;
 
         } else if (counter < start_index) {
             pe_rsc_trace(rsc, "Skipping %s on %s: old %d",
                          id, pe__node_name(node), counter);
             continue;
         }
 
         crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
         if (interval_ms == 0) {
             pe_rsc_trace(rsc, "Skipping %s on %s: non-recurring",
                          id, pe__node_name(node));
             continue;
         }
 
         status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
         if (pcmk__str_eq(status, "-1", pcmk__str_casei)) {
             pe_rsc_trace(rsc, "Skipping %s on %s: status",
                          id, pe__node_name(node));
             continue;
         }
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         /* create the action */
         key = pcmk__op_key(rsc->id, task, interval_ms);
         pe_rsc_trace(rsc, "Creating %s on %s", key, pe__node_name(node));
         custom_action(rsc, key, task, node, TRUE, TRUE, data_set);
     }
 }
 
 void
 calculate_active_ops(const GList *sorted_op_list, int *start_index,
                      int *stop_index)
 {
     int counter = -1;
     int implied_monitor_start = -1;
     int implied_clone_start = -1;
     const char *task = NULL;
     const char *status = NULL;
 
     *stop_index = -1;
     *start_index = -1;
 
     for (const GList *iter = sorted_op_list; iter != NULL; iter = iter->next) {
         const xmlNode *rsc_op = (const xmlNode *) iter->data;
 
         counter++;
 
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
 
         if (pcmk__str_eq(task, PCMK_ACTION_STOP, pcmk__str_casei)
             && pcmk__str_eq(status, "0", pcmk__str_casei)) {
             *stop_index = counter;
 
         } else if (pcmk__strcase_any_of(task, PCMK_ACTION_START,
                                         PCMK_ACTION_MIGRATE_FROM, NULL)) {
             *start_index = counter;
 
         } else if ((implied_monitor_start <= *stop_index)
                    && pcmk__str_eq(task, PCMK_ACTION_MONITOR,
                                    pcmk__str_casei)) {
             const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);
 
             if (pcmk__strcase_any_of(rc, "0", "8", NULL)) {
                 implied_monitor_start = counter;
             }
         } else if (pcmk__strcase_any_of(task, PCMK_ACTION_PROMOTE,
                                         PCMK_ACTION_DEMOTE, NULL)) {
             implied_clone_start = counter;
         }
     }
 
     if (*start_index == -1) {
         if (implied_clone_start != -1) {
             *start_index = implied_clone_start;
         } else if (implied_monitor_start != -1) {
             *start_index = implied_monitor_start;
         }
     }
 }
 
 // If resource history entry has shutdown lock, remember lock node and time
 static void
 unpack_shutdown_lock(const xmlNode *rsc_entry, pe_resource_t *rsc,
                      const pe_node_t *node, pe_working_set_t *data_set)
 {
     time_t lock_time = 0;   // When lock started (i.e. node shutdown time)
 
     if ((crm_element_value_epoch(rsc_entry, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
                                  &lock_time) == pcmk_ok) && (lock_time != 0)) {
 
         if ((data_set->shutdown_lock > 0)
             && (get_effective_time(data_set)
                 > (lock_time + data_set->shutdown_lock))) {
             pe_rsc_info(rsc, "Shutdown lock for %s on %s expired",
                         rsc->id, pe__node_name(node));
             pe__clear_resource_history(rsc, node, data_set);
         } else {
             /* @COMPAT I don't like breaking const signatures, but
              * rsc->lock_node should really be const -- we just can't change it
              * until the next API compatibility break.
              */
             rsc->lock_node = (pe_node_t *) node;
             rsc->lock_time = lock_time;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Unpack one lrm_resource entry from a node's CIB status
  *
  * \param[in,out] node       Node whose status is being unpacked
  * \param[in]     rsc_entry  lrm_resource XML being unpacked
  * \param[in,out] data_set   Cluster working set
  *
  * \return Resource corresponding to the entry, or NULL if no operation history
  */
 static pe_resource_t *
 unpack_lrm_resource(pe_node_t *node, const xmlNode *lrm_resource,
                     pe_working_set_t *data_set)
 {
     GList *gIter = NULL;
     int stop_index = -1;
     int start_index = -1;
     enum rsc_role_e req_role = pcmk_role_unknown;
 
     const char *rsc_id = ID(lrm_resource);
 
     pe_resource_t *rsc = NULL;
     GList *op_list = NULL;
     GList *sorted_op_list = NULL;
 
     xmlNode *rsc_op = NULL;
     xmlNode *last_failure = NULL;
 
     enum action_fail_response on_fail = pcmk_on_fail_ignore;
     enum rsc_role_e saved_role = pcmk_role_unknown;
 
     if (rsc_id == NULL) {
         crm_warn("Ignoring malformed " XML_LRM_TAG_RESOURCE
                  " entry without id");
         return NULL;
     }
     crm_trace("Unpacking " XML_LRM_TAG_RESOURCE " for %s on %s",
               rsc_id, pe__node_name(node));
 
     // Build a list of individual lrm_rsc_op entries, so we can sort them
     for (rsc_op = first_named_child(lrm_resource, XML_LRM_TAG_RSC_OP);
          rsc_op != NULL; rsc_op = crm_next_same_xml(rsc_op)) {
 
         op_list = g_list_prepend(op_list, rsc_op);
     }
 
     if (!pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)) {
         if (op_list == NULL) {
             // If there are no operations, there is nothing to do
             return NULL;
         }
     }
 
     /* find the resource */
     rsc = unpack_find_resource(data_set, node, rsc_id);
     if (rsc == NULL) {
         if (op_list == NULL) {
             // If there are no operations, there is nothing to do
             return NULL;
         } else {
             rsc = process_orphan_resource(lrm_resource, node, data_set);
         }
     }
     CRM_ASSERT(rsc != NULL);
 
     // Check whether the resource is "shutdown-locked" to this node
     if (pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)) {
         unpack_shutdown_lock(lrm_resource, rsc, node, data_set);
     }
 
     /* process operations */
     saved_role = rsc->role;
     rsc->role = pcmk_role_unknown;
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail);
     }
 
     /* create active recurring operations as optional */
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
     process_recurring(node, rsc, start_index, stop_index, sorted_op_list, data_set);
 
     /* no need to free the contents */
     g_list_free(sorted_op_list);
 
     process_rsc_state(rsc, node, on_fail);
 
     if (get_target_role(rsc, &req_role)) {
         if ((rsc->next_role == pcmk_role_unknown)
             || (req_role < rsc->next_role)) {
 
             pe__set_next_role(rsc, req_role, XML_RSC_ATTR_TARGET_ROLE);
 
         } else if (req_role > rsc->next_role) {
             pe_rsc_info(rsc, "%s: Not overwriting calculated next role %s"
                         " with requested next role %s",
                         rsc->id, role2text(rsc->next_role), role2text(req_role));
         }
     }
 
     if (saved_role > rsc->role) {
         rsc->role = saved_role;
     }
 
     return rsc;
 }
 
 static void
 handle_orphaned_container_fillers(const xmlNode *lrm_rsc_list,
                                   pe_working_set_t *data_set)
 {
     for (const xmlNode *rsc_entry = pcmk__xe_first_child(lrm_rsc_list);
          rsc_entry != NULL; rsc_entry = pcmk__xe_next(rsc_entry)) {
 
         pe_resource_t *rsc;
         pe_resource_t *container;
         const char *rsc_id;
         const char *container_id;
 
         if (!pcmk__str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, pcmk__str_casei)) {
             continue;
         }
 
         container_id = crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER);
         rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
         if (container_id == NULL || rsc_id == NULL) {
             continue;
         }
 
         container = pe_find_resource(data_set->resources, container_id);
         if (container == NULL) {
             continue;
         }
 
         rsc = pe_find_resource(data_set->resources, rsc_id);
         if (rsc == NULL ||
             !pcmk_is_set(rsc->flags, pe_rsc_orphan_container_filler) ||
             rsc->container != NULL) {
             continue;
         }
 
         pe_rsc_trace(rsc, "Mapped container of orphaned resource %s to %s",
                      rsc->id, container_id);
         rsc->container = container;
         container->fillers = g_list_append(container->fillers, rsc);
     }
 }
 
 /*!
  * \internal
  * \brief Unpack one node's lrm status section
  *
  * \param[in,out] node      Node whose status is being unpacked
  * \param[in]     xml       CIB node state XML
  * \param[in,out] data_set  Cluster working set
  */
 static void
 unpack_node_lrm(pe_node_t *node, const xmlNode *xml, pe_working_set_t *data_set)
 {
     bool found_orphaned_container_filler = false;
 
     // Drill down to lrm_resources section
     xml = find_xml_node(xml, XML_CIB_TAG_LRM, FALSE);
     if (xml == NULL) {
         return;
     }
     xml = find_xml_node(xml, XML_LRM_TAG_RESOURCES, FALSE);
     if (xml == NULL) {
         return;
     }
 
     // Unpack each lrm_resource entry
     for (const xmlNode *rsc_entry = first_named_child(xml, XML_LRM_TAG_RESOURCE);
          rsc_entry != NULL; rsc_entry = crm_next_same_xml(rsc_entry)) {
 
         pe_resource_t *rsc = unpack_lrm_resource(node, rsc_entry, data_set);
 
         if ((rsc != NULL)
             && pcmk_is_set(rsc->flags, pe_rsc_orphan_container_filler)) {
             found_orphaned_container_filler = true;
         }
     }
 
     /* Now that all resource state has been unpacked for this node, map any
      * orphaned container fillers to their container resource.
      */
     if (found_orphaned_container_filler) {
         handle_orphaned_container_fillers(xml, data_set);
     }
 }
 
 static void
 set_active(pe_resource_t * rsc)
 {
     const pe_resource_t *top = pe__const_top_resource(rsc, false);
 
     if (top && pcmk_is_set(top->flags, pe_rsc_promotable)) {
         rsc->role = pcmk_role_unpromoted;
     } else {
         rsc->role = pcmk_role_started;
     }
 }
 
 static void
 set_node_score(gpointer key, gpointer value, gpointer user_data)
 {
     pe_node_t *node = value;
     int *score = user_data;
 
     node->weight = *score;
 }
 
 #define XPATH_NODE_STATE "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS     \
                          "/" XML_CIB_TAG_STATE
 #define SUB_XPATH_LRM_RESOURCE "/" XML_CIB_TAG_LRM              \
                                "/" XML_LRM_TAG_RESOURCES        \
                                "/" XML_LRM_TAG_RESOURCE
 #define SUB_XPATH_LRM_RSC_OP "/" XML_LRM_TAG_RSC_OP
 
 static xmlNode *
 find_lrm_op(const char *resource, const char *op, const char *node, const char *source,
             int target_rc, pe_working_set_t *data_set)
 {
     GString *xpath = NULL;
     xmlNode *xml = NULL;
 
     CRM_CHECK((resource != NULL) && (op != NULL) && (node != NULL),
               return NULL);
 
     xpath = g_string_sized_new(256);
     pcmk__g_strcat(xpath,
                    XPATH_NODE_STATE "[@" XML_ATTR_UNAME "='", node, "']"
                    SUB_XPATH_LRM_RESOURCE "[@" XML_ATTR_ID "='", resource, "']"
                    SUB_XPATH_LRM_RSC_OP "[@" XML_LRM_ATTR_TASK "='", op, "'",
                    NULL);
 
     /* Need to check against transition_magic too? */
     if ((source != NULL) && (strcmp(op, PCMK_ACTION_MIGRATE_TO) == 0)) {
         pcmk__g_strcat(xpath,
                        " and @" XML_LRM_ATTR_MIGRATE_TARGET "='", source, "']",
                        NULL);
 
     } else if ((source != NULL)
                && (strcmp(op, PCMK_ACTION_MIGRATE_FROM) == 0)) {
         pcmk__g_strcat(xpath,
                        " and @" XML_LRM_ATTR_MIGRATE_SOURCE "='", source, "']",
                        NULL);
     } else {
         g_string_append_c(xpath, ']');
     }
 
     xml = get_xpath_object((const char *) xpath->str, data_set->input,
                            LOG_DEBUG);
     g_string_free(xpath, TRUE);
 
     if (xml && target_rc >= 0) {
         int rc = PCMK_OCF_UNKNOWN_ERROR;
         int status = PCMK_EXEC_ERROR;
 
         crm_element_value_int(xml, XML_LRM_ATTR_RC, &rc);
         crm_element_value_int(xml, XML_LRM_ATTR_OPSTATUS, &status);
         if ((rc != target_rc) || (status != PCMK_EXEC_DONE)) {
             return NULL;
         }
     }
     return xml;
 }
 
 static xmlNode *
 find_lrm_resource(const char *rsc_id, const char *node_name,
                   pe_working_set_t *data_set)
 {
     GString *xpath = NULL;
     xmlNode *xml = NULL;
 
     CRM_CHECK((rsc_id != NULL) && (node_name != NULL), return NULL);
 
     xpath = g_string_sized_new(256);
     pcmk__g_strcat(xpath,
                    XPATH_NODE_STATE "[@" XML_ATTR_UNAME "='", node_name, "']"
                    SUB_XPATH_LRM_RESOURCE "[@" XML_ATTR_ID "='", rsc_id, "']",
                    NULL);
 
     xml = get_xpath_object((const char *) xpath->str, data_set->input,
                            LOG_DEBUG);
 
     g_string_free(xpath, TRUE);
     return xml;
 }
 
 /*!
  * \internal
  * \brief Check whether a resource has no completed action history on a node
  *
  * \param[in,out] rsc        Resource to check
  * \param[in]     node_name  Node to check
  *
  * \return true if \p rsc_id is unknown on \p node_name, otherwise false
  */
 static bool
 unknown_on_node(pe_resource_t *rsc, const char *node_name)
 {
     bool result = false;
     xmlXPathObjectPtr search;
     GString *xpath = g_string_sized_new(256);
 
     pcmk__g_strcat(xpath,
                    XPATH_NODE_STATE "[@" XML_ATTR_UNAME "='", node_name, "']"
                    SUB_XPATH_LRM_RESOURCE "[@" XML_ATTR_ID "='", rsc->id, "']"
                    SUB_XPATH_LRM_RSC_OP "[@" XML_LRM_ATTR_RC "!='193']",
                    NULL);
     search = xpath_search(rsc->cluster->input, (const char *) xpath->str);
     result = (numXpathResults(search) == 0);
     freeXpathObject(search);
     g_string_free(xpath, TRUE);
     return result;
 }
 
 /*!
  * \brief Check whether a probe/monitor indicating the resource was not running
  * on a node happened after some event
  *
  * \param[in]     rsc_id     Resource being checked
  * \param[in]     node_name  Node being checked
  * \param[in]     xml_op     Event that monitor is being compared to
  * \param[in]     same_node  Whether the operations are on the same node
  * \param[in,out] data_set   Cluster working set
  *
  * \return true if such a monitor happened after event, false otherwise
  */
 static bool
 monitor_not_running_after(const char *rsc_id, const char *node_name,
                           const xmlNode *xml_op, bool same_node,
                           pe_working_set_t *data_set)
 {
     /* Any probe/monitor operation on the node indicating it was not running
      * there
      */
     xmlNode *monitor = find_lrm_op(rsc_id, PCMK_ACTION_MONITOR, node_name,
                                    NULL, PCMK_OCF_NOT_RUNNING, data_set);
 
     return (monitor && pe__is_newer_op(monitor, xml_op, same_node) > 0);
 }
 
 /*!
  * \brief Check whether any non-monitor operation on a node happened after some
  * event
  *
  * \param[in]     rsc_id    Resource being checked
  * \param[in]     node_name Node being checked
  * \param[in]     xml_op    Event that non-monitor is being compared to
  * \param[in]     same_node Whether the operations are on the same node
  * \param[in,out] data_set  Cluster working set
  *
  * \return true if such a operation happened after event, false otherwise
  */
 static bool
 non_monitor_after(const char *rsc_id, const char *node_name,
                   const xmlNode *xml_op, bool same_node,
                   pe_working_set_t *data_set)
 {
     xmlNode *lrm_resource = NULL;
 
     lrm_resource = find_lrm_resource(rsc_id, node_name, data_set);
     if (lrm_resource == NULL) {
         return false;
     }
 
     for (xmlNode *op = first_named_child(lrm_resource, XML_LRM_TAG_RSC_OP);
          op != NULL; op = crm_next_same_xml(op)) {
         const char * task = NULL;
 
         if (op == xml_op) {
             continue;
         }
 
         task = crm_element_value(op, XML_LRM_ATTR_TASK);
 
         if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_STOP,
                              PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
                              NULL)
             && pe__is_newer_op(op, xml_op, same_node) > 0) {
             return true;
         }
     }
 
     return false;
 }
 
 /*!
  * \brief Check whether the resource has newer state on a node after a migration
  * attempt
  *
  * \param[in]     rsc_id       Resource being checked
  * \param[in]     node_name    Node being checked
  * \param[in]     migrate_to   Any migrate_to event that is being compared to
  * \param[in]     migrate_from Any migrate_from event that is being compared to
  * \param[in,out] data_set     Cluster working set
  *
  * \return true if such a operation happened after event, false otherwise
  */
 static bool
 newer_state_after_migrate(const char *rsc_id, const char *node_name,
                           const xmlNode *migrate_to,
                           const xmlNode *migrate_from,
                           pe_working_set_t *data_set)
 {
     const xmlNode *xml_op = migrate_to;
     const char *source = NULL;
     const char *target = NULL;
     bool same_node = false;
 
     if (migrate_from) {
         xml_op = migrate_from;
     }
 
     source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
     target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
     /* It's preferred to compare to the migrate event on the same node if
      * existing, since call ids are more reliable.
      */
     if (pcmk__str_eq(node_name, target, pcmk__str_casei)) {
         if (migrate_from) {
            xml_op = migrate_from;
            same_node = true;
 
         } else {
            xml_op = migrate_to;
         }
 
     } else if (pcmk__str_eq(node_name, source, pcmk__str_casei)) {
         if (migrate_to) {
            xml_op = migrate_to;
            same_node = true;
 
         } else {
            xml_op = migrate_from;
         }
     }
 
     /* If there's any newer non-monitor operation on the node, or any newer
      * probe/monitor operation on the node indicating it was not running there,
      * the migration events potentially no longer matter for the node.
      */
     return non_monitor_after(rsc_id, node_name, xml_op, same_node, data_set)
            || monitor_not_running_after(rsc_id, node_name, xml_op, same_node,
                                         data_set);
 }
 
 /*!
  * \internal
  * \brief Parse migration source and target node names from history entry
  *
  * \param[in]  entry        Resource history entry for a migration action
  * \param[in]  source_node  If not NULL, source must match this node
  * \param[in]  target_node  If not NULL, target must match this node
  * \param[out] source_name  Where to store migration source node name
  * \param[out] target_name  Where to store migration target node name
  *
  * \return Standard Pacemaker return code
  */
 static int
 get_migration_node_names(const xmlNode *entry, const pe_node_t *source_node,
                          const pe_node_t *target_node,
                          const char **source_name, const char **target_name)
 {
     *source_name = crm_element_value(entry, XML_LRM_ATTR_MIGRATE_SOURCE);
     *target_name = crm_element_value(entry, XML_LRM_ATTR_MIGRATE_TARGET);
     if ((*source_name == NULL) || (*target_name == NULL)) {
         crm_err("Ignoring resource history entry %s without "
                 XML_LRM_ATTR_MIGRATE_SOURCE " and " XML_LRM_ATTR_MIGRATE_TARGET,
                 ID(entry));
         return pcmk_rc_unpack_error;
     }
 
     if ((source_node != NULL)
         && !pcmk__str_eq(*source_name, source_node->details->uname,
                          pcmk__str_casei|pcmk__str_null_matches)) {
         crm_err("Ignoring resource history entry %s because "
                 XML_LRM_ATTR_MIGRATE_SOURCE "='%s' does not match %s",
                 ID(entry), *source_name, pe__node_name(source_node));
         return pcmk_rc_unpack_error;
     }
 
     if ((target_node != NULL)
         && !pcmk__str_eq(*target_name, target_node->details->uname,
                          pcmk__str_casei|pcmk__str_null_matches)) {
         crm_err("Ignoring resource history entry %s because "
                 XML_LRM_ATTR_MIGRATE_TARGET "='%s' does not match %s",
                 ID(entry), *target_name, pe__node_name(target_node));
         return pcmk_rc_unpack_error;
     }
 
     return pcmk_rc_ok;
 }
 
 /*
  * \internal
  * \brief Add a migration source to a resource's list of dangling migrations
  *
  * If the migrate_to and migrate_from actions in a live migration both
  * succeeded, but there is no stop on the source, the migration is considered
  * "dangling." Add the source to the resource's dangling migration list, which
  * will be used to schedule a stop on the source without affecting the target.
  *
  * \param[in,out] rsc   Resource involved in migration
  * \param[in]     node  Migration source
  */
 static void
 add_dangling_migration(pe_resource_t *rsc, const pe_node_t *node)
 {
     pe_rsc_trace(rsc, "Dangling migration of %s requires stop on %s",
                  rsc->id, pe__node_name(node));
     rsc->role = pcmk_role_stopped;
     rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations,
                                               (gpointer) node);
 }
 
 /*!
  * \internal
  * \brief Update resource role etc. after a successful migrate_to action
  *
  * \param[in,out] history  Parsed action result history
  */
 static void
 unpack_migrate_to_success(struct action_history *history)
 {
     /* A complete migration sequence is:
      * 1. migrate_to on source node (which succeeded if we get to this function)
      * 2. migrate_from on target node
      * 3. stop on source node
      *
      * If no migrate_from has happened, the migration is considered to be
      * "partial". If the migrate_from succeeded but no stop has happened, the
      * migration is considered to be "dangling".
      *
      * If a successful migrate_to and stop have happened on the source node, we
      * still need to check for a partial migration, due to scenarios (easier to
      * produce with batch-limit=1) like:
      *
      * - A resource is migrating from node1 to node2, and a migrate_to is
      *   initiated for it on node1.
      *
      * - node2 goes into standby mode while the migrate_to is pending, which
      *   aborts the transition.
      *
      * - Upon completion of the migrate_to, a new transition schedules a stop
      *   on both nodes and a start on node1.
      *
      * - If the new transition is aborted for any reason while the resource is
      *   stopping on node1, the transition after that stop completes will see
      *   the migrate_to and stop on the source, but it's still a partial
      *   migration, and the resource must be stopped on node2 because it is
      *   potentially active there due to the migrate_to.
      *
      *   We also need to take into account that either node's history may be
      *   cleared at any point in the migration process.
      */
     int from_rc = PCMK_OCF_OK;
     int from_status = PCMK_EXEC_PENDING;
     pe_node_t *target_node = NULL;
     xmlNode *migrate_from = NULL;
     const char *source = NULL;
     const char *target = NULL;
     bool source_newer_op = false;
     bool target_newer_state = false;
     bool active_on_target = false;
 
     // Get source and target node names from XML
     if (get_migration_node_names(history->xml, history->node, NULL, &source,
                                  &target) != pcmk_rc_ok) {
         return;
     }
 
     // Check for newer state on the source
     source_newer_op = non_monitor_after(history->rsc->id, source, history->xml,
                                         true, history->rsc->cluster);
 
     // Check for a migrate_from action from this source on the target
     migrate_from = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_FROM,
                                target, source, -1, history->rsc->cluster);
     if (migrate_from != NULL) {
         if (source_newer_op) {
             /* There's a newer non-monitor operation on the source and a
              * migrate_from on the target, so this migrate_to is irrelevant to
              * the resource's state.
              */
             return;
         }
         crm_element_value_int(migrate_from, XML_LRM_ATTR_RC, &from_rc);
         crm_element_value_int(migrate_from, XML_LRM_ATTR_OPSTATUS,
                               &from_status);
     }
 
     /* If the resource has newer state on both the source and target after the
      * migration events, this migrate_to is irrelevant to the resource's state.
      */
     target_newer_state = newer_state_after_migrate(history->rsc->id, target,
                                                    history->xml, migrate_from,
                                                    history->rsc->cluster);
     if (source_newer_op && target_newer_state) {
         return;
     }
 
     /* Check for dangling migration (migrate_from succeeded but stop not done).
      * We know there's no stop because we already returned if the target has a
      * migrate_from and the source has any newer non-monitor operation.
      */
     if ((from_rc == PCMK_OCF_OK) && (from_status == PCMK_EXEC_DONE)) {
         add_dangling_migration(history->rsc, history->node);
         return;
     }
 
     /* Without newer state, this migrate_to implies the resource is active.
      * (Clones are not allowed to migrate, so role can't be promoted.)
      */
     history->rsc->role = pcmk_role_started;
 
     target_node = pe_find_node(history->rsc->cluster->nodes, target);
     active_on_target = !target_newer_state && (target_node != NULL)
                        && target_node->details->online;
 
     if (from_status != PCMK_EXEC_PENDING) { // migrate_from failed on target
         if (active_on_target) {
             native_add_running(history->rsc, target_node, history->rsc->cluster,
                                TRUE);
         } else {
             // Mark resource as failed, require recovery, and prevent migration
             pe__set_resource_flags(history->rsc, pe_rsc_failed|pe_rsc_stop);
             pe__clear_resource_flags(history->rsc, pe_rsc_allow_migrate);
         }
         return;
     }
 
     // The migrate_from is pending, complete but erased, or to be scheduled
 
     /* If there is no history at all for the resource on an online target, then
      * it was likely cleaned. Just return, and we'll schedule a probe. Once we
      * have the probe result, it will be reflected in target_newer_state.
      */
     if ((target_node != NULL) && target_node->details->online
         && unknown_on_node(history->rsc, target)) {
         return;
     }
 
     if (active_on_target) {
         pe_node_t *source_node = pe_find_node(history->rsc->cluster->nodes,
                                               source);
 
         native_add_running(history->rsc, target_node, history->rsc->cluster,
                            FALSE);
         if ((source_node != NULL) && source_node->details->online) {
             /* This is a partial migration: the migrate_to completed
              * successfully on the source, but the migrate_from has not
              * completed. Remember the source and target; if the newly
              * chosen target remains the same when we schedule actions
              * later, we may continue with the migration.
              */
             history->rsc->partial_migration_target = target_node;
             history->rsc->partial_migration_source = source_node;
         }
 
     } else if (!source_newer_op) {
         // Mark resource as failed, require recovery, and prevent migration
         pe__set_resource_flags(history->rsc, pe_rsc_failed|pe_rsc_stop);
         pe__clear_resource_flags(history->rsc, pe_rsc_allow_migrate);
     }
 }
 
 /*!
  * \internal
  * \brief Update resource role etc. after a failed migrate_to action
  *
  * \param[in,out] history  Parsed action result history
  */
 static void
 unpack_migrate_to_failure(struct action_history *history)
 {
     xmlNode *target_migrate_from = NULL;
     const char *source = NULL;
     const char *target = NULL;
 
     // Get source and target node names from XML
     if (get_migration_node_names(history->xml, history->node, NULL, &source,
                                  &target) != pcmk_rc_ok) {
         return;
     }
 
     /* If a migration failed, we have to assume the resource is active. Clones
      * are not allowed to migrate, so role can't be promoted.
      */
     history->rsc->role = pcmk_role_started;
 
     // Check for migrate_from on the target
     target_migrate_from = find_lrm_op(history->rsc->id,
                                       PCMK_ACTION_MIGRATE_FROM, target, source,
                                       PCMK_OCF_OK, history->rsc->cluster);
 
     if (/* If the resource state is unknown on the target, it will likely be
          * probed there.
          * Don't just consider it running there. We will get back here anyway in
          * case the probe detects it's running there.
          */
         !unknown_on_node(history->rsc, target)
         /* If the resource has newer state on the target after the migration
          * events, this migrate_to no longer matters for the target.
          */
         && !newer_state_after_migrate(history->rsc->id, target, history->xml,
                                       target_migrate_from,
                                       history->rsc->cluster)) {
         /* The resource has no newer state on the target, so assume it's still
          * active there.
          * (if it is up).
          */
         pe_node_t *target_node = pe_find_node(history->rsc->cluster->nodes,
                                               target);
 
         if (target_node && target_node->details->online) {
             native_add_running(history->rsc, target_node, history->rsc->cluster,
                                FALSE);
         }
 
     } else if (!non_monitor_after(history->rsc->id, source, history->xml, true,
                                   history->rsc->cluster)) {
         /* We know the resource has newer state on the target, but this
          * migrate_to still matters for the source as long as there's no newer
          * non-monitor operation there.
          */
 
         // Mark node as having dangling migration so we can force a stop later
         history->rsc->dangling_migrations =
             g_list_prepend(history->rsc->dangling_migrations,
                            (gpointer) history->node);
     }
 }
 
 /*!
  * \internal
  * \brief Update resource role etc. after a failed migrate_from action
  *
  * \param[in,out] history  Parsed action result history
  */
 static void
 unpack_migrate_from_failure(struct action_history *history)
 {
     xmlNode *source_migrate_to = NULL;
     const char *source = NULL;
     const char *target = NULL;
 
     // Get source and target node names from XML
     if (get_migration_node_names(history->xml, NULL, history->node, &source,
                                  &target) != pcmk_rc_ok) {
         return;
     }
 
     /* If a migration failed, we have to assume the resource is active. Clones
      * are not allowed to migrate, so role can't be promoted.
      */
     history->rsc->role = pcmk_role_started;
 
     // Check for a migrate_to on the source
     source_migrate_to = find_lrm_op(history->rsc->id, PCMK_ACTION_MIGRATE_TO,
                                     source, target, PCMK_OCF_OK,
                                     history->rsc->cluster);
 
     if (/* If the resource state is unknown on the source, it will likely be
          * probed there.
          * Don't just consider it running there. We will get back here anyway in
          * case the probe detects it's running there.
          */
         !unknown_on_node(history->rsc, source)
         /* If the resource has newer state on the source after the migration
          * events, this migrate_from no longer matters for the source.
          */
         && !newer_state_after_migrate(history->rsc->id, source,
                                       source_migrate_to, history->xml,
                                       history->rsc->cluster)) {
         /* The resource has no newer state on the source, so assume it's still
          * active there (if it is up).
          */
         pe_node_t *source_node = pe_find_node(history->rsc->cluster->nodes,
                                               source);
 
         if (source_node && source_node->details->online) {
             native_add_running(history->rsc, source_node, history->rsc->cluster,
                                TRUE);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Add an action to cluster's list of failed actions
  *
  * \param[in,out] history  Parsed action result history
  */
 static void
 record_failed_op(struct action_history *history)
 {
     if (!(history->node->details->online)) {
         return;
     }
 
     for (const xmlNode *xIter = history->rsc->cluster->failed->children;
          xIter != NULL; xIter = xIter->next) {
 
         const char *key = pe__xe_history_key(xIter);
         const char *uname = crm_element_value(xIter, XML_ATTR_UNAME);
 
         if (pcmk__str_eq(history->key, key, pcmk__str_none)
             && pcmk__str_eq(uname, history->node->details->uname,
                             pcmk__str_casei)) {
             crm_trace("Skipping duplicate entry %s on %s",
                       history->key, pe__node_name(history->node));
             return;
         }
     }
 
     crm_trace("Adding entry for %s on %s to failed action list",
               history->key, pe__node_name(history->node));
     crm_xml_add(history->xml, XML_ATTR_UNAME, history->node->details->uname);
     crm_xml_add(history->xml, XML_LRM_ATTR_RSCID, history->rsc->id);
     add_node_copy(history->rsc->cluster->failed, history->xml);
 }
 
 static char *
 last_change_str(const xmlNode *xml_op)
 {
     time_t when;
     char *result = NULL;
 
     if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
                                 &when) == pcmk_ok) {
         char *when_s = pcmk__epoch2str(&when, 0);
         const char *p = strchr(when_s, ' ');
 
         // Skip day of week to make message shorter
         if ((p != NULL) && (*(++p) != '\0')) {
             result = strdup(p);
             CRM_ASSERT(result != NULL);
         }
         free(when_s);
     }
 
     if (result == NULL) {
         result = strdup("unknown time");
         CRM_ASSERT(result != NULL);
     }
 
     return result;
 }
 
 /*!
  * \internal
  * \brief Compare two on-fail values
  *
  * \param[in] first   One on-fail value to compare
  * \param[in] second  The other on-fail value to compare
  *
  * \return A negative number if second is more severe than first, zero if they
  *         are equal, or a positive number if first is more severe than second.
  * \note This is only needed until the action_fail_response values can be
  *       renumbered at the next API compatibility break.
  */
 static int
 cmp_on_fail(enum action_fail_response first, enum action_fail_response second)
 {
     switch (first) {
         case pcmk_on_fail_demote:
             switch (second) {
                 case pcmk_on_fail_ignore:
                     return 1;
                 case pcmk_on_fail_demote:
                     return 0;
                 default:
                     return -1;
             }
             break;
 
         case pcmk_on_fail_reset_remote:
             switch (second) {
                 case pcmk_on_fail_ignore:
                 case pcmk_on_fail_demote:
                 case pcmk_on_fail_restart:
                     return 1;
                 case pcmk_on_fail_reset_remote:
                     return 0;
                 default:
                     return -1;
             }
             break;
 
         case pcmk_on_fail_restart_container:
             switch (second) {
                 case pcmk_on_fail_ignore:
                 case pcmk_on_fail_demote:
                 case pcmk_on_fail_restart:
                 case pcmk_on_fail_reset_remote:
                     return 1;
                 case pcmk_on_fail_restart_container:
                     return 0;
                 default:
                     return -1;
             }
             break;
 
         default:
             break;
     }
     switch (second) {
         case pcmk_on_fail_demote:
             return (first == pcmk_on_fail_ignore)? -1 : 1;
 
         case pcmk_on_fail_reset_remote:
             switch (first) {
                 case pcmk_on_fail_ignore:
                 case pcmk_on_fail_demote:
                 case pcmk_on_fail_restart:
                     return -1;
                 default:
                     return 1;
             }
             break;
 
         case pcmk_on_fail_restart_container:
             switch (first) {
                 case pcmk_on_fail_ignore:
                 case pcmk_on_fail_demote:
                 case pcmk_on_fail_restart:
                 case pcmk_on_fail_reset_remote:
                     return -1;
                 default:
                     return 1;
             }
             break;
 
         default:
             break;
     }
     return first - second;
 }
 
 /*!
  * \internal
  * \brief Ban a resource (or its clone if an anonymous instance) from all nodes
  *
  * \param[in,out] rsc  Resource to ban
  */
 static void
 ban_from_all_nodes(pe_resource_t *rsc)
 {
     int score = -INFINITY;
     pe_resource_t *fail_rsc = rsc;
 
     if (fail_rsc->parent != NULL) {
         pe_resource_t *parent = uber_parent(fail_rsc);
 
         if (pe_rsc_is_anon_clone(parent)) {
             /* For anonymous clones, if an operation with on-fail=stop fails for
              * any instance, the entire clone must stop.
              */
             fail_rsc = parent;
         }
     }
 
     // Ban the resource from all nodes
     crm_notice("%s will not be started under current conditions", fail_rsc->id);
     if (fail_rsc->allowed_nodes != NULL) {
         g_hash_table_destroy(fail_rsc->allowed_nodes);
     }
     fail_rsc->allowed_nodes = pe__node_list2table(rsc->cluster->nodes);
     g_hash_table_foreach(fail_rsc->allowed_nodes, set_node_score, &score);
 }
 
 /*!
  * \internal
  * \brief Update resource role, failure handling, etc., after a failed action
  *
  * \param[in,out] history       Parsed action result history
  * \param[out]    last_failure  Set this to action XML
  * \param[in,out] on_fail       What should be done about the result
  */
 static void
 unpack_rsc_op_failure(struct action_history *history, xmlNode **last_failure,
                       enum action_fail_response *on_fail)
 {
     bool is_probe = false;
     pe_action_t *action = NULL;
     char *last_change_s = NULL;
 
     *last_failure = history->xml;
 
     is_probe = pcmk_xe_is_probe(history->xml);
     last_change_s = last_change_str(history->xml);
 
     if (!pcmk_is_set(history->rsc->cluster->flags, pe_flag_symmetric_cluster)
         && (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
         crm_trace("Unexpected result (%s%s%s) was recorded for "
                   "%s of %s on %s at %s " CRM_XS " exit-status=%d id=%s",
                   services_ocf_exitcode_str(history->exit_status),
                   (pcmk__str_empty(history->exit_reason)? "" : ": "),
                   pcmk__s(history->exit_reason, ""),
                   (is_probe? "probe" : history->task), history->rsc->id,
                   pe__node_name(history->node), last_change_s,
                   history->exit_status, history->id);
     } else {
         crm_warn("Unexpected result (%s%s%s) was recorded for "
                   "%s of %s on %s at %s " CRM_XS " exit-status=%d id=%s",
                  services_ocf_exitcode_str(history->exit_status),
                  (pcmk__str_empty(history->exit_reason)? "" : ": "),
                  pcmk__s(history->exit_reason, ""),
                  (is_probe? "probe" : history->task), history->rsc->id,
                  pe__node_name(history->node), last_change_s,
                  history->exit_status, history->id);
 
         if (is_probe && (history->exit_status != PCMK_OCF_OK)
             && (history->exit_status != PCMK_OCF_NOT_RUNNING)
             && (history->exit_status != PCMK_OCF_RUNNING_PROMOTED)) {
 
             /* A failed (not just unexpected) probe result could mean the user
              * didn't know resources will be probed even where they can't run.
              */
             crm_notice("If it is not possible for %s to run on %s, see "
                        "the resource-discovery option for location constraints",
                        history->rsc->id, pe__node_name(history->node));
         }
 
         record_failed_op(history);
     }
 
     free(last_change_s);
 
     action = custom_action(history->rsc, strdup(history->key), history->task,
                            NULL, TRUE, FALSE, history->rsc->cluster);
     if (cmp_on_fail(*on_fail, action->on_fail) < 0) {
         pe_rsc_trace(history->rsc, "on-fail %s -> %s for %s (%s)",
                      fail2text(*on_fail), fail2text(action->on_fail),
                      action->uuid, history->key);
         *on_fail = action->on_fail;
     }
 
     if (strcmp(history->task, PCMK_ACTION_STOP) == 0) {
         resource_location(history->rsc, history->node, -INFINITY,
                           "__stop_fail__", history->rsc->cluster);
 
     } else if (strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0) {
         unpack_migrate_to_failure(history);
 
     } else if (strcmp(history->task, PCMK_ACTION_MIGRATE_FROM) == 0) {
         unpack_migrate_from_failure(history);
 
     } else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
         history->rsc->role = pcmk_role_promoted;
 
     } else if (strcmp(history->task, PCMK_ACTION_DEMOTE) == 0) {
         if (action->on_fail == pcmk_on_fail_block) {
             history->rsc->role = pcmk_role_promoted;
             pe__set_next_role(history->rsc, pcmk_role_stopped,
                               "demote with on-fail=block");
 
         } else if (history->exit_status == PCMK_OCF_NOT_RUNNING) {
             history->rsc->role = pcmk_role_stopped;
 
         } else {
             /* Staying in the promoted role would put the scheduler and
              * controller into a loop. Setting the role to unpromoted is not
              * dangerous because the resource will be stopped as part of
              * recovery, and any promotion will be ordered after that stop.
              */
             history->rsc->role = pcmk_role_unpromoted;
         }
     }
 
     if (is_probe && (history->exit_status == PCMK_OCF_NOT_INSTALLED)) {
         /* leave stopped */
         pe_rsc_trace(history->rsc, "Leaving %s stopped", history->rsc->id);
         history->rsc->role = pcmk_role_stopped;
 
     } else if (history->rsc->role < pcmk_role_started) {
         pe_rsc_trace(history->rsc, "Setting %s active", history->rsc->id);
         set_active(history->rsc);
     }
 
     pe_rsc_trace(history->rsc,
                  "Resource %s: role=%s, unclean=%s, on_fail=%s, fail_role=%s",
                  history->rsc->id, role2text(history->rsc->role),
                  pcmk__btoa(history->node->details->unclean),
                  fail2text(action->on_fail), role2text(action->fail_role));
 
     if ((action->fail_role != pcmk_role_started)
         && (history->rsc->next_role < action->fail_role)) {
         pe__set_next_role(history->rsc, action->fail_role, "failure");
     }
 
     if (action->fail_role == pcmk_role_stopped) {
         ban_from_all_nodes(history->rsc);
     }
 
     pe_free_action(action);
 }
 
 /*!
  * \internal
  * \brief Block a resource with a failed action if it cannot be recovered
  *
  * If resource action is a failed stop and fencing is not possible, mark the
  * resource as unmanaged and blocked, since recovery cannot be done.
  *
  * \param[in,out] history  Parsed action history entry
  */
 static void
 block_if_unrecoverable(struct action_history *history)
 {
     char *last_change_s = NULL;
 
     if (strcmp(history->task, PCMK_ACTION_STOP) != 0) {
         return; // All actions besides stop are always recoverable
     }
     if (pe_can_fence(history->node->details->data_set, history->node)) {
         return; // Failed stops are recoverable via fencing
     }
 
     last_change_s = last_change_str(history->xml);
     pe_proc_err("No further recovery can be attempted for %s "
                 "because %s on %s failed (%s%s%s) at %s "
                 CRM_XS " rc=%d id=%s",
                 history->rsc->id, history->task, pe__node_name(history->node),
                 services_ocf_exitcode_str(history->exit_status),
                 (pcmk__str_empty(history->exit_reason)? "" : ": "),
                 pcmk__s(history->exit_reason, ""),
                 last_change_s, history->exit_status, history->id);
 
     free(last_change_s);
 
     pe__clear_resource_flags(history->rsc, pe_rsc_managed);
     pe__set_resource_flags(history->rsc, pe_rsc_block);
 }
 
 /*!
  * \internal
  * \brief Update action history's execution status and why
  *
  * \param[in,out] history  Parsed action history entry
  * \param[out]    why      Where to store reason for update
  * \param[in]     value    New value
  * \param[in]     reason   Description of why value was changed
  */
 static inline void
 remap_because(struct action_history *history, const char **why, int value,
               const char *reason)
 {
     if (history->execution_status != value) {
         history->execution_status = value;
         *why = reason;
     }
 }
 
 /*!
  * \internal
  * \brief Remap informational monitor results and operation status
  *
  * For the monitor results, certain OCF codes are for providing extended information
  * to the user about services that aren't yet failed but not entirely healthy either.
  * These must be treated as the "normal" result by Pacemaker.
  *
  * For operation status, the action result can be used to determine an appropriate
  * status for the purposes of responding to the action.  The status provided by the
  * executor is not directly usable since the executor does not know what was expected.
  *
  * \param[in,out] history  Parsed action history entry
  * \param[in,out] on_fail  What should be done about the result
  * \param[in]     expired  Whether result is expired
  *
  * \note If the result is remapped and the node is not shutting down or failed,
  *       the operation will be recorded in the data set's list of failed operations
  *       to highlight it for the user.
  *
  * \note This may update the resource's current and next role.
  */
 static void
 remap_operation(struct action_history *history,
                 enum action_fail_response *on_fail, bool expired)
 {
     bool is_probe = false;
     int orig_exit_status = history->exit_status;
     int orig_exec_status = history->execution_status;
     const char *why = NULL;
     const char *task = history->task;
 
     // Remap degraded results to their successful counterparts
     history->exit_status = pcmk__effective_rc(history->exit_status);
     if (history->exit_status != orig_exit_status) {
         why = "degraded result";
         if (!expired && (!history->node->details->shutdown
                          || history->node->details->online)) {
             record_failed_op(history);
         }
     }
 
     if (!pe_rsc_is_bundled(history->rsc)
         && pcmk_xe_mask_probe_failure(history->xml)
         && ((history->execution_status != PCMK_EXEC_DONE)
             || (history->exit_status != PCMK_OCF_NOT_RUNNING))) {
         history->execution_status = PCMK_EXEC_DONE;
         history->exit_status = PCMK_OCF_NOT_RUNNING;
         why = "equivalent probe result";
     }
 
     /* If the executor reported an execution status of anything but done or
      * error, consider that final. But for done or error, we know better whether
      * it should be treated as a failure or not, because we know the expected
      * result.
      */
     switch (history->execution_status) {
         case PCMK_EXEC_DONE:
         case PCMK_EXEC_ERROR:
             break;
 
         // These should be treated as node-fatal
         case PCMK_EXEC_NO_FENCE_DEVICE:
         case PCMK_EXEC_NO_SECRETS:
             remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
                           "node-fatal error");
             goto remap_done;
 
         default:
             goto remap_done;
     }
 
     is_probe = pcmk_xe_is_probe(history->xml);
     if (is_probe) {
         task = "probe";
     }
 
     if (history->expected_exit_status < 0) {
         /* Pre-1.0 Pacemaker versions, and Pacemaker 1.1.6 or earlier with
          * Heartbeat 2.0.7 or earlier as the cluster layer, did not include the
          * expected exit status in the transition key, which (along with the
          * similar case of a corrupted transition key in the CIB) will be
          * reported to this function as -1. Pacemaker 2.0+ does not support
          * rolling upgrades from those versions or processing of saved CIB files
          * from those versions, so we do not need to care much about this case.
          */
         remap_because(history, &why, PCMK_EXEC_ERROR,
                       "obsolete history format");
         crm_warn("Expected result not found for %s on %s "
                  "(corrupt or obsolete CIB?)",
                  history->key, pe__node_name(history->node));
 
     } else if (history->exit_status == history->expected_exit_status) {
         remap_because(history, &why, PCMK_EXEC_DONE, "expected result");
 
     } else {
         remap_because(history, &why, PCMK_EXEC_ERROR, "unexpected result");
         pe_rsc_debug(history->rsc,
                      "%s on %s: expected %d (%s), got %d (%s%s%s)",
                      history->key, pe__node_name(history->node),
                      history->expected_exit_status,
                      services_ocf_exitcode_str(history->expected_exit_status),
                      history->exit_status,
                      services_ocf_exitcode_str(history->exit_status),
                      (pcmk__str_empty(history->exit_reason)? "" : ": "),
                      pcmk__s(history->exit_reason, ""));
     }
 
     switch (history->exit_status) {
         case PCMK_OCF_OK:
             if (is_probe
                 && (history->expected_exit_status == PCMK_OCF_NOT_RUNNING)) {
                 char *last_change_s = last_change_str(history->xml);
 
                 remap_because(history, &why, PCMK_EXEC_DONE, "probe");
                 pe_rsc_info(history->rsc, "Probe found %s active on %s at %s",
                             history->rsc->id, pe__node_name(history->node),
                             last_change_s);
                 free(last_change_s);
             }
             break;
 
         case PCMK_OCF_NOT_RUNNING:
             if (is_probe
                 || (history->expected_exit_status == history->exit_status)
                 || !pcmk_is_set(history->rsc->flags, pe_rsc_managed)) {
 
                 /* For probes, recurring monitors for the Stopped role, and
                  * unmanaged resources, "not running" is not considered a
                  * failure.
                  */
                 remap_because(history, &why, PCMK_EXEC_DONE, "exit status");
                 history->rsc->role = pcmk_role_stopped;
                 *on_fail = pcmk_on_fail_ignore;
                 pe__set_next_role(history->rsc, pcmk_role_unknown,
                                   "not running");
             }
             break;
 
         case PCMK_OCF_RUNNING_PROMOTED:
             if (is_probe
                 && (history->exit_status != history->expected_exit_status)) {
                 char *last_change_s = last_change_str(history->xml);
 
                 remap_because(history, &why, PCMK_EXEC_DONE, "probe");
                 pe_rsc_info(history->rsc,
                             "Probe found %s active and promoted on %s at %s",
                             history->rsc->id, pe__node_name(history->node),
                             last_change_s);
                 free(last_change_s);
             }
             if (!expired
                 || (history->exit_status == history->expected_exit_status)) {
                 history->rsc->role = pcmk_role_promoted;
             }
             break;
 
         case PCMK_OCF_FAILED_PROMOTED:
             if (!expired) {
                 history->rsc->role = pcmk_role_promoted;
             }
             remap_because(history, &why, PCMK_EXEC_ERROR, "exit status");
             break;
 
         case PCMK_OCF_NOT_CONFIGURED:
             remap_because(history, &why, PCMK_EXEC_ERROR_FATAL, "exit status");
             break;
 
         case PCMK_OCF_UNIMPLEMENT_FEATURE:
             {
                 guint interval_ms = 0;
                 crm_element_value_ms(history->xml, XML_LRM_ATTR_INTERVAL_MS,
                                      &interval_ms);
 
                 if (interval_ms == 0) {
                     if (!expired) {
                         block_if_unrecoverable(history);
                     }
                     remap_because(history, &why, PCMK_EXEC_ERROR_HARD,
                                   "exit status");
                 } else {
                     remap_because(history, &why, PCMK_EXEC_NOT_SUPPORTED,
                                   "exit status");
                 }
             }
             break;
 
         case PCMK_OCF_NOT_INSTALLED:
         case PCMK_OCF_INVALID_PARAM:
         case PCMK_OCF_INSUFFICIENT_PRIV:
             if (!expired) {
                 block_if_unrecoverable(history);
             }
             remap_because(history, &why, PCMK_EXEC_ERROR_HARD, "exit status");
             break;
 
         default:
             if (history->execution_status == PCMK_EXEC_DONE) {
                 char *last_change_s = last_change_str(history->xml);
 
                 crm_info("Treating unknown exit status %d from %s of %s "
                          "on %s at %s as failure",
                          history->exit_status, task, history->rsc->id,
                          pe__node_name(history->node), last_change_s);
                 remap_because(history, &why, PCMK_EXEC_ERROR,
                               "unknown exit status");
                 free(last_change_s);
             }
             break;
     }
 
 remap_done:
     if (why != NULL) {
         pe_rsc_trace(history->rsc,
                      "Remapped %s result from [%s: %s] to [%s: %s] "
                      "because of %s",
                      history->key, pcmk_exec_status_str(orig_exec_status),
                      crm_exit_str(orig_exit_status),
                      pcmk_exec_status_str(history->execution_status),
                      crm_exit_str(history->exit_status), why);
     }
 }
 
 // return TRUE if start or monitor last failure but parameters changed
 static bool
 should_clear_for_param_change(const xmlNode *xml_op, const char *task,
                               pe_resource_t *rsc, pe_node_t *node)
 {
     if (pcmk__str_any_of(task, PCMK_ACTION_START, PCMK_ACTION_MONITOR, NULL)) {
         if (pe__bundle_needs_remote_name(rsc)) {
             /* We haven't allocated resources yet, so we can't reliably
              * substitute addr parameters for the REMOTE_CONTAINER_HACK.
              * When that's needed, defer the check until later.
              */
             pe__add_param_check(xml_op, rsc, node, pcmk__check_last_failure,
                                 rsc->cluster);
 
         } else {
             op_digest_cache_t *digest_data = NULL;
 
             digest_data = rsc_action_digest_cmp(rsc, xml_op, node,
                                                 rsc->cluster);
             switch (digest_data->rc) {
                 case RSC_DIGEST_UNKNOWN:
                     crm_trace("Resource %s history entry %s on %s"
                               " has no digest to compare",
                               rsc->id, pe__xe_history_key(xml_op),
                               node->details->id);
                     break;
                 case RSC_DIGEST_MATCH:
                     break;
                 default:
                     return TRUE;
             }
         }
     }
     return FALSE;
 }
 
 // Order action after fencing of remote node, given connection rsc
 static void
 order_after_remote_fencing(pe_action_t *action, pe_resource_t *remote_conn,
                            pe_working_set_t *data_set)
 {
     pe_node_t *remote_node = pe_find_node(data_set->nodes, remote_conn->id);
 
     if (remote_node) {
         pe_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL,
                                          FALSE, data_set);
 
         order_actions(fence, action, pe_order_implies_then);
     }
 }
 
 static bool
 should_ignore_failure_timeout(const pe_resource_t *rsc, const char *task,
                               guint interval_ms, bool is_last_failure)
 {
     /* Clearing failures of recurring monitors has special concerns. The
      * executor reports only changes in the monitor result, so if the
      * monitor is still active and still getting the same failure result,
      * that will go undetected after the failure is cleared.
      *
      * Also, the operation history will have the time when the recurring
      * monitor result changed to the given code, not the time when the
      * result last happened.
      *
      * @TODO We probably should clear such failures only when the failure
      * timeout has passed since the last occurrence of the failed result.
      * However we don't record that information. We could maybe approximate
      * that by clearing only if there is a more recent successful monitor or
      * stop result, but we don't even have that information at this point
      * since we are still unpacking the resource's operation history.
      *
      * This is especially important for remote connection resources with a
      * reconnect interval, so in that case, we skip clearing failures
      * if the remote node hasn't been fenced.
      */
     if (rsc->remote_reconnect_ms
         && pcmk_is_set(rsc->cluster->flags, pe_flag_stonith_enabled)
         && (interval_ms != 0)
         && pcmk__str_eq(task, PCMK_ACTION_MONITOR, pcmk__str_casei)) {
 
         pe_node_t *remote_node = pe_find_node(rsc->cluster->nodes, rsc->id);
 
         if (remote_node && !remote_node->details->remote_was_fenced) {
             if (is_last_failure) {
                 crm_info("Waiting to clear monitor failure for remote node %s"
                          " until fencing has occurred", rsc->id);
             }
             return TRUE;
         }
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Check operation age and schedule failure clearing when appropriate
  *
  * This function has two distinct purposes. The first is to check whether an
  * operation history entry is expired (i.e. the resource has a failure timeout,
  * the entry is older than the timeout, and the resource either has no fail
  * count or its fail count is entirely older than the timeout). The second is to
  * schedule fail count clearing when appropriate (i.e. the operation is expired
  * and either the resource has an expired fail count or the operation is a
  * last_failure for a remote connection resource with a reconnect interval,
  * or the operation is a last_failure for a start or monitor operation and the
  * resource's parameters have changed since the operation).
  *
  * \param[in,out] history  Parsed action result history
  *
  * \return true if operation history entry is expired, otherwise false
  */
 static bool
 check_operation_expiry(struct action_history *history)
 {
     bool expired = false;
     bool is_last_failure = pcmk__ends_with(history->id, "_last_failure_0");
     time_t last_run = 0;
     int unexpired_fail_count = 0;
     const char *clear_reason = NULL;
 
     if (history->execution_status == PCMK_EXEC_NOT_INSTALLED) {
         pe_rsc_trace(history->rsc,
                      "Resource history entry %s on %s is not expired: "
                      "Not Installed does not expire",
                      history->id, pe__node_name(history->node));
         return false; // "Not installed" must always be cleared manually
     }
 
     if ((history->rsc->failure_timeout > 0)
         && (crm_element_value_epoch(history->xml, XML_RSC_OP_LAST_CHANGE,
                                     &last_run) == 0)) {
 
         // Resource has a failure-timeout, and history entry has a timestamp
 
         time_t now = get_effective_time(history->rsc->cluster);
         time_t last_failure = 0;
 
         // Is this particular operation history older than the failure timeout?
         if ((now >= (last_run + history->rsc->failure_timeout))
             && !should_ignore_failure_timeout(history->rsc, history->task,
                                               history->interval_ms,
                                               is_last_failure)) {
             expired = true;
         }
 
         // Does the resource as a whole have an unexpired fail count?
         unexpired_fail_count = pe_get_failcount(history->node, history->rsc,
                                                 &last_failure, pe_fc_effective,
                                                 history->xml);
 
         // Update scheduler recheck time according to *last* failure
         crm_trace("%s@%lld is %sexpired @%lld with unexpired_failures=%d timeout=%ds"
                   " last-failure@%lld",
                   history->id, (long long) last_run, (expired? "" : "not "),
                   (long long) now, unexpired_fail_count,
                   history->rsc->failure_timeout, (long long) last_failure);
         last_failure += history->rsc->failure_timeout + 1;
         if (unexpired_fail_count && (now < last_failure)) {
             pe__update_recheck_time(last_failure, history->rsc->cluster);
         }
     }
 
     if (expired) {
         if (pe_get_failcount(history->node, history->rsc, NULL, pe_fc_default,
                              history->xml)) {
             // There is a fail count ignoring timeout
 
             if (unexpired_fail_count == 0) {
                 // There is no fail count considering timeout
                 clear_reason = "it expired";
 
             } else {
                 /* This operation is old, but there is an unexpired fail count.
                  * In a properly functioning cluster, this should only be
                  * possible if this operation is not a failure (otherwise the
                  * fail count should be expired too), so this is really just a
                  * failsafe.
                  */
                 pe_rsc_trace(history->rsc,
                              "Resource history entry %s on %s is not expired: "
                              "Unexpired fail count",
                              history->id, pe__node_name(history->node));
                 expired = false;
             }
 
         } else if (is_last_failure
                    && (history->rsc->remote_reconnect_ms != 0)) {
             /* Clear any expired last failure when reconnect interval is set,
              * even if there is no fail count.
              */
             clear_reason = "reconnect interval is set";
         }
     }
 
     if (!expired && is_last_failure
         && should_clear_for_param_change(history->xml, history->task,
                                          history->rsc, history->node)) {
         clear_reason = "resource parameters have changed";
     }
 
     if (clear_reason != NULL) {
         // Schedule clearing of the fail count
         pe_action_t *clear_op = pe__clear_failcount(history->rsc, history->node,
                                                     clear_reason,
                                                     history->rsc->cluster);
 
         if (pcmk_is_set(history->rsc->cluster->flags, pe_flag_stonith_enabled)
             && (history->rsc->remote_reconnect_ms != 0)) {
             /* If we're clearing a remote connection due to a reconnect
              * interval, we want to wait until any scheduled fencing
              * completes.
              *
              * We could limit this to remote_node->details->unclean, but at
              * this point, that's always true (it won't be reliable until
              * after unpack_node_history() is done).
              */
             crm_info("Clearing %s failure will wait until any scheduled "
                      "fencing of %s completes",
                      history->task, history->rsc->id);
             order_after_remote_fencing(clear_op, history->rsc,
                                        history->rsc->cluster);
         }
     }
 
     if (expired && (history->interval_ms == 0)
         && pcmk__str_eq(history->task, PCMK_ACTION_MONITOR, pcmk__str_none)) {
         switch (history->exit_status) {
             case PCMK_OCF_OK:
             case PCMK_OCF_NOT_RUNNING:
             case PCMK_OCF_RUNNING_PROMOTED:
             case PCMK_OCF_DEGRADED:
             case PCMK_OCF_DEGRADED_PROMOTED:
                 // Don't expire probes that return these values
                 pe_rsc_trace(history->rsc,
                              "Resource history entry %s on %s is not expired: "
                              "Probe result",
                              history->id, pe__node_name(history->node));
                 expired = false;
                 break;
         }
     }
 
     return expired;
 }
 
 int
 pe__target_rc_from_xml(const xmlNode *xml_op)
 {
     int target_rc = 0;
     const char *key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);
 
     if (key == NULL) {
         return -1;
     }
     decode_transition_key(key, NULL, NULL, NULL, &target_rc);
     return target_rc;
 }
 
 /*!
  * \internal
  * \brief Get the failure handling for an action
  *
  * \param[in,out] history  Parsed action history entry
  *
  * \return Failure handling appropriate to action
  */
 static enum action_fail_response
 get_action_on_fail(struct action_history *history)
 {
     enum action_fail_response result = pcmk_on_fail_restart;
     pe_action_t *action = custom_action(history->rsc, strdup(history->key),
                                         history->task, NULL, TRUE, FALSE,
                                         history->rsc->cluster);
 
     result = action->on_fail;
     pe_free_action(action);
     return result;
 }
 
 /*!
  * \internal
  * \brief Update a resource's state for an action result
  *
  * \param[in,out] history       Parsed action history entry
  * \param[in]     exit_status   Exit status to base new state on
  * \param[in]     last_failure  Resource's last_failure entry, if known
  * \param[in,out] on_fail       Resource's current failure handling
  */
 static void
 update_resource_state(struct action_history *history, int exit_status,
                       const xmlNode *last_failure,
                       enum action_fail_response *on_fail)
 {
     bool clear_past_failure = false;
 
     if ((exit_status == PCMK_OCF_NOT_INSTALLED)
         || (!pe_rsc_is_bundled(history->rsc)
             && pcmk_xe_mask_probe_failure(history->xml))) {
         history->rsc->role = pcmk_role_stopped;
 
     } else if (exit_status == PCMK_OCF_NOT_RUNNING) {
         clear_past_failure = true;
 
     } else if (pcmk__str_eq(history->task, PCMK_ACTION_MONITOR,
                             pcmk__str_none)) {
         if ((last_failure != NULL)
             && pcmk__str_eq(history->key, pe__xe_history_key(last_failure),
                             pcmk__str_none)) {
             clear_past_failure = true;
         }
         if (history->rsc->role < pcmk_role_started) {
             set_active(history->rsc);
         }
 
     } else if (pcmk__str_eq(history->task, PCMK_ACTION_START, pcmk__str_none)) {
         history->rsc->role = pcmk_role_started;
         clear_past_failure = true;
 
     } else if (pcmk__str_eq(history->task, PCMK_ACTION_STOP, pcmk__str_none)) {
         history->rsc->role = pcmk_role_stopped;
         clear_past_failure = true;
 
     } else if (pcmk__str_eq(history->task, PCMK_ACTION_PROMOTE,
                             pcmk__str_none)) {
         history->rsc->role = pcmk_role_promoted;
         clear_past_failure = true;
 
     } else if (pcmk__str_eq(history->task, PCMK_ACTION_DEMOTE,
                             pcmk__str_none)) {
         if (*on_fail == pcmk_on_fail_demote) {
             // Demote clears an error only if on-fail=demote
             clear_past_failure = true;
         }
         history->rsc->role = pcmk_role_unpromoted;
 
     } else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_FROM,
                             pcmk__str_none)) {
         history->rsc->role = pcmk_role_started;
         clear_past_failure = true;
 
     } else if (pcmk__str_eq(history->task, PCMK_ACTION_MIGRATE_TO,
                             pcmk__str_none)) {
         unpack_migrate_to_success(history);
 
     } else if (history->rsc->role < pcmk_role_started) {
         pe_rsc_trace(history->rsc, "%s active on %s",
                      history->rsc->id, pe__node_name(history->node));
         set_active(history->rsc);
     }
 
     if (!clear_past_failure) {
         return;
     }
 
     switch (*on_fail) {
         case pcmk_on_fail_stop:
         case pcmk_on_fail_ban:
         case pcmk_on_fail_standby_node:
         case pcmk_on_fail_fence_node:
             pe_rsc_trace(history->rsc,
                          "%s (%s) is not cleared by a completed %s",
                          history->rsc->id, fail2text(*on_fail), history->task);
             break;
 
         case pcmk_on_fail_block:
         case pcmk_on_fail_ignore:
         case pcmk_on_fail_demote:
         case pcmk_on_fail_restart:
         case pcmk_on_fail_restart_container:
             *on_fail = pcmk_on_fail_ignore;
             pe__set_next_role(history->rsc, pcmk_role_unknown,
                               "clear past failures");
             break;
 
         case pcmk_on_fail_reset_remote:
             if (history->rsc->remote_reconnect_ms == 0) {
                 /* With no reconnect interval, the connection is allowed to
                  * start again after the remote node is fenced and
                  * completely stopped. (With a reconnect interval, we wait
                  * for the failure to be cleared entirely before attempting
                  * to reconnect.)
                  */
                 *on_fail = pcmk_on_fail_ignore;
                 pe__set_next_role(history->rsc, pcmk_role_unknown,
                                   "clear past failures and reset remote");
             }
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Check whether a given history entry matters for resource state
  *
  * \param[in] history  Parsed action history entry
  *
  * \return true if action can affect resource state, otherwise false
  */
 static inline bool
 can_affect_state(struct action_history *history)
 {
 #if 0
     /* @COMPAT It might be better to parse only actions we know we're interested
      * in, rather than exclude a couple we don't. However that would be a
      * behavioral change that should be done at a major or minor series release.
      * Currently, unknown operations can affect whether a resource is considered
      * active and/or failed.
      */
      return pcmk__str_any_of(history->task, PCMK_ACTION_MONITOR,
                              PCMK_ACTION_START, PCMK_ACTION_STOP,
                              PCMK_ACTION_PROMOTE, PCMK_ACTION_DEMOTE,
                              PCMK_ACTION_MIGRATE_TO, PCMK_ACTION_MIGRATE_FROM,
                              "asyncmon", NULL);
 #else
      return !pcmk__str_any_of(history->task, PCMK_ACTION_NOTIFY,
                               PCMK_ACTION_META_DATA, NULL);
 #endif
 }
 
 /*!
  * \internal
  * \brief Unpack execution/exit status and exit reason from a history entry
  *
  * \param[in,out] history  Action history entry to unpack
  *
  * \return Standard Pacemaker return code
  */
 static int
 unpack_action_result(struct action_history *history)
 {
     if ((crm_element_value_int(history->xml, XML_LRM_ATTR_OPSTATUS,
                                &(history->execution_status)) < 0)
         || (history->execution_status < PCMK_EXEC_PENDING)
         || (history->execution_status > PCMK_EXEC_MAX)
         || (history->execution_status == PCMK_EXEC_CANCELLED)) {
         crm_err("Ignoring resource history entry %s for %s on %s "
                 "with invalid " XML_LRM_ATTR_OPSTATUS " '%s'",
                 history->id, history->rsc->id, pe__node_name(history->node),
                 pcmk__s(crm_element_value(history->xml, XML_LRM_ATTR_OPSTATUS),
                         ""));
         return pcmk_rc_unpack_error;
     }
     if ((crm_element_value_int(history->xml, XML_LRM_ATTR_RC,
                                &(history->exit_status)) < 0)
         || (history->exit_status < 0) || (history->exit_status > CRM_EX_MAX)) {
 #if 0
         /* @COMPAT We should ignore malformed entries, but since that would
          * change behavior, it should be done at a major or minor series
          * release.
          */
         crm_err("Ignoring resource history entry %s for %s on %s "
                 "with invalid " XML_LRM_ATTR_RC " '%s'",
                 history->id, history->rsc->id, pe__node_name(history->node),
                 pcmk__s(crm_element_value(history->xml, XML_LRM_ATTR_RC),
                         ""));
         return pcmk_rc_unpack_error;
 #else
         history->exit_status = CRM_EX_ERROR;
 #endif
     }
     history->exit_reason = crm_element_value(history->xml,
                                              XML_LRM_ATTR_EXIT_REASON);
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Process an action history entry whose result expired
  *
  * \param[in,out] history           Parsed action history entry
  * \param[in]     orig_exit_status  Action exit status before remapping
  *
  * \return Standard Pacemaker return code (in particular, pcmk_rc_ok means the
  *         entry needs no further processing)
  */
 static int
 process_expired_result(struct action_history *history, int orig_exit_status)
 {
     if (!pe_rsc_is_bundled(history->rsc)
         && pcmk_xe_mask_probe_failure(history->xml)
         && (orig_exit_status != history->expected_exit_status)) {
 
         if (history->rsc->role <= pcmk_role_stopped) {
             history->rsc->role = pcmk_role_unknown;
         }
         crm_trace("Ignoring resource history entry %s for probe of %s on %s: "
                   "Masked failure expired",
                   history->id, history->rsc->id,
                   pe__node_name(history->node));
         return pcmk_rc_ok;
     }
 
     if (history->exit_status == history->expected_exit_status) {
         return pcmk_rc_undetermined; // Only failures expire
     }
 
     if (history->interval_ms == 0) {
         crm_notice("Ignoring resource history entry %s for %s of %s on %s: "
                    "Expired failure",
                    history->id, history->task, history->rsc->id,
                    pe__node_name(history->node));
         return pcmk_rc_ok;
     }
 
     if (history->node->details->online && !history->node->details->unclean) {
         /* Reschedule the recurring action. schedule_cancel() won't work at
          * this stage, so as a hacky workaround, forcibly change the restart
          * digest so pcmk__check_action_config() does what we want later.
          *
          * @TODO We should skip this if there is a newer successful monitor.
          *       Also, this causes rescheduling only if the history entry
          *       has an op-digest (which the expire-non-blocked-failure
          *       scheduler regression test doesn't, but that may not be a
          *       realistic scenario in production).
          */
         crm_notice("Rescheduling %s-interval %s of %s on %s "
                    "after failure expired",
                    pcmk__readable_interval(history->interval_ms), history->task,
                    history->rsc->id, pe__node_name(history->node));
         crm_xml_add(history->xml, XML_LRM_ATTR_RESTART_DIGEST,
                     "calculated-failure-timeout");
         return pcmk_rc_ok;
     }
 
     return pcmk_rc_undetermined;
 }
 
 /*!
  * \internal
  * \brief Process a masked probe failure
  *
  * \param[in,out] history           Parsed action history entry
  * \param[in]     orig_exit_status  Action exit status before remapping
  * \param[in]     last_failure      Resource's last_failure entry, if known
  * \param[in,out] on_fail           Resource's current failure handling
  */
 static void
 mask_probe_failure(struct action_history *history, int orig_exit_status,
                    const xmlNode *last_failure,
                    enum action_fail_response *on_fail)
 {
     pe_resource_t *ban_rsc = history->rsc;
 
     if (!pcmk_is_set(history->rsc->flags, pe_rsc_unique)) {
         ban_rsc = uber_parent(history->rsc);
     }
 
     crm_notice("Treating probe result '%s' for %s on %s as 'not running'",
                services_ocf_exitcode_str(orig_exit_status), history->rsc->id,
                pe__node_name(history->node));
     update_resource_state(history, history->expected_exit_status, last_failure,
                           on_fail);
     crm_xml_add(history->xml, XML_ATTR_UNAME, history->node->details->uname);
 
     record_failed_op(history);
     resource_location(ban_rsc, history->node, -INFINITY, "masked-probe-failure",
                       history->rsc->cluster);
 }
 
 /*!
  * \internal Check whether a given failure is for a given pending action
  *
  * \param[in] history       Parsed history entry for pending action
  * \param[in] last_failure  Resource's last_failure entry, if known
  *
  * \return true if \p last_failure is failure of pending action in \p history,
  *         otherwise false
  * \note Both \p history and \p last_failure must come from the same
  *       lrm_resource block, as node and resource are assumed to be the same.
  */
 static bool
 failure_is_newer(const struct action_history *history,
                  const xmlNode *last_failure)
 {
     guint failure_interval_ms = 0U;
     long long failure_change = 0LL;
     long long this_change = 0LL;
 
     if (last_failure == NULL) {
         return false; // Resource has no last_failure entry
     }
 
     if (!pcmk__str_eq(history->task,
                       crm_element_value(last_failure, XML_LRM_ATTR_TASK),
                       pcmk__str_none)) {
         return false; // last_failure is for different action
     }
 
     if ((crm_element_value_ms(last_failure, XML_LRM_ATTR_INTERVAL_MS,
                               &failure_interval_ms) != pcmk_ok)
         || (history->interval_ms != failure_interval_ms)) {
         return false; // last_failure is for action with different interval
     }
 
     if ((pcmk__scan_ll(crm_element_value(history->xml, XML_RSC_OP_LAST_CHANGE),
                        &this_change, 0LL) != pcmk_rc_ok)
         || (pcmk__scan_ll(crm_element_value(last_failure,
                                             XML_RSC_OP_LAST_CHANGE),
                           &failure_change, 0LL) != pcmk_rc_ok)
         || (failure_change < this_change)) {
         return false; // Failure is not known to be newer
     }
 
     return true;
 }
 
 /*!
  * \internal
  * \brief Update a resource's role etc. for a pending action
  *
  * \param[in,out] history       Parsed history entry for pending action
  * \param[in]     last_failure  Resource's last_failure entry, if known
  */
 static void
 process_pending_action(struct action_history *history,
                        const xmlNode *last_failure)
 {
     /* For recurring monitors, a failure is recorded only in RSC_last_failure_0,
      * and there might be a RSC_monitor_INTERVAL entry with the last successful
      * or pending result.
      *
      * If last_failure contains the failure of the pending recurring monitor
      * we're processing here, and is newer, the action is no longer pending.
      * (Pending results have call ID -1, which sorts last, so the last failure
      * if any should be known.)
      */
     if (failure_is_newer(history, last_failure)) {
         return;
     }
 
     if (strcmp(history->task, PCMK_ACTION_START) == 0) {
         pe__set_resource_flags(history->rsc, pe_rsc_start_pending);
         set_active(history->rsc);
 
     } else if (strcmp(history->task, PCMK_ACTION_PROMOTE) == 0) {
         history->rsc->role = pcmk_role_promoted;
 
     } else if ((strcmp(history->task, PCMK_ACTION_MIGRATE_TO) == 0)
                && history->node->details->unclean) {
         /* A migrate_to action is pending on a unclean source, so force a stop
          * on the target.
          */
         const char *migrate_target = NULL;
         pe_node_t *target = NULL;
 
         migrate_target = crm_element_value(history->xml,
                                            XML_LRM_ATTR_MIGRATE_TARGET);
         target = pe_find_node(history->rsc->cluster->nodes, migrate_target);
         if (target != NULL) {
             stop_action(history->rsc, target, FALSE);
         }
     }
 
     if (history->rsc->pending_task != NULL) {
         /* There should never be multiple pending actions, but as a failsafe,
          * just remember the first one processed for display purposes.
          */
         return;
     }
 
     if (pcmk_is_probe(history->task, history->interval_ms)) {
         /* Pending probes are currently never displayed, even if pending
          * operations are requested. If we ever want to change that,
          * enable the below and the corresponding part of
          * native.c:native_pending_task().
          */
 #if 0
         history->rsc->pending_task = strdup("probe");
         history->rsc->pending_node = history->node;
 #endif
     } else {
         history->rsc->pending_task = strdup(history->task);
         history->rsc->pending_node = history->node;
     }
 }
 
 static void
 unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
               xmlNode **last_failure, enum action_fail_response *on_fail)
 {
     int old_rc = 0;
     bool expired = false;
     pe_resource_t *parent = rsc;
     enum action_fail_response failure_strategy = pcmk_on_fail_restart;
 
     struct action_history history = {
         .rsc = rsc,
         .node = node,
         .xml = xml_op,
         .execution_status = PCMK_EXEC_UNKNOWN,
     };
 
     CRM_CHECK(rsc && node && xml_op, return);
 
     history.id = ID(xml_op);
     if (history.id == NULL) {
         crm_err("Ignoring resource history entry for %s on %s without ID",
                 rsc->id, pe__node_name(node));
         return;
     }
 
     // Task and interval
     history.task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     if (history.task == NULL) {
         crm_err("Ignoring resource history entry %s for %s on %s without "
                 XML_LRM_ATTR_TASK, history.id, rsc->id, pe__node_name(node));
         return;
     }
     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS,
                          &(history.interval_ms));
     if (!can_affect_state(&history)) {
         pe_rsc_trace(rsc,
                      "Ignoring resource history entry %s for %s on %s "
                      "with irrelevant action '%s'",
                      history.id, rsc->id, pe__node_name(node), history.task);
         return;
     }
 
     if (unpack_action_result(&history) != pcmk_rc_ok) {
         return; // Error already logged
     }
 
     history.expected_exit_status = pe__target_rc_from_xml(xml_op);
     history.key = pe__xe_history_key(xml_op);
     crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &(history.call_id));
 
     pe_rsc_trace(rsc, "Unpacking %s (%s call %d on %s): %s (%s)",
                  history.id, history.task, history.call_id, pe__node_name(node),
                  pcmk_exec_status_str(history.execution_status),
                  crm_exit_str(history.exit_status));
 
     if (node->details->unclean) {
         pe_rsc_trace(rsc,
                      "%s is running on %s, which is unclean (further action "
                      "depends on value of stop's on-fail attribute)",
                      rsc->id, pe__node_name(node));
     }
 
     expired = check_operation_expiry(&history);
     old_rc = history.exit_status;
 
     remap_operation(&history, on_fail, expired);
 
     if (expired && (process_expired_result(&history, old_rc) == pcmk_rc_ok)) {
         goto done;
     }
 
     if (!pe_rsc_is_bundled(rsc) && pcmk_xe_mask_probe_failure(xml_op)) {
         mask_probe_failure(&history, old_rc, *last_failure, on_fail);
         goto done;
     }
 
     if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
         parent = uber_parent(rsc);
     }
 
     switch (history.execution_status) {
         case PCMK_EXEC_PENDING:
             process_pending_action(&history, *last_failure);
             goto done;
 
         case PCMK_EXEC_DONE:
             update_resource_state(&history, history.exit_status, *last_failure,
                                   on_fail);
             goto done;
 
         case PCMK_EXEC_NOT_INSTALLED:
             failure_strategy = get_action_on_fail(&history);
             if (failure_strategy == pcmk_on_fail_ignore) {
                 crm_warn("Cannot ignore failed %s of %s on %s: "
                          "Resource agent doesn't exist "
                          CRM_XS " status=%d rc=%d id=%s",
                          history.task, rsc->id, pe__node_name(node),
                          history.execution_status, history.exit_status,
                          history.id);
                 /* Also for printing it as "FAILED" by marking it as pe_rsc_failed later */
                 *on_fail = pcmk_on_fail_ban;
             }
             resource_location(parent, node, -INFINITY, "hard-error",
                               rsc->cluster);
             unpack_rsc_op_failure(&history, last_failure, on_fail);
             goto done;
 
         case PCMK_EXEC_NOT_CONNECTED:
             if (pe__is_guest_or_remote_node(node)
                 && pcmk_is_set(node->details->remote_rsc->flags, pe_rsc_managed)) {
                 /* We should never get into a situation where a managed remote
                  * connection resource is considered OK but a resource action
                  * behind the connection gets a "not connected" status. But as a
                  * fail-safe in case a bug or unusual circumstances do lead to
                  * that, ensure the remote connection is considered failed.
                  */
                 pe__set_resource_flags(node->details->remote_rsc,
                                        pe_rsc_failed|pe_rsc_stop);
             }
             break; // Not done, do error handling
 
         case PCMK_EXEC_ERROR:
         case PCMK_EXEC_ERROR_HARD:
         case PCMK_EXEC_ERROR_FATAL:
         case PCMK_EXEC_TIMEOUT:
         case PCMK_EXEC_NOT_SUPPORTED:
         case PCMK_EXEC_INVALID:
             break; // Not done, do error handling
 
         default: // No other value should be possible at this point
             break;
     }
 
     failure_strategy = get_action_on_fail(&history);
     if ((failure_strategy == pcmk_on_fail_ignore)
         || ((failure_strategy == pcmk_on_fail_restart_container)
             && (strcmp(history.task, PCMK_ACTION_STOP) == 0))) {
 
         char *last_change_s = last_change_str(xml_op);
 
         crm_warn("Pretending failed %s (%s%s%s) of %s on %s at %s succeeded "
                  CRM_XS " %s",
                  history.task, services_ocf_exitcode_str(history.exit_status),
                  (pcmk__str_empty(history.exit_reason)? "" : ": "),
                  pcmk__s(history.exit_reason, ""), rsc->id, pe__node_name(node),
                  last_change_s, history.id);
         free(last_change_s);
 
         update_resource_state(&history, history.expected_exit_status,
                               *last_failure, on_fail);
         crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
         pe__set_resource_flags(rsc, pe_rsc_failure_ignored);
 
         record_failed_op(&history);
 
         if ((failure_strategy == pcmk_on_fail_restart_container)
             && cmp_on_fail(*on_fail, pcmk_on_fail_restart) <= 0) {
             *on_fail = failure_strategy;
         }
 
     } else {
         unpack_rsc_op_failure(&history, last_failure, on_fail);
 
         if (history.execution_status == PCMK_EXEC_ERROR_HARD) {
             uint8_t log_level = LOG_ERR;
 
             if (history.exit_status == PCMK_OCF_NOT_INSTALLED) {
                 log_level = LOG_NOTICE;
             }
             do_crm_log(log_level,
                        "Preventing %s from restarting on %s because "
                        "of hard failure (%s%s%s) " CRM_XS " %s",
                        parent->id, pe__node_name(node),
                        services_ocf_exitcode_str(history.exit_status),
                        (pcmk__str_empty(history.exit_reason)? "" : ": "),
                        pcmk__s(history.exit_reason, ""), history.id);
             resource_location(parent, node, -INFINITY, "hard-error",
                               rsc->cluster);
 
         } else if (history.execution_status == PCMK_EXEC_ERROR_FATAL) {
             crm_err("Preventing %s from restarting anywhere because "
                     "of fatal failure (%s%s%s) " CRM_XS " %s",
                     parent->id, services_ocf_exitcode_str(history.exit_status),
                     (pcmk__str_empty(history.exit_reason)? "" : ": "),
                     pcmk__s(history.exit_reason, ""), history.id);
             resource_location(parent, NULL, -INFINITY, "fatal-error",
                               rsc->cluster);
         }
     }
 
 done:
     pe_rsc_trace(rsc, "%s role on %s after %s is %s (next %s)",
                  rsc->id, pe__node_name(node), history.id,
                  role2text(rsc->role), role2text(rsc->next_role));
 }
 
 static void
 add_node_attrs(const xmlNode *xml_obj, pe_node_t *node, bool overwrite,
                pe_working_set_t *data_set)
 {
     const char *cluster_name = NULL;
 
     pe_rule_eval_data_t rule_data = {
         .node_hash = NULL,
         .role = pcmk_role_unknown,
         .now = data_set->now,
         .match_data = NULL,
         .rsc_data = NULL,
         .op_data = NULL
     };
 
     g_hash_table_insert(node->details->attrs,
                         strdup(CRM_ATTR_UNAME), strdup(node->details->uname));
 
     g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_ID),
                         strdup(node->details->id));
     if (pcmk__str_eq(node->details->id, data_set->dc_uuid, pcmk__str_casei)) {
         data_set->dc_node = node;
         node->details->is_dc = TRUE;
         g_hash_table_insert(node->details->attrs,
                             strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_TRUE));
     } else {
         g_hash_table_insert(node->details->attrs,
                             strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_FALSE));
     }
 
     cluster_name = g_hash_table_lookup(data_set->config_hash, "cluster-name");
     if (cluster_name) {
         g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_CLUSTER_NAME),
                             strdup(cluster_name));
     }
 
     pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_ATTR_SETS, &rule_data,
                                node->details->attrs, NULL, overwrite, data_set);
 
     pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_UTILIZATION, &rule_data,
                                node->details->utilization, NULL,
                                FALSE, data_set);
 
     if (pe_node_attribute_raw(node, CRM_ATTR_SITE_NAME) == NULL) {
         const char *site_name = pe_node_attribute_raw(node, "site-name");
 
         if (site_name) {
             g_hash_table_insert(node->details->attrs,
                                 strdup(CRM_ATTR_SITE_NAME),
                                 strdup(site_name));
 
         } else if (cluster_name) {
             /* Default to cluster-name if unset */
             g_hash_table_insert(node->details->attrs,
                                 strdup(CRM_ATTR_SITE_NAME),
                                 strdup(cluster_name));
         }
     }
 }
 
 static GList *
 extract_operations(const char *node, const char *rsc, xmlNode * rsc_entry, gboolean active_filter)
 {
     int counter = -1;
     int stop_index = -1;
     int start_index = -1;
 
     xmlNode *rsc_op = NULL;
 
     GList *gIter = NULL;
     GList *op_list = NULL;
     GList *sorted_op_list = NULL;
 
     /* extract operations */
     op_list = NULL;
     sorted_op_list = NULL;
 
     for (rsc_op = pcmk__xe_first_child(rsc_entry);
          rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op)) {
 
         if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP,
                          pcmk__str_none)) {
             crm_xml_add(rsc_op, "resource", rsc);
             crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
             op_list = g_list_prepend(op_list, rsc_op);
         }
     }
 
     if (op_list == NULL) {
         /* if there are no operations, there is nothing to do */
         return NULL;
     }
 
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 
     /* create active recurring operations as optional */
     if (active_filter == FALSE) {
         return sorted_op_list;
     }
 
     op_list = NULL;
 
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         counter++;
 
         if (start_index < stop_index) {
             crm_trace("Skipping %s: not active", ID(rsc_entry));
             break;
 
         } else if (counter < start_index) {
             crm_trace("Skipping %s: old", ID(rsc_op));
             continue;
         }
         op_list = g_list_append(op_list, rsc_op);
     }
 
     g_list_free(sorted_op_list);
     return op_list;
 }
 
 GList *
 find_operations(const char *rsc, const char *node, gboolean active_filter,
                 pe_working_set_t * data_set)
 {
     GList *output = NULL;
     GList *intermediate = NULL;
 
     xmlNode *tmp = NULL;
     xmlNode *status = find_xml_node(data_set->input, XML_CIB_TAG_STATUS, TRUE);
 
     pe_node_t *this_node = NULL;
 
     xmlNode *node_state = NULL;
 
     for (node_state = pcmk__xe_first_child(status); node_state != NULL;
          node_state = pcmk__xe_next(node_state)) {
 
         if (pcmk__str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, pcmk__str_none)) {
             const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);
 
             if (node != NULL && !pcmk__str_eq(uname, node, pcmk__str_casei)) {
                 continue;
             }
 
             this_node = pe_find_node(data_set->nodes, uname);
             if(this_node == NULL) {
                 CRM_LOG_ASSERT(this_node != NULL);
                 continue;
 
             } else if (pe__is_guest_or_remote_node(this_node)) {
                 determine_remote_online_status(data_set, this_node);
 
             } else {
                 determine_online_status(node_state, this_node, data_set);
             }
 
             if (this_node->details->online
                 || pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
                 /* offline nodes run no resources...
                  * unless stonith is enabled in which case we need to
                  *   make sure rsc start events happen after the stonith
                  */
                 xmlNode *lrm_rsc = NULL;
 
                 tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
                 tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);
 
                 for (lrm_rsc = pcmk__xe_first_child(tmp); lrm_rsc != NULL;
                      lrm_rsc = pcmk__xe_next(lrm_rsc)) {
 
                     if (pcmk__str_eq((const char *)lrm_rsc->name,
                                      XML_LRM_TAG_RESOURCE, pcmk__str_none)) {
 
                         const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID);
 
                         if (rsc != NULL && !pcmk__str_eq(rsc_id, rsc, pcmk__str_casei)) {
                             continue;
                         }
 
                         intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
                         output = g_list_concat(output, intermediate);
                     }
                 }
             }
         }
     }
 
     return output;
 }
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
index e244f70b09..9f4818425d 100644
--- a/lib/pengine/utils.c
+++ b/lib/pengine/utils.c
@@ -1,900 +1,900 @@
 /*
  * Copyright 2004-2023 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <glib.h>
 #include <stdbool.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/pengine/rules.h>
 #include <crm/pengine/internal.h>
 
 #include "pe_status_private.h"
 
 extern bool pcmk__is_daemon;
 
 gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data);
 
 /*!
  * \internal
  * \brief Check whether we can fence a particular node
  *
  * \param[in] data_set  Working set for cluster
  * \param[in] node      Name of node to check
  *
  * \return true if node can be fenced, false otherwise
  */
 bool
 pe_can_fence(const pe_working_set_t *data_set, const pe_node_t *node)
 {
     if (pe__is_guest_node(node)) {
         /* Guest nodes are fenced by stopping their container resource. We can
          * do that if the container's host is either online or fenceable.
          */
         pe_resource_t *rsc = node->details->remote_rsc->container;
 
         for (GList *n = rsc->running_on; n != NULL; n = n->next) {
             pe_node_t *container_node = n->data;
 
             if (!container_node->details->online
                 && !pe_can_fence(data_set, container_node)) {
                 return false;
             }
         }
         return true;
 
     } else if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
         return false; /* Turned off */
 
     } else if (!pcmk_is_set(data_set->flags, pe_flag_have_stonith_resource)) {
         return false; /* No devices */
 
-    } else if (pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
+    } else if (pcmk_is_set(data_set->flags, pcmk_sched_quorate)) {
         return true;
 
     } else if (data_set->no_quorum_policy == pcmk_no_quorum_ignore) {
         return true;
 
     } else if(node == NULL) {
         return false;
 
     } else if(node->details->online) {
         crm_notice("We can fence %s without quorum because they're in our membership",
                    pe__node_name(node));
         return true;
     }
 
     crm_trace("Cannot fence %s", pe__node_name(node));
     return false;
 }
 
 /*!
  * \internal
  * \brief Copy a node object
  *
  * \param[in] this_node  Node object to copy
  *
  * \return Newly allocated shallow copy of this_node
  * \note This function asserts on errors and is guaranteed to return non-NULL.
  */
 pe_node_t *
 pe__copy_node(const pe_node_t *this_node)
 {
     pe_node_t *new_node = NULL;
 
     CRM_ASSERT(this_node != NULL);
 
     new_node = calloc(1, sizeof(pe_node_t));
     CRM_ASSERT(new_node != NULL);
 
     new_node->rsc_discover_mode = this_node->rsc_discover_mode;
     new_node->weight = this_node->weight;
     new_node->fixed = this_node->fixed; // @COMPAT deprecated and unused
     new_node->count = this_node->count;
     new_node->details = this_node->details;
 
     return new_node;
 }
 
 /*!
  * \internal
  * \brief Create a node hash table from a node list
  *
  * \param[in] list  Node list
  *
  * \return Hash table equivalent of node list
  */
 GHashTable *
 pe__node_list2table(const GList *list)
 {
     GHashTable *result = NULL;
 
     result = pcmk__strkey_table(NULL, free);
     for (const GList *gIter = list; gIter != NULL; gIter = gIter->next) {
         pe_node_t *new_node = pe__copy_node((const pe_node_t *) gIter->data);
 
         g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
     }
     return result;
 }
 
 /*!
  * \internal
  * \brief Compare two nodes by name, with numeric portions sorted numerically
  *
  * Sort two node names case-insensitively like strcasecmp(), but with any
  * numeric portions of the name sorted numerically. For example, "node10" will
  * sort higher than "node9" but lower than "remotenode9".
  *
  * \param[in] a  First node to compare (can be \c NULL)
  * \param[in] b  Second node to compare (can be \c NULL)
  *
  * \retval -1 \c a comes before \c b (or \c a is \c NULL and \c b is not)
  * \retval  0 \c a and \c b are equal (or both are \c NULL)
  * \retval  1 \c a comes after \c b (or \c b is \c NULL and \c a is not)
  */
 gint
 pe__cmp_node_name(gconstpointer a, gconstpointer b)
 {
     const pe_node_t *node1 = (const pe_node_t *) a;
     const pe_node_t *node2 = (const pe_node_t *) b;
 
     if ((node1 == NULL) && (node2 == NULL)) {
         return 0;
     }
 
     if (node1 == NULL) {
         return -1;
     }
 
     if (node2 == NULL) {
         return 1;
     }
 
     return pcmk__numeric_strcasecmp(node1->details->uname,
                                     node2->details->uname);
 }
 
 /*!
  * \internal
  * \brief Output node weights to stdout
  *
  * \param[in]     rsc       Use allowed nodes for this resource
  * \param[in]     comment   Text description to prefix lines with
  * \param[in]     nodes     If rsc is not specified, use these nodes
  * \param[in,out] data_set  Cluster working set
  */
 static void
 pe__output_node_weights(const pe_resource_t *rsc, const char *comment,
                         GHashTable *nodes, pe_working_set_t *data_set)
 {
     pcmk__output_t *out = data_set->priv;
 
     // Sort the nodes so the output is consistent for regression tests
     GList *list = g_list_sort(g_hash_table_get_values(nodes),
                               pe__cmp_node_name);
 
     for (const GList *gIter = list; gIter != NULL; gIter = gIter->next) {
         const pe_node_t *node = (const pe_node_t *) gIter->data;
 
         out->message(out, "node-weight", rsc, comment, node->details->uname,
                      pcmk_readable_score(node->weight));
     }
     g_list_free(list);
 }
 
 /*!
  * \internal
  * \brief Log node weights at trace level
  *
  * \param[in] file      Caller's filename
  * \param[in] function  Caller's function name
  * \param[in] line      Caller's line number
  * \param[in] rsc       If not NULL, include this resource's ID in logs
  * \param[in] comment   Text description to prefix lines with
  * \param[in] nodes     Nodes whose scores should be logged
  */
 static void
 pe__log_node_weights(const char *file, const char *function, int line,
                      const pe_resource_t *rsc, const char *comment,
                      GHashTable *nodes)
 {
     GHashTableIter iter;
     pe_node_t *node = NULL;
 
     // Don't waste time if we're not tracing at this point
     pcmk__if_tracing({}, return);
 
     g_hash_table_iter_init(&iter, nodes);
     while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
         if (rsc) {
             qb_log_from_external_source(function, file,
                                         "%s: %s allocation score on %s: %s",
                                         LOG_TRACE, line, 0,
                                         comment, rsc->id,
                                         pe__node_name(node),
                                         pcmk_readable_score(node->weight));
         } else {
             qb_log_from_external_source(function, file, "%s: %s = %s",
                                         LOG_TRACE, line, 0,
                                         comment, pe__node_name(node),
                                         pcmk_readable_score(node->weight));
         }
     }
 }
 
 /*!
  * \internal
  * \brief Log or output node weights
  *
  * \param[in]     file      Caller's filename
  * \param[in]     function  Caller's function name
  * \param[in]     line      Caller's line number
  * \param[in]     to_log    Log if true, otherwise output
  * \param[in]     rsc       If not NULL, use this resource's ID in logs,
  *                          and show scores recursively for any children
  * \param[in]     comment   Text description to prefix lines with
  * \param[in]     nodes     Nodes whose scores should be shown
  * \param[in,out] data_set  Cluster working set
  */
 void
 pe__show_node_scores_as(const char *file, const char *function, int line,
                         bool to_log, const pe_resource_t *rsc,
                         const char *comment, GHashTable *nodes,
                         pe_working_set_t *data_set)
 {
     if (rsc != NULL && pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
         // Don't show allocation scores for orphans
         return;
     }
     if (nodes == NULL) {
         // Nothing to show
         return;
     }
 
     if (to_log) {
         pe__log_node_weights(file, function, line, rsc, comment, nodes);
     } else {
         pe__output_node_weights(rsc, comment, nodes, data_set);
     }
 
     // If this resource has children, repeat recursively for each
     if (rsc && rsc->children) {
         for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child = (pe_resource_t *) gIter->data;
 
             pe__show_node_scores_as(file, function, line, to_log, child,
                                     comment, child->allowed_nodes, data_set);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Compare two resources by priority
  *
  * \param[in] a  First resource to compare (can be \c NULL)
  * \param[in] b  Second resource to compare (can be \c NULL)
  *
  * \retval -1 \c a->priority > \c b->priority (or \c b is \c NULL and \c a is
  *            not)
  * \retval  0 \c a->priority == \c b->priority (or both \c a and \c b are
  *            \c NULL)
  * \retval  1 \c a->priority < \c b->priority (or \c a is \c NULL and \c b is
  *            not)
  */
 gint
 pe__cmp_rsc_priority(gconstpointer a, gconstpointer b)
 {
     const pe_resource_t *resource1 = (const pe_resource_t *)a;
     const pe_resource_t *resource2 = (const pe_resource_t *)b;
 
     if (a == NULL && b == NULL) {
         return 0;
     }
     if (a == NULL) {
         return 1;
     }
     if (b == NULL) {
         return -1;
     }
 
     if (resource1->priority > resource2->priority) {
         return -1;
     }
 
     if (resource1->priority < resource2->priority) {
         return 1;
     }
 
     return 0;
 }
 
 static void
 resource_node_score(pe_resource_t *rsc, const pe_node_t *node, int score,
                     const char *tag)
 {
     pe_node_t *match = NULL;
 
     if ((rsc->exclusive_discover || (node->rsc_discover_mode == pe_discover_never))
         && pcmk__str_eq(tag, "symmetric_default", pcmk__str_casei)) {
         /* This string comparision may be fragile, but exclusive resources and
          * exclusive nodes should not have the symmetric_default constraint
          * applied to them.
          */
         return;
 
     } else if (rsc->children) {
         GList *gIter = rsc->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
             resource_node_score(child_rsc, node, score, tag);
         }
     }
 
     match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
     if (match == NULL) {
         match = pe__copy_node(node);
         g_hash_table_insert(rsc->allowed_nodes, (gpointer) match->details->id, match);
     }
     match->weight = pcmk__add_scores(match->weight, score);
     pe_rsc_trace(rsc,
                  "Enabling %s preference (%s) for %s on %s (now %s)",
                  tag, pcmk_readable_score(score), rsc->id, pe__node_name(node),
                  pcmk_readable_score(match->weight));
 }
 
 void
 resource_location(pe_resource_t *rsc, const pe_node_t *node, int score,
                   const char *tag, pe_working_set_t *data_set)
 {
     if (node != NULL) {
         resource_node_score(rsc, node, score, tag);
 
     } else if (data_set != NULL) {
         GList *gIter = data_set->nodes;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_node_t *node_iter = (pe_node_t *) gIter->data;
 
             resource_node_score(rsc, node_iter, score, tag);
         }
 
     } else {
         GHashTableIter iter;
         pe_node_t *node_iter = NULL;
 
         g_hash_table_iter_init(&iter, rsc->allowed_nodes);
         while (g_hash_table_iter_next(&iter, NULL, (void **)&node_iter)) {
             resource_node_score(rsc, node_iter, score, tag);
         }
     }
 
     if (node == NULL && score == -INFINITY) {
         if (rsc->allocated_to) {
             crm_info("Deallocating %s from %s",
                      rsc->id, pe__node_name(rsc->allocated_to));
             free(rsc->allocated_to);
             rsc->allocated_to = NULL;
         }
     }
 }
 
 time_t
 get_effective_time(pe_working_set_t * data_set)
 {
     if(data_set) {
         if (data_set->now == NULL) {
             crm_trace("Recording a new 'now'");
             data_set->now = crm_time_new(NULL);
         }
         return crm_time_get_seconds_since_epoch(data_set->now);
     }
 
     crm_trace("Defaulting to 'now'");
     return time(NULL);
 }
 
 gboolean
 get_target_role(const pe_resource_t *rsc, enum rsc_role_e *role)
 {
     enum rsc_role_e local_role = pcmk_role_unknown;
     const char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
 
     CRM_CHECK(role != NULL, return FALSE);
 
     if (pcmk__str_eq(value, "started", pcmk__str_null_matches | pcmk__str_casei)
         || pcmk__str_eq("default", value, pcmk__str_casei)) {
         return FALSE;
     }
 
     local_role = text2role(value);
     if (local_role == pcmk_role_unknown) {
         pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
                          "because '%s' is not valid", rsc->id, value);
         return FALSE;
 
     } else if (local_role > pcmk_role_started) {
         if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
                         pe_rsc_promotable)) {
             if (local_role > pcmk_role_unpromoted) {
                 /* This is what we'd do anyway, just leave the default to avoid messing up the placement algorithm */
                 return FALSE;
             }
 
         } else {
             pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
                              "because '%s' only makes sense for promotable "
                              "clones", rsc->id, value);
             return FALSE;
         }
     }
 
     *role = local_role;
     return TRUE;
 }
 
 gboolean
 order_actions(pe_action_t * lh_action, pe_action_t * rh_action, enum pe_ordering order)
 {
     GList *gIter = NULL;
     pe_action_wrapper_t *wrapper = NULL;
     GList *list = NULL;
 
     if (order == pe_order_none) {
         return FALSE;
     }
 
     if (lh_action == NULL || rh_action == NULL) {
         return FALSE;
     }
 
     crm_trace("Creating action wrappers for ordering: %s then %s",
               lh_action->uuid, rh_action->uuid);
 
     /* Ensure we never create a dependency on ourselves... it's happened */
     CRM_ASSERT(lh_action != rh_action);
 
     /* Filter dups, otherwise update_action_states() has too much work to do */
     gIter = lh_action->actions_after;
     for (; gIter != NULL; gIter = gIter->next) {
         pe_action_wrapper_t *after = (pe_action_wrapper_t *) gIter->data;
 
         if (after->action == rh_action && (after->type & order)) {
             return FALSE;
         }
     }
 
     wrapper = calloc(1, sizeof(pe_action_wrapper_t));
     wrapper->action = rh_action;
     wrapper->type = order;
     list = lh_action->actions_after;
     list = g_list_prepend(list, wrapper);
     lh_action->actions_after = list;
 
     wrapper = calloc(1, sizeof(pe_action_wrapper_t));
     wrapper->action = lh_action;
     wrapper->type = order;
     list = rh_action->actions_before;
     list = g_list_prepend(list, wrapper);
     rh_action->actions_before = list;
     return TRUE;
 }
 
 void
 destroy_ticket(gpointer data)
 {
     pe_ticket_t *ticket = data;
 
     if (ticket->state) {
         g_hash_table_destroy(ticket->state);
     }
     free(ticket->id);
     free(ticket);
 }
 
 pe_ticket_t *
 ticket_new(const char *ticket_id, pe_working_set_t * data_set)
 {
     pe_ticket_t *ticket = NULL;
 
     if (pcmk__str_empty(ticket_id)) {
         return NULL;
     }
 
     if (data_set->tickets == NULL) {
         data_set->tickets = pcmk__strkey_table(free, destroy_ticket);
     }
 
     ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
     if (ticket == NULL) {
 
         ticket = calloc(1, sizeof(pe_ticket_t));
         if (ticket == NULL) {
             crm_err("Cannot allocate ticket '%s'", ticket_id);
             return NULL;
         }
 
         crm_trace("Creaing ticket entry for %s", ticket_id);
 
         ticket->id = strdup(ticket_id);
         ticket->granted = FALSE;
         ticket->last_granted = -1;
         ticket->standby = FALSE;
         ticket->state = pcmk__strkey_table(free, free);
 
         g_hash_table_insert(data_set->tickets, strdup(ticket->id), ticket);
     }
 
     return ticket;
 }
 
 const char *
 rsc_printable_id(const pe_resource_t *rsc)
 {
     return pcmk_is_set(rsc->flags, pe_rsc_unique)? rsc->id : ID(rsc->xml);
 }
 
 void
 pe__clear_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags)
 {
     pe__clear_resource_flags(rsc, flags);
     for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
         pe__clear_resource_flags_recursive((pe_resource_t *) gIter->data, flags);
     }
 }
 
 void
 pe__clear_resource_flags_on_all(pe_working_set_t *data_set, uint64_t flag)
 {
     for (GList *lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
         pe_resource_t *r = (pe_resource_t *) lpc->data;
         pe__clear_resource_flags_recursive(r, flag);
     }
 }
 
 void
 pe__set_resource_flags_recursive(pe_resource_t *rsc, uint64_t flags)
 {
     pe__set_resource_flags(rsc, flags);
     for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
         pe__set_resource_flags_recursive((pe_resource_t *) gIter->data, flags);
     }
 }
 
 void
 trigger_unfencing(pe_resource_t *rsc, pe_node_t *node, const char *reason,
                   pe_action_t *dependency, pe_working_set_t *data_set)
 {
     if (!pcmk_is_set(data_set->flags, pe_flag_enable_unfencing)) {
         /* No resources require it */
         return;
 
     } else if ((rsc != NULL)
                && !pcmk_is_set(rsc->flags, pe_rsc_fence_device)) {
         /* Wasn't a stonith device */
         return;
 
     } else if(node
               && node->details->online
               && node->details->unclean == FALSE
               && node->details->shutdown == FALSE) {
         pe_action_t *unfence = pe_fence_op(node, PCMK_ACTION_ON, FALSE, reason,
                                            FALSE, data_set);
 
         if(dependency) {
             order_actions(unfence, dependency, pe_order_optional);
         }
 
     } else if(rsc) {
         GHashTableIter iter;
 
         g_hash_table_iter_init(&iter, rsc->allowed_nodes);
         while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
             if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) {
                 trigger_unfencing(rsc, node, reason, dependency, data_set);
             }
         }
     }
 }
 
 gboolean
 add_tag_ref(GHashTable * tags, const char * tag_name,  const char * obj_ref)
 {
     pe_tag_t *tag = NULL;
     GList *gIter = NULL;
     gboolean is_existing = FALSE;
 
     CRM_CHECK(tags && tag_name && obj_ref, return FALSE);
 
     tag = g_hash_table_lookup(tags, tag_name);
     if (tag == NULL) {
         tag = calloc(1, sizeof(pe_tag_t));
         if (tag == NULL) {
             return FALSE;
         }
         tag->id = strdup(tag_name);
         tag->refs = NULL;
         g_hash_table_insert(tags, strdup(tag_name), tag);
     }
 
     for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
         const char *existing_ref = (const char *) gIter->data;
 
         if (pcmk__str_eq(existing_ref, obj_ref, pcmk__str_none)){
             is_existing = TRUE;
             break;
         }
     }
 
     if (is_existing == FALSE) {
         tag->refs = g_list_append(tag->refs, strdup(obj_ref));
         crm_trace("Added: tag=%s ref=%s", tag->id, obj_ref);
     }
 
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Check whether shutdown has been requested for a node
  *
  * \param[in] node  Node to check
  *
  * \return TRUE if node has shutdown attribute set and nonzero, FALSE otherwise
  * \note This differs from simply using node->details->shutdown in that it can
  *       be used before that has been determined (and in fact to determine it),
  *       and it can also be used to distinguish requested shutdown from implicit
  *       shutdown of remote nodes by virtue of their connection stopping.
  */
 bool
 pe__shutdown_requested(const pe_node_t *node)
 {
     const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
 
     return !pcmk__str_eq(shutdown, "0", pcmk__str_null_matches);
 }
 
 /*!
  * \internal
  * \brief Update a data set's "recheck by" time
  *
  * \param[in]     recheck   Epoch time when recheck should happen
  * \param[in,out] data_set  Current working set
  */
 void
 pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set)
 {
     if ((recheck > get_effective_time(data_set))
         && ((data_set->recheck_by == 0)
             || (data_set->recheck_by > recheck))) {
         data_set->recheck_by = recheck;
     }
 }
 
 /*!
  * \internal
  * \brief Extract nvpair blocks contained by a CIB XML element into a hash table
  *
  * \param[in]     xml_obj       XML element containing blocks of nvpair elements
  * \param[in]     set_name      If not NULL, only use blocks of this element
  * \param[in]     rule_data     Matching parameters to use when unpacking
  * \param[out]    hash          Where to store extracted name/value pairs
  * \param[in]     always_first  If not NULL, process block with this ID first
  * \param[in]     overwrite     Whether to replace existing values with same name
  * \param[in,out] data_set      Cluster working set containing \p xml_obj
  */
 void
 pe__unpack_dataset_nvpairs(const xmlNode *xml_obj, const char *set_name,
                            const pe_rule_eval_data_t *rule_data,
                            GHashTable *hash, const char *always_first,
                            gboolean overwrite, pe_working_set_t *data_set)
 {
     crm_time_t *next_change = crm_time_new_undefined();
 
     pe_eval_nvpairs(data_set->input, xml_obj, set_name, rule_data, hash,
                     always_first, overwrite, next_change);
     if (crm_time_is_defined(next_change)) {
         time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);
 
         pe__update_recheck_time(recheck, data_set);
     }
     crm_time_free(next_change);
 }
 
 bool
 pe__resource_is_disabled(const pe_resource_t *rsc)
 {
     const char *target_role = NULL;
 
     CRM_CHECK(rsc != NULL, return false);
     target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
     if (target_role) {
         enum rsc_role_e target_role_e = text2role(target_role);
 
         if ((target_role_e == pcmk_role_stopped)
             || ((target_role_e == pcmk_role_unpromoted)
                 && pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
                                pe_rsc_promotable))) {
             return true;
         }
     }
     return false;
 }
 
 /*!
  * \internal
  * \brief Check whether a resource is running only on given node
  *
  * \param[in] rsc   Resource to check
  * \param[in] node  Node to check
  *
  * \return true if \p rsc is running only on \p node, otherwise false
  */
 bool
 pe__rsc_running_on_only(const pe_resource_t *rsc, const pe_node_t *node)
 {
     return (rsc != NULL) && pcmk__list_of_1(rsc->running_on)
             && pe__same_node((const pe_node_t *) rsc->running_on->data, node);
 }
 
 bool
 pe__rsc_running_on_any(pe_resource_t *rsc, GList *node_list)
 {
     for (GList *ele = rsc->running_on; ele; ele = ele->next) {
         pe_node_t *node = (pe_node_t *) ele->data;
         if (pcmk__str_in_list(node->details->uname, node_list,
                               pcmk__str_star_matches|pcmk__str_casei)) {
             return true;
         }
     }
 
     return false;
 }
 
 bool
 pcmk__rsc_filtered_by_node(pe_resource_t *rsc, GList *only_node)
 {
     return (rsc->fns->active(rsc, FALSE) && !pe__rsc_running_on_any(rsc, only_node));
 }
 
 GList *
 pe__filter_rsc_list(GList *rscs, GList *filter)
 {
     GList *retval = NULL;
 
     for (GList *gIter = rscs; gIter; gIter = gIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
         /* I think the second condition is safe here for all callers of this
          * function.  If not, it needs to move into pe__node_text.
          */
         if (pcmk__str_in_list(rsc_printable_id(rsc), filter, pcmk__str_star_matches) ||
             (rsc->parent && pcmk__str_in_list(rsc_printable_id(rsc->parent), filter, pcmk__str_star_matches))) {
             retval = g_list_prepend(retval, rsc);
         }
     }
 
     return retval;
 }
 
 GList *
 pe__build_node_name_list(pe_working_set_t *data_set, const char *s) {
     GList *nodes = NULL;
 
     if (pcmk__str_eq(s, "*", pcmk__str_null_matches)) {
         /* Nothing was given so return a list of all node names.  Or, '*' was
          * given.  This would normally fall into the pe__unames_with_tag branch
          * where it will return an empty list.  Catch it here instead.
          */
         nodes = g_list_prepend(nodes, strdup("*"));
     } else {
         pe_node_t *node = pe_find_node(data_set->nodes, s);
 
         if (node) {
             /* The given string was a valid uname for a node.  Return a
              * singleton list containing just that uname.
              */
             nodes = g_list_prepend(nodes, strdup(s));
         } else {
             /* The given string was not a valid uname.  It's either a tag or
              * it's a typo or something.  In the first case, we'll return a
              * list of all the unames of the nodes with the given tag.  In the
              * second case, we'll return a NULL pointer and nothing will
              * get displayed.
              */
             nodes = pe__unames_with_tag(data_set, s);
         }
     }
 
     return nodes;
 }
 
 GList *
 pe__build_rsc_list(pe_working_set_t *data_set, const char *s) {
     GList *resources = NULL;
 
     if (pcmk__str_eq(s, "*", pcmk__str_null_matches)) {
         resources = g_list_prepend(resources, strdup("*"));
     } else {
         const uint32_t flags = pcmk_rsc_match_history|pcmk_rsc_match_basename;
         pe_resource_t *rsc = pe_find_resource_with_flags(data_set->resources, s,
                                                          flags);
 
         if (rsc) {
             /* A colon in the name we were given means we're being asked to filter
              * on a specific instance of a cloned resource.  Put that exact string
              * into the filter list.  Otherwise, use the printable ID of whatever
              * resource was found that matches what was asked for.
              */
             if (strstr(s, ":") != NULL) {
                 resources = g_list_prepend(resources, strdup(rsc->id));
             } else {
                 resources = g_list_prepend(resources, strdup(rsc_printable_id(rsc)));
             }
         } else {
             /* The given string was not a valid resource name. It's a tag or a
              * typo or something. See pe__build_node_name_list() for more
              * detail.
              */
             resources = pe__rscs_with_tag(data_set, s);
         }
     }
 
     return resources;
 }
 
 xmlNode *
 pe__failed_probe_for_rsc(const pe_resource_t *rsc, const char *name)
 {
     const pe_resource_t *parent = pe__const_top_resource(rsc, false);
     const char *rsc_id = rsc->id;
 
     if (parent->variant == pcmk_rsc_variant_clone) {
         rsc_id = pe__clone_child_id(parent);
     }
 
     for (xmlNode *xml_op = pcmk__xml_first_child(rsc->cluster->failed); xml_op != NULL;
          xml_op = pcmk__xml_next(xml_op)) {
         const char *value = NULL;
         char *op_id = NULL;
 
         /* This resource operation is not a failed probe. */
         if (!pcmk_xe_mask_probe_failure(xml_op)) {
             continue;
         }
 
         /* This resource operation was not run on the given node.  Note that if name is
          * NULL, this will always succeed.
          */
         value = crm_element_value(xml_op, XML_LRM_ATTR_TARGET);
         if (value == NULL || !pcmk__str_eq(value, name, pcmk__str_casei|pcmk__str_null_matches)) {
             continue;
         }
 
         if (!parse_op_key(pe__xe_history_key(xml_op), &op_id, NULL, NULL)) {
             continue; // This history entry is missing an operation key
         }
 
         /* This resource operation's ID does not match the rsc_id we are looking for. */
         if (!pcmk__str_eq(op_id, rsc_id, pcmk__str_none)) {
             free(op_id);
             continue;
         }
 
         free(op_id);
         return xml_op;
     }
 
     return NULL;
 }