diff --git a/daemons/based/cibmon.c b/daemons/based/cibmon.c
index c9b0784617..5bdc27e06c 100644
--- a/daemons/based/cibmon.c
+++ b/daemons/based/cibmon.c
@@ -1,251 +1,250 @@
 /*
- * Copyright 2004-2019 the Pacemaker project contributors
+ * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/mainloop.h>
 #include <crm/cib/internal.h>
 
 #include <crm/common/ipc.h>
 #include <crm/pengine/status.h>
-#include <../lib/pengine/unpack.h>
 
 #include <crm/cib.h>
 
 #ifdef HAVE_GETOPT_H
 #  include <getopt.h>
 #endif
 
 static int max_failures = 30;
 
 static gboolean log_diffs = FALSE;
 static gboolean log_updates = FALSE;
 
 static GMainLoop *mainloop = NULL;
 void usage(const char *cmd, crm_exit_t exit_status);
 void cib_connection_destroy(gpointer user_data);
 
 void cibmon_shutdown(int nsig);
 void cibmon_diff(const char *event, xmlNode * msg);
 
 static cib_t *cib = NULL;
 static xmlNode *cib_copy = NULL;
 
 #define OPTARGS	"V?m:du"
 
 int
 main(int argc, char **argv)
 {
     int argerr = 0;
     int flag;
     int attempts = 0;
     int rc = pcmk_ok;
 
 #ifdef HAVE_GETOPT_H
     int option_index = 0;
 
     static struct option long_options[] = {
         /* Top-level Options */
         {"verbose", 0, 0, 'V'},
         {"help", 0, 0, '?'},
         {"log-diffs", 0, 0, 'd'},
         {"log-updates", 0, 0, 'u'},
         {"max-conn-fail", 1, 0, 'm'},
         {0, 0, 0, 0}
     };
 #endif
 
     crm_log_cli_init("cibmon");
 
     crm_signal_handler(SIGTERM, cibmon_shutdown);
 
     while (1) {
 #ifdef HAVE_GETOPT_H
         flag = getopt_long(argc, argv, OPTARGS, long_options, &option_index);
 #else
         flag = getopt(argc, argv, OPTARGS);
 #endif
         if (flag == -1)
             break;
 
         switch (flag) {
             case 'V':
                 crm_bump_log_level(argc, argv);
                 break;
             case '?':
                 usage(crm_system_name, CRM_EX_OK);
                 break;
             case 'd':
                 log_diffs = TRUE;
                 break;
             case 'u':
                 log_updates = TRUE;
                 break;
             case 'm':
                 max_failures = crm_parse_int(optarg, "30");
                 break;
             default:
                 printf("Argument code 0%o (%c)" " is not (?yet?) supported\n", flag, flag);
                 ++argerr;
                 break;
         }
     }
 
     if (optind < argc) {
         printf("non-option ARGV-elements: ");
         while (optind < argc)
             printf("%s ", argv[optind++]);
         printf("\n");
     }
 
     if (optind > argc) {
         ++argerr;
     }
 
     if (argerr) {
         usage(crm_system_name, CRM_EX_USAGE);
     }
 
     cib = cib_new();
 
     do {
         sleep(1);
         rc = cib->cmds->signon(cib, crm_system_name, cib_query);
 
     } while (rc == -ENOTCONN && attempts++ < max_failures);
 
     if (rc != pcmk_ok) {
         crm_err("Signon to CIB failed: %s", pcmk_strerror(rc));
         goto fail;
     }
 
     crm_debug("Setting dnotify");
     rc = cib->cmds->set_connection_dnotify(cib, cib_connection_destroy);
     if (rc != pcmk_ok) {
         crm_err("Failed to set dnotify callback: %s", pcmk_strerror(rc));
         goto fail;
     }
 
     crm_debug("Setting diff callback");
     rc = cib->cmds->add_notify_callback(cib, T_CIB_DIFF_NOTIFY, cibmon_diff);
     if (rc != pcmk_ok) {
         crm_err("Failed to set diff callback: %s", pcmk_strerror(rc));
         goto fail;
     }
 
     mainloop = g_main_loop_new(NULL, FALSE);
     crm_info("Starting mainloop");
     g_main_loop_run(mainloop);
     crm_trace("%s exiting normally", crm_system_name);
     fflush(stderr);
     return CRM_EX_OK;
 
 fail:
     crm_err("Setup failed, could not monitor CIB actions");
     return CRM_EX_ERROR;
 }
 
 void
 usage(const char *cmd, crm_exit_t exit_status)
 {
     FILE *stream;
 
     stream = (exit_status == CRM_EX_OK)? stdout : stderr;
     fflush(stream);
 
     crm_exit(exit_status);
 }
 
 void
 cib_connection_destroy(gpointer user_data)
 {
     cib_t *conn = user_data;
 
     crm_err("Connection to the CIB terminated... exiting");
     conn->cmds->signoff(conn);  /* Ensure IPC is cleaned up */
     g_main_loop_quit(mainloop);
     return;
 }
 
 void
 cibmon_diff(const char *event, xmlNode * msg)
 {
     int rc = -1;
     const char *op = NULL;
     unsigned int log_level = LOG_INFO;
 
     xmlNode *diff = NULL;
     xmlNode *cib_last = NULL;
     xmlNode *update = get_message_xml(msg, F_CIB_UPDATE);
 
     if (msg == NULL) {
         crm_err("NULL update");
         return;
     }
 
     crm_element_value_int(msg, F_CIB_RC, &rc);
     op = crm_element_value(msg, F_CIB_OPERATION);
     diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
 
     if (rc < pcmk_ok) {
         log_level = LOG_WARNING;
         do_crm_log(log_level, "[%s] %s ABORTED: %s", event, op, pcmk_strerror(rc));
         return;
     }
 
     if (log_diffs) {
         xml_log_patchset(log_level, op, diff);
     }
 
     if (log_updates && update != NULL) {
         crm_log_xml_trace(update, "raw_update");
     }
 
     if (cib_copy != NULL) {
         cib_last = cib_copy;
         cib_copy = NULL;
         rc = cib_process_diff(op, cib_force_diff, NULL, NULL, diff, cib_last, &cib_copy, NULL);
 
         if (rc != pcmk_ok) {
             crm_debug("Update didn't apply, requesting full copy: %s", pcmk_strerror(rc));
             free_xml(cib_copy);
             cib_copy = NULL;
         }
     }
 
     if (cib_copy == NULL) {
         rc = cib->cmds->query(cib, NULL, &cib_copy, cib_scope_local | cib_sync_call);
     }
 
     if(rc == -EACCES) {
         crm_exit(CRM_EX_INSUFFICIENT_PRIV);
     }
 
     free_xml(cib_last);
 }
 
 void
 cibmon_shutdown(int nsig)
 {
     crm_exit(CRM_EX_OK);
 }
diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
index b4ce1574cd..aa69dc8a55 100644
--- a/include/crm/pengine/internal.h
+++ b/include/crm/pengine/internal.h
@@ -1,444 +1,470 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef PE_INTERNAL__H
 #  define PE_INTERNAL__H
 #  include <string.h>
 #  include <crm/pengine/status.h>
 #  include <crm/pengine/remote_internal.h>
 #  include <crm/common/output.h>
 
 #  define pe_rsc_info(rsc, fmt, args...)  crm_log_tag(LOG_INFO,  rsc ? rsc->id : "<NULL>", fmt, ##args)
 #  define pe_rsc_debug(rsc, fmt, args...) crm_log_tag(LOG_DEBUG, rsc ? rsc->id : "<NULL>", fmt, ##args)
 #  define pe_rsc_trace(rsc, fmt, args...) crm_log_tag(LOG_TRACE, rsc ? rsc->id : "<NULL>", fmt, ##args)
 
 #  define pe_err(fmt...) { was_processing_error = TRUE; crm_config_error = TRUE; crm_err(fmt); }
 #  define pe_warn(fmt...) { was_processing_warning = TRUE; crm_config_warning = TRUE; crm_warn(fmt); }
 #  define pe_proc_err(fmt...) { was_processing_error = TRUE; crm_err(fmt); }
 #  define pe_proc_warn(fmt...) { was_processing_warning = TRUE; crm_warn(fmt); }
 #  define pe_set_action_bit(action, bit) action->flags = crm_set_bit(__FUNCTION__, __LINE__, action->uuid, action->flags, bit)
 #  define pe_clear_action_bit(action, bit) action->flags = crm_clear_bit(__FUNCTION__, __LINE__, action->uuid, action->flags, bit)
 
+// Some warnings we don't want to print every transition
+
+enum pe_warn_once_e {
+    pe_wo_blind         = 0x0001,
+    pe_wo_restart_type  = 0x0002,
+    pe_wo_role_after    = 0x0004,
+    pe_wo_poweroff      = 0x0008,
+    pe_wo_require_all   = 0x0010,
+    pe_wo_order_score   = 0x0020,
+    pe_wo_neg_threshold = 0x0040,
+};
+
+extern uint32_t pe_wo;
+
+#define pe_warn_once(pe_wo_bit, fmt...) do {    \
+        if (is_not_set(pe_wo, pe_wo_bit)) {     \
+            if (pe_wo_bit == pe_wo_blind) {     \
+                crm_warn(fmt);                  \
+            } else {                            \
+                pe_warn(fmt);                   \
+            }                                   \
+            set_bit(pe_wo, pe_wo_bit);          \
+        }                                       \
+    } while (0);
+
+
 typedef struct pe__location_constraint_s {
     char *id;                           // Constraint XML ID
     pe_resource_t *rsc_lh;              // Resource being located
     enum rsc_role_e role_filter;        // Role to locate
     enum pe_discover_e discover_mode;   // Resource discovery
     GListPtr node_list_rh;              // List of pe_node_t*
 } pe__location_t;
 
 typedef struct pe__order_constraint_s {
     int id;
     enum pe_ordering type;
 
     void *lh_opaque;
     pe_resource_t *lh_rsc;
     pe_action_t *lh_action;
     char *lh_action_task;
 
     void *rh_opaque;
     pe_resource_t *rh_rsc;
     pe_action_t *rh_action;
     char *rh_action_task;
 } pe__ordering_t;
 
 typedef struct notify_data_s {
     GSList *keys;               // Environment variable name/value pairs
 
     const char *action;
 
     pe_action_t *pre;
     pe_action_t *post;
     pe_action_t *pre_done;
     pe_action_t *post_done;
 
     GListPtr active;            /* notify_entry_t*  */
     GListPtr inactive;          /* notify_entry_t*  */
     GListPtr start;             /* notify_entry_t*  */
     GListPtr stop;              /* notify_entry_t*  */
     GListPtr demote;            /* notify_entry_t*  */
     GListPtr promote;           /* notify_entry_t*  */
     GListPtr master;            /* notify_entry_t*  */
     GListPtr slave;             /* notify_entry_t*  */
     GHashTable *allowed_nodes;
 
 } notify_data_t;
 
 bool pe_can_fence(pe_working_set_t *data_set, pe_node_t *node);
 
 int merge_weights(int w1, int w2);
 void add_hash_param(GHashTable * hash, const char *name, const char *value);
 
 char *native_parameter(pe_resource_t * rsc, pe_node_t * node, gboolean create, const char *name,
                        pe_working_set_t * data_set);
 pe_node_t *native_location(const pe_resource_t *rsc, GList **list, int current);
 
 void pe_metadata(void);
 void verify_pe_options(GHashTable * options);
 
 void common_update_score(pe_resource_t * rsc, const char *id, int score);
 void native_add_running(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set);
 
 gboolean native_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
 gboolean group_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
 gboolean clone_unpack(pe_resource_t * rsc, pe_working_set_t * data_set);
 gboolean pe__unpack_bundle(pe_resource_t *rsc, pe_working_set_t *data_set);
 
 pe_resource_t *native_find_rsc(pe_resource_t *rsc, const char *id, const pe_node_t *node,
                                int flags);
 
 gboolean native_active(pe_resource_t * rsc, gboolean all);
 gboolean group_active(pe_resource_t * rsc, gboolean all);
 gboolean clone_active(pe_resource_t * rsc, gboolean all);
 gboolean pe__bundle_active(pe_resource_t *rsc, gboolean all);
 
 void native_print(pe_resource_t * rsc, const char *pre_text, long options, void *print_data);
 void group_print(pe_resource_t * rsc, const char *pre_text, long options, void *print_data);
 void clone_print(pe_resource_t * rsc, const char *pre_text, long options, void *print_data);
 void pe__print_bundle(pe_resource_t *rsc, const char *pre_text, long options,
                       void *print_data);
 
 int pe__name_and_nvpairs_xml(pcmk__output_t *out, bool is_list, const char *tag_name
                          , size_t pairs_count, ...);
 char *pe__node_display_name(pe_node_t *node, bool print_detail);
 
 int pe__ban_html(pcmk__output_t *out, va_list args);
 int pe__ban_text(pcmk__output_t *out, va_list args);
 int pe__ban_xml(pcmk__output_t *out, va_list args);
 int pe__clone_xml(pcmk__output_t *out, va_list args);
 int pe__clone_html(pcmk__output_t *out, va_list args);
 int pe__clone_text(pcmk__output_t *out, va_list args);
 int pe__cluster_counts_html(pcmk__output_t *out, va_list args);
 int pe__cluster_counts_text(pcmk__output_t *out, va_list args);
 int pe__cluster_counts_xml(pcmk__output_t *out, va_list args);
 int pe__cluster_dc_html(pcmk__output_t *out, va_list args);
 int pe__cluster_dc_text(pcmk__output_t *out, va_list args);
 int pe__cluster_dc_xml(pcmk__output_t *out, va_list args);
 int pe__cluster_maint_mode_html(pcmk__output_t *out, va_list args);
 int pe__cluster_maint_mode_text(pcmk__output_t *out, va_list args);
 int pe__cluster_options_html(pcmk__output_t *out, va_list args);
 int pe__cluster_options_log(pcmk__output_t *out, va_list args);
 int pe__cluster_options_text(pcmk__output_t *out, va_list args);
 int pe__cluster_options_xml(pcmk__output_t *out, va_list args);
 int pe__cluster_stack_html(pcmk__output_t *out, va_list args);
 int pe__cluster_stack_text(pcmk__output_t *out, va_list args);
 int pe__cluster_stack_xml(pcmk__output_t *out, va_list args);
 int pe__cluster_summary(pcmk__output_t *out, va_list args);
 int pe__cluster_summary_html(pcmk__output_t *out, va_list args);
 int pe__cluster_times_html(pcmk__output_t *out, va_list args);
 int pe__cluster_times_xml(pcmk__output_t *out, va_list args);
 int pe__cluster_times_text(pcmk__output_t *out, va_list args);
 int pe__failed_action_text(pcmk__output_t *out, va_list args);
 int pe__failed_action_xml(pcmk__output_t *out, va_list args);
 int pe__group_xml(pcmk__output_t *out, va_list args);
 int pe__group_html(pcmk__output_t *out, va_list args);
 int pe__group_text(pcmk__output_t *out, va_list args);
 int pe__bundle_xml(pcmk__output_t *out, va_list args);
 int pe__bundle_html(pcmk__output_t *out, va_list args);
 int pe__bundle_text(pcmk__output_t *out, va_list args);
 int pe__node_html(pcmk__output_t *out, va_list args);
 int pe__node_text(pcmk__output_t *out, va_list args);
 int pe__node_xml(pcmk__output_t *out, va_list args);
 int pe__node_attribute_html(pcmk__output_t *out, va_list args);
 int pe__node_attribute_text(pcmk__output_t *out, va_list args);
 int pe__node_attribute_xml(pcmk__output_t *out, va_list args);
 int pe__op_history_text(pcmk__output_t *out, va_list args);
 int pe__op_history_xml(pcmk__output_t *out, va_list args);
 int pe__resource_history_text(pcmk__output_t *out, va_list args);
 int pe__resource_history_xml(pcmk__output_t *out, va_list args);
 int pe__resource_xml(pcmk__output_t *out, va_list args);
 int pe__resource_html(pcmk__output_t *out, va_list args);
 int pe__resource_text(pcmk__output_t *out, va_list args);
 int pe__ticket_html(pcmk__output_t *out, va_list args);
 int pe__ticket_text(pcmk__output_t *out, va_list args);
 int pe__ticket_xml(pcmk__output_t *out, va_list args);
 
 void native_free(pe_resource_t * rsc);
 void group_free(pe_resource_t * rsc);
 void clone_free(pe_resource_t * rsc);
 void pe__free_bundle(pe_resource_t *rsc);
 
 enum rsc_role_e native_resource_state(const pe_resource_t * rsc, gboolean current);
 enum rsc_role_e group_resource_state(const pe_resource_t * rsc, gboolean current);
 enum rsc_role_e clone_resource_state(const pe_resource_t * rsc, gboolean current);
 enum rsc_role_e pe__bundle_resource_state(const pe_resource_t *rsc,
                                           gboolean current);
 
 void pe__count_common(pe_resource_t *rsc);
 void pe__count_bundle(pe_resource_t *rsc);
 
 gboolean common_unpack(xmlNode * xml_obj, pe_resource_t ** rsc, pe_resource_t * parent,
                        pe_working_set_t * data_set);
 void common_free(pe_resource_t * rsc);
 
 extern pe_node_t *node_copy(const pe_node_t *this_node);
 extern time_t get_effective_time(pe_working_set_t * data_set);
 
 /* Failure handling utilities (from failcounts.c) */
 
 // bit flags for fail count handling options
 enum pe_fc_flags_e {
     pe_fc_default   = 0x00,
     pe_fc_effective = 0x01, // don't count expired failures
     pe_fc_fillers   = 0x02, // if container, include filler failures in count
 };
 
 int pe_get_failcount(pe_node_t *node, pe_resource_t *rsc, time_t *last_failure,
                      uint32_t flags, xmlNode *xml_op,
                      pe_working_set_t *data_set);
 
 pe_action_t *pe__clear_failcount(pe_resource_t *rsc, pe_node_t *node,
                                  const char *reason,
                                  pe_working_set_t *data_set);
 
 /* Functions for finding/counting a resource's active nodes */
 
 pe_node_t *pe__find_active_on(const pe_resource_t *rsc,
                               unsigned int *count_all,
                               unsigned int *count_clean);
 pe_node_t *pe__find_active_requires(const pe_resource_t *rsc,
                                     unsigned int *count);
 
 static inline pe_node_t *
 pe__current_node(const pe_resource_t *rsc)
 {
     return pe__find_active_on(rsc, NULL, NULL);
 }
 
 
 /* Binary like operators for lists of nodes */
 extern void node_list_exclude(GHashTable * list, GListPtr list2, gboolean merge_scores);
 extern GListPtr node_list_dup(GListPtr list, gboolean reset, gboolean filter);
 
 extern GHashTable *node_hash_from_list(GListPtr list);
 static inline gpointer
 pe_hash_table_lookup(GHashTable * hash, gconstpointer key)
 {
     if (hash) {
         return g_hash_table_lookup(hash, key);
     }
     return NULL;
 }
 
 extern pe_action_t *get_pseudo_op(const char *name, pe_working_set_t * data_set);
 extern gboolean order_actions(pe_action_t * lh_action, pe_action_t * rh_action, enum pe_ordering order);
 
 GHashTable *node_hash_dup(GHashTable * hash);
 
 /* Printing functions for debug */
 extern void print_node(const char *pre_text, pe_node_t * node, gboolean details);
 extern void print_str_str(gpointer key, gpointer value, gpointer user_data);
 extern void pe__output_node(pe_node_t * node, gboolean details, pcmk__output_t *out);
 
 extern void dump_node_capacity(int level, const char *comment, pe_node_t * node);
 extern void dump_rsc_utilization(int level, const char *comment, pe_resource_t * rsc, pe_node_t * node);
 
 void pe__show_node_weights_as(const char *file, const char *function,
                               int line, bool to_log, pe_resource_t *rsc,
                               const char *comment, GHashTable *nodes);
 
 #define pe__show_node_weights(level, rsc, text, nodes)              \
         pe__show_node_weights_as(__FILE__, __FUNCTION__, __LINE__,  \
                                  (level), (rsc), (text), (nodes))
 
 /* Sorting functions */
 extern gint sort_rsc_priority(gconstpointer a, gconstpointer b);
 extern gint sort_rsc_index(gconstpointer a, gconstpointer b);
 
 extern xmlNode *find_rsc_op_entry(pe_resource_t * rsc, const char *key);
 
 extern pe_action_t *custom_action(pe_resource_t * rsc, char *key, const char *task, pe_node_t * on_node,
                                   gboolean optional, gboolean foo, pe_working_set_t * data_set);
 
 #  define delete_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DELETE, 0)
 #  define delete_action(rsc, node, optional) custom_action(		\
 		rsc, delete_key(rsc), CRMD_ACTION_DELETE, node,		\
 		optional, TRUE, data_set);
 
 #  define stopped_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOPPED, 0)
 #  define stopped_action(rsc, node, optional) custom_action(		\
 		rsc, stopped_key(rsc), CRMD_ACTION_STOPPED, node,	\
 		optional, TRUE, data_set);
 
 #  define stop_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STOP, 0)
 #  define stop_action(rsc, node, optional) custom_action(			\
 		rsc, stop_key(rsc), CRMD_ACTION_STOP, node,		\
 		optional, TRUE, data_set);
 
 #  define reload_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_RELOAD, 0)
 #  define start_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_START, 0)
 #  define start_action(rsc, node, optional) custom_action(		\
 		rsc, start_key(rsc), CRMD_ACTION_START, node,		\
 		optional, TRUE, data_set)
 
 #  define started_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_STARTED, 0)
 #  define started_action(rsc, node, optional) custom_action(		\
 		rsc, started_key(rsc), CRMD_ACTION_STARTED, node,	\
 		optional, TRUE, data_set)
 
 #  define promote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTE, 0)
 #  define promote_action(rsc, node, optional) custom_action(		\
 		rsc, promote_key(rsc), CRMD_ACTION_PROMOTE, node,	\
 		optional, TRUE, data_set)
 
 #  define promoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_PROMOTED, 0)
 #  define promoted_action(rsc, node, optional) custom_action(		\
 		rsc, promoted_key(rsc), CRMD_ACTION_PROMOTED, node,	\
 		optional, TRUE, data_set)
 
 #  define demote_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTE, 0)
 #  define demote_action(rsc, node, optional) custom_action(		\
 		rsc, demote_key(rsc), CRMD_ACTION_DEMOTE, node,		\
 		optional, TRUE, data_set)
 
 #  define demoted_key(rsc) pcmk__op_key(rsc->id, CRMD_ACTION_DEMOTED, 0)
 #  define demoted_action(rsc, node, optional) custom_action(		\
 		rsc, demoted_key(rsc), CRMD_ACTION_DEMOTED, node,	\
 		optional, TRUE, data_set)
 
 extern int pe_get_configured_timeout(pe_resource_t *rsc, const char *action,
                                      pe_working_set_t *data_set);
 
 extern pe_action_t *find_first_action(GListPtr input, const char *uuid, const char *task,
                                       pe_node_t * on_node);
 extern enum action_tasks get_complex_task(pe_resource_t * rsc, const char *name,
                                           gboolean allow_non_atomic);
 
 extern GListPtr find_actions(GListPtr input, const char *key, const pe_node_t *on_node);
 GList *find_actions_exact(GList *input, const char *key,
                           const pe_node_t *on_node);
 extern GListPtr find_recurring_actions(GListPtr input, pe_node_t * not_on_node);
 GList *pe__resource_actions(const pe_resource_t *rsc, const pe_node_t *node,
                             const char *task, bool require_node);
 
 extern void pe_free_action(pe_action_t * action);
 
 extern void resource_location(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag,
                               pe_working_set_t * data_set);
 
 extern gint sort_op_by_callid(gconstpointer a, gconstpointer b);
 extern gboolean get_target_role(pe_resource_t * rsc, enum rsc_role_e *role);
 
 extern pe_resource_t *find_clone_instance(pe_resource_t * rsc, const char *sub_id,
                                           pe_working_set_t * data_set);
 
 extern void destroy_ticket(gpointer data);
 extern pe_ticket_t *ticket_new(const char *ticket_id, pe_working_set_t * data_set);
 
 // Resources for manipulating resource names
 const char *pe_base_name_end(const char *id);
 char *clone_strip(const char *last_rsc_id);
 char *clone_zero(const char *last_rsc_id);
 
 static inline bool
 pe_base_name_eq(pe_resource_t *rsc, const char *id)
 {
     if (id && rsc && rsc->id) {
         // Number of characters in rsc->id before any clone suffix
         size_t base_len = pe_base_name_end(rsc->id) - rsc->id + 1;
 
         return (strlen(id) == base_len) && !strncmp(id, rsc->id, base_len);
     }
     return FALSE;
 }
 
 int pe__target_rc_from_xml(xmlNode *xml_op);
 
 gint sort_node_uname(gconstpointer a, gconstpointer b);
 bool is_set_recursive(pe_resource_t * rsc, long long flag, bool any);
 
 enum rsc_digest_cmp_val {
     /*! Digests are the same */
     RSC_DIGEST_MATCH = 0,
     /*! Params that require a restart changed */
     RSC_DIGEST_RESTART,
     /*! Some parameter changed.  */
     RSC_DIGEST_ALL,
     /*! rsc op didn't have a digest associated with it, so
      *  it is unknown if parameters changed or not. */
     RSC_DIGEST_UNKNOWN,
 };
 
 typedef struct op_digest_cache_s {
     enum rsc_digest_cmp_val rc;
     xmlNode *params_all;
     xmlNode *params_secure;
     xmlNode *params_restart;
     char *digest_all_calc;
     char *digest_secure_calc;
     char *digest_restart_calc;
 } op_digest_cache_t;
 
 op_digest_cache_t *rsc_action_digest_cmp(pe_resource_t * rsc, xmlNode * xml_op, pe_node_t * node,
                                          pe_working_set_t * data_set);
 
 pe_action_t *pe_fence_op(pe_node_t * node, const char *op, bool optional, const char *reason, bool priority_delay, pe_working_set_t * data_set);
 void trigger_unfencing(
     pe_resource_t * rsc, pe_node_t *node, const char *reason, pe_action_t *dependency, pe_working_set_t * data_set);
 
 void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrite);
 void pe_action_set_flag_reason(const char *function, long line, pe_action_t *action, pe_action_t *reason, const char *text, enum pe_action_flags flags, bool overwrite);
 
 #define pe_action_required(action, reason, text) pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, reason, text, pe_action_optional, FALSE)
 #define pe_action_implies(action, reason, flag) pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, reason, NULL, flag, FALSE)
 
 void set_bit_recursive(pe_resource_t * rsc, unsigned long long flag);
 void clear_bit_recursive(pe_resource_t * rsc, unsigned long long flag);
 
 gboolean add_tag_ref(GHashTable * tags, const char * tag_name,  const char * obj_ref);
 
 void print_rscs_brief(GListPtr rsc_list, const char * pre_text, long options,
                       void * print_data, gboolean print_all);
 int pe__rscs_brief_output(pcmk__output_t *out, GListPtr rsc_list, long options, gboolean print_all);
 void pe_fence_node(pe_working_set_t * data_set, pe_node_t * node, const char *reason, bool priority_delay);
 
 pe_node_t *pe_create_node(const char *id, const char *uname, const char *type,
                           const char *score, pe_working_set_t * data_set);
 void common_print(pe_resource_t * rsc, const char *pre_text, const char *name, pe_node_t *node, long options, void *print_data);
 int pe__common_output_text(pcmk__output_t *out, pe_resource_t * rsc, const char *name, pe_node_t *node, long options);
 int pe__common_output_html(pcmk__output_t *out, pe_resource_t * rsc, const char *name, pe_node_t *node, long options);
 pe_resource_t *pe__find_bundle_replica(const pe_resource_t *bundle,
                                        const pe_node_t *node);
 bool pe__bundle_needs_remote_name(pe_resource_t *rsc);
 const char *pe__add_bundle_remote_name(pe_resource_t *rsc, xmlNode *xml,
                                        const char *field);
 const char *pe_node_attribute_calculated(const pe_node_t *node,
                                          const char *name,
                                          const pe_resource_t *rsc);
 const char *pe_node_attribute_raw(pe_node_t *node, const char *name);
 bool pe__is_universal_clone(pe_resource_t *rsc,
                             pe_working_set_t *data_set);
 void pe__add_param_check(xmlNode *rsc_op, pe_resource_t *rsc, pe_node_t *node,
                          enum pe_check_parameters, pe_working_set_t *data_set);
 void pe__foreach_param_check(pe_working_set_t *data_set,
                              void (*cb)(pe_resource_t*, pe_node_t*, xmlNode*,
                                         enum pe_check_parameters,
                                         pe_working_set_t*));
 void pe__free_param_checks(pe_working_set_t *data_set);
 
 bool pe__shutdown_requested(pe_node_t *node);
 void pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set);
 
 #define BOOL2STR(x) ((x) ? "true" : "false")
 /*!
  * \internal
  * \brief Register xml formatting message functions.
  */
 void pe__register_messages(pcmk__output_t *out);
 
 void pe__unpack_dataset_nvpairs(xmlNode *xml_obj, const char *set_name,
                                 GHashTable *node_hash, GHashTable *hash,
                                 const char *always_first, gboolean overwrite,
                                 pe_working_set_t *data_set);
 
 bool pe__resource_is_disabled(pe_resource_t *rsc);
 pe_action_t *pe__clear_resource_history(pe_resource_t *rsc, pe_node_t *node,
                                         pe_working_set_t *data_set);
 
 #endif
diff --git a/lib/pacemaker/pcmk_sched_constraints.c b/lib/pacemaker/pcmk_sched_constraints.c
index fdc4f3a6e2..08dac7d8cb 100644
--- a/lib/pacemaker/pcmk_sched_constraints.c
+++ b/lib/pacemaker/pcmk_sched_constraints.c
@@ -1,3064 +1,3061 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <sys/types.h>
 #include <regex.h>
+#include <glib.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/iso8601.h>
-
-#include <glib.h>
-
 #include <crm/pengine/status.h>
-#include <pacemaker-internal.h>
+#include <crm/pengine/internal.h>
 #include <crm/pengine/rules.h>
-
-#include <../lib/pengine/unpack.h>
+#include <pacemaker-internal.h>
 
 enum pe_order_kind {
     pe_order_kind_optional,
     pe_order_kind_mandatory,
     pe_order_kind_serialize,
 };
 
 #define EXPAND_CONSTRAINT_IDREF(__set, __rsc, __name) do {				\
 	__rsc = pe_find_constraint_resource(data_set->resources, __name);		\
 	if(__rsc == NULL) {						\
 	    pcmk__config_err("%s: No resource found for %s", __set, __name);    \
 	    return FALSE;						\
 	}								\
     } while(0)
 
 enum pe_ordering get_flags(const char *id, enum pe_order_kind kind,
                            const char *action_first, const char *action_then, gboolean invert);
 enum pe_ordering get_asymmetrical_flags(enum pe_order_kind kind);
 static pe__location_t *generate_location_rule(pe_resource_t *rsc,
                                               xmlNode *rule_xml,
                                               const char *discovery,
                                               crm_time_t *next_change,
                                               pe_working_set_t *data_set,
                                               pe_match_data_t *match_data);
 
 static bool
 evaluate_lifetime(xmlNode *lifetime, pe_working_set_t *data_set)
 {
     bool result = FALSE;
     crm_time_t *next_change = crm_time_new_undefined();
 
     result = pe_evaluate_rules(lifetime, NULL, data_set->now, next_change);
     if (crm_time_is_defined(next_change)) {
         time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);
 
         pe__update_recheck_time(recheck, data_set);
     }
     crm_time_free(next_change);
     return result;
 }
 
 gboolean
 unpack_constraints(xmlNode * xml_constraints, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
     xmlNode *lifetime = NULL;
 
     for (xml_obj = __xml_first_child_element(xml_constraints); xml_obj != NULL;
          xml_obj = __xml_next_element(xml_obj)) {
         const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
         const char *tag = crm_element_name(xml_obj);
 
         if (id == NULL) {
             pcmk__config_err("Ignoring <%s> constraint without "
                              XML_ATTR_ID, tag);
             continue;
         }
 
         crm_trace("Processing constraint %s %s", tag, id);
 
         lifetime = first_named_child(xml_obj, "lifetime");
         if (lifetime) {
             pcmk__config_warn("Support for 'lifetime' attribute (in %s) is "
                               "deprecated (the rules it contains should "
                               "instead be direct descendents of the "
                               "constraint object)", id);
         }
 
         if (lifetime && !evaluate_lifetime(lifetime, data_set)) {
             crm_info("Constraint %s %s is not active", tag, id);
 
         } else if (safe_str_eq(XML_CONS_TAG_RSC_ORDER, tag)) {
             unpack_rsc_order(xml_obj, data_set);
 
         } else if (safe_str_eq(XML_CONS_TAG_RSC_DEPEND, tag)) {
             unpack_rsc_colocation(xml_obj, data_set);
 
         } else if (safe_str_eq(XML_CONS_TAG_RSC_LOCATION, tag)) {
             unpack_location(xml_obj, data_set);
 
         } else if (safe_str_eq(XML_CONS_TAG_RSC_TICKET, tag)) {
             unpack_rsc_ticket(xml_obj, data_set);
 
         } else {
             pe_err("Unsupported constraint type: %s", tag);
         }
     }
 
     return TRUE;
 }
 
 static const char *
 invert_action(const char *action)
 {
     if (safe_str_eq(action, RSC_START)) {
         return RSC_STOP;
 
     } else if (safe_str_eq(action, RSC_STOP)) {
         return RSC_START;
 
     } else if (safe_str_eq(action, RSC_PROMOTE)) {
         return RSC_DEMOTE;
 
     } else if (safe_str_eq(action, RSC_DEMOTE)) {
         return RSC_PROMOTE;
 
     } else if (safe_str_eq(action, RSC_PROMOTED)) {
         return RSC_DEMOTED;
 
     } else if (safe_str_eq(action, RSC_DEMOTED)) {
         return RSC_PROMOTED;
 
     } else if (safe_str_eq(action, RSC_STARTED)) {
         return RSC_STOPPED;
 
     } else if (safe_str_eq(action, RSC_STOPPED)) {
         return RSC_STARTED;
     }
     crm_warn("Unknown action '%s' specified in order constraint", action);
     return NULL;
 }
 
 static enum pe_order_kind
 get_ordering_type(xmlNode * xml_obj)
 {
     enum pe_order_kind kind_e = pe_order_kind_mandatory;
     const char *kind = crm_element_value(xml_obj, XML_ORDER_ATTR_KIND);
 
     if (kind == NULL) {
         const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
 
         kind_e = pe_order_kind_mandatory;
 
         if (score) {
             // @COMPAT deprecated informally since 1.0.7, formally since 2.0.1
             int score_i = char2score(score);
 
             if (score_i == 0) {
                 kind_e = pe_order_kind_optional;
             }
             pe_warn_once(pe_wo_order_score,
                          "Support for 'score' in rsc_order is deprecated "
                          "and will be removed in a future release (use 'kind' instead)");
         }
 
     } else if (safe_str_eq(kind, "Mandatory")) {
         kind_e = pe_order_kind_mandatory;
 
     } else if (safe_str_eq(kind, "Optional")) {
         kind_e = pe_order_kind_optional;
 
     } else if (safe_str_eq(kind, "Serialize")) {
         kind_e = pe_order_kind_serialize;
 
     } else {
         pcmk__config_err("Resetting '" XML_ORDER_ATTR_KIND "' for constraint "
                          "'%s' to Mandatory because '%s' is not valid",
                          crm_str(ID(xml_obj)), kind);
     }
     return kind_e;
 }
 
 static pe_resource_t *
 pe_find_constraint_resource(GListPtr rsc_list, const char *id)
 {
     GListPtr rIter = NULL;
 
     for (rIter = rsc_list; id && rIter; rIter = rIter->next) {
         pe_resource_t *parent = rIter->data;
         pe_resource_t *match = parent->fns->find_rsc(parent, id, NULL,
                                                      pe_find_renamed);
 
         if (match != NULL) {
             if(safe_str_neq(match->id, id)) {
                 /* We found an instance of a clone instead */
                 match = uber_parent(match);
                 crm_debug("Found %s for %s", match->id, id);
             }
             return match;
         }
     }
     crm_trace("No match for %s", id);
     return NULL;
 }
 
 static gboolean
 pe_find_constraint_tag(pe_working_set_t * data_set, const char * id, pe_tag_t ** tag)
 {
     gboolean rc = FALSE;
 
     *tag = NULL;
     rc = g_hash_table_lookup_extended(data_set->template_rsc_sets, id,
                                        NULL, (gpointer*) tag);
 
     if (rc == FALSE) {
         rc = g_hash_table_lookup_extended(data_set->tags, id,
                                           NULL, (gpointer*) tag);
 
         if (rc == FALSE) {
             crm_warn("No template or tag named '%s'", id);
             return FALSE;
 
         } else if (*tag == NULL) {
             crm_warn("No resource is tagged with '%s'", id);
             return FALSE;
         }
 
     } else if (*tag == NULL) {
         crm_warn("No resource is derived from template '%s'", id);
         return FALSE;
     }
 
     return rc;
 }
 
 static gboolean
 valid_resource_or_tag(pe_working_set_t * data_set, const char * id,
                       pe_resource_t ** rsc, pe_tag_t ** tag)
 {
     gboolean rc = FALSE;
 
     if (rsc) {
         *rsc = NULL;
         *rsc = pe_find_constraint_resource(data_set->resources, id);
         if (*rsc) {
             return TRUE;
         }
     }
 
     if (tag) {
         *tag = NULL;
         rc = pe_find_constraint_tag(data_set, id, tag);
     }
 
     return rc;
 }
 
 static gboolean
 order_is_symmetrical(xmlNode * xml_obj,
                      enum pe_order_kind parent_kind, const char * parent_symmetrical_s)
 {
     const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
     const char *kind_s = crm_element_value(xml_obj, XML_ORDER_ATTR_KIND);
     const char *score_s = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
     const char *symmetrical_s = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
     enum pe_order_kind kind = parent_kind;
 
     if (kind_s || score_s) {
         kind = get_ordering_type(xml_obj);
     }
 
     if (symmetrical_s == NULL) {
         symmetrical_s = parent_symmetrical_s;
     }
 
     if (symmetrical_s) {
         gboolean symmetrical = crm_is_true(symmetrical_s);
 
         if (symmetrical && kind == pe_order_kind_serialize) {
             pcmk__config_warn("Ignoring " XML_CONS_ATTR_SYMMETRICAL
                               " for '%s' because not valid with "
                               XML_ORDER_ATTR_KIND " of 'Serialize'", id);
             return FALSE;
         }
 
         return symmetrical;
 
     } else {
         if (kind == pe_order_kind_serialize) {
             return FALSE;
 
         } else {
             return TRUE;
         }
     }
 }
 
 static gboolean
 unpack_simple_rsc_order(xmlNode * xml_obj, pe_working_set_t * data_set)
 {
     int order_id = 0;
     pe_resource_t *rsc_then = NULL;
     pe_resource_t *rsc_first = NULL;
     gboolean invert_bool = TRUE;
     int min_required_before = 0;
     enum pe_order_kind kind = pe_order_kind_mandatory;
     enum pe_ordering cons_weight = pe_order_optional;
 
     const char *id_first = NULL;
     const char *id_then = NULL;
     const char *action_then = NULL;
     const char *action_first = NULL;
     const char *instance_then = NULL;
     const char *instance_first = NULL;
 
     const char *id = NULL;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     id = crm_element_value(xml_obj, XML_ATTR_ID);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     invert_bool = order_is_symmetrical(xml_obj, kind, NULL);
 
     id_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN);
     id_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST);
 
     action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
     action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
 
     instance_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_INSTANCE);
     instance_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_INSTANCE);
 
     if (action_first == NULL) {
         action_first = RSC_START;
     }
     if (action_then == NULL) {
         action_then = action_first;
     }
 
     if (id_first == NULL) {
         pcmk__config_err("Ignoring constraint '%s' without "
                          XML_ORDER_ATTR_FIRST, id);
         return FALSE;
     }
     if (id_then == NULL) {
         pcmk__config_err("Ignoring constraint '%s' without "
                          XML_ORDER_ATTR_THEN, id);
         return FALSE;
     }
 
     rsc_then = pe_find_constraint_resource(data_set->resources, id_then);
     rsc_first = pe_find_constraint_resource(data_set->resources, id_first);
 
     if (rsc_then == NULL) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "does not exist", id, id_then);
         return FALSE;
 
     } else if (rsc_first == NULL) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "does not exist", id, id_first);
         return FALSE;
 
     } else if (instance_then && pe_rsc_is_clone(rsc_then) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "is not a clone but instance '%s' was requested",
                          id, id_then, instance_then);
         return FALSE;
 
     } else if (instance_first && pe_rsc_is_clone(rsc_first) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "is not a clone but instance '%s' was requested",
                          id, id_first, instance_first);
         return FALSE;
     }
 
     if (instance_then) {
         rsc_then = find_clone_instance(rsc_then, instance_then, data_set);
         if (rsc_then == NULL) {
             pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
                               "does not have an instance '%s'",
                               id, id_then, instance_then);
             return FALSE;
         }
     }
 
     if (instance_first) {
         rsc_first = find_clone_instance(rsc_first, instance_first, data_set);
         if (rsc_first == NULL) {
             pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
                               "does not have an instance '%s'",
                               "'%s'", id, id_first, instance_first);
             return FALSE;
         }
     }
 
     cons_weight = pe_order_optional;
     kind = get_ordering_type(xml_obj);
 
     if (kind == pe_order_kind_optional && rsc_then->restart_type == pe_restart_restart) {
         crm_trace("Upgrade : recovery - implies right");
         cons_weight |= pe_order_implies_then;
     }
 
     if (invert_bool == FALSE) {
         cons_weight |= get_asymmetrical_flags(kind);
     } else {
         cons_weight |= get_flags(id, kind, action_first, action_then, FALSE);
     }
 
     if (pe_rsc_is_clone(rsc_first)) {
         /* If clone-min is set, require at least that number of instances to be
          * runnable before allowing dependencies to be runnable.
          */
         const char *min_clones_s = g_hash_table_lookup(rsc_first->meta,
                                                        XML_RSC_ATTR_INCARNATION_MIN);
 
         // @COMPAT 1.1.13: deprecated
         const char *require_all_s = crm_element_value(xml_obj, "require-all");
 
         if (min_clones_s) {
             min_required_before = crm_parse_int(min_clones_s, "0");
 
         } else if (require_all_s) {
             pe_warn_once(pe_wo_require_all,
                         "Support for require-all in ordering constraints "
                         "is deprecated and will be removed in a future release"
                         " (use clone-min clone meta-attribute instead)");
             if (crm_is_true(require_all_s) == FALSE) {
                 // require-all=false is deprecated equivalent of clone-min=1
                 min_required_before = 1;
             }
         }
     }
 
     /* If there is a minimum number of instances that must be runnable before
      * the 'then' action is runnable, we use a pseudo action as an intermediate step
      * start min number of clones -> pseudo action is runnable -> dependency runnable. */
     if (min_required_before) {
         GListPtr rIter = NULL;
         char *task = crm_strdup_printf(CRM_OP_RELAXED_CLONE ":%s", id);
         pe_action_t *unordered_action = get_pseudo_op(task, data_set);
         free(task);
 
         /* require the pseudo action to have "min_required_before" number of
          * actions to be considered runnable before allowing the pseudo action
          * to be runnable. */ 
         unordered_action->required_runnable_before = min_required_before;
         update_action_flags(unordered_action, pe_action_requires_any, __FUNCTION__, __LINE__);
 
         for (rIter = rsc_first->children; id && rIter; rIter = rIter->next) {
             pe_resource_t *child = rIter->data;
             /* order each clone instance before the pseudo action */
             custom_action_order(child, pcmk__op_key(child->id, action_first, 0),
                                 NULL, NULL, NULL, unordered_action,
                                 pe_order_one_or_more|pe_order_implies_then_printed,
                                 data_set);
         }
 
         /* order the "then" dependency to occur after the pseudo action only if
          * the pseudo action is runnable */ 
         order_id = custom_action_order(NULL, NULL, unordered_action, rsc_then,
                                        pcmk__op_key(rsc_then->id, action_then, 0),
                                        NULL, cons_weight|pe_order_runnable_left,
                                        data_set);
     } else {
         order_id = new_rsc_order(rsc_first, action_first, rsc_then, action_then, cons_weight, data_set);
     }
 
     pe_rsc_trace(rsc_first, "order-%d (%s): %s_%s before %s_%s flags=0x%.6x",
                  order_id, id, rsc_first->id, action_first, rsc_then->id, action_then, cons_weight);
 
     if (invert_bool == FALSE) {
         return TRUE;
     }
 
     action_then = invert_action(action_then);
     action_first = invert_action(action_first);
     if (action_then == NULL || action_first == NULL) {
         pcmk__config_err("Cannot invert constraint '%s' "
                          "(please specify inverse manually)", id);
         return TRUE;
     }
 
     cons_weight = pe_order_optional;
     if (kind == pe_order_kind_optional && rsc_then->restart_type == pe_restart_restart) {
         crm_trace("Upgrade : recovery - implies left");
         cons_weight |= pe_order_implies_first;
     }
 
     cons_weight |= get_flags(id, kind, action_first, action_then, TRUE);
 
     order_id = new_rsc_order(rsc_then, action_then, rsc_first, action_first, cons_weight, data_set);
 
     pe_rsc_trace(rsc_then, "order-%d (%s): %s_%s before %s_%s flags=0x%.6x",
                  order_id, id, rsc_then->id, action_then, rsc_first->id, action_first, cons_weight);
 
     return TRUE;
 }
 
 static gboolean
 expand_tags_in_sets(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
 {
     xmlNode *new_xml = NULL;
     xmlNode *set = NULL;
     gboolean any_refs = FALSE;
     const char *cons_id = NULL;
 
     *expanded_xml = NULL;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     new_xml = copy_xml(xml_obj);
     cons_id = ID(new_xml);
 
     for (set = __xml_first_child_element(new_xml); set != NULL;
          set = __xml_next_element(set)) {
 
         xmlNode *xml_rsc = NULL;
         GListPtr tag_refs = NULL;
         GListPtr gIter = NULL;
 
         if (safe_str_neq((const char *)set->name, XML_CONS_TAG_RSC_SET)) {
             continue;
         }
 
         for (xml_rsc = __xml_first_child_element(set); xml_rsc != NULL;
              xml_rsc = __xml_next_element(xml_rsc)) {
 
             pe_resource_t *rsc = NULL;
             pe_tag_t *tag = NULL;
             const char *id = ID(xml_rsc);
 
             if (safe_str_neq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF)) {
                 continue;
             }
 
             if (valid_resource_or_tag(data_set, id, &rsc, &tag) == FALSE) {
                 pcmk__config_err("Ignoring resource sets for constraint '%s' "
                                  "because '%s' is not a valid resource or tag",
                                  cons_id, id);
                 free_xml(new_xml);
                 return FALSE;
 
             } else if (rsc) {
                 continue;
 
             } else if (tag) {
                 /* The resource_ref under the resource_set references a template/tag */
                 xmlNode *last_ref = xml_rsc;
 
                 /* A sample:
 
                    Original XML:
 
                    <resource_set id="tag1-colocation-0" sequential="true">
                      <resource_ref id="rsc1"/>
                      <resource_ref id="tag1"/>
                      <resource_ref id="rsc4"/>
                    </resource_set>
 
                    Now we are appending rsc2 and rsc3 which are tagged with tag1 right after it:
 
                    <resource_set id="tag1-colocation-0" sequential="true">
                      <resource_ref id="rsc1"/>
                      <resource_ref id="tag1"/>
                      <resource_ref id="rsc2"/>
                      <resource_ref id="rsc3"/>
                      <resource_ref id="rsc4"/>
                    </resource_set>
 
                  */
 
                 for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
                     const char *obj_ref = (const char *) gIter->data;
                     xmlNode *new_rsc_ref = NULL;
 
                     new_rsc_ref = xmlNewDocRawNode(getDocPtr(set), NULL,
                                                    (pcmkXmlStr) XML_TAG_RESOURCE_REF, NULL);
                     crm_xml_add(new_rsc_ref, XML_ATTR_ID, obj_ref);
                     xmlAddNextSibling(last_ref, new_rsc_ref);
 
                     last_ref = new_rsc_ref;
                 }
 
                 any_refs = TRUE;
 
                 /* Do not directly free '<resource_ref id="tag1"/>'.
                    That would break the further __xml_next_element(xml_rsc)) and cause "Invalid read" seen by valgrind.
                    So just record it into a hash table for freeing it later.
                  */
                 tag_refs = g_list_append(tag_refs, xml_rsc);
             }
         }
 
         /* Now free '<resource_ref id="tag1"/>', and finally get:
 
            <resource_set id="tag1-colocation-0" sequential="true">
              <resource_ref id="rsc1"/>
              <resource_ref id="rsc2"/>
              <resource_ref id="rsc3"/>
              <resource_ref id="rsc4"/>
            </resource_set>
 
          */
         for (gIter = tag_refs; gIter != NULL; gIter = gIter->next) {
             xmlNode *tag_ref = gIter->data;
 
             free_xml(tag_ref);
         }
         g_list_free(tag_refs);
     }
 
     if (any_refs) {
         *expanded_xml = new_xml;
     } else {
         free_xml(new_xml);
     }
 
     return TRUE;
 }
 
 static gboolean
 tag_to_set(xmlNode * xml_obj, xmlNode ** rsc_set, const char * attr,
                 gboolean convert_rsc, pe_working_set_t * data_set)
 {
     const char *cons_id = NULL;
     const char *id = NULL;
 
     pe_resource_t *rsc = NULL;
     pe_tag_t *tag = NULL;
 
     *rsc_set = NULL;
 
     CRM_CHECK((xml_obj != NULL) && (attr != NULL), return FALSE);
 
     cons_id = ID(xml_obj);
     if (cons_id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     id = crm_element_value(xml_obj, attr);
     if (id == NULL) {
         return TRUE;
     }
 
     if (valid_resource_or_tag(data_set, id, &rsc, &tag) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag", cons_id, id);
         return FALSE;
 
     } else if (tag) {
         GListPtr gIter = NULL;
 
         /* A template/tag is referenced by the "attr" attribute (first, then, rsc or with-rsc).
            Add the template/tag's corresponding "resource_set" which contains the resources derived
            from it or tagged with it under the constraint. */
         *rsc_set = create_xml_node(xml_obj, XML_CONS_TAG_RSC_SET);
         crm_xml_add(*rsc_set, XML_ATTR_ID, id);
 
         for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
             const char *obj_ref = (const char *) gIter->data;
             xmlNode *rsc_ref = NULL;
 
             rsc_ref = create_xml_node(*rsc_set, XML_TAG_RESOURCE_REF);
             crm_xml_add(rsc_ref, XML_ATTR_ID, obj_ref);
         }
 
         /* Set sequential="false" for the resource_set */
         crm_xml_add(*rsc_set, "sequential", XML_BOOLEAN_FALSE);
 
     } else if (rsc && convert_rsc) {
         /* Even a regular resource is referenced by "attr", convert it into a resource_set.
            Because the other side of the constraint could be a template/tag reference. */
         xmlNode *rsc_ref = NULL;
 
         *rsc_set = create_xml_node(xml_obj, XML_CONS_TAG_RSC_SET);
         crm_xml_add(*rsc_set, XML_ATTR_ID, id);
 
         rsc_ref = create_xml_node(*rsc_set, XML_TAG_RESOURCE_REF);
         crm_xml_add(rsc_ref, XML_ATTR_ID, id);
 
     } else {
         return TRUE;
     }
 
     /* Remove the "attr" attribute referencing the template/tag */
     if (*rsc_set) {
         xml_remove_prop(xml_obj, attr);
     }
 
     return TRUE;
 }
 
 static gboolean unpack_rsc_location(xmlNode * xml_obj, pe_resource_t * rsc_lh, const char * role,
                              const char * score, pe_working_set_t * data_set, pe_match_data_t * match_data);
 
 static gboolean
 unpack_simple_location(xmlNode * xml_obj, pe_working_set_t * data_set)
 {
     const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
     const char *value = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);
 
     if(value) {
         pe_resource_t *rsc_lh = pe_find_constraint_resource(data_set->resources, value);
 
         return unpack_rsc_location(xml_obj, rsc_lh, NULL, NULL, data_set, NULL);
     }
 
     value = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE_PATTERN);
     if(value) {
         regex_t *r_patt = calloc(1, sizeof(regex_t));
         bool invert = FALSE;
         GListPtr rIter = NULL;
 
         if(value[0] == '!') {
             value++;
             invert = TRUE;
         }
 
         if (regcomp(r_patt, value, REG_EXTENDED)) {
             pcmk__config_err("Ignoring constraint '%s' because "
                              XML_LOC_ATTR_SOURCE_PATTERN
                              " has invalid value '%s'", id, value);
             regfree(r_patt);
             free(r_patt);
             return FALSE;
         }
 
         for (rIter = data_set->resources; rIter; rIter = rIter->next) {
             pe_resource_t *r = rIter->data;
             int nregs = 0;
             regmatch_t *pmatch = NULL;
             int status;
 
             if(r_patt->re_nsub > 0) {
                 nregs = r_patt->re_nsub + 1;
             } else {
                 nregs = 1;
             }
             pmatch = calloc(nregs, sizeof(regmatch_t));
 
             status = regexec(r_patt, r->id, nregs, pmatch, 0);
 
             if(invert == FALSE && status == 0) {
                 pe_re_match_data_t re_match_data = {
                                                 .string = r->id,
                                                 .nregs = nregs,
                                                 .pmatch = pmatch
                                                };
                 pe_match_data_t match_data = {
                                                 .re = &re_match_data,
                                                 .params = r->parameters,
                                                 .meta = r->meta,
                                              };
                 crm_debug("'%s' matched '%s' for %s", r->id, value, id);
                 unpack_rsc_location(xml_obj, r, NULL, NULL, data_set, &match_data);
 
             } else if (invert && (status != 0)) {
                 crm_debug("'%s' is an inverted match of '%s' for %s", r->id, value, id);
                 unpack_rsc_location(xml_obj, r, NULL, NULL, data_set, NULL);
 
             } else {
                 crm_trace("'%s' does not match '%s' for %s", r->id, value, id);
             }
 
             free(pmatch);
         }
 
         regfree(r_patt);
         free(r_patt);
     }
 
     return FALSE;
 }
 
 static gboolean
 unpack_rsc_location(xmlNode * xml_obj, pe_resource_t * rsc_lh, const char * role,
                     const char * score, pe_working_set_t * data_set, pe_match_data_t * match_data)
 {
     pe__location_t *location = NULL;
     const char *id_lh = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);
     const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
     const char *node = crm_element_value(xml_obj, XML_CIB_TAG_NODE);
     const char *discovery = crm_element_value(xml_obj, XML_LOCATION_ATTR_DISCOVERY);
 
     if (rsc_lh == NULL) {
         pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
                           "does not exist", id, id_lh);
         return FALSE;
     }
 
     if (score == NULL) {
         score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
     }
 
     if (node != NULL && score != NULL) {
         int score_i = char2score(score);
         pe_node_t *match = pe_find_node(data_set->nodes, node);
 
         if (!match) {
             return FALSE;
         }
         location = rsc2node_new(id, rsc_lh, score_i, discovery, match, data_set);
 
     } else {
         bool empty = TRUE;
         crm_time_t *next_change = crm_time_new_undefined();
 
         /* This loop is logically parallel to pe_evaluate_rules(), except
          * instead of checking whether any rule is active, we set up location
          * constraints for each active rule.
          */
         for (xmlNode *rule_xml = first_named_child(xml_obj, XML_TAG_RULE);
              rule_xml != NULL; rule_xml = crm_next_same_xml(rule_xml)) {
             empty = FALSE;
             crm_trace("Unpacking %s/%s", id, ID(rule_xml));
             generate_location_rule(rsc_lh, rule_xml, discovery, next_change,
                                    data_set, match_data);
         }
 
         if (empty) {
             pcmk__config_err("Ignoring constraint '%s' because it contains "
                              "no rules", id);
         }
 
         /* If there is a point in the future when the evaluation of a rule will
          * change, make sure the scheduler is re-run by that time.
          */
         if (crm_time_is_defined(next_change)) {
             time_t t = (time_t) crm_time_get_seconds_since_epoch(next_change);
 
             pe__update_recheck_time(t, data_set);
         }
         crm_time_free(next_change);
         return TRUE;
     }
 
     if (role == NULL) {
         role = crm_element_value(xml_obj, XML_RULE_ATTR_ROLE);
     }
 
     if (location && role) {
         if (text2role(role) == RSC_ROLE_UNKNOWN) {
             pe_err("Invalid constraint %s: Bad role %s", id, role);
             return FALSE;
 
         } else {
             enum rsc_role_e r = text2role(role);
             switch(r) {
                 case RSC_ROLE_UNKNOWN:
                 case RSC_ROLE_STARTED:
                 case RSC_ROLE_SLAVE:
                     /* Applies to all */
                     location->role_filter = RSC_ROLE_UNKNOWN;
                     break;
                 default:
                     location->role_filter = r;
                     break;
             }
         }
     }
 
     return TRUE;
 }
 
 static gboolean
 unpack_location_tags(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     const char *id_lh = NULL;
     const char *state_lh = NULL;
 
     pe_resource_t *rsc_lh = NULL;
 
     pe_tag_t *tag_lh = NULL;
 
     xmlNode *new_xml = NULL;
     xmlNode *rsc_set_lh = NULL;
 
     *expanded_xml = NULL;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     id = ID(xml_obj);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     /* Attempt to expand any template/tag references in possible resource sets. */
     expand_tags_in_sets(xml_obj, &new_xml, data_set);
     if (new_xml) {
         /* There are resource sets referencing templates. Return with the expanded XML. */
         crm_log_xml_trace(new_xml, "Expanded rsc_location...");
         *expanded_xml = new_xml;
         return TRUE;
     }
 
     id_lh = crm_element_value(xml_obj, XML_LOC_ATTR_SOURCE);
     if (id_lh == NULL) {
         return TRUE;
     }
 
     if (valid_resource_or_tag(data_set, id_lh, &rsc_lh, &tag_lh) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag", id, id_lh);
         return FALSE;
 
     } else if (rsc_lh) {
         /* No template is referenced. */
         return TRUE;
     }
 
     state_lh = crm_element_value(xml_obj, XML_RULE_ATTR_ROLE);
 
     new_xml = copy_xml(xml_obj);
 
     /* Convert the template/tag reference in "rsc" into a resource_set under the rsc_location constraint. */
     if (tag_to_set(new_xml, &rsc_set_lh, XML_LOC_ATTR_SOURCE, FALSE, data_set) == FALSE) {
         free_xml(new_xml);
         return FALSE;
     }
 
     if (rsc_set_lh) {
         if (state_lh) {
             /* A "rsc-role" is specified.
                Move it into the converted resource_set as a "role"" attribute. */
             crm_xml_add(rsc_set_lh, "role", state_lh);
             xml_remove_prop(new_xml, XML_RULE_ATTR_ROLE);
         }
         crm_log_xml_trace(new_xml, "Expanded rsc_location...");
         *expanded_xml = new_xml;
 
     } else {
         /* No sets */
         free_xml(new_xml);
     }
 
     return TRUE;
 }
 
 static gboolean
 unpack_location_set(xmlNode * location, xmlNode * set, pe_working_set_t * data_set)
 {
     xmlNode *xml_rsc = NULL;
     pe_resource_t *resource = NULL;
     const char *set_id;
     const char *role;
     const char *local_score;
 
     CRM_CHECK(set != NULL, return FALSE);
 
     set_id = ID(set);
     if (set_id == NULL) {
         pcmk__config_err("Ignoring " XML_CONS_TAG_RSC_SET " without "
                          XML_ATTR_ID " in constraint '%s'",
                          crm_str(ID(location)));
         return FALSE;
     }
 
     role = crm_element_value(set, "role");
     local_score = crm_element_value(set, XML_RULE_ATTR_SCORE);
 
     for (xml_rsc = __xml_first_child_element(set); xml_rsc != NULL;
          xml_rsc = __xml_next_element(xml_rsc)) {
 
         if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
             EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
             unpack_rsc_location(location, resource, role, local_score, data_set, NULL);
         }
     }
 
     return TRUE;
 }
 
 gboolean
 unpack_location(xmlNode * xml_obj, pe_working_set_t * data_set)
 {
     xmlNode *set = NULL;
     gboolean any_sets = FALSE;
 
     xmlNode *orig_xml = NULL;
     xmlNode *expanded_xml = NULL;
 
     if (unpack_location_tags(xml_obj, &expanded_xml, data_set) == FALSE) {
         return FALSE;
     }
 
     if (expanded_xml) {
         orig_xml = xml_obj;
         xml_obj = expanded_xml;
     }
 
     for (set = __xml_first_child_element(xml_obj); set != NULL;
          set = __xml_next_element(set)) {
 
         if (crm_str_eq((const char *)set->name, XML_CONS_TAG_RSC_SET, TRUE)) {
             any_sets = TRUE;
             set = expand_idref(set, data_set->input);
             if (unpack_location_set(xml_obj, set, data_set) == FALSE) {
                 if (expanded_xml) {
                     free_xml(expanded_xml);
                 }
                 return FALSE;
             }
         }
     }
 
     if (expanded_xml) {
         free_xml(expanded_xml);
         xml_obj = orig_xml;
     }
 
     if (any_sets == FALSE) {
         return unpack_simple_location(xml_obj, data_set);
     }
 
     return TRUE;
 }
 
 static int
 get_node_score(const char *rule, const char *score, gboolean raw, pe_node_t * node, pe_resource_t *rsc)
 {
     int score_f = 0;
 
     if (score == NULL) {
         pe_err("Rule %s: no score specified.  Assuming 0.", rule);
 
     } else if (raw) {
         score_f = char2score(score);
 
     } else {
         const char *attr_score = pe_node_attribute_calculated(node, score, rsc);
 
         if (attr_score == NULL) {
             crm_debug("Rule %s: node %s did not have a value for %s",
                       rule, node->details->uname, score);
             score_f = -INFINITY;
 
         } else {
             crm_debug("Rule %s: node %s had value %s for %s",
                       rule, node->details->uname, attr_score, score);
             score_f = char2score(attr_score);
         }
     }
     return score_f;
 }
 
 static pe__location_t *
 generate_location_rule(pe_resource_t *rsc, xmlNode *rule_xml,
                        const char *discovery, crm_time_t *next_change,
                        pe_working_set_t *data_set, pe_match_data_t *match_data)
 {
     const char *rule_id = NULL;
     const char *score = NULL;
     const char *boolean = NULL;
     const char *role = NULL;
 
     GListPtr gIter = NULL;
     GListPtr match_L = NULL;
 
     gboolean do_and = TRUE;
     gboolean accept = TRUE;
     gboolean raw_score = TRUE;
     gboolean score_allocated = FALSE;
 
     pe__location_t *location_rule = NULL;
 
     rule_xml = expand_idref(rule_xml, data_set->input);
     rule_id = crm_element_value(rule_xml, XML_ATTR_ID);
     boolean = crm_element_value(rule_xml, XML_RULE_ATTR_BOOLEAN_OP);
     role = crm_element_value(rule_xml, XML_RULE_ATTR_ROLE);
 
     crm_trace("Processing rule: %s", rule_id);
 
     if (role != NULL && text2role(role) == RSC_ROLE_UNKNOWN) {
         pe_err("Bad role specified for %s: %s", rule_id, role);
         return NULL;
     }
 
     score = crm_element_value(rule_xml, XML_RULE_ATTR_SCORE);
     if (score == NULL) {
         score = crm_element_value(rule_xml, XML_RULE_ATTR_SCORE_ATTRIBUTE);
         if (score != NULL) {
             raw_score = FALSE;
         }
     }
     if (safe_str_eq(boolean, "or")) {
         do_and = FALSE;
     }
 
     location_rule = rsc2node_new(rule_id, rsc, 0, discovery, NULL, data_set);
 
     if (location_rule == NULL) {
         return NULL;
     }
 
     if (match_data && match_data->re && match_data->re->nregs > 0 && match_data->re->pmatch[0].rm_so != -1) {
         if (raw_score == FALSE) {
             char *result = pe_expand_re_matches(score, match_data->re);
 
             if (result) {
                 score = (const char *) result;
                 score_allocated = TRUE;
             }
         }
     }
 
     if (role != NULL) {
         crm_trace("Setting role filter: %s", role);
         location_rule->role_filter = text2role(role);
         if (location_rule->role_filter == RSC_ROLE_SLAVE) {
             /* Any promotable clone cannot be promoted without being a slave first
              * Ergo, any constraint for the slave role applies to every role
              */
             location_rule->role_filter = RSC_ROLE_UNKNOWN;
         }
     }
     if (do_and) {
         GListPtr gIter = NULL;
 
         match_L = node_list_dup(data_set->nodes, TRUE, FALSE);
         for (gIter = match_L; gIter != NULL; gIter = gIter->next) {
             pe_node_t *node = (pe_node_t *) gIter->data;
 
             node->weight = get_node_score(rule_id, score, raw_score, node, rsc);
         }
     }
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         int score_f = 0;
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         accept = pe_test_rule(rule_xml, node->details->attrs, RSC_ROLE_UNKNOWN,
                               data_set->now, next_change, match_data);
 
         crm_trace("Rule %s %s on %s", ID(rule_xml), accept ? "passed" : "failed",
                   node->details->uname);
 
         score_f = get_node_score(rule_id, score, raw_score, node, rsc);
 /* 			if(accept && score_f == -INFINITY) { */
 /* 				accept = FALSE; */
 /* 			} */
 
         if (accept) {
             pe_node_t *local = pe_find_node_id(match_L, node->details->id);
 
             if (local == NULL && do_and) {
                 continue;
 
             } else if (local == NULL) {
                 local = node_copy(node);
                 match_L = g_list_append(match_L, local);
             }
 
             if (do_and == FALSE) {
                 local->weight = merge_weights(local->weight, score_f);
             }
             crm_trace("node %s now has weight %d", node->details->uname, local->weight);
 
         } else if (do_and && !accept) {
             /* remove it */
             pe_node_t *delete = pe_find_node_id(match_L, node->details->id);
 
             if (delete != NULL) {
                 match_L = g_list_remove(match_L, delete);
                 crm_trace("node %s did not match", node->details->uname);
             }
             free(delete);
         }
     }
 
     if (score_allocated == TRUE) {
         free((char *)score);
     }
 
     location_rule->node_list_rh = match_L;
     if (location_rule->node_list_rh == NULL) {
         crm_trace("No matching nodes for rule %s", rule_id);
         return NULL;
     }
 
     crm_trace("%s: %d nodes matched", rule_id, g_list_length(location_rule->node_list_rh));
     return location_rule;
 }
 
 static gint
 sort_cons_priority_lh(gconstpointer a, gconstpointer b)
 {
     const rsc_colocation_t *rsc_constraint1 = (const rsc_colocation_t *)a;
     const rsc_colocation_t *rsc_constraint2 = (const rsc_colocation_t *)b;
 
     if (a == NULL) {
         return 1;
     }
     if (b == NULL) {
         return -1;
     }
 
     CRM_ASSERT(rsc_constraint1->rsc_lh != NULL);
     CRM_ASSERT(rsc_constraint1->rsc_rh != NULL);
 
     if (rsc_constraint1->rsc_lh->priority > rsc_constraint2->rsc_lh->priority) {
         return -1;
     }
 
     if (rsc_constraint1->rsc_lh->priority < rsc_constraint2->rsc_lh->priority) {
         return 1;
     }
 
     /* Process clones before primitives and groups */
     if (rsc_constraint1->rsc_lh->variant > rsc_constraint2->rsc_lh->variant) {
         return -1;
     } else if (rsc_constraint1->rsc_lh->variant < rsc_constraint2->rsc_lh->variant) {
         return 1;
     }
 
     /* @COMPAT scheduler <2.0.0: Process promotable clones before nonpromotable
      * clones (probably unnecessary, but avoids having to update regression
      * tests)
      */
     if (rsc_constraint1->rsc_lh->variant == pe_clone) {
         if (is_set(rsc_constraint1->rsc_lh->flags, pe_rsc_promotable)
             && is_not_set(rsc_constraint2->rsc_lh->flags, pe_rsc_promotable)) {
             return -1;
         } else if (is_not_set(rsc_constraint1->rsc_lh->flags, pe_rsc_promotable)
             && is_set(rsc_constraint2->rsc_lh->flags, pe_rsc_promotable)) {
             return 1;
         }
     }
 
     return strcmp(rsc_constraint1->rsc_lh->id, rsc_constraint2->rsc_lh->id);
 }
 
 static gint
 sort_cons_priority_rh(gconstpointer a, gconstpointer b)
 {
     const rsc_colocation_t *rsc_constraint1 = (const rsc_colocation_t *)a;
     const rsc_colocation_t *rsc_constraint2 = (const rsc_colocation_t *)b;
 
     if (a == NULL) {
         return 1;
     }
     if (b == NULL) {
         return -1;
     }
 
     CRM_ASSERT(rsc_constraint1->rsc_lh != NULL);
     CRM_ASSERT(rsc_constraint1->rsc_rh != NULL);
 
     if (rsc_constraint1->rsc_rh->priority > rsc_constraint2->rsc_rh->priority) {
         return -1;
     }
 
     if (rsc_constraint1->rsc_rh->priority < rsc_constraint2->rsc_rh->priority) {
         return 1;
     }
 
     /* Process clones before primitives and groups */
     if (rsc_constraint1->rsc_rh->variant > rsc_constraint2->rsc_rh->variant) {
         return -1;
     } else if (rsc_constraint1->rsc_rh->variant < rsc_constraint2->rsc_rh->variant) {
         return 1;
     }
 
     /* @COMPAT scheduler <2.0.0: Process promotable clones before nonpromotable
      * clones (probably unnecessary, but avoids having to update regression
      * tests)
      */
     if (rsc_constraint1->rsc_rh->variant == pe_clone) {
         if (is_set(rsc_constraint1->rsc_rh->flags, pe_rsc_promotable)
             && is_not_set(rsc_constraint2->rsc_rh->flags, pe_rsc_promotable)) {
             return -1;
         } else if (is_not_set(rsc_constraint1->rsc_rh->flags, pe_rsc_promotable)
             && is_set(rsc_constraint2->rsc_rh->flags, pe_rsc_promotable)) {
             return 1;
         }
     }
 
     return strcmp(rsc_constraint1->rsc_rh->id, rsc_constraint2->rsc_rh->id);
 }
 
 static void
 anti_colocation_order(pe_resource_t * first_rsc, int first_role,
                       pe_resource_t * then_rsc, int then_role,
                       pe_working_set_t * data_set)
 {
     const char *first_tasks[] = { NULL, NULL };
     const char *then_tasks[] = { NULL, NULL };
     int first_lpc = 0;
     int then_lpc = 0;
 
     /* Actions to make first_rsc lose first_role */
     if (first_role == RSC_ROLE_MASTER) {
         first_tasks[0] = CRMD_ACTION_DEMOTE;
 
     } else {
         first_tasks[0] = CRMD_ACTION_STOP;
 
         if (first_role == RSC_ROLE_SLAVE) {
             first_tasks[1] = CRMD_ACTION_PROMOTE;
         }
     }
 
     /* Actions to make then_rsc gain then_role */
     if (then_role == RSC_ROLE_MASTER) {
         then_tasks[0] = CRMD_ACTION_PROMOTE;
 
     } else {
         then_tasks[0] = CRMD_ACTION_START;
 
         if (then_role == RSC_ROLE_SLAVE) {
             then_tasks[1] = CRMD_ACTION_DEMOTE;
         }
     }
 
     for (first_lpc = 0; first_lpc <= 1 && first_tasks[first_lpc] != NULL; first_lpc++) {
         for (then_lpc = 0; then_lpc <= 1 && then_tasks[then_lpc] != NULL; then_lpc++) {
             new_rsc_order(first_rsc, first_tasks[first_lpc], then_rsc, then_tasks[then_lpc],
                           pe_order_anti_colocation, data_set);
         }
     }
 }
 
 gboolean
 rsc_colocation_new(const char *id, const char *node_attr, int score,
                    pe_resource_t * rsc_lh, pe_resource_t * rsc_rh,
                    const char *state_lh, const char *state_rh, pe_working_set_t * data_set)
 {
     rsc_colocation_t *new_con = NULL;
 
     if ((rsc_lh == NULL) || (rsc_rh == NULL)) {
         pcmk__config_err("Ignoring colocation '%s' because resource "
                          "does not exist", id);
         return FALSE;
     }
 
     new_con = calloc(1, sizeof(rsc_colocation_t));
     if (new_con == NULL) {
         return FALSE;
     }
 
     if (state_lh == NULL || safe_str_eq(state_lh, RSC_ROLE_STARTED_S)) {
         state_lh = RSC_ROLE_UNKNOWN_S;
     }
 
     if (state_rh == NULL || safe_str_eq(state_rh, RSC_ROLE_STARTED_S)) {
         state_rh = RSC_ROLE_UNKNOWN_S;
     }
 
     new_con->id = id;
     new_con->rsc_lh = rsc_lh;
     new_con->rsc_rh = rsc_rh;
     new_con->score = score;
     new_con->role_lh = text2role(state_lh);
     new_con->role_rh = text2role(state_rh);
     new_con->node_attribute = node_attr;
 
     if (node_attr == NULL) {
         node_attr = CRM_ATTR_UNAME;
     }
 
     pe_rsc_trace(rsc_lh, "%s ==> %s (%s %d)", rsc_lh->id, rsc_rh->id, node_attr, score);
 
     rsc_lh->rsc_cons = g_list_insert_sorted(rsc_lh->rsc_cons, new_con, sort_cons_priority_rh);
 
     rsc_rh->rsc_cons_lhs =
         g_list_insert_sorted(rsc_rh->rsc_cons_lhs, new_con, sort_cons_priority_lh);
 
     data_set->colocation_constraints = g_list_append(data_set->colocation_constraints, new_con);
 
     if (score <= -INFINITY) {
         anti_colocation_order(rsc_lh, new_con->role_lh, rsc_rh, new_con->role_rh, data_set);
         anti_colocation_order(rsc_rh, new_con->role_rh, rsc_lh, new_con->role_lh, data_set);
     }
 
     return TRUE;
 }
 
 /* LHS before RHS */
 int
 new_rsc_order(pe_resource_t * lh_rsc, const char *lh_task,
               pe_resource_t * rh_rsc, const char *rh_task,
               enum pe_ordering type, pe_working_set_t * data_set)
 {
     char *lh_key = NULL;
     char *rh_key = NULL;
 
     CRM_CHECK(lh_rsc != NULL, return -1);
     CRM_CHECK(lh_task != NULL, return -1);
     CRM_CHECK(rh_rsc != NULL, return -1);
     CRM_CHECK(rh_task != NULL, return -1);
 
 #if 0
     /* We do not need to test if these reference stonith resources
      * because the fencer has access to them even when they're not "running"
      */
     if (validate_order_resources(lh_rsc, lh_task, rh_rsc, rh_task)) {
         return -1;
     }
 #endif
 
     lh_key = pcmk__op_key(lh_rsc->id, lh_task, 0);
     rh_key = pcmk__op_key(rh_rsc->id, rh_task, 0);
 
     return custom_action_order(lh_rsc, lh_key, NULL, rh_rsc, rh_key, NULL, type, data_set);
 }
 
 static char *
 task_from_action_or_key(pe_action_t *action, const char *key)
 {
     char *res = NULL;
 
     if (action) {
         res = strdup(action->task);
     } else if (key) {
         parse_op_key(key, NULL, &res, NULL);
     }
     return res;
 }
 
 /* when order constraints are made between two resources start and stop actions
  * those constraints have to be mirrored against the corresponding
  * migration actions to ensure start/stop ordering is preserved during
  * a migration */
 static void
 handle_migration_ordering(pe__ordering_t *order, pe_working_set_t *data_set)
 {
     char *lh_task = NULL;
     char *rh_task = NULL;
     gboolean rh_migratable;
     gboolean lh_migratable;
 
     if (order->lh_rsc == NULL || order->rh_rsc == NULL) {
         return;
     } else if (order->lh_rsc == order->rh_rsc) {
         return;
     /* don't mess with those constraints built between parent
      * resources and the children */
     } else if (is_parent(order->lh_rsc, order->rh_rsc)) {
         return;
     } else if (is_parent(order->rh_rsc, order->lh_rsc)) {
         return;
     }
 
     lh_migratable = is_set(order->lh_rsc->flags, pe_rsc_allow_migrate);
     rh_migratable = is_set(order->rh_rsc->flags, pe_rsc_allow_migrate);
 
     /* one of them has to be migratable for
      * the migrate ordering logic to be applied */
     if (lh_migratable == FALSE && rh_migratable == FALSE) {
         return;
     }
 
     /* at this point we have two resources which allow migrations that have an
      * order dependency set between them.  If those order dependencies involve
      * start/stop actions, we need to mirror the corresponding migrate actions
      * so order will be preserved. */
     lh_task = task_from_action_or_key(order->lh_action, order->lh_action_task);
     rh_task = task_from_action_or_key(order->rh_action, order->rh_action_task);
     if (lh_task == NULL || rh_task == NULL) {
         goto cleanup_order;
     }
 
     if (safe_str_eq(lh_task, RSC_START) && safe_str_eq(rh_task, RSC_START)) {
         int flags = pe_order_optional;
 
         if (lh_migratable && rh_migratable) {
             /* A start then B start
              * A migrate_from then B migrate_to */
             custom_action_order(order->lh_rsc,
                                 pcmk__op_key(order->lh_rsc->id, RSC_MIGRATED, 0),
                                 NULL, order->rh_rsc,
                                 pcmk__op_key(order->rh_rsc->id, RSC_MIGRATE, 0),
                                 NULL, flags, data_set);
         }
 
         if (rh_migratable) {
             if (lh_migratable) {
                 flags |= pe_order_apply_first_non_migratable;
             }
 
             /* A start then B start
              * A start then B migrate_to... only if A start is not a part of a migration*/
             custom_action_order(order->lh_rsc,
                                 pcmk__op_key(order->lh_rsc->id, RSC_START, 0),
                                 NULL, order->rh_rsc,
                                 pcmk__op_key(order->rh_rsc->id, RSC_MIGRATE, 0),
                                 NULL, flags, data_set);
         }
 
     } else if (rh_migratable == TRUE && safe_str_eq(lh_task, RSC_STOP) && safe_str_eq(rh_task, RSC_STOP)) {
         int flags = pe_order_optional;
 
         if (lh_migratable) {
             flags |= pe_order_apply_first_non_migratable;
         }
 
         /* rh side is at the bottom of the stack during a stop. If we have a constraint
          * stop B then stop A, if B is migrating via stop/start, and A is migrating using migration actions,
          * we need to enforce that A's migrate_to action occurs after B's stop action. */
         custom_action_order(order->lh_rsc,
                             pcmk__op_key(order->lh_rsc->id, RSC_STOP, 0), NULL,
                             order->rh_rsc,
                             pcmk__op_key(order->rh_rsc->id, RSC_MIGRATE, 0),
                             NULL, flags, data_set);
 
         /* We need to build the stop constraint against migrate_from as well
          * to account for partial migrations. */
         if (order->rh_rsc->partial_migration_target) {
             custom_action_order(order->lh_rsc,
                                 pcmk__op_key(order->lh_rsc->id, RSC_STOP, 0),
                                 NULL, order->rh_rsc,
                                 pcmk__op_key(order->rh_rsc->id, RSC_MIGRATED, 0),
                                 NULL, flags, data_set);
         }
 
     } else if (safe_str_eq(lh_task, RSC_PROMOTE) && safe_str_eq(rh_task, RSC_START)) {
         int flags = pe_order_optional;
 
         if (rh_migratable) {
             /* A promote then B start
              * A promote then B migrate_to */
             custom_action_order(order->lh_rsc,
                                 pcmk__op_key(order->lh_rsc->id, RSC_PROMOTE, 0),
                                 NULL, order->rh_rsc,
                                 pcmk__op_key(order->rh_rsc->id, RSC_MIGRATE, 0),
                                 NULL, flags, data_set);
         }
 
     } else if (safe_str_eq(lh_task, RSC_DEMOTE) && safe_str_eq(rh_task, RSC_STOP)) {
         int flags = pe_order_optional;
 
         if (rh_migratable) {
             /* A demote then B stop
              * A demote then B migrate_to */
             custom_action_order(order->lh_rsc, pcmk__op_key(order->lh_rsc->id, RSC_DEMOTE, 0), NULL,
                                 order->rh_rsc, pcmk__op_key(order->rh_rsc->id, RSC_MIGRATE, 0), NULL,
                                 flags, data_set);
 
             /* We need to build the demote constraint against migrate_from as well
              * to account for partial migrations. */
             if (order->rh_rsc->partial_migration_target) {
                 custom_action_order(order->lh_rsc,
                                     pcmk__op_key(order->lh_rsc->id, RSC_DEMOTE, 0),
                                     NULL, order->rh_rsc,
                                     pcmk__op_key(order->rh_rsc->id, RSC_MIGRATED, 0),
                                     NULL, flags, data_set);
             }
         }
     }
 
 cleanup_order:
     free(lh_task);
     free(rh_task);
 }
 
 /* LHS before RHS */
 int
 custom_action_order(pe_resource_t * lh_rsc, char *lh_action_task, pe_action_t * lh_action,
                     pe_resource_t * rh_rsc, char *rh_action_task, pe_action_t * rh_action,
                     enum pe_ordering type, pe_working_set_t * data_set)
 {
     pe__ordering_t *order = NULL;
 
     if (lh_rsc == NULL && lh_action) {
         lh_rsc = lh_action->rsc;
     }
     if (rh_rsc == NULL && rh_action) {
         rh_rsc = rh_action->rsc;
     }
 
     if ((lh_action == NULL && lh_rsc == NULL)
         || (rh_action == NULL && rh_rsc == NULL)) {
         crm_err("Invalid ordering (bug?)");
         free(lh_action_task);
         free(rh_action_task);
         return -1;
     }
 
     order = calloc(1, sizeof(pe__ordering_t));
 
     crm_trace("Creating[%d] %s %s %s - %s %s %s", data_set->order_id,
               lh_rsc?lh_rsc->id:"NA", lh_action_task, lh_action?lh_action->uuid:"NA",
               rh_rsc?rh_rsc->id:"NA", rh_action_task, rh_action?rh_action->uuid:"NA");
 
     /* CRM_ASSERT(data_set->order_id != 291); */
 
     order->id = data_set->order_id++;
     order->type = type;
     order->lh_rsc = lh_rsc;
     order->rh_rsc = rh_rsc;
     order->lh_action = lh_action;
     order->rh_action = rh_action;
     order->lh_action_task = lh_action_task;
     order->rh_action_task = rh_action_task;
 
     if (order->lh_action_task == NULL && lh_action) {
         order->lh_action_task = strdup(lh_action->uuid);
     }
 
     if (order->rh_action_task == NULL && rh_action) {
         order->rh_action_task = strdup(rh_action->uuid);
     }
 
     if (order->lh_rsc == NULL && lh_action) {
         order->lh_rsc = lh_action->rsc;
     }
 
     if (order->rh_rsc == NULL && rh_action) {
         order->rh_rsc = rh_action->rsc;
     }
 
     data_set->ordering_constraints = g_list_prepend(data_set->ordering_constraints, order);
     handle_migration_ordering(order, data_set);
 
     return order->id;
 }
 
 enum pe_ordering
 get_asymmetrical_flags(enum pe_order_kind kind)
 {
     enum pe_ordering flags = pe_order_optional;
 
     if (kind == pe_order_kind_mandatory) {
         flags |= pe_order_asymmetrical;
     } else if (kind == pe_order_kind_serialize) {
         flags |= pe_order_serialize_only;
     }
     return flags;
 }
 
 enum pe_ordering
 get_flags(const char *id, enum pe_order_kind kind,
           const char *action_first, const char *action_then, gboolean invert)
 {
     enum pe_ordering flags = pe_order_optional;
 
     if (invert && kind == pe_order_kind_mandatory) {
         crm_trace("Upgrade %s: implies left", id);
         flags |= pe_order_implies_first;
 
     } else if (kind == pe_order_kind_mandatory) {
         crm_trace("Upgrade %s: implies right", id);
         flags |= pe_order_implies_then;
         if (safe_str_eq(action_first, RSC_START)
             || safe_str_eq(action_first, RSC_PROMOTE)) {
             crm_trace("Upgrade %s: runnable", id);
             flags |= pe_order_runnable_left;
         }
 
     } else if (kind == pe_order_kind_serialize) {
         flags |= pe_order_serialize_only;
     }
 
     return flags;
 }
 
 static gboolean
 unpack_order_set(xmlNode * set, enum pe_order_kind parent_kind, pe_resource_t ** rsc,
                  pe_action_t ** begin, pe_action_t ** end, pe_action_t ** inv_begin,
                  pe_action_t ** inv_end, const char *parent_symmetrical_s,
                  pe_working_set_t * data_set)
 {
     xmlNode *xml_rsc = NULL;
     GListPtr set_iter = NULL;
     GListPtr resources = NULL;
 
     pe_resource_t *last = NULL;
     pe_resource_t *resource = NULL;
 
     int local_kind = parent_kind;
     gboolean sequential = FALSE;
     enum pe_ordering flags = pe_order_optional;
     gboolean symmetrical = TRUE;
 
     char *key = NULL;
     const char *id = ID(set);
     const char *action = crm_element_value(set, "action");
     const char *sequential_s = crm_element_value(set, "sequential");
     const char *kind_s = crm_element_value(set, XML_ORDER_ATTR_KIND);
 
     /*
        char *pseudo_id = NULL;
        char *end_id    = NULL;
        char *begin_id  = NULL;
      */
 
     if (action == NULL) {
         action = RSC_START;
     }
 
     if (kind_s) {
         local_kind = get_ordering_type(set);
     }
     if (sequential_s == NULL) {
         sequential_s = "1";
     }
 
     sequential = crm_is_true(sequential_s);
 
     symmetrical = order_is_symmetrical(set, parent_kind, parent_symmetrical_s);
     if (symmetrical) {
         flags = get_flags(id, local_kind, action, action, FALSE);
     } else {
         flags = get_asymmetrical_flags(local_kind);
     }
 
     for (xml_rsc = __xml_first_child_element(set); xml_rsc != NULL;
          xml_rsc = __xml_next_element(xml_rsc)) {
 
         if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
             EXPAND_CONSTRAINT_IDREF(id, resource, ID(xml_rsc));
             resources = g_list_append(resources, resource);
         }
     }
 
     if (pcmk__list_of_1(resources)) {
         crm_trace("Single set: %s", id);
         *rsc = resource;
         *end = NULL;
         *begin = NULL;
         *inv_end = NULL;
         *inv_begin = NULL;
         goto done;
     }
 
     /*
        pseudo_id = crm_strdup_printf("%s-%s", id, action);
        end_id    = crm_strdup_printf("%s-%s", pseudo_id, "end");
        begin_id  = crm_strdup_printf("%s-%s", pseudo_id, "begin");
      */
 
     *rsc = NULL;
     /*
      *end = get_pseudo_op(end_id, data_set);
      *begin = get_pseudo_op(begin_id, data_set);
 
      free(pseudo_id);
      free(begin_id);
      free(end_id);
      */
 
     set_iter = resources;
     while (set_iter != NULL) {
         resource = (pe_resource_t *) set_iter->data;
         set_iter = set_iter->next;
 
         key = pcmk__op_key(resource->id, action, 0);
 
         /*
            custom_action_order(NULL, NULL, *begin, resource, strdup(key), NULL,
            flags|pe_order_implies_first_printed, data_set);
 
            custom_action_order(resource, strdup(key), NULL, NULL, NULL, *end,
            flags|pe_order_implies_then_printed, data_set);
          */
 
         if (local_kind == pe_order_kind_serialize) {
             /* Serialize before everything that comes after */
 
             GListPtr gIter = NULL;
 
             for (gIter = set_iter; gIter != NULL; gIter = gIter->next) {
                 pe_resource_t *then_rsc = (pe_resource_t *) gIter->data;
                 char *then_key = pcmk__op_key(then_rsc->id, action, 0);
 
                 custom_action_order(resource, strdup(key), NULL, then_rsc, then_key, NULL,
                                     flags, data_set);
             }
 
         } else if (sequential) {
             if (last != NULL) {
                 new_rsc_order(last, action, resource, action, flags, data_set);
             }
             last = resource;
         }
         free(key);
     }
 
     if (symmetrical == FALSE) {
         goto done;
     }
 
     last = NULL;
     action = invert_action(action);
 
     /*
        pseudo_id = crm_strdup_printf("%s-%s", id, action);
        end_id    = crm_strdup_printf("%s-%s", pseudo_id, "end");
        begin_id  = crm_strdup_printf("%s-%s", pseudo_id, "begin");
 
        *inv_end = get_pseudo_op(end_id, data_set);
        *inv_begin = get_pseudo_op(begin_id, data_set);
 
        free(pseudo_id);
        free(begin_id);
        free(end_id);
      */
 
     flags = get_flags(id, local_kind, action, action, TRUE);
 
     set_iter = resources;
     while (set_iter != NULL) {
         resource = (pe_resource_t *) set_iter->data;
         set_iter = set_iter->next;
 
         /*
            key = pcmk__op_key(resource->id, action, 0);
 
            custom_action_order(NULL, NULL, *inv_begin, resource, strdup(key), NULL,
            flags|pe_order_implies_first_printed, data_set);
 
            custom_action_order(resource, key, NULL, NULL, NULL, *inv_end,
            flags|pe_order_implies_then_printed, data_set);
          */
 
         if (sequential) {
             if (last != NULL) {
                 new_rsc_order(resource, action, last, action, flags, data_set);
             }
             last = resource;
         }
     }
 
   done:
     g_list_free(resources);
     return TRUE;
 }
 
 static gboolean
 order_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, enum pe_order_kind kind,
                pe_working_set_t * data_set, gboolean invert, gboolean symmetrical)
 {
 
     xmlNode *xml_rsc = NULL;
     xmlNode *xml_rsc_2 = NULL;
 
     pe_resource_t *rsc_1 = NULL;
     pe_resource_t *rsc_2 = NULL;
 
     const char *action_1 = crm_element_value(set1, "action");
     const char *action_2 = crm_element_value(set2, "action");
 
     const char *sequential_1 = crm_element_value(set1, "sequential");
     const char *sequential_2 = crm_element_value(set2, "sequential");
 
     const char *require_all_s = crm_element_value(set1, "require-all");
     gboolean require_all = require_all_s ? crm_is_true(require_all_s) : TRUE;
 
     enum pe_ordering flags = pe_order_none;
 
     if (action_1 == NULL) {
         action_1 = RSC_START;
     };
 
     if (action_2 == NULL) {
         action_2 = RSC_START;
     };
 
     if (invert) {
         action_1 = invert_action(action_1);
         action_2 = invert_action(action_2);
     }
 
     if(safe_str_eq(RSC_STOP, action_1) || safe_str_eq(RSC_DEMOTE, action_1)) {
         /* Assuming: A -> ( B || C) -> D
          * The one-or-more logic only applies during the start/promote phase
          * During shutdown neither B nor can shutdown until D is down, so simply turn require_all back on.
          */
         require_all = TRUE;
     }
 
     if (symmetrical == FALSE) {
         flags = get_asymmetrical_flags(kind);
     } else {
         flags = get_flags(id, kind, action_2, action_1, invert);
     }
 
     /* If we have an un-ordered set1, whether it is sequential or not is irrelevant in regards to set2. */
     if (!require_all) {
         char *task = crm_strdup_printf(CRM_OP_RELAXED_SET ":%s", ID(set1));
         pe_action_t *unordered_action = get_pseudo_op(task, data_set);
 
         free(task);
         update_action_flags(unordered_action, pe_action_requires_any, __FUNCTION__, __LINE__);
 
         for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL;
              xml_rsc = __xml_next_element(xml_rsc)) {
 
             if (!crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                 continue;
             }
 
             EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
 
             /* Add an ordering constraint between every element in set1 and the pseudo action.
              * If any action in set1 is runnable the pseudo action will be runnable. */
             custom_action_order(rsc_1, pcmk__op_key(rsc_1->id, action_1, 0),
                                 NULL, NULL, NULL, unordered_action,
                                 pe_order_one_or_more|pe_order_implies_then_printed,
                                 data_set);
         }
         for (xml_rsc_2 = __xml_first_child_element(set2); xml_rsc_2 != NULL;
              xml_rsc_2 = __xml_next_element(xml_rsc_2)) {
 
             if (!crm_str_eq((const char *)xml_rsc_2->name, XML_TAG_RESOURCE_REF, TRUE)) {
                 continue;
             }
 
             EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
 
             /* Add an ordering constraint between the pseudo action and every element in set2.
              * If the pseudo action is runnable, every action in set2 will be runnable */
             custom_action_order(NULL, NULL, unordered_action,
                                 rsc_2, pcmk__op_key(rsc_2->id, action_2, 0),
                                 NULL, flags|pe_order_runnable_left, data_set);
         }
 
         return TRUE;
     }
 
     if (crm_is_true(sequential_1)) {
         if (invert == FALSE) {
             /* get the last one */
             const char *rid = NULL;
 
             for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL;
                  xml_rsc = __xml_next_element(xml_rsc)) {
 
                 if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                     rid = ID(xml_rsc);
                 }
             }
             EXPAND_CONSTRAINT_IDREF(id, rsc_1, rid);
 
         } else {
             /* get the first one */
             for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL;
                  xml_rsc = __xml_next_element(xml_rsc)) {
 
                 if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                     EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
                     break;
                 }
             }
         }
     }
 
     if (crm_is_true(sequential_2)) {
         if (invert == FALSE) {
             /* get the first one */
             for (xml_rsc = __xml_first_child_element(set2); xml_rsc != NULL;
                  xml_rsc = __xml_next_element(xml_rsc)) {
 
                 if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                     EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
                     break;
                 }
             }
 
         } else {
             /* get the last one */
             const char *rid = NULL;
 
             for (xml_rsc = __xml_first_child_element(set2); xml_rsc != NULL;
                  xml_rsc = __xml_next_element(xml_rsc)) {
 
                 if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                     rid = ID(xml_rsc);
                 }
             }
             EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);
         }
     }
 
     if (rsc_1 != NULL && rsc_2 != NULL) {
         new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);
 
     } else if (rsc_1 != NULL) {
         for (xml_rsc = __xml_first_child_element(set2); xml_rsc != NULL;
              xml_rsc = __xml_next_element(xml_rsc)) {
 
             if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                 EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
                 new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);
             }
         }
 
     } else if (rsc_2 != NULL) {
         xmlNode *xml_rsc = NULL;
 
         for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL;
              xml_rsc = __xml_next_element(xml_rsc)) {
 
             if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                 EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
                 new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);
             }
         }
 
     } else {
         for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL;
              xml_rsc = __xml_next_element(xml_rsc)) {
 
             if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                 xmlNode *xml_rsc_2 = NULL;
 
                 EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
 
                 for (xml_rsc_2 = __xml_first_child_element(set2);
                      xml_rsc_2 != NULL;
                      xml_rsc_2 = __xml_next_element(xml_rsc_2)) {
 
                     if (crm_str_eq((const char *)xml_rsc_2->name, XML_TAG_RESOURCE_REF, TRUE)) {
                         EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
                         new_rsc_order(rsc_1, action_1, rsc_2, action_2, flags, data_set);
                     }
                 }
             }
         }
     }
 
     return TRUE;
 }
 
 static gboolean
 unpack_order_tags(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     const char *id_first = NULL;
     const char *id_then = NULL;
     const char *action_first = NULL;
     const char *action_then = NULL;
 
     pe_resource_t *rsc_first = NULL;
     pe_resource_t *rsc_then = NULL;
     pe_tag_t *tag_first = NULL;
     pe_tag_t *tag_then = NULL;
 
     xmlNode *new_xml = NULL;
     xmlNode *rsc_set_first = NULL;
     xmlNode *rsc_set_then = NULL;
     gboolean any_sets = FALSE;
 
     *expanded_xml = NULL;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     id = ID(xml_obj);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     /* Attempt to expand any template/tag references in possible resource sets. */
     expand_tags_in_sets(xml_obj, &new_xml, data_set);
     if (new_xml) {
         /* There are resource sets referencing templates/tags. Return with the expanded XML. */
         crm_log_xml_trace(new_xml, "Expanded rsc_order...");
         *expanded_xml = new_xml;
         return TRUE;
     }
 
     id_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST);
     id_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN);
     if (id_first == NULL || id_then == NULL) {
         return TRUE;
     }
 
     if (valid_resource_or_tag(data_set, id_first, &rsc_first, &tag_first) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag", id, id_first);
         return FALSE;
     }
 
     if (valid_resource_or_tag(data_set, id_then, &rsc_then, &tag_then) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag", id, id_then);
         return FALSE;
     }
 
     if (rsc_first && rsc_then) {
         /* Neither side references any template/tag. */
         return TRUE;
     }
 
     action_first = crm_element_value(xml_obj, XML_ORDER_ATTR_FIRST_ACTION);
     action_then = crm_element_value(xml_obj, XML_ORDER_ATTR_THEN_ACTION);
 
     new_xml = copy_xml(xml_obj);
 
     /* Convert the template/tag reference in "first" into a resource_set under the order constraint. */
     if (tag_to_set(new_xml, &rsc_set_first, XML_ORDER_ATTR_FIRST, TRUE, data_set) == FALSE) {
         free_xml(new_xml);
         return FALSE;
     }
 
     if (rsc_set_first) {
         if (action_first) {
             /* A "first-action" is specified.
                Move it into the converted resource_set as an "action" attribute. */
             crm_xml_add(rsc_set_first, "action", action_first);
             xml_remove_prop(new_xml, XML_ORDER_ATTR_FIRST_ACTION);
         }
         any_sets = TRUE;
     }
 
     /* Convert the template/tag reference in "then" into a resource_set under the order constraint. */
     if (tag_to_set(new_xml, &rsc_set_then, XML_ORDER_ATTR_THEN, TRUE, data_set) == FALSE) {
         free_xml(new_xml);
         return FALSE;
     }
 
     if (rsc_set_then) {
         if (action_then) {
             /* A "then-action" is specified.
                Move it into the converted resource_set as an "action" attribute. */
             crm_xml_add(rsc_set_then, "action", action_then);
             xml_remove_prop(new_xml, XML_ORDER_ATTR_THEN_ACTION);
         }
         any_sets = TRUE;
     }
 
     if (any_sets) {
         crm_log_xml_trace(new_xml, "Expanded rsc_order...");
         *expanded_xml = new_xml;
     } else {
         free_xml(new_xml);
     }
 
     return TRUE;
 }
 
 gboolean
 unpack_rsc_order(xmlNode * xml_obj, pe_working_set_t * data_set)
 {
     gboolean any_sets = FALSE;
 
     pe_resource_t *rsc = NULL;
 
     /*
        pe_resource_t *last_rsc = NULL;
      */
 
     pe_action_t *set_end = NULL;
     pe_action_t *set_begin = NULL;
 
     pe_action_t *set_inv_end = NULL;
     pe_action_t *set_inv_begin = NULL;
 
     xmlNode *set = NULL;
     xmlNode *last = NULL;
 
     xmlNode *orig_xml = NULL;
     xmlNode *expanded_xml = NULL;
 
     /*
        pe_action_t *last_end = NULL;
        pe_action_t *last_begin = NULL;
        pe_action_t *last_inv_end = NULL;
        pe_action_t *last_inv_begin = NULL;
      */
 
     const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
     const char *invert = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
     enum pe_order_kind kind = get_ordering_type(xml_obj);
 
     gboolean invert_bool = order_is_symmetrical(xml_obj, kind, NULL);
     gboolean rc = TRUE;
 
     rc = unpack_order_tags(xml_obj, &expanded_xml, data_set);
     if (expanded_xml) {
         orig_xml = xml_obj;
         xml_obj = expanded_xml;
 
     } else if (rc == FALSE) {
         return FALSE;
     }
 
     for (set = __xml_first_child_element(xml_obj); set != NULL;
          set = __xml_next_element(set)) {
 
         if (crm_str_eq((const char *)set->name, XML_CONS_TAG_RSC_SET, TRUE)) {
             any_sets = TRUE;
             set = expand_idref(set, data_set->input);
             if (unpack_order_set(set, kind, &rsc, &set_begin, &set_end,
                                  &set_inv_begin, &set_inv_end, invert, data_set) == FALSE) {
                 return FALSE;
 
                 /* Expand orders in order_rsc_sets() instead of via pseudo actions. */
                 /*
                    } else if(last) {
                    const char *set_action = crm_element_value(set, "action");
                    const char *last_action = crm_element_value(last, "action");
                    enum pe_ordering flags = get_flags(id, kind, last_action, set_action, FALSE);
 
                    if(!set_action) { set_action = RSC_START; }
                    if(!last_action) { last_action = RSC_START; }
 
                    if(rsc == NULL && last_rsc == NULL) {
                    order_actions(last_end, set_begin, flags);
                    } else {
                    custom_action_order(
                    last_rsc, null_or_opkey(last_rsc, last_action), last_end,
                    rsc, null_or_opkey(rsc, set_action), set_begin,
                    flags, data_set);
                    }
 
                    if(crm_is_true(invert)) {
                    set_action = invert_action(set_action);
                    last_action = invert_action(last_action);
 
                    flags = get_flags(id, kind, last_action, set_action, TRUE);
                    if(rsc == NULL && last_rsc == NULL) {
                    order_actions(last_inv_begin, set_inv_end, flags);
 
                    } else {
                    custom_action_order(
                    last_rsc, null_or_opkey(last_rsc, last_action), last_inv_begin,
                    rsc, null_or_opkey(rsc, set_action), set_inv_end,
                    flags, data_set);
                    }
                    }
                  */
 
             } else if (         /* never called -- Now call it for supporting clones in resource sets */
                           last) {
                 if (order_rsc_sets(id, last, set, kind, data_set, FALSE, invert_bool) == FALSE) {
                     return FALSE;
                 }
 
                 if (invert_bool
                     && order_rsc_sets(id, set, last, kind, data_set, TRUE, invert_bool) == FALSE) {
                     return FALSE;
                 }
 
             }
             last = set;
             /*
                last_rsc = rsc;
                last_end = set_end;
                last_begin = set_begin;
                last_inv_end = set_inv_end;
                last_inv_begin = set_inv_begin;
              */
         }
     }
 
     if (expanded_xml) {
         free_xml(expanded_xml);
         xml_obj = orig_xml;
     }
 
     if (any_sets == FALSE) {
         return unpack_simple_rsc_order(xml_obj, data_set);
     }
 
     return TRUE;
 }
 
 static gboolean
 unpack_colocation_set(xmlNode * set, int score, pe_working_set_t * data_set)
 {
     xmlNode *xml_rsc = NULL;
     pe_resource_t *with = NULL;
     pe_resource_t *resource = NULL;
     const char *set_id = ID(set);
     const char *role = crm_element_value(set, "role");
     const char *sequential = crm_element_value(set, "sequential");
     const char *ordering = crm_element_value(set, "ordering");
     int local_score = score;
 
     const char *score_s = crm_element_value(set, XML_RULE_ATTR_SCORE);
 
     if (score_s) {
         local_score = char2score(score_s);
     }
 
     if(ordering == NULL) {
         ordering = "group";
     }
 
     if (sequential != NULL && crm_is_true(sequential) == FALSE) {
         return TRUE;
 
     } else if (local_score >= 0 && safe_str_eq(ordering, "group")) {
         for (xml_rsc = __xml_first_child_element(set); xml_rsc != NULL;
              xml_rsc = __xml_next_element(xml_rsc)) {
 
             if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                 EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
                 if (with != NULL) {
                     pe_rsc_trace(resource, "Colocating %s with %s", resource->id, with->id);
                     rsc_colocation_new(set_id, NULL, local_score, resource, with, role, role,
                                        data_set);
                 }
 
                 with = resource;
             }
         }
     } else if (local_score >= 0) {
         pe_resource_t *last = NULL;
         for (xml_rsc = __xml_first_child_element(set); xml_rsc != NULL;
              xml_rsc = __xml_next_element(xml_rsc)) {
 
             if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                 EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
                 if (last != NULL) {
                     pe_rsc_trace(resource, "Colocating %s with %s", last->id, resource->id);
                     rsc_colocation_new(set_id, NULL, local_score, last, resource, role, role,
                                        data_set);
                 }
 
                 last = resource;
             }
         }
 
     } else {
         /* Anti-colocating with every prior resource is
          * the only way to ensure the intuitive result
          * (i.e. that no one in the set can run with anyone else in the set)
          */
 
         for (xml_rsc = __xml_first_child_element(set); xml_rsc != NULL;
              xml_rsc = __xml_next_element(xml_rsc)) {
 
             if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                 xmlNode *xml_rsc_with = NULL;
 
                 EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
 
                 for (xml_rsc_with = __xml_first_child_element(set);
                      xml_rsc_with != NULL;
                      xml_rsc_with = __xml_next_element(xml_rsc_with)) {
 
                     if (crm_str_eq((const char *)xml_rsc_with->name, XML_TAG_RESOURCE_REF, TRUE)) {
                         if (safe_str_eq(resource->id, ID(xml_rsc_with))) {
                             break;
                         }
                         EXPAND_CONSTRAINT_IDREF(set_id, with, ID(xml_rsc_with));
                         pe_rsc_trace(resource, "Anti-Colocating %s with %s", resource->id,
                                      with->id);
                         rsc_colocation_new(set_id, NULL, local_score, resource, with, role, role,
                                            data_set);
                     }
                 }
             }
         }
     }
 
     return TRUE;
 }
 
 static gboolean
 colocate_rsc_sets(const char *id, xmlNode * set1, xmlNode * set2, int score,
                   pe_working_set_t * data_set)
 {
     xmlNode *xml_rsc = NULL;
     pe_resource_t *rsc_1 = NULL;
     pe_resource_t *rsc_2 = NULL;
 
     const char *role_1 = crm_element_value(set1, "role");
     const char *role_2 = crm_element_value(set2, "role");
 
     const char *sequential_1 = crm_element_value(set1, "sequential");
     const char *sequential_2 = crm_element_value(set2, "sequential");
 
     if (sequential_1 == NULL || crm_is_true(sequential_1)) {
         /* get the first one */
         for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL;
              xml_rsc = __xml_next_element(xml_rsc)) {
 
             if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                 EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
                 break;
             }
         }
     }
 
     if (sequential_2 == NULL || crm_is_true(sequential_2)) {
         /* get the last one */
         const char *rid = NULL;
 
         for (xml_rsc = __xml_first_child_element(set2); xml_rsc != NULL;
              xml_rsc = __xml_next_element(xml_rsc)) {
 
             if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                 rid = ID(xml_rsc);
             }
         }
         EXPAND_CONSTRAINT_IDREF(id, rsc_2, rid);
     }
 
     if (rsc_1 != NULL && rsc_2 != NULL) {
         rsc_colocation_new(id, NULL, score, rsc_1, rsc_2, role_1, role_2, data_set);
 
     } else if (rsc_1 != NULL) {
         for (xml_rsc = __xml_first_child_element(set2); xml_rsc != NULL;
              xml_rsc = __xml_next_element(xml_rsc)) {
 
             if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                 EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc));
                 rsc_colocation_new(id, NULL, score, rsc_1, rsc_2, role_1, role_2, data_set);
             }
         }
 
     } else if (rsc_2 != NULL) {
         for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL;
              xml_rsc = __xml_next_element(xml_rsc)) {
 
             if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                 EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
                 rsc_colocation_new(id, NULL, score, rsc_1, rsc_2, role_1, role_2, data_set);
             }
         }
 
     } else {
         for (xml_rsc = __xml_first_child_element(set1); xml_rsc != NULL;
              xml_rsc = __xml_next_element(xml_rsc)) {
 
             if (crm_str_eq((const char *)xml_rsc->name, XML_TAG_RESOURCE_REF, TRUE)) {
                 xmlNode *xml_rsc_2 = NULL;
 
                 EXPAND_CONSTRAINT_IDREF(id, rsc_1, ID(xml_rsc));
 
                 for (xml_rsc_2 = __xml_first_child_element(set2);
                      xml_rsc_2 != NULL;
                      xml_rsc_2 = __xml_next_element(xml_rsc_2)) {
 
                     if (crm_str_eq((const char *)xml_rsc_2->name, XML_TAG_RESOURCE_REF, TRUE)) {
                         EXPAND_CONSTRAINT_IDREF(id, rsc_2, ID(xml_rsc_2));
                         rsc_colocation_new(id, NULL, score, rsc_1, rsc_2, role_1, role_2, data_set);
                     }
                 }
             }
         }
     }
 
     return TRUE;
 }
 
 static gboolean
 unpack_simple_colocation(xmlNode * xml_obj, pe_working_set_t * data_set)
 {
     int score_i = 0;
 
     const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
     const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
 
     const char *id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
     const char *id_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET);
     const char *state_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
     const char *state_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE);
     const char *attr = crm_element_value(xml_obj, XML_COLOC_ATTR_NODE_ATTR);
     const char *symmetrical = crm_element_value(xml_obj, XML_CONS_ATTR_SYMMETRICAL);
 
     // experimental syntax from pacemaker-next (unlikely to be adopted as-is)
     const char *instance_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_INSTANCE);
     const char *instance_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_INSTANCE);
 
     pe_resource_t *rsc_lh = pe_find_constraint_resource(data_set->resources, id_lh);
     pe_resource_t *rsc_rh = pe_find_constraint_resource(data_set->resources, id_rh);
 
     if (rsc_lh == NULL) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "does not exist", id, id_lh);
         return FALSE;
 
     } else if (rsc_rh == NULL) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "does not exist", id, id_rh);
         return FALSE;
 
     } else if (instance_lh && pe_rsc_is_clone(rsc_lh) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "is not a clone but instance '%s' was requested",
                          id, id_lh, instance_lh);
         return FALSE;
 
     } else if (instance_rh && pe_rsc_is_clone(rsc_rh) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "is not a clone but instance '%s' was requested",
                          id, id_rh, instance_rh);
         return FALSE;
     }
 
     if (instance_lh) {
         rsc_lh = find_clone_instance(rsc_lh, instance_lh, data_set);
         if (rsc_lh == NULL) {
             pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
                               "does not have an instance '%s'",
                               id, id_lh, instance_lh);
             return FALSE;
         }
     }
 
     if (instance_rh) {
         rsc_rh = find_clone_instance(rsc_rh, instance_rh, data_set);
         if (rsc_rh == NULL) {
             pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
                               "does not have an instance '%s'",
                               "'%s'", id, id_rh, instance_rh);
             return FALSE;
         }
     }
 
     if (crm_is_true(symmetrical)) {
         pcmk__config_warn("The colocation constraint '"
                           XML_CONS_ATTR_SYMMETRICAL
                           "' attribute has been removed");
     }
 
     if (score) {
         score_i = char2score(score);
     }
 
     rsc_colocation_new(id, attr, score_i, rsc_lh, rsc_rh, state_lh, state_rh, data_set);
     return TRUE;
 }
 
 static gboolean
 unpack_colocation_tags(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     const char *id_lh = NULL;
     const char *id_rh = NULL;
     const char *state_lh = NULL;
     const char *state_rh = NULL;
 
     pe_resource_t *rsc_lh = NULL;
     pe_resource_t *rsc_rh = NULL;
 
     pe_tag_t *tag_lh = NULL;
     pe_tag_t *tag_rh = NULL;
 
     xmlNode *new_xml = NULL;
     xmlNode *rsc_set_lh = NULL;
     xmlNode *rsc_set_rh = NULL;
     gboolean any_sets = FALSE;
 
     *expanded_xml = NULL;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     id = ID(xml_obj);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     /* Attempt to expand any template/tag references in possible resource sets. */
     expand_tags_in_sets(xml_obj, &new_xml, data_set);
     if (new_xml) {
         /* There are resource sets referencing templates/tags. Return with the expanded XML. */
         crm_log_xml_trace(new_xml, "Expanded rsc_colocation...");
         *expanded_xml = new_xml;
         return TRUE;
     }
 
     id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
     id_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET);
     if (id_lh == NULL || id_rh == NULL) {
         return TRUE;
     }
 
     if (valid_resource_or_tag(data_set, id_lh, &rsc_lh, &tag_lh) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag", id, id_lh);
         return FALSE;
     }
 
     if (valid_resource_or_tag(data_set, id_rh, &rsc_rh, &tag_rh) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag", id, id_rh);
         return FALSE;
     }
 
     if (rsc_lh && rsc_rh) {
         /* Neither side references any template/tag. */
         return TRUE;
     }
 
     if (tag_lh && tag_rh) {
         /* A colocation constraint between two templates/tags makes no sense. */
         pcmk__config_err("Ignoring constraint '%s' because two templates or "
                          "tags cannot be colocated", id);
         return FALSE;
     }
 
     state_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
     state_rh = crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE);
 
     new_xml = copy_xml(xml_obj);
 
     /* Convert the template/tag reference in "rsc" into a resource_set under the colocation constraint. */
     if (tag_to_set(new_xml, &rsc_set_lh, XML_COLOC_ATTR_SOURCE, TRUE, data_set) == FALSE) {
         free_xml(new_xml);
         return FALSE;
     }
 
     if (rsc_set_lh) {
         if (state_lh) {
             /* A "rsc-role" is specified.
                Move it into the converted resource_set as a "role"" attribute. */
             crm_xml_add(rsc_set_lh, "role", state_lh);
             xml_remove_prop(new_xml, XML_COLOC_ATTR_SOURCE_ROLE);
         }
         any_sets = TRUE;
     }
 
     /* Convert the template/tag reference in "with-rsc" into a resource_set under the colocation constraint. */
     if (tag_to_set(new_xml, &rsc_set_rh, XML_COLOC_ATTR_TARGET, TRUE, data_set) == FALSE) {
         free_xml(new_xml);
         return FALSE;
     }
 
     if (rsc_set_rh) {
         if (state_rh) {
             /* A "with-rsc-role" is specified.
                Move it into the converted resource_set as a "role"" attribute. */
             crm_xml_add(rsc_set_rh, "role", state_rh);
             xml_remove_prop(new_xml, XML_COLOC_ATTR_TARGET_ROLE);
         }
         any_sets = TRUE;
     }
 
     if (any_sets) {
         crm_log_xml_trace(new_xml, "Expanded rsc_colocation...");
         *expanded_xml = new_xml;
     } else {
         free_xml(new_xml);
     }
 
     return TRUE;
 }
 
 gboolean
 unpack_rsc_colocation(xmlNode * xml_obj, pe_working_set_t * data_set)
 {
     int score_i = 0;
     xmlNode *set = NULL;
     xmlNode *last = NULL;
     gboolean any_sets = FALSE;
 
     xmlNode *orig_xml = NULL;
     xmlNode *expanded_xml = NULL;
 
     const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
     const char *score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
 
     gboolean rc = TRUE;
 
     if (score) {
         score_i = char2score(score);
     }
 
     rc = unpack_colocation_tags(xml_obj, &expanded_xml, data_set);
     if (expanded_xml) {
         orig_xml = xml_obj;
         xml_obj = expanded_xml;
 
     } else if (rc == FALSE) {
         return FALSE;
     }
 
     for (set = __xml_first_child_element(xml_obj); set != NULL;
          set = __xml_next_element(set)) {
 
         if (crm_str_eq((const char *)set->name, XML_CONS_TAG_RSC_SET, TRUE)) {
             any_sets = TRUE;
             set = expand_idref(set, data_set->input);
             if (unpack_colocation_set(set, score_i, data_set) == FALSE) {
                 return FALSE;
 
             } else if (last && colocate_rsc_sets(id, last, set, score_i, data_set) == FALSE) {
                 return FALSE;
             }
             last = set;
         }
     }
 
     if (expanded_xml) {
         free_xml(expanded_xml);
         xml_obj = orig_xml;
     }
 
     if (any_sets == FALSE) {
         return unpack_simple_colocation(xml_obj, data_set);
     }
 
     return TRUE;
 }
 
 gboolean
 rsc_ticket_new(const char *id, pe_resource_t * rsc_lh, pe_ticket_t * ticket,
                const char *state_lh, const char *loss_policy, pe_working_set_t * data_set)
 {
     rsc_ticket_t *new_rsc_ticket = NULL;
 
     if (rsc_lh == NULL) {
         pcmk__config_err("Ignoring ticket '%s' because resource "
                          "does not exist", id);
         return FALSE;
     }
 
     new_rsc_ticket = calloc(1, sizeof(rsc_ticket_t));
     if (new_rsc_ticket == NULL) {
         return FALSE;
     }
 
     if (state_lh == NULL || safe_str_eq(state_lh, RSC_ROLE_STARTED_S)) {
         state_lh = RSC_ROLE_UNKNOWN_S;
     }
 
     new_rsc_ticket->id = id;
     new_rsc_ticket->ticket = ticket;
     new_rsc_ticket->rsc_lh = rsc_lh;
     new_rsc_ticket->role_lh = text2role(state_lh);
 
     if (safe_str_eq(loss_policy, "fence")) {
         if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
             new_rsc_ticket->loss_policy = loss_ticket_fence;
         } else {
             pcmk__config_err("Resetting '" XML_TICKET_ATTR_LOSS_POLICY
                              "' for ticket '%s' to 'stop' "
                              "because fencing is not configured", ticket->id);
             loss_policy = "stop";
         }
     }
 
     if (new_rsc_ticket->loss_policy == loss_ticket_fence) {
         crm_debug("On loss of ticket '%s': Fence the nodes running %s (%s)",
                   new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
                   role2text(new_rsc_ticket->role_lh));
 
     } else if (safe_str_eq(loss_policy, "freeze")) {
         crm_debug("On loss of ticket '%s': Freeze %s (%s)",
                   new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
                   role2text(new_rsc_ticket->role_lh));
         new_rsc_ticket->loss_policy = loss_ticket_freeze;
 
     } else if (safe_str_eq(loss_policy, "demote")) {
         crm_debug("On loss of ticket '%s': Demote %s (%s)",
                   new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
                   role2text(new_rsc_ticket->role_lh));
         new_rsc_ticket->loss_policy = loss_ticket_demote;
 
     } else if (safe_str_eq(loss_policy, "stop")) {
         crm_debug("On loss of ticket '%s': Stop %s (%s)",
                   new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
                   role2text(new_rsc_ticket->role_lh));
         new_rsc_ticket->loss_policy = loss_ticket_stop;
 
     } else {
         if (new_rsc_ticket->role_lh == RSC_ROLE_MASTER) {
             crm_debug("On loss of ticket '%s': Default to demote %s (%s)",
                       new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
                       role2text(new_rsc_ticket->role_lh));
             new_rsc_ticket->loss_policy = loss_ticket_demote;
 
         } else {
             crm_debug("On loss of ticket '%s': Default to stop %s (%s)",
                       new_rsc_ticket->ticket->id, new_rsc_ticket->rsc_lh->id,
                       role2text(new_rsc_ticket->role_lh));
             new_rsc_ticket->loss_policy = loss_ticket_stop;
         }
     }
 
     pe_rsc_trace(rsc_lh, "%s (%s) ==> %s", rsc_lh->id, role2text(new_rsc_ticket->role_lh),
                  ticket->id);
 
     rsc_lh->rsc_tickets = g_list_append(rsc_lh->rsc_tickets, new_rsc_ticket);
 
     data_set->ticket_constraints = g_list_append(data_set->ticket_constraints, new_rsc_ticket);
 
     if (new_rsc_ticket->ticket->granted == FALSE || new_rsc_ticket->ticket->standby) {
         rsc_ticket_constraint(rsc_lh, new_rsc_ticket, data_set);
     }
 
     return TRUE;
 }
 
 static gboolean
 unpack_rsc_ticket_set(xmlNode * set, pe_ticket_t * ticket, const char *loss_policy,
                       pe_working_set_t * data_set)
 {
     xmlNode *xml_rsc = NULL;
     pe_resource_t *resource = NULL;
     const char *set_id = NULL;
     const char *role = NULL;
 
     CRM_CHECK(set != NULL, return FALSE);
     CRM_CHECK(ticket != NULL, return FALSE);
 
     set_id = ID(set);
     if (set_id == NULL) {
         pcmk__config_err("Ignoring <" XML_CONS_TAG_RSC_SET "> without "
                          XML_ATTR_ID);
         return FALSE;
     }
 
     role = crm_element_value(set, "role");
 
     for (xml_rsc = first_named_child(set, XML_TAG_RESOURCE_REF);
          xml_rsc != NULL; xml_rsc = crm_next_same_xml(xml_rsc)) {
 
         EXPAND_CONSTRAINT_IDREF(set_id, resource, ID(xml_rsc));
         pe_rsc_trace(resource, "Resource '%s' depends on ticket '%s'",
                      resource->id, ticket->id);
         rsc_ticket_new(set_id, resource, ticket, role, loss_policy, data_set);
     }
 
     return TRUE;
 }
 
 static gboolean
 unpack_simple_rsc_ticket(xmlNode * xml_obj, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     const char *ticket_str = crm_element_value(xml_obj, XML_TICKET_ATTR_TICKET);
     const char *loss_policy = crm_element_value(xml_obj, XML_TICKET_ATTR_LOSS_POLICY);
 
     pe_ticket_t *ticket = NULL;
 
     const char *id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
     const char *state_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
 
     // experimental syntax from pacemaker-next (unlikely to be adopted as-is)
     const char *instance_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_INSTANCE);
 
     pe_resource_t *rsc_lh = NULL;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     id = ID(xml_obj);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     if (ticket_str == NULL) {
         pcmk__config_err("Ignoring constraint '%s' without ticket specified",
                          id);
         return FALSE;
     } else {
         ticket = g_hash_table_lookup(data_set->tickets, ticket_str);
     }
 
     if (ticket == NULL) {
         pcmk__config_err("Ignoring constraint '%s' because ticket '%s' "
                          "does not exist", id, ticket_str);
         return FALSE;
     }
 
     if (id_lh == NULL) {
         pcmk__config_err("Ignoring constraint '%s' without resource", id);
         return FALSE;
     } else {
         rsc_lh = pe_find_constraint_resource(data_set->resources, id_lh);
     }
 
     if (rsc_lh == NULL) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "does not exist", id, id_lh);
         return FALSE;
 
     } else if (instance_lh && pe_rsc_is_clone(rsc_lh) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because resource '%s' "
                          "is not a clone but instance '%s' was requested",
                          id, id_lh, instance_lh);
         return FALSE;
     }
 
     if (instance_lh) {
         rsc_lh = find_clone_instance(rsc_lh, instance_lh, data_set);
         if (rsc_lh == NULL) {
             pcmk__config_warn("Ignoring constraint '%s' because resource '%s' "
                               "does not have an instance '%s'",
                               "'%s'", id, id_lh, instance_lh);
             return FALSE;
         }
     }
 
     rsc_ticket_new(id, rsc_lh, ticket, state_lh, loss_policy, data_set);
     return TRUE;
 }
 
 static gboolean
 unpack_rsc_ticket_tags(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     const char *id_lh = NULL;
     const char *state_lh = NULL;
 
     pe_resource_t *rsc_lh = NULL;
     pe_tag_t *tag_lh = NULL;
 
     xmlNode *new_xml = NULL;
     xmlNode *rsc_set_lh = NULL;
     gboolean any_sets = FALSE;
 
     *expanded_xml = NULL;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     id = ID(xml_obj);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     /* Attempt to expand any template/tag references in possible resource sets. */
     expand_tags_in_sets(xml_obj, &new_xml, data_set);
     if (new_xml) {
         /* There are resource sets referencing templates/tags. Return with the expanded XML. */
         crm_log_xml_trace(new_xml, "Expanded rsc_ticket...");
         *expanded_xml = new_xml;
         return TRUE;
     }
 
     id_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE);
     if (id_lh == NULL) {
         return TRUE;
     }
 
     if (valid_resource_or_tag(data_set, id_lh, &rsc_lh, &tag_lh) == FALSE) {
         pcmk__config_err("Ignoring constraint '%s' because '%s' is not a "
                          "valid resource or tag", id, id_lh);
         return FALSE;
 
     } else if (rsc_lh) {
         /* No template/tag is referenced. */
         return TRUE;
     }
 
     state_lh = crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE);
 
     new_xml = copy_xml(xml_obj);
 
     /* Convert the template/tag reference in "rsc" into a resource_set under the rsc_ticket constraint. */
     if (tag_to_set(new_xml, &rsc_set_lh, XML_COLOC_ATTR_SOURCE, FALSE, data_set) == FALSE) {
         free_xml(new_xml);
         return FALSE;
     }
 
     if (rsc_set_lh) {
         if (state_lh) {
             /* A "rsc-role" is specified.
                Move it into the converted resource_set as a "role"" attribute. */
             crm_xml_add(rsc_set_lh, "role", state_lh);
             xml_remove_prop(new_xml, XML_COLOC_ATTR_SOURCE_ROLE);
         }
         any_sets = TRUE;
     }
 
     if (any_sets) {
         crm_log_xml_trace(new_xml, "Expanded rsc_ticket...");
         *expanded_xml = new_xml;
     } else {
         free_xml(new_xml);
     }
 
     return TRUE;
 }
 
 gboolean
 unpack_rsc_ticket(xmlNode * xml_obj, pe_working_set_t * data_set)
 {
     xmlNode *set = NULL;
     gboolean any_sets = FALSE;
 
     const char *id = NULL;
     const char *ticket_str = crm_element_value(xml_obj, XML_TICKET_ATTR_TICKET);
     const char *loss_policy = crm_element_value(xml_obj, XML_TICKET_ATTR_LOSS_POLICY);
 
     pe_ticket_t *ticket = NULL;
 
     xmlNode *orig_xml = NULL;
     xmlNode *expanded_xml = NULL;
 
     gboolean rc = TRUE;
 
     CRM_CHECK(xml_obj != NULL, return FALSE);
 
     id = ID(xml_obj);
     if (id == NULL) {
         pcmk__config_err("Ignoring <%s> constraint without " XML_ATTR_ID,
                          crm_element_name(xml_obj));
         return FALSE;
     }
 
     if (data_set->tickets == NULL) {
         data_set->tickets =
             g_hash_table_new_full(crm_str_hash, g_str_equal, free, destroy_ticket);
     }
 
     if (ticket_str == NULL) {
         pcmk__config_err("Ignoring constraint '%s' without ticket", id);
         return FALSE;
     } else {
         ticket = g_hash_table_lookup(data_set->tickets, ticket_str);
     }
 
     if (ticket == NULL) {
         ticket = ticket_new(ticket_str, data_set);
         if (ticket == NULL) {
             return FALSE;
         }
     }
 
     rc = unpack_rsc_ticket_tags(xml_obj, &expanded_xml, data_set);
     if (expanded_xml) {
         orig_xml = xml_obj;
         xml_obj = expanded_xml;
 
     } else if (rc == FALSE) {
         return FALSE;
     }
 
     for (set = __xml_first_child_element(xml_obj); set != NULL;
          set = __xml_next_element(set)) {
 
         if (crm_str_eq((const char *)set->name, XML_CONS_TAG_RSC_SET, TRUE)) {
             any_sets = TRUE;
             set = expand_idref(set, data_set->input);
             if (unpack_rsc_ticket_set(set, ticket, loss_policy, data_set) == FALSE) {
                 return FALSE;
             }
         }
     }
 
     if (expanded_xml) {
         free_xml(expanded_xml);
         xml_obj = orig_xml;
     }
 
     if (any_sets == FALSE) {
         return unpack_simple_rsc_ticket(xml_obj, data_set);
     }
 
     return TRUE;
 }
diff --git a/lib/pengine/Makefile.am b/lib/pengine/Makefile.am
index eb981c93ae..faf36bdedc 100644
--- a/lib/pengine/Makefile.am
+++ b/lib/pengine/Makefile.am
@@ -1,50 +1,50 @@
 #
 # Copyright 2004-2018 the Pacemaker project contributors
 #
 # The version control history for this file may have further details.
 #
 # This source code is licensed under the GNU General Public License version 2
 # or later (GPLv2+) WITHOUT ANY WARRANTY.
 #
 include $(top_srcdir)/mk/common.mk
 
 SUBDIRS = tests
 
 ## libraries
 lib_LTLIBRARIES		= libpe_rules.la libpe_status.la
 
 ## SOURCES
-noinst_HEADERS		= unpack.h variant.h pe_status_private.h
+noinst_HEADERS		= variant.h pe_status_private.h
 
 libpe_rules_la_LDFLAGS	= -version-info 27:0:1
 
 libpe_rules_la_CFLAGS	= $(CFLAGS_HARDENED_LIB)
 libpe_rules_la_LDFLAGS	+= $(LDFLAGS_HARDENED_LIB)
 
 libpe_rules_la_LIBADD	= $(top_builddir)/lib/common/libcrmcommon.la
 libpe_rules_la_SOURCES	= rules.c rules_alerts.c common.c
 
 libpe_status_la_LDFLAGS	= -version-info 29:0:1
 
 libpe_status_la_CFLAGS	= $(CFLAGS_HARDENED_LIB)
 libpe_status_la_LDFLAGS	+= $(LDFLAGS_HARDENED_LIB)
 
 libpe_status_la_LIBADD	= @CURSESLIBS@ $(top_builddir)/lib/common/libcrmcommon.la
 # Use += rather than backlashed continuation lines for parsing by bumplibs.sh
 libpe_status_la_SOURCES	=
 libpe_status_la_SOURCES	+= bundle.c
 libpe_status_la_SOURCES	+= clone.c
 libpe_status_la_SOURCES	+= common.c
 libpe_status_la_SOURCES	+= complex.c
 libpe_status_la_SOURCES	+= failcounts.c
 libpe_status_la_SOURCES	+= group.c
 libpe_status_la_SOURCES	+= native.c
 libpe_status_la_SOURCES	+= remote.c
 libpe_status_la_SOURCES	+= rules.c
 libpe_status_la_SOURCES	+= status.c
 libpe_status_la_SOURCES	+= unpack.c
 libpe_status_la_SOURCES	+= utils.c
 libpe_status_la_SOURCES	+= pe_output.c
 
 clean-generic:
 	rm -f *.log *.debug *~
diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c
index 496452e4eb..e84b5c248f 100644
--- a/lib/pengine/complex.c
+++ b/lib/pengine/complex.c
@@ -1,975 +1,973 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm/pengine/rules.h>
 #include <crm/pengine/internal.h>
 #include <crm/msg_xml.h>
 
-#include <unpack.h>
-
 void populate_hash(xmlNode * nvpair_list, GHashTable * hash, const char **attrs, int attrs_length);
 
 resource_object_functions_t resource_class_functions[] = {
     {
          native_unpack,
          native_find_rsc,
          native_parameter,
          native_print,
          native_active,
          native_resource_state,
          native_location,
          native_free,
          pe__count_common,
     },
     {
          group_unpack,
          native_find_rsc,
          native_parameter,
          group_print,
          group_active,
          group_resource_state,
          native_location,
          group_free,
          pe__count_common,
     },
     {
          clone_unpack,
          native_find_rsc,
          native_parameter,
          clone_print,
          clone_active,
          clone_resource_state,
          native_location,
          clone_free,
          pe__count_common,
     },
     {
          pe__unpack_bundle,
          native_find_rsc,
          native_parameter,
          pe__print_bundle,
          pe__bundle_active,
          pe__bundle_resource_state,
          native_location,
          pe__free_bundle,
          pe__count_bundle,
     }
 };
 
 static enum pe_obj_types
 get_resource_type(const char *name)
 {
     if (safe_str_eq(name, XML_CIB_TAG_RESOURCE)) {
         return pe_native;
 
     } else if (safe_str_eq(name, XML_CIB_TAG_GROUP)) {
         return pe_group;
 
     } else if (safe_str_eq(name, XML_CIB_TAG_INCARNATION)) {
         return pe_clone;
 
     } else if (safe_str_eq(name, XML_CIB_TAG_MASTER)) {
         // @COMPAT deprecated since 2.0.0
         return pe_clone;
 
     } else if (safe_str_eq(name, XML_CIB_TAG_CONTAINER)) {
         return pe_container;
     }
 
     return pe_unknown;
 }
 
 static void
 dup_attr(gpointer key, gpointer value, gpointer user_data)
 {
     add_hash_param(user_data, key, value);
 }
 
 void
 get_meta_attributes(GHashTable * meta_hash, pe_resource_t * rsc,
                     pe_node_t * node, pe_working_set_t * data_set)
 {
     GHashTable *node_hash = NULL;
 
     if (node) {
         node_hash = node->details->attrs;
     }
 
     if (rsc->xml) {
         xmlAttrPtr xIter = NULL;
 
         for (xIter = rsc->xml->properties; xIter; xIter = xIter->next) {
             const char *prop_name = (const char *)xIter->name;
             const char *prop_value = crm_element_value(rsc->xml, prop_name);
 
             add_hash_param(meta_hash, prop_name, prop_value);
         }
     }
 
     pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_META_SETS, node_hash,
                                meta_hash, NULL, FALSE, data_set);
 
     /* set anything else based on the parent */
     if (rsc->parent != NULL) {
         g_hash_table_foreach(rsc->parent->meta, dup_attr, meta_hash);
     }
 
     /* and finally check the defaults */
     pe__unpack_dataset_nvpairs(data_set->rsc_defaults, XML_TAG_META_SETS,
                                node_hash, meta_hash, NULL, FALSE, data_set);
 }
 
 void
 get_rsc_attributes(GHashTable * meta_hash, pe_resource_t * rsc,
                    pe_node_t * node, pe_working_set_t * data_set)
 {
     GHashTable *node_hash = NULL;
 
     if (node) {
         node_hash = node->details->attrs;
     }
 
     pe__unpack_dataset_nvpairs(rsc->xml, XML_TAG_ATTR_SETS, node_hash,
                                meta_hash, NULL, FALSE, data_set);
 
     /* set anything else based on the parent */
     if (rsc->parent != NULL) {
         get_rsc_attributes(meta_hash, rsc->parent, node, data_set);
 
     } else {
         /* and finally check the defaults */
         pe__unpack_dataset_nvpairs(data_set->rsc_defaults, XML_TAG_ATTR_SETS,
                                    node_hash, meta_hash, NULL, FALSE, data_set);
     }
 }
 
 #if ENABLE_VERSIONED_ATTRS
 void
 pe_get_versioned_attributes(xmlNode * meta_hash, pe_resource_t * rsc,
                             pe_node_t * node, pe_working_set_t * data_set)
 {
     GHashTable *node_hash = NULL;
 
     if (node) {
         node_hash = node->details->attrs;
     }
 
     pe_unpack_versioned_attributes(data_set->input, rsc->xml, XML_TAG_ATTR_SETS, node_hash,
                                    meta_hash, data_set->now, NULL);
 
     /* set anything else based on the parent */
     if (rsc->parent != NULL) {
         pe_get_versioned_attributes(meta_hash, rsc->parent, node, data_set);
 
     } else {
         /* and finally check the defaults */
         pe_unpack_versioned_attributes(data_set->input, data_set->rsc_defaults, XML_TAG_ATTR_SETS,
                                        node_hash, meta_hash, data_set->now, NULL);
     }
 }
 #endif
 
 static char *
 template_op_key(xmlNode * op)
 {
     const char *name = crm_element_value(op, "name");
     const char *role = crm_element_value(op, "role");
     char *key = NULL;
 
     if (role == NULL || crm_str_eq(role, RSC_ROLE_STARTED_S, TRUE)
         || crm_str_eq(role, RSC_ROLE_SLAVE_S, TRUE)) {
         role = RSC_ROLE_UNKNOWN_S;
     }
 
     key = crm_strdup_printf("%s-%s", name, role);
     return key;
 }
 
 static gboolean
 unpack_template(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set)
 {
     xmlNode *cib_resources = NULL;
     xmlNode *template = NULL;
     xmlNode *new_xml = NULL;
     xmlNode *child_xml = NULL;
     xmlNode *rsc_ops = NULL;
     xmlNode *template_ops = NULL;
     const char *template_ref = NULL;
     const char *clone = NULL;
     const char *id = NULL;
 
     if (xml_obj == NULL) {
         pe_err("No resource object for template unpacking");
         return FALSE;
     }
 
     template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
     if (template_ref == NULL) {
         return TRUE;
     }
 
     id = ID(xml_obj);
     if (id == NULL) {
         pe_err("'%s' object must have a id", crm_element_name(xml_obj));
         return FALSE;
     }
 
     if (crm_str_eq(template_ref, id, TRUE)) {
         pe_err("The resource object '%s' should not reference itself", id);
         return FALSE;
     }
 
     cib_resources = get_xpath_object("//"XML_CIB_TAG_RESOURCES, data_set->input, LOG_TRACE);
     if (cib_resources == NULL) {
         pe_err("No resources configured");
         return FALSE;
     }
 
     template = find_entity(cib_resources, XML_CIB_TAG_RSC_TEMPLATE, template_ref);
     if (template == NULL) {
         pe_err("No template named '%s'", template_ref);
         return FALSE;
     }
 
     new_xml = copy_xml(template);
     xmlNodeSetName(new_xml, xml_obj->name);
     crm_xml_replace(new_xml, XML_ATTR_ID, id);
 
     clone = crm_element_value(xml_obj, XML_RSC_ATTR_INCARNATION);
     if(clone) {
         crm_xml_add(new_xml, XML_RSC_ATTR_INCARNATION, clone);
     }
 
     template_ops = find_xml_node(new_xml, "operations", FALSE);
 
     for (child_xml = __xml_first_child_element(xml_obj); child_xml != NULL;
          child_xml = __xml_next_element(child_xml)) {
         xmlNode *new_child = NULL;
 
         new_child = add_node_copy(new_xml, child_xml);
 
         if (crm_str_eq((const char *)new_child->name, "operations", TRUE)) {
             rsc_ops = new_child;
         }
     }
 
     if (template_ops && rsc_ops) {
         xmlNode *op = NULL;
         GHashTable *rsc_ops_hash = g_hash_table_new_full(crm_str_hash,
                                                          g_str_equal, free,
                                                          NULL);
 
         for (op = __xml_first_child_element(rsc_ops); op != NULL;
              op = __xml_next_element(op)) {
 
             char *key = template_op_key(op);
 
             g_hash_table_insert(rsc_ops_hash, key, op);
         }
 
         for (op = __xml_first_child_element(template_ops); op != NULL;
              op = __xml_next_element(op)) {
 
             char *key = template_op_key(op);
 
             if (g_hash_table_lookup(rsc_ops_hash, key) == NULL) {
                 add_node_copy(rsc_ops, op);
             }
 
             free(key);
         }
 
         if (rsc_ops_hash) {
             g_hash_table_destroy(rsc_ops_hash);
         }
 
         free_xml(template_ops);
     }
 
     /*free_xml(*expanded_xml); */
     *expanded_xml = new_xml;
 
     /* Disable multi-level templates for now */
     /*if(unpack_template(new_xml, expanded_xml, data_set) == FALSE) {
        free_xml(*expanded_xml);
        *expanded_xml = NULL;
 
        return FALSE;
        } */
 
     return TRUE;
 }
 
 static gboolean
 add_template_rsc(xmlNode * xml_obj, pe_working_set_t * data_set)
 {
     const char *template_ref = NULL;
     const char *id = NULL;
 
     if (xml_obj == NULL) {
         pe_err("No resource object for processing resource list of template");
         return FALSE;
     }
 
     template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE);
     if (template_ref == NULL) {
         return TRUE;
     }
 
     id = ID(xml_obj);
     if (id == NULL) {
         pe_err("'%s' object must have a id", crm_element_name(xml_obj));
         return FALSE;
     }
 
     if (crm_str_eq(template_ref, id, TRUE)) {
         pe_err("The resource object '%s' should not reference itself", id);
         return FALSE;
     }
 
     if (add_tag_ref(data_set->template_rsc_sets, template_ref, id) == FALSE) {
         return FALSE;
     }
 
     return TRUE;
 }
 
 static bool
 detect_promotable(pe_resource_t *rsc)
 {
     const char *promotable = g_hash_table_lookup(rsc->meta,
                                                  XML_RSC_ATTR_PROMOTABLE);
 
     if (crm_is_true(promotable)) {
         return TRUE;
     }
 
     // @COMPAT deprecated since 2.0.0
     if (safe_str_eq(crm_element_name(rsc->xml), XML_CIB_TAG_MASTER)) {
         /* @TODO in some future version, pe_warn_once() here,
          *       then drop support in even later version
          */
         g_hash_table_insert(rsc->meta, strdup(XML_RSC_ATTR_PROMOTABLE),
                             strdup(XML_BOOLEAN_TRUE));
         return TRUE;
     }
     return FALSE;
 }
 
 gboolean
 common_unpack(xmlNode * xml_obj, pe_resource_t ** rsc,
               pe_resource_t * parent, pe_working_set_t * data_set)
 {
     bool isdefault = FALSE;
     xmlNode *expanded_xml = NULL;
     xmlNode *ops = NULL;
     const char *value = NULL;
     const char *rclass = NULL; /* Look for this after any templates have been expanded */
     const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
     bool guest_node = FALSE;
     bool remote_node = FALSE;
     bool has_versioned_params = FALSE;
 
     crm_log_xml_trace(xml_obj, "Processing resource input...");
 
     if (id == NULL) {
         pe_err("Must specify id tag in <resource>");
         return FALSE;
 
     } else if (rsc == NULL) {
         pe_err("Nowhere to unpack resource into");
         return FALSE;
 
     }
 
     if (unpack_template(xml_obj, &expanded_xml, data_set) == FALSE) {
         return FALSE;
     }
 
     *rsc = calloc(1, sizeof(pe_resource_t));
     (*rsc)->cluster = data_set;
 
     if (expanded_xml) {
         crm_log_xml_trace(expanded_xml, "Expanded resource...");
         (*rsc)->xml = expanded_xml;
         (*rsc)->orig_xml = xml_obj;
 
     } else {
         (*rsc)->xml = xml_obj;
         (*rsc)->orig_xml = NULL;
     }
 
     /* Do not use xml_obj from here on, use (*rsc)->xml in case templates are involved */
     rclass = crm_element_value((*rsc)->xml, XML_AGENT_ATTR_CLASS);
     (*rsc)->parent = parent;
 
     ops = find_xml_node((*rsc)->xml, "operations", FALSE);
     (*rsc)->ops_xml = expand_idref(ops, data_set->input);
 
     (*rsc)->variant = get_resource_type(crm_element_name((*rsc)->xml));
     if ((*rsc)->variant == pe_unknown) {
         pe_err("Unknown resource type: %s", crm_element_name((*rsc)->xml));
         free(*rsc);
         return FALSE;
     }
 
     (*rsc)->parameters = crm_str_table_new();
 
 #if ENABLE_VERSIONED_ATTRS
     (*rsc)->versioned_parameters = create_xml_node(NULL, XML_TAG_RSC_VER_ATTRS);
 #endif
 
     (*rsc)->meta = crm_str_table_new();
 
     (*rsc)->allowed_nodes =
         g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free);
 
     (*rsc)->known_on = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL,
                                              free);
 
     value = crm_element_value((*rsc)->xml, XML_RSC_ATTR_INCARNATION);
     if (value) {
         (*rsc)->id = crm_strdup_printf("%s:%s", id, value);
         add_hash_param((*rsc)->meta, XML_RSC_ATTR_INCARNATION, value);
 
     } else {
         (*rsc)->id = strdup(id);
     }
 
     (*rsc)->fns = &resource_class_functions[(*rsc)->variant];
     pe_rsc_trace((*rsc), "Unpacking resource...");
 
     get_meta_attributes((*rsc)->meta, *rsc, NULL, data_set);
     get_rsc_attributes((*rsc)->parameters, *rsc, NULL, data_set);
 #if ENABLE_VERSIONED_ATTRS
     pe_get_versioned_attributes((*rsc)->versioned_parameters, *rsc, NULL, data_set);
 #endif
 
     (*rsc)->flags = 0;
     set_bit((*rsc)->flags, pe_rsc_runnable);
     set_bit((*rsc)->flags, pe_rsc_provisional);
 
     if (is_not_set(data_set->flags, pe_flag_maintenance_mode)) {
         set_bit((*rsc)->flags, pe_rsc_managed);
     }
 
     (*rsc)->rsc_cons = NULL;
     (*rsc)->rsc_tickets = NULL;
     (*rsc)->actions = NULL;
     (*rsc)->role = RSC_ROLE_STOPPED;
     (*rsc)->next_role = RSC_ROLE_UNKNOWN;
 
     (*rsc)->recovery_type = recovery_stop_start;
     (*rsc)->stickiness = 0;
     (*rsc)->migration_threshold = INFINITY;
     (*rsc)->failure_timeout = 0;
 
     value = g_hash_table_lookup((*rsc)->meta, XML_CIB_ATTR_PRIORITY);
     (*rsc)->priority = crm_parse_int(value, "0");
 
     value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_NOTIFY);
     if (crm_is_true(value)) {
         set_bit((*rsc)->flags, pe_rsc_notify);
     }
 
     if (xml_contains_remote_node((*rsc)->xml)) {
         (*rsc)->is_remote_node = TRUE;
         if (g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_CONTAINER)) {
             guest_node = TRUE;
         } else {
             remote_node = TRUE;
         }
     }
 
     value = g_hash_table_lookup((*rsc)->meta, XML_OP_ATTR_ALLOW_MIGRATE);
 #if ENABLE_VERSIONED_ATTRS
     has_versioned_params = xml_has_children((*rsc)->versioned_parameters);
 #endif
     if (crm_is_true(value) && has_versioned_params) {
         pe_rsc_trace((*rsc), "Migration is disabled for resources with versioned parameters");
     } else if (crm_is_true(value)) {
         set_bit((*rsc)->flags, pe_rsc_allow_migrate);
     } else if ((value == NULL) && remote_node && !has_versioned_params) {
         /* By default, we want remote nodes to be able
          * to float around the cluster without having to stop all the
          * resources within the remote-node before moving. Allowing
          * migration support enables this feature. If this ever causes
          * problems, migration support can be explicitly turned off with
          * allow-migrate=false.
          * We don't support migration for versioned resources, though. */
         set_bit((*rsc)->flags, pe_rsc_allow_migrate);
     }
 
     value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MANAGED);
     if (value != NULL && safe_str_neq("default", value)) {
         gboolean bool_value = TRUE;
 
         crm_str_to_boolean(value, &bool_value);
         if (bool_value == FALSE) {
             clear_bit((*rsc)->flags, pe_rsc_managed);
         } else {
             set_bit((*rsc)->flags, pe_rsc_managed);
         }
     }
 
     value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MAINTENANCE);
     if (value != NULL && safe_str_neq("default", value)) {
         gboolean bool_value = FALSE;
 
         crm_str_to_boolean(value, &bool_value);
         if (bool_value == TRUE) {
             clear_bit((*rsc)->flags, pe_rsc_managed);
             set_bit((*rsc)->flags, pe_rsc_maintenance);
         }
 
     }
     if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
         clear_bit((*rsc)->flags, pe_rsc_managed);
         set_bit((*rsc)->flags, pe_rsc_maintenance);
     }
 
     if (pe_rsc_is_clone(uber_parent(*rsc))) {
         value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_UNIQUE);
         if (crm_is_true(value)) {
             set_bit((*rsc)->flags, pe_rsc_unique);
         }
         if (detect_promotable(*rsc)) {
             set_bit((*rsc)->flags, pe_rsc_promotable);
         }
     } else {
         set_bit((*rsc)->flags, pe_rsc_unique);
     }
 
     pe_rsc_trace((*rsc), "Options for %s", (*rsc)->id);
 
     value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_RESTART);
     if (safe_str_eq(value, "restart")) {
         (*rsc)->restart_type = pe_restart_restart;
         pe_rsc_trace((*rsc), "\tDependency restart handling: restart");
         pe_warn_once(pe_wo_restart_type,
                      "Support for restart-type is deprecated and will be removed in a future release");
 
     } else {
         (*rsc)->restart_type = pe_restart_ignore;
         pe_rsc_trace((*rsc), "\tDependency restart handling: ignore");
     }
 
     value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MULTIPLE);
     if (safe_str_eq(value, "stop_only")) {
         (*rsc)->recovery_type = recovery_stop_only;
         pe_rsc_trace((*rsc), "\tMultiple running resource recovery: stop only");
 
     } else if (safe_str_eq(value, "block")) {
         (*rsc)->recovery_type = recovery_block;
         pe_rsc_trace((*rsc), "\tMultiple running resource recovery: block");
 
     } else {
         (*rsc)->recovery_type = recovery_stop_start;
         pe_rsc_trace((*rsc), "\tMultiple running resource recovery: stop/start");
     }
 
     value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_STICKINESS);
     if (value != NULL && safe_str_neq("default", value)) {
         (*rsc)->stickiness = char2score(value);
     }
 
     value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_FAIL_STICKINESS);
     if (value != NULL && safe_str_neq("default", value)) {
         (*rsc)->migration_threshold = char2score(value);
         if ((*rsc)->migration_threshold < 0) {
             /* @TODO We use 1 here to preserve previous behavior, but this
              * should probably use the default (INFINITY) or 0 (to disable)
              * instead.
              */
             pe_warn_once(pe_wo_neg_threshold,
                          XML_RSC_ATTR_FAIL_STICKINESS
                          " must be non-negative, using 1 instead");
             (*rsc)->migration_threshold = 1;
         }
     }
 
     if (safe_str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH)) {
         set_bit(data_set->flags, pe_flag_have_stonith_resource);
         set_bit((*rsc)->flags, pe_rsc_fence_device);
     }
 
     value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_REQUIRES);
 
   handle_requires_pref:
     if (safe_str_eq(value, "nothing")) {
 
     } else if (safe_str_eq(value, "quorum")) {
         set_bit((*rsc)->flags, pe_rsc_needs_quorum);
 
     } else if (safe_str_eq(value, "unfencing")) {
         if (is_set((*rsc)->flags, pe_rsc_fence_device)) {
             pcmk__config_warn("Resetting '" XML_RSC_ATTR_REQUIRES "' for %s "
                               "to 'quorum' because fencing devices cannot "
                               "require unfencing", (*rsc)->id);
             value = "quorum";
             isdefault = TRUE;
             goto handle_requires_pref;
 
         } else if (is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
             pcmk__config_warn("Resetting '" XML_RSC_ATTR_REQUIRES "' for %s "
                               "to 'quorum' because fencing is disabled",
                               (*rsc)->id);
             value = "quorum";
             isdefault = TRUE;
             goto handle_requires_pref;
 
         } else {
             set_bit((*rsc)->flags, pe_rsc_needs_fencing);
             set_bit((*rsc)->flags, pe_rsc_needs_unfencing);
         }
 
     } else if (safe_str_eq(value, "fencing")) {
         set_bit((*rsc)->flags, pe_rsc_needs_fencing);
         if (is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
             pcmk__config_warn("%s requires fencing but fencing is disabled",
                               (*rsc)->id);
         }
 
     } else {
         const char *orig_value = value;
 
         isdefault = TRUE;
         if(is_set((*rsc)->flags, pe_rsc_fence_device)) {
             value = "quorum";
 
         } else if (((*rsc)->variant == pe_native)
                    && safe_str_eq(crm_element_value((*rsc)->xml, XML_AGENT_ATTR_CLASS),
                                   PCMK_RESOURCE_CLASS_OCF)
                    && safe_str_eq(crm_element_value((*rsc)->xml, XML_AGENT_ATTR_PROVIDER), "pacemaker")
                    && safe_str_eq(crm_element_value((*rsc)->xml, XML_ATTR_TYPE), "remote")
             ) {
             value = "quorum";
 
         } else if (is_set(data_set->flags, pe_flag_enable_unfencing)) {
             value = "unfencing";
 
         } else if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
             value = "fencing";
 
         } else if (data_set->no_quorum_policy == no_quorum_ignore) {
             value = "nothing";
 
         } else {
             value = "quorum";
         }
 
         if (orig_value != NULL) {
             pcmk__config_err("Resetting '" XML_RSC_ATTR_REQUIRES "' for %s "
                              "to '%s' because '%s' is not valid",
                               (*rsc)->id, value, orig_value);
         }
 
         goto handle_requires_pref;
     }
 
     pe_rsc_trace((*rsc), "\tRequired to start: %s%s", value, isdefault?" (default)":"");
     value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_FAIL_TIMEOUT);
     if (value != NULL) {
         // Stored as seconds
         (*rsc)->failure_timeout = (int) (crm_parse_interval_spec(value) / 1000);
     }
 
     if (remote_node) {
         value = g_hash_table_lookup((*rsc)->parameters, XML_REMOTE_ATTR_RECONNECT_INTERVAL);
         if (value) {
             /* reconnect delay works by setting failure_timeout and preventing the
              * connection from starting until the failure is cleared. */
             (*rsc)->remote_reconnect_ms = crm_parse_interval_spec(value);
             /* we want to override any default failure_timeout in use when remote
              * reconnect_interval is in use. */ 
             (*rsc)->failure_timeout = (*rsc)->remote_reconnect_ms / 1000;
         }
     }
 
     get_target_role(*rsc, &((*rsc)->next_role));
     pe_rsc_trace((*rsc), "\tDesired next state: %s",
                  (*rsc)->next_role != RSC_ROLE_UNKNOWN ? role2text((*rsc)->next_role) : "default");
 
     if ((*rsc)->fns->unpack(*rsc, data_set) == FALSE) {
         return FALSE;
     }
 
     if (is_set(data_set->flags, pe_flag_symmetric_cluster)) {
         // This tag must stay exactly the same because it is tested elsewhere
         resource_location(*rsc, NULL, 0, "symmetric_default", data_set);
     } else if (guest_node) {
         /* remote resources tied to a container resource must always be allowed
          * to opt-in to the cluster. Whether the connection resource is actually
          * allowed to be placed on a node is dependent on the container resource */
         resource_location(*rsc, NULL, 0, "remote_connection_default", data_set);
     }
 
     pe_rsc_trace((*rsc), "\tAction notification: %s",
                  is_set((*rsc)->flags, pe_rsc_notify) ? "required" : "not required");
 
     (*rsc)->utilization = crm_str_table_new();
 
     pe__unpack_dataset_nvpairs((*rsc)->xml, XML_TAG_UTILIZATION, NULL,
                                (*rsc)->utilization, NULL, FALSE, data_set);
 
 /* 	data_set->resources = g_list_append(data_set->resources, (*rsc)); */
 
     if (expanded_xml) {
         if (add_template_rsc(xml_obj, data_set) == FALSE) {
             return FALSE;
         }
     }
     return TRUE;
 }
 
 void
 common_update_score(pe_resource_t * rsc, const char *id, int score)
 {
     pe_node_t *node = NULL;
 
     node = pe_hash_table_lookup(rsc->allowed_nodes, id);
     if (node != NULL) {
         pe_rsc_trace(rsc, "Updating score for %s on %s: %d + %d", rsc->id, id, node->weight, score);
         node->weight = merge_weights(node->weight, score);
     }
 
     if (rsc->children) {
         GListPtr gIter = rsc->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
             common_update_score(child_rsc, id, score);
         }
     }
 }
 
 gboolean
 is_parent(pe_resource_t *child, pe_resource_t *rsc)
 {
     pe_resource_t *parent = child;
 
     if (parent == NULL || rsc == NULL) {
         return FALSE;
     }
     while (parent->parent != NULL) {
         if (parent->parent == rsc) {
             return TRUE;
         }
         parent = parent->parent;
     }
     return FALSE;
 }
 
 pe_resource_t *
 uber_parent(pe_resource_t * rsc)
 {
     pe_resource_t *parent = rsc;
 
     if (parent == NULL) {
         return NULL;
     }
     while (parent->parent != NULL && parent->parent->variant != pe_container) {
         parent = parent->parent;
     }
     return parent;
 }
 
 void
 common_free(pe_resource_t * rsc)
 {
     if (rsc == NULL) {
         return;
     }
 
     pe_rsc_trace(rsc, "Freeing %s %d", rsc->id, rsc->variant);
 
     g_list_free(rsc->rsc_cons);
     g_list_free(rsc->rsc_cons_lhs);
     g_list_free(rsc->rsc_tickets);
     g_list_free(rsc->dangling_migrations);
 
     if (rsc->parameters != NULL) {
         g_hash_table_destroy(rsc->parameters);
     }
 #if ENABLE_VERSIONED_ATTRS
     if (rsc->versioned_parameters != NULL) {
         free_xml(rsc->versioned_parameters);
     }
 #endif
     if (rsc->meta != NULL) {
         g_hash_table_destroy(rsc->meta);
     }
     if (rsc->utilization != NULL) {
         g_hash_table_destroy(rsc->utilization);
     }
 
     if (rsc->parent == NULL && is_set(rsc->flags, pe_rsc_orphan)) {
         free_xml(rsc->xml);
         rsc->xml = NULL;
         free_xml(rsc->orig_xml);
         rsc->orig_xml = NULL;
 
         /* if rsc->orig_xml, then rsc->xml is an expanded xml from a template */
     } else if (rsc->orig_xml) {
         free_xml(rsc->xml);
         rsc->xml = NULL;
     }
     if (rsc->running_on) {
         g_list_free(rsc->running_on);
         rsc->running_on = NULL;
     }
     if (rsc->known_on) {
         g_hash_table_destroy(rsc->known_on);
         rsc->known_on = NULL;
     }
     if (rsc->actions) {
         g_list_free(rsc->actions);
         rsc->actions = NULL;
     }
     if (rsc->allowed_nodes) {
         g_hash_table_destroy(rsc->allowed_nodes);
         rsc->allowed_nodes = NULL;
     }
     g_list_free(rsc->fillers);
     g_list_free(rsc->rsc_location);
     pe_rsc_trace(rsc, "Resource freed");
     free(rsc->id);
     free(rsc->clone_name);
     free(rsc->allocated_to);
     free(rsc->variant_opaque);
     free(rsc->pending_task);
     free(rsc);
 }
 
 /*!
  * \brief
  * \internal Find a node (and optionally count all) where resource is active
  *
  * \param[in]  rsc          Resource to check
  * \param[out] count_all    If not NULL, will be set to count of active nodes
  * \param[out] count_clean  If not NULL, will be set to count of clean nodes
  *
  * \return An active node (or NULL if resource is not active anywhere)
  *
  * \note The order of preference is: an active node that is the resource's
  *       partial migration source; if the resource's "requires" is "quorum" or
  *       "nothing", the first active node in the list that is clean and online;
  *       the first active node in the list.
  */
 pe_node_t *
 pe__find_active_on(const pe_resource_t *rsc, unsigned int *count_all,
                    unsigned int *count_clean)
 {
     pe_node_t *active = NULL;
     pe_node_t *node = NULL;
     bool keep_looking = FALSE;
     bool is_happy = FALSE;
 
     if (count_all) {
         *count_all = 0;
     }
     if (count_clean) {
         *count_clean = 0;
     }
     if (rsc == NULL) {
         return NULL;
     }
 
     for (GList *node_iter = rsc->running_on; node_iter != NULL;
          node_iter = node_iter->next) {
 
         node = node_iter->data;
         keep_looking = FALSE;
 
         is_happy = node->details->online && !node->details->unclean;
 
         if (count_all) {
             ++*count_all;
         }
         if (count_clean && is_happy) {
             ++*count_clean;
         }
         if (count_all || count_clean) {
             // If we're counting, we need to go through entire list
             keep_looking = TRUE;
         }
 
         if (rsc->partial_migration_source != NULL) {
             if (node->details == rsc->partial_migration_source->details) {
                 // This is the migration source
                 active = node;
             } else {
                 keep_looking = TRUE;
             }
         } else if (is_not_set(rsc->flags, pe_rsc_needs_fencing)) {
             if (is_happy && (!active || !active->details->online
                              || active->details->unclean)) {
                 // This is the first clean node
                 active = node;
             } else {
                 keep_looking = TRUE;
             }
         }
         if (active == NULL) {
             // This is first node in list
             active = node;
         }
 
         if (keep_looking == FALSE) {
             // Don't waste time iterating if we don't have to
             break;
         }
     }
     return active;
 }
 
 /*!
  * \brief
  * \internal Find and count active nodes according to "requires"
  *
  * \param[in]  rsc    Resource to check
  * \param[out] count  If not NULL, will be set to count of active nodes
  *
  * \return An active node (or NULL if resource is not active anywhere)
  *
  * \note This is a convenience wrapper for pe__find_active_on() where the count
  *       of all active nodes or only clean active nodes is desired according to
  *       the "requires" meta-attribute.
  */
 pe_node_t *
 pe__find_active_requires(const pe_resource_t *rsc, unsigned int *count)
 {
     if (rsc && is_not_set(rsc->flags, pe_rsc_needs_fencing)) {
         return pe__find_active_on(rsc, NULL, count);
     }
     return pe__find_active_on(rsc, count, NULL);
 }
 
 void
 pe__count_common(pe_resource_t *rsc)
 {
     if (rsc->children != NULL) {
         for (GList *item = rsc->children; item != NULL; item = item->next) {
             ((pe_resource_t *) item->data)->fns->count(item->data);
         }
 
     } else if (is_not_set(rsc->flags, pe_rsc_orphan)
                || (rsc->role > RSC_ROLE_STOPPED)) {
         rsc->cluster->ninstances++;
         if (pe__resource_is_disabled(rsc)) {
             rsc->cluster->disabled_resources++;
         }
         if (is_set(rsc->flags, pe_rsc_block)) {
             rsc->cluster->blocked_resources++;
         }
     }
 }
diff --git a/lib/pengine/pe_status_private.h b/lib/pengine/pe_status_private.h
index 4587db6347..0f3814ff10 100644
--- a/lib/pengine/pe_status_private.h
+++ b/lib/pengine/pe_status_private.h
@@ -1,48 +1,68 @@
 /*
- * Copyright 2018 Andrew Beekhof <andrew@beekhof.net>
+ * Copyright 2018-2020 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef PE_STATUS_PRIVATE__H
 #  define PE_STATUS_PRIVATE__H
 
 /* This header is for the sole use of libpe_status, so that functions can be
  * declared with G_GNUC_INTERNAL for efficiency.
  */
 
 #  if CURSES_ENABLED
 #    define status_printw(fmt, args...) printw(fmt, ##args)
 #  else
 #    define status_printw(fmt, args...) \
    crm_err("printw support requires ncurses to be available during configure"); \
    do_crm_log(LOG_WARNING, fmt, ##args);
 #  endif
 
 #  define status_print(fmt, args...)           \
    if(options & pe_print_html) {           \
        FILE *stream = print_data;      \
        fprintf(stream, fmt, ##args);       \
    } else if(options & pe_print_ncurses) {     \
        status_printw(fmt, ##args);     \
    } else if(options & pe_print_printf) {      \
        FILE *stream = print_data;      \
        fprintf(stream, fmt, ##args);       \
    } else if(options & pe_print_xml) {     \
        FILE *stream = print_data;      \
        fprintf(stream, fmt, ##args);       \
    } else if(options & pe_print_log) {     \
        int log_level = *(int*)print_data;  \
        do_crm_log(log_level, fmt, ##args); \
    }
 
 G_GNUC_INTERNAL
 pe_resource_t *pe__create_clone_child(pe_resource_t *rsc,
                                       pe_working_set_t *data_set);
 
 G_GNUC_INTERNAL
 void pe__force_anon(const char *standard, pe_resource_t *rsc, const char *rid,
                     pe_working_set_t *data_set);
 
+G_GNUC_INTERNAL
+gboolean unpack_remote_nodes(xmlNode *xml_resources, pe_working_set_t *data_set);
+
+G_GNUC_INTERNAL
+gboolean unpack_resources(xmlNode *xml_resources, pe_working_set_t *data_set);
+
+G_GNUC_INTERNAL
+gboolean unpack_config(xmlNode *config, pe_working_set_t *data_set);
+
+G_GNUC_INTERNAL
+gboolean unpack_nodes(xmlNode *xml_nodes, pe_working_set_t *data_set);
+
+G_GNUC_INTERNAL
+gboolean unpack_tags(xmlNode *xml_tags, pe_working_set_t *data_set);
+
+G_GNUC_INTERNAL
+gboolean unpack_status(xmlNode *status, pe_working_set_t *data_set);
+
 #endif  // PE_STATUS_PRIVATE__H
diff --git a/lib/pengine/status.c b/lib/pengine/status.c
index eabb6f6d78..8dc5095c15 100644
--- a/lib/pengine/status.c
+++ b/lib/pengine/status.c
@@ -1,440 +1,440 @@
 /*
- * Copyright 2004-2019 the Pacemaker project contributors
+ * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <glib.h>
 
 #include <crm/pengine/internal.h>
-#include <unpack.h>
+#include <pe_status_private.h>
 
 /*!
  * \brief Create a new working set
  *
  * \return New, initialized working set on success, else NULL (and set errno)
  * \note Only pe_working_set_t objects created with this function (as opposed
  *       to statically declared or directly allocated) should be used with the
  *       functions in this library, to allow for future extensions to the
  *       data type. The caller is responsible for freeing the memory with
  *       pe_free_working_set() when the instance is no longer needed.
  */
 pe_working_set_t *
 pe_new_working_set()
 {
     pe_working_set_t *data_set = calloc(1, sizeof(pe_working_set_t));
 
     if (data_set != NULL) {
         set_working_set_defaults(data_set);
     }
     return data_set;
 }
 
 /*!
  * \brief Free a working set
  *
  * \param[in] data_set  Working set to free
  */
 void
 pe_free_working_set(pe_working_set_t *data_set)
 {
     if (data_set != NULL) {
         pe_reset_working_set(data_set);
         free(data_set);
     }
 }
 
 /*
  * Unpack everything
  * At the end you'll have:
  *  - A list of nodes
  *  - A list of resources (each with any dependencies on other resources)
  *  - A list of constraints between resources and nodes
  *  - A list of constraints between start/stop actions
  *  - A list of nodes that need to be stonith'd
  *  - A list of nodes that need to be shutdown
  *  - A list of the possible stop/start actions (without dependencies)
  */
 gboolean
 cluster_status(pe_working_set_t * data_set)
 {
     xmlNode *config = get_xpath_object("//"XML_CIB_TAG_CRMCONFIG, data_set->input, LOG_TRACE);
     xmlNode *cib_nodes = get_xpath_object("//"XML_CIB_TAG_NODES, data_set->input, LOG_TRACE);
     xmlNode *cib_resources = get_xpath_object("//"XML_CIB_TAG_RESOURCES, data_set->input, LOG_TRACE);
     xmlNode *cib_status = get_xpath_object("//"XML_CIB_TAG_STATUS, data_set->input, LOG_TRACE);
     xmlNode *cib_tags = get_xpath_object("//" XML_CIB_TAG_TAGS, data_set->input,
                                          LOG_NEVER);
     const char *value = crm_element_value(data_set->input, XML_ATTR_HAVE_QUORUM);
 
     crm_trace("Beginning unpack");
 
     /* reset remaining global variables */
     data_set->failed = create_xml_node(NULL, "failed-ops");
 
     if (data_set->input == NULL) {
         return FALSE;
     }
 
     if (data_set->now == NULL) {
         data_set->now = crm_time_new(NULL);
     }
 
     if (data_set->dc_uuid == NULL) {
         data_set->dc_uuid = crm_element_value_copy(data_set->input,
                                                    XML_ATTR_DC_UUID);
     }
 
     clear_bit(data_set->flags, pe_flag_have_quorum);
     if (crm_is_true(value)) {
         set_bit(data_set->flags, pe_flag_have_quorum);
     }
 
     data_set->op_defaults = get_xpath_object("//" XML_CIB_TAG_OPCONFIG,
                                              data_set->input, LOG_NEVER);
     data_set->rsc_defaults = get_xpath_object("//" XML_CIB_TAG_RSCCONFIG,
                                               data_set->input, LOG_NEVER);
 
     unpack_config(config, data_set);
 
    if (is_not_set(data_set->flags, pe_flag_quick_location)
        && is_not_set(data_set->flags, pe_flag_have_quorum)
        && data_set->no_quorum_policy != no_quorum_ignore) {
         crm_warn("Fencing and resource management disabled due to lack of quorum");
     }
 
     unpack_nodes(cib_nodes, data_set);
 
     if(is_not_set(data_set->flags, pe_flag_quick_location)) {
         unpack_remote_nodes(cib_resources, data_set);
     }
 
     unpack_resources(cib_resources, data_set);
     unpack_tags(cib_tags, data_set);
 
     if(is_not_set(data_set->flags, pe_flag_quick_location)) {
         unpack_status(cib_status, data_set);
     }
 
     if (is_not_set(data_set->flags, pe_flag_no_counts)) {
         for (GList *item = data_set->resources; item != NULL;
              item = item->next) {
             ((pe_resource_t *) (item->data))->fns->count(item->data);
         }
     }
 
     set_bit(data_set->flags, pe_flag_have_status);
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Free a list of pe_resource_t
  *
  * \param[in] resources  List to free
  *
  * \note When a working set's resource list is freed, that includes the original
  *       storage for the uname and id of any Pacemaker Remote nodes in the
  *       working set's node list, so take care not to use those afterward.
  * \todo Refactor pe_node_t to strdup() the node name.
  */
 static void
 pe_free_resources(GListPtr resources)
 {
     pe_resource_t *rsc = NULL;
     GListPtr iterator = resources;
 
     while (iterator != NULL) {
         rsc = (pe_resource_t *) iterator->data;
         iterator = iterator->next;
         rsc->fns->free(rsc);
     }
     if (resources != NULL) {
         g_list_free(resources);
     }
 }
 
 static void
 pe_free_actions(GListPtr actions)
 {
     GListPtr iterator = actions;
 
     while (iterator != NULL) {
         pe_free_action(iterator->data);
         iterator = iterator->next;
     }
     if (actions != NULL) {
         g_list_free(actions);
     }
 }
 
 static void
 pe_free_nodes(GListPtr nodes)
 {
     for (GList *iterator = nodes; iterator != NULL; iterator = iterator->next) {
         pe_node_t *node = (pe_node_t *) iterator->data;
 
         // Shouldn't be possible, but to be safe ...
         if (node == NULL) {
             continue;
         }
         if (node->details == NULL) {
             free(node);
             continue;
         }
 
         /* This is called after pe_free_resources(), which means that we can't
          * use node->details->uname for Pacemaker Remote nodes.
          */
         crm_trace("Freeing node %s", (pe__is_guest_or_remote_node(node)?
                   "(guest or remote)" : node->details->uname));
 
         if (node->details->attrs != NULL) {
             g_hash_table_destroy(node->details->attrs);
         }
         if (node->details->utilization != NULL) {
             g_hash_table_destroy(node->details->utilization);
         }
         if (node->details->digest_cache != NULL) {
             g_hash_table_destroy(node->details->digest_cache);
         }
         g_list_free(node->details->running_rsc);
         g_list_free(node->details->allocated_rsc);
         free(node->details);
         free(node);
     }
     if (nodes != NULL) {
         g_list_free(nodes);
     }
 }
 
 static void
 pe__free_ordering(GListPtr constraints)
 {
     GListPtr iterator = constraints;
 
     while (iterator != NULL) {
         pe__ordering_t *order = iterator->data;
 
         iterator = iterator->next;
 
         free(order->lh_action_task);
         free(order->rh_action_task);
         free(order);
     }
     if (constraints != NULL) {
         g_list_free(constraints);
     }
 }
 
 static void
 pe__free_location(GListPtr constraints)
 {
     GListPtr iterator = constraints;
 
     while (iterator != NULL) {
         pe__location_t *cons = iterator->data;
 
         iterator = iterator->next;
 
         g_list_free_full(cons->node_list_rh, free);
         free(cons->id);
         free(cons);
     }
     if (constraints != NULL) {
         g_list_free(constraints);
     }
 }
 
 /*!
  * \brief Reset working set to default state without freeing it or constraints
  *
  * \param[in,out] data_set  Working set to reset
  *
  * \deprecated This function is deprecated as part of the API;
  *             pe_reset_working_set() should be used instead.
  */
 void
 cleanup_calculations(pe_working_set_t * data_set)
 {
     if (data_set == NULL) {
         return;
     }
 
     clear_bit(data_set->flags, pe_flag_have_status);
     if (data_set->config_hash != NULL) {
         g_hash_table_destroy(data_set->config_hash);
     }
 
     if (data_set->singletons != NULL) {
         g_hash_table_destroy(data_set->singletons);
     }
 
     if (data_set->tickets) {
         g_hash_table_destroy(data_set->tickets);
     }
 
     if (data_set->template_rsc_sets) {
         g_hash_table_destroy(data_set->template_rsc_sets);
     }
 
     if (data_set->tags) {
         g_hash_table_destroy(data_set->tags);
     }
 
     free(data_set->dc_uuid);
 
     crm_trace("deleting resources");
     pe_free_resources(data_set->resources);
 
     crm_trace("deleting actions");
     pe_free_actions(data_set->actions);
 
     crm_trace("deleting nodes");
     pe_free_nodes(data_set->nodes);
 
     pe__free_param_checks(data_set);
     g_list_free(data_set->stop_needed);
     free_xml(data_set->graph);
     crm_time_free(data_set->now);
     free_xml(data_set->input);
     free_xml(data_set->failed);
 
     set_working_set_defaults(data_set);
 
     CRM_CHECK(data_set->ordering_constraints == NULL,;
         );
     CRM_CHECK(data_set->placement_constraints == NULL,;
         );
 }
 
 /*!
  * \brief Reset a working set to default state without freeing it
  *
  * \param[in,out] data_set  Working set to reset
  */
 void
 pe_reset_working_set(pe_working_set_t *data_set)
 {
     if (data_set == NULL) {
         return;
     }
 
     crm_trace("Deleting %d ordering constraints",
               g_list_length(data_set->ordering_constraints));
     pe__free_ordering(data_set->ordering_constraints);
     data_set->ordering_constraints = NULL;
 
     crm_trace("Deleting %d location constraints",
               g_list_length(data_set->placement_constraints));
     pe__free_location(data_set->placement_constraints);
     data_set->placement_constraints = NULL;
 
     crm_trace("Deleting %d colocation constraints",
               g_list_length(data_set->colocation_constraints));
     g_list_free_full(data_set->colocation_constraints, free);
     data_set->colocation_constraints = NULL;
 
     crm_trace("Deleting %d ticket constraints",
               g_list_length(data_set->ticket_constraints));
     g_list_free_full(data_set->ticket_constraints, free);
     data_set->ticket_constraints = NULL;
 
     cleanup_calculations(data_set);
 }
 
 void
 set_working_set_defaults(pe_working_set_t * data_set)
 {
     memset(data_set, 0, sizeof(pe_working_set_t));
 
     data_set->order_id = 1;
     data_set->action_id = 1;
     data_set->no_quorum_policy = no_quorum_freeze;
 
     data_set->flags = 0x0ULL;
     set_bit(data_set->flags, pe_flag_stop_rsc_orphans);
     set_bit(data_set->flags, pe_flag_symmetric_cluster);
     set_bit(data_set->flags, pe_flag_stop_action_orphans);
 #ifdef DEFAULT_CONCURRENT_FENCING_TRUE
     set_bit(data_set->flags, pe_flag_concurrent_fencing);
 #endif
 }
 
 pe_resource_t *
 pe_find_resource(GListPtr rsc_list, const char *id)
 {
     return pe_find_resource_with_flags(rsc_list, id, pe_find_renamed);
 }
 
 pe_resource_t *
 pe_find_resource_with_flags(GListPtr rsc_list, const char *id, enum pe_find flags)
 {
     GListPtr rIter = NULL;
 
     for (rIter = rsc_list; id && rIter; rIter = rIter->next) {
         pe_resource_t *parent = rIter->data;
 
         pe_resource_t *match =
             parent->fns->find_rsc(parent, id, NULL, flags);
         if (match != NULL) {
             return match;
         }
     }
     crm_trace("No match for %s", id);
     return NULL;
 }
 
 pe_node_t *
 pe_find_node_any(GListPtr nodes, const char *id, const char *uname)
 {
     pe_node_t *match = pe_find_node_id(nodes, id);
 
     if (match) {
         return match;
     }
     crm_trace("Looking up %s via its uname instead", uname);
     return pe_find_node(nodes, uname);
 }
 
 pe_node_t *
 pe_find_node_id(GListPtr nodes, const char *id)
 {
     GListPtr gIter = nodes;
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         if (node && safe_str_eq(node->details->id, id)) {
             return node;
         }
     }
     /* error */
     return NULL;
 }
 
 pe_node_t *
 pe_find_node(GListPtr nodes, const char *uname)
 {
     GListPtr gIter = nodes;
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         if (node && safe_str_eq(node->details->uname, uname)) {
             return node;
         }
     }
     /* error */
     return NULL;
 }
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index aa51f9e02d..37138cedd8 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -1,3873 +1,3871 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <string.h>
 #include <glib.h>
 #include <time.h>
 
 #include <crm/crm.h>
 #include <crm/services.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crm/common/util.h>
 #include <crm/pengine/rules.h>
 #include <crm/pengine/internal.h>
 #include <crm/common/iso8601_internal.h>
-#include <unpack.h>
 #include <pe_status_private.h>
 
 CRM_TRACE_INIT_DATA(pe_status);
 
 #define set_config_flag(data_set, option, flag) do {			\
 	const char *tmp = pe_pref(data_set->config_hash, option);	\
 	if(tmp) {							\
 	    if(crm_is_true(tmp)) {					\
 		set_bit(data_set->flags, flag);			\
 	    } else {							\
 		clear_bit(data_set->flags, flag);		\
 	    }								\
 	}								\
     } while(0)
 
 static void unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
                           xmlNode **last_failure,
                           enum action_fail_response *failed,
                           pe_working_set_t *data_set);
 static void determine_remote_online_status(pe_working_set_t *data_set,
                                            pe_node_t *this_node);
 static void add_node_attrs(xmlNode *attrs, pe_node_t *node, bool overwrite,
                            pe_working_set_t *data_set);
 static void determine_online_status(xmlNode *node_state, pe_node_t *this_node,
                                     pe_working_set_t *data_set);
 
+static void unpack_lrm_resources(pe_node_t *node, xmlNode *lrm_state,
+                                 pe_working_set_t *data_set);
+
 
 // Bitmask for warnings we only want to print once
 uint32_t pe_wo = 0;
 
 static gboolean
 is_dangling_guest_node(pe_node_t *node)
 {
     /* we are looking for a remote-node that was supposed to be mapped to a
      * container resource, but all traces of that container have disappeared 
      * from both the config and the status section. */
     if (pe__is_guest_or_remote_node(node) &&
         node->details->remote_rsc &&
         node->details->remote_rsc->container == NULL &&
         is_set(node->details->remote_rsc->flags, pe_rsc_orphan_container_filler)) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 
 /*!
  * \brief Schedule a fence action for a node
  *
  * \param[in,out] data_set  Current working set of cluster
  * \param[in,out] node      Node to fence
  * \param[in]     reason    Text description of why fencing is needed
  * \param[in]     priority_delay  Whether to consider `priority-fencing-delay`
  */
 void
 pe_fence_node(pe_working_set_t * data_set, pe_node_t * node,
               const char *reason, bool priority_delay)
 {
     CRM_CHECK(node, return);
 
     /* A guest node is fenced by marking its container as failed */
     if (pe__is_guest_node(node)) {
         pe_resource_t *rsc = node->details->remote_rsc->container;
 
         if (is_set(rsc->flags, pe_rsc_failed) == FALSE) {
             if (!is_set(rsc->flags, pe_rsc_managed)) {
                 crm_notice("Not fencing guest node %s "
                            "(otherwise would because %s): "
                            "its guest resource %s is unmanaged",
                            node->details->uname, reason, rsc->id);
             } else {
                 crm_warn("Guest node %s will be fenced "
                          "(by recovering its guest resource %s): %s",
                          node->details->uname, rsc->id, reason);
 
                 /* We don't mark the node as unclean because that would prevent the
                  * node from running resources. We want to allow it to run resources
                  * in this transition if the recovery succeeds.
                  */
                 node->details->remote_requires_reset = TRUE;
                 set_bit(rsc->flags, pe_rsc_failed);
             }
         }
 
     } else if (is_dangling_guest_node(node)) {
         crm_info("Cleaning up dangling connection for guest node %s: "
                  "fencing was already done because %s, "
                  "and guest resource no longer exists",
                  node->details->uname, reason);
         set_bit(node->details->remote_rsc->flags, pe_rsc_failed);
 
     } else if (pe__is_remote_node(node)) {
         pe_resource_t *rsc = node->details->remote_rsc;
 
         if (rsc && (!is_set(rsc->flags, pe_rsc_managed))) {
             crm_notice("Not fencing remote node %s "
                        "(otherwise would because %s): connection is unmanaged",
                        node->details->uname, reason);
         } else if(node->details->remote_requires_reset == FALSE) {
             node->details->remote_requires_reset = TRUE;
             crm_warn("Remote node %s %s: %s",
                      node->details->uname,
                      pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
                      reason);
         }
         node->details->unclean = TRUE;
         // No need to apply `priority-fencing-delay` for remote nodes
         pe_fence_op(node, NULL, TRUE, reason, FALSE, data_set);
 
     } else if (node->details->unclean) {
         crm_trace("Cluster node %s %s because %s",
                   node->details->uname,
                   pe_can_fence(data_set, node)? "would also be fenced" : "also is unclean",
                   reason);
 
     } else {
         crm_warn("Cluster node %s %s: %s",
                  node->details->uname,
                  pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
                  reason);
         node->details->unclean = TRUE;
         pe_fence_op(node, NULL, TRUE, reason, priority_delay, data_set);
     }
 }
 
 // @TODO xpaths can't handle templates, rules, or id-refs
 
 // nvpair with provides or requires set to unfencing
 #define XPATH_UNFENCING_NVPAIR XML_CIB_TAG_NVPAIR                \
     "[(@" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_PROVIDES "'"    \
     "or @" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_REQUIRES "') " \
     "and @" XML_NVPAIR_ATTR_VALUE "='unfencing']"
 
 // unfencing in rsc_defaults or any resource
 #define XPATH_ENABLE_UNFENCING \
     "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RESOURCES   \
     "//" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR                                               \
     "|/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RSCCONFIG  \
     "/" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR
 
 static
 void set_if_xpath(unsigned long long flag, const char *xpath,
                   pe_working_set_t *data_set)
 {
     xmlXPathObjectPtr result = NULL;
 
     if (is_not_set(data_set->flags, flag)) {
         result = xpath_search(data_set->input, xpath);
         if (result && (numXpathResults(result) > 0)) {
             set_bit(data_set->flags, flag);
         }
         freeXpathObject(result);
     }
 }
 
 gboolean
 unpack_config(xmlNode * config, pe_working_set_t * data_set)
 {
     const char *value = NULL;
     GHashTable *config_hash = crm_str_table_new();
 
     data_set->config_hash = config_hash;
 
     pe__unpack_dataset_nvpairs(config, XML_CIB_TAG_PROPSET, NULL, config_hash,
                                CIB_OPTIONS_FIRST, FALSE, data_set);
 
     verify_pe_options(data_set->config_hash);
 
     set_config_flag(data_set, "enable-startup-probes", pe_flag_startup_probes);
     if(is_not_set(data_set->flags, pe_flag_startup_probes)) {
         crm_info("Startup probes: disabled (dangerous)");
     }
 
     value = pe_pref(data_set->config_hash, XML_ATTR_HAVE_WATCHDOG);
     if (value && crm_is_true(value)) {
         crm_notice("Watchdog will be used via SBD if fencing is required "
                    "and stonith-watchdog-timeout is nonzero");
         set_bit(data_set->flags, pe_flag_have_stonith_resource);
     }
 
     /* Set certain flags via xpath here, so they can be used before the relevant
      * configuration sections are unpacked.
      */
     set_if_xpath(pe_flag_enable_unfencing, XPATH_ENABLE_UNFENCING, data_set);
 
     value = pe_pref(data_set->config_hash, "stonith-timeout");
     data_set->stonith_timeout = (int) crm_parse_interval_spec(value);
     crm_debug("STONITH timeout: %d", data_set->stonith_timeout);
 
     set_config_flag(data_set, "stonith-enabled", pe_flag_stonith_enabled);
     crm_debug("STONITH of failed nodes is %s",
               is_set(data_set->flags, pe_flag_stonith_enabled) ? "enabled" : "disabled");
 
     data_set->stonith_action = pe_pref(data_set->config_hash, "stonith-action");
     if (!strcmp(data_set->stonith_action, "poweroff")) {
         pe_warn_once(pe_wo_poweroff,
                      "Support for stonith-action of 'poweroff' is deprecated "
                      "and will be removed in a future release (use 'off' instead)");
         data_set->stonith_action = "off";
     }
     crm_trace("STONITH will %s nodes", data_set->stonith_action);
 
     set_config_flag(data_set, "concurrent-fencing", pe_flag_concurrent_fencing);
     crm_debug("Concurrent fencing is %s",
               is_set(data_set->flags, pe_flag_concurrent_fencing) ? "enabled" : "disabled");
 
     // Default value -1 means `priority-fencing-delay` is disabled
     data_set->priority_fencing_delay = -1;
     value = pe_pref(data_set->config_hash,
                     XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY);
     if (value) {
         data_set->priority_fencing_delay = crm_parse_interval_spec(value) / 1000;
         crm_trace("Priority fencing delay is %ds", data_set->priority_fencing_delay);
     }
 
     set_config_flag(data_set, "stop-all-resources", pe_flag_stop_everything);
     crm_debug("Stop all active resources: %s",
               is_set(data_set->flags, pe_flag_stop_everything) ? "true" : "false");
 
     set_config_flag(data_set, "symmetric-cluster", pe_flag_symmetric_cluster);
     if (is_set(data_set->flags, pe_flag_symmetric_cluster)) {
         crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
     }
 
     value = pe_pref(data_set->config_hash, "no-quorum-policy");
 
     if (safe_str_eq(value, "ignore")) {
         data_set->no_quorum_policy = no_quorum_ignore;
 
     } else if (safe_str_eq(value, "freeze")) {
         data_set->no_quorum_policy = no_quorum_freeze;
 
     } else if (safe_str_eq(value, "suicide")) {
         if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
             int do_panic = 0;
 
             crm_element_value_int(data_set->input, XML_ATTR_QUORUM_PANIC,
                                   &do_panic);
             if (do_panic || is_set(data_set->flags, pe_flag_have_quorum)) {
                 data_set->no_quorum_policy = no_quorum_suicide;
             } else {
                 crm_notice("Resetting no-quorum-policy to 'stop': cluster has never had quorum");
                 data_set->no_quorum_policy = no_quorum_stop;
             }
         } else {
             pcmk__config_err("Resetting no-quorum-policy to 'stop' because "
                              "fencing is disabled");
             data_set->no_quorum_policy = no_quorum_stop;
         }
 
     } else {
         data_set->no_quorum_policy = no_quorum_stop;
     }
 
     switch (data_set->no_quorum_policy) {
         case no_quorum_freeze:
             crm_debug("On loss of quorum: Freeze resources");
             break;
         case no_quorum_stop:
             crm_debug("On loss of quorum: Stop ALL resources");
             break;
         case no_quorum_suicide:
             crm_notice("On loss of quorum: Fence all remaining nodes");
             break;
         case no_quorum_ignore:
             crm_notice("On loss of quorum: Ignore");
             break;
     }
 
     set_config_flag(data_set, "stop-orphan-resources", pe_flag_stop_rsc_orphans);
     crm_trace("Orphan resources are %s",
               is_set(data_set->flags, pe_flag_stop_rsc_orphans) ? "stopped" : "ignored");
 
     set_config_flag(data_set, "stop-orphan-actions", pe_flag_stop_action_orphans);
     crm_trace("Orphan resource actions are %s",
               is_set(data_set->flags, pe_flag_stop_action_orphans) ? "stopped" : "ignored");
 
     set_config_flag(data_set, "remove-after-stop", pe_flag_remove_after_stop);
     crm_trace("Stopped resources are removed from the status section: %s",
               is_set(data_set->flags, pe_flag_remove_after_stop) ? "true" : "false");
 
     set_config_flag(data_set, "maintenance-mode", pe_flag_maintenance_mode);
     crm_trace("Maintenance mode: %s",
               is_set(data_set->flags, pe_flag_maintenance_mode) ? "true" : "false");
 
     set_config_flag(data_set, "start-failure-is-fatal", pe_flag_start_failure_fatal);
     crm_trace("Start failures are %s",
               is_set(data_set->flags,
                      pe_flag_start_failure_fatal) ? "always fatal" : "handled by failcount");
 
     if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
         set_config_flag(data_set, "startup-fencing", pe_flag_startup_fencing);
     }
     if (is_set(data_set->flags, pe_flag_startup_fencing)) {
         crm_trace("Unseen nodes will be fenced");
     } else {
         pe_warn_once(pe_wo_blind, "Blind faith: not fencing unseen nodes");
     }
 
     pcmk__score_red = char2score(pe_pref(data_set->config_hash, "node-health-red"));
     pcmk__score_green = char2score(pe_pref(data_set->config_hash, "node-health-green"));
     pcmk__score_yellow = char2score(pe_pref(data_set->config_hash, "node-health-yellow"));
 
     crm_debug("Node scores: 'red' = %s, 'yellow' = %s, 'green' = %s",
              pe_pref(data_set->config_hash, "node-health-red"),
              pe_pref(data_set->config_hash, "node-health-yellow"),
              pe_pref(data_set->config_hash, "node-health-green"));
 
     data_set->placement_strategy = pe_pref(data_set->config_hash, "placement-strategy");
     crm_trace("Placement strategy: %s", data_set->placement_strategy);
 
     set_config_flag(data_set, "shutdown-lock", pe_flag_shutdown_lock);
     crm_trace("Resources will%s be locked to cleanly shut down nodes",
               (is_set(data_set->flags, pe_flag_shutdown_lock)? "" : " not"));
     if (is_set(data_set->flags, pe_flag_shutdown_lock)) {
         value = pe_pref(data_set->config_hash,
                         XML_CONFIG_ATTR_SHUTDOWN_LOCK_LIMIT);
         data_set->shutdown_lock = crm_parse_interval_spec(value) / 1000;
         crm_trace("Shutdown locks expire after %us", data_set->shutdown_lock);
     }
 
     return TRUE;
 }
 
 static void
 destroy_digest_cache(gpointer ptr)
 {
     op_digest_cache_t *data = ptr;
 
     free_xml(data->params_all);
     free_xml(data->params_secure);
     free_xml(data->params_restart);
 
     free(data->digest_all_calc);
     free(data->digest_restart_calc);
     free(data->digest_secure_calc);
 
     free(data);
 }
 
 pe_node_t *
 pe_create_node(const char *id, const char *uname, const char *type,
                const char *score, pe_working_set_t * data_set)
 {
     pe_node_t *new_node = NULL;
 
     if (pe_find_node(data_set->nodes, uname) != NULL) {
         pcmk__config_warn("More than one node entry has name '%s'", uname);
     }
 
     new_node = calloc(1, sizeof(pe_node_t));
     if (new_node == NULL) {
         return NULL;
     }
 
     new_node->weight = char2score(score);
     new_node->fixed = FALSE;
     new_node->details = calloc(1, sizeof(struct pe_node_shared_s));
 
     if (new_node->details == NULL) {
         free(new_node);
         return NULL;
     }
 
     crm_trace("Creating node for entry %s/%s", uname, id);
     new_node->details->id = id;
     new_node->details->uname = uname;
     new_node->details->online = FALSE;
     new_node->details->shutdown = FALSE;
     new_node->details->rsc_discovery_enabled = TRUE;
     new_node->details->running_rsc = NULL;
     new_node->details->type = node_ping;
 
     if (safe_str_eq(type, "remote")) {
         new_node->details->type = node_remote;
         set_bit(data_set->flags, pe_flag_have_remote_nodes);
     } else if ((type == NULL) || safe_str_eq(type, "member")) {
         new_node->details->type = node_member;
     }
 
     new_node->details->attrs = crm_str_table_new();
 
     if (pe__is_guest_or_remote_node(new_node)) {
         g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
                             strdup("remote"));
     } else {
         g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
                             strdup("cluster"));
     }
 
     new_node->details->utilization = crm_str_table_new();
 
     new_node->details->digest_cache = g_hash_table_new_full(crm_str_hash,
                                                             g_str_equal, free,
                                                             destroy_digest_cache);
 
     data_set->nodes = g_list_insert_sorted(data_set->nodes, new_node, sort_node_uname);
     return new_node;
 }
 
 static const char *
 expand_remote_rsc_meta(xmlNode *xml_obj, xmlNode *parent, pe_working_set_t *data)
 {
     xmlNode *attr_set = NULL;
     xmlNode *attr = NULL;
 
     const char *container_id = ID(xml_obj);
     const char *remote_name = NULL;
     const char *remote_server = NULL;
     const char *remote_port = NULL;
     const char *connect_timeout = "60s";
     const char *remote_allow_migrate=NULL;
     const char *is_managed = NULL;
 
     for (attr_set = __xml_first_child_element(xml_obj); attr_set != NULL;
          attr_set = __xml_next_element(attr_set)) {
         if (safe_str_neq((const char *)attr_set->name, XML_TAG_META_SETS)) {
             continue;
         }
 
         for (attr = __xml_first_child_element(attr_set); attr != NULL;
              attr = __xml_next_element(attr)) {
             const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
             const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
 
             if (safe_str_eq(name, XML_RSC_ATTR_REMOTE_NODE)) {
                 remote_name = value;
             } else if (safe_str_eq(name, "remote-addr")) {
                 remote_server = value;
             } else if (safe_str_eq(name, "remote-port")) {
                 remote_port = value;
             } else if (safe_str_eq(name, "remote-connect-timeout")) {
                 connect_timeout = value;
             } else if (safe_str_eq(name, "remote-allow-migrate")) {
                 remote_allow_migrate=value;
             } else if (safe_str_eq(name, XML_RSC_ATTR_MANAGED)) {
                 is_managed = value;
             }
         }
     }
 
     if (remote_name == NULL) {
         return NULL;
     }
 
     if (pe_find_resource(data->resources, remote_name) != NULL) {
         return NULL;
     }
 
     pe_create_remote_xml(parent, remote_name, container_id,
                          remote_allow_migrate, is_managed,
                          connect_timeout, remote_server, remote_port);
     return remote_name;
 }
 
 static void
 handle_startup_fencing(pe_working_set_t *data_set, pe_node_t *new_node)
 {
     if ((new_node->details->type == node_remote) && (new_node->details->remote_rsc == NULL)) {
         /* Ignore fencing for remote nodes that don't have a connection resource
          * associated with them. This happens when remote node entries get left
          * in the nodes section after the connection resource is removed.
          */
         return;
     }
 
     if (is_set(data_set->flags, pe_flag_startup_fencing)) {
         // All nodes are unclean until we've seen their status entry
         new_node->details->unclean = TRUE;
 
     } else {
         // Blind faith ...
         new_node->details->unclean = FALSE;
     }
 
     /* We need to be able to determine if a node's status section
      * exists or not separate from whether the node is unclean. */
     new_node->details->unseen = TRUE;
 }
 
 gboolean
 unpack_nodes(xmlNode * xml_nodes, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
     pe_node_t *new_node = NULL;
     const char *id = NULL;
     const char *uname = NULL;
     const char *type = NULL;
     const char *score = NULL;
 
     for (xml_obj = __xml_first_child_element(xml_nodes); xml_obj != NULL;
          xml_obj = __xml_next_element(xml_obj)) {
 
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, TRUE)) {
             new_node = NULL;
 
             id = crm_element_value(xml_obj, XML_ATTR_ID);
             uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
             type = crm_element_value(xml_obj, XML_ATTR_TYPE);
             score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
             crm_trace("Processing node %s/%s", uname, id);
 
             if (id == NULL) {
                 pcmk__config_err("Ignoring <" XML_CIB_TAG_NODE
                                  "> entry in configuration without id");
                 continue;
             }
             new_node = pe_create_node(id, uname, type, score, data_set);
 
             if (new_node == NULL) {
                 return FALSE;
             }
 
 /* 		if(data_set->have_quorum == FALSE */
 /* 		   && data_set->no_quorum_policy == no_quorum_stop) { */
 /* 			/\* start shutting resources down *\/ */
 /* 			new_node->weight = -INFINITY; */
 /* 		} */
 
             handle_startup_fencing(data_set, new_node);
 
             add_node_attrs(xml_obj, new_node, FALSE, data_set);
             pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_UTILIZATION, NULL,
                                        new_node->details->utilization, NULL,
                                        FALSE, data_set);
 
             crm_trace("Done with node %s", crm_element_value(xml_obj, XML_ATTR_UNAME));
         }
     }
 
     if (data_set->localhost && pe_find_node(data_set->nodes, data_set->localhost) == NULL) {
         crm_info("Creating a fake local node");
         pe_create_node(data_set->localhost, data_set->localhost, NULL, 0,
                        data_set);
     }
 
     return TRUE;
 }
 
 static void
 setup_container(pe_resource_t * rsc, pe_working_set_t * data_set)
 {
     const char *container_id = NULL;
 
     if (rsc->children) {
         GListPtr gIter = rsc->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
             setup_container(child_rsc, data_set);
         }
         return;
     }
 
     container_id = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_CONTAINER);
     if (container_id && safe_str_neq(container_id, rsc->id)) {
         pe_resource_t *container = pe_find_resource(data_set->resources, container_id);
 
         if (container) {
             rsc->container = container;
             set_bit(container->flags, pe_rsc_is_container);
             container->fillers = g_list_append(container->fillers, rsc);
             pe_rsc_trace(rsc, "Resource %s's container is %s", rsc->id, container_id);
         } else {
             pe_err("Resource %s: Unknown resource container (%s)", rsc->id, container_id);
         }
     }
 }
 
 gboolean
 unpack_remote_nodes(xmlNode * xml_resources, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
 
     /* Create remote nodes and guest nodes from the resource configuration
      * before unpacking resources.
      */
     for (xml_obj = __xml_first_child_element(xml_resources); xml_obj != NULL;
          xml_obj = __xml_next_element(xml_obj)) {
 
         const char *new_node_id = NULL;
 
         /* Check for remote nodes, which are defined by ocf:pacemaker:remote
          * primitives.
          */
         if (xml_contains_remote_node(xml_obj)) {
             new_node_id = ID(xml_obj);
             /* The "pe_find_node" check is here to make sure we don't iterate over
              * an expanded node that has already been added to the node list. */
             if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
                 crm_trace("Found remote node %s defined by resource %s",
                           new_node_id, ID(xml_obj));
                 pe_create_node(new_node_id, new_node_id, "remote", NULL,
                                data_set);
             }
             continue;
         }
 
         /* Check for guest nodes, which are defined by special meta-attributes
          * of a primitive of any type (for example, VirtualDomain or Xen).
          */
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RESOURCE, TRUE)) {
             /* This will add an ocf:pacemaker:remote primitive to the
              * configuration for the guest node's connection, to be unpacked
              * later.
              */
             new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources, data_set);
             if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
                 crm_trace("Found guest node %s in resource %s",
                           new_node_id, ID(xml_obj));
                 pe_create_node(new_node_id, new_node_id, "remote", NULL,
                                data_set);
             }
             continue;
         }
 
         /* Check for guest nodes inside a group. Clones are currently not
          * supported as guest nodes.
          */
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, TRUE)) {
             xmlNode *xml_obj2 = NULL;
             for (xml_obj2 = __xml_first_child_element(xml_obj); xml_obj2 != NULL;
                  xml_obj2 = __xml_next_element(xml_obj2)) {
 
                 new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources, data_set);
 
                 if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
                     crm_trace("Found guest node %s in resource %s inside group %s",
                               new_node_id, ID(xml_obj2), ID(xml_obj));
                     pe_create_node(new_node_id, new_node_id, "remote", NULL,
                                    data_set);
                 }
             }
         }
     }
     return TRUE;
 }
 
 /* Call this after all the nodes and resources have been
  * unpacked, but before the status section is read.
  *
  * A remote node's online status is reflected by the state
  * of the remote node's connection resource. We need to link
  * the remote node to this connection resource so we can have
  * easy access to the connection resource during the scheduler calculations.
  */
 static void
 link_rsc2remotenode(pe_working_set_t *data_set, pe_resource_t *new_rsc)
 {
     pe_node_t *remote_node = NULL;
 
     if (new_rsc->is_remote_node == FALSE) {
         return;
     }
 
     if (is_set(data_set->flags, pe_flag_quick_location)) {
         /* remote_nodes and remote_resources are not linked in quick location calculations */
         return;
     }
 
     remote_node = pe_find_node(data_set->nodes, new_rsc->id);
     CRM_CHECK(remote_node != NULL, return;);
 
     pe_rsc_trace(new_rsc, "Linking remote connection resource %s to node %s",
                  new_rsc->id, remote_node->details->uname);
     remote_node->details->remote_rsc = new_rsc;
 
     if (new_rsc->container == NULL) {
         /* Handle start-up fencing for remote nodes (as opposed to guest nodes)
          * the same as is done for cluster nodes.
          */
         handle_startup_fencing(data_set, remote_node);
 
     } else {
         /* pe_create_node() marks the new node as "remote" or "cluster"; now
          * that we know the node is a guest node, update it correctly.
          */
         g_hash_table_replace(remote_node->details->attrs, strdup(CRM_ATTR_KIND),
                              strdup("container"));
     }
 }
 
 static void
 destroy_tag(gpointer data)
 {
     pe_tag_t *tag = data;
 
     if (tag) {
         free(tag->id);
         g_list_free_full(tag->refs, free);
         free(tag);
     }
 }
 
 /*!
  * \internal
  * \brief Parse configuration XML for resource information
  *
  * \param[in]     xml_resources  Top of resource configuration XML
  * \param[in,out] data_set       Where to put resource information
  *
  * \return TRUE
  *
  * \note unpack_remote_nodes() MUST be called before this, so that the nodes can
  *       be used when common_unpack() calls resource_location()
  */
 gboolean
 unpack_resources(xmlNode * xml_resources, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
     GListPtr gIter = NULL;
 
     data_set->template_rsc_sets = g_hash_table_new_full(crm_str_hash,
                                                         g_str_equal, free,
                                                         destroy_tag);
 
     for (xml_obj = __xml_first_child_element(xml_resources); xml_obj != NULL;
          xml_obj = __xml_next_element(xml_obj)) {
 
         pe_resource_t *new_rsc = NULL;
 
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE, TRUE)) {
             const char *template_id = ID(xml_obj);
 
             if (template_id && g_hash_table_lookup_extended(data_set->template_rsc_sets,
                                                             template_id, NULL, NULL) == FALSE) {
                 /* Record the template's ID for the knowledge of its existence anyway. */
                 g_hash_table_insert(data_set->template_rsc_sets, strdup(template_id), NULL);
             }
             continue;
         }
 
         crm_trace("Beginning unpack... <%s id=%s... >", crm_element_name(xml_obj), ID(xml_obj));
         if (common_unpack(xml_obj, &new_rsc, NULL, data_set)) {
             data_set->resources = g_list_append(data_set->resources, new_rsc);
             pe_rsc_trace(new_rsc, "Added resource %s", new_rsc->id);
 
         } else {
             pcmk__config_err("Ignoring <%s> resource '%s' "
                              "because configuration is invalid",
                              crm_element_name(xml_obj), crm_str(ID(xml_obj)));
             if (new_rsc != NULL && new_rsc->fns != NULL) {
                 new_rsc->fns->free(new_rsc);
             }
         }
     }
 
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
         setup_container(rsc, data_set);
         link_rsc2remotenode(data_set, rsc);
     }
 
     data_set->resources = g_list_sort(data_set->resources, sort_rsc_priority);
     if (is_set(data_set->flags, pe_flag_quick_location)) {
         /* Ignore */
 
     } else if (is_set(data_set->flags, pe_flag_stonith_enabled)
                && is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {
 
         pcmk__config_err("Resource start-up disabled since no STONITH resources have been defined");
         pcmk__config_err("Either configure some or disable STONITH with the stonith-enabled option");
         pcmk__config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
     }
 
     return TRUE;
 }
 
 gboolean
 unpack_tags(xmlNode * xml_tags, pe_working_set_t * data_set)
 {
     xmlNode *xml_tag = NULL;
 
     data_set->tags = g_hash_table_new_full(crm_str_hash, g_str_equal, free,
                                            destroy_tag);
 
     for (xml_tag = __xml_first_child_element(xml_tags); xml_tag != NULL;
          xml_tag = __xml_next_element(xml_tag)) {
 
         xmlNode *xml_obj_ref = NULL;
         const char *tag_id = ID(xml_tag);
 
         if (crm_str_eq((const char *)xml_tag->name, XML_CIB_TAG_TAG, TRUE) == FALSE) {
             continue;
         }
 
         if (tag_id == NULL) {
             pcmk__config_err("Ignoring <%s> without " XML_ATTR_ID,
                              crm_element_name(xml_tag));
             continue;
         }
 
         for (xml_obj_ref = __xml_first_child_element(xml_tag); xml_obj_ref != NULL;
              xml_obj_ref = __xml_next_element(xml_obj_ref)) {
 
             const char *obj_ref = ID(xml_obj_ref);
 
             if (crm_str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, TRUE) == FALSE) {
                 continue;
             }
 
             if (obj_ref == NULL) {
                 pcmk__config_err("Ignoring <%s> for tag '%s' without " XML_ATTR_ID,
                                  crm_element_name(xml_obj_ref), tag_id);
                 continue;
             }
 
             if (add_tag_ref(data_set->tags, tag_id, obj_ref) == FALSE) {
                 return FALSE;
             }
         }
     }
 
     return TRUE;
 }
 
 /* The ticket state section:
  * "/cib/status/tickets/ticket_state" */
 static gboolean
 unpack_ticket_state(xmlNode * xml_ticket, pe_working_set_t * data_set)
 {
     const char *ticket_id = NULL;
     const char *granted = NULL;
     const char *last_granted = NULL;
     const char *standby = NULL;
     xmlAttrPtr xIter = NULL;
 
     pe_ticket_t *ticket = NULL;
 
     ticket_id = ID(xml_ticket);
     if (ticket_id == NULL || strlen(ticket_id) == 0) {
         return FALSE;
     }
 
     crm_trace("Processing ticket state for %s", ticket_id);
 
     ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
     if (ticket == NULL) {
         ticket = ticket_new(ticket_id, data_set);
         if (ticket == NULL) {
             return FALSE;
         }
     }
 
     for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
         const char *prop_name = (const char *)xIter->name;
         const char *prop_value = crm_element_value(xml_ticket, prop_name);
 
         if (crm_str_eq(prop_name, XML_ATTR_ID, TRUE)) {
             continue;
         }
         g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
     }
 
     granted = g_hash_table_lookup(ticket->state, "granted");
     if (granted && crm_is_true(granted)) {
         ticket->granted = TRUE;
         crm_info("We have ticket '%s'", ticket->id);
     } else {
         ticket->granted = FALSE;
         crm_info("We do not have ticket '%s'", ticket->id);
     }
 
     last_granted = g_hash_table_lookup(ticket->state, "last-granted");
     if (last_granted) {
         ticket->last_granted = crm_parse_int(last_granted, 0);
     }
 
     standby = g_hash_table_lookup(ticket->state, "standby");
     if (standby && crm_is_true(standby)) {
         ticket->standby = TRUE;
         if (ticket->granted) {
             crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
         }
     } else {
         ticket->standby = FALSE;
     }
 
     crm_trace("Done with ticket state for %s", ticket_id);
 
     return TRUE;
 }
 
 static gboolean
 unpack_tickets_state(xmlNode * xml_tickets, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
 
     for (xml_obj = __xml_first_child_element(xml_tickets); xml_obj != NULL;
          xml_obj = __xml_next_element(xml_obj)) {
 
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, TRUE) == FALSE) {
             continue;
         }
         unpack_ticket_state(xml_obj, data_set);
     }
 
     return TRUE;
 }
 
 static void
 unpack_handle_remote_attrs(pe_node_t *this_node, xmlNode *state, pe_working_set_t * data_set) 
 {
     const char *resource_discovery_enabled = NULL;
     xmlNode *attrs = NULL;
     pe_resource_t *rsc = NULL;
 
     if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
         return;
     }
 
     if ((this_node == NULL) || !pe__is_guest_or_remote_node(this_node)) {
         return;
     }
     crm_trace("Processing remote node id=%s, uname=%s", this_node->details->id, this_node->details->uname);
 
     this_node->details->remote_maintenance =
         crm_atoi(crm_element_value(state, XML_NODE_IS_MAINTENANCE), "0");
 
     rsc = this_node->details->remote_rsc;
     if (this_node->details->remote_requires_reset == FALSE) {
         this_node->details->unclean = FALSE;
         this_node->details->unseen = FALSE;
     }
     attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
     add_node_attrs(attrs, this_node, TRUE, data_set);
 
     if (pe__shutdown_requested(this_node)) {
         crm_info("Node %s is shutting down", this_node->details->uname);
         this_node->details->shutdown = TRUE;
         if (rsc) {
             rsc->next_role = RSC_ROLE_STOPPED;
         }
     }
  
     if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
         crm_info("Node %s is in standby-mode", this_node->details->uname);
         this_node->details->standby = TRUE;
     }
 
     if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance")) ||
         (rsc && !is_set(rsc->flags, pe_rsc_managed))) {
         crm_info("Node %s is in maintenance-mode", this_node->details->uname);
         this_node->details->maintenance = TRUE;
     }
 
     resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
     if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
         if (pe__is_remote_node(this_node)
             && is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
             crm_warn("Ignoring %s attribute on remote node %s because stonith is disabled",
                      XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
         } else {
             /* This is either a remote node with fencing enabled, or a guest
              * node. We don't care whether fencing is enabled when fencing guest
              * nodes, because they are "fenced" by recovering their containing
              * resource.
              */
             crm_info("Node %s has resource discovery disabled", this_node->details->uname);
             this_node->details->rsc_discovery_enabled = FALSE;
         }
     }
 }
 
 static bool
 unpack_node_loop(xmlNode * status, bool fence, pe_working_set_t * data_set) 
 {
     bool changed = false;
     xmlNode *lrm_rsc = NULL;
 
     for (xmlNode *state = __xml_first_child_element(status); state != NULL;
          state = __xml_next_element(state)) {
 
         const char *id = NULL;
         const char *uname = NULL;
         pe_node_t *this_node = NULL;
         bool process = FALSE;
 
         if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
             continue;
         }
 
         id = crm_element_value(state, XML_ATTR_ID);
         uname = crm_element_value(state, XML_ATTR_UNAME);
         this_node = pe_find_node_any(data_set->nodes, id, uname);
 
         if (this_node == NULL) {
             crm_info("Node %s is unknown", id);
             continue;
 
         } else if (this_node->details->unpacked) {
             crm_trace("Node %s was already processed", id);
             continue;
 
         } else if (!pe__is_guest_or_remote_node(this_node)
                    && is_set(data_set->flags, pe_flag_stonith_enabled)) {
             // A redundant test, but preserves the order for regression tests
             process = TRUE;
 
         } else if (pe__is_guest_or_remote_node(this_node)) {
             bool check = FALSE;
             pe_resource_t *rsc = this_node->details->remote_rsc;
 
             if(fence) {
                 check = TRUE;
 
             } else if(rsc == NULL) {
                 /* Not ready yet */
 
             } else if (pe__is_guest_node(this_node)
                        && rsc->role == RSC_ROLE_STARTED
                        && rsc->container->role == RSC_ROLE_STARTED) {
                 /* Both the connection and its containing resource need to be
                  * known to be up before we process resources running in it.
                  */
                 check = TRUE;
                 crm_trace("Checking node %s/%s/%s status %d/%d/%d", id, rsc->id, rsc->container->id, fence, rsc->role, RSC_ROLE_STARTED);
 
             } else if (!pe__is_guest_node(this_node)
                        && ((rsc->role == RSC_ROLE_STARTED)
                            || is_set(data_set->flags, pe_flag_shutdown_lock))) {
                 check = TRUE;
                 crm_trace("Checking node %s/%s status %d/%d/%d", id, rsc->id, fence, rsc->role, RSC_ROLE_STARTED);
             }
 
             if (check) {
                 determine_remote_online_status(data_set, this_node);
                 unpack_handle_remote_attrs(this_node, state, data_set);
                 process = TRUE;
             }
 
         } else if (this_node->details->online) {
             process = TRUE;
 
         } else if (fence) {
             process = TRUE;
 
         } else if (is_set(data_set->flags, pe_flag_shutdown_lock)) {
             process = TRUE;
         }
 
         if(process) {
             crm_trace("Processing lrm resource entries on %shealthy%s node: %s",
                       fence?"un":"",
                       (pe__is_guest_or_remote_node(this_node)? " remote" : ""),
                       this_node->details->uname);
             changed = TRUE;
             this_node->details->unpacked = TRUE;
 
             lrm_rsc = find_xml_node(state, XML_CIB_TAG_LRM, FALSE);
             lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
             unpack_lrm_resources(this_node, lrm_rsc, data_set);
         }
     }
     return changed;
 }
 
 /* remove nodes that are down, stopping */
 /* create positive rsc_to_node constraints between resources and the nodes they are running on */
 /* anything else? */
 gboolean
 unpack_status(xmlNode * status, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     const char *uname = NULL;
 
     xmlNode *state = NULL;
     pe_node_t *this_node = NULL;
 
     crm_trace("Beginning unpack");
 
     if (data_set->tickets == NULL) {
         data_set->tickets = g_hash_table_new_full(crm_str_hash, g_str_equal,
                                                   free, destroy_ticket);
     }
 
     for (state = __xml_first_child_element(status); state != NULL;
          state = __xml_next_element(state)) {
 
         if (crm_str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, TRUE)) {
             unpack_tickets_state((xmlNode *) state, data_set);
 
         } else if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE)) {
             xmlNode *attrs = NULL;
             const char *resource_discovery_enabled = NULL;
 
             id = crm_element_value(state, XML_ATTR_ID);
             uname = crm_element_value(state, XML_ATTR_UNAME);
             this_node = pe_find_node_any(data_set->nodes, id, uname);
 
             if (uname == NULL) {
                 /* error */
                 continue;
 
             } else if (this_node == NULL) {
                 pcmk__config_warn("Ignoring recorded node status for '%s' "
                                   "because no longer in configuration", uname);
                 continue;
 
             } else if (pe__is_guest_or_remote_node(this_node)) {
                 /* online state for remote nodes is determined by the
                  * rsc state after all the unpacking is done. we do however
                  * need to mark whether or not the node has been fenced as this plays
                  * a role during unpacking cluster node resource state */
                 this_node->details->remote_was_fenced = 
                     crm_atoi(crm_element_value(state, XML_NODE_IS_FENCED), "0");
                 continue;
             }
 
             crm_trace("Processing node id=%s, uname=%s", id, uname);
 
             /* Mark the node as provisionally clean
              * - at least we have seen it in the current cluster's lifetime
              */
             this_node->details->unclean = FALSE;
             this_node->details->unseen = FALSE;
             attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
             add_node_attrs(attrs, this_node, TRUE, data_set);
 
             if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
                 crm_info("Node %s is in standby-mode", this_node->details->uname);
                 this_node->details->standby = TRUE;
             }
 
             if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance"))) {
                 crm_info("Node %s is in maintenance-mode", this_node->details->uname);
                 this_node->details->maintenance = TRUE;
             }
 
             resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
             if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
                 crm_warn("ignoring %s attribute on node %s, disabling resource discovery is not allowed on cluster nodes",
                     XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
             }
 
             crm_trace("determining node state");
             determine_online_status(state, this_node, data_set);
 
             if (is_not_set(data_set->flags, pe_flag_have_quorum)
                 && this_node->details->online
                 && (data_set->no_quorum_policy == no_quorum_suicide)) {
                 /* Everything else should flow from this automatically
                  * (at least until the scheduler becomes able to migrate off
                  * healthy resources)
                  */
                 pe_fence_node(data_set, this_node, "cluster does not have quorum", FALSE);
             }
         }
     }
 
 
     while(unpack_node_loop(status, FALSE, data_set)) {
         crm_trace("Start another loop");
     }
 
     // Now catch any nodes we didn't see
     unpack_node_loop(status, is_set(data_set->flags, pe_flag_stonith_enabled), data_set);
 
     /* Now that we know where resources are, we can schedule stops of containers
      * with failed bundle connections
      */
     if (data_set->stop_needed != NULL) {
         for (GList *item = data_set->stop_needed; item; item = item->next) {
             pe_resource_t *container = item->data;
             pe_node_t *node = pe__current_node(container);
 
             if (node) {
                 stop_action(container, node, FALSE);
             }
         }
         g_list_free(data_set->stop_needed);
         data_set->stop_needed = NULL;
     }
 
     for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         pe_node_t *this_node = gIter->data;
 
         if (this_node == NULL) {
             continue;
         } else if (!pe__is_guest_or_remote_node(this_node)) {
             continue;
         } else if(this_node->details->unpacked) {
             continue;
         }
         determine_remote_online_status(data_set, this_node);
     }
 
     return TRUE;
 }
 
 static gboolean
 determine_online_status_no_fencing(pe_working_set_t * data_set, xmlNode * node_state,
                                    pe_node_t * this_node)
 {
     gboolean online = FALSE;
     const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
     const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
     const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
 
     if (!crm_is_true(in_cluster)) {
         crm_trace("Node is down: in_cluster=%s", crm_str(in_cluster));
 
     } else if (safe_str_eq(is_peer, ONLINESTATUS)) {
         if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {
             online = TRUE;
         } else {
             crm_debug("Node is not ready to run resources: %s", join);
         }
 
     } else if (this_node->details->expected_up == FALSE) {
         crm_trace("Controller is down: in_cluster=%s", crm_str(in_cluster));
         crm_trace("\tis_peer=%s, join=%s, expected=%s",
                   crm_str(is_peer), crm_str(join), crm_str(exp_state));
 
     } else {
         /* mark it unclean */
         pe_fence_node(data_set, this_node, "peer is unexpectedly down", FALSE);
         crm_info("\tin_cluster=%s, is_peer=%s, join=%s, expected=%s",
                  crm_str(in_cluster), crm_str(is_peer), crm_str(join), crm_str(exp_state));
     }
     return online;
 }
 
 static gboolean
 determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_state,
                                 pe_node_t * this_node)
 {
     gboolean online = FALSE;
     gboolean do_terminate = FALSE;
     bool crmd_online = FALSE;
     const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
     const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
     const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
     const char *terminate = pe_node_attribute_raw(this_node, "terminate");
 
 /*
   - XML_NODE_IN_CLUSTER    ::= true|false
   - XML_NODE_IS_PEER       ::= online|offline
   - XML_NODE_JOIN_STATE    ::= member|down|pending|banned
   - XML_NODE_EXPECTED      ::= member|down
 */
 
     if (crm_is_true(terminate)) {
         do_terminate = TRUE;
 
     } else if (terminate != NULL && strlen(terminate) > 0) {
         /* could be a time() value */
         char t = terminate[0];
 
         if (t != '0' && isdigit(t)) {
             do_terminate = TRUE;
         }
     }
 
     crm_trace("%s: in_cluster=%s, is_peer=%s, join=%s, expected=%s, term=%d",
               this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
               crm_str(join), crm_str(exp_state), do_terminate);
 
     online = crm_is_true(in_cluster);
     crmd_online = safe_str_eq(is_peer, ONLINESTATUS);
     if (exp_state == NULL) {
         exp_state = CRMD_JOINSTATE_DOWN;
     }
 
     if (this_node->details->shutdown) {
         crm_debug("%s is shutting down", this_node->details->uname);
 
         /* Slightly different criteria since we can't shut down a dead peer */
         online = crmd_online;
 
     } else if (in_cluster == NULL) {
         pe_fence_node(data_set, this_node, "peer has not been seen by the cluster", FALSE);
 
     } else if (safe_str_eq(join, CRMD_JOINSTATE_NACK)) {
         pe_fence_node(data_set, this_node, "peer failed the pacemaker membership criteria", FALSE);
 
     } else if (do_terminate == FALSE && safe_str_eq(exp_state, CRMD_JOINSTATE_DOWN)) {
 
         if (crm_is_true(in_cluster) || crmd_online) {
             crm_info("- Node %s is not ready to run resources", this_node->details->uname);
             this_node->details->standby = TRUE;
             this_node->details->pending = TRUE;
 
         } else {
             crm_trace("%s is down or still coming up", this_node->details->uname);
         }
 
     } else if (do_terminate && safe_str_eq(join, CRMD_JOINSTATE_DOWN)
                && crm_is_true(in_cluster) == FALSE && !crmd_online) {
         crm_info("Node %s was just shot", this_node->details->uname);
         online = FALSE;
 
     } else if (crm_is_true(in_cluster) == FALSE) {
         // Consider `priority-fencing-delay` for lost nodes
         pe_fence_node(data_set, this_node, "peer is no longer part of the cluster", TRUE);
 
     } else if (!crmd_online) {
         pe_fence_node(data_set, this_node, "peer process is no longer available", FALSE);
 
         /* Everything is running at this point, now check join state */
     } else if (do_terminate) {
         pe_fence_node(data_set, this_node, "termination was requested", FALSE);
 
     } else if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {
         crm_info("Node %s is active", this_node->details->uname);
 
     } else if (safe_str_eq(join, CRMD_JOINSTATE_PENDING)
                || safe_str_eq(join, CRMD_JOINSTATE_DOWN)) {
         crm_info("Node %s is not ready to run resources", this_node->details->uname);
         this_node->details->standby = TRUE;
         this_node->details->pending = TRUE;
 
     } else {
         pe_fence_node(data_set, this_node, "peer was in an unknown state", FALSE);
         crm_warn("%s: in-cluster=%s, is-peer=%s, join=%s, expected=%s, term=%d, shutdown=%d",
                  this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
                  crm_str(join), crm_str(exp_state), do_terminate, this_node->details->shutdown);
     }
 
     return online;
 }
 
 static void
 determine_remote_online_status(pe_working_set_t * data_set, pe_node_t * this_node)
 {
     pe_resource_t *rsc = this_node->details->remote_rsc;
     pe_resource_t *container = NULL;
     pe_node_t *host = NULL;
 
     /* If there is a node state entry for a (former) Pacemaker Remote node
      * but no resource creating that node, the node's connection resource will
      * be NULL. Consider it an offline remote node in that case.
      */
     if (rsc == NULL) {
         this_node->details->online = FALSE;
         goto remote_online_done;
     }
 
     container = rsc->container;
 
     if (container && pcmk__list_of_1(rsc->running_on)) {
         host = rsc->running_on->data;
     }
 
     /* If the resource is currently started, mark it online. */
     if (rsc->role == RSC_ROLE_STARTED) {
         crm_trace("%s node %s presumed ONLINE because connection resource is started",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->online = TRUE;
     }
 
     /* consider this node shutting down if transitioning start->stop */
     if (rsc->role == RSC_ROLE_STARTED && rsc->next_role == RSC_ROLE_STOPPED) {
         crm_trace("%s node %s shutting down because connection resource is stopping",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->shutdown = TRUE;
     }
 
     /* Now check all the failure conditions. */
     if(container && is_set(container->flags, pe_rsc_failed)) {
         crm_trace("Guest node %s UNCLEAN because guest resource failed",
                   this_node->details->id);
         this_node->details->online = FALSE;
         this_node->details->remote_requires_reset = TRUE;
 
     } else if(is_set(rsc->flags, pe_rsc_failed)) {
         crm_trace("%s node %s OFFLINE because connection resource failed",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->online = FALSE;
 
     } else if (rsc->role == RSC_ROLE_STOPPED
         || (container && container->role == RSC_ROLE_STOPPED)) {
 
         crm_trace("%s node %s OFFLINE because its resource is stopped",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->online = FALSE;
         this_node->details->remote_requires_reset = FALSE;
 
     } else if (host && (host->details->online == FALSE)
                && host->details->unclean) {
         crm_trace("Guest node %s UNCLEAN because host is unclean",
                   this_node->details->id);
         this_node->details->online = FALSE;
         this_node->details->remote_requires_reset = TRUE;
     }
 
 remote_online_done:
     crm_trace("Remote node %s online=%s",
         this_node->details->id, this_node->details->online ? "TRUE" : "FALSE");
 }
 
 static void
 determine_online_status(xmlNode * node_state, pe_node_t * this_node, pe_working_set_t * data_set)
 {
     gboolean online = FALSE;
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
 
     CRM_CHECK(this_node != NULL, return);
 
     this_node->details->shutdown = FALSE;
     this_node->details->expected_up = FALSE;
 
     if (pe__shutdown_requested(this_node)) {
         this_node->details->shutdown = TRUE;
 
     } else if (safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) {
         this_node->details->expected_up = TRUE;
     }
 
     if (this_node->details->type == node_ping) {
         this_node->details->unclean = FALSE;
         online = FALSE;         /* As far as resource management is concerned,
                                  * the node is safely offline.
                                  * Anyone caught abusing this logic will be shot
                                  */
 
     } else if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
         online = determine_online_status_no_fencing(data_set, node_state, this_node);
 
     } else {
         online = determine_online_status_fencing(data_set, node_state, this_node);
     }
 
     if (online) {
         this_node->details->online = TRUE;
 
     } else {
         /* remove node from contention */
         this_node->fixed = TRUE;
         this_node->weight = -INFINITY;
     }
 
     if (online && this_node->details->shutdown) {
         /* don't run resources here */
         this_node->fixed = TRUE;
         this_node->weight = -INFINITY;
     }
 
     if (this_node->details->type == node_ping) {
         crm_info("Node %s is not a pacemaker node", this_node->details->uname);
 
     } else if (this_node->details->unclean) {
         pe_proc_warn("Node %s is unclean", this_node->details->uname);
 
     } else if (this_node->details->online) {
         crm_info("Node %s is %s", this_node->details->uname,
                  this_node->details->shutdown ? "shutting down" :
                  this_node->details->pending ? "pending" :
                  this_node->details->standby ? "standby" :
                  this_node->details->maintenance ? "maintenance" : "online");
 
     } else {
         crm_trace("Node %s is offline", this_node->details->uname);
     }
 }
 
 /*!
  * \internal
  * \brief Find the end of a resource's name, excluding any clone suffix
  *
  * \param[in] id  Resource ID to check
  *
  * \return Pointer to last character of resource's base name
  */
 const char *
 pe_base_name_end(const char *id)
 {
     if (!pcmk__str_empty(id)) {
         const char *end = id + strlen(id) - 1;
 
         for (const char *s = end; s > id; --s) {
             switch (*s) {
                 case '0':
                 case '1':
                 case '2':
                 case '3':
                 case '4':
                 case '5':
                 case '6':
                 case '7':
                 case '8':
                 case '9':
                     break;
                 case ':':
                     return (s == end)? s : (s - 1);
                 default:
                     return end;
             }
         }
         return end;
     }
     return NULL;
 }
 
 /*!
  * \internal
  * \brief Get a resource name excluding any clone suffix
  *
  * \param[in] last_rsc_id  Resource ID to check
  *
  * \return Pointer to newly allocated string with resource's base name
  * \note It is the caller's responsibility to free() the result.
  *       This asserts on error, so callers can assume result is not NULL.
  */
 char *
 clone_strip(const char *last_rsc_id)
 {
     const char *end = pe_base_name_end(last_rsc_id);
     char *basename = NULL;
 
     CRM_ASSERT(end);
     basename = strndup(last_rsc_id, end - last_rsc_id + 1);
     CRM_ASSERT(basename);
     return basename;
 }
 
 /*!
  * \internal
  * \brief Get the name of the first instance of a cloned resource
  *
  * \param[in] last_rsc_id  Resource ID to check
  *
  * \return Pointer to newly allocated string with resource's base name plus :0
  * \note It is the caller's responsibility to free() the result.
  *       This asserts on error, so callers can assume result is not NULL.
  */
 char *
 clone_zero(const char *last_rsc_id)
 {
     const char *end = pe_base_name_end(last_rsc_id);
     size_t base_name_len = end - last_rsc_id + 1;
     char *zero = NULL;
 
     CRM_ASSERT(end);
     zero = calloc(base_name_len + 3, sizeof(char));
     CRM_ASSERT(zero);
     memcpy(zero, last_rsc_id, base_name_len);
     zero[base_name_len] = ':';
     zero[base_name_len + 1] = '0';
     return zero;
 }
 
 static pe_resource_t *
 create_fake_resource(const char *rsc_id, xmlNode * rsc_entry, pe_working_set_t * data_set)
 {
     pe_resource_t *rsc = NULL;
     xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
 
     copy_in_properties(xml_rsc, rsc_entry);
     crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
     crm_log_xml_debug(xml_rsc, "Orphan resource");
 
     if (!common_unpack(xml_rsc, &rsc, NULL, data_set)) {
         return NULL;
     }
 
     if (xml_contains_remote_node(xml_rsc)) {
         pe_node_t *node;
 
         crm_debug("Detected orphaned remote node %s", rsc_id);
         node = pe_find_node(data_set->nodes, rsc_id);
         if (node == NULL) {
 	        node = pe_create_node(rsc_id, rsc_id, "remote", NULL, data_set);
         }
         link_rsc2remotenode(data_set, rsc);
 
         if (node) {
             crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
             node->details->shutdown = TRUE;
         }
     }
 
     if (crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER)) {
         /* This orphaned rsc needs to be mapped to a container. */
         crm_trace("Detected orphaned container filler %s", rsc_id);
         set_bit(rsc->flags, pe_rsc_orphan_container_filler);
     }
     set_bit(rsc->flags, pe_rsc_orphan);
     data_set->resources = g_list_append(data_set->resources, rsc);
     return rsc;
 }
 
 /*!
  * \internal
  * \brief Create orphan instance for anonymous clone resource history
  */
 static pe_resource_t *
 create_anonymous_orphan(pe_resource_t *parent, const char *rsc_id,
                         pe_node_t *node, pe_working_set_t *data_set)
 {
     pe_resource_t *top = pe__create_clone_child(parent, data_set);
 
     // find_rsc() because we might be a cloned group
     pe_resource_t *orphan = top->fns->find_rsc(top, rsc_id, NULL, pe_find_clone);
 
     pe_rsc_debug(parent, "Created orphan %s for %s: %s on %s",
                  top->id, parent->id, rsc_id, node->details->uname);
     return orphan;
 }
 
 /*!
  * \internal
  * \brief Check a node for an instance of an anonymous clone
  *
  * Return a child instance of the specified anonymous clone, in order of
  * preference: (1) the instance running on the specified node, if any;
  * (2) an inactive instance (i.e. within the total of clone-max instances);
  * (3) a newly created orphan (i.e. clone-max instances are already active).
  *
  * \param[in] data_set  Cluster information
  * \param[in] node      Node on which to check for instance
  * \param[in] parent    Clone to check
  * \param[in] rsc_id    Name of cloned resource in history (without instance)
  */
 static pe_resource_t *
 find_anonymous_clone(pe_working_set_t * data_set, pe_node_t * node, pe_resource_t * parent,
                      const char *rsc_id)
 {
     GListPtr rIter = NULL;
     pe_resource_t *rsc = NULL;
     pe_resource_t *inactive_instance = NULL;
     gboolean skip_inactive = FALSE;
 
     CRM_ASSERT(parent != NULL);
     CRM_ASSERT(pe_rsc_is_clone(parent));
     CRM_ASSERT(is_not_set(parent->flags, pe_rsc_unique));
 
     // Check for active (or partially active, for cloned groups) instance
     pe_rsc_trace(parent, "Looking for %s on %s in %s", rsc_id, node->details->uname, parent->id);
     for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
         GListPtr locations = NULL;
         pe_resource_t *child = rIter->data;
 
         /* Check whether this instance is already known to be active or pending
          * anywhere, at this stage of unpacking. Because this function is called
          * for a resource before the resource's individual operation history
          * entries are unpacked, locations will generally not contain the
          * desired node.
          *
          * However, there are three exceptions:
          * (1) when child is a cloned group and we have already unpacked the
          *     history of another member of the group on the same node;
          * (2) when we've already unpacked the history of another numbered
          *     instance on the same node (which can happen if globally-unique
          *     was flipped from true to false); and
          * (3) when we re-run calculations on the same data set as part of a
          *     simulation.
          */
         child->fns->location(child, &locations, 2);
         if (locations) {
             /* We should never associate the same numbered anonymous clone
              * instance with multiple nodes, and clone instances can't migrate,
              * so there must be only one location, regardless of history.
              */
             CRM_LOG_ASSERT(locations->next == NULL);
 
             if (((pe_node_t *)locations->data)->details == node->details) {
                 /* This child instance is active on the requested node, so check
                  * for a corresponding configured resource. We use find_rsc()
                  * instead of child because child may be a cloned group, and we
                  * need the particular member corresponding to rsc_id.
                  *
                  * If the history entry is orphaned, rsc will be NULL.
                  */
                 rsc = parent->fns->find_rsc(child, rsc_id, NULL, pe_find_clone);
                 if (rsc) {
                     /* If there are multiple instance history entries for an
                      * anonymous clone in a single node's history (which can
                      * happen if globally-unique is switched from true to
                      * false), we want to consider the instances beyond the
                      * first as orphans, even if there are inactive instance
                      * numbers available.
                      */
                     if (rsc->running_on) {
                         crm_notice("Active (now-)anonymous clone %s has "
                                    "multiple (orphan) instance histories on %s",
                                    parent->id, node->details->uname);
                         skip_inactive = TRUE;
                         rsc = NULL;
                     } else {
                         pe_rsc_trace(parent, "Resource %s, active", rsc->id);
                     }
                 }
             }
             g_list_free(locations);
 
         } else {
             pe_rsc_trace(parent, "Resource %s, skip inactive", child->id);
             if (!skip_inactive && !inactive_instance
                 && is_not_set(child->flags, pe_rsc_block)) {
                 // Remember one inactive instance in case we don't find active
                 inactive_instance = parent->fns->find_rsc(child, rsc_id, NULL,
                                                           pe_find_clone);
 
                 /* ... but don't use it if it was already associated with a
                  * pending action on another node
                  */
                 if (inactive_instance && inactive_instance->pending_node
                     && (inactive_instance->pending_node->details != node->details)) {
                     inactive_instance = NULL;
                 }
             }
         }
     }
 
     if ((rsc == NULL) && !skip_inactive && (inactive_instance != NULL)) {
         pe_rsc_trace(parent, "Resource %s, empty slot", inactive_instance->id);
         rsc = inactive_instance;
     }
 
     /* If the resource has "requires" set to "quorum" or "nothing", and we don't
      * have a clone instance for every node, we don't want to consume a valid
      * instance number for unclean nodes. Such instances may appear to be active
      * according to the history, but should be considered inactive, so we can
      * start an instance elsewhere. Treat such instances as orphans.
      *
      * An exception is instances running on guest nodes -- since guest node
      * "fencing" is actually just a resource stop, requires shouldn't apply.
      *
      * @TODO Ideally, we'd use an inactive instance number if it is not needed
      * for any clean instances. However, we don't know that at this point.
      */
     if ((rsc != NULL) && is_not_set(rsc->flags, pe_rsc_needs_fencing)
         && (!node->details->online || node->details->unclean)
         && !pe__is_guest_node(node)
         && !pe__is_universal_clone(parent, data_set)) {
 
         rsc = NULL;
     }
 
     if (rsc == NULL) {
         rsc = create_anonymous_orphan(parent, rsc_id, node, data_set);
         pe_rsc_trace(parent, "Resource %s, orphan", rsc->id);
     }
     return rsc;
 }
 
 static pe_resource_t *
 unpack_find_resource(pe_working_set_t * data_set, pe_node_t * node, const char *rsc_id,
                      xmlNode * rsc_entry)
 {
     pe_resource_t *rsc = NULL;
     pe_resource_t *parent = NULL;
 
     crm_trace("looking for %s", rsc_id);
     rsc = pe_find_resource(data_set->resources, rsc_id);
 
     if (rsc == NULL) {
         /* If we didn't find the resource by its name in the operation history,
          * check it again as a clone instance. Even when clone-max=0, we create
          * a single :0 orphan to match against here.
          */
         char *clone0_id = clone_zero(rsc_id);
         pe_resource_t *clone0 = pe_find_resource(data_set->resources, clone0_id);
 
         if (clone0 && is_not_set(clone0->flags, pe_rsc_unique)) {
             rsc = clone0;
             parent = uber_parent(clone0);
             crm_trace("%s found as %s (%s)", rsc_id, clone0_id, parent->id);
         } else {
             crm_trace("%s is not known as %s either (orphan)",
                       rsc_id, clone0_id);
         }
         free(clone0_id);
 
     } else if (rsc->variant > pe_native) {
         crm_trace("Resource history for %s is orphaned because it is no longer primitive",
                   rsc_id);
         return NULL;
 
     } else {
         parent = uber_parent(rsc);
     }
 
     if (pe_rsc_is_anon_clone(parent)) {
 
         if (pe_rsc_is_bundled(parent)) {
             rsc = pe__find_bundle_replica(parent->parent, node);
         } else {
             char *base = clone_strip(rsc_id);
 
             rsc = find_anonymous_clone(data_set, node, parent, base);
             free(base);
             CRM_ASSERT(rsc != NULL);
         }
     }
 
     if (rsc && safe_str_neq(rsc_id, rsc->id)
         && safe_str_neq(rsc_id, rsc->clone_name)) {
 
         free(rsc->clone_name);
         rsc->clone_name = strdup(rsc_id);
         pe_rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
                      rsc_id, node->details->uname, rsc->id,
                      (is_set(rsc->flags, pe_rsc_orphan)? " (ORPHAN)" : ""));
     }
     return rsc;
 }
 
 static pe_resource_t *
 process_orphan_resource(xmlNode * rsc_entry, pe_node_t * node, pe_working_set_t * data_set)
 {
     pe_resource_t *rsc = NULL;
     const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
 
     crm_debug("Detected orphan resource %s on %s", rsc_id, node->details->uname);
     rsc = create_fake_resource(rsc_id, rsc_entry, data_set);
 
     if (is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
         clear_bit(rsc->flags, pe_rsc_managed);
 
     } else {
         CRM_CHECK(rsc != NULL, return NULL);
         pe_rsc_trace(rsc, "Added orphan %s", rsc->id);
         resource_location(rsc, NULL, -INFINITY, "__orphan_do_not_run__", data_set);
     }
     return rsc;
 }
 
 static void
 process_rsc_state(pe_resource_t * rsc, pe_node_t * node,
                   enum action_fail_response on_fail,
                   xmlNode * migrate_op, pe_working_set_t * data_set)
 {
     pe_node_t *tmpnode = NULL;
     char *reason = NULL;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
                  rsc->id, role2text(rsc->role), node->details->uname, fail2text(on_fail));
 
     /* process current state */
     if (rsc->role != RSC_ROLE_UNKNOWN) {
         pe_resource_t *iter = rsc;
 
         while (iter) {
             if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
                 pe_node_t *n = node_copy(node);
 
                 pe_rsc_trace(rsc, "%s (aka. %s) known on %s", rsc->id, rsc->clone_name,
                              n->details->uname);
                 g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
             }
             if (is_set(iter->flags, pe_rsc_unique)) {
                 break;
             }
             iter = iter->parent;
         }
     }
 
     /* If a managed resource is believed to be running, but node is down ... */
     if (rsc->role > RSC_ROLE_STOPPED
         && node->details->online == FALSE
         && node->details->maintenance == FALSE
         && is_set(rsc->flags, pe_rsc_managed)) {
 
         gboolean should_fence = FALSE;
 
         /* If this is a guest node, fence it (regardless of whether fencing is
          * enabled, because guest node fencing is done by recovery of the
          * container resource rather than by the fencer). Mark the resource
          * we're processing as failed. When the guest comes back up, its
          * operation history in the CIB will be cleared, freeing the affected
          * resource to run again once we are sure we know its state.
          */
         if (pe__is_guest_node(node)) {
             set_bit(rsc->flags, pe_rsc_failed);
             should_fence = TRUE;
 
         } else if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
             if (pe__is_remote_node(node) && node->details->remote_rsc
                 && is_not_set(node->details->remote_rsc->flags, pe_rsc_failed)) {
 
                 /* Setting unseen means that fencing of the remote node will
                  * occur only if the connection resource is not going to start
                  * somewhere. This allows connection resources on a failed
                  * cluster node to move to another node without requiring the
                  * remote nodes to be fenced as well.
                  */
                 node->details->unseen = TRUE;
                 reason = crm_strdup_printf("%s is active there (fencing will be"
                                            " revoked if remote connection can "
                                            "be re-established elsewhere)",
                                            rsc->id);
             }
             should_fence = TRUE;
         }
 
         if (should_fence) {
             if (reason == NULL) {
                reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
             }
             pe_fence_node(data_set, node, reason, FALSE);
         }
         free(reason);
     }
 
     if (node->details->unclean) {
         /* No extra processing needed
          * Also allows resources to be started again after a node is shot
          */
         on_fail = action_fail_ignore;
     }
 
     switch (on_fail) {
         case action_fail_ignore:
             /* nothing to do */
             break;
 
         case action_fail_fence:
             /* treat it as if it is still running
              * but also mark the node as unclean
              */
             reason = crm_strdup_printf("%s failed there", rsc->id);
             pe_fence_node(data_set, node, reason, FALSE);
             free(reason);
             break;
 
         case action_fail_standby:
             node->details->standby = TRUE;
             node->details->standby_onfail = TRUE;
             break;
 
         case action_fail_block:
             /* is_managed == FALSE will prevent any
              * actions being sent for the resource
              */
             clear_bit(rsc->flags, pe_rsc_managed);
             set_bit(rsc->flags, pe_rsc_block);
             break;
 
         case action_fail_migrate:
             /* make sure it comes up somewhere else
              * or not at all
              */
             resource_location(rsc, node, -INFINITY, "__action_migration_auto__", data_set);
             break;
 
         case action_fail_stop:
             rsc->next_role = RSC_ROLE_STOPPED;
             break;
 
         case action_fail_recover:
             if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
                 set_bit(rsc->flags, pe_rsc_failed);
                 stop_action(rsc, node, FALSE);
             }
             break;
 
         case action_fail_restart_container:
             set_bit(rsc->flags, pe_rsc_failed);
 
             if (rsc->container && pe_rsc_is_bundled(rsc)) {
                 /* A bundle's remote connection can run on a different node than
                  * the bundle's container. We don't necessarily know where the
                  * container is running yet, so remember it and add a stop
                  * action for it later.
                  */
                 data_set->stop_needed = g_list_prepend(data_set->stop_needed,
                                                        rsc->container);
             } else if (rsc->container) {
                 stop_action(rsc->container, node, FALSE);
             } else if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
                 stop_action(rsc, node, FALSE);
             }
             break;
 
         case action_fail_reset_remote:
             set_bit(rsc->flags, pe_rsc_failed);
             if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
                 tmpnode = NULL;
                 if (rsc->is_remote_node) {
                     tmpnode = pe_find_node(data_set->nodes, rsc->id);
                 }
                 if (tmpnode &&
                     pe__is_remote_node(tmpnode) &&
                     tmpnode->details->remote_was_fenced == 0) {
 
                     /* The remote connection resource failed in a way that
                      * should result in fencing the remote node.
                      */
                     pe_fence_node(data_set, tmpnode,
                                   "remote connection is unrecoverable", FALSE);
                 }
             }
 
             /* require the stop action regardless if fencing is occurring or not. */
             if (rsc->role > RSC_ROLE_STOPPED) {
                 stop_action(rsc, node, FALSE);
             }
 
             /* if reconnect delay is in use, prevent the connection from exiting the
              * "STOPPED" role until the failure is cleared by the delay timeout. */
             if (rsc->remote_reconnect_ms) {
                 rsc->next_role = RSC_ROLE_STOPPED;
             }
             break;
     }
 
     /* ensure a remote-node connection failure forces an unclean remote-node
      * to be fenced. By setting unseen = FALSE, the remote-node failure will
      * result in a fencing operation regardless if we're going to attempt to 
      * reconnect to the remote-node in this transition or not. */
     if (is_set(rsc->flags, pe_rsc_failed) && rsc->is_remote_node) {
         tmpnode = pe_find_node(data_set->nodes, rsc->id);
         if (tmpnode && tmpnode->details->unclean) {
             tmpnode->details->unseen = FALSE;
         }
     }
 
     if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
         if (is_set(rsc->flags, pe_rsc_orphan)) {
             if (is_set(rsc->flags, pe_rsc_managed)) {
                 pcmk__config_warn("Detected active orphan %s running on %s",
                                   rsc->id, node->details->uname);
             } else {
                 pcmk__config_warn("Resource '%s' must be stopped manually on "
                                   "%s because cluster is configured not to "
                                   "stop active orphans",
                                   rsc->id, node->details->uname);
             }
         }
 
         native_add_running(rsc, node, data_set);
         if (on_fail != action_fail_ignore) {
             set_bit(rsc->flags, pe_rsc_failed);
         }
 
     } else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
         /* Only do this for older status sections that included instance numbers
          * Otherwise stopped instances will appear as orphans
          */
         pe_rsc_trace(rsc, "Resetting clone_name %s for %s (stopped)", rsc->clone_name, rsc->id);
         free(rsc->clone_name);
         rsc->clone_name = NULL;
 
     } else {
         GList *possible_matches = pe__resource_actions(rsc, node, RSC_STOP,
                                                        FALSE);
         GListPtr gIter = possible_matches;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_action_t *stop = (pe_action_t *) gIter->data;
 
             stop->flags |= pe_action_optional;
         }
 
         g_list_free(possible_matches);
     }
 }
 
 /* create active recurring operations as optional */
 static void
 process_recurring(pe_node_t * node, pe_resource_t * rsc,
                   int start_index, int stop_index,
                   GListPtr sorted_op_list, pe_working_set_t * data_set)
 {
     int counter = -1;
     const char *task = NULL;
     const char *status = NULL;
     GListPtr gIter = sorted_op_list;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "%s: Start index %d, stop index = %d", rsc->id, start_index, stop_index);
 
     for (; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         guint interval_ms = 0;
         char *key = NULL;
         const char *id = ID(rsc_op);
 
         counter++;
 
         if (node->details->online == FALSE) {
             pe_rsc_trace(rsc, "Skipping %s/%s: node is offline", rsc->id, node->details->uname);
             break;
 
             /* Need to check if there's a monitor for role="Stopped" */
         } else if (start_index < stop_index && counter <= stop_index) {
             pe_rsc_trace(rsc, "Skipping %s/%s: resource is not active", id, node->details->uname);
             continue;
 
         } else if (counter < start_index) {
             pe_rsc_trace(rsc, "Skipping %s/%s: old %d", id, node->details->uname, counter);
             continue;
         }
 
         crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
         if (interval_ms == 0) {
             pe_rsc_trace(rsc, "Skipping %s/%s: non-recurring", id, node->details->uname);
             continue;
         }
 
         status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
         if (safe_str_eq(status, "-1")) {
             pe_rsc_trace(rsc, "Skipping %s/%s: status", id, node->details->uname);
             continue;
         }
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         /* create the action */
         key = pcmk__op_key(rsc->id, task, interval_ms);
         pe_rsc_trace(rsc, "Creating %s/%s", key, node->details->uname);
         custom_action(rsc, key, task, node, TRUE, TRUE, data_set);
     }
 }
 
 void
 calculate_active_ops(GListPtr sorted_op_list, int *start_index, int *stop_index)
 {
     int counter = -1;
     int implied_monitor_start = -1;
     int implied_clone_start = -1;
     const char *task = NULL;
     const char *status = NULL;
     GListPtr gIter = sorted_op_list;
 
     *stop_index = -1;
     *start_index = -1;
 
     for (; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         counter++;
 
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
 
         if (safe_str_eq(task, CRMD_ACTION_STOP)
             && safe_str_eq(status, "0")) {
             *stop_index = counter;
 
         } else if (safe_str_eq(task, CRMD_ACTION_START) || safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
             *start_index = counter;
 
         } else if ((implied_monitor_start <= *stop_index) && safe_str_eq(task, CRMD_ACTION_STATUS)) {
             const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);
 
             if (safe_str_eq(rc, "0") || safe_str_eq(rc, "8")) {
                 implied_monitor_start = counter;
             }
         } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE) || safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
             implied_clone_start = counter;
         }
     }
 
     if (*start_index == -1) {
         if (implied_clone_start != -1) {
             *start_index = implied_clone_start;
         } else if (implied_monitor_start != -1) {
             *start_index = implied_monitor_start;
         }
     }
 }
 
 // If resource history entry has shutdown lock, remember lock node and time
 static void
 unpack_shutdown_lock(xmlNode *rsc_entry, pe_resource_t *rsc, pe_node_t *node,
                      pe_working_set_t *data_set)
 {
     time_t lock_time = 0;   // When lock started (i.e. node shutdown time)
 
     if ((crm_element_value_epoch(rsc_entry, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
                                  &lock_time) == pcmk_ok) && (lock_time != 0)) {
 
         if ((data_set->shutdown_lock > 0)
             && (get_effective_time(data_set)
                 > (lock_time + data_set->shutdown_lock))) {
             pe_rsc_info(rsc, "Shutdown lock for %s on %s expired",
                         rsc->id, node->details->uname);
             pe__clear_resource_history(rsc, node, data_set);
         } else {
             rsc->lock_node = node;
             rsc->lock_time = lock_time;
         }
     }
 }
 
 static pe_resource_t *
 unpack_lrm_rsc_state(pe_node_t * node, xmlNode * rsc_entry, pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     int stop_index = -1;
     int start_index = -1;
     enum rsc_role_e req_role = RSC_ROLE_UNKNOWN;
 
     const char *task = NULL;
     const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
 
     pe_resource_t *rsc = NULL;
     GListPtr op_list = NULL;
     GListPtr sorted_op_list = NULL;
 
     xmlNode *migrate_op = NULL;
     xmlNode *rsc_op = NULL;
     xmlNode *last_failure = NULL;
 
     enum action_fail_response on_fail = FALSE;
     enum rsc_role_e saved_role = RSC_ROLE_UNKNOWN;
 
     crm_trace("[%s] Processing %s on %s",
               crm_element_name(rsc_entry), rsc_id, node->details->uname);
 
     /* extract operations */
     op_list = NULL;
     sorted_op_list = NULL;
 
     for (rsc_op = __xml_first_child_element(rsc_entry); rsc_op != NULL;
          rsc_op = __xml_next_element(rsc_op)) {
         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             op_list = g_list_prepend(op_list, rsc_op);
         }
     }
 
     if (is_not_set(data_set->flags, pe_flag_shutdown_lock)) {
         if (op_list == NULL) {
             // If there are no operations, there is nothing to do
             return NULL;
         }
     }
 
     /* find the resource */
     rsc = unpack_find_resource(data_set, node, rsc_id, rsc_entry);
     if (rsc == NULL) {
         if (op_list == NULL) {
             // If there are no operations, there is nothing to do
             return NULL;
         } else {
             rsc = process_orphan_resource(rsc_entry, node, data_set);
         }
     }
     CRM_ASSERT(rsc != NULL);
 
     // Check whether the resource is "shutdown-locked" to this node
     if (is_set(data_set->flags, pe_flag_shutdown_lock)) {
         unpack_shutdown_lock(rsc_entry, rsc, node, data_set);
     }
 
     /* process operations */
     saved_role = rsc->role;
     on_fail = action_fail_ignore;
     rsc->role = RSC_ROLE_UNKNOWN;
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
             migrate_op = rsc_op;
         }
 
         unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail, data_set);
     }
 
     /* create active recurring operations as optional */
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
     process_recurring(node, rsc, start_index, stop_index, sorted_op_list, data_set);
 
     /* no need to free the contents */
     g_list_free(sorted_op_list);
 
     process_rsc_state(rsc, node, on_fail, migrate_op, data_set);
 
     if (get_target_role(rsc, &req_role)) {
         if (rsc->next_role == RSC_ROLE_UNKNOWN || req_role < rsc->next_role) {
             pe_rsc_debug(rsc, "%s: Overwriting calculated next role %s"
                          " with requested next role %s",
                          rsc->id, role2text(rsc->next_role), role2text(req_role));
             rsc->next_role = req_role;
 
         } else if (req_role > rsc->next_role) {
             pe_rsc_info(rsc, "%s: Not overwriting calculated next role %s"
                         " with requested next role %s",
                         rsc->id, role2text(rsc->next_role), role2text(req_role));
         }
     }
 
     if (saved_role > rsc->role) {
         rsc->role = saved_role;
     }
 
     return rsc;
 }
 
 static void
 handle_orphaned_container_fillers(xmlNode * lrm_rsc_list, pe_working_set_t * data_set)
 {
     xmlNode *rsc_entry = NULL;
     for (rsc_entry = __xml_first_child_element(lrm_rsc_list); rsc_entry != NULL;
          rsc_entry = __xml_next_element(rsc_entry)) {
 
         pe_resource_t *rsc;
         pe_resource_t *container;
         const char *rsc_id;
         const char *container_id;
 
         if (safe_str_neq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE)) {
             continue;
         }
 
         container_id = crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER);
         rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
         if (container_id == NULL || rsc_id == NULL) {
             continue;
         }
 
         container = pe_find_resource(data_set->resources, container_id);
         if (container == NULL) {
             continue;
         }
 
         rsc = pe_find_resource(data_set->resources, rsc_id);
         if (rsc == NULL ||
             is_set(rsc->flags, pe_rsc_orphan_container_filler) == FALSE ||
             rsc->container != NULL) {
             continue;
         }
 
         pe_rsc_trace(rsc, "Mapped container of orphaned resource %s to %s",
                      rsc->id, container_id);
         rsc->container = container;
         container->fillers = g_list_append(container->fillers, rsc);
     }
 }
 
-gboolean
-unpack_lrm_resources(pe_node_t * node, xmlNode * lrm_rsc_list, pe_working_set_t * data_set)
+static void
+unpack_lrm_resources(pe_node_t *node, xmlNode *lrm_rsc_list,
+                     pe_working_set_t *data_set)
 {
     xmlNode *rsc_entry = NULL;
     gboolean found_orphaned_container_filler = FALSE;
 
-    CRM_CHECK(node != NULL, return FALSE);
-
-    crm_trace("Unpacking resources on %s", node->details->uname);
-
     for (rsc_entry = __xml_first_child_element(lrm_rsc_list); rsc_entry != NULL;
          rsc_entry = __xml_next_element(rsc_entry)) {
 
         if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
             pe_resource_t *rsc = unpack_lrm_rsc_state(node, rsc_entry, data_set);
             if (!rsc) {
                 continue;
             }
             if (is_set(rsc->flags, pe_rsc_orphan_container_filler)) {
                 found_orphaned_container_filler = TRUE;
             }
         }
     }
 
     /* now that all the resource state has been unpacked for this node
      * we have to go back and map any orphaned container fillers to their
      * container resource */
     if (found_orphaned_container_filler) {
         handle_orphaned_container_fillers(lrm_rsc_list, data_set);
     }
-    return TRUE;
 }
 
 static void
 set_active(pe_resource_t * rsc)
 {
     pe_resource_t *top = uber_parent(rsc);
 
     if (top && is_set(top->flags, pe_rsc_promotable)) {
         rsc->role = RSC_ROLE_SLAVE;
     } else {
         rsc->role = RSC_ROLE_STARTED;
     }
 }
 
 static void
 set_node_score(gpointer key, gpointer value, gpointer user_data)
 {
     pe_node_t *node = value;
     int *score = user_data;
 
     node->weight = *score;
 }
 
 #define STATUS_PATH_MAX 1024
 static xmlNode *
 find_lrm_op(const char *resource, const char *op, const char *node, const char *source,
             bool success_only, pe_working_set_t *data_set)
 {
     int offset = 0;
     char xpath[STATUS_PATH_MAX];
     xmlNode *xml = NULL;
 
     offset += snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//node_state[@uname='%s']", node);
     offset +=
         snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//" XML_LRM_TAG_RESOURCE "[@id='%s']",
                  resource);
 
     /* Need to check against transition_magic too? */
     if (source && safe_str_eq(op, CRMD_ACTION_MIGRATE)) {
         offset +=
             snprintf(xpath + offset, STATUS_PATH_MAX - offset,
                      "/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_target='%s']", op,
                      source);
     } else if (source && safe_str_eq(op, CRMD_ACTION_MIGRATED)) {
         offset +=
             snprintf(xpath + offset, STATUS_PATH_MAX - offset,
                      "/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_source='%s']", op,
                      source);
     } else {
         offset +=
             snprintf(xpath + offset, STATUS_PATH_MAX - offset,
                      "/" XML_LRM_TAG_RSC_OP "[@operation='%s']", op);
     }
 
     CRM_LOG_ASSERT(offset > 0);
     xml = get_xpath_object(xpath, data_set->input, LOG_DEBUG);
 
     if (xml && success_only) {
         int rc = PCMK_OCF_UNKNOWN_ERROR;
         int status = PCMK_LRM_OP_ERROR;
 
         crm_element_value_int(xml, XML_LRM_ATTR_RC, &rc);
         crm_element_value_int(xml, XML_LRM_ATTR_OPSTATUS, &status);
         if ((rc != PCMK_OCF_OK) || (status != PCMK_LRM_OP_DONE)) {
             return NULL;
         }
     }
     return xml;
 }
 
 static int
 pe__call_id(xmlNode *op_xml)
 {
     int id = 0;
 
     if (op_xml) {
         crm_element_value_int(op_xml, XML_LRM_ATTR_CALLID, &id);
     }
     return id;
 }
 
 /*!
  * \brief Check whether a stop happened on the same node after some event
  *
  * \param[in] rsc       Resource being checked
  * \param[in] node      Node being checked
  * \param[in] xml_op    Event that stop is being compared to
  * \param[in] data_set  Cluster working set
  *
  * \return TRUE if stop happened after event, FALSE otherwise
  *
  * \note This is really unnecessary, but kept as a safety mechanism. We
  *       currently don't save more than one successful event in history, so this
  *       only matters when processing really old CIB files that we don't
  *       technically support anymore, or as preparation for logging an extended
  *       history in the future.
  */
 static bool
 stop_happened_after(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
                     pe_working_set_t *data_set)
 {
     xmlNode *stop_op = find_lrm_op(rsc->id, CRMD_ACTION_STOP,
                                    node->details->uname, NULL, TRUE, data_set);
 
     return (stop_op && (pe__call_id(stop_op) > pe__call_id(xml_op)));
 }
 
 static void
 unpack_migrate_to_success(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
                           pe_working_set_t *data_set)
 {
     /* A successful migration sequence is:
      *    migrate_to on source node
      *    migrate_from on target node
      *    stop on source node
      *
      * If a migrate_to is followed by a stop, the entire migration (successful
      * or failed) is complete, and we don't care what happened on the target.
      *
      * If no migrate_from has happened, the migration is considered to be
      * "partial". If the migrate_from failed, make sure the resource gets
      * stopped on both source and target (if up).
      *
      * If the migrate_to and migrate_from both succeeded (which also implies the
      * resource is no longer running on the source), but there is no stop, the
      * migration is considered to be "dangling". Schedule a stop on the source
      * in this case.
      */
     int from_rc = 0;
     int from_status = 0;
     pe_node_t *target_node = NULL;
     pe_node_t *source_node = NULL;
     xmlNode *migrate_from = NULL;
     const char *source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
     const char *target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
     // Sanity check
     CRM_CHECK(source && target && !strcmp(source, node->details->uname), return);
 
     if (stop_happened_after(rsc, node, xml_op, data_set)) {
         return;
     }
 
     // Clones are not allowed to migrate, so role can't be master
     rsc->role = RSC_ROLE_STARTED;
 
     target_node = pe_find_node(data_set->nodes, target);
     source_node = pe_find_node(data_set->nodes, source);
 
     // Check whether there was a migrate_from action on the target
     migrate_from = find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, target,
                                source, FALSE, data_set);
     if (migrate_from) {
         crm_element_value_int(migrate_from, XML_LRM_ATTR_RC, &from_rc);
         crm_element_value_int(migrate_from, XML_LRM_ATTR_OPSTATUS, &from_status);
         pe_rsc_trace(rsc, "%s op on %s exited with status=%d, rc=%d",
                      ID(migrate_from), target, from_status, from_rc);
     }
 
     if (migrate_from && from_rc == PCMK_OCF_OK
         && from_status == PCMK_LRM_OP_DONE) {
         /* The migrate_to and migrate_from both succeeded, so mark the migration
          * as "dangling". This will be used to schedule a stop action on the
          * source without affecting the target.
          */
         pe_rsc_trace(rsc, "Detected dangling migration op: %s on %s", ID(xml_op),
                      source);
         rsc->role = RSC_ROLE_STOPPED;
         rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);
 
     } else if (migrate_from && (from_status != PCMK_LRM_OP_PENDING)) { // Failed
         if (target_node && target_node->details->online) {
             pe_rsc_trace(rsc, "Marking active on %s %p %d", target, target_node,
                          target_node->details->online);
             native_add_running(rsc, target_node, data_set);
         }
 
     } else { // Pending, or complete but erased
         if (target_node && target_node->details->online) {
             pe_rsc_trace(rsc, "Marking active on %s %p %d", target, target_node,
                          target_node->details->online);
 
             native_add_running(rsc, target_node, data_set);
             if (source_node && source_node->details->online) {
                 /* This is a partial migration: the migrate_to completed
                  * successfully on the source, but the migrate_from has not
                  * completed. Remember the source and target; if the newly
                  * chosen target remains the same when we schedule actions
                  * later, we may continue with the migration.
                  */
                 rsc->partial_migration_target = target_node;
                 rsc->partial_migration_source = source_node;
             }
         } else {
             /* Consider it failed here - forces a restart, prevents migration */
             set_bit(rsc->flags, pe_rsc_failed);
             clear_bit(rsc->flags, pe_rsc_allow_migrate);
         }
     }
 }
 
 static void
 unpack_migrate_to_failure(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
                           pe_working_set_t *data_set)
 {
     int target_stop_id = 0;
     int target_migrate_from_id = 0;
     xmlNode *target_stop = NULL;
     xmlNode *target_migrate_from = NULL;
     const char *source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
     const char *target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
     // Sanity check
     CRM_CHECK(source && target && !strcmp(source, node->details->uname), return);
 
     /* If a migration failed, we have to assume the resource is active. Clones
      * are not allowed to migrate, so role can't be master.
      */
     rsc->role = RSC_ROLE_STARTED;
 
     // Check for stop on the target
     target_stop = find_lrm_op(rsc->id, CRMD_ACTION_STOP, target, NULL,
                               TRUE, data_set);
     target_stop_id = pe__call_id(target_stop);
 
     // Check for migrate_from on the target
     target_migrate_from = find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, target,
                                       source, TRUE, data_set);
     target_migrate_from_id = pe__call_id(target_migrate_from);
 
     if ((target_stop == NULL) || (target_stop_id < target_migrate_from_id)) {
         /* There was no stop on the source, or a stop that happened before a
          * migrate_from, so assume the resource is still active on the target
          * (if it is up).
          */
         pe_node_t *target_node = pe_find_node(data_set->nodes, target);
 
         pe_rsc_trace(rsc, "stop (%d) + migrate_from (%d)",
                      target_stop_id, target_migrate_from_id);
         if (target_node && target_node->details->online) {
             native_add_running(rsc, target_node, data_set);
         }
 
     } else if (target_migrate_from == NULL) {
         /* We know there was a stop on the target, but there may not have been a
          * migrate_from (the stop could have happened before migrate_from was
          * scheduled or attempted).
          *
          * That means this could be a "dangling" migration. But first, check
          * whether there is a newer migrate_from or start on the source node --
          * it's possible the failed migration was followed by a successful
          * full restart or migration in the reverse direction, in which case we
          * don't want to force it to stop.
          */
         xmlNode *source_migrate_from = NULL;
         xmlNode *source_start = NULL;
         int source_migrate_to_id = pe__call_id(xml_op);
 
         source_migrate_from = find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, source,
                                           NULL, TRUE, data_set);
         if (pe__call_id(source_migrate_from) > source_migrate_to_id) {
             return;
         }
 
         source_start = find_lrm_op(rsc->id, CRMD_ACTION_START, source, NULL,
                                    TRUE, data_set);
         if (pe__call_id(source_start) > source_migrate_to_id) {
             return;
         }
 
         // Mark node as having dangling migration so we can force a stop later
         rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);
     }
 }
 
 static void
 unpack_migrate_from_failure(pe_resource_t *rsc, pe_node_t *node,
                             xmlNode *xml_op, pe_working_set_t *data_set)
 {
     xmlNode *source_stop = NULL;
     xmlNode *source_migrate_to = NULL;
     const char *source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
     const char *target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
     // Sanity check
     CRM_CHECK(source && target && !strcmp(target, node->details->uname), return);
 
     /* If a migration failed, we have to assume the resource is active. Clones
      * are not allowed to migrate, so role can't be master.
      */
     rsc->role = RSC_ROLE_STARTED;
 
     // Check for a stop on the source
     source_stop = find_lrm_op(rsc->id, CRMD_ACTION_STOP, source, NULL,
                               TRUE, data_set);
 
     // Check for a migrate_to on the source
     source_migrate_to = find_lrm_op(rsc->id, CRMD_ACTION_MIGRATE,
                                     source, target, TRUE, data_set);
 
     if ((source_stop == NULL)
         || (pe__call_id(source_stop) < pe__call_id(source_migrate_to))) {
         /* There was no stop on the source, or a stop that happened before
          * migrate_to, so assume the resource is still active on the source (if
          * it is up).
          */
         pe_node_t *source_node = pe_find_node(data_set->nodes, source);
 
         if (source_node && source_node->details->online) {
             native_add_running(rsc, source_node, data_set);
         }
     }
 }
 
 static void
 record_failed_op(xmlNode *op, const pe_node_t *node,
                  const pe_resource_t *rsc, pe_working_set_t *data_set)
 {
     xmlNode *xIter = NULL;
     const char *op_key = crm_element_value(op, XML_LRM_ATTR_TASK_KEY);
 
     if (node->details->online == FALSE) {
         return;
     }
 
     for (xIter = data_set->failed->children; xIter; xIter = xIter->next) {
         const char *key = crm_element_value(xIter, XML_LRM_ATTR_TASK_KEY);
         const char *uname = crm_element_value(xIter, XML_ATTR_UNAME);
 
         if(safe_str_eq(op_key, key) && safe_str_eq(uname, node->details->uname)) {
             crm_trace("Skipping duplicate entry %s on %s", op_key, node->details->uname);
             return;
         }
     }
 
     crm_trace("Adding entry %s on %s", op_key, node->details->uname);
     crm_xml_add(op, XML_ATTR_UNAME, node->details->uname);
     crm_xml_add(op, XML_LRM_ATTR_RSCID, rsc->id);
     add_node_copy(data_set->failed, op);
 }
 
 static const char *get_op_key(xmlNode *xml_op)
 {
     const char *key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
     if(key == NULL) {
         key = ID(xml_op);
     }
     return key;
 }
 
 static const char *
 last_change_str(xmlNode *xml_op)
 {
     time_t when;
     const char *when_s = NULL;
 
     if (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
                                 &when) == pcmk_ok) {
         when_s = pcmk__epoch2str(&when);
         if (when_s) {
             // Skip day of week to make message shorter
             when_s = strchr(when_s, ' ');
             if (when_s) {
                 ++when_s;
             }
         }
     }
     return ((when_s && *when_s)? when_s : "unknown time");
 }
 
 static void
 unpack_rsc_op_failure(pe_resource_t * rsc, pe_node_t * node, int rc, xmlNode * xml_op, xmlNode ** last_failure,
                       enum action_fail_response * on_fail, pe_working_set_t * data_set)
 {
     guint interval_ms = 0;
     bool is_probe = false;
     pe_action_t *action = NULL;
 
     const char *key = get_op_key(xml_op);
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     const char *exit_reason = crm_element_value(xml_op,
                                                 XML_LRM_ATTR_EXIT_REASON);
 
     CRM_ASSERT(rsc);
     CRM_CHECK(task != NULL, return);
 
     *last_failure = xml_op;
 
     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
     if ((interval_ms == 0) && !strcmp(task, CRMD_ACTION_STATUS)) {
         is_probe = true;
     }
 
     if (exit_reason == NULL) {
         exit_reason = "";
     }
 
     if (is_not_set(data_set->flags, pe_flag_symmetric_cluster)
         && (rc == PCMK_OCF_NOT_INSTALLED)) {
         crm_trace("Unexpected result (%s%s%s) was recorded for "
                   "%s of %s on %s at %s " CRM_XS " rc=%d id=%s",
                   services_ocf_exitcode_str(rc),
                   (*exit_reason? ": " : ""), exit_reason,
                   (is_probe? "probe" : task), rsc->id, node->details->uname,
                   last_change_str(xml_op), rc, ID(xml_op));
     } else {
         crm_warn("Unexpected result (%s%s%s) was recorded for "
                   "%s of %s on %s at %s " CRM_XS " rc=%d id=%s",
                  services_ocf_exitcode_str(rc),
                  (*exit_reason? ": " : ""), exit_reason,
                  (is_probe? "probe" : task), rsc->id, node->details->uname,
                  last_change_str(xml_op), rc, ID(xml_op));
 
         if (is_probe && (rc != PCMK_OCF_OK)
             && (rc != PCMK_OCF_NOT_RUNNING)
             && (rc != PCMK_OCF_RUNNING_MASTER)) {
 
             /* A failed (not just unexpected) probe result could mean the user
              * didn't know resources will be probed even where they can't run.
              */
             crm_notice("If it is not possible for %s to run on %s, see "
                        "the resource-discovery option for location constraints",
                        rsc->id, node->details->uname);
         }
 
         record_failed_op(xml_op, node, rsc, data_set);
     }
 
     action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);
     if ((action->on_fail <= action_fail_fence && *on_fail < action->on_fail) ||
         (action->on_fail == action_fail_reset_remote && *on_fail <= action_fail_recover) ||
         (action->on_fail == action_fail_restart_container && *on_fail <= action_fail_recover) ||
         (*on_fail == action_fail_restart_container && action->on_fail >= action_fail_migrate)) {
         pe_rsc_trace(rsc, "on-fail %s -> %s for %s (%s)", fail2text(*on_fail),
                      fail2text(action->on_fail), action->uuid, key);
         *on_fail = action->on_fail;
     }
 
     if (!strcmp(task, CRMD_ACTION_STOP)) {
         resource_location(rsc, node, -INFINITY, "__stop_fail__", data_set);
 
     } else if (!strcmp(task, CRMD_ACTION_MIGRATE)) {
         unpack_migrate_to_failure(rsc, node, xml_op, data_set);
 
     } else if (!strcmp(task, CRMD_ACTION_MIGRATED)) {
         unpack_migrate_from_failure(rsc, node, xml_op, data_set);
 
     } else if (!strcmp(task, CRMD_ACTION_PROMOTE)) {
         rsc->role = RSC_ROLE_MASTER;
 
     } else if (!strcmp(task, CRMD_ACTION_DEMOTE)) {
         if (action->on_fail == action_fail_block) {
             rsc->role = RSC_ROLE_MASTER;
             rsc->next_role = RSC_ROLE_STOPPED;
 
         } else if(rc == PCMK_OCF_NOT_RUNNING) {
             rsc->role = RSC_ROLE_STOPPED;
 
         } else {
             /* Staying in master role would put the scheduler and controller
              * into a loop. Setting slave role is not dangerous because the
              * resource will be stopped as part of recovery, and any master
              * promotion will be ordered after that stop.
              */
             rsc->role = RSC_ROLE_SLAVE;
         }
     }
 
     if(is_probe && rc == PCMK_OCF_NOT_INSTALLED) {
         /* leave stopped */
         pe_rsc_trace(rsc, "Leaving %s stopped", rsc->id);
         rsc->role = RSC_ROLE_STOPPED;
 
     } else if (rsc->role < RSC_ROLE_STARTED) {
         pe_rsc_trace(rsc, "Setting %s active", rsc->id);
         set_active(rsc);
     }
 
     pe_rsc_trace(rsc, "Resource %s: role=%s, unclean=%s, on_fail=%s, fail_role=%s",
                  rsc->id, role2text(rsc->role),
                  node->details->unclean ? "true" : "false",
                  fail2text(action->on_fail), role2text(action->fail_role));
 
     if (action->fail_role != RSC_ROLE_STARTED && rsc->next_role < action->fail_role) {
         rsc->next_role = action->fail_role;
     }
 
     if (action->fail_role == RSC_ROLE_STOPPED) {
         int score = -INFINITY;
 
         pe_resource_t *fail_rsc = rsc;
 
         if (fail_rsc->parent) {
             pe_resource_t *parent = uber_parent(fail_rsc);
 
             if (pe_rsc_is_clone(parent)
                 && is_not_set(parent->flags, pe_rsc_unique)) {
                 /* For clone resources, if a child fails on an operation
                  * with on-fail = stop, all the resources fail.  Do this by preventing
                  * the parent from coming up again. */
                 fail_rsc = parent;
             }
         }
         crm_notice("%s will not be started under current conditions",
                    fail_rsc->id);
         /* make sure it doesn't come up again */
         if (fail_rsc->allowed_nodes != NULL) {
             g_hash_table_destroy(fail_rsc->allowed_nodes);
         }
         fail_rsc->allowed_nodes = node_hash_from_list(data_set->nodes);
         g_hash_table_foreach(fail_rsc->allowed_nodes, set_node_score, &score);
     }
 
     pe_free_action(action);
 }
 
 /*!
  * \internal
  * \brief Remap operation status based on action result
  *
  * Given an action result, determine an appropriate operation status for the
  * purposes of responding to the action (the status provided by the executor is
  * not directly usable since the executor does not know what was expected).
  *
  * \param[in,out] rsc        Resource that operation history entry is for
  * \param[in]     rc         Actual return code of operation
  * \param[in]     target_rc  Expected return code of operation
  * \param[in]     node       Node where operation was executed
  * \param[in]     xml_op     Operation history entry XML from CIB status
  * \param[in,out] on_fail    What should be done about the result
  * \param[in]     data_set   Current cluster working set
  *
  * \return Operation status based on return code and action info
  * \note This may update the resource's current and next role.
  */
 static int
 determine_op_status(
     pe_resource_t *rsc, int rc, int target_rc, pe_node_t * node, xmlNode * xml_op, enum action_fail_response * on_fail, pe_working_set_t * data_set) 
 {
     guint interval_ms = 0;
     bool is_probe = false;
     int result = PCMK_LRM_OP_DONE;
     const char *key = get_op_key(xml_op);
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     const char *exit_reason = crm_element_value(xml_op,
                                                 XML_LRM_ATTR_EXIT_REASON);
 
     CRM_ASSERT(rsc);
     CRM_CHECK(task != NULL, return PCMK_LRM_OP_ERROR);
 
     if (exit_reason == NULL) {
         exit_reason = "";
     }
 
     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
     if ((interval_ms == 0) && !strcmp(task, CRMD_ACTION_STATUS)) {
         is_probe = true;
         task = "probe";
     }
 
     if (target_rc < 0) {
         /* Pre-1.0 Pacemaker versions, and Pacemaker 1.1.6 or earlier with
          * Heartbeat 2.0.7 or earlier as the cluster layer, did not include the
          * target_rc in the transition key, which (along with the similar case
          * of a corrupted transition key in the CIB) will be reported to this
          * function as -1. Pacemaker 2.0+ does not support rolling upgrades from
          * those versions or processing of saved CIB files from those versions,
          * so we do not need to care much about this case.
          */
         result = PCMK_LRM_OP_ERROR;
         crm_warn("Expected result not found for %s on %s (corrupt or obsolete CIB?)",
                  key, node->details->uname);
 
     } else if (target_rc != rc) {
         result = PCMK_LRM_OP_ERROR;
         pe_rsc_debug(rsc, "%s on %s: expected %d (%s), got %d (%s%s%s)",
                      key, node->details->uname,
                      target_rc, services_ocf_exitcode_str(target_rc),
                      rc, services_ocf_exitcode_str(rc),
                      (*exit_reason? ": " : ""), exit_reason);
     }
 
     switch (rc) {
         case PCMK_OCF_OK:
             if (is_probe && (target_rc == PCMK_OCF_NOT_RUNNING)) {
                 result = PCMK_LRM_OP_DONE;
                 pe_rsc_info(rsc, "Probe found %s active on %s at %s",
                             rsc->id, node->details->uname,
                             last_change_str(xml_op));
             }
             break;
 
         case PCMK_OCF_NOT_RUNNING:
             if (is_probe || target_rc == rc || is_not_set(rsc->flags, pe_rsc_managed)) {
                 result = PCMK_LRM_OP_DONE;
                 rsc->role = RSC_ROLE_STOPPED;
 
                 /* clear any previous failure actions */
                 *on_fail = action_fail_ignore;
                 rsc->next_role = RSC_ROLE_UNKNOWN;
             }
             break;
 
         case PCMK_OCF_RUNNING_MASTER:
             if (is_probe && (rc != target_rc)) {
                 result = PCMK_LRM_OP_DONE;
                 pe_rsc_info(rsc,
                             "Probe found %s active and promoted on %s at %s",
                             rsc->id, node->details->uname,
                             last_change_str(xml_op));
             }
             rsc->role = RSC_ROLE_MASTER;
             break;
 
         case PCMK_OCF_DEGRADED_MASTER:
         case PCMK_OCF_FAILED_MASTER:
             rsc->role = RSC_ROLE_MASTER;
             result = PCMK_LRM_OP_ERROR;
             break;
 
         case PCMK_OCF_NOT_CONFIGURED:
             result = PCMK_LRM_OP_ERROR_FATAL;
             break;
 
         case PCMK_OCF_UNIMPLEMENT_FEATURE:
             if (interval_ms > 0) {
                 result = PCMK_LRM_OP_NOTSUPPORTED;
                 break;
             }
             // fall through
         case PCMK_OCF_NOT_INSTALLED:
         case PCMK_OCF_INVALID_PARAM:
         case PCMK_OCF_INSUFFICIENT_PRIV:
             if (!pe_can_fence(data_set, node)
                 && !strcmp(task, CRMD_ACTION_STOP)) {
                 /* If a stop fails and we can't fence, there's nothing else we can do */
                 pe_proc_err("No further recovery can be attempted for %s "
                             "because %s on %s failed (%s%s%s) at %s "
                             CRM_XS " rc=%d id=%s", rsc->id, task,
                             node->details->uname, services_ocf_exitcode_str(rc),
                             (*exit_reason? ": " : ""), exit_reason,
                             last_change_str(xml_op), rc, ID(xml_op));
                 clear_bit(rsc->flags, pe_rsc_managed);
                 set_bit(rsc->flags, pe_rsc_block);
             }
             result = PCMK_LRM_OP_ERROR_HARD;
             break;
 
         default:
             if (result == PCMK_LRM_OP_DONE) {
                 crm_info("Treating unknown exit status %d from %s of %s "
                          "on %s at %s as failure",
                          rc, task, rsc->id, node->details->uname,
                          last_change_str(xml_op));
                 result = PCMK_LRM_OP_ERROR;
             }
             break;
     }
     return result;
 }
 
 // return TRUE if start or monitor last failure but parameters changed
 static bool
 should_clear_for_param_change(xmlNode *xml_op, const char *task,
                               pe_resource_t *rsc, pe_node_t *node,
                               pe_working_set_t *data_set)
 {
     if (!strcmp(task, "start") || !strcmp(task, "monitor")) {
 
         if (pe__bundle_needs_remote_name(rsc)) {
             /* We haven't allocated resources yet, so we can't reliably
              * substitute addr parameters for the REMOTE_CONTAINER_HACK.
              * When that's needed, defer the check until later.
              */
             pe__add_param_check(xml_op, rsc, node, pe_check_last_failure,
                                 data_set);
 
         } else {
             op_digest_cache_t *digest_data = NULL;
 
             digest_data = rsc_action_digest_cmp(rsc, xml_op, node, data_set);
             switch (digest_data->rc) {
                 case RSC_DIGEST_UNKNOWN:
                     crm_trace("Resource %s history entry %s on %s"
                               " has no digest to compare",
                               rsc->id, get_op_key(xml_op), node->details->id);
                     break;
                 case RSC_DIGEST_MATCH:
                     break;
                 default:
                     return TRUE;
             }
         }
     }
     return FALSE;
 }
 
 // Order action after fencing of remote node, given connection rsc
 static void
 order_after_remote_fencing(pe_action_t *action, pe_resource_t *remote_conn,
                            pe_working_set_t *data_set)
 {
     pe_node_t *remote_node = pe_find_node(data_set->nodes, remote_conn->id);
 
     if (remote_node) {
         pe_action_t *fence = pe_fence_op(remote_node, NULL, TRUE, NULL,
                                          FALSE, data_set);
 
         order_actions(fence, action, pe_order_implies_then);
     }
 }
 
 static bool
 should_ignore_failure_timeout(pe_resource_t *rsc, xmlNode *xml_op,
                               const char *task, guint interval_ms,
                               bool is_last_failure, pe_working_set_t *data_set)
 {
     /* Clearing failures of recurring monitors has special concerns. The
      * executor reports only changes in the monitor result, so if the
      * monitor is still active and still getting the same failure result,
      * that will go undetected after the failure is cleared.
      *
      * Also, the operation history will have the time when the recurring
      * monitor result changed to the given code, not the time when the
      * result last happened.
      *
      * @TODO We probably should clear such failures only when the failure
      * timeout has passed since the last occurrence of the failed result.
      * However we don't record that information. We could maybe approximate
      * that by clearing only if there is a more recent successful monitor or
      * stop result, but we don't even have that information at this point
      * since we are still unpacking the resource's operation history.
      *
      * This is especially important for remote connection resources with a
      * reconnect interval, so in that case, we skip clearing failures
      * if the remote node hasn't been fenced.
      */
     if (rsc->remote_reconnect_ms
         && is_set(data_set->flags, pe_flag_stonith_enabled)
         && (interval_ms != 0) && safe_str_eq(task, CRMD_ACTION_STATUS)) {
 
         pe_node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);
 
         if (remote_node && !remote_node->details->remote_was_fenced) {
             if (is_last_failure) {
                 crm_info("Waiting to clear monitor failure for remote node %s"
                          " until fencing has occurred", rsc->id);
             }
             return TRUE;
         }
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Check operation age and schedule failure clearing when appropriate
  *
  * This function has two distinct purposes. The first is to check whether an
  * operation history entry is expired (i.e. the resource has a failure timeout,
  * the entry is older than the timeout, and the resource either has no fail
  * count or its fail count is entirely older than the timeout). The second is to
  * schedule fail count clearing when appropriate (i.e. the operation is expired
  * and either the resource has an expired fail count or the operation is a
  * last_failure for a remote connection resource with a reconnect interval,
  * or the operation is a last_failure for a start or monitor operation and the
  * resource's parameters have changed since the operation).
  *
  * \param[in] rsc       Resource that operation happened to
  * \param[in] node      Node that operation happened on
  * \param[in] rc        Actual result of operation
  * \param[in] xml_op    Operation history entry XML
  * \param[in] data_set  Current working set
  *
  * \return TRUE if operation history entry is expired, FALSE otherwise
  */
 static bool
 check_operation_expiry(pe_resource_t *rsc, pe_node_t *node, int rc,
                        xmlNode *xml_op, pe_working_set_t *data_set)
 {
     bool expired = FALSE;
     bool is_last_failure = pcmk__ends_with(ID(xml_op), "_last_failure_0");
     time_t last_run = 0;
     guint interval_ms = 0;
     int unexpired_fail_count = 0;
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     const char *clear_reason = NULL;
 
     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
 
     if ((rsc->failure_timeout > 0)
         && (crm_element_value_epoch(xml_op, XML_RSC_OP_LAST_CHANGE,
                                     &last_run) == 0)) {
 
         // Resource has a failure-timeout, and history entry has a timestamp
 
         time_t now = get_effective_time(data_set);
         time_t last_failure = 0;
 
         // Is this particular operation history older than the failure timeout?
         if ((now >= (last_run + rsc->failure_timeout))
             && !should_ignore_failure_timeout(rsc, xml_op, task, interval_ms,
                                               is_last_failure, data_set)) {
             expired = TRUE;
         }
 
         // Does the resource as a whole have an unexpired fail count?
         unexpired_fail_count = pe_get_failcount(node, rsc, &last_failure,
                                                 pe_fc_effective, xml_op,
                                                 data_set);
 
         // Update scheduler recheck time according to *last* failure
         crm_trace("%s@%lld is %sexpired @%lld with unexpired_failures=%d timeout=%ds"
                   " last-failure@%lld",
                   ID(xml_op), (long long) last_run, (expired? "" : "not "),
                   (long long) now, unexpired_fail_count, rsc->failure_timeout,
                   (long long) last_failure);
         last_failure += rsc->failure_timeout + 1;
         if (unexpired_fail_count && (now < last_failure)) {
             pe__update_recheck_time(last_failure, data_set);
         }
     }
 
     if (expired) {
         if (pe_get_failcount(node, rsc, NULL, pe_fc_default, xml_op, data_set)) {
 
             // There is a fail count ignoring timeout
 
             if (unexpired_fail_count == 0) {
                 // There is no fail count considering timeout
                 clear_reason = "it expired";
 
             } else {
                 /* This operation is old, but there is an unexpired fail count.
                  * In a properly functioning cluster, this should only be
                  * possible if this operation is not a failure (otherwise the
                  * fail count should be expired too), so this is really just a
                  * failsafe.
                  */
                 expired = FALSE;
             }
 
         } else if (is_last_failure && rsc->remote_reconnect_ms) {
             /* Clear any expired last failure when reconnect interval is set,
              * even if there is no fail count.
              */
             clear_reason = "reconnect interval is set";
         }
     }
 
     if (!expired && is_last_failure
         && should_clear_for_param_change(xml_op, task, rsc, node, data_set)) {
         clear_reason = "resource parameters have changed";
     }
 
     if (clear_reason != NULL) {
         // Schedule clearing of the fail count
         pe_action_t *clear_op = pe__clear_failcount(rsc, node, clear_reason,
                                                     data_set);
 
         if (is_set(data_set->flags, pe_flag_stonith_enabled)
             && rsc->remote_reconnect_ms) {
             /* If we're clearing a remote connection due to a reconnect
              * interval, we want to wait until any scheduled fencing
              * completes.
              *
              * We could limit this to remote_node->details->unclean, but at
              * this point, that's always true (it won't be reliable until
              * after unpack_node_loop() is done).
              */
             crm_info("Clearing %s failure will wait until any scheduled "
                      "fencing of %s completes", task, rsc->id);
             order_after_remote_fencing(clear_op, rsc, data_set);
         }
     }
 
     if (expired && (interval_ms == 0) && safe_str_eq(task, CRMD_ACTION_STATUS)) {
         switch(rc) {
             case PCMK_OCF_OK:
             case PCMK_OCF_NOT_RUNNING:
             case PCMK_OCF_RUNNING_MASTER:
             case PCMK_OCF_DEGRADED:
             case PCMK_OCF_DEGRADED_MASTER:
                 // Don't expire probes that return these values
                 expired = FALSE;
                 break;
         }
     }
 
     return expired;
 }
 
 int pe__target_rc_from_xml(xmlNode *xml_op)
 {
     int target_rc = 0;
     const char *key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);
 
     if (key == NULL) {
         return -1;
     }
     decode_transition_key(key, NULL, NULL, NULL, &target_rc);
     return target_rc;
 }
 
 static enum action_fail_response
 get_action_on_fail(pe_resource_t *rsc, const char *key, const char *task, pe_working_set_t * data_set) 
 {
     int result = action_fail_recover;
     pe_action_t *action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);
 
     result = action->on_fail;
     pe_free_action(action);
 
     return result;
 }
 
 static void
 update_resource_state(pe_resource_t * rsc, pe_node_t * node, xmlNode * xml_op, const char * task, int rc,
                       xmlNode * last_failure, enum action_fail_response * on_fail, pe_working_set_t * data_set)
 {
     gboolean clear_past_failure = FALSE;
 
     CRM_ASSERT(rsc);
     CRM_ASSERT(xml_op);
 
     if (rc == PCMK_OCF_NOT_RUNNING) {
         clear_past_failure = TRUE;
 
     } else if (rc == PCMK_OCF_NOT_INSTALLED) {
         rsc->role = RSC_ROLE_STOPPED;
 
     } else if (safe_str_eq(task, CRMD_ACTION_STATUS)) {
         if (last_failure) {
             const char *op_key = get_op_key(xml_op);
             const char *last_failure_key = get_op_key(last_failure);
 
             if (safe_str_eq(op_key, last_failure_key)) {
                 clear_past_failure = TRUE;
             }
         }
 
         if (rsc->role < RSC_ROLE_STARTED) {
             set_active(rsc);
         }
 
     } else if (safe_str_eq(task, CRMD_ACTION_START)) {
         rsc->role = RSC_ROLE_STARTED;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_STOP)) {
         rsc->role = RSC_ROLE_STOPPED;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
         rsc->role = RSC_ROLE_MASTER;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
         /* Demote from Master does not clear an error */
         rsc->role = RSC_ROLE_SLAVE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
         rsc->role = RSC_ROLE_STARTED;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_MIGRATE)) {
         unpack_migrate_to_success(rsc, node, xml_op, data_set);
 
     } else if (rsc->role < RSC_ROLE_STARTED) {
         pe_rsc_trace(rsc, "%s active on %s", rsc->id, node->details->uname);
         set_active(rsc);
     }
 
     /* clear any previous failure actions */
     if (clear_past_failure) {
         switch (*on_fail) {
             case action_fail_stop:
             case action_fail_fence:
             case action_fail_migrate:
             case action_fail_standby:
                 pe_rsc_trace(rsc, "%s.%s is not cleared by a completed stop",
                              rsc->id, fail2text(*on_fail));
                 break;
 
             case action_fail_block:
             case action_fail_ignore:
             case action_fail_recover:
             case action_fail_restart_container:
                 *on_fail = action_fail_ignore;
                 rsc->next_role = RSC_ROLE_UNKNOWN;
                 break;
             case action_fail_reset_remote:
                 if (rsc->remote_reconnect_ms == 0) {
                     /* With no reconnect interval, the connection is allowed to
                      * start again after the remote node is fenced and
                      * completely stopped. (With a reconnect interval, we wait
                      * for the failure to be cleared entirely before attempting
                      * to reconnect.)
                      */
                     *on_fail = action_fail_ignore;
                     rsc->next_role = RSC_ROLE_UNKNOWN;
                 }
                 break;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Remap informational monitor results to usual values
  *
  * Certain OCF result codes are for providing extended information to the
  * user about services that aren't yet failed but not entirely healthy either.
  * These must be treated as the "normal" result by pacemaker.
  *
  * \param[in] rc        Actual result of a monitor action
  * \param[in] xml_op    Operation history XML
  * \param[in] node      Node that operation happened on
  * \param[in] rsc       Resource that operation happened to
  * \param[in] data_set  Cluster working set
  *
  * \return Result code that pacemaker should use
  *
  * \note If the result is remapped, and the node is not shutting down or failed,
  *       the operation will be recorded in the data set's list of failed
  *       operations, to highlight it for the user.
  */
 static int
 remap_monitor_rc(int rc, xmlNode *xml_op, const pe_node_t *node,
                  const pe_resource_t *rsc, pe_working_set_t *data_set)
 {
     int remapped_rc = rc;
 
     switch (rc) {
         case PCMK_OCF_DEGRADED:
             remapped_rc = PCMK_OCF_OK;
             break;
 
         case PCMK_OCF_DEGRADED_MASTER:
             remapped_rc = PCMK_OCF_RUNNING_MASTER;
             break;
 
         default:
             break;
     }
 
     if (rc != remapped_rc) {
         crm_trace("Remapping monitor result %d to %d", rc, remapped_rc);
         if (!node->details->shutdown || node->details->online) {
             record_failed_op(xml_op, node, rsc, data_set);
         }
     }
     return remapped_rc;
 }
 
 static void
 unpack_rsc_op(pe_resource_t *rsc, pe_node_t *node, xmlNode *xml_op,
               xmlNode **last_failure, enum action_fail_response *on_fail,
               pe_working_set_t *data_set)
 {
     int rc = 0;
     int task_id = 0;
     int target_rc = 0;
     int status = PCMK_LRM_OP_UNKNOWN;
     guint interval_ms = 0;
     const char *task = NULL;
     const char *task_key = NULL;
     const char *exit_reason = NULL;
     bool expired = FALSE;
     pe_resource_t *parent = rsc;
     enum action_fail_response failure_strategy = action_fail_recover;
 
     CRM_CHECK(rsc && node && xml_op, return);
 
     target_rc = pe__target_rc_from_xml(xml_op);
     task_key = get_op_key(xml_op);
     task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     exit_reason = crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON);
     if (exit_reason == NULL) {
         exit_reason = "";
     }
 
     crm_element_value_int(xml_op, XML_LRM_ATTR_RC, &rc);
     crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &task_id);
     crm_element_value_int(xml_op, XML_LRM_ATTR_OPSTATUS, &status);
     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
 
     CRM_CHECK(task != NULL, return);
     CRM_CHECK(status <= PCMK_LRM_OP_INVALID, return);
     CRM_CHECK(status >= PCMK_LRM_OP_PENDING, return);
 
     if (!strcmp(task, CRMD_ACTION_NOTIFY) ||
         !strcmp(task, CRMD_ACTION_METADATA)) {
         /* safe to ignore these */
         return;
     }
 
     if (is_not_set(rsc->flags, pe_rsc_unique)) {
         parent = uber_parent(rsc);
     }
 
     pe_rsc_trace(rsc, "Unpacking task %s/%s (call_id=%d, status=%d, rc=%d) on %s (role=%s)",
                  task_key, task, task_id, status, rc, node->details->uname, role2text(rsc->role));
 
     if (node->details->unclean) {
         pe_rsc_trace(rsc, "Node %s (where %s is running) is unclean."
                      " Further action depends on the value of the stop's on-fail attribute",
                      node->details->uname, rsc->id);
     }
 
     /* It should be possible to call remap_monitor_rc() first then call
      * check_operation_expiry() only if rc != target_rc, because there should
      * never be a fail count without at least one unexpected result in the
      * resource history. That would be more efficient by avoiding having to call
      * check_operation_expiry() for expected results.
      *
      * However, we do have such configurations in the scheduler regression
      * tests, even if it shouldn't be possible with the current code. It's
      * probably a good idea anyway, but that would require updating the test
      * inputs to something currently possible.
      */
 
     if ((status != PCMK_LRM_OP_NOT_INSTALLED)
         && check_operation_expiry(rsc, node, rc, xml_op, data_set)) {
         expired = TRUE;
     }
 
     if (!strcmp(task, CRMD_ACTION_STATUS)) {
         rc = remap_monitor_rc(rc, xml_op, node, rsc, data_set);
     }
 
     if (expired && (rc != target_rc)) {
         const char *magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);
 
         if (interval_ms == 0) {
             crm_notice("Ignoring expired %s failure on %s "
                        CRM_XS " actual=%d expected=%d magic=%s",
                        task_key, node->details->uname, rc, target_rc, magic);
             goto done;
 
         } else if(node->details->online && node->details->unclean == FALSE) {
             /* Reschedule the recurring monitor. CancelXmlOp() won't work at
              * this stage, so as a hacky workaround, forcibly change the restart
              * digest so check_action_definition() does what we want later.
              *
              * @TODO We should skip this if there is a newer successful monitor.
              *       Also, this causes rescheduling only if the history entry
              *       has an op-digest (which the expire-non-blocked-failure
              *       scheduler regression test doesn't, but that may not be a
              *       realistic scenario in production).
              */
             crm_notice("Rescheduling %s after failure expired on %s "
                        CRM_XS " actual=%d expected=%d magic=%s",
                        task_key, node->details->uname, rc, target_rc, magic);
             crm_xml_add(xml_op, XML_LRM_ATTR_RESTART_DIGEST, "calculated-failure-timeout");
             goto done;
         }
     }
 
     /* If the executor reported an operation status of anything but done or
      * error, consider that final. But for done or error, we know better whether
      * it should be treated as a failure or not, because we know the expected
      * result.
      */
     if(status == PCMK_LRM_OP_DONE || status == PCMK_LRM_OP_ERROR) {
         status = determine_op_status(rsc, rc, target_rc, node, xml_op, on_fail, data_set);
         pe_rsc_trace(rsc, "Remapped %s status to %d", task_key, status);
     }
 
     switch (status) {
         case PCMK_LRM_OP_CANCELLED:
             // Should never happen
             pe_err("Resource history contains cancellation '%s' "
                    "(%s of %s on %s at %s)",
                    ID(xml_op), task, rsc->id, node->details->uname,
                    last_change_str(xml_op));
             break;
 
         case PCMK_LRM_OP_PENDING:
             if (!strcmp(task, CRMD_ACTION_START)) {
                 set_bit(rsc->flags, pe_rsc_start_pending);
                 set_active(rsc);
 
             } else if (!strcmp(task, CRMD_ACTION_PROMOTE)) {
                 rsc->role = RSC_ROLE_MASTER;
 
             } else if (!strcmp(task, CRMD_ACTION_MIGRATE) && node->details->unclean) {
                 /* If a pending migrate_to action is out on a unclean node,
                  * we have to force the stop action on the target. */
                 const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
                 pe_node_t *target = pe_find_node(data_set->nodes, migrate_target);
                 if (target) {
                     stop_action(rsc, target, FALSE);
                 }
             }
 
             if (rsc->pending_task == NULL) {
                 if ((interval_ms != 0) || strcmp(task, CRMD_ACTION_STATUS)) {
                     rsc->pending_task = strdup(task);
                     rsc->pending_node = node;
                 } else {
                     /* Pending probes are not printed, even if pending
                      * operations are requested. If someone ever requests that
                      * behavior, enable the below and the corresponding part of
                      * native.c:native_pending_task().
                      */
 #if 0
                     rsc->pending_task = strdup("probe");
                     rsc->pending_node = node;
 #endif
                 }
             }
             break;
 
         case PCMK_LRM_OP_DONE:
             pe_rsc_trace(rsc, "%s of %s on %s completed at %s " CRM_XS " id=%s",
                          task, rsc->id, node->details->uname,
                          last_change_str(xml_op), ID(xml_op));
             update_resource_state(rsc, node, xml_op, task, rc, *last_failure, on_fail, data_set);
             break;
 
         case PCMK_LRM_OP_NOT_INSTALLED:
             failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
             if (failure_strategy == action_fail_ignore) {
                 crm_warn("Cannot ignore failed %s of %s on %s: "
                          "Resource agent doesn't exist "
                          CRM_XS " status=%d rc=%d id=%s",
                          task, rsc->id, node->details->uname, status, rc,
                          ID(xml_op));
                 /* Also for printing it as "FAILED" by marking it as pe_rsc_failed later */
                 *on_fail = action_fail_migrate;
             }
             resource_location(parent, node, -INFINITY, "hard-error", data_set);
             unpack_rsc_op_failure(rsc, node, rc, xml_op, last_failure, on_fail, data_set);
             break;
 
         case PCMK_LRM_OP_NOT_CONNECTED:
             if (pe__is_guest_or_remote_node(node)
                 && is_set(node->details->remote_rsc->flags, pe_rsc_managed)) {
                 /* We should never get into a situation where a managed remote
                  * connection resource is considered OK but a resource action
                  * behind the connection gets a "not connected" status. But as a
                  * fail-safe in case a bug or unusual circumstances do lead to
                  * that, ensure the remote connection is considered failed.
                  */
                 set_bit(node->details->remote_rsc->flags, pe_rsc_failed);
             }
 
             // fall through
 
         case PCMK_LRM_OP_ERROR:
         case PCMK_LRM_OP_ERROR_HARD:
         case PCMK_LRM_OP_ERROR_FATAL:
         case PCMK_LRM_OP_TIMEOUT:
         case PCMK_LRM_OP_NOTSUPPORTED:
         case PCMK_LRM_OP_INVALID:
 
             failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
             if ((failure_strategy == action_fail_ignore)
                 || (failure_strategy == action_fail_restart_container
                     && !strcmp(task, CRMD_ACTION_STOP))) {
 
                 crm_warn("Pretending failed %s (%s%s%s) of %s on %s at %s "
                          "succeeded " CRM_XS " rc=%d id=%s",
                          task, services_ocf_exitcode_str(rc),
                          (*exit_reason? ": " : ""), exit_reason, rsc->id,
                          node->details->uname, last_change_str(xml_op), rc,
                          ID(xml_op));
 
                 update_resource_state(rsc, node, xml_op, task, target_rc, *last_failure, on_fail, data_set);
                 crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
                 set_bit(rsc->flags, pe_rsc_failure_ignored);
 
                 record_failed_op(xml_op, node, rsc, data_set);
 
                 if (failure_strategy == action_fail_restart_container && *on_fail <= action_fail_recover) {
                     *on_fail = failure_strategy;
                 }
 
             } else {
                 unpack_rsc_op_failure(rsc, node, rc, xml_op, last_failure, on_fail, data_set);
 
                 if(status == PCMK_LRM_OP_ERROR_HARD) {
                     do_crm_log(rc != PCMK_OCF_NOT_INSTALLED?LOG_ERR:LOG_NOTICE,
                                "Preventing %s from restarting on %s because "
                                "of hard failure (%s%s%s)" CRM_XS " rc=%d id=%s",
                                parent->id, node->details->uname,
                                services_ocf_exitcode_str(rc),
                                (*exit_reason? ": " : ""), exit_reason,
                                rc, ID(xml_op));
                     resource_location(parent, node, -INFINITY, "hard-error", data_set);
 
                 } else if(status == PCMK_LRM_OP_ERROR_FATAL) {
                     crm_err("Preventing %s from restarting anywhere because "
                             "of fatal failure (%s%s%s) " CRM_XS " rc=%d id=%s",
                             parent->id, services_ocf_exitcode_str(rc),
                             (*exit_reason? ": " : ""), exit_reason,
                             rc, ID(xml_op));
                     resource_location(parent, NULL, -INFINITY, "fatal-error", data_set);
                 }
             }
             break;
     }
 
   done:
     pe_rsc_trace(rsc, "Resource %s after %s: role=%s, next=%s",
                  rsc->id, task, role2text(rsc->role),
                  role2text(rsc->next_role));
 }
 
 static void
 add_node_attrs(xmlNode *xml_obj, pe_node_t *node, bool overwrite,
                pe_working_set_t *data_set)
 {
     const char *cluster_name = NULL;
 
     g_hash_table_insert(node->details->attrs,
                         strdup(CRM_ATTR_UNAME), strdup(node->details->uname));
 
     g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_ID),
                         strdup(node->details->id));
     if (safe_str_eq(node->details->id, data_set->dc_uuid)) {
         data_set->dc_node = node;
         node->details->is_dc = TRUE;
         g_hash_table_insert(node->details->attrs,
                             strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_TRUE));
     } else {
         g_hash_table_insert(node->details->attrs,
                             strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_FALSE));
     }
 
     cluster_name = g_hash_table_lookup(data_set->config_hash, "cluster-name");
     if (cluster_name) {
         g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_CLUSTER_NAME),
                             strdup(cluster_name));
     }
 
     pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_ATTR_SETS, NULL,
                                node->details->attrs, NULL, overwrite, data_set);
 
     if (pe_node_attribute_raw(node, CRM_ATTR_SITE_NAME) == NULL) {
         const char *site_name = pe_node_attribute_raw(node, "site-name");
 
         if (site_name) {
             g_hash_table_insert(node->details->attrs,
                                 strdup(CRM_ATTR_SITE_NAME),
                                 strdup(site_name));
 
         } else if (cluster_name) {
             /* Default to cluster-name if unset */
             g_hash_table_insert(node->details->attrs,
                                 strdup(CRM_ATTR_SITE_NAME),
                                 strdup(cluster_name));
         }
     }
 }
 
 static GListPtr
 extract_operations(const char *node, const char *rsc, xmlNode * rsc_entry, gboolean active_filter)
 {
     int counter = -1;
     int stop_index = -1;
     int start_index = -1;
 
     xmlNode *rsc_op = NULL;
 
     GListPtr gIter = NULL;
     GListPtr op_list = NULL;
     GListPtr sorted_op_list = NULL;
 
     /* extract operations */
     op_list = NULL;
     sorted_op_list = NULL;
 
     for (rsc_op = __xml_first_child_element(rsc_entry);
          rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             crm_xml_add(rsc_op, "resource", rsc);
             crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
             op_list = g_list_prepend(op_list, rsc_op);
         }
     }
 
     if (op_list == NULL) {
         /* if there are no operations, there is nothing to do */
         return NULL;
     }
 
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 
     /* create active recurring operations as optional */
     if (active_filter == FALSE) {
         return sorted_op_list;
     }
 
     op_list = NULL;
 
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         counter++;
 
         if (start_index < stop_index) {
             crm_trace("Skipping %s: not active", ID(rsc_entry));
             break;
 
         } else if (counter < start_index) {
             crm_trace("Skipping %s: old", ID(rsc_op));
             continue;
         }
         op_list = g_list_append(op_list, rsc_op);
     }
 
     g_list_free(sorted_op_list);
     return op_list;
 }
 
 GListPtr
 find_operations(const char *rsc, const char *node, gboolean active_filter,
                 pe_working_set_t * data_set)
 {
     GListPtr output = NULL;
     GListPtr intermediate = NULL;
 
     xmlNode *tmp = NULL;
     xmlNode *status = find_xml_node(data_set->input, XML_CIB_TAG_STATUS, TRUE);
 
     pe_node_t *this_node = NULL;
 
     xmlNode *node_state = NULL;
 
     for (node_state = __xml_first_child_element(status); node_state != NULL;
          node_state = __xml_next_element(node_state)) {
 
         if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
             const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);
 
             if (node != NULL && safe_str_neq(uname, node)) {
                 continue;
             }
 
             this_node = pe_find_node(data_set->nodes, uname);
             if(this_node == NULL) {
                 CRM_LOG_ASSERT(this_node != NULL);
                 continue;
 
             } else if (pe__is_guest_or_remote_node(this_node)) {
                 determine_remote_online_status(data_set, this_node);
 
             } else {
                 determine_online_status(node_state, this_node, data_set);
             }
 
             if (this_node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) {
                 /* offline nodes run no resources...
                  * unless stonith is enabled in which case we need to
                  *   make sure rsc start events happen after the stonith
                  */
                 xmlNode *lrm_rsc = NULL;
 
                 tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
                 tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);
 
                 for (lrm_rsc = __xml_first_child_element(tmp); lrm_rsc != NULL;
                      lrm_rsc = __xml_next_element(lrm_rsc)) {
                     if (crm_str_eq((const char *)lrm_rsc->name, XML_LRM_TAG_RESOURCE, TRUE)) {
 
                         const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID);
 
                         if (rsc != NULL && safe_str_neq(rsc_id, rsc)) {
                             continue;
                         }
 
                         intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
                         output = g_list_concat(output, intermediate);
                     }
                 }
             }
         }
     }
 
     return output;
 }
diff --git a/lib/pengine/unpack.h b/lib/pengine/unpack.h
deleted file mode 100644
index 6f7226a05e..0000000000
--- a/lib/pengine/unpack.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright 2004-2020 the Pacemaker project contributors
- *
- * The version control history for this file may have further details.
- *
- * This source code is licensed under the GNU Lesser General Public License
- * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
- */
-
-#ifndef PENGINE_UNPACK__H
-#  define PENGINE_UNPACK__H
-
-extern gboolean unpack_remote_nodes(xmlNode * xml_resources, pe_working_set_t * data_set);
-
-extern gboolean unpack_resources(xmlNode * xml_resources, pe_working_set_t * data_set);
-
-extern gboolean unpack_config(xmlNode * config, pe_working_set_t * data_set);
-
-extern gboolean unpack_nodes(xmlNode * xml_nodes, pe_working_set_t * data_set);
-
-extern gboolean unpack_tags(xmlNode * xml_tags, pe_working_set_t * data_set);
-
-extern gboolean unpack_status(xmlNode * status, pe_working_set_t * data_set);
-
-extern gint sort_op_by_callid(gconstpointer a, gconstpointer b);
-
-extern gboolean unpack_lrm_resources(pe_node_t * node, xmlNode * lrm_state,
-                                     pe_working_set_t * data_set);
-
-// Some warnings we don't want to print every transition
-
-enum pe_warn_once_e {
-    pe_wo_blind         = 0x0001,
-    pe_wo_restart_type  = 0x0002,
-    pe_wo_role_after    = 0x0004,
-    pe_wo_poweroff      = 0x0008,
-    pe_wo_require_all   = 0x0010,
-    pe_wo_order_score   = 0x0020,
-    pe_wo_neg_threshold = 0x0040,
-};
-
-extern uint32_t pe_wo;
-
-#define pe_warn_once(pe_wo_bit, fmt...) do {    \
-        if (is_not_set(pe_wo, pe_wo_bit)) {     \
-            if (pe_wo_bit == pe_wo_blind) {     \
-                crm_warn(fmt);                  \
-            } else {                            \
-                pe_warn(fmt);                   \
-            }                                   \
-            set_bit(pe_wo, pe_wo_bit);          \
-        }                                       \
-    } while (0);
-
-#endif
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
index dbc73b8d4f..d26a1dfe89 100644
--- a/lib/pengine/utils.c
+++ b/lib/pengine/utils.c
@@ -1,2766 +1,2764 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/util.h>
 
 #include <ctype.h>
 #include <glib.h>
 #include <stdbool.h>
 
 #include <crm/pengine/rules.h>
 #include <crm/pengine/internal.h>
 
-#include <unpack.h>
-
 extern xmlNode *get_object_root(const char *object_type, xmlNode * the_root);
 void print_str_str(gpointer key, gpointer value, gpointer user_data);
 gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data);
 void unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container,
                       pe_working_set_t * data_set);
 static xmlNode *find_rsc_op_entry_helper(pe_resource_t * rsc, const char *key,
                                          gboolean include_disabled);
 
 #if ENABLE_VERSIONED_ATTRS
 pe_rsc_action_details_t *
 pe_rsc_action_details(pe_action_t *action)
 {
     pe_rsc_action_details_t *details;
 
     CRM_CHECK(action != NULL, return NULL);
 
     if (action->action_details == NULL) {
         action->action_details = calloc(1, sizeof(pe_rsc_action_details_t));
         CRM_CHECK(action->action_details != NULL, return NULL);
     }
 
     details = (pe_rsc_action_details_t *) action->action_details;
     if (details->versioned_parameters == NULL) {
         details->versioned_parameters = create_xml_node(NULL,
                                                         XML_TAG_OP_VER_ATTRS);
     }
     if (details->versioned_meta == NULL) {
         details->versioned_meta = create_xml_node(NULL, XML_TAG_OP_VER_META);
     }
     return details;
 }
 
 static void
 pe_free_rsc_action_details(pe_action_t *action)
 {
     pe_rsc_action_details_t *details;
 
     if ((action == NULL) || (action->action_details == NULL)) {
         return;
     }
 
     details = (pe_rsc_action_details_t *) action->action_details;
 
     if (details->versioned_parameters) {
         free_xml(details->versioned_parameters);
     }
     if (details->versioned_meta) {
         free_xml(details->versioned_meta);
     }
 
     action->action_details = NULL;
 }
 #endif
 
 /*!
  * \internal
  * \brief Check whether we can fence a particular node
  *
  * \param[in] data_set  Working set for cluster
  * \param[in] node      Name of node to check
  *
  * \return true if node can be fenced, false otherwise
  */
 bool
 pe_can_fence(pe_working_set_t *data_set, pe_node_t *node)
 {
     if (pe__is_guest_node(node)) {
         /* Guest nodes are fenced by stopping their container resource. We can
          * do that if the container's host is either online or fenceable.
          */
         pe_resource_t *rsc = node->details->remote_rsc->container;
 
         for (GList *n = rsc->running_on; n != NULL; n = n->next) {
             pe_node_t *container_node = n->data;
 
             if (!container_node->details->online
                 && !pe_can_fence(data_set, container_node)) {
                 return false;
             }
         }
         return true;
 
     } else if(is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
         return false; /* Turned off */
 
     } else if (is_not_set(data_set->flags, pe_flag_have_stonith_resource)) {
         return false; /* No devices */
 
     } else if (is_set(data_set->flags, pe_flag_have_quorum)) {
         return true;
 
     } else if (data_set->no_quorum_policy == no_quorum_ignore) {
         return true;
 
     } else if(node == NULL) {
         return false;
 
     } else if(node->details->online) {
         crm_notice("We can fence %s without quorum because they're in our membership", node->details->uname);
         return true;
     }
 
     crm_trace("Cannot fence %s", node->details->uname);
     return false;
 }
 
 pe_node_t *
 node_copy(const pe_node_t *this_node)
 {
     pe_node_t *new_node = NULL;
 
     CRM_CHECK(this_node != NULL, return NULL);
 
     new_node = calloc(1, sizeof(pe_node_t));
     CRM_ASSERT(new_node != NULL);
 
     crm_trace("Copying %p (%s) to %p", this_node, this_node->details->uname, new_node);
 
     new_node->rsc_discover_mode = this_node->rsc_discover_mode;
     new_node->weight = this_node->weight;
     new_node->fixed = this_node->fixed;
     new_node->details = this_node->details;
 
     return new_node;
 }
 
 /* any node in list1 or list2 and not in the other gets a score of -INFINITY */
 void
 node_list_exclude(GHashTable * hash, GListPtr list, gboolean merge_scores)
 {
     GHashTable *result = hash;
     pe_node_t *other_node = NULL;
     GListPtr gIter = list;
 
     GHashTableIter iter;
     pe_node_t *node = NULL;
 
     g_hash_table_iter_init(&iter, hash);
     while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
 
         other_node = pe_find_node_id(list, node->details->id);
         if (other_node == NULL) {
             node->weight = -INFINITY;
         } else if (merge_scores) {
             node->weight = merge_weights(node->weight, other_node->weight);
         }
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         other_node = pe_hash_table_lookup(result, node->details->id);
 
         if (other_node == NULL) {
             pe_node_t *new_node = node_copy(node);
 
             new_node->weight = -INFINITY;
             g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
         }
     }
 }
 
 GHashTable *
 node_hash_from_list(GListPtr list)
 {
     GListPtr gIter = list;
     GHashTable *result = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL,
                                                free);
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
         pe_node_t *n = node_copy(node);
 
         g_hash_table_insert(result, (gpointer) n->details->id, n);
     }
 
     return result;
 }
 
 GListPtr
 node_list_dup(GListPtr list1, gboolean reset, gboolean filter)
 {
     GListPtr result = NULL;
     GListPtr gIter = list1;
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_node_t *new_node = NULL;
         pe_node_t *this_node = (pe_node_t *) gIter->data;
 
         if (filter && this_node->weight < 0) {
             continue;
         }
 
         new_node = node_copy(this_node);
         if (reset) {
             new_node->weight = 0;
         }
         if (new_node != NULL) {
             result = g_list_prepend(result, new_node);
         }
     }
 
     return result;
 }
 
 gint
 sort_node_uname(gconstpointer a, gconstpointer b)
 {
     const char *name_a = ((const pe_node_t *) a)->details->uname;
     const char *name_b = ((const pe_node_t *) b)->details->uname;
 
     while (*name_a && *name_b) {
         if (isdigit(*name_a) && isdigit(*name_b)) {
             // If node names contain a number, sort numerically
 
             char *end_a = NULL;
             char *end_b = NULL;
             long num_a = strtol(name_a, &end_a, 10);
             long num_b = strtol(name_b, &end_b, 10);
 
             // allow ordering e.g. 007 > 7
             size_t len_a = end_a - name_a;
             size_t len_b = end_b - name_b;
 
             if (num_a < num_b) {
                 return -1;
             } else if (num_a > num_b) {
                 return 1;
             } else if (len_a < len_b) {
                 return -1;
             } else if (len_a > len_b) {
                 return 1;
             }
             name_a = end_a;
             name_b = end_b;
         } else {
             // Compare non-digits case-insensitively
             int lower_a = tolower(*name_a);
             int lower_b = tolower(*name_b);
 
             if (lower_a < lower_b) {
                 return -1;
             } else if (lower_a > lower_b) {
                 return 1;
             }
             ++name_a;
             ++name_b;
         }
     }
     if (!*name_a && *name_b) {
         return -1;
     } else if (*name_a && !*name_b) {
         return 1;
     }
     return 0;
 }
 
 /*!
  * \internal
  * \brief Output node weights to stdout
  *
  * \param[in] rsc       Use allowed nodes for this resource
  * \param[in] comment   Text description to prefix lines with
  * \param[in] nodes     If rsc is not specified, use these nodes
  */
 static void
 pe__output_node_weights(pe_resource_t *rsc, const char *comment,
                         GHashTable *nodes)
 {
     char score[128]; // Stack-allocated since this is called frequently
 
     // Sort the nodes so the output is consistent for regression tests
     GList *list = g_list_sort(g_hash_table_get_values(nodes), sort_node_uname);
 
     for (GList *gIter = list; gIter != NULL; gIter = gIter->next) {
         pe_node_t *node = (pe_node_t *) gIter->data;
 
         score2char_stack(node->weight, score, sizeof(score));
         if (rsc) {
             printf("%s: %s allocation score on %s: %s\n",
                    comment, rsc->id, node->details->uname, score);
         } else {
             printf("%s: %s = %s\n", comment, node->details->uname, score);
         }
     }
     g_list_free(list);
 }
 
 /*!
  * \internal
  * \brief Log node weights at trace level
  *
  * \param[in] file      Caller's filename
  * \param[in] function  Caller's function name
  * \param[in] line      Caller's line number
  * \param[in] rsc       Use allowed nodes for this resource
  * \param[in] comment   Text description to prefix lines with
  * \param[in] nodes     If rsc is not specified, use these nodes
  */
 static void
 pe__log_node_weights(const char *file, const char *function, int line,
                      pe_resource_t *rsc, const char *comment, GHashTable *nodes)
 {
     GHashTableIter iter;
     pe_node_t *node = NULL;
     char score[128]; // Stack-allocated since this is called frequently
 
     // Don't waste time if we're not tracing at this point
     pcmk__log_else(LOG_TRACE, return);
 
     g_hash_table_iter_init(&iter, nodes);
     while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
         score2char_stack(node->weight, score, sizeof(score));
         if (rsc) {
             qb_log_from_external_source(function, file,
                                         "%s: %s allocation score on %s: %s",
                                         LOG_TRACE, line, 0,
                                         comment, rsc->id,
                                         node->details->uname, score);
         } else {
             qb_log_from_external_source(function, file, "%s: %s = %s",
                                         LOG_TRACE, line, 0,
                                         comment, node->details->uname,
                                         score);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Log or output node weights
  *
  * \param[in] file      Caller's filename
  * \param[in] function  Caller's function name
  * \param[in] line      Caller's line number
  * \param[in] to_log    Log if true, otherwise output
  * \param[in] rsc       Use allowed nodes for this resource
  * \param[in] comment   Text description to prefix lines with
  * \param[in] nodes     If rsc is not specified, use these nodes
  */
 void
 pe__show_node_weights_as(const char *file, const char *function, int line,
                          bool to_log, pe_resource_t *rsc, const char *comment,
                          GHashTable *nodes)
 {
     if (rsc != NULL) {
         if (is_set(rsc->flags, pe_rsc_orphan)) {
             // Don't show allocation scores for orphans
             return;
         }
         nodes = rsc->allowed_nodes;
     }
     if (nodes == NULL) {
         // Nothing to show
         return;
     }
 
     if (to_log) {
         pe__log_node_weights(file, function, line, rsc, comment, nodes);
     } else {
         pe__output_node_weights(rsc, comment, nodes);
     }
 
     // If this resource has children, repeat recursively for each
     if (rsc && rsc->children) {
         for (GList *gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child = (pe_resource_t *) gIter->data;
 
             pe__show_node_weights_as(file, function, line, to_log, child,
                                      comment, nodes);
         }
     }
 }
 
 static void
 append_dump_text(gpointer key, gpointer value, gpointer user_data)
 {
     char **dump_text = user_data;
     char *new_text = crm_strdup_printf("%s %s=%s",
                                        *dump_text, (char *)key, (char *)value);
 
     free(*dump_text);
     *dump_text = new_text;
 }
 
 void
 dump_node_capacity(int level, const char *comment, pe_node_t * node)
 {
     char *dump_text = crm_strdup_printf("%s: %s capacity:",
                                         comment, node->details->uname);
 
     g_hash_table_foreach(node->details->utilization, append_dump_text, &dump_text);
 
     if (level == LOG_STDOUT) {
         fprintf(stdout, "%s\n", dump_text);
     } else {
         crm_trace("%s", dump_text);
     }
 
     free(dump_text);
 }
 
 void
 dump_rsc_utilization(int level, const char *comment, pe_resource_t * rsc, pe_node_t * node)
 {
     char *dump_text = crm_strdup_printf("%s: %s utilization on %s:",
                                         comment, rsc->id, node->details->uname);
 
     g_hash_table_foreach(rsc->utilization, append_dump_text, &dump_text);
     switch (level) {
         case LOG_STDOUT:
             fprintf(stdout, "%s\n", dump_text);
             break;
         case LOG_NEVER:
             break;
         default:
             crm_trace("%s", dump_text);
     }
     free(dump_text);
 }
 
 gint
 sort_rsc_index(gconstpointer a, gconstpointer b)
 {
     const pe_resource_t *resource1 = (const pe_resource_t *)a;
     const pe_resource_t *resource2 = (const pe_resource_t *)b;
 
     if (a == NULL && b == NULL) {
         return 0;
     }
     if (a == NULL) {
         return 1;
     }
     if (b == NULL) {
         return -1;
     }
 
     if (resource1->sort_index > resource2->sort_index) {
         return -1;
     }
 
     if (resource1->sort_index < resource2->sort_index) {
         return 1;
     }
 
     return 0;
 }
 
 gint
 sort_rsc_priority(gconstpointer a, gconstpointer b)
 {
     const pe_resource_t *resource1 = (const pe_resource_t *)a;
     const pe_resource_t *resource2 = (const pe_resource_t *)b;
 
     if (a == NULL && b == NULL) {
         return 0;
     }
     if (a == NULL) {
         return 1;
     }
     if (b == NULL) {
         return -1;
     }
 
     if (resource1->priority > resource2->priority) {
         return -1;
     }
 
     if (resource1->priority < resource2->priority) {
         return 1;
     }
 
     return 0;
 }
 
 pe_action_t *
 custom_action(pe_resource_t * rsc, char *key, const char *task,
               pe_node_t * on_node, gboolean optional, gboolean save_action,
               pe_working_set_t * data_set)
 {
     pe_action_t *action = NULL;
     GListPtr possible_matches = NULL;
 
     CRM_CHECK(key != NULL, return NULL);
     CRM_CHECK(task != NULL, free(key); return NULL);
 
     if (save_action && rsc != NULL) {
         possible_matches = find_actions(rsc->actions, key, on_node);
     } else if(save_action) {
 #if 0
         action = g_hash_table_lookup(data_set->singletons, key);
 #else
         /* More expensive but takes 'node' into account */
         possible_matches = find_actions(data_set->actions, key, on_node);
 #endif
     }
 
     if(data_set->singletons == NULL) {
         data_set->singletons = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL);
     }
 
     if (possible_matches != NULL) {
         if (pcmk__list_of_multiple(possible_matches)) {
             pe_warn("Action %s for %s on %s exists %d times",
                     task, rsc ? rsc->id : "<NULL>",
                     on_node ? on_node->details->uname : "<NULL>", g_list_length(possible_matches));
         }
 
         action = g_list_nth_data(possible_matches, 0);
         pe_rsc_trace(rsc, "Found existing action %d (%s) for %s (%s) on %s",
                      action->id, action->uuid,
                      (rsc? rsc->id : "no resource"), task,
                      (on_node? on_node->details->uname : "no node"));
         g_list_free(possible_matches);
     }
 
     if (action == NULL) {
         if (save_action) {
             pe_rsc_trace(rsc, "Creating %s action %d: %s for %s (%s) on %s",
                          (optional? "optional" : "mandatory"),
                          data_set->action_id, key,
                          (rsc? rsc->id : "no resource"), task,
                          (on_node? on_node->details->uname : "no node"));
         }
 
         action = calloc(1, sizeof(pe_action_t));
         if (save_action) {
             action->id = data_set->action_id++;
         } else {
             action->id = 0;
         }
         action->rsc = rsc;
         CRM_ASSERT(task != NULL);
         action->task = strdup(task);
         if (on_node) {
             action->node = node_copy(on_node);
         }
         action->uuid = strdup(key);
 
         if (safe_str_eq(task, CRM_OP_LRM_DELETE)) {
             // Resource history deletion for a node can be done on the DC
             pe_set_action_bit(action, pe_action_dc);
         }
 
         pe_set_action_bit(action, pe_action_runnable);
         if (optional) {
             pe_set_action_bit(action, pe_action_optional);
         } else {
             pe_clear_action_bit(action, pe_action_optional);
         }
 
         action->extra = crm_str_table_new();
         action->meta = crm_str_table_new();
 
         if (save_action) {
             data_set->actions = g_list_prepend(data_set->actions, action);
             if(rsc == NULL) {
                 g_hash_table_insert(data_set->singletons, action->uuid, action);
             }
         }
 
         if (rsc != NULL) {
             action->op_entry = find_rsc_op_entry_helper(rsc, key, TRUE);
 
             unpack_operation(action, action->op_entry, rsc->container, data_set);
 
             if (save_action) {
                 rsc->actions = g_list_prepend(rsc->actions, action);
             }
         }
 
         if (save_action) {
             pe_rsc_trace(rsc, "Action %d created", action->id);
         }
     }
 
     if (!optional && is_set(action->flags, pe_action_optional)) {
         pe_rsc_trace(rsc, "Unset optional on action %d", action->id);
         pe_clear_action_bit(action, pe_action_optional);
     }
 
     if (rsc != NULL) {
         enum action_tasks a_task = text2task(action->task);
         int warn_level = LOG_TRACE;
 
         if (save_action) {
             warn_level = LOG_WARNING;
         }
 
         if (is_set(action->flags, pe_action_have_node_attrs) == FALSE
             && action->node != NULL && action->op_entry != NULL) {
             pe_set_action_bit(action, pe_action_have_node_attrs);
             pe__unpack_dataset_nvpairs(action->op_entry, XML_TAG_ATTR_SETS,
                                        action->node->details->attrs,
                                        action->extra, NULL, FALSE, data_set);
         }
 
         if (is_set(action->flags, pe_action_pseudo)) {
             /* leave untouched */
 
         } else if (action->node == NULL) {
             pe_rsc_trace(rsc, "Unset runnable on %s", action->uuid);
             pe_clear_action_bit(action, pe_action_runnable);
 
         } else if (is_not_set(rsc->flags, pe_rsc_managed)
                    && g_hash_table_lookup(action->meta,
                                           XML_LRM_ATTR_INTERVAL_MS) == NULL) {
             crm_debug("Action %s (unmanaged)", action->uuid);
             pe_rsc_trace(rsc, "Set optional on %s", action->uuid);
             pe_set_action_bit(action, pe_action_optional);
 /*   			action->runnable = FALSE; */
 
         } else if (is_not_set(action->flags, pe_action_dc)
                    && !(action->node->details->online)
                    && (!pe__is_guest_node(action->node)
                        || action->node->details->remote_requires_reset)) {
             pe_clear_action_bit(action, pe_action_runnable);
             do_crm_log(warn_level, "Action %s on %s is unrunnable (offline)",
                        action->uuid, action->node->details->uname);
             if (is_set(action->rsc->flags, pe_rsc_managed)
                 && save_action && a_task == stop_rsc
                 && action->node->details->unclean == FALSE) {
                 pe_fence_node(data_set, action->node, "resource actions are unrunnable", FALSE);
             }
 
         } else if (is_not_set(action->flags, pe_action_dc)
                    && action->node->details->pending) {
             pe_clear_action_bit(action, pe_action_runnable);
             do_crm_log(warn_level, "Action %s on %s is unrunnable (pending)",
                        action->uuid, action->node->details->uname);
 
         } else if (action->needs == rsc_req_nothing) {
             pe_rsc_trace(rsc, "Action %s does not require anything", action->uuid);
             pe_action_set_reason(action, NULL, TRUE);
             if (pe__is_guest_node(action->node)
                 && !pe_can_fence(data_set, action->node)) {
                 /* An action that requires nothing usually does not require any
                  * fencing in order to be runnable. However, there is an
                  * exception: an action cannot be completed if it is on a guest
                  * node whose host is unclean and cannot be fenced.
                  */
                 pe_clear_action_bit(action, pe_action_runnable);
                 crm_debug("%s\t%s (cancelled : host cannot be fenced)",
                           action->node->details->uname, action->uuid);
             } else {
                 pe_set_action_bit(action, pe_action_runnable);
             }
 #if 0
             /*
              * No point checking this
              * - if we don't have quorum we can't stonith anyway
              */
         } else if (action->needs == rsc_req_stonith) {
             crm_trace("Action %s requires only stonith", action->uuid);
             action->runnable = TRUE;
 #endif
         } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE
                    && data_set->no_quorum_policy == no_quorum_stop) {
             pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, NULL, "no quorum", pe_action_runnable, TRUE);
             crm_debug("%s\t%s (cancelled : quorum)", action->node->details->uname, action->uuid);
 
         } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE
                    && data_set->no_quorum_policy == no_quorum_freeze) {
             pe_rsc_trace(rsc, "Check resource is already active: %s %s %s %s", rsc->id, action->uuid, role2text(rsc->next_role), role2text(rsc->role));
             if (rsc->fns->active(rsc, TRUE) == FALSE || rsc->next_role > rsc->role) {
                 pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, NULL, "quorum freeze", pe_action_runnable, TRUE);
                 pe_rsc_debug(rsc, "%s\t%s (cancelled : quorum freeze)",
                              action->node->details->uname, action->uuid);
             }
 
         } else if(is_not_set(action->flags, pe_action_runnable)) {
             pe_rsc_trace(rsc, "Action %s is runnable", action->uuid);
             //pe_action_set_reason(action, NULL, TRUE);
             pe_set_action_bit(action, pe_action_runnable);
         }
 
         if (save_action) {
             switch (a_task) {
                 case stop_rsc:
                     set_bit(rsc->flags, pe_rsc_stopping);
                     break;
                 case start_rsc:
                     clear_bit(rsc->flags, pe_rsc_starting);
                     if (is_set(action->flags, pe_action_runnable)) {
                         set_bit(rsc->flags, pe_rsc_starting);
                     }
                     break;
                 default:
                     break;
             }
         }
     }
 
     free(key);
     return action;
 }
 
 static const char *
 unpack_operation_on_fail(pe_action_t * action)
 {
 
     const char *value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL);
 
     if (safe_str_eq(action->task, CRMD_ACTION_STOP) && safe_str_eq(value, "standby")) {
         pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for %s stop "
                          "action to default value because 'standby' is not "
                          "allowed for stop", action->rsc->id);
         return NULL;
     } else if (safe_str_eq(action->task, CRMD_ACTION_DEMOTE) && !value) {
         /* demote on_fail defaults to master monitor value if present */
         xmlNode *operation = NULL;
         const char *name = NULL;
         const char *role = NULL;
         const char *on_fail = NULL;
         const char *interval_spec = NULL;
         const char *enabled = NULL;
 
         CRM_CHECK(action->rsc != NULL, return NULL);
 
         for (operation = __xml_first_child_element(action->rsc->ops_xml);
              operation && !value; operation = __xml_next_element(operation)) {
 
             if (!crm_str_eq((const char *)operation->name, "op", TRUE)) {
                 continue;
             }
             name = crm_element_value(operation, "name");
             role = crm_element_value(operation, "role");
             on_fail = crm_element_value(operation, XML_OP_ATTR_ON_FAIL);
             enabled = crm_element_value(operation, "enabled");
             interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
             if (!on_fail) {
                 continue;
             } else if (enabled && !crm_is_true(enabled)) {
                 continue;
             } else if (safe_str_neq(name, "monitor") || safe_str_neq(role, "Master")) {
                 continue;
             } else if (crm_parse_interval_spec(interval_spec) == 0) {
                 continue;
             }
 
             value = on_fail;
         }
     } else if (safe_str_eq(action->task, CRM_OP_LRM_DELETE)) {
         value = "ignore";
     }
 
     return value;
 }
 
 static xmlNode *
 find_min_interval_mon(pe_resource_t * rsc, gboolean include_disabled)
 {
     guint interval_ms = 0;
     guint min_interval_ms = G_MAXUINT;
     const char *name = NULL;
     const char *value = NULL;
     const char *interval_spec = NULL;
     xmlNode *op = NULL;
     xmlNode *operation = NULL;
 
     for (operation = __xml_first_child_element(rsc->ops_xml); operation != NULL;
          operation = __xml_next_element(operation)) {
 
         if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
             name = crm_element_value(operation, "name");
             interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
             value = crm_element_value(operation, "enabled");
             if (!include_disabled && value && crm_is_true(value) == FALSE) {
                 continue;
             }
 
             if (safe_str_neq(name, RSC_STATUS)) {
                 continue;
             }
 
             interval_ms = crm_parse_interval_spec(interval_spec);
 
             if (interval_ms && (interval_ms < min_interval_ms)) {
                 min_interval_ms = interval_ms;
                 op = operation;
             }
         }
     }
 
     return op;
 }
 
 static int
 unpack_start_delay(const char *value, GHashTable *meta)
 {
     int start_delay = 0;
 
     if (value != NULL) {
         start_delay = crm_get_msec(value);
 
         if (start_delay < 0) {
             start_delay = 0;
         }
 
         if (meta) {
             g_hash_table_replace(meta, strdup(XML_OP_ATTR_START_DELAY), crm_itoa(start_delay));
         }
     }
 
     return start_delay;
 }
 
 // true if value contains valid, non-NULL interval origin for recurring op
 static bool
 unpack_interval_origin(const char *value, xmlNode *xml_obj, guint interval_ms,
                        crm_time_t *now, long long *start_delay)
 {
     long long result = 0;
     guint interval_sec = interval_ms / 1000;
     crm_time_t *origin = NULL;
 
     // Ignore unspecified values and non-recurring operations
     if ((value == NULL) || (interval_ms == 0) || (now == NULL)) {
         return false;
     }
 
     // Parse interval origin from text
     origin = crm_time_new(value);
     if (origin == NULL) {
         pcmk__config_err("Ignoring '" XML_OP_ATTR_ORIGIN "' for operation "
                          "'%s' because '%s' is not valid",
                          (ID(xml_obj)? ID(xml_obj) : "(missing ID)"), value);
         return false;
     }
 
     // Get seconds since origin (negative if origin is in the future)
     result = crm_time_get_seconds(now) - crm_time_get_seconds(origin);
     crm_time_free(origin);
 
     // Calculate seconds from closest interval to now
     result = result % interval_sec;
 
     // Calculate seconds remaining until next interval
     result = ((result <= 0)? 0 : interval_sec) - result;
     crm_info("Calculated a start delay of %llds for operation '%s'",
              result,
              (ID(xml_obj)? ID(xml_obj) : "(unspecified)"));
 
     if (start_delay != NULL) {
         *start_delay = result * 1000; // milliseconds
     }
     return true;
 }
 
 static int
 unpack_timeout(const char *value)
 {
     int timeout = crm_get_msec(value);
 
     if (timeout < 0) {
         timeout = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
     }
     return timeout;
 }
 
 int
 pe_get_configured_timeout(pe_resource_t *rsc, const char *action, pe_working_set_t *data_set)
 {
     xmlNode *child = NULL;
     const char *timeout = NULL;
     int timeout_ms = 0;
 
     for (child = first_named_child(rsc->ops_xml, XML_ATTR_OP);
          child != NULL; child = crm_next_same_xml(child)) {
         if (safe_str_eq(action, crm_element_value(child, XML_NVPAIR_ATTR_NAME))) {
             timeout = crm_element_value(child, XML_ATTR_TIMEOUT);
             break;
         }
     }
 
     if (timeout == NULL && data_set->op_defaults) {
         GHashTable *action_meta = crm_str_table_new();
         pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS,
                                    NULL, action_meta, NULL, FALSE, data_set);
         timeout = g_hash_table_lookup(action_meta, XML_ATTR_TIMEOUT);
     }
 
     // @TODO check meta-attributes (including versioned meta-attributes)
     // @TODO maybe use min-interval monitor timeout as default for monitors
 
     timeout_ms = crm_get_msec(timeout);
     if (timeout_ms < 0) {
         timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
     }
     return timeout_ms;
 }
 
 #if ENABLE_VERSIONED_ATTRS
 static void
 unpack_versioned_meta(xmlNode *versioned_meta, xmlNode *xml_obj,
                       guint interval_ms, crm_time_t *now)
 {
     xmlNode *attrs = NULL;
     xmlNode *attr = NULL;
 
     for (attrs = __xml_first_child_element(versioned_meta); attrs != NULL;
          attrs = __xml_next_element(attrs)) {
 
         for (attr = __xml_first_child_element(attrs); attr != NULL;
              attr = __xml_next_element(attr)) {
 
             const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
             const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
 
             if (safe_str_eq(name, XML_OP_ATTR_START_DELAY)) {
                 int start_delay = unpack_start_delay(value, NULL);
 
                 crm_xml_add_int(attr, XML_NVPAIR_ATTR_VALUE, start_delay);
             } else if (safe_str_eq(name, XML_OP_ATTR_ORIGIN)) {
                 long long start_delay = 0;
 
                 if (unpack_interval_origin(value, xml_obj, interval_ms, now,
                                            &start_delay)) {
                     crm_xml_add(attr, XML_NVPAIR_ATTR_NAME,
                                 XML_OP_ATTR_START_DELAY);
                     crm_xml_add_ll(attr, XML_NVPAIR_ATTR_VALUE, start_delay);
                 }
             } else if (safe_str_eq(name, XML_ATTR_TIMEOUT)) {
                 int timeout = unpack_timeout(value);
 
                 crm_xml_add_int(attr, XML_NVPAIR_ATTR_VALUE, timeout);
             }
         }
     }
 }
 #endif
 
 /*!
  * \brief Unpack operation XML into an action structure
  *
  * Unpack an operation's meta-attributes (normalizing the interval, timeout,
  * and start delay values as integer milliseconds), requirements, and
  * failure policy.
  *
  * \param[in,out] action     Action to unpack into
  * \param[in]     xml_obj    Operation XML (or NULL if all defaults)
  * \param[in]     container  Resource that contains affected resource, if any
  * \param[in]     data_set   Cluster state
  */
 void
 unpack_operation(pe_action_t * action, xmlNode * xml_obj, pe_resource_t * container,
                  pe_working_set_t * data_set)
 {
     guint interval_ms = 0;
     int timeout = 0;
     char *value_ms = NULL;
     const char *value = NULL;
     const char *field = NULL;
     char *default_timeout = NULL;
 #if ENABLE_VERSIONED_ATTRS
     pe_rsc_action_details_t *rsc_details = NULL;
 #endif
 
     CRM_CHECK(action && action->rsc, return);
 
     // Cluster-wide <op_defaults> <meta_attributes>
     pe__unpack_dataset_nvpairs(data_set->op_defaults, XML_TAG_META_SETS, NULL,
                                action->meta, NULL, FALSE, data_set);
 
     // Probe timeouts default differently, so handle timeout default later
     default_timeout = g_hash_table_lookup(action->meta, XML_ATTR_TIMEOUT);
     if (default_timeout) {
         default_timeout = strdup(default_timeout);
         g_hash_table_remove(action->meta, XML_ATTR_TIMEOUT);
     }
 
     if (xml_obj) {
         xmlAttrPtr xIter = NULL;
 
         // <op> <meta_attributes> take precedence over defaults
         pe__unpack_dataset_nvpairs(xml_obj, XML_TAG_META_SETS, NULL,
                                    action->meta, NULL, TRUE, data_set);
 
 #if ENABLE_VERSIONED_ATTRS
         rsc_details = pe_rsc_action_details(action);
         pe_unpack_versioned_attributes(data_set->input, xml_obj,
                                        XML_TAG_ATTR_SETS, NULL,
                                        rsc_details->versioned_parameters,
                                        data_set->now, NULL);
         pe_unpack_versioned_attributes(data_set->input, xml_obj,
                                        XML_TAG_META_SETS, NULL,
                                        rsc_details->versioned_meta,
                                        data_set->now, NULL);
 #endif
 
         /* Anything set as an <op> XML property has highest precedence.
          * This ensures we use the name and interval from the <op> tag.
          */
         for (xIter = xml_obj->properties; xIter; xIter = xIter->next) {
             const char *prop_name = (const char *)xIter->name;
             const char *prop_value = crm_element_value(xml_obj, prop_name);
 
             g_hash_table_replace(action->meta, strdup(prop_name), strdup(prop_value));
         }
     }
 
     g_hash_table_remove(action->meta, "id");
 
     // Normalize interval to milliseconds
     field = XML_LRM_ATTR_INTERVAL;
     value = g_hash_table_lookup(action->meta, field);
     if (value != NULL) {
         interval_ms = crm_parse_interval_spec(value);
 
     } else if ((xml_obj == NULL) && !strcmp(action->task, RSC_STATUS)) {
         /* An orphaned recurring monitor will not have any XML. However, we
          * want the interval to be set, so the action can be properly detected
          * as a recurring monitor. Parse it from the key in this case.
          */
         parse_op_key(action->uuid, NULL, NULL, &interval_ms);
     }
     if (interval_ms > 0) {
         value_ms = crm_strdup_printf("%u", interval_ms);
         g_hash_table_replace(action->meta, strdup(field), value_ms);
 
     } else if (value) {
         g_hash_table_remove(action->meta, field);
     }
 
     // Handle timeout default, now that we know the interval
     if (g_hash_table_lookup(action->meta, XML_ATTR_TIMEOUT)) {
         free(default_timeout);
 
     } else {
         // Probe timeouts default to minimum-interval monitor's
         if (safe_str_eq(action->task, RSC_STATUS) && (interval_ms == 0)) {
 
             xmlNode *min_interval_mon = find_min_interval_mon(action->rsc, FALSE);
 
             if (min_interval_mon) {
                 value = crm_element_value(min_interval_mon, XML_ATTR_TIMEOUT);
                 if (value) {
                     crm_trace("\t%s defaults to minimum-interval monitor's timeout '%s'",
                               action->uuid, value);
                     free(default_timeout);
                     default_timeout = strdup(value);
                 }
             }
         }
 
         if (default_timeout) {
             g_hash_table_insert(action->meta, strdup(XML_ATTR_TIMEOUT),
                                 default_timeout);
         }
     }
 
     if (safe_str_neq(action->task, RSC_START)
         && safe_str_neq(action->task, RSC_PROMOTE)) {
         action->needs = rsc_req_nothing;
         value = "nothing (not start/promote)";
 
     } else if (is_set(action->rsc->flags, pe_rsc_needs_fencing)) {
         action->needs = rsc_req_stonith;
         value = "fencing (resource)";
 
     } else if (is_set(action->rsc->flags, pe_rsc_needs_quorum)) {
         action->needs = rsc_req_quorum;
         value = "quorum (resource)";
 
     } else {
         action->needs = rsc_req_nothing;
         value = "nothing (resource)";
     }
 
     pe_rsc_trace(action->rsc, "\tAction %s requires: %s", action->uuid, value);
 
     value = unpack_operation_on_fail(action);
 
     if (value == NULL) {
 
     } else if (safe_str_eq(value, "block")) {
         action->on_fail = action_fail_block;
         g_hash_table_insert(action->meta, strdup(XML_OP_ATTR_ON_FAIL), strdup("block"));
         value = "block"; // The above could destroy the original string
 
     } else if (safe_str_eq(value, "fence")) {
         action->on_fail = action_fail_fence;
         value = "node fencing";
 
         if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
             pcmk__config_err("Resetting '" XML_OP_ATTR_ON_FAIL "' for "
                              "operation '%s' to 'stop' because 'fence' is not "
                              "valid when fencing is disabled", action->uuid);
             action->on_fail = action_fail_stop;
             action->fail_role = RSC_ROLE_STOPPED;
             value = "stop resource";
         }
 
     } else if (safe_str_eq(value, "standby")) {
         action->on_fail = action_fail_standby;
         value = "node standby";
 
     } else if (safe_str_eq(value, "ignore")
                || safe_str_eq(value, "nothing")) {
         action->on_fail = action_fail_ignore;
         value = "ignore";
 
     } else if (safe_str_eq(value, "migrate")) {
         action->on_fail = action_fail_migrate;
         value = "force migration";
 
     } else if (safe_str_eq(value, "stop")) {
         action->on_fail = action_fail_stop;
         action->fail_role = RSC_ROLE_STOPPED;
         value = "stop resource";
 
     } else if (safe_str_eq(value, "restart")) {
         action->on_fail = action_fail_recover;
         value = "restart (and possibly migrate)";
 
     } else if (safe_str_eq(value, "restart-container")) {
         if (container) {
             action->on_fail = action_fail_restart_container;
             value = "restart container (and possibly migrate)";
 
         } else {
             value = NULL;
         }
 
     } else {
         pe_err("Resource %s: Unknown failure type (%s)", action->rsc->id, value);
         value = NULL;
     }
 
     /* defaults */
     if (value == NULL && container) {
         action->on_fail = action_fail_restart_container;
         value = "restart container (and possibly migrate) (default)";
 
     /* For remote nodes, ensure that any failure that results in dropping an
      * active connection to the node results in fencing of the node.
      *
      * There are only two action failures that don't result in fencing.
      * 1. probes - probe failures are expected.
      * 2. start - a start failure indicates that an active connection does not already
      * exist. The user can set op on-fail=fence if they really want to fence start
      * failures. */
     } else if (((value == NULL) || !is_set(action->rsc->flags, pe_rsc_managed)) &&
                 (pe__resource_is_remote_conn(action->rsc, data_set) &&
                !(safe_str_eq(action->task, CRMD_ACTION_STATUS) && (interval_ms == 0)) &&
                 (safe_str_neq(action->task, CRMD_ACTION_START)))) {
 
         if (!is_set(action->rsc->flags, pe_rsc_managed)) {
             action->on_fail = action_fail_stop;
             action->fail_role = RSC_ROLE_STOPPED;
             value = "stop unmanaged remote node (enforcing default)";
 
         } else {
             if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
                 value = "fence remote node (default)";
             } else {
                 value = "recover remote node connection (default)";
             }
 
             if (action->rsc->remote_reconnect_ms) {
                 action->fail_role = RSC_ROLE_STOPPED;
             }
             action->on_fail = action_fail_reset_remote;
         }
 
     } else if (value == NULL && safe_str_eq(action->task, CRMD_ACTION_STOP)) {
         if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
             action->on_fail = action_fail_fence;
             value = "resource fence (default)";
 
         } else {
             action->on_fail = action_fail_block;
             value = "resource block (default)";
         }
 
     } else if (value == NULL) {
         action->on_fail = action_fail_recover;
         value = "restart (and possibly migrate) (default)";
     }
 
     pe_rsc_trace(action->rsc, "\t%s failure handling: %s", action->task, value);
 
     value = NULL;
     if (xml_obj != NULL) {
         value = g_hash_table_lookup(action->meta, "role_after_failure");
         if (value) {
             pe_warn_once(pe_wo_role_after,
                         "Support for role_after_failure is deprecated and will be removed in a future release");
         }
     }
     if (value != NULL && action->fail_role == RSC_ROLE_UNKNOWN) {
         action->fail_role = text2role(value);
     }
     /* defaults */
     if (action->fail_role == RSC_ROLE_UNKNOWN) {
         if (safe_str_eq(action->task, CRMD_ACTION_PROMOTE)) {
             action->fail_role = RSC_ROLE_SLAVE;
         } else {
             action->fail_role = RSC_ROLE_STARTED;
         }
     }
     pe_rsc_trace(action->rsc, "\t%s failure results in: %s", action->task,
                  role2text(action->fail_role));
 
     value = g_hash_table_lookup(action->meta, XML_OP_ATTR_START_DELAY);
     if (value) {
         unpack_start_delay(value, action->meta);
     } else {
         long long start_delay = 0;
 
         value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ORIGIN);
         if (unpack_interval_origin(value, xml_obj, interval_ms, data_set->now,
                                    &start_delay)) {
             g_hash_table_replace(action->meta, strdup(XML_OP_ATTR_START_DELAY),
                                  crm_strdup_printf("%lld", start_delay));
         }
     }
 
     value = g_hash_table_lookup(action->meta, XML_ATTR_TIMEOUT);
     timeout = unpack_timeout(value);
     g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT), crm_itoa(timeout));
 
 #if ENABLE_VERSIONED_ATTRS
     unpack_versioned_meta(rsc_details->versioned_meta, xml_obj, interval_ms,
                           data_set->now);
 #endif
 }
 
 static xmlNode *
 find_rsc_op_entry_helper(pe_resource_t * rsc, const char *key, gboolean include_disabled)
 {
     guint interval_ms = 0;
     gboolean do_retry = TRUE;
     char *local_key = NULL;
     const char *name = NULL;
     const char *value = NULL;
     const char *interval_spec = NULL;
     char *match_key = NULL;
     xmlNode *op = NULL;
     xmlNode *operation = NULL;
 
   retry:
     for (operation = __xml_first_child_element(rsc->ops_xml); operation != NULL;
          operation = __xml_next_element(operation)) {
         if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
             name = crm_element_value(operation, "name");
             interval_spec = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
             value = crm_element_value(operation, "enabled");
             if (!include_disabled && value && crm_is_true(value) == FALSE) {
                 continue;
             }
 
             interval_ms = crm_parse_interval_spec(interval_spec);
             match_key = pcmk__op_key(rsc->id, name, interval_ms);
             if (safe_str_eq(key, match_key)) {
                 op = operation;
             }
             free(match_key);
 
             if (rsc->clone_name) {
                 match_key = pcmk__op_key(rsc->clone_name, name, interval_ms);
                 if (safe_str_eq(key, match_key)) {
                     op = operation;
                 }
                 free(match_key);
             }
 
             if (op != NULL) {
                 free(local_key);
                 return op;
             }
         }
     }
 
     free(local_key);
     if (do_retry == FALSE) {
         return NULL;
     }
 
     do_retry = FALSE;
     if (strstr(key, CRMD_ACTION_MIGRATE) || strstr(key, CRMD_ACTION_MIGRATED)) {
         local_key = pcmk__op_key(rsc->id, "migrate", 0);
         key = local_key;
         goto retry;
 
     } else if (strstr(key, "_notify_")) {
         local_key = pcmk__op_key(rsc->id, "notify", 0);
         key = local_key;
         goto retry;
     }
 
     return NULL;
 }
 
 xmlNode *
 find_rsc_op_entry(pe_resource_t * rsc, const char *key)
 {
     return find_rsc_op_entry_helper(rsc, key, FALSE);
 }
 
 void
 print_node(const char *pre_text, pe_node_t * node, gboolean details)
 {
     if (node == NULL) {
         crm_trace("%s%s: <NULL>", pre_text == NULL ? "" : pre_text, pre_text == NULL ? "" : ": ");
         return;
     }
 
     CRM_ASSERT(node->details);
     crm_trace("%s%s%sNode %s: (weight=%d, fixed=%s)",
               pre_text == NULL ? "" : pre_text,
               pre_text == NULL ? "" : ": ",
               node->details->online ? "" : "Unavailable/Unclean ",
               node->details->uname, node->weight, node->fixed ? "True" : "False");
 
     if (details) {
         int log_level = LOG_TRACE;
 
         char *pe_mutable = strdup("\t\t");
         GListPtr gIter = node->details->running_rsc;
 
         crm_trace("\t\t===Node Attributes");
         g_hash_table_foreach(node->details->attrs, print_str_str, pe_mutable);
         free(pe_mutable);
 
         crm_trace("\t\t=== Resources");
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
             rsc->fns->print(rsc, "\t\t", pe_print_log|pe_print_pending,
                             &log_level);
         }
     }
 }
 
 /*
  * Used by the HashTable for-loop
  */
 void
 print_str_str(gpointer key, gpointer value, gpointer user_data)
 {
     crm_trace("%s%s %s ==> %s",
               user_data == NULL ? "" : (char *)user_data,
               user_data == NULL ? "" : ": ", (char *)key, (char *)value);
 }
 
 void
 pe_free_action(pe_action_t * action)
 {
     if (action == NULL) {
         return;
     }
     g_list_free_full(action->actions_before, free);     /* pe_action_wrapper_t* */
     g_list_free_full(action->actions_after, free);      /* pe_action_wrapper_t* */
     if (action->extra) {
         g_hash_table_destroy(action->extra);
     }
     if (action->meta) {
         g_hash_table_destroy(action->meta);
     }
 #if ENABLE_VERSIONED_ATTRS
     if (action->rsc) {
         pe_free_rsc_action_details(action);
     }
 #endif
     free(action->cancel_task);
     free(action->reason);
     free(action->task);
     free(action->uuid);
     free(action->node);
     free(action);
 }
 
 GListPtr
 find_recurring_actions(GListPtr input, pe_node_t * not_on_node)
 {
     const char *value = NULL;
     GListPtr result = NULL;
     GListPtr gIter = input;
 
     CRM_CHECK(input != NULL, return NULL);
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         value = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL_MS);
         if (value == NULL) {
             /* skip */
         } else if (safe_str_eq(value, "0")) {
             /* skip */
         } else if (safe_str_eq(CRMD_ACTION_CANCEL, action->task)) {
             /* skip */
         } else if (not_on_node == NULL) {
             crm_trace("(null) Found: %s", action->uuid);
             result = g_list_prepend(result, action);
 
         } else if (action->node == NULL) {
             /* skip */
         } else if (action->node->details != not_on_node->details) {
             crm_trace("Found: %s", action->uuid);
             result = g_list_prepend(result, action);
         }
     }
 
     return result;
 }
 
 enum action_tasks
 get_complex_task(pe_resource_t * rsc, const char *name, gboolean allow_non_atomic)
 {
     enum action_tasks task = text2task(name);
 
     if (rsc == NULL) {
         return task;
 
     } else if (allow_non_atomic == FALSE || rsc->variant == pe_native) {
         switch (task) {
             case stopped_rsc:
             case started_rsc:
             case action_demoted:
             case action_promoted:
                 crm_trace("Folding %s back into its atomic counterpart for %s", name, rsc->id);
                 return task - 1;
                 break;
             default:
                 break;
         }
     }
     return task;
 }
 
 pe_action_t *
 find_first_action(GListPtr input, const char *uuid, const char *task, pe_node_t * on_node)
 {
     GListPtr gIter = NULL;
 
     CRM_CHECK(uuid || task, return NULL);
 
     for (gIter = input; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if (uuid != NULL && safe_str_neq(uuid, action->uuid)) {
             continue;
 
         } else if (task != NULL && safe_str_neq(task, action->task)) {
             continue;
 
         } else if (on_node == NULL) {
             return action;
 
         } else if (action->node == NULL) {
             continue;
 
         } else if (on_node->details == action->node->details) {
             return action;
         }
     }
 
     return NULL;
 }
 
 GListPtr
 find_actions(GListPtr input, const char *key, const pe_node_t *on_node)
 {
     GListPtr gIter = input;
     GListPtr result = NULL;
 
     CRM_CHECK(key != NULL, return NULL);
 
     for (; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if (safe_str_neq(key, action->uuid)) {
             crm_trace("%s does not match action %s", key, action->uuid);
             continue;
 
         } else if (on_node == NULL) {
             crm_trace("Action %s matches (ignoring node)", key);
             result = g_list_prepend(result, action);
 
         } else if (action->node == NULL) {
             crm_trace("Action %s matches (unallocated, assigning to %s)",
                       key, on_node->details->uname);
 
             action->node = node_copy(on_node);
             result = g_list_prepend(result, action);
 
         } else if (on_node->details == action->node->details) {
             crm_trace("Action %s on %s matches", key, on_node->details->uname);
             result = g_list_prepend(result, action);
 
         } else {
             crm_trace("Action %s on node %s does not match requested node %s",
                       key, action->node->details->uname,
                       on_node->details->uname);
         }
     }
 
     return result;
 }
 
 GList *
 find_actions_exact(GList *input, const char *key, const pe_node_t *on_node)
 {
     GList *result = NULL;
 
     CRM_CHECK(key != NULL, return NULL);
 
     if (on_node == NULL) {
         crm_trace("Not searching for action %s because node not specified",
                   key);
         return NULL;
     }
 
     for (GList *gIter = input; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         if (action->node == NULL) {
             crm_trace("Skipping comparison of %s vs action %s without node",
                       key, action->uuid);
 
         } else if (safe_str_neq(key, action->uuid)) {
             crm_trace("Desired action %s doesn't match %s", key, action->uuid);
 
         } else if (safe_str_neq(on_node->details->id,
                                 action->node->details->id)) {
             crm_trace("Action %s desired node ID %s doesn't match %s",
                       key, on_node->details->id, action->node->details->id);
 
         } else {
             crm_trace("Action %s matches", key);
             result = g_list_prepend(result, action);
         }
     }
 
     return result;
 }
 
 /*!
  * \brief Find all actions of given type for a resource
  *
  * \param[in] rsc           Resource to search
  * \param[in] node          Find only actions scheduled on this node
  * \param[in] task          Action name to search for
  * \param[in] require_node  If TRUE, NULL node or action node will not match
  *
  * \return List of actions found (or NULL if none)
  * \note If node is not NULL and require_node is FALSE, matching actions
  *       without a node will be assigned to node.
  */
 GList *
 pe__resource_actions(const pe_resource_t *rsc, const pe_node_t *node,
                      const char *task, bool require_node)
 {
     GList *result = NULL;
     char *key = pcmk__op_key(rsc->id, task, 0);
 
     if (require_node) {
         result = find_actions_exact(rsc->actions, key, node);
     } else {
         result = find_actions(rsc->actions, key, node);
     }
     free(key);
     return result;
 }
 
 static void
 resource_node_score(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag)
 {
     pe_node_t *match = NULL;
 
     if ((rsc->exclusive_discover || (node->rsc_discover_mode == pe_discover_never))
         && safe_str_eq(tag, "symmetric_default")) {
         /* This string comparision may be fragile, but exclusive resources and
          * exclusive nodes should not have the symmetric_default constraint
          * applied to them.
          */
         return;
 
     } else if (rsc->children) {
         GListPtr gIter = rsc->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
             resource_node_score(child_rsc, node, score, tag);
         }
     }
 
     pe_rsc_trace(rsc, "Setting %s for %s on %s: %d", tag, rsc->id, node->details->uname, score);
     match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
     if (match == NULL) {
         match = node_copy(node);
         g_hash_table_insert(rsc->allowed_nodes, (gpointer) match->details->id, match);
     }
     match->weight = merge_weights(match->weight, score);
 }
 
 void
 resource_location(pe_resource_t * rsc, pe_node_t * node, int score, const char *tag,
                   pe_working_set_t * data_set)
 {
     if (node != NULL) {
         resource_node_score(rsc, node, score, tag);
 
     } else if (data_set != NULL) {
         GListPtr gIter = data_set->nodes;
 
         for (; gIter != NULL; gIter = gIter->next) {
             pe_node_t *node_iter = (pe_node_t *) gIter->data;
 
             resource_node_score(rsc, node_iter, score, tag);
         }
 
     } else {
         GHashTableIter iter;
         pe_node_t *node_iter = NULL;
 
         g_hash_table_iter_init(&iter, rsc->allowed_nodes);
         while (g_hash_table_iter_next(&iter, NULL, (void **)&node_iter)) {
             resource_node_score(rsc, node_iter, score, tag);
         }
     }
 
     if (node == NULL && score == -INFINITY) {
         if (rsc->allocated_to) {
             crm_info("Deallocating %s from %s", rsc->id, rsc->allocated_to->details->uname);
             free(rsc->allocated_to);
             rsc->allocated_to = NULL;
         }
     }
 }
 
 #define sort_return(an_int, why) do {					\
 	free(a_uuid);						\
 	free(b_uuid);						\
 	crm_trace("%s (%d) %c %s (%d) : %s",				\
 		  a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=',	\
 		  b_xml_id, b_call_id, why);				\
 	return an_int;							\
     } while(0)
 
 gint
 sort_op_by_callid(gconstpointer a, gconstpointer b)
 {
     int a_call_id = -1;
     int b_call_id = -1;
 
     char *a_uuid = NULL;
     char *b_uuid = NULL;
 
     const xmlNode *xml_a = a;
     const xmlNode *xml_b = b;
 
     const char *a_xml_id = crm_element_value(xml_a, XML_ATTR_ID);
     const char *b_xml_id = crm_element_value(xml_b, XML_ATTR_ID);
 
     if (safe_str_eq(a_xml_id, b_xml_id)) {
         /* We have duplicate lrm_rsc_op entries in the status
          * section which is unlikely to be a good thing
          *    - we can handle it easily enough, but we need to get
          *    to the bottom of why it's happening.
          */
         pe_err("Duplicate lrm_rsc_op entries named %s", a_xml_id);
         sort_return(0, "duplicate");
     }
 
     crm_element_value_int(xml_a, XML_LRM_ATTR_CALLID, &a_call_id);
     crm_element_value_int(xml_b, XML_LRM_ATTR_CALLID, &b_call_id);
 
     if (a_call_id == -1 && b_call_id == -1) {
         /* both are pending ops so it doesn't matter since
          *   stops are never pending
          */
         sort_return(0, "pending");
 
     } else if (a_call_id >= 0 && a_call_id < b_call_id) {
         sort_return(-1, "call id");
 
     } else if (b_call_id >= 0 && a_call_id > b_call_id) {
         sort_return(1, "call id");
 
     } else if (b_call_id >= 0 && a_call_id == b_call_id) {
         /*
          * The op and last_failed_op are the same
          * Order on last-rc-change
          */
         time_t last_a = -1;
         time_t last_b = -1;
 
         crm_element_value_epoch(xml_a, XML_RSC_OP_LAST_CHANGE, &last_a);
         crm_element_value_epoch(xml_b, XML_RSC_OP_LAST_CHANGE, &last_b);
 
         crm_trace("rc-change: %lld vs %lld",
                   (long long) last_a, (long long) last_b);
         if (last_a >= 0 && last_a < last_b) {
             sort_return(-1, "rc-change");
 
         } else if (last_b >= 0 && last_a > last_b) {
             sort_return(1, "rc-change");
         }
         sort_return(0, "rc-change");
 
     } else {
         /* One of the inputs is a pending operation
          * Attempt to use XML_ATTR_TRANSITION_MAGIC to determine its age relative to the other
          */
 
         int a_id = -1;
         int b_id = -1;
 
         const char *a_magic = crm_element_value(xml_a, XML_ATTR_TRANSITION_MAGIC);
         const char *b_magic = crm_element_value(xml_b, XML_ATTR_TRANSITION_MAGIC);
 
         CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
         if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL,
                                      NULL)) {
             sort_return(0, "bad magic a");
         }
         if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL,
                                      NULL)) {
             sort_return(0, "bad magic b");
         }
         /* try to determine the relative age of the operation...
          * some pending operations (e.g. a start) may have been superseded
          *   by a subsequent stop
          *
          * [a|b]_id == -1 means it's a shutdown operation and _always_ comes last
          */
         if (safe_str_neq(a_uuid, b_uuid) || a_id == b_id) {
             /*
              * some of the logic in here may be redundant...
              *
              * if the UUID from the TE doesn't match then one better
              *   be a pending operation.
              * pending operations don't survive between elections and joins
              *   because we query the LRM directly
              */
 
             if (b_call_id == -1) {
                 sort_return(-1, "transition + call");
 
             } else if (a_call_id == -1) {
                 sort_return(1, "transition + call");
             }
 
         } else if ((a_id >= 0 && a_id < b_id) || b_id == -1) {
             sort_return(-1, "transition");
 
         } else if ((b_id >= 0 && a_id > b_id) || a_id == -1) {
             sort_return(1, "transition");
         }
     }
 
     /* we should never end up here */
     CRM_CHECK(FALSE, sort_return(0, "default"));
 
 }
 
 time_t
 get_effective_time(pe_working_set_t * data_set)
 {
     if(data_set) {
         if (data_set->now == NULL) {
             crm_trace("Recording a new 'now'");
             data_set->now = crm_time_new(NULL);
         }
         return crm_time_get_seconds_since_epoch(data_set->now);
     }
 
     crm_trace("Defaulting to 'now'");
     return time(NULL);
 }
 
 gboolean
 get_target_role(pe_resource_t * rsc, enum rsc_role_e * role)
 {
     enum rsc_role_e local_role = RSC_ROLE_UNKNOWN;
     const char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
 
     CRM_CHECK(role != NULL, return FALSE);
 
     if (value == NULL || safe_str_eq("started", value)
         || safe_str_eq("default", value)) {
         return FALSE;
     }
 
     local_role = text2role(value);
     if (local_role == RSC_ROLE_UNKNOWN) {
         pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
                          "because '%s' is not valid", rsc->id, value);
         return FALSE;
 
     } else if (local_role > RSC_ROLE_STARTED) {
         if (is_set(uber_parent(rsc)->flags, pe_rsc_promotable)) {
             if (local_role > RSC_ROLE_SLAVE) {
                 /* This is what we'd do anyway, just leave the default to avoid messing up the placement algorithm */
                 return FALSE;
             }
 
         } else {
             pcmk__config_err("Ignoring '" XML_RSC_ATTR_TARGET_ROLE "' for %s "
                              "because '%s' only makes sense for promotable "
                              "clones", rsc->id, value);
             return FALSE;
         }
     }
 
     *role = local_role;
     return TRUE;
 }
 
 gboolean
 order_actions(pe_action_t * lh_action, pe_action_t * rh_action, enum pe_ordering order)
 {
     GListPtr gIter = NULL;
     pe_action_wrapper_t *wrapper = NULL;
     GListPtr list = NULL;
 
     if (order == pe_order_none) {
         return FALSE;
     }
 
     if (lh_action == NULL || rh_action == NULL) {
         return FALSE;
     }
 
     crm_trace("Ordering Action %s before %s", lh_action->uuid, rh_action->uuid);
 
     /* Ensure we never create a dependency on ourselves... it's happened */
     CRM_ASSERT(lh_action != rh_action);
 
     /* Filter dups, otherwise update_action_states() has too much work to do */
     gIter = lh_action->actions_after;
     for (; gIter != NULL; gIter = gIter->next) {
         pe_action_wrapper_t *after = (pe_action_wrapper_t *) gIter->data;
 
         if (after->action == rh_action && (after->type & order)) {
             return FALSE;
         }
     }
 
     wrapper = calloc(1, sizeof(pe_action_wrapper_t));
     wrapper->action = rh_action;
     wrapper->type = order;
 
     list = lh_action->actions_after;
     list = g_list_prepend(list, wrapper);
     lh_action->actions_after = list;
 
     wrapper = NULL;
 
 /* 	order |= pe_order_implies_then; */
 /* 	order ^= pe_order_implies_then; */
 
     wrapper = calloc(1, sizeof(pe_action_wrapper_t));
     wrapper->action = lh_action;
     wrapper->type = order;
     list = rh_action->actions_before;
     list = g_list_prepend(list, wrapper);
     rh_action->actions_before = list;
     return TRUE;
 }
 
 pe_action_t *
 get_pseudo_op(const char *name, pe_working_set_t * data_set)
 {
     pe_action_t *op = NULL;
 
     if(data_set->singletons) {
         op = g_hash_table_lookup(data_set->singletons, name);
     }
     if (op == NULL) {
         op = custom_action(NULL, strdup(name), name, NULL, TRUE, TRUE, data_set);
         set_bit(op->flags, pe_action_pseudo);
         set_bit(op->flags, pe_action_runnable);
     }
 
     return op;
 }
 
 void
 destroy_ticket(gpointer data)
 {
     pe_ticket_t *ticket = data;
 
     if (ticket->state) {
         g_hash_table_destroy(ticket->state);
     }
     free(ticket->id);
     free(ticket);
 }
 
 pe_ticket_t *
 ticket_new(const char *ticket_id, pe_working_set_t * data_set)
 {
     pe_ticket_t *ticket = NULL;
 
     if (ticket_id == NULL || strlen(ticket_id) == 0) {
         return NULL;
     }
 
     if (data_set->tickets == NULL) {
         data_set->tickets =
             g_hash_table_new_full(crm_str_hash, g_str_equal, free,
                                   destroy_ticket);
     }
 
     ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
     if (ticket == NULL) {
 
         ticket = calloc(1, sizeof(pe_ticket_t));
         if (ticket == NULL) {
             crm_err("Cannot allocate ticket '%s'", ticket_id);
             return NULL;
         }
 
         crm_trace("Creaing ticket entry for %s", ticket_id);
 
         ticket->id = strdup(ticket_id);
         ticket->granted = FALSE;
         ticket->last_granted = -1;
         ticket->standby = FALSE;
         ticket->state = crm_str_table_new();
 
         g_hash_table_insert(data_set->tickets, strdup(ticket->id), ticket);
     }
 
     return ticket;
 }
 
 static void
 filter_parameters(xmlNode * param_set, const char *param_string, bool need_present)
 {
     if (param_set && param_string) {
         xmlAttrPtr xIter = param_set->properties;
 
         while (xIter) {
             const char *prop_name = (const char *)xIter->name;
             char *name = crm_strdup_printf(" %s ", prop_name);
             char *match = strstr(param_string, name);
 
             free(name);
 
             //  Do now, because current entry might get removed below
             xIter = xIter->next;
 
             if (need_present && match == NULL) {
                 crm_trace("%s not found in %s", prop_name, param_string);
                 xml_remove_prop(param_set, prop_name);
 
             } else if (need_present == FALSE && match) {
                 crm_trace("%s found in %s", prop_name, param_string);
                 xml_remove_prop(param_set, prop_name);
             }
         }
     }
 }
 
 #if ENABLE_VERSIONED_ATTRS
 static void
 append_versioned_params(xmlNode *versioned_params, const char *ra_version, xmlNode *params)
 {
     GHashTable *hash = pe_unpack_versioned_parameters(versioned_params, ra_version);
     char *key = NULL;
     char *value = NULL;
     GHashTableIter iter;
 
     g_hash_table_iter_init(&iter, hash);
     while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
         crm_xml_add(params, key, value);
     }
     g_hash_table_destroy(hash);
 }
 #endif
 
 /*!
  * \internal
  * \brief Calculate action digests and store in node's digest cache
  *
  * \param[in] rsc          Resource that action was for
  * \param[in] task         Name of action performed
  * \param[in] key          Action's task key
  * \param[in] node         Node action was performed on
  * \param[in] xml_op       XML of operation in CIB status (if available)
  * \param[in] calc_secure  Whether to calculate secure digest
  * \param[in] data_set     Cluster working set
  *
  * \return Pointer to node's digest cache entry
  */
 static op_digest_cache_t *
 rsc_action_digest(pe_resource_t *rsc, const char *task, const char *key,
                   pe_node_t *node, xmlNode *xml_op, bool calc_secure,
                   pe_working_set_t *data_set)
 {
     op_digest_cache_t *data = NULL;
 
     data = g_hash_table_lookup(node->details->digest_cache, key);
     if (data == NULL) {
         GHashTable *local_rsc_params = crm_str_table_new();
         pe_action_t *action = custom_action(rsc, strdup(key), task, node, TRUE, FALSE, data_set);
 #if ENABLE_VERSIONED_ATTRS
         xmlNode *local_versioned_params = create_xml_node(NULL, XML_TAG_RSC_VER_ATTRS);
         const char *ra_version = NULL;
 #endif
 
         const char *op_version;
         const char *restart_list = NULL;
         const char *secure_list = " passwd password ";
 
         data = calloc(1, sizeof(op_digest_cache_t));
         CRM_ASSERT(data != NULL);
 
         get_rsc_attributes(local_rsc_params, rsc, node, data_set);
 #if ENABLE_VERSIONED_ATTRS
         pe_get_versioned_attributes(local_versioned_params, rsc, node, data_set);
 #endif
 
         data->params_all = create_xml_node(NULL, XML_TAG_PARAMS);
 
         // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
         if (pe__add_bundle_remote_name(rsc, data->params_all,
                                        XML_RSC_ATTR_REMOTE_RA_ADDR)) {
             crm_trace("Set address for bundle connection %s (on %s)",
                       rsc->id, node->details->uname);
         }
 
         g_hash_table_foreach(local_rsc_params, hash2field, data->params_all);
         g_hash_table_foreach(action->extra, hash2field, data->params_all);
         g_hash_table_foreach(rsc->parameters, hash2field, data->params_all);
         g_hash_table_foreach(action->meta, hash2metafield, data->params_all);
 
         if(xml_op) {
             secure_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_SECURE);
             restart_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_RESTART);
 
             op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
 #if ENABLE_VERSIONED_ATTRS
             ra_version = crm_element_value(xml_op, XML_ATTR_RA_VERSION);
 #endif
 
         } else {
             op_version = CRM_FEATURE_SET;
         }
 
 #if ENABLE_VERSIONED_ATTRS
         append_versioned_params(local_versioned_params, ra_version, data->params_all);
         append_versioned_params(rsc->versioned_parameters, ra_version, data->params_all);
 
         {
             pe_rsc_action_details_t *details = pe_rsc_action_details(action);
             append_versioned_params(details->versioned_parameters, ra_version, data->params_all);
         }
 #endif
 
         pcmk__filter_op_for_digest(data->params_all);
 
         g_hash_table_destroy(local_rsc_params);
         pe_free_action(action);
 
         data->digest_all_calc = calculate_operation_digest(data->params_all, op_version);
 
         if (calc_secure) {
             data->params_secure = copy_xml(data->params_all);
             if(secure_list) {
                 filter_parameters(data->params_secure, secure_list, FALSE);
             }
             data->digest_secure_calc = calculate_operation_digest(data->params_secure, op_version);
         }
 
         if(xml_op && crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST) != NULL) {
             data->params_restart = copy_xml(data->params_all);
             if (restart_list) {
                 filter_parameters(data->params_restart, restart_list, TRUE);
             }
             data->digest_restart_calc = calculate_operation_digest(data->params_restart, op_version);
         }
 
         g_hash_table_insert(node->details->digest_cache, strdup(key), data);
     }
 
     return data;
 }
 
 op_digest_cache_t *
 rsc_action_digest_cmp(pe_resource_t * rsc, xmlNode * xml_op, pe_node_t * node,
                       pe_working_set_t * data_set)
 {
     op_digest_cache_t *data = NULL;
 
     char *key = NULL;
     guint interval_ms = 0;
 
     const char *op_version;
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     const char *digest_all;
     const char *digest_restart;
 
     CRM_ASSERT(node != NULL);
 
     op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
     digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
     digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);
 
     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
     key = pcmk__op_key(rsc->id, task, interval_ms);
     data = rsc_action_digest(rsc, task, key, node, xml_op,
                              is_set(data_set->flags, pe_flag_sanitized),
                              data_set);
 
     data->rc = RSC_DIGEST_MATCH;
     if (digest_restart && data->digest_restart_calc && strcmp(data->digest_restart_calc, digest_restart) != 0) {
         pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (restart:%s) %s",
                  key, node->details->uname,
                  crm_str(digest_restart), data->digest_restart_calc,
                  op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
         data->rc = RSC_DIGEST_RESTART;
 
     } else if (digest_all == NULL) {
         /* it is unknown what the previous op digest was */
         data->rc = RSC_DIGEST_UNKNOWN;
 
     } else if (strcmp(digest_all, data->digest_all_calc) != 0) {
         pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (%s:%s) %s",
                  key, node->details->uname,
                  crm_str(digest_all), data->digest_all_calc,
                  (interval_ms > 0)? "reschedule" : "reload",
                  op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
         data->rc = RSC_DIGEST_ALL;
     }
 
     free(key);
     return data;
 }
 
 /*!
  * \internal
  * \brief Create an unfencing summary for use in special node attribute
  *
  * Create a string combining a fence device's resource ID, agent type, and
  * parameter digest (whether for all parameters or just non-private parameters).
  * This can be stored in a special node attribute, allowing us to detect changes
  * in either the agent type or parameters, to know whether unfencing must be
  * redone or can be safely skipped when the device's history is cleaned.
  *
  * \param[in] rsc_id        Fence device resource ID
  * \param[in] agent_type    Fence device agent
  * \param[in] param_digest  Fence device parameter digest
  *
  * \return Newly allocated string with unfencing digest
  * \note The caller is responsible for freeing the result.
  */
 static inline char *
 create_unfencing_summary(const char *rsc_id, const char *agent_type,
                          const char *param_digest)
 {
     return crm_strdup_printf("%s:%s:%s", rsc_id, agent_type, param_digest);
 }
 
 /*!
  * \internal
  * \brief Check whether a node can skip unfencing
  *
  * Check whether a fence device's current definition matches a node's
  * stored summary of when it was last unfenced by the device.
  *
  * \param[in] rsc_id        Fence device's resource ID
  * \param[in] agent         Fence device's agent type
  * \param[in] digest_calc   Fence device's current parameter digest
  * \param[in] node_summary  Value of node's special unfencing node attribute
  *                          (a comma-separated list of unfencing summaries for
  *                          all devices that have unfenced this node)
  *
  * \return TRUE if digest matches, FALSE otherwise
  */
 static bool
 unfencing_digest_matches(const char *rsc_id, const char *agent,
                          const char *digest_calc, const char *node_summary)
 {
     bool matches = FALSE;
 
     if (rsc_id && agent && digest_calc && node_summary) {
         char *search_secure = create_unfencing_summary(rsc_id, agent,
                                                        digest_calc);
 
         /* The digest was calculated including the device ID and agent,
          * so there is no risk of collision using strstr().
          */
         matches = (strstr(node_summary, search_secure) != NULL);
         crm_trace("Calculated unfencing digest '%s' %sfound in '%s'",
                   search_secure, matches? "" : "not ", node_summary);
         free(search_secure);
     }
     return matches;
 }
 
 /* Magic string to use as action name for digest cache entries used for
  * unfencing checks. This is not a real action name (i.e. "on"), so
  * check_action_definition() won't confuse these entries with real actions.
  */
 #define STONITH_DIGEST_TASK "stonith-on"
 
 /*!
  * \internal
  * \brief Calculate fence device digests and digest comparison result
  *
  * \param[in] rsc       Fence device resource
  * \param[in] agent     Fence device's agent type
  * \param[in] node      Node with digest cache to use
  * \param[in] data_set  Cluster working set
  *
  * \return Node's digest cache entry
  */
 static op_digest_cache_t *
 fencing_action_digest_cmp(pe_resource_t *rsc, const char *agent,
                           pe_node_t *node, pe_working_set_t *data_set)
 {
     const char *node_summary = NULL;
 
     // Calculate device's current parameter digests
     char *key = pcmk__op_key(rsc->id, STONITH_DIGEST_TASK, 0);
     op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key,
                                                 node, NULL, TRUE, data_set);
 
     free(key);
 
     // Check whether node has special unfencing summary node attribute
     node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL);
     if (node_summary == NULL) {
         data->rc = RSC_DIGEST_UNKNOWN;
         return data;
     }
 
     // Check whether full parameter digest matches
     if (unfencing_digest_matches(rsc->id, agent, data->digest_all_calc,
                                  node_summary)) {
         data->rc = RSC_DIGEST_MATCH;
         return data;
     }
 
     // Check whether secure parameter digest matches
     node_summary = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE);
     if (unfencing_digest_matches(rsc->id, agent, data->digest_secure_calc,
                                  node_summary)) {
         data->rc = RSC_DIGEST_MATCH;
         if (is_set(data_set->flags, pe_flag_stdout)) {
             printf("Only 'private' parameters to %s for unfencing %s changed\n",
                    rsc->id, node->details->uname);
         }
         return data;
     }
 
     // Parameters don't match
     data->rc = RSC_DIGEST_ALL;
     if (is_set(data_set->flags, (pe_flag_sanitized|pe_flag_stdout))
         && data->digest_secure_calc) {
         char *digest = create_unfencing_summary(rsc->id, agent,
                                                 data->digest_secure_calc);
 
         printf("Parameters to %s for unfencing %s changed, try '%s'\n",
                rsc->id, node->details->uname, digest);
         free(digest);
     }
     return data;
 }
 
 const char *rsc_printable_id(pe_resource_t *rsc)
 {
     if (is_not_set(rsc->flags, pe_rsc_unique)) {
         return ID(rsc->xml);
     }
     return rsc->id;
 }
 
 void
 clear_bit_recursive(pe_resource_t * rsc, unsigned long long flag)
 {
     GListPtr gIter = rsc->children;
 
     clear_bit(rsc->flags, flag);
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         clear_bit_recursive(child_rsc, flag);
     }
 }
 
 void
 set_bit_recursive(pe_resource_t * rsc, unsigned long long flag)
 {
     GListPtr gIter = rsc->children;
 
     set_bit(rsc->flags, flag);
     for (; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 
         set_bit_recursive(child_rsc, flag);
     }
 }
 
 static GListPtr
 find_unfencing_devices(GListPtr candidates, GListPtr matches) 
 {
     for (GListPtr gIter = candidates; gIter != NULL; gIter = gIter->next) {
         pe_resource_t *candidate = gIter->data;
         const char *provides = g_hash_table_lookup(candidate->meta, XML_RSC_ATTR_PROVIDES);
         const char *requires = g_hash_table_lookup(candidate->meta, XML_RSC_ATTR_REQUIRES);
 
         if(candidate->children) {
             matches = find_unfencing_devices(candidate->children, matches);
         } else if (is_not_set(candidate->flags, pe_rsc_fence_device)) {
             continue;
 
         } else if (crm_str_eq(provides, "unfencing", FALSE) || crm_str_eq(requires, "unfencing", FALSE)) {
             matches = g_list_prepend(matches, candidate);
         }
     }
     return matches;
 }
 
 static int
 node_priority_fencing_delay(pe_node_t * node, pe_working_set_t * data_set)
 {
     int member_count = 0;
     int online_count = 0;
     int top_priority = 0;
     int lowest_priority = 0;
     GListPtr gIter = NULL;
 
     // `priority-fencing-delay` is disabled
     if (data_set->priority_fencing_delay < 0) {
         return -1;
     }
 
     /* No need to delay fencing if the fencing target is not a normal cluster
      * member, for example if it's a remote node or a guest node. */
     if (node->details->type != node_member) {
         return 0;
     }
 
     // No need to delay fencing if the fencing target is in our partition
     if (node->details->online) {
         return 0;
     }
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         pe_node_t *n =  gIter->data;
 
         if (n->details->type != node_member) {
             continue;
         }
 
         member_count ++;
 
         if (n->details->online) {
             online_count++;
         }
 
         if (member_count == 1
             || n->details->priority > top_priority) {
             top_priority = n->details->priority;
         }
 
         if (member_count == 1
             || n->details->priority < lowest_priority) {
             lowest_priority = n->details->priority;
         }
     }
 
     // No need to delay if we have more than half of the cluster members
     if (online_count > member_count / 2) {
         return 0;
     }
 
     /* All the nodes have equal priority.
      * Any configured corresponding `pcmk_delay_base/max` will be applied. */
     if (lowest_priority == top_priority) {
         return -1;
     }
 
     if (node->details->priority < top_priority) {
         return 0;
     }
 
     return data_set->priority_fencing_delay;
 }
 
 pe_action_t *
 pe_fence_op(pe_node_t * node, const char *op, bool optional, const char *reason,
             bool priority_delay, pe_working_set_t * data_set)
 {
     char *op_key = NULL;
     pe_action_t *stonith_op = NULL;
 
     if(op == NULL) {
         op = data_set->stonith_action;
     }
 
     op_key = crm_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op);
 
     if(data_set->singletons) {
         stonith_op = g_hash_table_lookup(data_set->singletons, op_key);
     }
 
     if(stonith_op == NULL) {
         stonith_op = custom_action(NULL, op_key, CRM_OP_FENCE, node, TRUE, TRUE, data_set);
 
         add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname);
         add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id);
         add_hash_param(stonith_op->meta, "stonith_action", op);
 
         if (pe__is_guest_or_remote_node(node)
             && is_set(data_set->flags, pe_flag_enable_unfencing)) {
             /* Extra work to detect device changes on remotes
              *
              * We may do this for all nodes in the future, but for now
              * the check_action_definition() based stuff works fine.
              */
             long max = 1024;
             long digests_all_offset = 0;
             long digests_secure_offset = 0;
 
             char *digests_all = calloc(max, sizeof(char));
             char *digests_secure = calloc(max, sizeof(char));
             GListPtr matches = find_unfencing_devices(data_set->resources, NULL);
 
             for (GListPtr gIter = matches; gIter != NULL; gIter = gIter->next) {
                 pe_resource_t *match = gIter->data;
                 const char *agent = g_hash_table_lookup(match->meta,
                                                         XML_ATTR_TYPE);
                 op_digest_cache_t *data = NULL;
 
                 data = fencing_action_digest_cmp(match, agent, node, data_set);
                 if(data->rc == RSC_DIGEST_ALL) {
                     optional = FALSE;
                     crm_notice("Unfencing %s (remote): because the definition of %s changed", node->details->uname, match->id);
                     if (is_set(data_set->flags, pe_flag_stdout)) {
                         fprintf(stdout, "  notice: Unfencing %s (remote): because the definition of %s changed\n", node->details->uname, match->id);
                     }
                 }
 
                 digests_all_offset += snprintf(
                     digests_all+digests_all_offset, max-digests_all_offset,
                     "%s:%s:%s,", match->id, agent, data->digest_all_calc);
 
                 digests_secure_offset += snprintf(
                     digests_secure+digests_secure_offset, max-digests_secure_offset,
                     "%s:%s:%s,", match->id, agent, data->digest_secure_calc);
             }
             g_hash_table_insert(stonith_op->meta,
                                 strdup(XML_OP_ATTR_DIGESTS_ALL),
                                 digests_all);
             g_hash_table_insert(stonith_op->meta,
                                 strdup(XML_OP_ATTR_DIGESTS_SECURE),
                                 digests_secure);
         }
 
     } else {
         free(op_key);
     }
 
     if (data_set->priority_fencing_delay >= 0
 
             /* It's a suitable case where `priority-fencing-delay` applies.
              * At least add `priority-fencing-delay` field as an indicator. */
         && (priority_delay
 
             /* Re-calculate priority delay for the suitable case when
              * pe_fence_op() is called again by stage6() after node priority has
              * been actually calculated with native_add_running() */
             || g_hash_table_lookup(stonith_op->meta,
                                    XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY) != NULL)) {
 
             /* Add `priority-fencing-delay` to the fencing op even if it's 0 for
              * the targeting node. So that it takes precedence over any possible
              * `pcmk_delay_base/max`.
              */
             char *delay_s = crm_itoa(node_priority_fencing_delay(node, data_set));
 
             g_hash_table_insert(stonith_op->meta,
                                 strdup(XML_CONFIG_ATTR_PRIORITY_FENCING_DELAY),
                                 delay_s);
     }
 
     if(optional == FALSE && pe_can_fence(data_set, node)) {
         pe_action_required(stonith_op, NULL, reason);
     } else if(reason && stonith_op->reason == NULL) {
         stonith_op->reason = strdup(reason);
     }
 
     return stonith_op;
 }
 
 void
 trigger_unfencing(
     pe_resource_t * rsc, pe_node_t *node, const char *reason, pe_action_t *dependency, pe_working_set_t * data_set) 
 {
     if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) {
         /* No resources require it */
         return;
 
     } else if (rsc != NULL && is_not_set(rsc->flags, pe_rsc_fence_device)) {
         /* Wasn't a stonith device */
         return;
 
     } else if(node
               && node->details->online
               && node->details->unclean == FALSE
               && node->details->shutdown == FALSE) {
         pe_action_t *unfence = pe_fence_op(node, "on", FALSE, reason, FALSE, data_set);
 
         if(dependency) {
             order_actions(unfence, dependency, pe_order_optional);
         }
 
     } else if(rsc) {
         GHashTableIter iter;
 
         g_hash_table_iter_init(&iter, rsc->allowed_nodes);
         while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
             if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) {
                 trigger_unfencing(rsc, node, reason, dependency, data_set);
             }
         }
     }
 }
 
 gboolean
 add_tag_ref(GHashTable * tags, const char * tag_name,  const char * obj_ref)
 {
     pe_tag_t *tag = NULL;
     GListPtr gIter = NULL;
     gboolean is_existing = FALSE;
 
     CRM_CHECK(tags && tag_name && obj_ref, return FALSE);
 
     tag = g_hash_table_lookup(tags, tag_name);
     if (tag == NULL) {
         tag = calloc(1, sizeof(pe_tag_t));
         if (tag == NULL) {
             return FALSE;
         }
         tag->id = strdup(tag_name);
         tag->refs = NULL;
         g_hash_table_insert(tags, strdup(tag_name), tag);
     }
 
     for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
         const char *existing_ref = (const char *) gIter->data;
 
         if (crm_str_eq(existing_ref, obj_ref, TRUE)){
             is_existing = TRUE;
             break;
         }
     }
 
     if (is_existing == FALSE) {
         tag->refs = g_list_append(tag->refs, strdup(obj_ref));
         crm_trace("Added: tag=%s ref=%s", tag->id, obj_ref);
     }
 
     return TRUE;
 }
 
 void pe_action_set_flag_reason(const char *function, long line,
                                pe_action_t *action, pe_action_t *reason, const char *text,
                                enum pe_action_flags flags, bool overwrite)
 {
     bool unset = FALSE;
     bool update = FALSE;
     const char *change = NULL;
 
     if(is_set(flags, pe_action_runnable)) {
         unset = TRUE;
         change = "unrunnable";
     } else if(is_set(flags, pe_action_optional)) {
         unset = TRUE;
         change = "required";
     } else if(is_set(flags, pe_action_migrate_runnable)) {
         unset = TRUE;
         overwrite = TRUE;
         change = "unrunnable";
     } else if(is_set(flags, pe_action_dangle)) {
         change = "dangling";
     } else if(is_set(flags, pe_action_requires_any)) {
         change = "required";
     } else {
         crm_err("Unknown flag change to %x by %s: 0x%s",
                 flags, action->uuid, (reason? reason->uuid : "0"));
     }
 
     if(unset) {
         if(is_set(action->flags, flags)) {
             action->flags = crm_clear_bit(function, line, action->uuid, action->flags, flags);
             update = TRUE;
         }
 
     } else {
         if(is_not_set(action->flags, flags)) {
             action->flags = crm_set_bit(function, line, action->uuid, action->flags, flags);
             update = TRUE;
         }
     }
 
     if((change && update) || text) {
         char *reason_text = NULL;
         if(reason == NULL) {
             pe_action_set_reason(action, text, overwrite);
 
         } else if(reason->rsc == NULL) {
             reason_text = crm_strdup_printf("%s %s%c %s", change, reason->task, text?':':0, text?text:"");
         } else {
             reason_text = crm_strdup_printf("%s %s %s%c %s", change, reason->rsc->id, reason->task, text?':':0, text?text:"NA");
         }
 
         if(reason_text && action->rsc != reason->rsc) {
             pe_action_set_reason(action, reason_text, overwrite);
         }
         free(reason_text);
     }
  }
 
 void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrite) 
 {
     if (action->reason != NULL && overwrite) {
         pe_rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'",
                      action->uuid, action->reason, crm_str(reason));
         free(action->reason);
     } else if (action->reason == NULL) {
         pe_rsc_trace(action->rsc, "Set %s reason to '%s'",
                      action->uuid, crm_str(reason));
     } else {
         // crm_assert(action->reason != NULL && !overwrite);
         return;
     }
 
     if (reason != NULL) {
         action->reason = strdup(reason);
     } else {
         action->reason = NULL;
     }
 }
 
 /*!
  * \internal
  * \brief Check whether shutdown has been requested for a node
  *
  * \param[in] node  Node to check
  *
  * \return TRUE if node has shutdown attribute set and nonzero, FALSE otherwise
  * \note This differs from simply using node->details->shutdown in that it can
  *       be used before that has been determined (and in fact to determine it),
  *       and it can also be used to distinguish requested shutdown from implicit
  *       shutdown of remote nodes by virtue of their connection stopping.
  */
 bool
 pe__shutdown_requested(pe_node_t *node)
 {
     const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
 
     return shutdown && strcmp(shutdown, "0");
 }
 
 /*!
  * \internal
  * \brief Update a data set's "recheck by" time
  *
  * \param[in]     recheck   Epoch time when recheck should happen
  * \param[in,out] data_set  Current working set
  */
 void
 pe__update_recheck_time(time_t recheck, pe_working_set_t *data_set)
 {
     if ((recheck > get_effective_time(data_set))
         && ((data_set->recheck_by == 0)
             || (data_set->recheck_by > recheck))) {
         data_set->recheck_by = recheck;
     }
 }
 
 /*!
  * \internal
  * \brief Wrapper for pe_unpack_nvpairs() using a cluster working set
  */
 void
 pe__unpack_dataset_nvpairs(xmlNode *xml_obj, const char *set_name,
                            GHashTable *node_hash, GHashTable *hash,
                            const char *always_first, gboolean overwrite,
                            pe_working_set_t *data_set)
 {
     crm_time_t *next_change = crm_time_new_undefined();
 
     pe_unpack_nvpairs(data_set->input, xml_obj, set_name, node_hash, hash,
                       always_first, overwrite, data_set->now, next_change);
     if (crm_time_is_defined(next_change)) {
         time_t recheck = (time_t) crm_time_get_seconds_since_epoch(next_change);
 
         pe__update_recheck_time(recheck, data_set);
     }
     crm_time_free(next_change);
 }
 
 bool
 pe__resource_is_disabled(pe_resource_t *rsc)
 {
     const char *target_role = NULL;
 
     CRM_CHECK(rsc != NULL, return false);
     target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
     if (target_role) {
         enum rsc_role_e target_role_e = text2role(target_role);
 
         if ((target_role_e == RSC_ROLE_STOPPED)
             || ((target_role_e == RSC_ROLE_SLAVE)
                 && is_set(uber_parent(rsc)->flags, pe_rsc_promotable))) {
             return true;
         }
     }
     return false;
 }
 
 /*!
  * \internal
  * \brief Create an action to clear a resource's history from CIB
  *
  * \param[in] rsc   Resource to clear
  * \param[in] node  Node to clear history on
  *
  * \return New action to clear resource history
  */
 pe_action_t *
 pe__clear_resource_history(pe_resource_t *rsc, pe_node_t *node,
                            pe_working_set_t *data_set)
 {
     char *key = NULL;
 
     CRM_ASSERT(rsc && node);
     key = pcmk__op_key(rsc->id, CRM_OP_LRM_DELETE, 0);
     return custom_action(rsc, key, CRM_OP_LRM_DELETE, node, FALSE, TRUE,
                          data_set);
 }