diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
index 7c0232c933..affbb25275 100644
--- a/include/crm/pengine/internal.h
+++ b/include/crm/pengine/internal.h
@@ -1,305 +1,313 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 #ifndef PE_INTERNAL__H
 #  define PE_INTERNAL__H
 #  include <crm/pengine/status.h>
 #  include <crm/pengine/remote.h>
 
 #  define pe_rsc_info(rsc, fmt, args...)  crm_log_tag(LOG_INFO,  rsc ? rsc->id : "<NULL>", fmt, ##args)
 #  define pe_rsc_debug(rsc, fmt, args...) crm_log_tag(LOG_DEBUG, rsc ? rsc->id : "<NULL>", fmt, ##args)
 #  define pe_rsc_trace(rsc, fmt, args...) crm_log_tag(LOG_TRACE, rsc ? rsc->id : "<NULL>", fmt, ##args)
 
 #  define pe_err(fmt...) { was_processing_error = TRUE; crm_config_error = TRUE; crm_err(fmt); }
 #  define pe_warn(fmt...) { was_processing_warning = TRUE; crm_config_warning = TRUE; crm_warn(fmt); }
 #  define pe_proc_err(fmt...) { was_processing_error = TRUE; crm_err(fmt); }
 #  define pe_proc_warn(fmt...) { was_processing_warning = TRUE; crm_warn(fmt); }
 #  define pe_set_action_bit(action, bit) action->flags = crm_set_bit(__FUNCTION__, __LINE__, action->uuid, action->flags, bit)
 #  define pe_clear_action_bit(action, bit) action->flags = crm_clear_bit(__FUNCTION__, __LINE__, action->uuid, action->flags, bit)
 
 typedef struct notify_data_s {
     GHashTable *keys;
 
     const char *action;
 
     action_t *pre;
     action_t *post;
     action_t *pre_done;
     action_t *post_done;
 
     GListPtr active;            /* notify_entry_t*  */
     GListPtr inactive;          /* notify_entry_t*  */
     GListPtr start;             /* notify_entry_t*  */
     GListPtr stop;              /* notify_entry_t*  */
     GListPtr demote;            /* notify_entry_t*  */
     GListPtr promote;           /* notify_entry_t*  */
     GListPtr master;            /* notify_entry_t*  */
     GListPtr slave;             /* notify_entry_t*  */
     GHashTable *allowed_nodes;
 
 } notify_data_t;
 
 bool pe_can_fence(pe_working_set_t *data_set, node_t *node);
 
 int merge_weights(int w1, int w2);
 void add_hash_param(GHashTable * hash, const char *name, const char *value);
 void append_hashtable(gpointer key, gpointer value, gpointer user_data);
 
 char *native_parameter(resource_t * rsc, node_t * node, gboolean create, const char *name,
                        pe_working_set_t * data_set);
 node_t *native_location(resource_t * rsc, GListPtr * list, gboolean current);
 
 void pe_metadata(void);
 void verify_pe_options(GHashTable * options);
 
 void common_update_score(resource_t * rsc, const char *id, int score);
 void native_add_running(resource_t * rsc, node_t * node, pe_working_set_t * data_set);
 node_t *rsc_known_on(resource_t * rsc, GListPtr * list);
 
 gboolean native_unpack(resource_t * rsc, pe_working_set_t * data_set);
 gboolean group_unpack(resource_t * rsc, pe_working_set_t * data_set);
 gboolean clone_unpack(resource_t * rsc, pe_working_set_t * data_set);
 gboolean master_unpack(resource_t * rsc, pe_working_set_t * data_set);
 gboolean container_unpack(resource_t * rsc, pe_working_set_t * data_set);
 
 resource_t *native_find_rsc(resource_t * rsc, const char *id, node_t * node, int flags);
 
 gboolean native_active(resource_t * rsc, gboolean all);
 gboolean group_active(resource_t * rsc, gboolean all);
 gboolean clone_active(resource_t * rsc, gboolean all);
 gboolean master_active(resource_t * rsc, gboolean all);
 gboolean container_active(resource_t * rsc, gboolean all);
 
 void native_print(resource_t * rsc, const char *pre_text, long options, void *print_data);
 void group_print(resource_t * rsc, const char *pre_text, long options, void *print_data);
 void clone_print(resource_t * rsc, const char *pre_text, long options, void *print_data);
 void master_print(resource_t * rsc, const char *pre_text, long options, void *print_data);
 void container_print(resource_t * rsc, const char *pre_text, long options, void *print_data);
 
 void native_free(resource_t * rsc);
 void group_free(resource_t * rsc);
 void clone_free(resource_t * rsc);
 void master_free(resource_t * rsc);
 void container_free(resource_t * rsc);
 
 enum rsc_role_e native_resource_state(const resource_t * rsc, gboolean current);
 enum rsc_role_e group_resource_state(const resource_t * rsc, gboolean current);
 enum rsc_role_e clone_resource_state(const resource_t * rsc, gboolean current);
 enum rsc_role_e master_resource_state(const resource_t * rsc, gboolean current);
 enum rsc_role_e container_resource_state(const resource_t * rsc, gboolean current);
 
 gboolean common_unpack(xmlNode * xml_obj, resource_t ** rsc, resource_t * parent,
                        pe_working_set_t * data_set);
 void common_free(resource_t * rsc);
 
 extern pe_working_set_t *pe_dataset;
 
 extern node_t *node_copy(const node_t *this_node);
 extern time_t get_effective_time(pe_working_set_t * data_set);
 
 /* Failure handling utilities (from failcounts.c) */
+
+// bit flags for fail count handling options
+enum pe_fc_flags_e {
+    pe_fc_default   = 0x00,
+    pe_fc_effective = 0x01, // don't count expired failures
+};
+
 int pe_get_failcount(node_t *node, resource_t *rsc, time_t *last_failure,
-                     bool effective, xmlNode *xml_op, pe_working_set_t *data_set);
+                     uint32_t flags, xmlNode *xml_op,
+                     pe_working_set_t *data_set);
 extern int get_failcount_all(node_t * node, resource_t * rsc, time_t *last_failure,
                              pe_working_set_t * data_set);
 
 /* Binary like operators for lists of nodes */
 extern void node_list_exclude(GHashTable * list, GListPtr list2, gboolean merge_scores);
 extern GListPtr node_list_dup(GListPtr list, gboolean reset, gboolean filter);
 extern GListPtr node_list_from_hash(GHashTable * hash, gboolean reset, gboolean filter);
 
 extern GHashTable *node_hash_from_list(GListPtr list);
 static inline gpointer
 pe_hash_table_lookup(GHashTable * hash, gconstpointer key)
 {
     if (hash) {
         return g_hash_table_lookup(hash, key);
     }
     return NULL;
 }
 
 extern action_t *get_pseudo_op(const char *name, pe_working_set_t * data_set);
 extern gboolean order_actions(action_t * lh_action, action_t * rh_action, enum pe_ordering order);
 
 GHashTable *node_hash_dup(GHashTable * hash);
 extern GListPtr node_list_and(GListPtr list1, GListPtr list2, gboolean filter);
 
 extern GListPtr node_list_xor(GListPtr list1, GListPtr list2, gboolean filter);
 
 extern GListPtr node_list_minus(GListPtr list1, GListPtr list2, gboolean filter);
 
 extern void pe_free_shallow(GListPtr alist);
 extern void pe_free_shallow_adv(GListPtr alist, gboolean with_data);
 
 /* Printing functions for debug */
 extern void print_node(const char *pre_text, node_t * node, gboolean details);
 
 extern void print_resource(int log_level, const char *pre_text, resource_t * rsc, gboolean details);
 
 extern void dump_node_scores_worker(int level, const char *file, const char *function, int line,
                                     resource_t * rsc, const char *comment, GHashTable * nodes);
 
 extern void dump_node_capacity(int level, const char *comment, node_t * node);
 extern void dump_rsc_utilization(int level, const char *comment, resource_t * rsc, node_t * node);
 
 #  define dump_node_scores(level, rsc, text, nodes) do {		\
         dump_node_scores_worker(level, __FILE__, __FUNCTION__, __LINE__, rsc, text, nodes); \
     } while(0)
 
 /* Sorting functions */
 extern gint sort_rsc_priority(gconstpointer a, gconstpointer b);
 extern gint sort_rsc_index(gconstpointer a, gconstpointer b);
 
 extern xmlNode *find_rsc_op_entry(resource_t * rsc, const char *key);
 
 extern action_t *custom_action(resource_t * rsc, char *key, const char *task, node_t * on_node,
                                gboolean optional, gboolean foo, pe_working_set_t * data_set);
 
 #  define delete_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_DELETE, 0)
 #  define delete_action(rsc, node, optional) custom_action(		\
 		rsc, delete_key(rsc), CRMD_ACTION_DELETE, node,		\
 		optional, TRUE, data_set);
 
 #  define stopped_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_STOPPED, 0)
 #  define stopped_action(rsc, node, optional) custom_action(		\
 		rsc, stopped_key(rsc), CRMD_ACTION_STOPPED, node,	\
 		optional, TRUE, data_set);
 
 #  define stop_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_STOP, 0)
 #  define stop_action(rsc, node, optional) custom_action(			\
 		rsc, stop_key(rsc), CRMD_ACTION_STOP, node,		\
 		optional, TRUE, data_set);
 
 #  define reload_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_RELOAD, 0)
 #  define start_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_START, 0)
 #  define start_action(rsc, node, optional) custom_action(		\
 		rsc, start_key(rsc), CRMD_ACTION_START, node,		\
 		optional, TRUE, data_set)
 
 #  define started_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_STARTED, 0)
 #  define started_action(rsc, node, optional) custom_action(		\
 		rsc, started_key(rsc), CRMD_ACTION_STARTED, node,	\
 		optional, TRUE, data_set)
 
 #  define promote_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_PROMOTE, 0)
 #  define promote_action(rsc, node, optional) custom_action(		\
 		rsc, promote_key(rsc), CRMD_ACTION_PROMOTE, node,	\
 		optional, TRUE, data_set)
 
 #  define promoted_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_PROMOTED, 0)
 #  define promoted_action(rsc, node, optional) custom_action(		\
 		rsc, promoted_key(rsc), CRMD_ACTION_PROMOTED, node,	\
 		optional, TRUE, data_set)
 
 #  define demote_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_DEMOTE, 0)
 #  define demote_action(rsc, node, optional) custom_action(		\
 		rsc, demote_key(rsc), CRMD_ACTION_DEMOTE, node,		\
 		optional, TRUE, data_set)
 
 #  define demoted_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_DEMOTED, 0)
 #  define demoted_action(rsc, node, optional) custom_action(		\
 		rsc, demoted_key(rsc), CRMD_ACTION_DEMOTED, node,	\
 		optional, TRUE, data_set)
 
 extern int pe_get_configured_timeout(resource_t *rsc, const char *action,
                                      pe_working_set_t *data_set);
 
 extern action_t *find_first_action(GListPtr input, const char *uuid, const char *task,
                                    node_t * on_node);
 extern enum action_tasks get_complex_task(resource_t * rsc, const char *name,
                                           gboolean allow_non_atomic);
 
 extern GListPtr find_actions(GListPtr input, const char *key, const node_t *on_node);
 extern GListPtr find_actions_exact(GListPtr input, const char *key, node_t * on_node);
 extern GListPtr find_recurring_actions(GListPtr input, node_t * not_on_node);
 
 extern void pe_free_action(action_t * action);
 
 extern void resource_location(resource_t * rsc, node_t * node, int score, const char *tag,
                               pe_working_set_t * data_set);
 
 extern gint sort_op_by_callid(gconstpointer a, gconstpointer b);
 extern gboolean get_target_role(resource_t * rsc, enum rsc_role_e *role);
 
 extern resource_t *find_clone_instance(resource_t * rsc, const char *sub_id,
                                        pe_working_set_t * data_set);
 
 extern void destroy_ticket(gpointer data);
 extern ticket_t *ticket_new(const char *ticket_id, pe_working_set_t * data_set);
 
 char *clone_strip(const char *last_rsc_id);
 char *clone_zero(const char *last_rsc_id);
 
 int get_target_rc(xmlNode * xml_op);
 
 gint sort_node_uname(gconstpointer a, gconstpointer b);
 bool is_set_recursive(resource_t * rsc, long long flag, bool any);
 
 enum rsc_digest_cmp_val {
     /*! Digests are the same */
     RSC_DIGEST_MATCH = 0,
     /*! Params that require a restart changed */
     RSC_DIGEST_RESTART,
     /*! Some parameter changed.  */
     RSC_DIGEST_ALL,
     /*! rsc op didn't have a digest associated with it, so
      *  it is unknown if parameters changed or not. */
     RSC_DIGEST_UNKNOWN,
 };
 
 typedef struct op_digest_cache_s {
     enum rsc_digest_cmp_val rc;
     xmlNode *params_all;
     xmlNode *params_secure;
     xmlNode *params_restart;
     char *digest_all_calc;
     char *digest_secure_calc;
     char *digest_restart_calc;
 } op_digest_cache_t;
 
 op_digest_cache_t *rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node,
                                          pe_working_set_t * data_set);
 
 action_t *pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe_working_set_t * data_set);
 void trigger_unfencing(
     resource_t * rsc, node_t *node, const char *reason, action_t *dependency, pe_working_set_t * data_set);
 
 void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrite);
 void pe_action_set_flag_reason(const char *function, long line, pe_action_t *action, pe_action_t *reason, const char *text, enum pe_action_flags flags, bool overwrite);
 
 #define pe_action_required(action, reason, text) pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, reason, text, pe_action_optional, FALSE)
 #define pe_action_implies(action, reason, flag) pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, reason, NULL, flag, FALSE)
 
 void set_bit_recursive(resource_t * rsc, unsigned long long flag);
 void clear_bit_recursive(resource_t * rsc, unsigned long long flag);
 
 gboolean add_tag_ref(GHashTable * tags, const char * tag_name,  const char * obj_ref);
 
 void print_rscs_brief(GListPtr rsc_list, const char * pre_text, long options,
                       void * print_data, gboolean print_all);
 void pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason);
 
 node_t *pe_create_node(const char *id, const char *uname, const char *type,
                        const char *score, pe_working_set_t * data_set);
 bool remote_id_conflict(const char *remote_name, pe_working_set_t *data);
 void common_print(resource_t * rsc, const char *pre_text, const char *name, node_t *node, long options, void *print_data);
 resource_t *find_container_child(const char *stem, resource_t * rsc, node_t *node);
 bool fix_remote_addr(resource_t * rsc);
 const char *pe_node_attribute_calculated(pe_node_t *node, const char *name, resource_t *rsc);
 const char *pe_node_attribute_raw(pe_node_t *node, const char *name);
 
 #endif
diff --git a/lib/pengine/failcounts.c b/lib/pengine/failcounts.c
index efc70f38be..53bc0f8d76 100644
--- a/lib/pengine/failcounts.c
+++ b/lib/pengine/failcounts.c
@@ -1,333 +1,335 @@
 /*
  * Copyright (C) 2008-2017 Andrew Beekhof <andrew@beekhof.net>
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/types.h>
 #include <regex.h>
 #include <glib.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/util.h>
 #include <crm/pengine/internal.h>
 
 static gboolean
 is_matched_failure(const char *rsc_id, xmlNode *conf_op_xml,
                    xmlNode *lrm_op_xml)
 {
     gboolean matched = FALSE;
     const char *conf_op_name = NULL;
     int conf_op_interval = 0;
     const char *lrm_op_task = NULL;
     int lrm_op_interval = 0;
     const char *lrm_op_id = NULL;
     char *last_failure_key = NULL;
 
     if (rsc_id == NULL || conf_op_xml == NULL || lrm_op_xml == NULL) {
         return FALSE;
     }
 
     conf_op_name = crm_element_value(conf_op_xml, "name");
     conf_op_interval = crm_get_msec(crm_element_value(conf_op_xml, "interval"));
     lrm_op_task = crm_element_value(lrm_op_xml, XML_LRM_ATTR_TASK);
     crm_element_value_int(lrm_op_xml, XML_LRM_ATTR_INTERVAL, &lrm_op_interval);
 
     if (safe_str_eq(conf_op_name, lrm_op_task) == FALSE
         || conf_op_interval != lrm_op_interval) {
         return FALSE;
     }
 
     lrm_op_id = ID(lrm_op_xml);
     last_failure_key = generate_op_key(rsc_id, "last_failure", 0);
 
     if (safe_str_eq(last_failure_key, lrm_op_id)) {
         matched = TRUE;
 
     } else {
         char *expected_op_key = generate_op_key(rsc_id, conf_op_name,
                                                 conf_op_interval);
 
         if (safe_str_eq(expected_op_key, lrm_op_id)) {
             int rc = 0;
             int target_rc = get_target_rc(lrm_op_xml);
 
             crm_element_value_int(lrm_op_xml, XML_LRM_ATTR_RC, &rc);
             if (rc != target_rc) {
                 matched = TRUE;
             }
         }
         free(expected_op_key);
     }
 
     free(last_failure_key);
     return matched;
 }
 
 static gboolean
 block_failure(node_t *node, resource_t *rsc, xmlNode *xml_op,
               pe_working_set_t *data_set)
 {
     char *xml_name = clone_strip(rsc->id);
     char *xpath = crm_strdup_printf("//primitive[@id='%s']//op[@on-fail='block']",
                                     xml_name);
     xmlXPathObject *xpathObj = xpath_search(rsc->xml, xpath);
     gboolean should_block = FALSE;
 
     free(xpath);
 
 #if 0
     /* A good idea? */
     if (rsc->container == NULL && is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
         /* In this case, stop on-fail defaults to block in unpack_operation() */
         return TRUE;
     }
 #endif
 
     if (xpathObj) {
         int max = numXpathResults(xpathObj);
         int lpc = 0;
 
         for (lpc = 0; lpc < max; lpc++) {
             xmlNode *pref = getXpathResult(xpathObj, lpc);
 
             if (xml_op) {
                 should_block = is_matched_failure(xml_name, pref, xml_op);
                 if (should_block) {
                     break;
                 }
 
             } else {
                 const char *conf_op_name = NULL;
                 int conf_op_interval = 0;
                 char *lrm_op_xpath = NULL;
                 xmlXPathObject *lrm_op_xpathObj = NULL;
 
                 conf_op_name = crm_element_value(pref, "name");
                 conf_op_interval = crm_get_msec(crm_element_value(pref, "interval"));
 
                 lrm_op_xpath = crm_strdup_printf("//node_state[@uname='%s']"
                                                "//lrm_resource[@id='%s']"
                                                "/lrm_rsc_op[@operation='%s'][@interval='%d']",
                                                node->details->uname, xml_name,
                                                conf_op_name, conf_op_interval);
                 lrm_op_xpathObj = xpath_search(data_set->input, lrm_op_xpath);
 
                 free(lrm_op_xpath);
 
                 if (lrm_op_xpathObj) {
                     int max2 = numXpathResults(lrm_op_xpathObj);
                     int lpc2 = 0;
 
                     for (lpc2 = 0; lpc2 < max2; lpc2++) {
                         xmlNode *lrm_op_xml = getXpathResult(lrm_op_xpathObj,
                                                              lpc2);
 
                         should_block = is_matched_failure(xml_name, pref,
                                                           lrm_op_xml);
                         if (should_block) {
                             break;
                         }
                     }
                 }
                 freeXpathObject(lrm_op_xpathObj);
 
                 if (should_block) {
                     break;
                 }
             }
         }
     }
 
     free(xml_name);
     freeXpathObject(xpathObj);
 
     return should_block;
 }
 
 /*!
  * \internal
  * \brief Get resource name as used in failure-related node attributes
  *
  * \param[in] rsc  Resource to check
  *
  * \return Newly allocated string containing resource's fail name
  * \note The caller is responsible for freeing the result.
  */
 static inline char *
 rsc_fail_name(resource_t *rsc)
 {
     const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);
 
     return is_set(rsc->flags, pe_rsc_unique)? strdup(name) : clone_strip(name);
 }
 
 /*!
  * \internal
  * \brief Compile regular expression to match a failure-related node attribute
  *
  * \param[in]  prefix    Attribute prefix to match
  * \param[in]  rsc_name  Resource name to match as used in failure attributes
  * \param[in]  is_legacy Whether DC uses per-resource fail counts
  * \param[in]  is_unique Whether the resource is a globally unique clone
  * \param[out] re        Where to store resulting regular expression
  *
  * \note Fail attributes are named like PREFIX-RESOURCE#OP_INTERVAL.
  *       The caller is responsible for freeing re with regfree().
  */
 static void
 generate_fail_regex(const char *prefix, const char *rsc_name,
                     gboolean is_legacy, gboolean is_unique, regex_t *re)
 {
     char *pattern;
 
     /* @COMPAT DC < 1.1.17: Fail counts used to be per-resource rather than
      * per-operation.
      */
     const char *op_pattern = (is_legacy? "" : "#.+_[0-9]+");
 
     /* Ignore instance numbers for anything other than globally unique clones.
      * Anonymous clone fail counts could contain an instance number if the
      * clone was initially unique, failed, then was converted to anonymous.
      * @COMPAT Also, before 1.1.8, anonymous clone fail counts always contained
      * clone instance numbers.
      */
     const char *instance_pattern = (is_unique? "" : "(:[0-9]+)?");
 
     pattern = crm_strdup_printf("^%s-%s%s%s$", prefix, rsc_name,
                                 instance_pattern, op_pattern);
     CRM_LOG_ASSERT(regcomp(re, pattern, REG_EXTENDED|REG_NOSUB) == 0);
     free(pattern);
 }
 
 /*!
  * \internal
  * \brief Compile regular expressions to match failure-related node attributes
  *
  * \param[in]  rsc             Resource being checked for failures
  * \param[in]  data_set        Data set (for CRM feature set version)
  * \param[out] failcount_re    Storage for regular expression for fail count
  * \param[out] lastfailure_re  Storage for regular expression for last failure
  *
  * \note The caller is responsible for freeing the expressions with regfree().
  */
 static void
 generate_fail_regexes(resource_t *rsc, pe_working_set_t *data_set,
                       regex_t *failcount_re, regex_t *lastfailure_re)
 {
     char *rsc_name = rsc_fail_name(rsc);
     const char *version = crm_element_value(data_set->input, XML_ATTR_CRM_VERSION);
     gboolean is_legacy = (compare_version(version, "3.0.13") < 0);
 
     generate_fail_regex(CRM_FAIL_COUNT_PREFIX, rsc_name, is_legacy,
                         is_set(rsc->flags, pe_rsc_unique), failcount_re);
 
     generate_fail_regex(CRM_LAST_FAILURE_PREFIX, rsc_name, is_legacy,
                         is_set(rsc->flags, pe_rsc_unique), lastfailure_re);
 
     free(rsc_name);
 }
 
 int
 pe_get_failcount(node_t *node, resource_t *rsc, time_t *last_failure,
-                 bool effective, xmlNode *xml_op, pe_working_set_t *data_set)
+                 uint32_t flags, xmlNode *xml_op, pe_working_set_t *data_set)
 {
     char *key = NULL;
     const char *value = NULL;
     regex_t failcount_re, lastfailure_re;
     int failcount = 0;
     time_t last = 0;
     GHashTableIter iter;
 
     generate_fail_regexes(rsc, data_set, &failcount_re, &lastfailure_re);
 
     /* Resource fail count is sum of all matching operation fail counts */
     g_hash_table_iter_init(&iter, node->details->attrs);
     while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
         if (regexec(&failcount_re, key, 0, NULL, 0) == 0) {
             failcount = merge_weights(failcount, char2score(value));
         } else if (regexec(&lastfailure_re, key, 0, NULL, 0) == 0) {
             last = QB_MAX(last, crm_int_helper(value, NULL));
         }
     }
 
     regfree(&failcount_re);
     regfree(&lastfailure_re);
 
     if ((failcount > 0) && (last > 0) && (last_failure != NULL)) {
         *last_failure = last;
     }
 
     /* If failure blocks the resource, disregard any failure timeout */
     if ((failcount > 0) && rsc->failure_timeout
         && block_failure(node, rsc, xml_op, data_set)) {
 
         pe_warn("Ignoring failure timeout %d for %s because it conflicts with on-fail=block",
                 rsc->id, rsc->failure_timeout);
         rsc->failure_timeout = 0;
     }
 
     /* If all failures have expired, ignore fail count */
-    if (effective && (failcount > 0) && (last > 0) && rsc->failure_timeout) {
+    if (is_set(flags, pe_fc_effective) && (failcount > 0) && (last > 0)
+        && rsc->failure_timeout) {
+
         time_t now = get_effective_time(data_set);
 
         if (now > (last + rsc->failure_timeout)) {
             crm_debug("Failcount for %s on %s expired after %ds",
                       rsc->id, node->details->uname, rsc->failure_timeout);
             failcount = 0;
         }
     }
 
     if (failcount > 0) {
         char *score = score2char(failcount);
 
         crm_info("%s has failed %s times on %s",
                  rsc->id, score, node->details->uname);
         free(score);
     }
 
     return failcount;
 }
 
 /* If it's a resource container, get its failcount plus all the failcounts of
  * the resources within it
  */
 int
 get_failcount_all(node_t *node, resource_t *rsc, time_t *last_failure,
                   pe_working_set_t *data_set)
 {
-    int failcount_all = 0;
-
-    failcount_all = pe_get_failcount(node, rsc, last_failure, TRUE, NULL, data_set);
+    int failcount_all = pe_get_failcount(node, rsc, last_failure,
+                                         pe_fc_effective, NULL, data_set);
 
     if (rsc->fillers) {
         GListPtr gIter = NULL;
 
         for (gIter = rsc->fillers; gIter != NULL; gIter = gIter->next) {
             resource_t *filler = (resource_t *) gIter->data;
             time_t filler_last_failure = 0;
 
-            failcount_all += pe_get_failcount(node, filler, &filler_last_failure,
-                                              TRUE, NULL, data_set);
+            failcount_all += pe_get_failcount(node, filler,
+                                              &filler_last_failure,
+                                              pe_fc_effective, NULL, data_set);
 
             if (last_failure && filler_last_failure > *last_failure) {
                 *last_failure = filler_last_failure;
             }
         }
 
         if (failcount_all != 0) {
             char *score = score2char(failcount_all);
 
             crm_info("Container %s and the resources within it have failed %s times on %s",
                      rsc->id, score, node->details->uname);
             free(score);
         }
     }
 
     return failcount_all;
 }
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index c6232205a7..95c3c83555 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -1,3453 +1,3455 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 #include <crm_internal.h>
 
 #include <glib.h>
 
 #include <crm/crm.h>
 #include <crm/services.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crm/common/util.h>
 #include <crm/pengine/rules.h>
 #include <crm/pengine/internal.h>
 #include <unpack.h>
 
 CRM_TRACE_INIT_DATA(pe_status);
 
 #define set_config_flag(data_set, option, flag) do {			\
 	const char *tmp = pe_pref(data_set->config_hash, option);	\
 	if(tmp) {							\
 	    if(crm_is_true(tmp)) {					\
 		set_bit(data_set->flags, flag);			\
 	    } else {							\
 		clear_bit(data_set->flags, flag);		\
 	    }								\
 	}								\
     } while(0)
 
 gboolean unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, xmlNode ** last_failure,
                        enum action_fail_response *failed, pe_working_set_t * data_set);
 static gboolean determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node);
 
 // Bitmask for warnings we only want to print once
 uint32_t pe_wo = 0;
 
 static gboolean
 is_dangling_container_remote_node(node_t *node)
 {
     /* we are looking for a remote-node that was supposed to be mapped to a
      * container resource, but all traces of that container have disappeared 
      * from both the config and the status section. */
     if (is_remote_node(node) &&
         node->details->remote_rsc &&
         node->details->remote_rsc->container == NULL &&
         is_set(node->details->remote_rsc->flags, pe_rsc_orphan_container_filler)) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 
 /*!
  * \brief Schedule a fence action for a node
  *
  * \param[in,out] data_set  Current working set of cluster
  * \param[in,out] node      Node to fence
  * \param[in]     reason    Text description of why fencing is needed
  */
 void
 pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason)
 {
     CRM_CHECK(node, return);
 
     /* A guest node is fenced by marking its container as failed */
     if (is_container_remote_node(node)) {
         resource_t *rsc = node->details->remote_rsc->container;
 
         if (is_set(rsc->flags, pe_rsc_failed) == FALSE) {
             if (!is_set(rsc->flags, pe_rsc_managed)) {
                 crm_notice("Not fencing guest node %s "
                            "(otherwise would because %s): "
                            "its guest resource %s is unmanaged",
                            node->details->uname, reason, rsc->id);
             } else {
                 crm_warn("Guest node %s will be fenced "
                          "(by recovering its guest resource %s): %s",
                          node->details->uname, rsc->id, reason);
 
                 /* We don't mark the node as unclean because that would prevent the
                  * node from running resources. We want to allow it to run resources
                  * in this transition if the recovery succeeds.
                  */
                 node->details->remote_requires_reset = TRUE;
                 set_bit(rsc->flags, pe_rsc_failed);
             }
         }
 
     } else if (is_dangling_container_remote_node(node)) {
         crm_info("Cleaning up dangling connection for guest node %s: "
                  "fencing was already done because %s, "
                  "and guest resource no longer exists",
                  node->details->uname, reason);
         set_bit(node->details->remote_rsc->flags, pe_rsc_failed);
 
     } else if (is_baremetal_remote_node(node)) {
         resource_t *rsc = node->details->remote_rsc;
 
         if (rsc && (!is_set(rsc->flags, pe_rsc_managed))) {
             crm_notice("Not fencing remote node %s "
                        "(otherwise would because %s): connection is unmanaged",
                        node->details->uname, reason);
         } else if(node->details->remote_requires_reset == FALSE) {
             node->details->remote_requires_reset = TRUE;
             crm_warn("Remote node %s %s: %s",
                      node->details->uname,
                      pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
                      reason);
         }
         node->details->unclean = TRUE;
         pe_fence_op(node, NULL, TRUE, reason, data_set);
 
     } else if (node->details->unclean) {
         crm_trace("Cluster node %s %s because %s",
                   node->details->uname,
                   pe_can_fence(data_set, node)? "would also be fenced" : "also is unclean",
                   reason);
 
     } else {
         crm_warn("Cluster node %s %s: %s",
                  node->details->uname,
                  pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
                  reason);
         node->details->unclean = TRUE;
         pe_fence_op(node, NULL, TRUE, reason, data_set);
     }
 }
 
 // @TODO xpaths can't handle templates, rules, or id-refs
 
 // nvpair with provides or requires set to unfencing
 #define XPATH_UNFENCING_NVPAIR XML_CIB_TAG_NVPAIR                \
     "[(@" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_PROVIDES "'"    \
     "or @" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_REQUIRES "') " \
     "and @" XML_NVPAIR_ATTR_VALUE "='unfencing']"
 
 // unfencing in rsc_defaults or any resource
 #define XPATH_ENABLE_UNFENCING \
     "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RESOURCES   \
     "//" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR                                               \
     "|/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RSCCONFIG  \
     "/" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR
 
 static
 void set_if_xpath(unsigned long long flag, const char *xpath,
                   pe_working_set_t *data_set)
 {
     xmlXPathObjectPtr result = NULL;
 
     if (is_not_set(data_set->flags, flag)) {
         result = xpath_search(data_set->input, xpath);
         if (result && (numXpathResults(result) > 0)) {
             set_bit(data_set->flags, flag);
         }
         freeXpathObject(result);
     }
 }
 
 gboolean
 unpack_config(xmlNode * config, pe_working_set_t * data_set)
 {
     const char *value = NULL;
     GHashTable *config_hash = crm_str_table_new();
 
     data_set->config_hash = config_hash;
 
     unpack_instance_attributes(data_set->input, config, XML_CIB_TAG_PROPSET, NULL, config_hash,
                                CIB_OPTIONS_FIRST, FALSE, data_set->now);
 
     verify_pe_options(data_set->config_hash);
 
     set_config_flag(data_set, "enable-startup-probes", pe_flag_startup_probes);
     if(is_not_set(data_set->flags, pe_flag_startup_probes)) {
         crm_info("Startup probes: disabled (dangerous)");
     }
 
     value = pe_pref(data_set->config_hash, XML_ATTR_HAVE_WATCHDOG);
     if (value && crm_is_true(value)) {
         crm_notice("Watchdog will be used via SBD if fencing is required");
         set_bit(data_set->flags, pe_flag_have_stonith_resource);
     }
 
     /* Set certain flags via xpath here, so they can be used before the relevant
      * configuration sections are unpacked.
      */
     set_if_xpath(pe_flag_enable_unfencing, XPATH_ENABLE_UNFENCING, data_set);
 
     value = pe_pref(data_set->config_hash, "stonith-timeout");
     data_set->stonith_timeout = crm_get_msec(value);
     crm_debug("STONITH timeout: %d", data_set->stonith_timeout);
 
     set_config_flag(data_set, "stonith-enabled", pe_flag_stonith_enabled);
     crm_debug("STONITH of failed nodes is %s",
               is_set(data_set->flags, pe_flag_stonith_enabled) ? "enabled" : "disabled");
 
     data_set->stonith_action = pe_pref(data_set->config_hash, "stonith-action");
     crm_trace("STONITH will %s nodes", data_set->stonith_action);
 
     set_config_flag(data_set, "concurrent-fencing", pe_flag_concurrent_fencing);
     crm_debug("Concurrent fencing is %s",
               is_set(data_set->flags, pe_flag_concurrent_fencing) ? "enabled" : "disabled");
 
     set_config_flag(data_set, "stop-all-resources", pe_flag_stop_everything);
     crm_debug("Stop all active resources: %s",
               is_set(data_set->flags, pe_flag_stop_everything) ? "true" : "false");
 
     set_config_flag(data_set, "symmetric-cluster", pe_flag_symmetric_cluster);
     if (is_set(data_set->flags, pe_flag_symmetric_cluster)) {
         crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
     }
 
     value = pe_pref(data_set->config_hash, "default-resource-stickiness");
     if (value) {
         pe_warn_once(pe_wo_default_stick,
                      "Support for 'default-resource-stickiness' cluster property"
                      " is deprecated and will be removed in a future release"
                      " (use resource-stickiness in rsc_defaults instead)");
     }
     data_set->default_resource_stickiness = char2score(value);
     crm_debug("Default stickiness: %d", data_set->default_resource_stickiness);
 
     value = pe_pref(data_set->config_hash, "no-quorum-policy");
 
     if (safe_str_eq(value, "ignore")) {
         data_set->no_quorum_policy = no_quorum_ignore;
 
     } else if (safe_str_eq(value, "freeze")) {
         data_set->no_quorum_policy = no_quorum_freeze;
 
     } else if (safe_str_eq(value, "suicide")) {
         if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
             int do_panic = 0;
 
             crm_element_value_int(data_set->input, XML_ATTR_QUORUM_PANIC,
                                   &do_panic);
             if (do_panic || is_set(data_set->flags, pe_flag_have_quorum)) {
                 data_set->no_quorum_policy = no_quorum_suicide;
             } else {
                 crm_notice("Resetting no-quorum-policy to 'stop': cluster has never had quorum");
                 data_set->no_quorum_policy = no_quorum_stop;
             }
         } else {
             crm_config_err("Resetting no-quorum-policy to 'stop': stonith is not configured");
             data_set->no_quorum_policy = no_quorum_stop;
         }
 
     } else {
         data_set->no_quorum_policy = no_quorum_stop;
     }
 
     switch (data_set->no_quorum_policy) {
         case no_quorum_freeze:
             crm_debug("On loss of CCM Quorum: Freeze resources");
             break;
         case no_quorum_stop:
             crm_debug("On loss of CCM Quorum: Stop ALL resources");
             break;
         case no_quorum_suicide:
             crm_notice("On loss of CCM Quorum: Fence all remaining nodes");
             break;
         case no_quorum_ignore:
             crm_notice("On loss of CCM Quorum: Ignore");
             break;
     }
 
     set_config_flag(data_set, "stop-orphan-resources", pe_flag_stop_rsc_orphans);
     crm_trace("Orphan resources are %s",
               is_set(data_set->flags, pe_flag_stop_rsc_orphans) ? "stopped" : "ignored");
 
     set_config_flag(data_set, "stop-orphan-actions", pe_flag_stop_action_orphans);
     crm_trace("Orphan resource actions are %s",
               is_set(data_set->flags, pe_flag_stop_action_orphans) ? "stopped" : "ignored");
 
     set_config_flag(data_set, "remove-after-stop", pe_flag_remove_after_stop);
     crm_trace("Stopped resources are removed from the status section: %s",
               is_set(data_set->flags, pe_flag_remove_after_stop) ? "true" : "false");
 
     set_config_flag(data_set, "maintenance-mode", pe_flag_maintenance_mode);
     crm_trace("Maintenance mode: %s",
               is_set(data_set->flags, pe_flag_maintenance_mode) ? "true" : "false");
 
     if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
         clear_bit(data_set->flags, pe_flag_is_managed_default);
     } else if (pe_pref(data_set->config_hash, "is-managed-default")) {
         set_config_flag(data_set, "is-managed-default", pe_flag_is_managed_default);
         pe_warn_once(pe_wo_default_isman,
                      "Support for 'is-managed-default' cluster property"
                      " is deprecated and will be removed in a future release"
                      " (use is-managed in rsc_defaults instead)");
     }
     crm_trace("By default resources are %smanaged",
               is_set(data_set->flags, pe_flag_is_managed_default) ? "" : "not ");
 
     set_config_flag(data_set, "start-failure-is-fatal", pe_flag_start_failure_fatal);
     crm_trace("Start failures are %s",
               is_set(data_set->flags,
                      pe_flag_start_failure_fatal) ? "always fatal" : "handled by failcount");
 
     if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
         set_config_flag(data_set, "startup-fencing", pe_flag_startup_fencing);
     }
     if (is_set(data_set->flags, pe_flag_startup_fencing)) {
         crm_trace("Unseen nodes will be fenced");
     } else {
         pe_warn_once(pe_wo_blind, "Blind faith: not fencing unseen nodes");
     }
 
     node_score_red = char2score(pe_pref(data_set->config_hash, "node-health-red"));
     node_score_green = char2score(pe_pref(data_set->config_hash, "node-health-green"));
     node_score_yellow = char2score(pe_pref(data_set->config_hash, "node-health-yellow"));
 
     crm_debug("Node scores: 'red' = %s, 'yellow' = %s, 'green' = %s",
              pe_pref(data_set->config_hash, "node-health-red"),
              pe_pref(data_set->config_hash, "node-health-yellow"),
              pe_pref(data_set->config_hash, "node-health-green"));
 
     data_set->placement_strategy = pe_pref(data_set->config_hash, "placement-strategy");
     crm_trace("Placement strategy: %s", data_set->placement_strategy);
 
     return TRUE;
 }
 
 static void
 destroy_digest_cache(gpointer ptr)
 {
     op_digest_cache_t *data = ptr;
 
     free_xml(data->params_all);
     free_xml(data->params_secure);
     free_xml(data->params_restart);
 
     free(data->digest_all_calc);
     free(data->digest_restart_calc);
     free(data->digest_secure_calc);
 
     free(data);
 }
 
 node_t *
 pe_create_node(const char *id, const char *uname, const char *type,
                const char *score, pe_working_set_t * data_set)
 {
     node_t *new_node = NULL;
 
     if (pe_find_node(data_set->nodes, uname) != NULL) {
         crm_config_warn("Detected multiple node entries with uname=%s"
                         " - this is rarely intended", uname);
     }
 
     new_node = calloc(1, sizeof(node_t));
     if (new_node == NULL) {
         return NULL;
     }
 
     new_node->weight = char2score(score);
     new_node->fixed = FALSE;
     new_node->details = calloc(1, sizeof(struct node_shared_s));
 
     if (new_node->details == NULL) {
         free(new_node);
         return NULL;
     }
 
     crm_trace("Creating node for entry %s/%s", uname, id);
     new_node->details->id = id;
     new_node->details->uname = uname;
     new_node->details->online = FALSE;
     new_node->details->shutdown = FALSE;
     new_node->details->rsc_discovery_enabled = TRUE;
     new_node->details->running_rsc = NULL;
     new_node->details->type = node_ping;
 
     if (safe_str_eq(type, "remote")) {
         new_node->details->type = node_remote;
         set_bit(data_set->flags, pe_flag_have_remote_nodes);
     } else if (type == NULL || safe_str_eq(type, "member")
         || safe_str_eq(type, NORMALNODE)) {
         new_node->details->type = node_member;
     }
 
     new_node->details->attrs = crm_str_table_new();
 
     if (is_remote_node(new_node)) {
         g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
                             strdup("remote"));
     } else {
         g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
                             strdup("cluster"));
     }
 
     new_node->details->utilization = crm_str_table_new();
 
     new_node->details->digest_cache =
         g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
                               destroy_digest_cache);
 
     data_set->nodes = g_list_insert_sorted(data_set->nodes, new_node, sort_node_uname);
     return new_node;
 }
 
 bool
 remote_id_conflict(const char *remote_name, pe_working_set_t *data) 
 {
     bool match = FALSE;
 #if 1
     pe_find_resource(data->resources, remote_name);
 #else
     if (data->name_check == NULL) {
         data->name_check = g_hash_table_new(crm_str_hash, g_str_equal);
         for (xml_rsc = __xml_first_child(parent); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) {
             const char *id = ID(xml_rsc);
 
             /* avoiding heap allocation here because we know the duration of this hashtable allows us to */
             g_hash_table_insert(data->name_check, (char *) id, (char *) id);
         }
     }
     if (g_hash_table_lookup(data->name_check, remote_name)) {
         match = TRUE;
     }
 #endif
     if (match) {
         crm_err("Invalid remote-node name, a resource called '%s' already exists.", remote_name);
         return NULL;
     }
 
     return match;
 }
 
 
 static const char *
 expand_remote_rsc_meta(xmlNode *xml_obj, xmlNode *parent, pe_working_set_t *data)
 {
     xmlNode *attr_set = NULL;
     xmlNode *attr = NULL;
 
     const char *container_id = ID(xml_obj);
     const char *remote_name = NULL;
     const char *remote_server = NULL;
     const char *remote_port = NULL;
     const char *connect_timeout = "60s";
     const char *remote_allow_migrate=NULL;
     const char *container_managed = NULL;
 
     for (attr_set = __xml_first_child(xml_obj); attr_set != NULL; attr_set = __xml_next_element(attr_set)) {
         if (safe_str_neq((const char *)attr_set->name, XML_TAG_META_SETS)) {
             continue;
         }
 
         for (attr = __xml_first_child(attr_set); attr != NULL; attr = __xml_next_element(attr)) {
             const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
             const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
 
             if (safe_str_eq(name, XML_RSC_ATTR_REMOTE_NODE)) {
                 remote_name = value;
             } else if (safe_str_eq(name, "remote-addr")) {
                 remote_server = value;
             } else if (safe_str_eq(name, "remote-port")) {
                 remote_port = value;
             } else if (safe_str_eq(name, "remote-connect-timeout")) {
                 connect_timeout = value;
             } else if (safe_str_eq(name, "remote-allow-migrate")) {
                 remote_allow_migrate=value;
             } else if (safe_str_eq(name, XML_RSC_ATTR_MANAGED)) {
                 container_managed = value;
             }
         }
     }
 
     if (remote_name == NULL) {
         return NULL;
     }
 
     if (remote_id_conflict(remote_name, data)) {
         return NULL;
     }
 
     pe_create_remote_xml(parent, remote_name, container_id,
                          remote_allow_migrate, container_managed, "30s", "30s",
                          connect_timeout, remote_server, remote_port);
     return remote_name;
 }
 
 static void
 handle_startup_fencing(pe_working_set_t *data_set, node_t *new_node)
 {
     if ((new_node->details->type == node_remote) && (new_node->details->remote_rsc == NULL)) {
         /* Ignore fencing for remote nodes that don't have a connection resource
          * associated with them. This happens when remote node entries get left
          * in the nodes section after the connection resource is removed.
          */
         return;
     }
 
     if (is_set(data_set->flags, pe_flag_startup_fencing)) {
         // All nodes are unclean until we've seen their status entry
         new_node->details->unclean = TRUE;
 
     } else {
         // Blind faith ...
         new_node->details->unclean = FALSE;
     }
 
     /* We need to be able to determine if a node's status section
      * exists or not separate from whether the node is unclean. */
     new_node->details->unseen = TRUE;
 }
 
 gboolean
 unpack_nodes(xmlNode * xml_nodes, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
     node_t *new_node = NULL;
     const char *id = NULL;
     const char *uname = NULL;
     const char *type = NULL;
     const char *score = NULL;
 
     for (xml_obj = __xml_first_child(xml_nodes); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, TRUE)) {
             new_node = NULL;
 
             id = crm_element_value(xml_obj, XML_ATTR_ID);
             uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
             type = crm_element_value(xml_obj, XML_ATTR_TYPE);
             score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
             crm_trace("Processing node %s/%s", uname, id);
 
             if (id == NULL) {
                 crm_config_err("Must specify id tag in <node>");
                 continue;
             }
             new_node = pe_create_node(id, uname, type, score, data_set);
 
             if (new_node == NULL) {
                 return FALSE;
             }
 
 /* 		if(data_set->have_quorum == FALSE */
 /* 		   && data_set->no_quorum_policy == no_quorum_stop) { */
 /* 			/\* start shutting resources down *\/ */
 /* 			new_node->weight = -INFINITY; */
 /* 		} */
 
             handle_startup_fencing(data_set, new_node);
 
             add_node_attrs(xml_obj, new_node, FALSE, data_set);
             unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_UTILIZATION, NULL,
                                        new_node->details->utilization, NULL, FALSE, data_set->now);
 
             crm_trace("Done with node %s", crm_element_value(xml_obj, XML_ATTR_UNAME));
         }
     }
 
     if (data_set->localhost && pe_find_node(data_set->nodes, data_set->localhost) == NULL) {
         crm_info("Creating a fake local node");
         pe_create_node(data_set->localhost, data_set->localhost, NULL, 0,
                        data_set);
     }
 
     return TRUE;
 }
 
 static void
 setup_container(resource_t * rsc, pe_working_set_t * data_set)
 {
     const char *container_id = NULL;
 
     if (rsc->children) {
         GListPtr gIter = rsc->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             resource_t *child_rsc = (resource_t *) gIter->data;
 
             setup_container(child_rsc, data_set);
         }
         return;
     }
 
     container_id = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_CONTAINER);
     if (container_id && safe_str_neq(container_id, rsc->id)) {
         resource_t *container = pe_find_resource(data_set->resources, container_id);
 
         if (container) {
             rsc->container = container;
             container->fillers = g_list_append(container->fillers, rsc);
             pe_rsc_trace(rsc, "Resource %s's container is %s", rsc->id, container_id);
         } else {
             pe_err("Resource %s: Unknown resource container (%s)", rsc->id, container_id);
         }
     }
 }
 
 gboolean
 unpack_remote_nodes(xmlNode * xml_resources, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
 
     /* generate remote nodes from resource config before unpacking resources */
     for (xml_obj = __xml_first_child(xml_resources); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
         const char *new_node_id = NULL;
 
         /* first check if this is a bare metal remote node. Bare metal remote nodes
          * are defined as a resource primitive only. */
         if (xml_contains_remote_node(xml_obj)) {
             new_node_id = ID(xml_obj);
             /* The "pe_find_node" check is here to make sure we don't iterate over
              * an expanded node that has already been added to the node list. */
             if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
                 crm_trace("Found baremetal remote node %s in container resource %s", new_node_id, ID(xml_obj));
                 pe_create_node(new_node_id, new_node_id, "remote", NULL,
                                data_set);
             }
             continue;
         }
 
         /* Now check for guest remote nodes.
          * guest remote nodes are defined within a resource primitive.
          * Example1: a vm resource might be configured as a remote node.
          * Example2: a vm resource might be configured within a group to be a remote node.
          * Note: right now we only support guest remote nodes in as a standalone primitive
          * or a primitive within a group. No cloned primitives can be a guest remote node
          * right now */
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RESOURCE, TRUE)) {
             /* expands a metadata defined remote resource into the xml config
              * as an actual rsc primitive to be unpacked later. */
             new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources, data_set);
 
             if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
                 crm_trace("Found guest remote node %s in container resource %s", new_node_id, ID(xml_obj));
                 pe_create_node(new_node_id, new_node_id, "remote", NULL,
                                data_set);
             }
             continue;
 
         } else if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, TRUE)) {
             xmlNode *xml_obj2 = NULL;
             /* search through a group to see if any of the primitive contain a remote node. */
             for (xml_obj2 = __xml_first_child(xml_obj); xml_obj2 != NULL; xml_obj2 = __xml_next_element(xml_obj2)) {
 
                 new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources, data_set);
 
                 if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
                     crm_trace("Found guest remote node %s in container resource %s which is in group %s", new_node_id, ID(xml_obj2), ID(xml_obj));
                     pe_create_node(new_node_id, new_node_id, "remote", NULL,
                                    data_set);
                 }
             }
         }
     }
     return TRUE;
 }
 
 
 /* Call this after all the nodes and resources have been
  * unpacked, but before the status section is read.
  *
  * A remote node's online status is reflected by the state
  * of the remote node's connection resource. We need to link
  * the remote node to this connection resource so we can have
  * easy access to the connection resource during the PE calculations.
  */
 static void
 link_rsc2remotenode(pe_working_set_t *data_set, resource_t *new_rsc)
 {
     node_t *remote_node = NULL;
 
     if (new_rsc->is_remote_node == FALSE) {
         return;
     }
 
     if (is_set(data_set->flags, pe_flag_quick_location)) {
         /* remote_nodes and remote_resources are not linked in quick location calculations */
         return;
     }
 
     print_resource(LOG_DEBUG_3, "Linking remote-node connection resource, ", new_rsc, FALSE);
 
     remote_node = pe_find_node(data_set->nodes, new_rsc->id);
     CRM_CHECK(remote_node != NULL, return;);
 
     remote_node->details->remote_rsc = new_rsc;
     /* If this is a baremetal remote-node (no container resource
      * associated with it) then we need to handle startup fencing the same way
      * as cluster nodes. */
     if (new_rsc->container == NULL) {
         handle_startup_fencing(data_set, remote_node);
     } else {
         /* At this point we know if the remote node is a container or baremetal
          * remote node, update the #kind attribute if a container is involved */
         g_hash_table_replace(remote_node->details->attrs, strdup(CRM_ATTR_KIND),
                              strdup("container"));
     }
 }
 
 static void
 destroy_tag(gpointer data)
 {
     tag_t *tag = data;
 
     if (tag) {
         free(tag->id);
         g_list_free_full(tag->refs, free);
         free(tag);
     }
 }
 
 /*!
  * \internal
  * \brief Parse configuration XML for resource information
  *
  * \param[in]     xml_resources  Top of resource configuration XML
  * \param[in,out] data_set       Where to put resource information
  *
  * \return TRUE
  *
  * \note unpack_remote_nodes() MUST be called before this, so that the nodes can
  *       be used when common_unpack() calls resource_location()
  */
 gboolean
 unpack_resources(xmlNode * xml_resources, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
     GListPtr gIter = NULL;
 
     data_set->template_rsc_sets =
         g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
                               destroy_tag);
 
     for (xml_obj = __xml_first_child(xml_resources); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
         resource_t *new_rsc = NULL;
 
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE, TRUE)) {
             const char *template_id = ID(xml_obj);
 
             if (template_id && g_hash_table_lookup_extended(data_set->template_rsc_sets,
                                                             template_id, NULL, NULL) == FALSE) {
                 /* Record the template's ID for the knowledge of its existence anyway. */
                 g_hash_table_insert(data_set->template_rsc_sets, strdup(template_id), NULL);
             }
             continue;
         }
 
         crm_trace("Beginning unpack... <%s id=%s... >", crm_element_name(xml_obj), ID(xml_obj));
         if (common_unpack(xml_obj, &new_rsc, NULL, data_set)) {
             data_set->resources = g_list_append(data_set->resources, new_rsc);
             print_resource(LOG_DEBUG_3, "Added ", new_rsc, FALSE);
 
         } else {
             crm_config_err("Failed unpacking %s %s",
                            crm_element_name(xml_obj), crm_element_value(xml_obj, XML_ATTR_ID));
             if (new_rsc != NULL && new_rsc->fns != NULL) {
                 new_rsc->fns->free(new_rsc);
             }
         }
     }
 
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         setup_container(rsc, data_set);
         link_rsc2remotenode(data_set, rsc);
     }
 
     data_set->resources = g_list_sort(data_set->resources, sort_rsc_priority);
     if (is_set(data_set->flags, pe_flag_quick_location)) {
         /* Ignore */
 
     } else if (is_set(data_set->flags, pe_flag_stonith_enabled)
                && is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {
 
         crm_config_err("Resource start-up disabled since no STONITH resources have been defined");
         crm_config_err("Either configure some or disable STONITH with the stonith-enabled option");
         crm_config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
     }
 
     return TRUE;
 }
 
 gboolean
 unpack_tags(xmlNode * xml_tags, pe_working_set_t * data_set)
 {
     xmlNode *xml_tag = NULL;
 
     data_set->tags =
         g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, destroy_tag);
 
     for (xml_tag = __xml_first_child(xml_tags); xml_tag != NULL; xml_tag = __xml_next_element(xml_tag)) {
         xmlNode *xml_obj_ref = NULL;
         const char *tag_id = ID(xml_tag);
 
         if (crm_str_eq((const char *)xml_tag->name, XML_CIB_TAG_TAG, TRUE) == FALSE) {
             continue;
         }
 
         if (tag_id == NULL) {
             crm_config_err("Failed unpacking %s: %s should be specified",
                            crm_element_name(xml_tag), XML_ATTR_ID);
             continue;
         }
 
         for (xml_obj_ref = __xml_first_child(xml_tag); xml_obj_ref != NULL; xml_obj_ref = __xml_next_element(xml_obj_ref)) {
             const char *obj_ref = ID(xml_obj_ref);
 
             if (crm_str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, TRUE) == FALSE) {
                 continue;
             }
 
             if (obj_ref == NULL) {
                 crm_config_err("Failed unpacking %s for tag %s: %s should be specified",
                                crm_element_name(xml_obj_ref), tag_id, XML_ATTR_ID);
                 continue;
             }
 
             if (add_tag_ref(data_set->tags, tag_id, obj_ref) == FALSE) {
                 return FALSE;
             }
         }
     }
 
     return TRUE;
 }
 
 /* The ticket state section:
  * "/cib/status/tickets/ticket_state" */
 static gboolean
 unpack_ticket_state(xmlNode * xml_ticket, pe_working_set_t * data_set)
 {
     const char *ticket_id = NULL;
     const char *granted = NULL;
     const char *last_granted = NULL;
     const char *standby = NULL;
     xmlAttrPtr xIter = NULL;
 
     ticket_t *ticket = NULL;
 
     ticket_id = ID(xml_ticket);
     if (ticket_id == NULL || strlen(ticket_id) == 0) {
         return FALSE;
     }
 
     crm_trace("Processing ticket state for %s", ticket_id);
 
     ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
     if (ticket == NULL) {
         ticket = ticket_new(ticket_id, data_set);
         if (ticket == NULL) {
             return FALSE;
         }
     }
 
     for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
         const char *prop_name = (const char *)xIter->name;
         const char *prop_value = crm_element_value(xml_ticket, prop_name);
 
         if (crm_str_eq(prop_name, XML_ATTR_ID, TRUE)) {
             continue;
         }
         g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
     }
 
     granted = g_hash_table_lookup(ticket->state, "granted");
     if (granted && crm_is_true(granted)) {
         ticket->granted = TRUE;
         crm_info("We have ticket '%s'", ticket->id);
     } else {
         ticket->granted = FALSE;
         crm_info("We do not have ticket '%s'", ticket->id);
     }
 
     last_granted = g_hash_table_lookup(ticket->state, "last-granted");
     if (last_granted) {
         ticket->last_granted = crm_parse_int(last_granted, 0);
     }
 
     standby = g_hash_table_lookup(ticket->state, "standby");
     if (standby && crm_is_true(standby)) {
         ticket->standby = TRUE;
         if (ticket->granted) {
             crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
         }
     } else {
         ticket->standby = FALSE;
     }
 
     crm_trace("Done with ticket state for %s", ticket_id);
 
     return TRUE;
 }
 
 static gboolean
 unpack_tickets_state(xmlNode * xml_tickets, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
 
     for (xml_obj = __xml_first_child(xml_tickets); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, TRUE) == FALSE) {
             continue;
         }
         unpack_ticket_state(xml_obj, data_set);
     }
 
     return TRUE;
 }
 
 /* @COMPAT DC < 1.1.7: Compatibility with the deprecated ticket state section:
  * "/cib/status/tickets/instance_attributes" */
 static void
 get_ticket_state_legacy(gpointer key, gpointer value, gpointer user_data)
 {
     const char *long_key = key;
     char *state_key = NULL;
 
     const char *granted_prefix = "granted-ticket-";
     const char *last_granted_prefix = "last-granted-";
     static int granted_prefix_strlen = 0;
     static int last_granted_prefix_strlen = 0;
 
     const char *ticket_id = NULL;
     const char *is_granted = NULL;
     const char *last_granted = NULL;
     const char *sep = NULL;
 
     ticket_t *ticket = NULL;
     pe_working_set_t *data_set = user_data;
 
     if (granted_prefix_strlen == 0) {
         granted_prefix_strlen = strlen(granted_prefix);
     }
 
     if (last_granted_prefix_strlen == 0) {
         last_granted_prefix_strlen = strlen(last_granted_prefix);
     }
 
     if (strstr(long_key, granted_prefix) == long_key) {
         ticket_id = long_key + granted_prefix_strlen;
         if (strlen(ticket_id)) {
             state_key = strdup("granted");
             is_granted = value;
         }
     } else if (strstr(long_key, last_granted_prefix) == long_key) {
         ticket_id = long_key + last_granted_prefix_strlen;
         if (strlen(ticket_id)) {
             state_key = strdup("last-granted");
             last_granted = value;
         }
     } else if ((sep = strrchr(long_key, '-'))) {
         ticket_id = sep + 1;
         state_key = strndup(long_key, strlen(long_key) - strlen(sep));
     }
 
     if (ticket_id == NULL || strlen(ticket_id) == 0) {
         free(state_key);
         return;
     }
 
     if (state_key == NULL || strlen(state_key) == 0) {
         free(state_key);
         return;
     }
 
     ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
     if (ticket == NULL) {
         ticket = ticket_new(ticket_id, data_set);
         if (ticket == NULL) {
             free(state_key);
             return;
         }
     }
 
     g_hash_table_replace(ticket->state, state_key, strdup(value));
 
     if (is_granted) {
         if (crm_is_true(is_granted)) {
             ticket->granted = TRUE;
             crm_info("We have ticket '%s'", ticket->id);
         } else {
             ticket->granted = FALSE;
             crm_info("We do not have ticket '%s'", ticket->id);
         }
 
     } else if (last_granted) {
         ticket->last_granted = crm_parse_int(last_granted, 0);
     }
 }
 
 static void
 unpack_handle_remote_attrs(node_t *this_node, xmlNode *state, pe_working_set_t * data_set) 
 {
     const char *resource_discovery_enabled = NULL;
     xmlNode *attrs = NULL;
     resource_t *rsc = NULL;
     const char *shutdown = NULL;
 
     if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
         return;
     }
 
     if ((this_node == NULL) || (is_remote_node(this_node) == FALSE)) {
         return;
     }
     crm_trace("Processing remote node id=%s, uname=%s", this_node->details->id, this_node->details->uname);
 
     this_node->details->remote_maintenance =
         crm_atoi(crm_element_value(state, XML_NODE_IS_MAINTENANCE), "0");
 
     rsc = this_node->details->remote_rsc;
     if (this_node->details->remote_requires_reset == FALSE) {
         this_node->details->unclean = FALSE;
         this_node->details->unseen = FALSE;
     }
     attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
     add_node_attrs(attrs, this_node, TRUE, data_set);
 
     shutdown = pe_node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN);
     if (shutdown != NULL && safe_str_neq("0", shutdown)) {
         crm_info("Node %s is shutting down", this_node->details->uname);
         this_node->details->shutdown = TRUE;
         if (rsc) {
             rsc->next_role = RSC_ROLE_STOPPED;
         }
     }
  
     if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
         crm_info("Node %s is in standby-mode", this_node->details->uname);
         this_node->details->standby = TRUE;
     }
 
     if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance")) ||
         (rsc && !is_set(rsc->flags, pe_rsc_managed))) {
         crm_info("Node %s is in maintenance-mode", this_node->details->uname);
         this_node->details->maintenance = TRUE;
     }
 
     resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
     if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
         if (is_baremetal_remote_node(this_node) && is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
             crm_warn("ignoring %s attribute on baremetal remote node %s, disabling resource discovery requires stonith to be enabled.",
                      XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
         } else {
             /* if we're here, this is either a baremetal node and fencing is enabled,
              * or this is a container node which we don't care if fencing is enabled 
              * or not on. container nodes are 'fenced' by recovering the container resource
              * regardless of whether fencing is enabled. */
             crm_info("Node %s has resource discovery disabled", this_node->details->uname);
             this_node->details->rsc_discovery_enabled = FALSE;
         }
     }
 }
 
 static bool
 unpack_node_loop(xmlNode * status, bool fence, pe_working_set_t * data_set) 
 {
     bool changed = false;
     xmlNode *lrm_rsc = NULL;
 
     for (xmlNode *state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
         const char *id = NULL;
         const char *uname = NULL;
         node_t *this_node = NULL;
         bool process = FALSE;
 
         if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
             continue;
         }
 
         id = crm_element_value(state, XML_ATTR_ID);
         uname = crm_element_value(state, XML_ATTR_UNAME);
         this_node = pe_find_node_any(data_set->nodes, id, uname);
 
         if (this_node == NULL) {
             crm_info("Node %s is unknown", id);
             continue;
 
         } else if (this_node->details->unpacked) {
             crm_info("Node %s is already processed", id);
             continue;
 
         } else if (is_remote_node(this_node) == FALSE && is_set(data_set->flags, pe_flag_stonith_enabled)) {
             // A redundant test, but preserves the order for regression tests
             process = TRUE;
 
         } else if (is_remote_node(this_node)) {
             resource_t *rsc = this_node->details->remote_rsc;
 
             if (fence || (rsc && rsc->role == RSC_ROLE_STARTED)) {
                 determine_remote_online_status(data_set, this_node);
                 unpack_handle_remote_attrs(this_node, state, data_set);
                 process = TRUE;
             }
 
         } else if (this_node->details->online) {
             process = TRUE;
 
         } else if (fence) {
             process = TRUE;
         }
 
         if(process) {
             crm_trace("Processing lrm resource entries on %shealthy%s node: %s",
                       fence?"un":"", is_remote_node(this_node)?" remote":"",
                       this_node->details->uname);
             changed = TRUE;
             this_node->details->unpacked = TRUE;
 
             lrm_rsc = find_xml_node(state, XML_CIB_TAG_LRM, FALSE);
             lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
             unpack_lrm_resources(this_node, lrm_rsc, data_set);
         }
     }
     return changed;
 }
 
 /* remove nodes that are down, stopping */
 /* create +ve rsc_to_node constraints between resources and the nodes they are running on */
 /* anything else? */
 gboolean
 unpack_status(xmlNode * status, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     const char *uname = NULL;
 
     xmlNode *state = NULL;
     node_t *this_node = NULL;
 
     crm_trace("Beginning unpack");
 
     if (data_set->tickets == NULL) {
         data_set->tickets =
             g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, destroy_ticket);
     }
 
     for (state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
         if (crm_str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, TRUE)) {
             xmlNode *xml_tickets = state;
             GHashTable *state_hash = NULL;
 
             /* @COMPAT DC < 1.1.7: Compatibility with the deprecated ticket state section:
              * Unpack the attributes in the deprecated "/cib/status/tickets/instance_attributes" if it exists. */
             state_hash = crm_str_table_new();
 
             unpack_instance_attributes(data_set->input, xml_tickets, XML_TAG_ATTR_SETS, NULL,
                                        state_hash, NULL, TRUE, data_set->now);
 
             g_hash_table_foreach(state_hash, get_ticket_state_legacy, data_set);
 
             if (state_hash) {
                 g_hash_table_destroy(state_hash);
             }
 
             /* Unpack the new "/cib/status/tickets/ticket_state"s */
             unpack_tickets_state(xml_tickets, data_set);
         }
 
         if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE)) {
             xmlNode *attrs = NULL;
             const char *resource_discovery_enabled = NULL;
 
             id = crm_element_value(state, XML_ATTR_ID);
             uname = crm_element_value(state, XML_ATTR_UNAME);
             this_node = pe_find_node_any(data_set->nodes, id, uname);
 
             if (uname == NULL) {
                 /* error */
                 continue;
 
             } else if (this_node == NULL) {
                 crm_config_warn("Node %s in status section no longer exists", uname);
                 continue;
 
             } else if (is_remote_node(this_node)) {
                 /* online state for remote nodes is determined by the
                  * rsc state after all the unpacking is done. we do however
                  * need to mark whether or not the node has been fenced as this plays
                  * a role during unpacking cluster node resource state */
                 this_node->details->remote_was_fenced = 
                     crm_atoi(crm_element_value(state, XML_NODE_IS_FENCED), "0");
                 continue;
             }
 
             crm_trace("Processing node id=%s, uname=%s", id, uname);
 
             /* Mark the node as provisionally clean
              * - at least we have seen it in the current cluster's lifetime
              */
             this_node->details->unclean = FALSE;
             this_node->details->unseen = FALSE;
             attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
             add_node_attrs(attrs, this_node, TRUE, data_set);
 
             if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
                 crm_info("Node %s is in standby-mode", this_node->details->uname);
                 this_node->details->standby = TRUE;
             }
 
             if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance"))) {
                 crm_info("Node %s is in maintenance-mode", this_node->details->uname);
                 this_node->details->maintenance = TRUE;
             }
 
             resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
             if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
                 crm_warn("ignoring %s attribute on node %s, disabling resource discovery is not allowed on cluster nodes",
                     XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
             }
 
             crm_trace("determining node state");
             determine_online_status(state, this_node, data_set);
 
             if (is_not_set(data_set->flags, pe_flag_have_quorum)
                 && this_node->details->online
                 && (data_set->no_quorum_policy == no_quorum_suicide)) {
                 /* Everything else should flow from this automatically
                  * At least until the PE becomes able to migrate off healthy resources
                  */
                 pe_fence_node(data_set, this_node, "cluster does not have quorum");
             }
         }
     }
 
 
     while(unpack_node_loop(status, FALSE, data_set)) {
         crm_trace("Start another loop");
     }
 
     // Now catch any nodes we didn't see
     unpack_node_loop(status, is_set(data_set->flags, pe_flag_stonith_enabled), data_set);
 
     for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *this_node = gIter->data;
 
         if (this_node == NULL) {
             continue;
         } else if(is_remote_node(this_node) == FALSE) {
             continue;
         } else if(this_node->details->unpacked) {
             continue;
         }
         determine_remote_online_status(data_set, this_node);
     }
 
     return TRUE;
 }
 
 static gboolean
 determine_online_status_no_fencing(pe_working_set_t * data_set, xmlNode * node_state,
                                    node_t * this_node)
 {
     gboolean online = FALSE;
     const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
     const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
     const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
 
     if (!crm_is_true(in_cluster)) {
         crm_trace("Node is down: in_cluster=%s", crm_str(in_cluster));
 
     } else if (safe_str_eq(is_peer, ONLINESTATUS)) {
         if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {
             online = TRUE;
         } else {
             crm_debug("Node is not ready to run resources: %s", join);
         }
 
     } else if (this_node->details->expected_up == FALSE) {
         crm_trace("CRMd is down: in_cluster=%s", crm_str(in_cluster));
         crm_trace("\tis_peer=%s, join=%s, expected=%s",
                   crm_str(is_peer), crm_str(join), crm_str(exp_state));
 
     } else {
         /* mark it unclean */
         pe_fence_node(data_set, this_node, "peer is unexpectedly down");
         crm_info("\tin_cluster=%s, is_peer=%s, join=%s, expected=%s",
                  crm_str(in_cluster), crm_str(is_peer), crm_str(join), crm_str(exp_state));
     }
     return online;
 }
 
 static gboolean
 determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_state,
                                 node_t * this_node)
 {
     gboolean online = FALSE;
     gboolean do_terminate = FALSE;
     const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
     const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
     const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
     const char *terminate = pe_node_attribute_raw(this_node, "terminate");
 
 /*
   - XML_NODE_IN_CLUSTER    ::= true|false
   - XML_NODE_IS_PEER       ::= true|false|online|offline
   - XML_NODE_JOIN_STATE    ::= member|down|pending|banned
   - XML_NODE_EXPECTED      ::= member|down
 */
 
     if (crm_is_true(terminate)) {
         do_terminate = TRUE;
 
     } else if (terminate != NULL && strlen(terminate) > 0) {
         /* could be a time() value */
         char t = terminate[0];
 
         if (t != '0' && isdigit(t)) {
             do_terminate = TRUE;
         }
     }
 
     crm_trace("%s: in_cluster=%s, is_peer=%s, join=%s, expected=%s, term=%d",
               this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
               crm_str(join), crm_str(exp_state), do_terminate);
 
     online = crm_is_true(in_cluster);
     if (safe_str_eq(is_peer, ONLINESTATUS)) {
         is_peer = XML_BOOLEAN_YES;
     }
     if (exp_state == NULL) {
         exp_state = CRMD_JOINSTATE_DOWN;
     }
 
     if (this_node->details->shutdown) {
         crm_debug("%s is shutting down", this_node->details->uname);
 
         /* Slightly different criteria since we can't shut down a dead peer */
         online = crm_is_true(is_peer);
 
     } else if (in_cluster == NULL) {
         pe_fence_node(data_set, this_node, "peer has not been seen by the cluster");
 
     } else if (safe_str_eq(join, CRMD_JOINSTATE_NACK)) {
         pe_fence_node(data_set, this_node, "peer failed the pacemaker membership criteria");
 
     } else if (do_terminate == FALSE && safe_str_eq(exp_state, CRMD_JOINSTATE_DOWN)) {
 
         if (crm_is_true(in_cluster) || crm_is_true(is_peer)) {
             crm_info("- Node %s is not ready to run resources", this_node->details->uname);
             this_node->details->standby = TRUE;
             this_node->details->pending = TRUE;
 
         } else {
             crm_trace("%s is down or still coming up", this_node->details->uname);
         }
 
     } else if (do_terminate && safe_str_eq(join, CRMD_JOINSTATE_DOWN)
                && crm_is_true(in_cluster) == FALSE && crm_is_true(is_peer) == FALSE) {
         crm_info("Node %s was just shot", this_node->details->uname);
         online = FALSE;
 
     } else if (crm_is_true(in_cluster) == FALSE) {
         pe_fence_node(data_set, this_node, "peer is no longer part of the cluster");
 
     } else if (crm_is_true(is_peer) == FALSE) {
         pe_fence_node(data_set, this_node, "peer process is no longer available");
 
         /* Everything is running at this point, now check join state */
     } else if (do_terminate) {
         pe_fence_node(data_set, this_node, "termination was requested");
 
     } else if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {
         crm_info("Node %s is active", this_node->details->uname);
 
     } else if (safe_str_eq(join, CRMD_JOINSTATE_PENDING)
                || safe_str_eq(join, CRMD_JOINSTATE_DOWN)) {
         crm_info("Node %s is not ready to run resources", this_node->details->uname);
         this_node->details->standby = TRUE;
         this_node->details->pending = TRUE;
 
     } else {
         pe_fence_node(data_set, this_node, "peer was in an unknown state");
         crm_warn("%s: in-cluster=%s, is-peer=%s, join=%s, expected=%s, term=%d, shutdown=%d",
                  this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
                  crm_str(join), crm_str(exp_state), do_terminate, this_node->details->shutdown);
     }
 
     return online;
 }
 
 static gboolean
 determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node)
 {
     resource_t *rsc = this_node->details->remote_rsc;
     resource_t *container = NULL;
     pe_node_t *host = NULL;
 
     /* If there is a node state entry for a (former) Pacemaker Remote node
      * but no resource creating that node, the node's connection resource will
      * be NULL. Consider it an offline remote node in that case.
      */
     if (rsc == NULL) {
         this_node->details->online = FALSE;
         goto remote_online_done;
     }
 
     container = rsc->container;
 
     if (container && (g_list_length(rsc->running_on) == 1)) {
         host = rsc->running_on->data;
     }
 
     /* If the resource is currently started, mark it online. */
     if (rsc->role == RSC_ROLE_STARTED) {
         crm_trace("%s node %s presumed ONLINE because connection resource is started",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->online = TRUE;
     }
 
     /* consider this node shutting down if transitioning start->stop */
     if (rsc->role == RSC_ROLE_STARTED && rsc->next_role == RSC_ROLE_STOPPED) {
         crm_trace("%s node %s shutting down because connection resource is stopping",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->shutdown = TRUE;
     }
 
     /* Now check all the failure conditions. */
     if(container && is_set(container->flags, pe_rsc_failed)) {
         crm_trace("Guest node %s UNCLEAN because guest resource failed",
                   this_node->details->id);
         this_node->details->online = FALSE;
         this_node->details->remote_requires_reset = TRUE;
 
     } else if(is_set(rsc->flags, pe_rsc_failed)) {
         crm_trace("%s node %s OFFLINE because connection resource failed",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->online = FALSE;
 
     } else if (rsc->role == RSC_ROLE_STOPPED
         || (container && container->role == RSC_ROLE_STOPPED)) {
 
         crm_trace("%s node %s OFFLINE because its resource is stopped",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->online = FALSE;
         this_node->details->remote_requires_reset = FALSE;
 
     } else if (host && (host->details->online == FALSE)
                && host->details->unclean) {
         crm_trace("Guest node %s UNCLEAN because host is unclean",
                   this_node->details->id);
         this_node->details->online = FALSE;
         this_node->details->remote_requires_reset = TRUE;
     }
 
 remote_online_done:
     crm_trace("Remote node %s online=%s",
         this_node->details->id, this_node->details->online ? "TRUE" : "FALSE");
     return this_node->details->online;
 }
 
 gboolean
 determine_online_status(xmlNode * node_state, node_t * this_node, pe_working_set_t * data_set)
 {
     gboolean online = FALSE;
     const char *shutdown = NULL;
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
 
     if (this_node == NULL) {
         crm_config_err("No node to check");
         return online;
     }
 
     this_node->details->shutdown = FALSE;
     this_node->details->expected_up = FALSE;
     shutdown = pe_node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN);
 
     if (shutdown != NULL && safe_str_neq("0", shutdown)) {
         this_node->details->shutdown = TRUE;
 
     } else if (safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) {
         this_node->details->expected_up = TRUE;
     }
 
     if (this_node->details->type == node_ping) {
         this_node->details->unclean = FALSE;
         online = FALSE;         /* As far as resource management is concerned,
                                  * the node is safely offline.
                                  * Anyone caught abusing this logic will be shot
                                  */
 
     } else if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
         online = determine_online_status_no_fencing(data_set, node_state, this_node);
 
     } else {
         online = determine_online_status_fencing(data_set, node_state, this_node);
     }
 
     if (online) {
         this_node->details->online = TRUE;
 
     } else {
         /* remove node from contention */
         this_node->fixed = TRUE;
         this_node->weight = -INFINITY;
     }
 
     if (online && this_node->details->shutdown) {
         /* don't run resources here */
         this_node->fixed = TRUE;
         this_node->weight = -INFINITY;
     }
 
     if (this_node->details->type == node_ping) {
         crm_info("Node %s is not a pacemaker node", this_node->details->uname);
 
     } else if (this_node->details->unclean) {
         pe_proc_warn("Node %s is unclean", this_node->details->uname);
 
     } else if (this_node->details->online) {
         crm_info("Node %s is %s", this_node->details->uname,
                  this_node->details->shutdown ? "shutting down" :
                  this_node->details->pending ? "pending" :
                  this_node->details->standby ? "standby" :
                  this_node->details->maintenance ? "maintenance" : "online");
 
     } else {
         crm_trace("Node %s is offline", this_node->details->uname);
     }
 
     return online;
 }
 
 char *
 clone_strip(const char *last_rsc_id)
 {
     int lpc = 0;
     char *zero = NULL;
 
     CRM_CHECK(last_rsc_id != NULL, return NULL);
     lpc = strlen(last_rsc_id);
     while (--lpc > 0) {
         switch (last_rsc_id[lpc]) {
             case 0:
                 crm_err("Empty string: %s", last_rsc_id);
                 return NULL;
                 break;
             case '0':
             case '1':
             case '2':
             case '3':
             case '4':
             case '5':
             case '6':
             case '7':
             case '8':
             case '9':
                 break;
             case ':':
                 zero = calloc(1, lpc + 1);
                 memcpy(zero, last_rsc_id, lpc);
                 zero[lpc] = 0;
                 return zero;
             default:
                 goto done;
         }
     }
   done:
     zero = strdup(last_rsc_id);
     return zero;
 }
 
 char *
 clone_zero(const char *last_rsc_id)
 {
     int lpc = 0;
     char *zero = NULL;
 
     CRM_CHECK(last_rsc_id != NULL, return NULL);
     if (last_rsc_id != NULL) {
         lpc = strlen(last_rsc_id);
     }
 
     while (--lpc > 0) {
         switch (last_rsc_id[lpc]) {
             case 0:
                 return NULL;
                 break;
             case '0':
             case '1':
             case '2':
             case '3':
             case '4':
             case '5':
             case '6':
             case '7':
             case '8':
             case '9':
                 break;
             case ':':
                 zero = calloc(1, lpc + 3);
                 memcpy(zero, last_rsc_id, lpc);
                 zero[lpc] = ':';
                 zero[lpc + 1] = '0';
                 zero[lpc + 2] = 0;
                 return zero;
             default:
                 goto done;
         }
     }
   done:
     lpc = strlen(last_rsc_id);
     zero = calloc(1, lpc + 3);
     memcpy(zero, last_rsc_id, lpc);
     zero[lpc] = ':';
     zero[lpc + 1] = '0';
     zero[lpc + 2] = 0;
     crm_trace("%s -> %s", last_rsc_id, zero);
     return zero;
 }
 
 static resource_t *
 create_fake_resource(const char *rsc_id, xmlNode * rsc_entry, pe_working_set_t * data_set)
 {
     resource_t *rsc = NULL;
     xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
 
     copy_in_properties(xml_rsc, rsc_entry);
     crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
     crm_log_xml_debug(xml_rsc, "Orphan resource");
 
     if (!common_unpack(xml_rsc, &rsc, NULL, data_set)) {
         return NULL;
     }
 
     if (xml_contains_remote_node(xml_rsc)) {
         node_t *node;
 
         crm_debug("Detected orphaned remote node %s", rsc_id);
         node = pe_find_node(data_set->nodes, rsc_id);
         if (node == NULL) {
 	        node = pe_create_node(rsc_id, rsc_id, "remote", NULL, data_set);
         }
         link_rsc2remotenode(data_set, rsc);
 
         if (node) {
             crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
             node->details->shutdown = TRUE;
         }
     }
 
     if (crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER)) {
         /* This orphaned rsc needs to be mapped to a container. */
         crm_trace("Detected orphaned container filler %s", rsc_id);
         set_bit(rsc->flags, pe_rsc_orphan_container_filler);
     }
     set_bit(rsc->flags, pe_rsc_orphan);
     data_set->resources = g_list_append(data_set->resources, rsc);
     return rsc;
 }
 
 extern resource_t *create_child_clone(resource_t * rsc, int sub_id, pe_working_set_t * data_set);
 
 static resource_t *
 find_anonymous_clone(pe_working_set_t * data_set, node_t * node, resource_t * parent,
                      const char *rsc_id)
 {
     GListPtr rIter = NULL;
     resource_t *rsc = NULL;
     gboolean skip_inactive = FALSE;
 
     CRM_ASSERT(parent != NULL);
     CRM_ASSERT(pe_rsc_is_clone(parent));
     CRM_ASSERT(is_not_set(parent->flags, pe_rsc_unique));
 
     /* Find an instance active (or partially active for grouped clones) on the specified node */
     pe_rsc_trace(parent, "Looking for %s on %s in %s", rsc_id, node->details->uname, parent->id);
     for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
         GListPtr nIter = NULL;
         GListPtr locations = NULL;
         resource_t *child = rIter->data;
 
         child->fns->location(child, &locations, TRUE);
         if (locations == NULL) {
             pe_rsc_trace(child, "Resource %s, skip inactive", child->id);
             continue;
         }
 
         for (nIter = locations; nIter && rsc == NULL; nIter = nIter->next) {
             node_t *childnode = nIter->data;
 
             if (childnode->details == node->details) {
                 /* ->find_rsc() because we might be a cloned group */
                 rsc = parent->fns->find_rsc(child, rsc_id, NULL, pe_find_clone);
                 if(rsc) {
                     pe_rsc_trace(rsc, "Resource %s, active", rsc->id);
                 }
             }
 
             /* Keep this block, it means we'll do the right thing if
              * anyone toggles the unique flag to 'off'
              */
             if (rsc && rsc->running_on) {
                 crm_notice("/Anonymous/ clone %s is already running on %s",
                            parent->id, node->details->uname);
                 skip_inactive = TRUE;
                 rsc = NULL;
             }
         }
 
         g_list_free(locations);
     }
 
     /* Find an inactive instance */
     if (skip_inactive == FALSE) {
         pe_rsc_trace(parent, "Looking for %s anywhere", rsc_id);
         for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
             GListPtr locations = NULL;
             resource_t *child = rIter->data;
 
             if (is_set(child->flags, pe_rsc_block)) {
                 pe_rsc_trace(child, "Skip: blocked in stopped state");
                 continue;
             }
 
             child->fns->location(child, &locations, TRUE);
             if (locations == NULL) {
                 /* ->find_rsc() because we might be a cloned group */
                 rsc = parent->fns->find_rsc(child, rsc_id, NULL, pe_find_clone);
                 pe_rsc_trace(parent, "Resource %s, empty slot", rsc->id);
             }
             g_list_free(locations);
         }
     }
 
     if (rsc == NULL) {
         /* Create an extra orphan */
         resource_t *top = create_child_clone(parent, -1, data_set);
 
         /* ->find_rsc() because we might be a cloned group */
         rsc = top->fns->find_rsc(top, rsc_id, NULL, pe_find_clone);
         CRM_ASSERT(rsc != NULL);
 
         pe_rsc_debug(parent, "Created orphan %s for %s: %s on %s", top->id, parent->id, rsc_id,
                      node->details->uname);
     }
 
     if (safe_str_neq(rsc_id, rsc->id)) {
         pe_rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
                     rsc_id, node->details->uname, rsc->id,
                     is_set(rsc->flags, pe_rsc_orphan) ? " (ORPHAN)" : "");
     }
 
     return rsc;
 }
 
 static resource_t *
 unpack_find_resource(pe_working_set_t * data_set, node_t * node, const char *rsc_id,
                      xmlNode * rsc_entry)
 {
     resource_t *rsc = NULL;
     resource_t *parent = NULL;
 
     crm_trace("looking for %s", rsc_id);
     rsc = pe_find_resource(data_set->resources, rsc_id);
 
     /* no match */
     if (rsc == NULL) {
         /* Even when clone-max=0, we still create a single :0 orphan to match against */
         char *tmp = clone_zero(rsc_id);
         resource_t *clone0 = pe_find_resource(data_set->resources, tmp);
 
         if (clone0 && is_not_set(clone0->flags, pe_rsc_unique)) {
             rsc = clone0;
         } else {
             crm_trace("%s is not known as %s either", rsc_id, tmp);
         }
 
         parent = uber_parent(clone0);
         free(tmp);
 
         crm_trace("%s not found: %s", rsc_id, parent ? parent->id : "orphan");
 
     } else if (rsc->variant > pe_native) {
         crm_trace("%s is no longer a primitive resource, the lrm_resource entry is obsolete",
                   rsc_id);
         return NULL;
 
     } else {
         parent = uber_parent(rsc);
     }
 
     if(parent && parent->parent) {
         rsc = find_container_child(rsc_id, rsc, node);
 
     } else if (pe_rsc_is_clone(parent)) {
         if (is_not_set(parent->flags, pe_rsc_unique)) {
             char *base = clone_strip(rsc_id);
 
             rsc = find_anonymous_clone(data_set, node, parent, base);
             CRM_ASSERT(rsc != NULL);
             free(base);
         }
 
         if (rsc && safe_str_neq(rsc_id, rsc->id)) {
             free(rsc->clone_name);
             rsc->clone_name = strdup(rsc_id);
         }
     }
 
     return rsc;
 }
 
 static resource_t *
 process_orphan_resource(xmlNode * rsc_entry, node_t * node, pe_working_set_t * data_set)
 {
     resource_t *rsc = NULL;
     const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
 
     crm_debug("Detected orphan resource %s on %s", rsc_id, node->details->uname);
     rsc = create_fake_resource(rsc_id, rsc_entry, data_set);
 
     if (is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
         clear_bit(rsc->flags, pe_rsc_managed);
 
     } else {
         print_resource(LOG_DEBUG_3, "Added orphan", rsc, FALSE);
 
         CRM_CHECK(rsc != NULL, return NULL);
         resource_location(rsc, NULL, -INFINITY, "__orphan_dont_run__", data_set);
     }
     return rsc;
 }
 
 static void
 process_rsc_state(resource_t * rsc, node_t * node,
                   enum action_fail_response on_fail,
                   xmlNode * migrate_op, pe_working_set_t * data_set)
 {
     node_t *tmpnode = NULL;
     char *reason = NULL;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
                  rsc->id, role2text(rsc->role), node->details->uname, fail2text(on_fail));
 
     /* process current state */
     if (rsc->role != RSC_ROLE_UNKNOWN) {
         resource_t *iter = rsc;
 
         while (iter) {
             if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
                 node_t *n = node_copy(node);
 
                 pe_rsc_trace(rsc, "%s (aka. %s) known on %s", rsc->id, rsc->clone_name,
                              n->details->uname);
                 g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
             }
             if (is_set(iter->flags, pe_rsc_unique)) {
                 break;
             }
             iter = iter->parent;
         }
     }
 
     /* If a managed resource is believed to be running, but node is down ... */
     if (rsc->role > RSC_ROLE_STOPPED
         && node->details->online == FALSE
         && node->details->maintenance == FALSE
         && is_set(rsc->flags, pe_rsc_managed)) {
 
         gboolean should_fence = FALSE;
 
         /* If this is a guest node, fence it (regardless of whether fencing is
          * enabled, because guest node fencing is done by recovery of the
          * container resource rather than by stonithd). Mark the resource
          * we're processing as failed. When the guest comes back up, its
          * operation history in the CIB will be cleared, freeing the affected
          * resource to run again once we are sure we know its state.
          */
         if (is_container_remote_node(node)) {
             set_bit(rsc->flags, pe_rsc_failed);
             should_fence = TRUE;
 
         } else if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
             if (is_baremetal_remote_node(node) && node->details->remote_rsc
                 && is_not_set(node->details->remote_rsc->flags, pe_rsc_failed)) {
 
                 /* setting unseen = true means that fencing of the remote node will
                  * only occur if the connection resource is not going to start somewhere.
                  * This allows connection resources on a failed cluster-node to move to
                  * another node without requiring the baremetal remote nodes to be fenced
                  * as well. */
                 node->details->unseen = TRUE;
                 reason = crm_strdup_printf("%s is active there (fencing will be"
                                            " revoked if remote connection can "
                                            "be re-established elsewhere)",
                                            rsc->id);
             }
             should_fence = TRUE;
         }
 
         if (should_fence) {
             if (reason == NULL) {
                reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
             }
             pe_fence_node(data_set, node, reason);
         }
         free(reason);
     }
 
     if (node->details->unclean) {
         /* No extra processing needed
          * Also allows resources to be started again after a node is shot
          */
         on_fail = action_fail_ignore;
     }
 
     switch (on_fail) {
         case action_fail_ignore:
             /* nothing to do */
             break;
 
         case action_fail_fence:
             /* treat it as if it is still running
              * but also mark the node as unclean
              */
             reason = crm_strdup_printf("%s failed there", rsc->id);
             pe_fence_node(data_set, node, reason);
             free(reason);
             break;
 
         case action_fail_standby:
             node->details->standby = TRUE;
             node->details->standby_onfail = TRUE;
             break;
 
         case action_fail_block:
             /* is_managed == FALSE will prevent any
              * actions being sent for the resource
              */
             clear_bit(rsc->flags, pe_rsc_managed);
             set_bit(rsc->flags, pe_rsc_block);
             break;
 
         case action_fail_migrate:
             /* make sure it comes up somewhere else
              * or not at all
              */
             resource_location(rsc, node, -INFINITY, "__action_migration_auto__", data_set);
             break;
 
         case action_fail_stop:
             rsc->next_role = RSC_ROLE_STOPPED;
             break;
 
         case action_fail_recover:
             if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
                 set_bit(rsc->flags, pe_rsc_failed);
                 stop_action(rsc, node, FALSE);
             }
             break;
 
         case action_fail_restart_container:
             set_bit(rsc->flags, pe_rsc_failed);
 
             if (rsc->container) {
                 stop_action(rsc->container, node, FALSE);
             } else if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
                 stop_action(rsc, node, FALSE);
             }
             break;
 
         case action_fail_reset_remote:
             set_bit(rsc->flags, pe_rsc_failed);
             if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
                 tmpnode = NULL;
                 if (rsc->is_remote_node) {
                     tmpnode = pe_find_node(data_set->nodes, rsc->id);
                 }
                 if (tmpnode &&
                     is_baremetal_remote_node(tmpnode) &&
                     tmpnode->details->remote_was_fenced == 0) {
 
                     /* connection resource to baremetal resource failed in a way that
                      * should result in fencing the remote-node. */
                     pe_fence_node(data_set, tmpnode,
                                   "remote connection is unrecoverable");
                 }
             }
 
             /* require the stop action regardless if fencing is occurring or not. */
             if (rsc->role > RSC_ROLE_STOPPED) {
                 stop_action(rsc, node, FALSE);
             }
 
             /* if reconnect delay is in use, prevent the connection from exiting the
              * "STOPPED" role until the failure is cleared by the delay timeout. */
             if (rsc->remote_reconnect_interval) {
                 rsc->next_role = RSC_ROLE_STOPPED;
             }
             break;
     }
 
     /* ensure a remote-node connection failure forces an unclean remote-node
      * to be fenced. By setting unseen = FALSE, the remote-node failure will
      * result in a fencing operation regardless if we're going to attempt to 
      * reconnect to the remote-node in this transition or not. */
     if (is_set(rsc->flags, pe_rsc_failed) && rsc->is_remote_node) {
         tmpnode = pe_find_node(data_set->nodes, rsc->id);
         if (tmpnode && tmpnode->details->unclean) {
             tmpnode->details->unseen = FALSE;
         }
     }
 
     if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
         if (is_set(rsc->flags, pe_rsc_orphan)) {
             if (is_set(rsc->flags, pe_rsc_managed)) {
                 crm_config_warn("Detected active orphan %s running on %s",
                                 rsc->id, node->details->uname);
             } else {
                 crm_config_warn("Cluster configured not to stop active orphans."
                                 " %s must be stopped manually on %s",
                                 rsc->id, node->details->uname);
             }
         }
 
         native_add_running(rsc, node, data_set);
         if (on_fail != action_fail_ignore) {
             set_bit(rsc->flags, pe_rsc_failed);
         }
 
     } else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
         /* Only do this for older status sections that included instance numbers
          * Otherwise stopped instances will appear as orphans
          */
         pe_rsc_trace(rsc, "Resetting clone_name %s for %s (stopped)", rsc->clone_name, rsc->id);
         free(rsc->clone_name);
         rsc->clone_name = NULL;
 
     } else {
         char *key = stop_key(rsc);
         GListPtr possible_matches = find_actions(rsc->actions, key, node);
         GListPtr gIter = possible_matches;
 
         for (; gIter != NULL; gIter = gIter->next) {
             action_t *stop = (action_t *) gIter->data;
 
             stop->flags |= pe_action_optional;
         }
 
         g_list_free(possible_matches);
         free(key);
     }
 }
 
 /* create active recurring operations as optional */
 static void
 process_recurring(node_t * node, resource_t * rsc,
                   int start_index, int stop_index,
                   GListPtr sorted_op_list, pe_working_set_t * data_set)
 {
     int counter = -1;
     const char *task = NULL;
     const char *status = NULL;
     GListPtr gIter = sorted_op_list;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "%s: Start index %d, stop index = %d", rsc->id, start_index, stop_index);
 
     for (; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         int interval = 0;
         char *key = NULL;
         const char *id = ID(rsc_op);
         const char *interval_s = NULL;
 
         counter++;
 
         if (node->details->online == FALSE) {
             pe_rsc_trace(rsc, "Skipping %s/%s: node is offline", rsc->id, node->details->uname);
             break;
 
             /* Need to check if there's a monitor for role="Stopped" */
         } else if (start_index < stop_index && counter <= stop_index) {
             pe_rsc_trace(rsc, "Skipping %s/%s: resource is not active", id, node->details->uname);
             continue;
 
         } else if (counter < start_index) {
             pe_rsc_trace(rsc, "Skipping %s/%s: old %d", id, node->details->uname, counter);
             continue;
         }
 
         interval_s = crm_element_value(rsc_op, XML_LRM_ATTR_INTERVAL);
         interval = crm_parse_int(interval_s, "0");
         if (interval == 0) {
             pe_rsc_trace(rsc, "Skipping %s/%s: non-recurring", id, node->details->uname);
             continue;
         }
 
         status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
         if (safe_str_eq(status, "-1")) {
             pe_rsc_trace(rsc, "Skipping %s/%s: status", id, node->details->uname);
             continue;
         }
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         /* create the action */
         key = generate_op_key(rsc->id, task, interval);
         pe_rsc_trace(rsc, "Creating %s/%s", key, node->details->uname);
         custom_action(rsc, key, task, node, TRUE, TRUE, data_set);
     }
 }
 
 void
 calculate_active_ops(GListPtr sorted_op_list, int *start_index, int *stop_index)
 {
     int counter = -1;
     int implied_monitor_start = -1;
     int implied_master_start = -1;
     const char *task = NULL;
     const char *status = NULL;
     GListPtr gIter = sorted_op_list;
 
     *stop_index = -1;
     *start_index = -1;
 
     for (; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         counter++;
 
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
 
         if (safe_str_eq(task, CRMD_ACTION_STOP)
             && safe_str_eq(status, "0")) {
             *stop_index = counter;
 
         } else if (safe_str_eq(task, CRMD_ACTION_START) || safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
             *start_index = counter;
 
         } else if ((implied_monitor_start <= *stop_index) && safe_str_eq(task, CRMD_ACTION_STATUS)) {
             const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);
 
             if (safe_str_eq(rc, "0") || safe_str_eq(rc, "8")) {
                 implied_monitor_start = counter;
             }
         } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE) || safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
             implied_master_start = counter;
         }
     }
 
     if (*start_index == -1) {
         if (implied_master_start != -1) {
             *start_index = implied_master_start;
         } else if (implied_monitor_start != -1) {
             *start_index = implied_monitor_start;
         }
     }
 }
 
 static resource_t *
 unpack_lrm_rsc_state(node_t * node, xmlNode * rsc_entry, pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     int stop_index = -1;
     int start_index = -1;
     enum rsc_role_e req_role = RSC_ROLE_UNKNOWN;
 
     const char *task = NULL;
     const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
 
     resource_t *rsc = NULL;
     GListPtr op_list = NULL;
     GListPtr sorted_op_list = NULL;
 
     xmlNode *migrate_op = NULL;
     xmlNode *rsc_op = NULL;
     xmlNode *last_failure = NULL;
 
     enum action_fail_response on_fail = FALSE;
     enum rsc_role_e saved_role = RSC_ROLE_UNKNOWN;
 
     crm_trace("[%s] Processing %s on %s",
               crm_element_name(rsc_entry), rsc_id, node->details->uname);
 
     /* extract operations */
     op_list = NULL;
     sorted_op_list = NULL;
 
     for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             op_list = g_list_prepend(op_list, rsc_op);
         }
     }
 
     if (op_list == NULL) {
         /* if there are no operations, there is nothing to do */
         return NULL;
     }
 
     /* find the resource */
     rsc = unpack_find_resource(data_set, node, rsc_id, rsc_entry);
     if (rsc == NULL) {
         rsc = process_orphan_resource(rsc_entry, node, data_set);
     }
     CRM_ASSERT(rsc != NULL);
 
     /* process operations */
     saved_role = rsc->role;
     on_fail = action_fail_ignore;
     rsc->role = RSC_ROLE_UNKNOWN;
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
             migrate_op = rsc_op;
         }
 
         unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail, data_set);
     }
 
     /* create active recurring operations as optional */
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
     process_recurring(node, rsc, start_index, stop_index, sorted_op_list, data_set);
 
     /* no need to free the contents */
     g_list_free(sorted_op_list);
 
     process_rsc_state(rsc, node, on_fail, migrate_op, data_set);
 
     if (get_target_role(rsc, &req_role)) {
         if (rsc->next_role == RSC_ROLE_UNKNOWN || req_role < rsc->next_role) {
             pe_rsc_debug(rsc, "%s: Overwriting calculated next role %s"
                          " with requested next role %s",
                          rsc->id, role2text(rsc->next_role), role2text(req_role));
             rsc->next_role = req_role;
 
         } else if (req_role > rsc->next_role) {
             pe_rsc_info(rsc, "%s: Not overwriting calculated next role %s"
                         " with requested next role %s",
                         rsc->id, role2text(rsc->next_role), role2text(req_role));
         }
     }
 
     if (saved_role > rsc->role) {
         rsc->role = saved_role;
     }
 
     return rsc;
 }
 
 static void
 handle_orphaned_container_fillers(xmlNode * lrm_rsc_list, pe_working_set_t * data_set)
 {
     xmlNode *rsc_entry = NULL;
     for (rsc_entry = __xml_first_child(lrm_rsc_list); rsc_entry != NULL;
         rsc_entry = __xml_next_element(rsc_entry)) {
 
         resource_t *rsc;
         resource_t *container;
         const char *rsc_id;
         const char *container_id;
 
         if (safe_str_neq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE)) {
             continue;
         }
 
         container_id = crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER);
         rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
         if (container_id == NULL || rsc_id == NULL) {
             continue;
         }
 
         container = pe_find_resource(data_set->resources, container_id);
         if (container == NULL) {
             continue;
         }
 
         rsc = pe_find_resource(data_set->resources, rsc_id);
         if (rsc == NULL ||
             is_set(rsc->flags, pe_rsc_orphan_container_filler) == FALSE ||
             rsc->container != NULL) {
             continue;
         }
 
         pe_rsc_trace(rsc, "Mapped orphaned rsc %s's container to  %s", rsc->id, container_id);
         rsc->container = container;
         container->fillers = g_list_append(container->fillers, rsc);
     }
 }
 
 gboolean
 unpack_lrm_resources(node_t * node, xmlNode * lrm_rsc_list, pe_working_set_t * data_set)
 {
     xmlNode *rsc_entry = NULL;
     gboolean found_orphaned_container_filler = FALSE;
 
     CRM_CHECK(node != NULL, return FALSE);
 
     crm_trace("Unpacking resources on %s", node->details->uname);
 
     for (rsc_entry = __xml_first_child(lrm_rsc_list); rsc_entry != NULL;
          rsc_entry = __xml_next_element(rsc_entry)) {
 
         if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
             resource_t *rsc = unpack_lrm_rsc_state(node, rsc_entry, data_set);
             if (!rsc) {
                 continue;
             }
             if (is_set(rsc->flags, pe_rsc_orphan_container_filler)) {
                 found_orphaned_container_filler = TRUE;
             }
         }
     }
 
     /* now that all the resource state has been unpacked for this node
      * we have to go back and map any orphaned container fillers to their
      * container resource */
     if (found_orphaned_container_filler) {
         handle_orphaned_container_fillers(lrm_rsc_list, data_set);
     }
     return TRUE;
 }
 
 static void
 set_active(resource_t * rsc)
 {
     resource_t *top = uber_parent(rsc);
 
     if (top && top->variant == pe_master) {
         rsc->role = RSC_ROLE_SLAVE;
     } else {
         rsc->role = RSC_ROLE_STARTED;
     }
 }
 
 static void
 set_node_score(gpointer key, gpointer value, gpointer user_data)
 {
     node_t *node = value;
     int *score = user_data;
 
     node->weight = *score;
 }
 
 #define STATUS_PATH_MAX 1024
 static xmlNode *
 find_lrm_op(const char *resource, const char *op, const char *node, const char *source,
             pe_working_set_t * data_set)
 {
     int offset = 0;
     char xpath[STATUS_PATH_MAX];
 
     offset += snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//node_state[@uname='%s']", node);
     offset +=
         snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//" XML_LRM_TAG_RESOURCE "[@id='%s']",
                  resource);
 
     /* Need to check against transition_magic too? */
     if (source && safe_str_eq(op, CRMD_ACTION_MIGRATE)) {
         offset +=
             snprintf(xpath + offset, STATUS_PATH_MAX - offset,
                      "/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_target='%s']", op,
                      source);
     } else if (source && safe_str_eq(op, CRMD_ACTION_MIGRATED)) {
         offset +=
             snprintf(xpath + offset, STATUS_PATH_MAX - offset,
                      "/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_source='%s']", op,
                      source);
     } else {
         offset +=
             snprintf(xpath + offset, STATUS_PATH_MAX - offset,
                      "/" XML_LRM_TAG_RSC_OP "[@operation='%s']", op);
     }
 
     CRM_LOG_ASSERT(offset > 0);
     return get_xpath_object(xpath, data_set->input, LOG_DEBUG);
 }
 
 static void
 unpack_rsc_migration(resource_t *rsc, node_t *node, xmlNode *xml_op, pe_working_set_t * data_set)
 {
 
     /*
      * The normal sequence is (now): migrate_to(Src) -> migrate_from(Tgt) -> stop(Src)
      *
      * So if a migrate_to is followed by a stop, then we don't need to care what
      * happened on the target node
      *
      * Without the stop, we need to look for a successful migrate_from.
      * This would also imply we're no longer running on the source
      *
      * Without the stop, and without a migrate_from op we make sure the resource
      * gets stopped on both source and target (assuming the target is up)
      *
      */
     int stop_id = 0;
     int task_id = 0;
     xmlNode *stop_op =
         find_lrm_op(rsc->id, CRMD_ACTION_STOP, node->details->id, NULL, data_set);
 
     if (stop_op) {
         crm_element_value_int(stop_op, XML_LRM_ATTR_CALLID, &stop_id);
     }
 
     crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &task_id);
 
     if (stop_op == NULL || stop_id < task_id) {
         int from_rc = 0, from_status = 0;
         const char *migrate_source =
             crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
         const char *migrate_target =
             crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
         node_t *target = pe_find_node(data_set->nodes, migrate_target);
         node_t *source = pe_find_node(data_set->nodes, migrate_source);
         xmlNode *migrate_from =
             find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, migrate_target, migrate_source,
                         data_set);
 
         rsc->role = RSC_ROLE_STARTED;       /* can be master? */
         if (migrate_from) {
             crm_element_value_int(migrate_from, XML_LRM_ATTR_RC, &from_rc);
             crm_element_value_int(migrate_from, XML_LRM_ATTR_OPSTATUS, &from_status);
             pe_rsc_trace(rsc, "%s op on %s exited with status=%d, rc=%d",
                          ID(migrate_from), migrate_target, from_status, from_rc);
         }
 
         if (migrate_from && from_rc == PCMK_OCF_OK
             && from_status == PCMK_LRM_OP_DONE) {
             pe_rsc_trace(rsc, "Detected dangling migration op: %s on %s", ID(xml_op),
                          migrate_source);
 
             /* all good
              * just need to arrange for the stop action to get sent
              * but _without_ affecting the target somehow
              */
             rsc->role = RSC_ROLE_STOPPED;
             rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);
 
         } else if (migrate_from) {  /* Failed */
             if (target && target->details->online) {
                 pe_rsc_trace(rsc, "Marking active on %s %p %d", migrate_target, target,
                              target->details->online);
                 native_add_running(rsc, target, data_set);
             }
 
         } else {    /* Pending or complete but erased */
             if (target && target->details->online) {
                 pe_rsc_trace(rsc, "Marking active on %s %p %d", migrate_target, target,
                              target->details->online);
 
                 native_add_running(rsc, target, data_set);
                 if (source && source->details->online) {
                     /* If we make it here we have a partial migration.  The migrate_to
                      * has completed but the migrate_from on the target has not. Hold on
                      * to the target and source on the resource. Later on if we detect that
                      * the resource is still going to run on that target, we may continue
                      * the migration */
                     rsc->partial_migration_target = target;
                     rsc->partial_migration_source = source;
                 }
             } else {
                 /* Consider it failed here - forces a restart, prevents migration */
                 set_bit(rsc->flags, pe_rsc_failed);
                 clear_bit(rsc->flags, pe_rsc_allow_migrate);
             }
         }
     }
 }
 
 static void
 unpack_rsc_migration_failure(resource_t *rsc, node_t *node, xmlNode *xml_op, pe_working_set_t * data_set) 
 {
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
 
     CRM_ASSERT(rsc);
     if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
         int stop_id = 0;
         int migrate_id = 0;
         const char *migrate_source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
         const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
         xmlNode *stop_op =
             find_lrm_op(rsc->id, CRMD_ACTION_STOP, migrate_source, NULL, data_set);
         xmlNode *migrate_op =
             find_lrm_op(rsc->id, CRMD_ACTION_MIGRATE, migrate_source, migrate_target,
                         data_set);
 
         if (stop_op) {
             crm_element_value_int(stop_op, XML_LRM_ATTR_CALLID, &stop_id);
         }
         if (migrate_op) {
             crm_element_value_int(migrate_op, XML_LRM_ATTR_CALLID, &migrate_id);
         }
 
         /* Get our state right */
         rsc->role = RSC_ROLE_STARTED;   /* can be master? */
 
         if (stop_op == NULL || stop_id < migrate_id) {
             node_t *source = pe_find_node(data_set->nodes, migrate_source);
 
             if (source && source->details->online) {
                 native_add_running(rsc, source, data_set);
             }
         }
 
     } else if (safe_str_eq(task, CRMD_ACTION_MIGRATE)) {
         int stop_id = 0;
         int migrate_id = 0;
         const char *migrate_source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
         const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
         xmlNode *stop_op =
             find_lrm_op(rsc->id, CRMD_ACTION_STOP, migrate_target, NULL, data_set);
         xmlNode *migrate_op =
             find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, migrate_target, migrate_source,
                         data_set);
 
         if (stop_op) {
             crm_element_value_int(stop_op, XML_LRM_ATTR_CALLID, &stop_id);
         }
         if (migrate_op) {
             crm_element_value_int(migrate_op, XML_LRM_ATTR_CALLID, &migrate_id);
         }
 
         /* Get our state right */
         rsc->role = RSC_ROLE_STARTED;   /* can be master? */
 
         if (stop_op == NULL || stop_id < migrate_id) {
             node_t *target = pe_find_node(data_set->nodes, migrate_target);
 
             pe_rsc_trace(rsc, "Stop: %p %d, Migrated: %p %d", stop_op, stop_id, migrate_op,
                          migrate_id);
             if (target && target->details->online) {
                 native_add_running(rsc, target, data_set);
             }
 
         } else if (migrate_op == NULL) {
             /* Make sure it gets cleaned up, the stop may pre-date the migrate_from */
             rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);
         }
     }
 }
 
 static void
 record_failed_op(xmlNode *op, node_t* node, pe_working_set_t * data_set)
 {
     xmlNode *xIter = NULL;
     const char *op_key = crm_element_value(op, XML_LRM_ATTR_TASK_KEY);
 
     if (node->details->online == FALSE) {
         return;
     }
 
     for (xIter = data_set->failed->children; xIter; xIter = xIter->next) {
         const char *key = crm_element_value(xIter, XML_LRM_ATTR_TASK_KEY);
         const char *uname = crm_element_value(xIter, XML_ATTR_UNAME);
 
         if(safe_str_eq(op_key, key) && safe_str_eq(uname, node->details->uname)) {
             crm_trace("Skipping duplicate entry %s on %s", op_key, node->details->uname);
             return;
         }
     }
 
     crm_trace("Adding entry %s on %s", op_key, node->details->uname);
     crm_xml_add(op, XML_ATTR_UNAME, node->details->uname);
     add_node_copy(data_set->failed, op);
 }
 
 static const char *get_op_key(xmlNode *xml_op)
 {
     const char *key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
     if(key == NULL) {
         key = ID(xml_op);
     }
     return key;
 }
 
 static void
 unpack_rsc_op_failure(resource_t * rsc, node_t * node, int rc, xmlNode * xml_op, xmlNode ** last_failure,
                       enum action_fail_response * on_fail, pe_working_set_t * data_set)
 {
     int interval = 0;
     bool is_probe = FALSE;
     action_t *action = NULL;
 
     const char *key = get_op_key(xml_op);
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     const char *op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
 
     CRM_ASSERT(rsc);
 
     *last_failure = xml_op;
 
     crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);
     if(interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
         is_probe = TRUE;
         pe_rsc_trace(rsc, "is a probe: %s", key);
     }
 
     if (rc != PCMK_OCF_NOT_INSTALLED || is_set(data_set->flags, pe_flag_symmetric_cluster)) {
         crm_warn("Processing failed op %s for %s on %s: %s (%d)",
                  task, rsc->id, node->details->uname, services_ocf_exitcode_str(rc),
                  rc);
 
         record_failed_op(xml_op, node, data_set);
 
     } else {
         crm_trace("Processing failed op %s for %s on %s: %s (%d)",
                  task, rsc->id, node->details->uname, services_ocf_exitcode_str(rc),
                  rc);
     }
 
     action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);
     if ((action->on_fail <= action_fail_fence && *on_fail < action->on_fail) ||
         (action->on_fail == action_fail_reset_remote && *on_fail <= action_fail_recover) ||
         (action->on_fail == action_fail_restart_container && *on_fail <= action_fail_recover) ||
         (*on_fail == action_fail_restart_container && action->on_fail >= action_fail_migrate)) {
         pe_rsc_trace(rsc, "on-fail %s -> %s for %s (%s)", fail2text(*on_fail),
                      fail2text(action->on_fail), action->uuid, key);
         *on_fail = action->on_fail;
     }
 
     if (safe_str_eq(task, CRMD_ACTION_STOP)) {
         resource_location(rsc, node, -INFINITY, "__stop_fail__", data_set);
 
     } else if (safe_str_eq(task, CRMD_ACTION_MIGRATE) || safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
         unpack_rsc_migration_failure(rsc, node, xml_op, data_set);
 
     } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
         rsc->role = RSC_ROLE_MASTER;
 
     } else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
         /*
          * staying in role=master ends up putting the PE/TE into a loop
          * setting role=slave is not dangerous because no master will be
          * promoted until the failed resource has been fully stopped
          */
         if (action->on_fail == action_fail_block) {
             rsc->role = RSC_ROLE_MASTER;
             rsc->next_role = RSC_ROLE_STOPPED;
 
         } else if(rc == PCMK_OCF_NOT_RUNNING) {
             rsc->role = RSC_ROLE_STOPPED;
 
         } else {
             crm_warn("Forcing %s to stop after a failed demote action", rsc->id);
             rsc->role = RSC_ROLE_SLAVE;
             rsc->next_role = RSC_ROLE_STOPPED;
         }
 
     } else if (compare_version("2.0", op_version) > 0 && safe_str_eq(task, CRMD_ACTION_START)) {
         crm_warn("Compatibility handling for failed op %s on %s", key, node->details->uname);
         resource_location(rsc, node, -INFINITY, "__legacy_start__", data_set);
     }
 
     if(is_probe && rc == PCMK_OCF_NOT_INSTALLED) {
         /* leave stopped */
         pe_rsc_trace(rsc, "Leaving %s stopped", rsc->id);
         rsc->role = RSC_ROLE_STOPPED;
 
     } else if (rsc->role < RSC_ROLE_STARTED) {
         pe_rsc_trace(rsc, "Setting %s active", rsc->id);
         set_active(rsc);
     }
 
     pe_rsc_trace(rsc, "Resource %s: role=%s, unclean=%s, on_fail=%s, fail_role=%s",
                  rsc->id, role2text(rsc->role),
                  node->details->unclean ? "true" : "false",
                  fail2text(action->on_fail), role2text(action->fail_role));
 
     if (action->fail_role != RSC_ROLE_STARTED && rsc->next_role < action->fail_role) {
         rsc->next_role = action->fail_role;
     }
 
     if (action->fail_role == RSC_ROLE_STOPPED) {
         int score = -INFINITY;
 
         resource_t *fail_rsc = rsc;
 
         if (fail_rsc->parent) {
             resource_t *parent = uber_parent(fail_rsc);
 
             if (pe_rsc_is_clone(parent)
                 && is_not_set(parent->flags, pe_rsc_unique)) {
                 /* for clone and master resources, if a child fails on an operation
                  * with on-fail = stop, all the resources fail.  Do this by preventing
                  * the parent from coming up again. */
                 fail_rsc = parent;
             }
         }
         crm_warn("Making sure %s doesn't come up again", fail_rsc->id);
         /* make sure it doesn't come up again */
         g_hash_table_destroy(fail_rsc->allowed_nodes);
         fail_rsc->allowed_nodes = node_hash_from_list(data_set->nodes);
         g_hash_table_foreach(fail_rsc->allowed_nodes, set_node_score, &score);
     }
 
     pe_free_action(action);
 }
 
 static int
 determine_op_status(
     resource_t *rsc, int rc, int target_rc, node_t * node, xmlNode * xml_op, enum action_fail_response * on_fail, pe_working_set_t * data_set) 
 {
     int interval = 0;
     int result = PCMK_LRM_OP_DONE;
 
     const char *key = get_op_key(xml_op);
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
 
     bool is_probe = FALSE;
 
     CRM_ASSERT(rsc);
     crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);
     if (interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
         is_probe = TRUE;
     }
 
     if (target_rc >= 0 && target_rc != rc) {
         result = PCMK_LRM_OP_ERROR;
         pe_rsc_debug(rsc, "%s on %s returned '%s' (%d) instead of the expected value: '%s' (%d)",
                      key, node->details->uname,
                      services_ocf_exitcode_str(rc), rc,
                      services_ocf_exitcode_str(target_rc), target_rc);
     }
 
     /* we could clean this up significantly except for old LRMs and CRMs that
      * didn't include target_rc and liked to remap status
      */
     switch (rc) {
         case PCMK_OCF_OK:
             if (is_probe && target_rc == 7) {
                 result = PCMK_LRM_OP_DONE;
                 pe_rsc_info(rsc, "Operation %s found resource %s active on %s",
                             task, rsc->id, node->details->uname);
 
                 /* legacy code for pre-0.6.5 operations */
             } else if (target_rc < 0 && interval > 0 && rsc->role == RSC_ROLE_MASTER) {
                 /* catch status ops that return 0 instead of 8 while they
                  *   are supposed to be in master mode
                  */
                 result = PCMK_LRM_OP_ERROR;
             }
             break;
 
         case PCMK_OCF_NOT_RUNNING:
             if (is_probe || target_rc == rc || is_not_set(rsc->flags, pe_rsc_managed)) {
                 result = PCMK_LRM_OP_DONE;
                 rsc->role = RSC_ROLE_STOPPED;
 
                 /* clear any previous failure actions */
                 *on_fail = action_fail_ignore;
                 rsc->next_role = RSC_ROLE_UNKNOWN;
 
             } else if (safe_str_neq(task, CRMD_ACTION_STOP)) {
                 result = PCMK_LRM_OP_ERROR;
             }
             break;
 
         case PCMK_OCF_RUNNING_MASTER:
             if (is_probe) {
                 result = PCMK_LRM_OP_DONE;
                 pe_rsc_info(rsc, "Operation %s found resource %s active in master mode on %s",
                             task, rsc->id, node->details->uname);
 
             } else if (target_rc == rc) {
                 /* nothing to do */
 
             } else if (target_rc >= 0) {
                 result = PCMK_LRM_OP_ERROR;
 
                 /* legacy code for pre-0.6.5 operations */
             } else if (safe_str_neq(task, CRMD_ACTION_STATUS)
                        || rsc->role != RSC_ROLE_MASTER) {
                 result = PCMK_LRM_OP_ERROR;
                 if (rsc->role != RSC_ROLE_MASTER) {
                     crm_err("%s reported %s in master mode on %s",
                             key, rsc->id, node->details->uname);
                 }
             }
             rsc->role = RSC_ROLE_MASTER;
             break;
 
         case PCMK_OCF_DEGRADED_MASTER:
         case PCMK_OCF_FAILED_MASTER:
             rsc->role = RSC_ROLE_MASTER;
             result = PCMK_LRM_OP_ERROR;
             break;
 
         case PCMK_OCF_NOT_CONFIGURED:
             result = PCMK_LRM_OP_ERROR_FATAL;
             break;
 
         case PCMK_OCF_NOT_INSTALLED:
         case PCMK_OCF_INVALID_PARAM:
         case PCMK_OCF_INSUFFICIENT_PRIV:
         case PCMK_OCF_UNIMPLEMENT_FEATURE:
             if (rc == PCMK_OCF_UNIMPLEMENT_FEATURE && interval > 0) {
                 result = PCMK_LRM_OP_NOTSUPPORTED;
                 break;
 
             } else if (pe_can_fence(data_set, node) == FALSE
                && safe_str_eq(task, CRMD_ACTION_STOP)) {
                 /* If a stop fails and we can't fence, there's nothing else we can do */
                 pe_proc_err("No further recovery can be attempted for %s: %s action failed with '%s' (%d)",
                             rsc->id, task, services_ocf_exitcode_str(rc), rc);
                 clear_bit(rsc->flags, pe_rsc_managed);
                 set_bit(rsc->flags, pe_rsc_block);
             }
             result = PCMK_LRM_OP_ERROR_HARD;
             break;
 
         default:
             if (result == PCMK_LRM_OP_DONE) {
                 crm_info("Treating %s (rc=%d) on %s as an ERROR",
                          key, rc, node->details->uname);
                 result = PCMK_LRM_OP_ERROR;
             }
     }
 
     return result;
 }
 
 static bool check_operation_expiry(resource_t *rsc, node_t *node, int rc, xmlNode *xml_op, pe_working_set_t * data_set)
 {
     bool expired = FALSE;
     time_t last_failure = 0;
     int interval = 0;
     int failure_timeout = rsc->failure_timeout;
     const char *key = get_op_key(xml_op);
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     const char *clear_reason = NULL;
 
     /* clearing recurring monitor operation failures automatically
      * needs to be carefully considered */
     if (safe_str_eq(crm_element_value(xml_op, XML_LRM_ATTR_TASK), "monitor") &&
         safe_str_neq(crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL), "0")) {
 
         /* TODO, in the future we should consider not clearing recurring monitor
          * op failures unless the last action for a resource was a "stop" action.
          * otherwise it is possible that clearing the monitor failure will result
          * in the resource being in an undeterministic state.
          *
          * For now we handle this potential undeterministic condition for remote
          * node connection resources by not clearing a recurring monitor op failure
          * until after the node has been fenced. */
 
         if (is_set(data_set->flags, pe_flag_stonith_enabled) &&
             (rsc->remote_reconnect_interval)) {
 
             node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);
             if (remote_node && remote_node->details->remote_was_fenced == 0) {
                 if (strstr(ID(xml_op), "last_failure")) {
                     crm_info("Waiting to clear monitor failure for remote node %s until fencing has occurred", rsc->id); 
                 }
                 /* disabling failure timeout for this operation because we believe
                  * fencing of the remote node should occur first. */ 
                 failure_timeout = 0;
             }
         }
     }
 
     if (failure_timeout > 0) {
         int last_run = 0;
 
         if (crm_element_value_int(xml_op, XML_RSC_OP_LAST_CHANGE, &last_run) == 0) {
             time_t now = get_effective_time(data_set);
 
             if (now > (last_run + failure_timeout)) {
                 expired = TRUE;
             }
         }
     }
 
     if (expired) {
         if (failure_timeout > 0) {
-            if (pe_get_failcount(node, rsc, &last_failure, FALSE, xml_op, data_set)) {
+            if (pe_get_failcount(node, rsc, &last_failure, pe_fc_default,
+                                 xml_op, data_set)) {
 
-                if (pe_get_failcount(node, rsc, &last_failure, TRUE, xml_op, data_set) == 0) {
+                if (pe_get_failcount(node, rsc, &last_failure, pe_fc_effective,
+                                     xml_op, data_set) == 0) {
                     clear_reason = "it expired";
                 } else {
                     expired = FALSE;
                 }
 
             } else if (rsc->remote_reconnect_interval && strstr(ID(xml_op), "last_failure")) {
                 /* always clear last failure when reconnect interval is set */
                 clear_reason = "reconnect interval is set";
             }
         }
 
     } else if (strstr(ID(xml_op), "last_failure") &&
                ((strcmp(task, "start") == 0) || (strcmp(task, "monitor") == 0))) {
 
         op_digest_cache_t *digest_data = NULL;
 
         digest_data = rsc_action_digest_cmp(rsc, xml_op, node, data_set);
 
         if (digest_data->rc == RSC_DIGEST_UNKNOWN) {
             crm_trace("rsc op %s/%s on node %s does not have a op digest to compare against", rsc->id,
                       key, node->details->id);
         } else if (digest_data->rc != RSC_DIGEST_MATCH) {
             clear_reason = "resource parameters have changed";
         }
     }
 
     if (clear_reason != NULL) {
         char *key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
         action_t *clear_op = custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT,
                                            node, FALSE, TRUE, data_set);
 
         add_hash_param(clear_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
 
         crm_notice("Clearing failure of %s on %s because %s " CRM_XS " %s",
                    rsc->id, node->details->uname, clear_reason, clear_op->uuid);
     }
 
     crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);
     if(expired && interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
         switch(rc) {
             case PCMK_OCF_OK:
             case PCMK_OCF_NOT_RUNNING:
             case PCMK_OCF_RUNNING_MASTER:
             case PCMK_OCF_DEGRADED:
             case PCMK_OCF_DEGRADED_MASTER:
                 /* Don't expire probes that return these values */ 
                 expired = FALSE;
                 break;
         }
     }
     
     return expired;
 }
 
 int get_target_rc(xmlNode *xml_op)
 {
     int dummy = 0;
     int target_rc = 0;
     char *dummy_string = NULL;
     const char *key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);
     if (key == NULL) {
         return -1;
     }
 
     decode_transition_key(key, &dummy_string, &dummy, &dummy, &target_rc);
     free(dummy_string);
 
     return target_rc;
 }
 
 static enum action_fail_response
 get_action_on_fail(resource_t *rsc, const char *key, const char *task, pe_working_set_t * data_set) 
 {
     int result = action_fail_recover;
     action_t *action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);
 
     result = action->on_fail;
     pe_free_action(action);
 
     return result;
 }
 
 static void
 update_resource_state(resource_t * rsc, node_t * node, xmlNode * xml_op, const char * task, int rc,
                       xmlNode * last_failure, enum action_fail_response * on_fail, pe_working_set_t * data_set)
 {
     gboolean clear_past_failure = FALSE;
 
     CRM_ASSERT(rsc);
     CRM_ASSERT(xml_op);
 
     if (rc == PCMK_OCF_NOT_RUNNING) {
         clear_past_failure = TRUE;
 
     } else if (rc == PCMK_OCF_NOT_INSTALLED) {
         rsc->role = RSC_ROLE_STOPPED;
 
     } else if (safe_str_eq(task, CRMD_ACTION_STATUS)) {
         if (last_failure) {
             const char *op_key = get_op_key(xml_op);
             const char *last_failure_key = get_op_key(last_failure);
 
             if (safe_str_eq(op_key, last_failure_key)) {
                 clear_past_failure = TRUE;
             }
         }
 
         if (rsc->role < RSC_ROLE_STARTED) {
             set_active(rsc);
         }
 
     } else if (safe_str_eq(task, CRMD_ACTION_START)) {
         rsc->role = RSC_ROLE_STARTED;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_STOP)) {
         rsc->role = RSC_ROLE_STOPPED;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
         rsc->role = RSC_ROLE_MASTER;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
         /* Demote from Master does not clear an error */
         rsc->role = RSC_ROLE_SLAVE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
         rsc->role = RSC_ROLE_STARTED;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_MIGRATE)) {
         unpack_rsc_migration(rsc, node, xml_op, data_set);
 
     } else if (rsc->role < RSC_ROLE_STARTED) {
         pe_rsc_trace(rsc, "%s active on %s", rsc->id, node->details->uname);
         set_active(rsc);
     }
 
     /* clear any previous failure actions */
     if (clear_past_failure) {
         switch (*on_fail) {
             case action_fail_stop:
             case action_fail_fence:
             case action_fail_migrate:
             case action_fail_standby:
                 pe_rsc_trace(rsc, "%s.%s is not cleared by a completed stop",
                              rsc->id, fail2text(*on_fail));
                 break;
 
             case action_fail_block:
             case action_fail_ignore:
             case action_fail_recover:
             case action_fail_restart_container:
                 *on_fail = action_fail_ignore;
                 rsc->next_role = RSC_ROLE_UNKNOWN;
                 break;
             case action_fail_reset_remote:
                 if (rsc->remote_reconnect_interval == 0) {
                     /* when reconnect delay is not in use, the connection is allowed
                      * to start again after the remote node is fenced and completely
                      * stopped. Otherwise, with reconnect delay we wait for the failure
                      * to be cleared entirely before reconnected can be attempted. */ 
                     *on_fail = action_fail_ignore;
                     rsc->next_role = RSC_ROLE_UNKNOWN;
                 }
                 break;
         }
     }
 }
 
 
 gboolean
 unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, xmlNode ** last_failure,
               enum action_fail_response * on_fail, pe_working_set_t * data_set)
 {
     int task_id = 0;
 
     const char *key = NULL;
     const char *task = NULL;
     const char *task_key = NULL;
 
     int rc = 0;
     int status = PCMK_LRM_OP_PENDING-1;
     int target_rc = get_target_rc(xml_op);
     int interval = 0;
 
     gboolean expired = FALSE;
     resource_t *parent = rsc;
     enum action_fail_response failure_strategy = action_fail_recover;
 
     CRM_CHECK(rsc != NULL, return FALSE);
     CRM_CHECK(node != NULL, return FALSE);
     CRM_CHECK(xml_op != NULL, return FALSE);
 
     task_key = get_op_key(xml_op);
 
     task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);
 
     crm_element_value_int(xml_op, XML_LRM_ATTR_RC, &rc);
     crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &task_id);
     crm_element_value_int(xml_op, XML_LRM_ATTR_OPSTATUS, &status);
     crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);
 
     CRM_CHECK(task != NULL, return FALSE);
     CRM_CHECK(status <= PCMK_LRM_OP_NOT_INSTALLED, return FALSE);
     CRM_CHECK(status >= PCMK_LRM_OP_PENDING, return FALSE);
 
     if (safe_str_eq(task, CRMD_ACTION_NOTIFY) ||
         safe_str_eq(task, CRMD_ACTION_METADATA)) {
         /* safe to ignore these */
         return TRUE;
     }
 
     if (is_not_set(rsc->flags, pe_rsc_unique)) {
         parent = uber_parent(rsc);
     }
 
     pe_rsc_trace(rsc, "Unpacking task %s/%s (call_id=%d, status=%d, rc=%d) on %s (role=%s)",
                  task_key, task, task_id, status, rc, node->details->uname, role2text(rsc->role));
 
     if (node->details->unclean) {
         pe_rsc_trace(rsc, "Node %s (where %s is running) is unclean."
                      " Further action depends on the value of the stop's on-fail attribute",
                      node->details->uname, rsc->id);
     }
 
     if (status == PCMK_LRM_OP_ERROR) {
         /* Older versions set this if rc != 0 but it's up to us to decide */
         status = PCMK_LRM_OP_DONE;
     }
 
     if(status != PCMK_LRM_OP_NOT_INSTALLED) {
         expired = check_operation_expiry(rsc, node, rc, xml_op, data_set);
     }
 
     /* Degraded results are informational only, re-map them to their error-free equivalents */
     if (rc == PCMK_OCF_DEGRADED && safe_str_eq(task, CRMD_ACTION_STATUS)) {
         rc = PCMK_OCF_OK;
 
         /* Add them to the failed list to highlight them for the user */
         if ((node->details->shutdown == FALSE) || (node->details->online == TRUE)) {
             crm_trace("Remapping %d to %d", PCMK_OCF_DEGRADED, PCMK_OCF_OK);
             record_failed_op(xml_op, node, data_set);
         }
 
     } else if (rc == PCMK_OCF_DEGRADED_MASTER && safe_str_eq(task, CRMD_ACTION_STATUS)) {
         rc = PCMK_OCF_RUNNING_MASTER;
 
         /* Add them to the failed list to highlight them for the user */
         if ((node->details->shutdown == FALSE) || (node->details->online == TRUE)) {
             crm_trace("Remapping %d to %d", PCMK_OCF_DEGRADED_MASTER, PCMK_OCF_RUNNING_MASTER);
             record_failed_op(xml_op, node, data_set);
         }
     }
 
     if (expired && target_rc != rc) {
         const char *magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);
 
         pe_rsc_debug(rsc, "Expired operation '%s' on %s returned '%s' (%d) instead of the expected value: '%s' (%d)",
                      key, node->details->uname,
                      services_ocf_exitcode_str(rc), rc,
                      services_ocf_exitcode_str(target_rc), target_rc);
 
         if(interval == 0) {
             crm_notice("Ignoring expired calculated failure %s (rc=%d, magic=%s) on %s",
                        task_key, rc, magic, node->details->uname);
             goto done;
 
         } else if(node->details->online && node->details->unclean == FALSE) {
             crm_notice("Re-initiated expired calculated failure %s (rc=%d, magic=%s) on %s",
                        task_key, rc, magic, node->details->uname);
             /* This is SO horrible, but we don't have access to CancelXmlOp() yet */
             crm_xml_add(xml_op, XML_LRM_ATTR_RESTART_DIGEST, "calculated-failure-timeout");
             goto done;
         }
     }
 
     if(status == PCMK_LRM_OP_DONE || status == PCMK_LRM_OP_ERROR) {
         status = determine_op_status(rsc, rc, target_rc, node, xml_op, on_fail, data_set);
     }
 
     pe_rsc_trace(rsc, "Handling status: %d", status);
     switch (status) {
         case PCMK_LRM_OP_CANCELLED:
             /* do nothing?? */
             pe_err("Don't know what to do for cancelled ops yet");
             break;
 
         case PCMK_LRM_OP_PENDING:
             if (safe_str_eq(task, CRMD_ACTION_START)) {
                 set_bit(rsc->flags, pe_rsc_start_pending);
                 set_active(rsc);
 
             } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
                 rsc->role = RSC_ROLE_MASTER;
 
             } else if (safe_str_eq(task, CRMD_ACTION_MIGRATE) && node->details->unclean) {
                 /* If a pending migrate_to action is out on a unclean node,
                  * we have to force the stop action on the target. */
                 const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
                 node_t *target = pe_find_node(data_set->nodes, migrate_target);
                 if (target) {
                     stop_action(rsc, target, FALSE);
                 }
             }
 
             if (rsc->pending_task == NULL) {
                 if (safe_str_eq(task, CRMD_ACTION_STATUS) && interval == 0) {
                     /* Pending probes are not printed, even if pending
                      * operations are requested. If someone ever requests that
                      * behavior, uncomment this and the corresponding part of
                      * native.c:native_pending_task().
                      */
                     /*rsc->pending_task = strdup("probe");*/
 
                 } else {
                     rsc->pending_task = strdup(task);
                 }
             }
             break;
 
         case PCMK_LRM_OP_DONE:
             pe_rsc_trace(rsc, "%s/%s completed on %s", rsc->id, task, node->details->uname);
             update_resource_state(rsc, node, xml_op, task, rc, *last_failure, on_fail, data_set);
             break;
 
         case PCMK_LRM_OP_NOT_INSTALLED:
             failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
             if (failure_strategy == action_fail_ignore) {
                 crm_warn("Cannot ignore failed %s (status=%d, rc=%d) on %s: "
                          "Resource agent doesn't exist",
                          task_key, status, rc, node->details->uname);
                 /* Also for printing it as "FAILED" by marking it as pe_rsc_failed later */
                 *on_fail = action_fail_migrate;
             }
             resource_location(parent, node, -INFINITY, "hard-error", data_set);
             unpack_rsc_op_failure(rsc, node, rc, xml_op, last_failure, on_fail, data_set);
             break;
 
         case PCMK_LRM_OP_ERROR:
         case PCMK_LRM_OP_ERROR_HARD:
         case PCMK_LRM_OP_ERROR_FATAL:
         case PCMK_LRM_OP_TIMEOUT:
         case PCMK_LRM_OP_NOTSUPPORTED:
 
             failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
             if ((failure_strategy == action_fail_ignore)
                 || (failure_strategy == action_fail_restart_container
                     && safe_str_eq(task, CRMD_ACTION_STOP))) {
 
                 crm_warn("Pretending the failure of %s (rc=%d) on %s succeeded",
                          task_key, rc, node->details->uname);
 
                 update_resource_state(rsc, node, xml_op, task, target_rc, *last_failure, on_fail, data_set);
                 crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
                 set_bit(rsc->flags, pe_rsc_failure_ignored);
 
                 record_failed_op(xml_op, node, data_set);
 
                 if (failure_strategy == action_fail_restart_container && *on_fail <= action_fail_recover) {
                     *on_fail = failure_strategy;
                 }
 
             } else {
                 unpack_rsc_op_failure(rsc, node, rc, xml_op, last_failure, on_fail, data_set);
 
                 if(status == PCMK_LRM_OP_ERROR_HARD) {
                     do_crm_log(rc != PCMK_OCF_NOT_INSTALLED?LOG_ERR:LOG_NOTICE,
                                "Preventing %s from re-starting on %s: operation %s failed '%s' (%d)",
                                parent->id, node->details->uname,
                                task, services_ocf_exitcode_str(rc), rc);
 
                     resource_location(parent, node, -INFINITY, "hard-error", data_set);
 
                 } else if(status == PCMK_LRM_OP_ERROR_FATAL) {
                     crm_err("Preventing %s from re-starting anywhere: operation %s failed '%s' (%d)",
                             parent->id, task, services_ocf_exitcode_str(rc), rc);
 
                     resource_location(parent, NULL, -INFINITY, "fatal-error", data_set);
                 }
             }
             break;
     }
 
   done:
     pe_rsc_trace(rsc, "Resource %s after %s: role=%s, next=%s", rsc->id, task, role2text(rsc->role), role2text(rsc->next_role));
     return TRUE;
 }
 
 gboolean
 add_node_attrs(xmlNode * xml_obj, node_t * node, gboolean overwrite, pe_working_set_t * data_set)
 {
     const char *cluster_name = NULL;
 
     g_hash_table_insert(node->details->attrs,
                         strdup(CRM_ATTR_UNAME), strdup(node->details->uname));
 
     g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_ID),
                         strdup(node->details->id));
     if (safe_str_eq(node->details->id, data_set->dc_uuid)) {
         data_set->dc_node = node;
         node->details->is_dc = TRUE;
         g_hash_table_insert(node->details->attrs,
                             strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_TRUE));
     } else {
         g_hash_table_insert(node->details->attrs,
                             strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_FALSE));
     }
 
     cluster_name = g_hash_table_lookup(data_set->config_hash, "cluster-name");
     if (cluster_name) {
         g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_CLUSTER_NAME),
                             strdup(cluster_name));
     }
 
     unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_ATTR_SETS, NULL,
                                node->details->attrs, NULL, overwrite, data_set->now);
 
     if (pe_node_attribute_raw(node, CRM_ATTR_SITE_NAME) == NULL) {
         const char *site_name = pe_node_attribute_raw(node, "site-name");
 
         if (site_name) {
             g_hash_table_insert(node->details->attrs,
                                 strdup(CRM_ATTR_SITE_NAME),
                                 strdup(site_name));
 
         } else if (cluster_name) {
             /* Default to cluster-name if unset */
             g_hash_table_insert(node->details->attrs,
                                 strdup(CRM_ATTR_SITE_NAME),
                                 strdup(cluster_name));
         }
     }
     return TRUE;
 }
 
 static GListPtr
 extract_operations(const char *node, const char *rsc, xmlNode * rsc_entry, gboolean active_filter)
 {
     int counter = -1;
     int stop_index = -1;
     int start_index = -1;
 
     xmlNode *rsc_op = NULL;
 
     GListPtr gIter = NULL;
     GListPtr op_list = NULL;
     GListPtr sorted_op_list = NULL;
 
     /* extract operations */
     op_list = NULL;
     sorted_op_list = NULL;
 
     for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             crm_xml_add(rsc_op, "resource", rsc);
             crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
             op_list = g_list_prepend(op_list, rsc_op);
         }
     }
 
     if (op_list == NULL) {
         /* if there are no operations, there is nothing to do */
         return NULL;
     }
 
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 
     /* create active recurring operations as optional */
     if (active_filter == FALSE) {
         return sorted_op_list;
     }
 
     op_list = NULL;
 
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         counter++;
 
         if (start_index < stop_index) {
             crm_trace("Skipping %s: not active", ID(rsc_entry));
             break;
 
         } else if (counter < start_index) {
             crm_trace("Skipping %s: old", ID(rsc_op));
             continue;
         }
         op_list = g_list_append(op_list, rsc_op);
     }
 
     g_list_free(sorted_op_list);
     return op_list;
 }
 
 GListPtr
 find_operations(const char *rsc, const char *node, gboolean active_filter,
                 pe_working_set_t * data_set)
 {
     GListPtr output = NULL;
     GListPtr intermediate = NULL;
 
     xmlNode *tmp = NULL;
     xmlNode *status = find_xml_node(data_set->input, XML_CIB_TAG_STATUS, TRUE);
 
     node_t *this_node = NULL;
 
     xmlNode *node_state = NULL;
 
     for (node_state = __xml_first_child(status); node_state != NULL;
          node_state = __xml_next_element(node_state)) {
 
         if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
             const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);
 
             if (node != NULL && safe_str_neq(uname, node)) {
                 continue;
             }
 
             this_node = pe_find_node(data_set->nodes, uname);
             if(this_node == NULL) {
                 CRM_LOG_ASSERT(this_node != NULL);
                 continue;
 
             } else if (is_remote_node(this_node)) {
                 determine_remote_online_status(data_set, this_node);
 
             } else {
                 determine_online_status(node_state, this_node, data_set);
             }
 
             if (this_node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) {
                 /* offline nodes run no resources...
                  * unless stonith is enabled in which case we need to
                  *   make sure rsc start events happen after the stonith
                  */
                 xmlNode *lrm_rsc = NULL;
 
                 tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
                 tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);
 
                 for (lrm_rsc = __xml_first_child(tmp); lrm_rsc != NULL;
                      lrm_rsc = __xml_next_element(lrm_rsc)) {
                     if (crm_str_eq((const char *)lrm_rsc->name, XML_LRM_TAG_RESOURCE, TRUE)) {
 
                         const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID);
 
                         if (rsc != NULL && safe_str_neq(rsc_id, rsc)) {
                             continue;
                         }
 
                         intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
                         output = g_list_concat(output, intermediate);
                     }
                 }
             }
         }
     }
 
     return output;
 }
diff --git a/pengine/allocate.c b/pengine/allocate.c
index 42544c0b96..b550b01638 100644
--- a/pengine/allocate.c
+++ b/pengine/allocate.c
@@ -1,2553 +1,2558 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <glib.h>
 
 #include <crm/pengine/status.h>
 #include <pengine.h>
 #include <allocate.h>
 #include <utils.h>
 
 CRM_TRACE_INIT_DATA(pe_allocate);
 
 void set_alloc_actions(pe_working_set_t * data_set);
 extern void ReloadRsc(resource_t * rsc, node_t *node, pe_working_set_t * data_set);
 extern gboolean DeleteRsc(resource_t * rsc, node_t * node, gboolean optional, pe_working_set_t * data_set);
 static void apply_remote_node_ordering(pe_working_set_t *data_set);
 static enum remote_connection_state get_remote_node_state(pe_node_t *node);
 
 enum remote_connection_state {
     remote_state_unknown = 0,
     remote_state_alive = 1,
     remote_state_resting = 2,
     remote_state_failed = 3,
     remote_state_stopped = 4
 };
 
 resource_alloc_functions_t resource_class_alloc_functions[] = {
     {
      native_merge_weights,
      native_color,
      native_create_actions,
      native_create_probe,
      native_internal_constraints,
      native_rsc_colocation_lh,
      native_rsc_colocation_rh,
      native_rsc_location,
      native_action_flags,
      native_update_actions,
      native_expand,
      native_append_meta,
      },
     {
      group_merge_weights,
      group_color,
      group_create_actions,
      native_create_probe,
      group_internal_constraints,
      group_rsc_colocation_lh,
      group_rsc_colocation_rh,
      group_rsc_location,
      group_action_flags,
      group_update_actions,
      group_expand,
      group_append_meta,
      },
     {
      clone_merge_weights,
      clone_color,
      clone_create_actions,
      clone_create_probe,
      clone_internal_constraints,
      clone_rsc_colocation_lh,
      clone_rsc_colocation_rh,
      clone_rsc_location,
      clone_action_flags,
      container_update_actions,
      clone_expand,
      clone_append_meta,
      },
     {
      master_merge_weights,
      master_color,
      master_create_actions,
      clone_create_probe,
      master_internal_constraints,
      clone_rsc_colocation_lh,
      master_rsc_colocation_rh,
      clone_rsc_location,
      clone_action_flags,
      container_update_actions,
      clone_expand,
      master_append_meta,
      },
     {
      container_merge_weights,
      container_color,
      container_create_actions,
      container_create_probe,
      container_internal_constraints,
      container_rsc_colocation_lh,
      container_rsc_colocation_rh,
      container_rsc_location,
      container_action_flags,
      container_update_actions,
      container_expand,
      container_append_meta,
      }
 };
 
 gboolean
 update_action_flags(action_t * action, enum pe_action_flags flags, const char *source, int line)
 {
     static unsigned long calls = 0;
     gboolean changed = FALSE;
     gboolean clear = is_set(flags, pe_action_clear);
     enum pe_action_flags last = action->flags;
 
     if (clear) {
         action->flags = crm_clear_bit(source, line, action->uuid, action->flags, flags);
     } else {
         action->flags = crm_set_bit(source, line, action->uuid, action->flags, flags);
     }
 
     if (last != action->flags) {
         calls++;
         changed = TRUE;
         /* Useful for tracking down _who_ changed a specific flag */
         /* CRM_ASSERT(calls != 534); */
         clear_bit(flags, pe_action_clear);
         crm_trace("%s on %s: %sset flags 0x%.6x (was 0x%.6x, now 0x%.6x, %lu, %s)",
                   action->uuid, action->node ? action->node->details->uname : "[none]",
                   clear ? "un-" : "", flags, last, action->flags, calls, source);
     }
 
     return changed;
 }
 
 static gboolean
 check_rsc_parameters(resource_t * rsc, node_t * node, xmlNode * rsc_entry,
                      gboolean active_here, pe_working_set_t * data_set)
 {
     int attr_lpc = 0;
     gboolean force_restart = FALSE;
     gboolean delete_resource = FALSE;
     gboolean changed = FALSE;
 
     const char *value = NULL;
     const char *old_value = NULL;
 
     const char *attr_list[] = {
         XML_ATTR_TYPE,
         XML_AGENT_ATTR_CLASS,
         XML_AGENT_ATTR_PROVIDER
     };
 
     for (; attr_lpc < DIMOF(attr_list); attr_lpc++) {
         value = crm_element_value(rsc->xml, attr_list[attr_lpc]);
         old_value = crm_element_value(rsc_entry, attr_list[attr_lpc]);
         if (value == old_value  /* i.e. NULL */
             || crm_str_eq(value, old_value, TRUE)) {
             continue;
         }
 
         changed = TRUE;
         trigger_unfencing(rsc, node, "Device definition changed", NULL, data_set);
         if (active_here) {
             force_restart = TRUE;
             crm_notice("Forcing restart of %s on %s, %s changed: %s -> %s",
                        rsc->id, node->details->uname, attr_list[attr_lpc],
                        crm_str(old_value), crm_str(value));
         }
     }
     if (force_restart) {
         /* make sure the restart happens */
         stop_action(rsc, node, FALSE);
         set_bit(rsc->flags, pe_rsc_start_pending);
         delete_resource = TRUE;
 
     } else if (changed) {
         delete_resource = TRUE;
     }
     return delete_resource;
 }
 
 static void
 CancelXmlOp(resource_t * rsc, xmlNode * xml_op, node_t * active_node,
             const char *reason, pe_working_set_t * data_set)
 {
     int interval = 0;
     action_t *cancel = NULL;
 
     char *key = NULL;
     const char *task = NULL;
     const char *call_id = NULL;
     const char *interval_s = NULL;
 
     CRM_CHECK(xml_op != NULL, return);
     CRM_CHECK(active_node != NULL, return);
 
     task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
     interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
 
     interval = crm_parse_int(interval_s, "0");
 
     /* we need to reconstruct the key because of the way we used to construct resource IDs */
     key = generate_op_key(rsc->id, task, interval);
 
     crm_info("Action %s on %s will be stopped: %s",
              key, active_node->details->uname, reason ? reason : "unknown");
 
     /* TODO: This looks highly dangerous if we ever try to schedule 'key' too */
     cancel = custom_action(rsc, strdup(key), RSC_CANCEL, active_node, FALSE, TRUE, data_set);
 
     free(cancel->task);
     free(cancel->cancel_task);
     cancel->task = strdup(RSC_CANCEL);
     cancel->cancel_task = strdup(task);
 
     add_hash_param(cancel->meta, XML_LRM_ATTR_TASK, task);
     add_hash_param(cancel->meta, XML_LRM_ATTR_CALLID, call_id);
     add_hash_param(cancel->meta, XML_LRM_ATTR_INTERVAL, interval_s);
 
     custom_action_order(rsc, stop_key(rsc), NULL, rsc, NULL, cancel, pe_order_optional, data_set);
     free(key);
     key = NULL;
 }
 
 static gboolean
 check_action_definition(resource_t * rsc, node_t * active_node, xmlNode * xml_op,
                         pe_working_set_t * data_set)
 {
     char *key = NULL;
     int interval = 0;
     const char *interval_s = NULL;
     const op_digest_cache_t *digest_data = NULL;
     gboolean did_change = FALSE;
 
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     const char *op_version;
     const char *digest_secure = NULL;
 
     CRM_CHECK(active_node != NULL, return FALSE);
     if (safe_str_eq(task, RSC_STOP)) {
         return FALSE;
     }
 
     interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
     interval = crm_parse_int(interval_s, "0");
 
     if (interval > 0) {
         xmlNode *op_match = NULL;
 
         /* we need to reconstruct the key because of the way we used to construct resource IDs */
         key = generate_op_key(rsc->id, task, interval);
 
         pe_rsc_trace(rsc, "Checking parameters for %s", key);
         op_match = find_rsc_op_entry(rsc, key);
 
         if (op_match == NULL && is_set(data_set->flags, pe_flag_stop_action_orphans)) {
             CancelXmlOp(rsc, xml_op, active_node, "orphan", data_set);
             free(key);
             return TRUE;
 
         } else if (op_match == NULL) {
             pe_rsc_debug(rsc, "Orphan action detected: %s on %s", key, active_node->details->uname);
             free(key);
             return TRUE;
         }
         free(key);
         key = NULL;
     }
 
     crm_trace("Testing %s_%s_%d on %s",
               rsc->id, task, interval, active_node->details->uname);
     if (interval == 0 && safe_str_eq(task, RSC_STATUS)) {
         /* Reload based on the start action not a probe */
         task = RSC_START;
 
     } else if (interval == 0 && safe_str_eq(task, RSC_MIGRATED)) {
         /* Reload based on the start action not a migrate */
         task = RSC_START;
     } else if (interval == 0 && safe_str_eq(task, RSC_PROMOTE)) {
         /* Reload based on the start action not a promote */
         task = RSC_START;
     }
 
     op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
     digest_data = rsc_action_digest_cmp(rsc, xml_op, active_node, data_set);
 
     if(is_set(data_set->flags, pe_flag_sanitized)) {
         digest_secure = crm_element_value(xml_op, XML_LRM_ATTR_SECURE_DIGEST);
     }
 
     if(digest_data->rc != RSC_DIGEST_MATCH
        && digest_secure
        && digest_data->digest_secure_calc
        && strcmp(digest_data->digest_secure_calc, digest_secure) == 0) {
         if (is_set(data_set->flags, pe_flag_sanitized)) {
             printf("Only 'private' parameters to %s_%s_%d on %s changed: %s\n",
                    rsc->id, task, interval, active_node->details->uname,
                    crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
         }
 
     } else if (digest_data->rc == RSC_DIGEST_RESTART) {
         /* Changes that force a restart */
         const char *digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);
 
         did_change = TRUE;
         key = generate_op_key(rsc->id, task, interval);
         crm_log_xml_info(digest_data->params_restart, "params:restart");
         pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (restart:%s) %s",
                  key, active_node->details->uname,
                  crm_str(digest_restart), digest_data->digest_restart_calc,
                  op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
 
         custom_action(rsc, key, task, NULL, FALSE, TRUE, data_set);
         trigger_unfencing(rsc, active_node, "Device parameters changed", NULL, data_set);
 
     } else if ((digest_data->rc == RSC_DIGEST_ALL) || (digest_data->rc == RSC_DIGEST_UNKNOWN)) {
         /* Changes that can potentially be handled by a reload */
         const char *digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);
         const char *digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
 
         did_change = TRUE;
         trigger_unfencing(rsc, active_node, "Device parameters changed (reload)", NULL, data_set);
         crm_log_xml_info(digest_data->params_all, "params:reload");
         key = generate_op_key(rsc->id, task, interval);
         pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (reload:%s) %s",
                  key, active_node->details->uname,
                  crm_str(digest_all), digest_data->digest_all_calc, op_version,
                  crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
 
         if (interval > 0) {
             action_t *op = NULL;
 
 #if 0
             /* Always reload/restart the entire resource */
             ReloadRsc(rsc, active_node, data_set);
 #else
             /* Re-sending the recurring op is sufficient - the old one will be cancelled automatically */
             op = custom_action(rsc, key, task, active_node, TRUE, TRUE, data_set);
             set_bit(op->flags, pe_action_reschedule);
 #endif
 
         } else if (digest_restart && rsc->isolation_wrapper == NULL && (uber_parent(rsc))->isolation_wrapper == NULL) {
             pe_rsc_trace(rsc, "Reloading '%s' action for resource %s", task, rsc->id);
 
             /* Reload this resource */
             ReloadRsc(rsc, active_node, data_set);
             free(key);
 
         } else {
             pe_rsc_trace(rsc, "Resource %s doesn't know how to reload", rsc->id);
 
             /* Re-send the start/demote/promote op
              * Recurring ops will be detected independently
              */
             custom_action(rsc, key, task, NULL, FALSE, TRUE, data_set);
         }
     }
 
     return did_change;
 }
 
 
 static void
 check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     int offset = -1;
     int interval = 0;
     int stop_index = 0;
     int start_index = 0;
 
     const char *task = NULL;
     const char *interval_s = NULL;
 
     xmlNode *rsc_op = NULL;
     GListPtr op_list = NULL;
     GListPtr sorted_op_list = NULL;
     gboolean is_probe = FALSE;
     gboolean did_change = FALSE;
 
     CRM_CHECK(node != NULL, return);
 
     if (is_set(rsc->flags, pe_rsc_orphan)) {
         resource_t *parent = uber_parent(rsc);
         if(parent == NULL
            || pe_rsc_is_clone(parent) == FALSE
            || is_set(parent->flags, pe_rsc_unique)) {
             pe_rsc_trace(rsc, "Skipping param check for %s and deleting: orphan", rsc->id);
             DeleteRsc(rsc, node, FALSE, data_set);
         } else {
             pe_rsc_trace(rsc, "Skipping param check for %s (orphan clone)", rsc->id);
         }
         return;
 
     } else if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
         if (check_rsc_parameters(rsc, node, rsc_entry, FALSE, data_set)) {
             DeleteRsc(rsc, node, FALSE, data_set);
         }
         pe_rsc_trace(rsc, "Skipping param check for %s: no longer active on %s",
                      rsc->id, node->details->uname);
         return;
     }
 
     pe_rsc_trace(rsc, "Processing %s on %s", rsc->id, node->details->uname);
 
     if (check_rsc_parameters(rsc, node, rsc_entry, TRUE, data_set)) {
         DeleteRsc(rsc, node, FALSE, data_set);
     }
 
     for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             op_list = g_list_prepend(op_list, rsc_op);
         }
     }
 
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         offset++;
 
         if (start_index < stop_index) {
             /* stopped */
             continue;
         } else if (offset < start_index) {
             /* action occurred prior to a start */
             continue;
         }
 
         is_probe = FALSE;
         did_change = FALSE;
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
 
         interval_s = crm_element_value(rsc_op, XML_LRM_ATTR_INTERVAL);
         interval = crm_parse_int(interval_s, "0");
 
         if (interval == 0 && safe_str_eq(task, RSC_STATUS)) {
             is_probe = TRUE;
         }
 
         if (interval > 0 &&
             (is_set(rsc->flags, pe_rsc_maintenance) || node->details->maintenance)) {
             CancelXmlOp(rsc, rsc_op, node, "maintenance mode", data_set);
 
         } else if (is_probe || safe_str_eq(task, RSC_START) || safe_str_eq(task, RSC_PROMOTE) || interval > 0
                    || safe_str_eq(task, RSC_MIGRATED)) {
             did_change = check_action_definition(rsc, node, rsc_op, data_set);
         }
 
-        if (did_change && pe_get_failcount(node, rsc, NULL, TRUE, NULL, data_set)) {
+        if (did_change && pe_get_failcount(node, rsc, NULL, pe_fc_effective,
+                                           NULL, data_set)) {
+
             char *key = NULL;
             action_t *action_clear = NULL;
 
             key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
             action_clear =
                 custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE, data_set);
             set_bit(action_clear->flags, pe_action_runnable);
 
             crm_notice("Clearing failure of %s on %s "
                        "because action definition changed " CRM_XS " %s",
                        rsc->id, node->details->uname, action_clear->uuid);
         }
     }
 
     g_list_free(sorted_op_list);
 
 }
 
 static GListPtr
 find_rsc_list(GListPtr result, resource_t * rsc, const char *id, gboolean renamed_clones,
               gboolean partial, pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     gboolean match = FALSE;
 
     if (id == NULL) {
         return NULL;
 
     } else if (rsc == NULL && data_set) {
 
         for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
             resource_t *child = (resource_t *) gIter->data;
 
             result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
         }
 
         return result;
 
     } else if (rsc == NULL) {
         return NULL;
     }
 
     if (partial) {
         if (strstr(rsc->id, id)) {
             match = TRUE;
 
         } else if (renamed_clones && rsc->clone_name && strstr(rsc->clone_name, id)) {
             match = TRUE;
         }
 
     } else {
         if (strcmp(rsc->id, id) == 0) {
             match = TRUE;
 
         } else if (renamed_clones && rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
             match = TRUE;
         }
     }
 
     if (match) {
         result = g_list_prepend(result, rsc);
     }
 
     if (rsc->children) {
         gIter = rsc->children;
         for (; gIter != NULL; gIter = gIter->next) {
             resource_t *child = (resource_t *) gIter->data;
 
             result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
         }
     }
 
     return result;
 }
 
 static void
 check_actions(pe_working_set_t * data_set)
 {
     const char *id = NULL;
     node_t *node = NULL;
     xmlNode *lrm_rscs = NULL;
     xmlNode *status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);
 
     xmlNode *node_state = NULL;
 
     for (node_state = __xml_first_child(status); node_state != NULL;
          node_state = __xml_next_element(node_state)) {
         if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
             id = crm_element_value(node_state, XML_ATTR_ID);
             lrm_rscs = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
             lrm_rscs = find_xml_node(lrm_rscs, XML_LRM_TAG_RESOURCES, FALSE);
 
             node = pe_find_node_id(data_set->nodes, id);
 
             if (node == NULL) {
                 continue;
 
             /* Still need to check actions for a maintenance node to cancel existing monitor operations */
             } else if (can_run_resources(node) == FALSE && node->details->maintenance == FALSE) {
                 crm_trace("Skipping param check for %s: can't run resources",
                           node->details->uname);
                 continue;
             }
 
             crm_trace("Processing node %s", node->details->uname);
             if (node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) {
                 xmlNode *rsc_entry = NULL;
 
                 for (rsc_entry = __xml_first_child(lrm_rscs); rsc_entry != NULL;
                      rsc_entry = __xml_next_element(rsc_entry)) {
                     if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
 
                         if (xml_has_children(rsc_entry)) {
                             GListPtr gIter = NULL;
                             GListPtr result = NULL;
                             const char *rsc_id = ID(rsc_entry);
 
                             CRM_CHECK(rsc_id != NULL, return);
 
                             result = find_rsc_list(NULL, NULL, rsc_id, TRUE, FALSE, data_set);
                             for (gIter = result; gIter != NULL; gIter = gIter->next) {
                                 resource_t *rsc = (resource_t *) gIter->data;
 
                                 if (rsc->variant != pe_native) {
                                     continue;
                                 }
                                 check_actions_for(rsc_entry, rsc, node, data_set);
                             }
                             g_list_free(result);
                         }
                     }
                 }
             }
         }
     }
 }
 
 static gboolean
 apply_placement_constraints(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
 
     crm_trace("Applying constraints...");
 
     for (gIter = data_set->placement_constraints; gIter != NULL; gIter = gIter->next) {
         rsc_to_node_t *cons = (rsc_to_node_t *) gIter->data;
 
         cons->rsc_lh->cmds->rsc_location(cons->rsc_lh, cons);
     }
 
     return TRUE;
 
 }
 
 static gboolean
 failcount_clear_action_exists(node_t * node, resource_t * rsc)
 {
     gboolean rc = FALSE;
     char *key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
     GListPtr list = find_actions_exact(rsc->actions, key, node);
 
     if (list) {
         rc = TRUE;
     }
     g_list_free(list);
     free(key);
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Force resource away if failures hit migration threshold
  *
  * \param[in,out] rsc       Resource to check for failures
  * \param[in,out] node      Node to check for failures
  * \param[in,out] data_set  Cluster working set to update
  */
 static void
 check_migration_threshold(resource_t *rsc, node_t *node,
                           pe_working_set_t *data_set)
 {
     int fail_count, countdown;
     resource_t *failed;
 
     /* Migration threshold of 0 means never force away */
     if (rsc->migration_threshold == 0) {
         return;
     }
 
     // If we're ignoring failures, also ignore the migration threshold
     if (is_set(rsc->flags, pe_rsc_failure_ignored)) {
         return;
     }
 
     /* If there are no failures, there's no need to force away */
     fail_count = get_failcount_all(node, rsc, NULL, data_set);
     if (fail_count <= 0) {
         return;
     }
 
     /* How many more times recovery will be tried on this node */
     countdown = QB_MAX(rsc->migration_threshold - fail_count, 0);
 
     /* If failed resource has a parent, we'll force the parent away */
     failed = rsc;
     if (is_not_set(rsc->flags, pe_rsc_unique)) {
         failed = uber_parent(rsc);
     }
 
     if (countdown == 0) {
         resource_location(failed, node, -INFINITY, "__fail_limit__", data_set);
         crm_warn("Forcing %s away from %s after %d failures (max=%d)",
                  failed->id, node->details->uname, fail_count,
                  rsc->migration_threshold);
     } else {
         crm_info("%s can fail %d more times on %s before being forced off",
                  failed->id, countdown, node->details->uname);
     }
 }
 
 static void
 common_apply_stickiness(resource_t * rsc, node_t * node, pe_working_set_t * data_set)
 {
     if (rsc->children) {
         GListPtr gIter = rsc->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             resource_t *child_rsc = (resource_t *) gIter->data;
 
             common_apply_stickiness(child_rsc, node, data_set);
         }
         return;
     }
 
     if (is_set(rsc->flags, pe_rsc_managed)
         && rsc->stickiness != 0 && g_list_length(rsc->running_on) == 1) {
         node_t *current = pe_find_node_id(rsc->running_on, node->details->id);
         node_t *match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
 
         if (current == NULL) {
 
         } else if (match != NULL || is_set(data_set->flags, pe_flag_symmetric_cluster)) {
             resource_t *sticky_rsc = rsc;
 
             resource_location(sticky_rsc, node, rsc->stickiness, "stickiness", data_set);
             pe_rsc_debug(sticky_rsc, "Resource %s: preferring current location"
                          " (node=%s, weight=%d)", sticky_rsc->id,
                          node->details->uname, rsc->stickiness);
         } else {
             GHashTableIter iter;
             node_t *nIter = NULL;
 
             pe_rsc_debug(rsc, "Ignoring stickiness for %s: the cluster is asymmetric"
                          " and node %s is not explicitly allowed", rsc->id, node->details->uname);
             g_hash_table_iter_init(&iter, rsc->allowed_nodes);
             while (g_hash_table_iter_next(&iter, NULL, (void **)&nIter)) {
                 crm_err("%s[%s] = %d", rsc->id, nIter->details->uname, nIter->weight);
             }
         }
     }
 
     /* Check the migration threshold only if a failcount clear action
      * has not already been placed for this resource on the node.
      * There is no sense in potentially forcing the resource from this
      * node if the failcount is being reset anyway. */
     if (failcount_clear_action_exists(node, rsc) == FALSE) {
         check_migration_threshold(rsc, node, data_set);
     }
 }
 
 void
 complex_set_cmds(resource_t * rsc)
 {
     GListPtr gIter = rsc->children;
 
     rsc->cmds = &resource_class_alloc_functions[rsc->variant];
 
     for (; gIter != NULL; gIter = gIter->next) {
         resource_t *child_rsc = (resource_t *) gIter->data;
 
         complex_set_cmds(child_rsc);
     }
 }
 
 void
 set_alloc_actions(pe_working_set_t * data_set)
 {
 
     GListPtr gIter = data_set->resources;
 
     for (; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         complex_set_cmds(rsc);
     }
 }
 
 static void
 calculate_system_health(gpointer gKey, gpointer gValue, gpointer user_data)
 {
     const char *key = (const char *)gKey;
     const char *value = (const char *)gValue;
     int *system_health = (int *)user_data;
 
     if (!gKey || !gValue || !user_data) {
         return;
     }
 
     if (crm_starts_with(key, "#health")) {
         int score;
 
         /* Convert the value into an integer */
         score = char2score(value);
 
         /* Add it to the running total */
         *system_health = merge_weights(score, *system_health);
     }
 }
 
 static gboolean
 apply_system_health(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     const char *health_strategy = pe_pref(data_set->config_hash, "node-health-strategy");
     int base_health = 0;
 
     if (health_strategy == NULL || safe_str_eq(health_strategy, "none")) {
         /* Prevent any accidental health -> score translation */
         node_score_red = 0;
         node_score_yellow = 0;
         node_score_green = 0;
         return TRUE;
 
     } else if (safe_str_eq(health_strategy, "migrate-on-red")) {
 
         /* Resources on nodes which have health values of red are
          * weighted away from that node.
          */
         node_score_red = -INFINITY;
         node_score_yellow = 0;
         node_score_green = 0;
 
     } else if (safe_str_eq(health_strategy, "only-green")) {
 
         /* Resources on nodes which have health values of red or yellow
          * are forced away from that node.
          */
         node_score_red = -INFINITY;
         node_score_yellow = -INFINITY;
         node_score_green = 0;
 
     } else if (safe_str_eq(health_strategy, "progressive")) {
         /* Same as the above, but use the r/y/g scores provided by the user
          * Defaults are provided by the pe_prefs table
          * Also, custom health "base score" can be used
          */
         base_health = crm_parse_int(pe_pref(data_set->config_hash, "node-health-base"), "0");
 
     } else if (safe_str_eq(health_strategy, "custom")) {
 
         /* Requires the admin to configure the rsc_location constaints for
          * processing the stored health scores
          */
         /* TODO: Check for the existence of appropriate node health constraints */
         return TRUE;
 
     } else {
         crm_err("Unknown node health strategy: %s", health_strategy);
         return FALSE;
     }
 
     crm_info("Applying automated node health strategy: %s", health_strategy);
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         int system_health = base_health;
         node_t *node = (node_t *) gIter->data;
 
         /* Search through the node hash table for system health entries. */
         g_hash_table_foreach(node->details->attrs, calculate_system_health, &system_health);
 
         crm_info(" Node %s has an combined system health of %d",
                  node->details->uname, system_health);
 
         /* If the health is non-zero, then create a new rsc2node so that the
          * weight will be added later on.
          */
         if (system_health != 0) {
 
             GListPtr gIter2 = data_set->resources;
 
             for (; gIter2 != NULL; gIter2 = gIter2->next) {
                 resource_t *rsc = (resource_t *) gIter2->data;
 
                 rsc2node_new(health_strategy, rsc, system_health, NULL, node, data_set);
             }
         }
     }
 
     return TRUE;
 }
 
 gboolean
 stage0(pe_working_set_t * data_set)
 {
     xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);
 
     if (data_set->input == NULL) {
         return FALSE;
     }
 
     if (is_set(data_set->flags, pe_flag_have_status) == FALSE) {
         crm_trace("Calculating status");
         cluster_status(data_set);
     }
 
     set_alloc_actions(data_set);
     apply_system_health(data_set);
     unpack_constraints(cib_constraints, data_set);
 
     return TRUE;
 }
 
 /*
  * Check nodes for resources started outside of the LRM
  */
 gboolean
 probe_resources(pe_working_set_t * data_set)
 {
     action_t *probe_node_complete = NULL;
 
     for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
         const char *probed = pe_node_attribute_raw(node, CRM_OP_PROBED);
 
         if (node->details->online == FALSE) {
 
             if (is_baremetal_remote_node(node) && node->details->remote_rsc
                 && (get_remote_node_state(node) == remote_state_failed)) {
 
                 pe_fence_node(data_set, node, "the connection is unrecoverable");
             }
             continue;
 
         } else if (node->details->unclean) {
             continue;
 
         } else if (node->details->rsc_discovery_enabled == FALSE) {
             /* resource discovery is disabled for this node */
             continue;
         }
 
         if (probed != NULL && crm_is_true(probed) == FALSE) {
             action_t *probe_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_REPROBE, node->details->uname),
                                                CRM_OP_REPROBE, node, FALSE, TRUE, data_set);
 
             add_hash_param(probe_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
             continue;
         }
 
         for (GListPtr gIter2 = data_set->resources; gIter2 != NULL; gIter2 = gIter2->next) {
             resource_t *rsc = (resource_t *) gIter2->data;
 
             rsc->cmds->create_probe(rsc, node, probe_node_complete, FALSE, data_set);
         }
     }
     return TRUE;
 }
 
 static void
 rsc_discover_filter(resource_t *rsc, node_t *node)
 {
     GListPtr gIter = rsc->children;
     resource_t *top = uber_parent(rsc);
     node_t *match;
 
     if (rsc->exclusive_discover == FALSE && top->exclusive_discover == FALSE) {
         return;
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         resource_t *child_rsc = (resource_t *) gIter->data;
         rsc_discover_filter(child_rsc, node);
     }
 
     match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
     if (match && match->rsc_discover_mode != pe_discover_exclusive) {
         match->weight = -INFINITY;
     }
 }
 
 /*
  * Count how many valid nodes we have (so we know the maximum number of
  *  colors we can resolve).
  *
  * Apply node constraints (i.e. filter the "allowed_nodes" part of resources)
  */
 gboolean
 stage2(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
 
     crm_trace("Applying placement constraints");
 
     gIter = data_set->nodes;
     for (; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
         if (node == NULL) {
             /* error */
 
         } else if (node->weight >= 0.0  /* global weight */
                    && node->details->online && node->details->type != node_ping) {
             data_set->max_valid_nodes++;
         }
     }
 
     apply_placement_constraints(data_set);
 
     gIter = data_set->nodes;
     for (; gIter != NULL; gIter = gIter->next) {
         GListPtr gIter2 = NULL;
         node_t *node = (node_t *) gIter->data;
 
         gIter2 = data_set->resources;
         for (; gIter2 != NULL; gIter2 = gIter2->next) {
             resource_t *rsc = (resource_t *) gIter2->data;
 
             common_apply_stickiness(rsc, node, data_set);
             rsc_discover_filter(rsc, node);
         }
     }
 
     return TRUE;
 }
 
 /*
  * Create internal resource constraints before allocation
  */
 gboolean
 stage3(pe_working_set_t * data_set)
 {
 
     GListPtr gIter = data_set->resources;
 
     for (; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         rsc->cmds->internal_constraints(rsc, data_set);
     }
 
     return TRUE;
 }
 
 /*
  * Check for orphaned or redefined actions
  */
 gboolean
 stage4(pe_working_set_t * data_set)
 {
     check_actions(data_set);
     return TRUE;
 }
 
 static gint
 sort_rsc_process_order(gconstpointer a, gconstpointer b, gpointer data)
 {
     int rc = 0;
     int r1_weight = -INFINITY;
     int r2_weight = -INFINITY;
 
     const char *reason = "existence";
 
     const GListPtr nodes = (GListPtr) data;
     resource_t *resource1 = (resource_t *) convert_const_pointer(a);
     resource_t *resource2 = (resource_t *) convert_const_pointer(b);
 
     node_t *r1_node = NULL;
     node_t *r2_node = NULL;
     GListPtr gIter = NULL;
     GHashTable *r1_nodes = NULL;
     GHashTable *r2_nodes = NULL;
 
     if (a == NULL && b == NULL) {
         goto done;
     }
     if (a == NULL) {
         return 1;
     }
     if (b == NULL) {
         return -1;
     }
 
     reason = "priority";
     r1_weight = resource1->priority;
     r2_weight = resource2->priority;
 
     if (r1_weight > r2_weight) {
         rc = -1;
         goto done;
     }
 
     if (r1_weight < r2_weight) {
         rc = 1;
         goto done;
     }
 
     reason = "no node list";
     if (nodes == NULL) {
         goto done;
     }
 
     r1_nodes =
         rsc_merge_weights(resource1, resource1->id, NULL, NULL, 1,
                           pe_weights_forward | pe_weights_init);
     dump_node_scores(LOG_TRACE, NULL, resource1->id, r1_nodes);
     r2_nodes =
         rsc_merge_weights(resource2, resource2->id, NULL, NULL, 1,
                           pe_weights_forward | pe_weights_init);
     dump_node_scores(LOG_TRACE, NULL, resource2->id, r2_nodes);
 
     /* Current location score */
     reason = "current location";
     r1_weight = -INFINITY;
     r2_weight = -INFINITY;
 
     if (resource1->running_on) {
         r1_node = g_list_nth_data(resource1->running_on, 0);
         r1_node = g_hash_table_lookup(r1_nodes, r1_node->details->id);
         if (r1_node != NULL) {
             r1_weight = r1_node->weight;
         }
     }
     if (resource2->running_on) {
         r2_node = g_list_nth_data(resource2->running_on, 0);
         r2_node = g_hash_table_lookup(r2_nodes, r2_node->details->id);
         if (r2_node != NULL) {
             r2_weight = r2_node->weight;
         }
     }
 
     if (r1_weight > r2_weight) {
         rc = -1;
         goto done;
     }
 
     if (r1_weight < r2_weight) {
         rc = 1;
         goto done;
     }
 
     reason = "score";
     for (gIter = nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
         r1_node = NULL;
         r2_node = NULL;
 
         r1_weight = -INFINITY;
         if (r1_nodes) {
             r1_node = g_hash_table_lookup(r1_nodes, node->details->id);
         }
         if (r1_node) {
             r1_weight = r1_node->weight;
         }
 
         r2_weight = -INFINITY;
         if (r2_nodes) {
             r2_node = g_hash_table_lookup(r2_nodes, node->details->id);
         }
         if (r2_node) {
             r2_weight = r2_node->weight;
         }
 
         if (r1_weight > r2_weight) {
             rc = -1;
             goto done;
         }
 
         if (r1_weight < r2_weight) {
             rc = 1;
             goto done;
         }
     }
 
   done:
     crm_trace("%s (%d) on %s %c %s (%d) on %s: %s",
               resource1->id, r1_weight, r1_node ? r1_node->details->id : "n/a",
               rc < 0 ? '>' : rc > 0 ? '<' : '=',
               resource2->id, r2_weight, r2_node ? r2_node->details->id : "n/a", reason);
 
     if (r1_nodes) {
         g_hash_table_destroy(r1_nodes);
     }
     if (r2_nodes) {
         g_hash_table_destroy(r2_nodes);
     }
 
     return rc;
 }
 
 static void
 allocate_resources(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
 
     if (is_set(data_set->flags, pe_flag_have_remote_nodes)) {
         /* Force remote connection resources to be allocated first. This
          * also forces any colocation dependencies to be allocated as well */
         for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
             resource_t *rsc = (resource_t *) gIter->data;
             if (rsc->is_remote_node == FALSE) {
                 continue;
             }
             pe_rsc_trace(rsc, "Allocating: %s", rsc->id);
             /* For remote node connection resources, always prefer the partial
              * migration target during resource allocation, if the rsc is in the
              * middle of a migration.
              */
             rsc->cmds->allocate(rsc, rsc->partial_migration_target, data_set);
         }
     }
 
     /* now do the rest of the resources */
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
         if (rsc->is_remote_node == TRUE) {
             continue;
         }
         pe_rsc_trace(rsc, "Allocating: %s", rsc->id);
         rsc->cmds->allocate(rsc, NULL, data_set);
     }
 }
 
 /* We always use pe_order_preserve with these convenience functions to exempt
  * internally generated constraints from the prohibition of user constraints
  * involving remote connection resources.
  *
  * The start ordering additionally uses pe_order_runnable_left so that the
  * specified action is not runnable if the start is not runnable.
  */
 
 static inline void
 order_start_then_action(resource_t *lh_rsc, action_t *rh_action,
                         enum pe_ordering extra, pe_working_set_t *data_set)
 {
     if (lh_rsc && rh_action && data_set) {
         custom_action_order(lh_rsc, start_key(lh_rsc), NULL,
                             rh_action->rsc, NULL, rh_action,
                             pe_order_preserve | pe_order_runnable_left | extra,
                             data_set);
     }
 }
 
 static inline void
 order_action_then_stop(action_t *lh_action, resource_t *rh_rsc,
                        enum pe_ordering extra, pe_working_set_t *data_set)
 {
     if (lh_action && rh_rsc && data_set) {
         custom_action_order(lh_action->rsc, NULL, lh_action,
                             rh_rsc, stop_key(rh_rsc), NULL,
                             pe_order_preserve | extra, data_set);
     }
 }
 
 static void
 cleanup_orphans(resource_t * rsc, pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
 
     if (is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
         return;
     }
 
     /* Don't recurse into ->children, those are just unallocated clone instances */
     if(is_not_set(rsc->flags, pe_rsc_orphan)) {
         return;
     }
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
-        if (node->details->online && pe_get_failcount(node, rsc, NULL, TRUE, NULL, data_set)) {
+        if (node->details->online
+            && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
+                                data_set)) {
+
             char *key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
             action_t *clear_op = custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT,
                                                node, FALSE, TRUE, data_set);
 
             add_hash_param(clear_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
 
             pe_rsc_info(rsc,
                         "Clearing failure of %s on %s because it is orphaned "
                         CRM_XS " %s",
                         rsc->id, node->details->uname, clear_op->uuid);
 
             /* We can't use order_action_then_stop() here because its
              * pe_order_preserve breaks things
              */
             custom_action_order(clear_op->rsc, NULL, clear_op,
                                 rsc, stop_key(rsc), NULL,
                                 pe_order_optional, data_set);
         }
     }
 }
 
 gboolean
 stage5(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
 
     if (safe_str_neq(data_set->placement_strategy, "default")) {
         GListPtr nodes = g_list_copy(data_set->nodes);
 
         nodes = g_list_sort_with_data(nodes, sort_node_weight, NULL);
 
         data_set->resources =
             g_list_sort_with_data(data_set->resources, sort_rsc_process_order, nodes);
 
         g_list_free(nodes);
     }
 
     gIter = data_set->nodes;
     for (; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
         dump_node_capacity(show_utilization ? 0 : utilization_log_level, "Original", node);
     }
 
     crm_trace("Allocating services");
     /* Take (next) highest resource, assign it and create its actions */
 
     allocate_resources(data_set);
 
     gIter = data_set->nodes;
     for (; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
         dump_node_capacity(show_utilization ? 0 : utilization_log_level, "Remaining", node);
     }
 
     if (is_set(data_set->flags, pe_flag_startup_probes)) {
         crm_trace("Calculating needed probes");
         /* This code probably needs optimization
          * ptest -x with 100 nodes, 100 clones and clone-max=100:
 
          With probes:
 
          ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
          ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
          ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
          ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:292 Check actions
          ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
          ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: stage5: allocate.c:881 Allocating services
          ptest[14781]: 2010/09/27_17:56:49 notice: TRACE: stage5: allocate.c:894 Calculating needed probes
          ptest[14781]: 2010/09/27_17:56:51 notice: TRACE: stage5: allocate.c:899 Creating actions
          ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: stage5: allocate.c:905 Creating done
          ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
          ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
          36s
          ptest[14781]: 2010/09/27_17:57:28 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
 
          Without probes:
 
          ptest[14637]: 2010/09/27_17:56:21 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
          ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
          ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
          ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:292 Check actions
          ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
          ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: stage5: allocate.c:881 Allocating services
          ptest[14637]: 2010/09/27_17:56:24 notice: TRACE: stage5: allocate.c:899 Creating actions
          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: stage5: allocate.c:905 Creating done
          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
         */
 
         probe_resources(data_set);
     }
 
     crm_trace("Handle orphans");
 
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
         cleanup_orphans(rsc, data_set);
     }
 
     crm_trace("Creating actions");
 
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         rsc->cmds->create_actions(rsc, data_set);
     }
 
     crm_trace("Creating done");
     return TRUE;
 }
 
 static gboolean
 is_managed(const resource_t * rsc)
 {
     GListPtr gIter = rsc->children;
 
     if (is_set(rsc->flags, pe_rsc_managed)) {
         return TRUE;
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         resource_t *child_rsc = (resource_t *) gIter->data;
 
         if (is_managed(child_rsc)) {
             return TRUE;
         }
     }
 
     return FALSE;
 }
 
 static gboolean
 any_managed_resources(pe_working_set_t * data_set)
 {
 
     GListPtr gIter = data_set->resources;
 
     for (; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         if (is_managed(rsc)) {
             return TRUE;
         }
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Create pseudo-op for guest node fence, and order relative to it
  *
  * \param[in] node      Guest node to fence
  * \param[in] done      STONITH_DONE operation
  * \param[in] data_set  Working set of CIB state
  */
 static void
 fence_guest(pe_node_t *node, pe_action_t *done, pe_working_set_t *data_set)
 {
     resource_t *container = node->details->remote_rsc->container;
     pe_action_t *stop = NULL;
     pe_action_t *stonith_op = NULL;
 
     /* The fence action is just a label; we don't do anything differently for
      * off vs. reboot. We specify it explicitly, rather than let it default to
      * cluster's default action, because we are not _initiating_ fencing -- we
      * are creating a pseudo-event to describe fencing that is already occurring
      * by other means (container recovery).
      */
     const char *fence_action = "off";
 
     /* Check whether guest's container resource is has any explicit stop or
      * start (the stop may be implied by fencing of the guest's host).
      */
     if (container) {
         stop = find_first_action(container->actions, NULL, CRMD_ACTION_STOP, NULL);
 
         if (find_first_action(container->actions, NULL, CRMD_ACTION_START, NULL)) {
             fence_action = "reboot";
         }
     }
 
     /* Create a fence pseudo-event, so we have an event to order actions
      * against, and crmd can always detect it.
      */
     stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean", data_set);
     update_action_flags(stonith_op, pe_action_pseudo | pe_action_runnable,
                         __FUNCTION__, __LINE__);
 
     /* We want to imply stops/demotes after the guest is stopped, not wait until
      * it is restarted, so we always order pseudo-fencing after stop, not start
      * (even though start might be closer to what is done for a real reboot).
      */
     if(stop && is_set(stop->flags, pe_action_pseudo)) {
         pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE, NULL, data_set);
         crm_info("Implying guest node %s is down (action %d) after %s fencing",
                  node->details->uname, stonith_op->id, stop->node->details->uname);
         order_actions(parent_stonith_op, stonith_op,
                       pe_order_runnable_left|pe_order_implies_then);
 
     } else if (stop) {
         order_actions(stop, stonith_op,
                       pe_order_runnable_left|pe_order_implies_then);
         crm_info("Implying guest node %s is down (action %d) "
                  "after container %s is stopped (action %d)",
                  node->details->uname, stonith_op->id,
                  container->id, stop->id);
     } else {
         crm_info("Implying guest node %s is down (action %d) ",
                  node->details->uname, stonith_op->id);
     }
 
     /* @TODO: Order pseudo-fence after any (optional) fence of guest's host */
 
     /* Order/imply other actions relative to pseudo-fence as with real fence */
     stonith_constraints(node, stonith_op, data_set);
     order_actions(stonith_op, done, pe_order_implies_then);
 }
 
 /*
  * Create dependencies for stonith and shutdown operations
  */
 gboolean
 stage6(pe_working_set_t * data_set)
 {
     action_t *dc_down = NULL;
     action_t *dc_fence = NULL;
     action_t *stonith_op = NULL;
     action_t *last_stonith = NULL;
     gboolean integrity_lost = FALSE;
     action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);
     action_t *done = get_pseudo_op(STONITH_DONE, data_set);
     gboolean need_stonith = TRUE;
     GListPtr gIter;
     GListPtr stonith_ops = NULL;
 
     /* Remote ordering constraints need to happen prior to calculate
      * fencing because it is one more place we will mark the node as
      * dirty.
      *
      * A nice side-effect of doing it first is that we can remove a
      * bunch of special logic from apply_*_ordering() because its
      * already part of pe_fence_node()
      */
     crm_trace("Creating remote ordering constraints");
     apply_remote_node_ordering(data_set);
 
     crm_trace("Processing fencing and shutdown cases");
     if (any_managed_resources(data_set) == FALSE) {
         crm_notice("Delaying fencing operations until there are resources to manage");
         need_stonith = FALSE;
     }
 
     /* Check each node for stonith/shutdown */
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
         /* Guest nodes are "fenced" by recovering their container resource,
          * so handle them separately.
          */
         if (is_container_remote_node(node)) {
             if (node->details->remote_requires_reset && need_stonith) {
                 fence_guest(node, done, data_set);
             }
             continue;
         }
 
         stonith_op = NULL;
 
         if (node->details->unclean
             && need_stonith && pe_can_fence(data_set, node)) {
 
             stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", data_set);
             pe_warn("Scheduling Node %s for STONITH", node->details->uname);
 
             stonith_constraints(node, stonith_op, data_set);
 
             if (node->details->is_dc) {
                 dc_down = stonith_op;
                 dc_fence = stonith_op;
 
             } else if (is_set(data_set->flags, pe_flag_concurrent_fencing) == FALSE) {
                 if (last_stonith) {
                     order_actions(last_stonith, stonith_op, pe_order_optional);
                 }
                 last_stonith = stonith_op;
 
             } else {
                 order_actions(stonith_op, done, pe_order_implies_then);
                 stonith_ops = g_list_append(stonith_ops, stonith_op);
             }
 
         } else if (node->details->online && node->details->shutdown &&
                 /* TODO define what a shutdown op means for a remote node.
                  * For now we do not send shutdown operations for remote nodes, but
                  * if we can come up with a good use for this in the future, we will. */
                     is_remote_node(node) == FALSE) {
 
             action_t *down_op = NULL;
 
             crm_notice("Scheduling Node %s for shutdown", node->details->uname);
 
             down_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_SHUTDOWN, node->details->uname),
                                     CRM_OP_SHUTDOWN, node, FALSE, TRUE, data_set);
 
             shutdown_constraints(node, down_op, data_set);
             add_hash_param(down_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
 
             if (node->details->is_dc) {
                 dc_down = down_op;
             }
         }
 
         if (node->details->unclean && stonith_op == NULL) {
             integrity_lost = TRUE;
             pe_warn("Node %s is unclean!", node->details->uname);
         }
     }
 
     if (integrity_lost) {
         if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
             pe_warn("YOUR RESOURCES ARE NOW LIKELY COMPROMISED");
             pe_err("ENABLE STONITH TO KEEP YOUR RESOURCES SAFE");
 
         } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE) {
             crm_notice("Cannot fence unclean nodes until quorum is"
                        " attained (or no-quorum-policy is set to ignore)");
         }
     }
 
     if (dc_down != NULL) {
         GListPtr gIter = NULL;
 
         crm_trace("Ordering shutdowns before %s on %s (DC)",
                   dc_down->task, dc_down->node->details->uname);
 
         add_hash_param(dc_down->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
 
         for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
             action_t *node_stop = (action_t *) gIter->data;
 
             if (safe_str_neq(CRM_OP_SHUTDOWN, node_stop->task)) {
                 continue;
             } else if (node_stop->node->details->is_dc) {
                 continue;
             }
 
             crm_debug("Ordering shutdown on %s before %s on %s",
                       node_stop->node->details->uname,
                       dc_down->task, dc_down->node->details->uname);
 
             order_actions(node_stop, dc_down, pe_order_optional);
         }
 
         if (last_stonith) {
             if (dc_down != last_stonith) {
                 order_actions(last_stonith, dc_down, pe_order_optional);
             }
 
         } else {
             GListPtr gIter2 = NULL;
 
             for (gIter2 = stonith_ops; gIter2 != NULL; gIter2 = gIter2->next) {
                 stonith_op = (action_t *) gIter2->data;
 
                 if (dc_down != stonith_op) {
                     order_actions(stonith_op, dc_down, pe_order_optional);
                 }
             }
         }
     }
 
 
     if (dc_fence) {
         order_actions(dc_down, done, pe_order_implies_then);
 
     } else if (last_stonith) {
         order_actions(last_stonith, done, pe_order_implies_then);
     }
 
     order_actions(done, all_stopped, pe_order_implies_then);
 
     g_list_free(stonith_ops);
     return TRUE;
 }
 
 /*
  * Determine the sets of independent actions and the correct order for the
  *  actions in each set.
  *
  * Mark dependencies of un-runnable actions un-runnable
  *
  */
 static GListPtr
 find_actions_by_task(GListPtr actions, resource_t * rsc, const char *original_key)
 {
     GListPtr list = NULL;
 
     list = find_actions(actions, original_key, NULL);
     if (list == NULL) {
         /* we're potentially searching a child of the original resource */
         char *key = NULL;
         char *tmp = NULL;
         char *task = NULL;
         int interval = 0;
 
         if (parse_op_key(original_key, &tmp, &task, &interval)) {
             key = generate_op_key(rsc->id, task, interval);
             /* crm_err("looking up %s instead of %s", key, original_key); */
             /* slist_iter(action, action_t, actions, lpc, */
             /*         crm_err("  - %s", action->uuid)); */
             list = find_actions(actions, key, NULL);
 
         } else {
             crm_err("search key: %s", original_key);
         }
 
         free(key);
         free(tmp);
         free(task);
     }
 
     return list;
 }
 
 static void
 rsc_order_then(action_t * lh_action, resource_t * rsc, order_constraint_t * order)
 {
     GListPtr gIter = NULL;
     GListPtr rh_actions = NULL;
     action_t *rh_action = NULL;
     enum pe_ordering type = order->type;
 
     CRM_CHECK(rsc != NULL, return);
     CRM_CHECK(order != NULL, return);
 
     rh_action = order->rh_action;
     crm_trace("Processing RH of ordering constraint %d", order->id);
 
     if (rh_action != NULL) {
         rh_actions = g_list_prepend(NULL, rh_action);
 
     } else if (rsc != NULL) {
         rh_actions = find_actions_by_task(rsc->actions, rsc, order->rh_action_task);
     }
 
     if (rh_actions == NULL) {
         pe_rsc_trace(rsc, "No RH-Side (%s/%s) found for constraint..."
                      " ignoring", rsc->id, order->rh_action_task);
         if (lh_action) {
             pe_rsc_trace(rsc, "LH-Side was: %s", lh_action->uuid);
         }
         return;
     }
 
     if (lh_action && lh_action->rsc == rsc && is_set(lh_action->flags, pe_action_dangle)) {
         pe_rsc_trace(rsc, "Detected dangling operation %s -> %s", lh_action->uuid,
                      order->rh_action_task);
         clear_bit(type, pe_order_implies_then);
     }
 
     gIter = rh_actions;
     for (; gIter != NULL; gIter = gIter->next) {
         action_t *rh_action_iter = (action_t *) gIter->data;
 
         if (lh_action) {
             order_actions(lh_action, rh_action_iter, type);
 
         } else if (type & pe_order_implies_then) {
             update_action_flags(rh_action_iter, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
             crm_warn("Unrunnable %s 0x%.6x", rh_action_iter->uuid, type);
         } else {
             crm_warn("neither %s 0x%.6x", rh_action_iter->uuid, type);
         }
     }
 
     g_list_free(rh_actions);
 }
 
 static void
 rsc_order_first(resource_t * lh_rsc, order_constraint_t * order, pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     GListPtr lh_actions = NULL;
     action_t *lh_action = order->lh_action;
     resource_t *rh_rsc = order->rh_rsc;
 
     crm_trace("Processing LH of ordering constraint %d", order->id);
     CRM_ASSERT(lh_rsc != NULL);
 
     if (lh_action != NULL) {
         lh_actions = g_list_prepend(NULL, lh_action);
 
     } else if (lh_action == NULL) {
         lh_actions = find_actions_by_task(lh_rsc->actions, lh_rsc, order->lh_action_task);
     }
 
     if (lh_actions == NULL && lh_rsc != rh_rsc) {
         char *key = NULL;
         char *rsc_id = NULL;
         char *op_type = NULL;
         int interval = 0;
 
         parse_op_key(order->lh_action_task, &rsc_id, &op_type, &interval);
         key = generate_op_key(lh_rsc->id, op_type, interval);
 
         if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_STOPPED && safe_str_eq(op_type, RSC_STOP)) {
             free(key);
             pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
                          lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
 
         } else if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_SLAVE && safe_str_eq(op_type, RSC_DEMOTE)) {
             free(key);
             pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
                          lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
 
         } else {
             pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - creating",
                          lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
             lh_action = custom_action(lh_rsc, key, op_type, NULL, TRUE, TRUE, data_set);
             lh_actions = g_list_prepend(NULL, lh_action);
         }
 
         free(op_type);
         free(rsc_id);
     }
 
     gIter = lh_actions;
     for (; gIter != NULL; gIter = gIter->next) {
         action_t *lh_action_iter = (action_t *) gIter->data;
 
         if (rh_rsc == NULL && order->rh_action) {
             rh_rsc = order->rh_action->rsc;
         }
         if (rh_rsc) {
             rsc_order_then(lh_action_iter, rh_rsc, order);
 
         } else if (order->rh_action) {
             order_actions(lh_action_iter, order->rh_action, order->type);
         }
     }
 
     g_list_free(lh_actions);
 }
 
 extern gboolean update_action(action_t * action);
 extern void update_colo_start_chain(action_t * action);
 
 static int
 is_recurring_action(action_t *action) 
 {
     const char *interval_s = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL);
     int interval = crm_parse_int(interval_s, "0");
     if(interval > 0) {
         return TRUE;
     }
     return FALSE;
 }
 
 static void
 apply_container_ordering(action_t *action, pe_working_set_t *data_set)
 {
     /* VMs are also classified as containers for these purposes... in
      * that they both involve a 'thing' running on a real or remote
      * cluster node.
      *
      * This allows us to be smarter about the type and extent of
      * recovery actions required in various scenarios
      */
     resource_t *remote_rsc = NULL;
     resource_t *container = NULL;
     enum action_tasks task = text2task(action->task);
 
     CRM_ASSERT(action->rsc);
     CRM_ASSERT(action->node);
     CRM_ASSERT(is_remote_node(action->node));
 
     remote_rsc = action->node->details->remote_rsc;
     CRM_ASSERT(remote_rsc);
 
     container = remote_rsc->container;
     CRM_ASSERT(container);
 
     if(is_set(container->flags, pe_rsc_failed)) {
         pe_fence_node(data_set, action->node, "container failed");
     }
 
     crm_trace("Order %s action %s relative to %s%s for %s%s",
               action->task, action->uuid,
               is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
               remote_rsc->id,
               is_set(container->flags, pe_rsc_failed)? "failed " : "",
               container->id);
 
     if (safe_str_eq(action->task, CRMD_ACTION_MIGRATE)
         || safe_str_eq(action->task, CRMD_ACTION_MIGRATE)) {
         /* Migration ops map to "no_action", but we need to apply the same
          * ordering as for stop or demote (see get_router_node()).
          */
         task = stop_rsc;
     }
 
     switch (task) {
         case start_rsc:
         case action_promote:
             /* Force resource recovery if the container is recovered */
             order_start_then_action(container, action, pe_order_implies_then,
                                     data_set);
 
             /* Wait for the connection resource to be up too */
             order_start_then_action(remote_rsc, action, pe_order_none,
                                     data_set);
             break;
 
         case stop_rsc:
         case action_demote:
             if (is_set(container->flags, pe_rsc_failed)) {
                 /* When the container representing a guest node fails, any stop
                  * or demote actions for resources running on the guest node
                  * are implied by the container stopping. This is similar to
                  * how fencing operations work for cluster nodes and remote
                  * nodes.
                  */
             } else {
                 /* Ensure the operation happens before the connection is brought
                  * down.
                  *
                  * If we really wanted to, we could order these after the
                  * connection start, IFF the container's current role was
                  * stopped (otherwise we re-introduce an ordering loop when the
                  * connection is restarting).
                  */
                 order_action_then_stop(action, remote_rsc, pe_order_none,
                                        data_set);
             }
             break;
 
         default:
             /* Wait for the connection resource to be up */
             if (is_recurring_action(action)) {
                 /* In case we ever get the recovery logic wrong, force
                  * recurring monitors to be restarted, even if just
                  * the connection was re-established
                  */
                 if(task != no_action) {
                     order_start_then_action(remote_rsc, action,
                                             pe_order_implies_then, data_set);
                 }
             } else {
                 order_start_then_action(remote_rsc, action, pe_order_none,
                                         data_set);
             }
             break;
     }
 }
 
 static enum remote_connection_state
 get_remote_node_state(pe_node_t *node) 
 {
     resource_t *remote_rsc = NULL;
     node_t *cluster_node = NULL;
 
     CRM_ASSERT(node);
 
     remote_rsc = node->details->remote_rsc;
     CRM_ASSERT(remote_rsc);
 
     if(remote_rsc->running_on) {
         cluster_node = remote_rsc->running_on->data;
     }
 
 
     /* If the cluster node the remote connection resource resides on
      * is unclean or went offline, we can't process any operations
      * on that remote node until after it starts elsewhere.
      */
     if(remote_rsc->next_role == RSC_ROLE_STOPPED || remote_rsc->allocated_to == NULL) {
         /* The connection resource is not going to run anywhere */
 
         if (cluster_node && cluster_node->details->unclean) {
             /* The remote connection is failed because its resource is on a
              * failed node and can't be recovered elsewhere, so we must fence.
              */
             return remote_state_failed;
         }
 
         if (is_not_set(remote_rsc->flags, pe_rsc_failed)) {
             /* Connection resource is cleanly stopped */
             return remote_state_stopped;
         }
 
         /* Connection resource is failed */
 
         if ((remote_rsc->next_role == RSC_ROLE_STOPPED)
             && remote_rsc->remote_reconnect_interval
             && node->details->remote_was_fenced) {
 
             /* We won't know whether the connection is recoverable until the
              * reconnect interval expires and we reattempt connection.
              */
             return remote_state_unknown;
         }
 
         /* The remote connection is in a failed state. If there are any
          * resources known to be active on it (stop) or in an unknown state
          * (probe), we must assume the worst and fence it.
          */
         return remote_state_failed;
 
     } else if (cluster_node == NULL) {
         /* Connection is recoverable but not currently running anywhere, see if we can recover it first */
         return remote_state_unknown;
 
     } else if(cluster_node->details->unclean == TRUE
               || cluster_node->details->online == FALSE) {
         /* Connection is running on a dead node, see if we can recover it first */
         return remote_state_resting;
 
     } else if (g_list_length(remote_rsc->running_on) > 1
                && remote_rsc->partial_migration_source
                && remote_rsc->partial_migration_target) {
         /* We're in the middle of migrating a connection resource,
          * wait until after the resource migrates before performing
          * any actions.
          */
         return remote_state_resting;
 
     }
     return remote_state_alive;
 }
 
 static void
 apply_remote_ordering(action_t *action, pe_working_set_t *data_set)
 {
     resource_t *remote_rsc = NULL;
     node_t *cluster_node = NULL;
     enum action_tasks task = text2task(action->task);
     enum remote_connection_state state = get_remote_node_state(action->node);
 
     enum pe_ordering order_opts = pe_order_none;
 
     if (action->rsc == NULL) {
         return;
     }
 
     CRM_ASSERT(action->node);
     CRM_ASSERT(is_remote_node(action->node));
 
     remote_rsc = action->node->details->remote_rsc;
     CRM_ASSERT(remote_rsc);
 
     if(remote_rsc->running_on) {
         cluster_node = remote_rsc->running_on->data;
     }
 
     crm_trace("Order %s action %s relative to %s%s (state %d)",
               action->task, action->uuid,
               is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
               remote_rsc->id, state);
 
     if (safe_str_eq(action->task, CRMD_ACTION_MIGRATE)
         || safe_str_eq(action->task, CRMD_ACTION_MIGRATE)) {
         /* Migration ops map to "no_action", but we need to apply the same
          * ordering as for stop or demote (see get_router_node()).
          */
         task = stop_rsc;
     }
 
     switch (task) {
         case start_rsc:
         case action_promote:
             order_opts = pe_order_none;
 
             if (state == remote_state_failed) {
                 /* Force recovery, by making this action required */
                 order_opts |= pe_order_implies_then;
             }
 
             /* Ensure connection is up before running this action */
             order_start_then_action(remote_rsc, action, order_opts, data_set);
             break;
 
         case stop_rsc:
             /* Handle special case with remote node where stop actions need to be
              * ordered after the connection resource starts somewhere else.
              */
             if(state == remote_state_resting) {
                 /* Wait for the connection resource to be up and assume everything is as we left it */
                 order_start_then_action(remote_rsc, action, pe_order_none,
                                         data_set);
 
             } else {
                 if(state == remote_state_failed) {
                     /* We would only be here if the resource is
                      * running on the remote node.  Since we have no
                      * way to stop it, it is necessary to fence the
                      * node.
                      */
                     pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable");
                 }
 
                 order_action_then_stop(action, remote_rsc,
                                        pe_order_implies_first, data_set);
             }
             break;
 
         case action_demote:
             /* Only order this demote relative to the connection start if the
              * connection isn't being torn down. Otherwise, the demote would be
              * blocked because the connection start would not be allowed.
              */
             if(state == remote_state_resting || state == remote_state_unknown) {
                 order_start_then_action(remote_rsc, action, pe_order_none,
                                         data_set);
             } /* Otherwise we can rely on the stop ordering */
             break;
 
         default:
             /* Wait for the connection resource to be up */
             if (is_recurring_action(action)) {
                 /* In case we ever get the recovery logic wrong, force
                  * recurring monitors to be restarted, even if just
                  * the connection was re-established
                  */
                 order_start_then_action(remote_rsc, action,
                                         pe_order_implies_then, data_set);
 
             } else {
                 if(task == monitor_rsc && state == remote_state_failed) {
                     /* We would only be here if we do not know the
                      * state of the resource on the remote node.
                      * Since we have no way to find out, it is
                      * necessary to fence the node.
                      */
                     pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable");
                 }
 
                 if(cluster_node && state == remote_state_stopped) {
                     /* The connection is currently up, but is going
                      * down permanently.
                      *
                      * Make sure we check services are actually
                      * stopped _before_ we let the connection get
                      * closed
                      */
                     order_action_then_stop(action, remote_rsc,
                                            pe_order_runnable_left, data_set);
 
                 } else {
                     order_start_then_action(remote_rsc, action, pe_order_none,
                                             data_set);
                 }
             }
             break;
     }
 }
 
 static void
 apply_remote_node_ordering(pe_working_set_t *data_set)
 {
     if (is_set(data_set->flags, pe_flag_have_remote_nodes) == FALSE) {
         return;
     }
 
     for (GListPtr gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         action_t *action = (action_t *) gIter->data;
         resource_t *remote = NULL;
 
         // We are only interested in resource actions
         if (action->rsc == NULL) {
             continue;
         }
 
         /* Special case: If we are clearing the failcount of an actual
          * remote connection resource, then make sure this happens before
          * any start of the resource in this transition.
          */
         if (action->rsc->is_remote_node &&
             safe_str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT)) {
 
             custom_action_order(action->rsc,
                 NULL,
                 action,
                 action->rsc,
                 generate_op_key(action->rsc->id, RSC_START, 0),
                 NULL,
                 pe_order_optional,
                 data_set);
 
             continue;
         }
 
         // We are only interested in actions allocated to a node
         if (action->node == NULL) {
             continue;
         }
 
         if (is_remote_node(action->node) == FALSE) {
             continue;
         }
 
         /* We are only interested in real actions.
          *
          * @TODO This is probably wrong; pseudo-actions might be converted to
          * real actions and vice versa later in update_actions() at the end of
          * stage7().
          */
         if (is_set(action->flags, pe_action_pseudo)) {
             continue;
         }
 
         remote = action->node->details->remote_rsc;
         if (remote == NULL) {
             // Orphaned
             continue;
         }
 
         /* The action occurs across a remote connection, so create
          * ordering constraints that guarantee the action occurs while the node
          * is active (after start, before stop ... things like that).
          *
          * This is somewhat brittle in that we need to make sure the results of
          * this ordering are compatible with the result of get_router_node().
          * It would probably be better to add XML_LRM_ATTR_ROUTER_NODE as part
          * of this logic rather than action2xml().
          */
         if (remote->container) {
             crm_trace("Container ordering for %s", action->uuid);
             apply_container_ordering(action, data_set);
 
         } else {
             crm_trace("Remote ordering for %s", action->uuid);
             apply_remote_ordering(action, data_set);
         }
     }
 }
 
 static void
 order_probes(pe_working_set_t * data_set) 
 {
 #if 0
     GListPtr gIter = NULL;
 
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         /* Given "A then B", we would prefer to wait for A to be
          * started before probing B.
          *
          * If A was a filesystem on which the binaries and data for B
          * lived, it would have been useful if the author of B's agent
          * could assume that A is running before B.monitor will be
          * called.
          *
          * However we can't _only_ probe once A is running, otherwise
          * we'd not detect the state of B if A could not be started
          * for some reason.
          *
          * In practice however, we cannot even do an opportunistic
          * version of this because B may be moving:
          *
          *   B.probe -> B.start
          *   B.probe -> B.stop
          *   B.stop -> B.start
          *   A.stop -> A.start
          *   A.start -> B.probe
          *
          * So far so good, but if we add the result of this code:
          *
          *   B.stop -> A.stop
          *
          * Then we get a loop:
          *
          *   B.probe -> B.stop -> A.stop -> A.start -> B.probe
          *
          * We could kill the 'B.probe -> B.stop' dependency, but that
          * could mean stopping B "too" soon, because B.start must wait
          * for the probes to complete.
          *
          * Another option is to allow it only if A is a non-unique
          * clone with clone-max == node-max (since we'll never be
          * moving it).  However, we could still be stopping one
          * instance at the same time as starting another.
 
          * The complexity of checking for allowed conditions combined
          * with the ever narrowing usecase suggests that this code
          * should remain disabled until someone gets smarter.
          */
         action_t *start = NULL;
         GListPtr actions = NULL;
         GListPtr probes = NULL;
         char *key = NULL;
 
         key = start_key(rsc);
         actions = find_actions(rsc->actions, key, NULL);
         free(key);
 
         if (actions) {
             start = actions->data;
             g_list_free(actions);
         }
 
         if(start == NULL) {
             crm_err("No start action for %s", rsc->id);
             continue;
         }
 
         key = generate_op_key(rsc->id, CRMD_ACTION_STATUS, 0);
         probes = find_actions(rsc->actions, key, NULL);
         free(key);
 
         for (actions = start->actions_before; actions != NULL; actions = actions->next) {
             action_wrapper_t *before = (action_wrapper_t *) actions->data;
 
             GListPtr pIter = NULL;
             action_t *first = before->action;
             resource_t *first_rsc = first->rsc;
 
             if(first->required_runnable_before) {
                 GListPtr clone_actions = NULL;
                 for (clone_actions = first->actions_before; clone_actions != NULL; clone_actions = clone_actions->next) {
                     before = (action_wrapper_t *) clone_actions->data;
 
                     crm_trace("Testing %s -> %s (%p) for %s", first->uuid, before->action->uuid, before->action->rsc, start->uuid);
 
                     CRM_ASSERT(before->action->rsc);
                     first_rsc = before->action->rsc;
                     break;
                 }
 
             } else if(safe_str_neq(first->task, RSC_START)) {
                 crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
             }
 
             if(first_rsc == NULL) {
                 continue;
 
             } else if(uber_parent(first_rsc) == uber_parent(start->rsc)) {
                 crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
                 continue;
 
             } else if(FALSE && pe_rsc_is_clone(uber_parent(first_rsc)) == FALSE) {
                 crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
                 continue;
             }
 
             crm_err("Applying %s before %s %d", first->uuid, start->uuid, uber_parent(first_rsc)->variant);
 
             for (pIter = probes; pIter != NULL; pIter = pIter->next) {
                 action_t *probe = (action_t *) pIter->data;
 
                 crm_err("Ordering %s before %s", first->uuid, probe->uuid);
                 order_actions(first, probe, pe_order_optional);
             }
         }
     }
 #endif
 }
 
 gboolean
 stage7(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
 
     crm_trace("Applying ordering constraints");
 
     /* Don't ask me why, but apparently they need to be processed in
      * the order they were created in... go figure
      *
      * Also g_list_append() has horrendous performance characteristics
      * So we need to use g_list_prepend() and then reverse the list here
      */
     data_set->ordering_constraints = g_list_reverse(data_set->ordering_constraints);
 
     for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
         order_constraint_t *order = (order_constraint_t *) gIter->data;
         resource_t *rsc = order->lh_rsc;
 
         crm_trace("Applying ordering constraint: %d", order->id);
 
         if (rsc != NULL) {
             crm_trace("rsc_action-to-*");
             rsc_order_first(rsc, order, data_set);
             continue;
         }
 
         rsc = order->rh_rsc;
         if (rsc != NULL) {
             crm_trace("action-to-rsc_action");
             rsc_order_then(order->lh_action, rsc, order);
 
         } else {
             crm_trace("action-to-action");
             order_actions(order->lh_action, order->rh_action, order->type);
         }
     }
 
     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         action_t *action = (action_t *) gIter->data;
 
         update_colo_start_chain(action);
     }
 
     crm_trace("Ordering probes");
     order_probes(data_set);
 
     crm_trace("Updating %d actions", g_list_length(data_set->actions));
     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         action_t *action = (action_t *) gIter->data;
 
         update_action(action);
     }
 
     LogNodeActions(data_set, FALSE);
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         LogActions(rsc, data_set, FALSE);
     }
     return TRUE;
 }
 
 int transition_id = -1;
 
 /*
  * Create a dependency graph to send to the transitioner (via the CRMd)
  */
 gboolean
 stage8(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     const char *value = NULL;
 
     transition_id++;
     crm_trace("Creating transition graph %d.", transition_id);
 
     data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);
 
     value = pe_pref(data_set->config_hash, "cluster-delay");
     crm_xml_add(data_set->graph, "cluster-delay", value);
 
     value = pe_pref(data_set->config_hash, "stonith-timeout");
     crm_xml_add(data_set->graph, "stonith-timeout", value);
 
     crm_xml_add(data_set->graph, "failed-stop-offset", "INFINITY");
 
     if (is_set(data_set->flags, pe_flag_start_failure_fatal)) {
         crm_xml_add(data_set->graph, "failed-start-offset", "INFINITY");
     } else {
         crm_xml_add(data_set->graph, "failed-start-offset", "1");
     }
 
     value = pe_pref(data_set->config_hash, "batch-limit");
     crm_xml_add(data_set->graph, "batch-limit", value);
 
     crm_xml_add_int(data_set->graph, "transition_id", transition_id);
 
     value = pe_pref(data_set->config_hash, "migration-limit");
     if (crm_int_helper(value, NULL) > 0) {
         crm_xml_add(data_set->graph, "migration-limit", value);
     }
 
 /* errors...
    slist_iter(action, action_t, action_list, lpc,
    if(action->optional == FALSE && action->runnable == FALSE) {
    print_action("Ignoring", action, TRUE);
    }
    );
 */
 
     gIter = data_set->resources;
     for (; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         pe_rsc_trace(rsc, "processing actions for rsc=%s", rsc->id);
         rsc->cmds->expand(rsc, data_set);
     }
 
     crm_log_xml_trace(data_set->graph, "created resource-driven action list");
 
     /* pseudo action to distribute list of nodes with maintenance state update */
     add_maintenance_update(data_set);
 
     /* catch any non-resource specific actions */
     crm_trace("processing non-resource actions");
 
     gIter = data_set->actions;
     for (; gIter != NULL; gIter = gIter->next) {
         action_t *action = (action_t *) gIter->data;
 
         if (action->rsc
             && action->node
             && action->node->details->shutdown
             && is_not_set(action->rsc->flags, pe_rsc_maintenance)
             && is_not_set(action->flags, pe_action_optional)
             && is_not_set(action->flags, pe_action_runnable)
             && crm_str_eq(action->task, RSC_STOP, TRUE)
             ) {
             /* Eventually we should just ignore the 'fence' case
              * But for now it's the best way to detect (in CTS) when
              * CIB resource updates are being lost
              */
             if (is_set(data_set->flags, pe_flag_have_quorum)
                 || data_set->no_quorum_policy == no_quorum_ignore) {
                 crm_crit("Cannot %s node '%s' because of %s:%s%s (%s)",
                          action->node->details->unclean ? "fence" : "shut down",
                          action->node->details->uname, action->rsc->id,
                          is_not_set(action->rsc->flags, pe_rsc_managed) ? " unmanaged" : " blocked",
                          is_set(action->rsc->flags, pe_rsc_failed) ? " failed" : "",
                          action->uuid);
             }
         }
 
         graph_element_from_action(action, data_set);
     }
 
     crm_log_xml_trace(data_set->graph, "created generic action list");
     crm_trace("Created transition graph %d.", transition_id);
 
     return TRUE;
 }
 
 void
 LogNodeActions(pe_working_set_t * data_set, gboolean terminal)
 {
     GListPtr gIter = NULL;
 
     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         char *node_name = NULL;
         char *task = NULL;
         action_t *action = (action_t *) gIter->data;
 
         if (action->rsc != NULL) {
             continue;
         } else if (is_set(action->flags, pe_action_optional)) {
             continue;
         }
 
         if (is_container_remote_node(action->node)) {
             node_name = crm_strdup_printf("%s (resource: %s)", action->node->details->uname, action->node->details->remote_rsc->container->id);
         } else if(action->node) {
             node_name = crm_strdup_printf("%s", action->node->details->uname);
         }
 
 
         if (safe_str_eq(action->task, CRM_OP_SHUTDOWN)) {
             task = strdup("Shutdown");
         } else if (safe_str_eq(action->task, CRM_OP_FENCE)) {
             const char *op = g_hash_table_lookup(action->meta, "stonith_action");
             task = crm_strdup_printf("Fence (%s)", op);
         }
 
         if(task == NULL) {
             /* Nothing to report */
         } else if(terminal && action->reason) {
             printf(" * %s %s '%s'\n", task, node_name, action->reason);
         } else if(terminal) {
             printf(" * %s %s\n", task, node_name);
         } else if(action->reason) {
             crm_notice(" * %s %s '%s'\n", task, node_name, action->reason);
         } else {
             crm_notice(" * %s %s\n", task, node_name);
         }
 
         free(node_name);
         free(task);
     }
 }
 
 void
 cleanup_alloc_calculations(pe_working_set_t * data_set)
 {
     if (data_set == NULL) {
         return;
     }
 
     crm_trace("deleting %d order cons: %p",
               g_list_length(data_set->ordering_constraints), data_set->ordering_constraints);
     pe_free_ordering(data_set->ordering_constraints);
     data_set->ordering_constraints = NULL;
 
     crm_trace("deleting %d node cons: %p",
               g_list_length(data_set->placement_constraints), data_set->placement_constraints);
     pe_free_rsc_to_node(data_set->placement_constraints);
     data_set->placement_constraints = NULL;
 
     crm_trace("deleting %d inter-resource cons: %p",
               g_list_length(data_set->colocation_constraints), data_set->colocation_constraints);
     g_list_free_full(data_set->colocation_constraints, free);
     data_set->colocation_constraints = NULL;
 
     crm_trace("deleting %d ticket deps: %p",
               g_list_length(data_set->ticket_constraints), data_set->ticket_constraints);
     g_list_free_full(data_set->ticket_constraints, free);
     data_set->ticket_constraints = NULL;
 
     cleanup_calculations(data_set);
 }
diff --git a/tools/crm_mon.c b/tools/crm_mon.c
index 651f3a4e16..401a0d403d 100644
--- a/tools/crm_mon.c
+++ b/tools/crm_mon.c
@@ -1,4247 +1,4247 @@
 /*
  * Copyright (C) 2004-2015 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <libgen.h>
 #include <sys/utsname.h>
 
 #include <crm/msg_xml.h>
 #include <crm/services.h>
 #include <crm/lrmd.h>
 #include <crm/common/internal.h>  /* crm_ends_with_ext */
 #include <crm/common/ipc.h>
 #include <crm/common/mainloop.h>
 #include <crm/common/util.h>
 #include <crm/common/xml.h>
 
 #include <crm/cib/internal.h>
 #include <crm/pengine/status.h>
 #include <crm/pengine/internal.h>
 #include <../lib/pengine/unpack.h>
 #include <../pengine/pengine.h>
 #include <crm/stonith-ng.h>
 
 extern void cleanup_alloc_calculations(pe_working_set_t * data_set);
 
 void clean_up(int rc);
 void crm_diff_update(const char *event, xmlNode * msg);
 gboolean mon_refresh_display(gpointer user_data);
 int cib_connect(gboolean full);
 void mon_st_callback(stonith_t * st, stonith_event_t * e);
 static char *get_node_display_name(node_t *node);
 
 /*
  * Definitions indicating which items to print
  */
 
 #define mon_show_times      (0x0001U)
 #define mon_show_stack      (0x0002U)
 #define mon_show_dc         (0x0004U)
 #define mon_show_count      (0x0008U)
 #define mon_show_nodes      (0x0010U)
 #define mon_show_resources  (0x0020U)
 #define mon_show_attributes (0x0040U)
 #define mon_show_failcounts (0x0080U)
 #define mon_show_operations (0x0100U)
 #define mon_show_tickets    (0x0200U)
 #define mon_show_bans       (0x0400U)
 
 #define mon_show_headers    (mon_show_times | mon_show_stack | mon_show_dc | mon_show_count)
 #define mon_show_default    (mon_show_headers | mon_show_nodes | mon_show_resources)
 #define mon_show_all        (mon_show_default | mon_show_attributes | mon_show_failcounts \
                      | mon_show_operations | mon_show_tickets | mon_show_bans)
 
 unsigned int show = mon_show_default;
 
 /*
  * Definitions indicating how to output
  */
 
 enum mon_output_format_e {
     mon_output_none,
     mon_output_monitor,
     mon_output_plain,
     mon_output_console,
     mon_output_xml,
     mon_output_html,
     mon_output_cgi
 } output_format = mon_output_console;
 
 char *output_filename = NULL;   /* if sending output to a file, its name */
 
 /* other globals */
 char *xml_file = NULL;
 char *pid_file = NULL;
 char *snmp_target = NULL;
 char *snmp_community = NULL;
 
 gboolean group_by_node = FALSE;
 gboolean inactive_resources = FALSE;
 int reconnect_msec = 5000;
 gboolean daemonize = FALSE;
 GMainLoop *mainloop = NULL;
 guint timer_id = 0;
 GList *attr_list = NULL;
 
 const char *crm_mail_host = NULL;
 const char *crm_mail_prefix = NULL;
 const char *crm_mail_from = NULL;
 const char *crm_mail_to = NULL;
 const char *external_agent = NULL;
 const char *external_recipient = NULL;
 
 cib_t *cib = NULL;
 stonith_t *st = NULL;
 xmlNode *current_cib = NULL;
 
 gboolean one_shot = FALSE;
 gboolean has_warnings = FALSE;
 gboolean print_timing = FALSE;
 gboolean watch_fencing = FALSE;
 gboolean print_brief = FALSE;
 gboolean print_pending = TRUE;
 gboolean print_clone_detail = FALSE;
 
 /* FIXME allow, detect, and correctly interpret glob pattern or regex? */
 const char *print_neg_location_prefix = "";
 
 /* Never display node attributes whose name starts with one of these prefixes */
 #define FILTER_STR { CRM_FAIL_COUNT_PREFIX, CRM_LAST_FAILURE_PREFIX,       \
                      "shutdown", "terminate", "standby", "probe_complete", \
                      "#", NULL }
 
 long last_refresh = 0;
 crm_trigger_t *refresh_trigger = NULL;
 
 /*
  * 1.3.6.1.4.1.32723 has been assigned to the project by IANA
  * http://www.iana.org/assignments/enterprise-numbers
  */
 #define PACEMAKER_PREFIX "1.3.6.1.4.1.32723"
 #define PACEMAKER_TRAP_PREFIX PACEMAKER_PREFIX ".1"
 
 #define snmp_crm_trap_oid   PACEMAKER_TRAP_PREFIX
 #define snmp_crm_oid_node   PACEMAKER_TRAP_PREFIX ".1"
 #define snmp_crm_oid_rsc    PACEMAKER_TRAP_PREFIX ".2"
 #define snmp_crm_oid_task   PACEMAKER_TRAP_PREFIX ".3"
 #define snmp_crm_oid_desc   PACEMAKER_TRAP_PREFIX ".4"
 #define snmp_crm_oid_status PACEMAKER_TRAP_PREFIX ".5"
 #define snmp_crm_oid_rc     PACEMAKER_TRAP_PREFIX ".6"
 #define snmp_crm_oid_trc    PACEMAKER_TRAP_PREFIX ".7"
 
 /* Define exit codes for monitoring-compatible output */
 #define MON_STATUS_OK   (0)
 #define MON_STATUS_WARN (1)
 
 /* Convenience macro for prettifying output (e.g. "node" vs "nodes") */
 #define s_if_plural(i) (((i) == 1)? "" : "s")
 
 #if CURSES_ENABLED
 #  define print_dot() if (output_format == mon_output_console) { \
 	printw(".");				\
 	clrtoeol();				\
 	refresh();				\
     } else {					\
 	fprintf(stdout, ".");			\
     }
 #else
 #  define print_dot() fprintf(stdout, ".");
 #endif
 
 #if CURSES_ENABLED
 #  define print_as(fmt, args...) if (output_format == mon_output_console) { \
 	printw(fmt, ##args);				\
 	clrtoeol();					\
 	refresh();					\
     } else {						\
 	fprintf(stdout, fmt, ##args);			\
     }
 #else
 #  define print_as(fmt, args...) fprintf(stdout, fmt, ##args);
 #endif
 
 static void
 blank_screen(void)
 {
 #if CURSES_ENABLED
     int lpc = 0;
 
     for (lpc = 0; lpc < LINES; lpc++) {
         move(lpc, 0);
         clrtoeol();
     }
     move(0, 0);
     refresh();
 #endif
 }
 
 static gboolean
 mon_timer_popped(gpointer data)
 {
     int rc = pcmk_ok;
 
 #if CURSES_ENABLED
     if (output_format == mon_output_console) {
         clear();
         refresh();
     }
 #endif
 
     if (timer_id > 0) {
         g_source_remove(timer_id);
     }
 
     print_as("Reconnecting...\n");
     rc = cib_connect(TRUE);
 
     if (rc != pcmk_ok) {
         timer_id = g_timeout_add(reconnect_msec, mon_timer_popped, NULL);
     }
     return FALSE;
 }
 
 static void
 mon_cib_connection_destroy(gpointer user_data)
 {
     print_as("Connection to the CIB terminated\n");
     if (cib) {
         cib->cmds->signoff(cib);
         timer_id = g_timeout_add(reconnect_msec, mon_timer_popped, NULL);
     }
     return;
 }
 
 /*
  * Mainloop signal handler.
  */
 static void
 mon_shutdown(int nsig)
 {
     clean_up(EX_OK);
 }
 
 #if ON_DARWIN
 #  define sighandler_t sig_t
 #endif
 
 #if CURSES_ENABLED
 #  ifndef HAVE_SIGHANDLER_T
 typedef void (*sighandler_t) (int);
 #  endif
 static sighandler_t ncurses_winch_handler;
 static void
 mon_winresize(int nsig)
 {
     static int not_done;
     int lines = 0, cols = 0;
 
     if (!not_done++) {
         if (ncurses_winch_handler)
             /* the original ncurses WINCH signal handler does the
              * magic of retrieving the new window size;
              * otherwise, we'd have to use ioctl or tgetent */
             (*ncurses_winch_handler) (SIGWINCH);
         getmaxyx(stdscr, lines, cols);
         resizeterm(lines, cols);
         mainloop_set_trigger(refresh_trigger);
     }
     not_done--;
 }
 #endif
 
 int
 cib_connect(gboolean full)
 {
     int rc = pcmk_ok;
     static gboolean need_pass = TRUE;
 
     CRM_CHECK(cib != NULL, return -EINVAL);
 
     if (getenv("CIB_passwd") != NULL) {
         need_pass = FALSE;
     }
 
     if (watch_fencing && st == NULL) {
         st = stonith_api_new();
     }
 
     if (watch_fencing && st->state == stonith_disconnected) {
         crm_trace("Connecting to stonith");
         rc = st->cmds->connect(st, crm_system_name, NULL);
         if (rc == pcmk_ok) {
             crm_trace("Setting up stonith callbacks");
             st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, mon_st_callback);
         }
     }
 
     if (cib->state != cib_connected_query && cib->state != cib_connected_command) {
         crm_trace("Connecting to the CIB");
         if ((output_format == mon_output_console) && need_pass && (cib->variant == cib_remote)) {
             need_pass = FALSE;
             print_as("Password:");
         }
 
         rc = cib->cmds->signon(cib, crm_system_name, cib_query);
 
         if (rc != pcmk_ok) {
             return rc;
         }
 
         rc = cib->cmds->query(cib, NULL, &current_cib, cib_scope_local | cib_sync_call);
         if (rc == pcmk_ok) {
             mon_refresh_display(NULL);
         }
 
         if (rc == pcmk_ok && full) {
             if (rc == pcmk_ok) {
                 rc = cib->cmds->set_connection_dnotify(cib, mon_cib_connection_destroy);
                 if (rc == -EPROTONOSUPPORT) {
                     print_as
                         ("Notification setup not supported, won't be able to reconnect after failure");
                     if (output_format == mon_output_console) {
                         sleep(2);
                     }
                     rc = pcmk_ok;
                 }
 
             }
 
             if (rc == pcmk_ok) {
                 cib->cmds->del_notify_callback(cib, T_CIB_DIFF_NOTIFY, crm_diff_update);
                 rc = cib->cmds->add_notify_callback(cib, T_CIB_DIFF_NOTIFY, crm_diff_update);
             }
 
             if (rc != pcmk_ok) {
                 print_as("Notification setup failed, could not monitor CIB actions");
                 if (output_format == mon_output_console) {
                     sleep(2);
                 }
                 clean_up(-rc);
             }
         }
     }
     return rc;
 }
 
 /* *INDENT-OFF* */
 static struct crm_option long_options[] = {
     /* Top-level Options */
     {"help",           0, 0, '?', "\tThis text"},
     {"version",        0, 0, '$', "\tVersion information"  },
     {"verbose",        0, 0, 'V', "\tIncrease debug output"},
     {"quiet",          0, 0, 'Q', "\tDisplay only essential output" },
 
     {"-spacer-",	1, 0, '-', "\nModes (mutually exclusive):"},
     {"as-html",        1, 0, 'h', "\tWrite cluster status to the named html file"},
     {"as-xml",         0, 0, 'X', "\t\tWrite cluster status as xml to stdout. This will enable one-shot mode."},
     {"web-cgi",        0, 0, 'w', "\t\tWeb mode with output suitable for CGI (preselected when run as *.cgi)"},
     {"simple-status",  0, 0, 's', "\tDisplay the cluster status once as a simple one line output (suitable for nagios)"},
     {"snmp-traps",     1, 0, 'S', "\tSend SNMP traps to this station", !ENABLE_SNMP},
     {"snmp-community", 1, 0, 'C', "Specify community for SNMP traps(default is NULL)", !ENABLE_SNMP},
     {"mail-to",        1, 0, 'T', "\tSend Mail alerts to this user.  See also --mail-from, --mail-host, --mail-prefix", !ENABLE_ESMTP},
 
     {"-spacer-",	1, 0, '-', "\nDisplay Options:"},
     {"group-by-node",  0, 0, 'n', "\tGroup resources by node"     },
     {"inactive",       0, 0, 'r', "\t\tDisplay inactive resources"  },
     {"failcounts",     0, 0, 'f', "\tDisplay resource fail counts"},
     {"operations",     0, 0, 'o', "\tDisplay resource operation history" },
     {"timing-details", 0, 0, 't', "\tDisplay resource operation history with timing details" },
     {"tickets",        0, 0, 'c', "\t\tDisplay cluster tickets"},
     {"watch-fencing",  0, 0, 'W', "\tListen for fencing events. For use with --external-agent, --mail-to and/or --snmp-traps where supported"},
     {"neg-locations",  2, 0, 'L', "Display negative location constraints [optionally filtered by id prefix]"},
     {"show-node-attributes", 0, 0, 'A', "Display node attributes" },
     {"hide-headers",   0, 0, 'D', "\tHide all headers" },
     {"show-detail",    0, 0, 'R', "\tShow more details (node IDs, individual clone instances)" },
     {"brief",          0, 0, 'b', "\t\tBrief output" },
     {"pending",        0, 0, 'j', "\t\tDisplay pending state if 'record-pending' is enabled", pcmk_option_hidden},
 
     {"-spacer-",	1, 0, '-', "\nAdditional Options:"},
     {"interval",       1, 0, 'i', "\tUpdate frequency in seconds" },
     {"one-shot",       0, 0, '1', "\t\tDisplay the cluster status once on the console and exit"},
     {"disable-ncurses",0, 0, 'N', "\tDisable the use of ncurses", !CURSES_ENABLED},
     {"daemonize",      0, 0, 'd', "\tRun in the background as a daemon"},
     {"pid-file",       1, 0, 'p', "\t(Advanced) Daemon pid file location"},
     {"mail-from",      1, 0, 'F', "\tMail alerts should come from the named user", !ENABLE_ESMTP},
     {"mail-host",      1, 0, 'H', "\tMail alerts should be sent via the named host", !ENABLE_ESMTP},
     {"mail-prefix",    1, 0, 'P', "Subjects for mail alerts should start with this string", !ENABLE_ESMTP},
     {"external-agent",    1, 0, 'E', "A program to run when resource operations take place."},
     {"external-recipient",1, 0, 'e', "A recipient for your program (assuming you want the program to send something to someone)."},
 
 
     {"xml-file",       1, 0, 'x', NULL, pcmk_option_hidden},
 
     {"-spacer-",	1, 0, '-', "\nExamples:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', "Display the cluster status on the console with updates as they occur:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_mon", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Display the cluster status on the console just once then exit:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_mon -1", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Display your cluster status, group resources by node, and include inactive resources in the list:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_mon --group-by-node --inactive", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Start crm_mon as a background daemon and have it write the cluster status to an HTML file:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_mon --daemonize --as-html /path/to/docroot/filename.html", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Start crm_mon and export the current cluster status as xml to stdout, then exit.:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_mon --as-xml", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Start crm_mon as a background daemon and have it send email alerts:", pcmk_option_paragraph|!ENABLE_ESMTP},
     {"-spacer-",	1, 0, '-', " crm_mon --daemonize --mail-to user@example.com --mail-host mail.example.com", pcmk_option_example|!ENABLE_ESMTP},
     {"-spacer-",	1, 0, '-', "Start crm_mon as a background daemon and have it send SNMP alerts:", pcmk_option_paragraph|!ENABLE_SNMP},
     {"-spacer-",	1, 0, '-', " crm_mon --daemonize --snmp-traps snmptrapd.example.com", pcmk_option_example|!ENABLE_SNMP},
 
     {NULL, 0, 0, 0}
 };
 /* *INDENT-ON* */
 
 #if CURSES_ENABLED
 static const char *
 get_option_desc(char c)
 {
     int lpc;
 
     for (lpc = 0; long_options[lpc].name != NULL; lpc++) {
 
         if (long_options[lpc].name[0] == '-')
             continue;
 
         if (long_options[lpc].val == c) {
             const char * tab = NULL;
             tab = strrchr(long_options[lpc].desc, '\t');
             return tab ? ++tab : long_options[lpc].desc;
         }
     }
 
     return NULL;
 }
 
 #define print_option_help(option, condition) \
     print_as("%c %c: \t%s\n", ((condition)? '*': ' '), option, get_option_desc(option));
 
 static gboolean
 detect_user_input(GIOChannel *channel, GIOCondition condition, gpointer unused)
 {
     int c;
     gboolean config_mode = FALSE;
 
     while (1) {
 
         /* Get user input */
         c = getchar();
 
         switch (c) {
             case 'c':
                 show ^= mon_show_tickets;
                 break;
             case 'f':
                 show ^= mon_show_failcounts;
                 break;
             case 'n':
                 group_by_node = ! group_by_node;
                 break;
             case 'o':
                 show ^= mon_show_operations;
                 if ((show & mon_show_operations) == 0) {
                     print_timing = 0;
                 }
                 break;
             case 'r':
                 inactive_resources = ! inactive_resources;
                 break;
             case 'R':
                 print_clone_detail = ! print_clone_detail;
                 break;
             case 't':
                 print_timing = ! print_timing;
                 if (print_timing) {
                     show |= mon_show_operations;
                 }
                 break;
             case 'A':
                 show ^= mon_show_attributes;
                 break;
             case 'L':
                 show ^= mon_show_bans;
                 break;
             case 'D':
                 /* If any header is shown, clear them all, otherwise set them all */
                 if (show & mon_show_headers) {
                     show &= ~mon_show_headers;
                 } else {
                     show |= mon_show_headers;
                 }
                 break;
             case 'b':
                 print_brief = ! print_brief;
                 break;
             case 'j':
                 print_pending = ! print_pending;
                 break;
             case '?':
                 config_mode = TRUE;
                 break;
             default:
                 goto refresh;
         }
 
         if (!config_mode)
             goto refresh;
 
         blank_screen();
 
         print_as("Display option change mode\n");
         print_as("\n");
         print_option_help('c', show & mon_show_tickets);
         print_option_help('f', show & mon_show_failcounts);
         print_option_help('n', group_by_node);
         print_option_help('o', show & mon_show_operations);
         print_option_help('r', inactive_resources);
         print_option_help('t', print_timing);
         print_option_help('A', show & mon_show_attributes);
         print_option_help('L', show & mon_show_bans);
         print_option_help('D', (show & mon_show_headers) == 0);
         print_option_help('R', print_clone_detail);
         print_option_help('b', print_brief);
         print_option_help('j', print_pending);
         print_as("\n");
         print_as("Toggle fields via field letter, type any other key to return");
     }
 
 refresh:
     mon_refresh_display(NULL);
     return TRUE;
 }
 #endif
 
 int
 main(int argc, char **argv)
 {
     int flag;
     int argerr = 0;
     int exit_code = 0;
     int option_index = 0;
 
     pid_file = strdup("/tmp/ClusterMon.pid");
     crm_log_cli_init("crm_mon");
     crm_set_options(NULL, "mode [options]", long_options,
                     "Provides a summary of cluster's current state."
                     "\n\nOutputs varying levels of detail in a number of different formats.\n");
 
 #if !defined (ON_DARWIN) && !defined (ON_BSD)
     /* prevent zombies */
     signal(SIGCLD, SIG_IGN);
 #endif
 
     if (crm_ends_with_ext(argv[0], ".cgi") == TRUE) {
         output_format = mon_output_cgi;
         one_shot = TRUE;
     }
 
     while (1) {
         flag = crm_get_option(argc, argv, &option_index);
         if (flag == -1)
             break;
 
         switch (flag) {
             case 'V':
                 crm_bump_log_level(argc, argv);
                 break;
             case 'Q':
                 show &= ~mon_show_times;
                 break;
             case 'i':
                 reconnect_msec = crm_get_msec(optarg);
                 break;
             case 'n':
                 group_by_node = TRUE;
                 break;
             case 'r':
                 inactive_resources = TRUE;
                 break;
             case 'W':
                 watch_fencing = TRUE;
                 break;
             case 'd':
                 daemonize = TRUE;
                 break;
             case 't':
                 print_timing = TRUE;
                 show |= mon_show_operations;
                 break;
             case 'o':
                 show |= mon_show_operations;
                 break;
             case 'f':
                 show |= mon_show_failcounts;
                 break;
             case 'A':
                 show |= mon_show_attributes;
                 break;
             case 'L':
                 show |= mon_show_bans;
                 print_neg_location_prefix = optarg? optarg : "";
                 break;
             case 'D':
                 show &= ~mon_show_headers;
                 break;
             case 'b':
                 print_brief = TRUE;
                 break;
             case 'j':
                 print_pending = TRUE;
                 break;
             case 'R':
                 print_clone_detail = TRUE;
                 break;
             case 'c':
                 show |= mon_show_tickets;
                 break;
             case 'p':
                 free(pid_file);
                 if(optarg == NULL) {
                     return crm_help(flag, EX_USAGE);
                 }
                 pid_file = strdup(optarg);
                 break;
             case 'x':
                 if(optarg == NULL) {
                     return crm_help(flag, EX_USAGE);
                 }
                 xml_file = strdup(optarg);
                 one_shot = TRUE;
                 break;
             case 'h':
                 if(optarg == NULL) {
                     return crm_help(flag, EX_USAGE);
                 }
                 argerr += (output_format != mon_output_console);
                 output_format = mon_output_html;
                 output_filename = strdup(optarg);
                 umask(S_IWGRP | S_IWOTH);
                 break;
             case 'X':
                 argerr += (output_format != mon_output_console);
                 output_format = mon_output_xml;
                 one_shot = TRUE;
                 break;
             case 'w':
                 /* do not allow argv[0] and argv[1...] redundancy */
                 argerr += (output_format != mon_output_console);
                 output_format = mon_output_cgi;
                 one_shot = TRUE;
                 break;
             case 's':
                 argerr += (output_format != mon_output_console);
                 output_format = mon_output_monitor;
                 one_shot = TRUE;
                 break;
             case 'S':
                 snmp_target = optarg;
                 break;
             case 'T':
                 crm_mail_to = optarg;
                 break;
             case 'F':
                 crm_mail_from = optarg;
                 break;
             case 'H':
                 crm_mail_host = optarg;
                 break;
             case 'P':
                 crm_mail_prefix = optarg;
                 break;
             case 'E':
                 external_agent = optarg;
                 break;
             case 'e':
                 external_recipient = optarg;
                 break;
             case '1':
                 one_shot = TRUE;
                 break;
             case 'N':
                 if (output_format == mon_output_console) {
                     output_format = mon_output_plain;
                 }
                 break;
             case 'C':
                 snmp_community = optarg;
                 break;
             case '$':
             case '?':
                 return crm_help(flag, EX_OK);
                 break;
             default:
                 printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
                 ++argerr;
                 break;
         }
     }
 
     /* Extra sanity checks when in CGI mode */
     if (output_format == mon_output_cgi) {
         argerr += (optind < argc);
         argerr += (output_filename != NULL);
         argerr += (xml_file != NULL);
         argerr += (snmp_target != NULL);
         argerr += (crm_mail_to != NULL);
         argerr += (external_agent != NULL);
         argerr += (daemonize == TRUE);  /* paranoia */
 
     } else if (optind < argc) {
         printf("non-option ARGV-elements: ");
         while (optind < argc)
             printf("%s ", argv[optind++]);
         printf("\n");
     }
 
     if (argerr) {
         if (output_format == mon_output_cgi) {
             fprintf(stdout, "Content-Type: text/plain\n"
                             "Status: 500\n\n");
             return EX_USAGE;
         }
         return crm_help('?', EX_USAGE);
     }
 
     /* XML output always prints everything */
     if (output_format == mon_output_xml) {
         show = mon_show_all;
         print_timing = TRUE;
     }
 
     if (one_shot) {
         if (output_format == mon_output_console) {
             output_format = mon_output_plain;
         }
 
     } else if (daemonize) {
         if ((output_format == mon_output_console) || (output_format == mon_output_plain)) {
             output_format = mon_output_none;
         }
         crm_enable_stderr(FALSE);
 
         if ((output_format != mon_output_html) && (output_format != mon_output_xml)
             && !snmp_target && !crm_mail_to && !external_agent) {
             printf
                 ("Looks like you forgot to specify one or more of: --as-html, --as-xml, --mail-to, --snmp-target, --external-agent\n");
             return crm_help('?', EX_USAGE);
         }
 
         crm_make_daemon(crm_system_name, TRUE, pid_file);
 
     } else if (output_format == mon_output_console) {
 #if CURSES_ENABLED
         initscr();
         cbreak();
         noecho();
         crm_enable_stderr(FALSE);
 #else
         one_shot = TRUE;
         output_format = mon_output_plain;
         printf("Defaulting to one-shot mode\n");
         printf("You need to have curses available at compile time to enable console mode\n");
 #endif
     }
 
     crm_info("Starting %s", crm_system_name);
     if (xml_file != NULL) {
         current_cib = filename2xml(xml_file);
         mon_refresh_display(NULL);
         return exit_code;
     }
 
     if (current_cib == NULL) {
         cib = cib_new();
 
         do {
             if (!one_shot) {
                 print_as("Attempting connection to the cluster...\n");
             }
             exit_code = cib_connect(!one_shot);
 
             if (one_shot) {
                 break;
 
             } else if (exit_code != pcmk_ok) {
                 sleep(reconnect_msec / 1000);
 #if CURSES_ENABLED
                 if (output_format == mon_output_console) {
                     clear();
                     refresh();
                 }
 #endif
             }
 
         } while (exit_code == -ENOTCONN);
 
         if (exit_code != pcmk_ok) {
             if (output_format == mon_output_monitor) {
                 printf("CLUSTER WARN: Connection to cluster failed: %s\n", pcmk_strerror(exit_code));
                 clean_up(MON_STATUS_WARN);
             } else {
                 print_as("\nConnection to cluster failed: %s\n", pcmk_strerror(exit_code));
             }
             if (output_format == mon_output_console) {
                 sleep(2);
             }
             clean_up(-exit_code);
         }
     }
 
     if (one_shot) {
         return exit_code;
     }
 
     mainloop = g_main_new(FALSE);
 
     mainloop_add_signal(SIGTERM, mon_shutdown);
     mainloop_add_signal(SIGINT, mon_shutdown);
 #if CURSES_ENABLED
     if (output_format == mon_output_console) {
         ncurses_winch_handler = signal(SIGWINCH, mon_winresize);
         if (ncurses_winch_handler == SIG_DFL ||
             ncurses_winch_handler == SIG_IGN || ncurses_winch_handler == SIG_ERR)
             ncurses_winch_handler = NULL;
         g_io_add_watch(g_io_channel_unix_new(STDIN_FILENO), G_IO_IN, detect_user_input, NULL);
     }
 #endif
     refresh_trigger = mainloop_add_trigger(G_PRIORITY_LOW, mon_refresh_display, NULL);
 
     g_main_run(mainloop);
     g_main_destroy(mainloop);
 
     crm_info("Exiting %s", crm_system_name);
 
     clean_up(0);
     return 0;                   /* never reached */
 }
 
 #define mon_warn(fmt...) do {			\
 	if (!has_warnings) {			\
 	    print_as("CLUSTER WARN:");		\
 	} else {				\
 	    print_as(",");			\
 	}					\
 	print_as(fmt);				\
 	has_warnings = TRUE;			\
     } while(0)
 
 static int
 count_resources(pe_working_set_t * data_set, resource_t * rsc)
 {
     int count = 0;
     GListPtr gIter = NULL;
 
     if (rsc == NULL) {
         gIter = data_set->resources;
     } else if (rsc->children) {
         gIter = rsc->children;
     } else {
         return is_not_set(rsc->flags, pe_rsc_orphan);
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         count += count_resources(data_set, gIter->data);
     }
     return count;
 }
 
 /*!
  * \internal
  * \brief Print one-line status suitable for use with monitoring software
  *
  * \param[in] data_set  Working set of CIB state
  *
  * \note This function's output (and the return code when the program exits)
  *       should conform to https://www.monitoring-plugins.org/doc/guidelines.html
  */
 static void
 print_simple_status(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     int nodes_online = 0;
     int nodes_standby = 0;
     int nodes_maintenance = 0;
 
     if (data_set->dc_node == NULL) {
         mon_warn(" No DC");
     }
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
         if (node->details->standby && node->details->online) {
             nodes_standby++;
         } else if (node->details->maintenance && node->details->online) {
             nodes_maintenance++;
         } else if (node->details->online) {
             nodes_online++;
         } else {
             mon_warn(" offline node: %s", node->details->uname);
         }
     }
 
     if (!has_warnings) {
         int nresources = count_resources(data_set, NULL);
 
         print_as("CLUSTER OK: %d node%s online", nodes_online, s_if_plural(nodes_online));
         if (nodes_standby > 0) {
             print_as(", %d standby node%s", nodes_standby, s_if_plural(nodes_standby));
         }
         if (nodes_maintenance > 0) {
             print_as(", %d maintenance node%s", nodes_maintenance, s_if_plural(nodes_maintenance));
         }
         print_as(", %d resource%s configured", nresources, s_if_plural(nresources));
     }
 
     print_as("\n");
 }
 
 /*!
  * \internal
  * \brief Print a [name]=[value][units] pair, optionally using time string
  *
  * \param[in] stream      File stream to display output to
  * \param[in] name        Name to display
  * \param[in] value       Value to display (or NULL to convert time instead)
  * \param[in] units       Units to display (or NULL for no units)
  * \param[in] epoch_time  Epoch time to convert if value is NULL
  */
 static void
 print_nvpair(FILE *stream, const char *name, const char *value,
              const char *units, time_t epoch_time)
 {
     /* print name= */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as(" %s=", name);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
         case mon_output_xml:
             fprintf(stream, " %s=", name);
             break;
 
         default:
             break;
     }
 
     /* If we have a value (and optionally units), print it */
     if (value) {
         switch (output_format) {
             case mon_output_plain:
             case mon_output_console:
                 print_as("%s%s", value, (units? units : ""));
                 break;
 
             case mon_output_html:
             case mon_output_cgi:
                 fprintf(stream, "%s%s", value, (units? units : ""));
                 break;
 
             case mon_output_xml:
                 fprintf(stream, "\"%s%s\"", value, (units? units : ""));
                 break;
 
             default:
                 break;
         }
 
     /* Otherwise print user-friendly time string */
     } else {
         static char empty_str[] = "";
         char *c, *date_str = asctime(localtime(&epoch_time));
 
         for (c = (date_str != NULL) ? date_str : empty_str; *c != '\0'; ++c) {
             if (*c == '\n') {
                 *c = '\0';
                 break;
             }
         }
         switch (output_format) {
             case mon_output_plain:
             case mon_output_console:
                 print_as("'%s'", date_str);
                 break;
 
             case mon_output_html:
             case mon_output_cgi:
             case mon_output_xml:
                 fprintf(stream, "\"%s\"", date_str);
                 break;
 
             default:
                 break;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Print whatever is needed to start a node section
  *
  * \param[in] stream     File stream to display output to
  * \param[in] node       Node to print
  */
 static void
 print_node_start(FILE *stream, node_t *node)
 {
     char *node_name;
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             node_name = get_node_display_name(node);
             print_as("* Node %s:\n", node_name);
             free(node_name);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             node_name = get_node_display_name(node);
             fprintf(stream, "  <h3>Node: %s</h3>\n  <ul>\n", node_name);
             free(node_name);
             break;
 
         case mon_output_xml:
             fprintf(stream, "        <node name=\"%s\">\n", node->details->uname);
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print whatever is needed to end a node section
  *
  * \param[in] stream     File stream to display output to
  */
 static void
 print_node_end(FILE *stream)
 {
     switch (output_format) {
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "  </ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "        </node>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print resources section heading appropriate to options
  *
  * \param[in] stream      File stream to display output to
  */
 static void
 print_resources_heading(FILE *stream)
 {
     const char *heading;
 
     if (group_by_node) {
 
         /* Active resources have already been printed by node */
         heading = (inactive_resources? "Inactive resources" : NULL);
 
     } else if (inactive_resources) {
         heading = "Full list of resources";
 
     } else {
         heading = "Active resources";
     }
 
     /* Print section heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n%s:\n\n", heading);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <hr />\n <h2>%s</h2>\n", heading);
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <resources>\n");
             break;
 
         default:
             break;
     }
 
 }
 
 /*!
  * \internal
  * \brief Print whatever resource section closing is appropriate
  *
  * \param[in] stream     File stream to display output to
  */
 static void
 print_resources_closing(FILE *stream, gboolean printed_heading)
 {
     const char *heading;
 
     /* What type of resources we did or did not display */
     if (group_by_node) {
         heading = "inactive ";
     } else if (inactive_resources) {
         heading = "";
     } else {
         heading = "active ";
     }
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             if (!printed_heading) {
                 print_as("\nNo %sresources\n\n", heading);
             }
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             if (!printed_heading) {
                 fprintf(stream, " <hr />\n <h2>No %sresources</h2>\n", heading);
             }
             break;
 
         case mon_output_xml:
             fprintf(stream, "    %s\n",
                     (printed_heading? "</resources>" : "<resources/>"));
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print whatever resource section(s) are appropriate
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Cluster state to display
  * \param[in] print_opts  Bitmask of pe_print_options
  */
 static void
 print_resources(FILE *stream, pe_working_set_t *data_set, int print_opts)
 {
     GListPtr rsc_iter;
     const char *prefix = NULL;
     gboolean printed_heading = FALSE;
     gboolean brief_output = print_brief;
 
     /* If we already showed active resources by node, and
      * we're not showing inactive resources, we have nothing to do
      */
     if (group_by_node && !inactive_resources) {
         return;
     }
 
     /* XML uses an indent, and ignores brief option for resources */
     if (output_format == mon_output_xml) {
         prefix = "        ";
         brief_output = FALSE;
     }
 
     /* If we haven't already printed resources grouped by node,
      * and brief output was requested, print resource summary */
     if (brief_output && !group_by_node) {
         print_resources_heading(stream);
         printed_heading = TRUE;
         print_rscs_brief(data_set->resources, NULL, print_opts, stream,
                          inactive_resources);
     }
 
     /* For each resource, display it if appropriate */
     for (rsc_iter = data_set->resources; rsc_iter != NULL; rsc_iter = rsc_iter->next) {
         resource_t *rsc = (resource_t *) rsc_iter->data;
 
         /* Complex resources may have some sub-resources active and some inactive */
         gboolean is_active = rsc->fns->active(rsc, TRUE);
         gboolean partially_active = rsc->fns->active(rsc, FALSE);
 
         /* Skip inactive orphans (deleted but still in CIB) */
         if (is_set(rsc->flags, pe_rsc_orphan) && !is_active) {
             continue;
 
         /* Skip active resources if we already displayed them by node */
         } else if (group_by_node) {
             if (is_active) {
                 continue;
             }
 
         /* Skip primitives already counted in a brief summary */
         } else if (brief_output && (rsc->variant == pe_native)) {
             continue;
 
         /* Skip resources that aren't at least partially active,
          * unless we're displaying inactive resources
          */
         } else if (!partially_active && !inactive_resources) {
             continue;
         }
 
         /* Print this resource */
         if (printed_heading == FALSE) {
             print_resources_heading(stream);
             printed_heading = TRUE;
         }
         rsc->fns->print(rsc, prefix, print_opts, stream);
     }
 
     print_resources_closing(stream, printed_heading);
 }
 
 /*!
  * \internal
  * \brief Print heading for resource history
  *
  * \param[in] stream      File stream to display output to
  * \param[in] data_set    Current state of CIB
  * \param[in] node        Node that ran this resource
  * \param[in] rsc         Resource to print
  * \param[in] rsc_id      ID of resource to print
  * \param[in] all         Whether to print every resource or just failed ones
  */
 static void
 print_rsc_history_start(FILE *stream, pe_working_set_t *data_set, node_t *node,
                         resource_t *rsc, const char *rsc_id, gboolean all)
 {
     time_t last_failure = 0;
     int failcount = rsc?
-                    pe_get_failcount(node, rsc, &last_failure, FALSE, NULL,
-                                     data_set)
+                    pe_get_failcount(node, rsc, &last_failure, pe_fc_default,
+                                     NULL, data_set)
                     : 0;
 
     if (!all && !failcount && (last_failure <= 0)) {
         return;
     }
 
     /* Print resource ID */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("   %s:", rsc_id);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "   <li>%s:", rsc_id);
             break;
 
         case mon_output_xml:
             fprintf(stream, "            <resource_history id=\"%s\"", rsc_id);
             break;
 
         default:
             break;
     }
 
     /* If resource is an orphan, that's all we can say about it */
     if (rsc == NULL) {
         switch (output_format) {
             case mon_output_plain:
             case mon_output_console:
                 print_as(" orphan");
                 break;
 
             case mon_output_html:
             case mon_output_cgi:
                 fprintf(stream, " orphan");
                 break;
 
             case mon_output_xml:
                 fprintf(stream, " orphan=\"true\"");
                 break;
 
             default:
                 break;
         }
 
     /* If resource is not an orphan, print some details */
     } else if (all || failcount || (last_failure > 0)) {
 
         /* Print migration threshold */
         switch (output_format) {
             case mon_output_plain:
             case mon_output_console:
                 print_as(" migration-threshold=%d", rsc->migration_threshold);
                 break;
 
             case mon_output_html:
             case mon_output_cgi:
                 fprintf(stream, " migration-threshold=%d", rsc->migration_threshold);
                 break;
 
             case mon_output_xml:
                 fprintf(stream, " orphan=\"false\" migration-threshold=\"%d\"",
                         rsc->migration_threshold);
                 break;
 
             default:
                 break;
         }
 
         /* Print fail count if any */
         if (failcount > 0) {
             switch (output_format) {
                 case mon_output_plain:
                 case mon_output_console:
                     print_as(" " CRM_FAIL_COUNT_PREFIX "=%d", failcount);
                     break;
 
                 case mon_output_html:
                 case mon_output_cgi:
                     fprintf(stream, " " CRM_FAIL_COUNT_PREFIX "=%d", failcount);
                     break;
 
                 case mon_output_xml:
                     fprintf(stream, " " CRM_FAIL_COUNT_PREFIX "=\"%d\"",
                             failcount);
                     break;
 
                 default:
                     break;
             }
         }
 
         /* Print last failure time if any */
         if (last_failure > 0) {
             print_nvpair(stream, CRM_LAST_FAILURE_PREFIX, NULL, NULL,
                          last_failure);
         }
     }
 
     /* End the heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "\n    <ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, ">\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print closing for resource history
  *
  * \param[in] stream      File stream to display output to
  */
 static void
 print_rsc_history_end(FILE *stream)
 {
     switch (output_format) {
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "    </ul>\n   </li>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "            </resource_history>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print operation history
  *
  * \param[in] stream      File stream to display output to
  * \param[in] data_set    Current state of CIB
  * \param[in] node        Node this operation is for
  * \param[in] xml_op      Root of XML tree describing this operation
  * \param[in] task        Task parsed from this operation's XML
  * \param[in] interval    Interval parsed from this operation's XML
  * \param[in] rc          Return code parsed from this operation's XML
  */
 static void
 print_op_history(FILE *stream, pe_working_set_t *data_set, node_t *node,
                  xmlNode *xml_op, const char *task, const char *interval, int rc)
 {
     const char *value = NULL;
     const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
 
     /* Begin the operation description */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("    + (%s) %s:", call, task);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "     <li>(%s) %s:", call, task);
             break;
 
         case mon_output_xml:
             fprintf(stream, "                <operation_history call=\"%s\" task=\"%s\"",
                     call, task);
             break;
 
         default:
             break;
     }
 
     /* Add name=value pairs as appropriate */
     if (safe_str_neq(interval, "0")) {
         print_nvpair(stream, "interval", interval, "ms", 0);
     }
     if (print_timing) {
         int int_value;
         const char *attr;
 
         attr = XML_RSC_OP_LAST_CHANGE;
         value = crm_element_value(xml_op, attr);
         if (value) {
             int_value = crm_parse_int(value, NULL);
             if (int_value > 0) {
                 print_nvpair(stream, attr, NULL, NULL, int_value);
             }
         }
 
         attr = XML_RSC_OP_LAST_RUN;
         value = crm_element_value(xml_op, attr);
         if (value) {
             int_value = crm_parse_int(value, NULL);
             if (int_value > 0) {
                 print_nvpair(stream, attr, NULL, NULL, int_value);
             }
         }
 
         attr = XML_RSC_OP_T_EXEC;
         value = crm_element_value(xml_op, attr);
         if (value) {
             print_nvpair(stream, attr, value, "ms", 0);
         }
 
         attr = XML_RSC_OP_T_QUEUE;
         value = crm_element_value(xml_op, attr);
         if (value) {
             print_nvpair(stream, attr, value, "ms", 0);
         }
     }
 
     /* End the operation description */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as(" rc=%d (%s)\n", rc, services_ocf_exitcode_str(rc));
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " rc=%d (%s)</li>\n", rc, services_ocf_exitcode_str(rc));
             break;
 
         case mon_output_xml:
             fprintf(stream, " rc=\"%d\" rc_text=\"%s\" />\n", rc, services_ocf_exitcode_str(rc));
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print resource operation/failure history
  *
  * \param[in] stream      File stream to display output to
  * \param[in] data_set    Current state of CIB
  * \param[in] node        Node that ran this resource
  * \param[in] rsc_entry   Root of XML tree describing resource status
  * \param[in] operations  Whether to print operations or just failcounts
  */
 static void
 print_rsc_history(FILE *stream, pe_working_set_t *data_set, node_t *node,
                   xmlNode *rsc_entry, gboolean operations)
 {
     GListPtr gIter = NULL;
     GListPtr op_list = NULL;
     gboolean printed = FALSE;
     const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
     resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
     xmlNode *rsc_op = NULL;
 
     /* If we're not showing operations, just print the resource failure summary */
     if (operations == FALSE) {
         print_rsc_history_start(stream, data_set, node, rsc, rsc_id, FALSE);
         print_rsc_history_end(stream);
         return;
     }
 
     /* Create a list of this resource's operations */
     for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next(rsc_op)) {
         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             op_list = g_list_append(op_list, rsc_op);
         }
     }
     op_list = g_list_sort(op_list, sort_op_by_callid);
 
     /* Print each operation */
     for (gIter = op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *xml_op = (xmlNode *) gIter->data;
         const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
         const char *interval = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
         const char *op_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC);
         int rc = crm_parse_int(op_rc, "0");
 
         /* Display 0-interval monitors as "probe" */
         if (safe_str_eq(task, CRMD_ACTION_STATUS) && safe_str_eq(interval, "0")) {
             task = "probe";
         }
 
         /* Ignore notifies and some probes */
         if (safe_str_eq(task, CRMD_ACTION_NOTIFY) || (safe_str_eq(task, "probe") && (rc == 7))) {
             continue;
         }
 
         /* If this is the first printed operation, print heading for resource */
         if (printed == FALSE) {
             printed = TRUE;
             print_rsc_history_start(stream, data_set, node, rsc, rsc_id, TRUE);
         }
 
         /* Print the operation */
         print_op_history(stream, data_set, node, xml_op, task, interval, rc);
     }
 
     /* Free the list we created (no need to free the individual items) */
     g_list_free(op_list);
 
     /* If we printed anything, close the resource */
     if (printed) {
         print_rsc_history_end(stream);
     }
 }
 
 /*!
  * \internal
  * \brief Print node operation/failure history
  *
  * \param[in] stream      File stream to display output to
  * \param[in] data_set    Current state of CIB
  * \param[in] node_state  Root of XML tree describing node status
  * \param[in] operations  Whether to print operations or just failcounts
  */
 static void
 print_node_history(FILE *stream, pe_working_set_t *data_set,
                    xmlNode *node_state, gboolean operations)
 {
     node_t *node = pe_find_node_id(data_set->nodes, ID(node_state));
     xmlNode *lrm_rsc = NULL;
     xmlNode *rsc_entry = NULL;
 
     if (node && node->details && node->details->online) {
         print_node_start(stream, node);
 
         lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
         lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
 
         /* Print history of each of the node's resources */
         for (rsc_entry = __xml_first_child(lrm_rsc); rsc_entry != NULL;
              rsc_entry = __xml_next(rsc_entry)) {
 
             if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
                 print_rsc_history(stream, data_set, node, rsc_entry, operations);
             }
         }
 
         print_node_end(stream);
     }
 }
 
 /*!
  * \internal
  * \brief Print extended information about an attribute if appropriate
  *
  * \param[in] data_set  Working set of CIB state
  *
  * \return TRUE if extended information was printed, FALSE otherwise
  * \note Currently, extended information is only supported for ping/pingd
  *       resources, for which a message will be printed if connectivity is lost
  *       or degraded.
  */
 static gboolean
 print_attr_msg(FILE *stream, node_t * node, GListPtr rsc_list, const char *attrname, const char *attrvalue)
 {
     GListPtr gIter = NULL;
 
     for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
         const char *type = g_hash_table_lookup(rsc->meta, "type");
 
         if (rsc->children != NULL) {
             if (print_attr_msg(stream, node, rsc->children, attrname, attrvalue)) {
                 return TRUE;
             }
         }
 
         if (safe_str_eq(type, "ping") || safe_str_eq(type, "pingd")) {
             const char *name = g_hash_table_lookup(rsc->parameters, "name");
 
             if (name == NULL) {
                 name = "pingd";
             }
 
             /* To identify the resource with the attribute name. */
             if (safe_str_eq(name, attrname)) {
                 int host_list_num = 0;
                 int expected_score = 0;
                 int value = crm_parse_int(attrvalue, "0");
                 const char *hosts = g_hash_table_lookup(rsc->parameters, "host_list");
                 const char *multiplier = g_hash_table_lookup(rsc->parameters, "multiplier");
 
                 if(hosts) {
                     char **host_list = g_strsplit(hosts, " ", 0);
                     host_list_num = g_strv_length(host_list);
                     g_strfreev(host_list);
                 }
 
                 /* pingd multiplier is the same as the default value. */
                 expected_score = host_list_num * crm_parse_int(multiplier, "1");
 
                 switch (output_format) {
                     case mon_output_plain:
                     case mon_output_console:
                         if (value <= 0) {
                             print_as("\t: Connectivity is lost");
                         } else if (value < expected_score) {
                             print_as("\t: Connectivity is degraded (Expected=%d)", expected_score);
                         }
                         break;
 
                     case mon_output_html:
                     case mon_output_cgi:
                         if (value <= 0) {
                             fprintf(stream, " <b>(connectivity is lost)</b>");
                         } else if (value < expected_score) {
                             fprintf(stream, " <b>(connectivity is degraded -- expected %d)</b>",
                                     expected_score);
                         }
                         break;
 
                     case mon_output_xml:
                         fprintf(stream, " expected=\"%d\"", expected_score);
                         break;
 
                     default:
                         break;
                 }
                 return TRUE;
             }
         }
     }
     return FALSE;
 }
 
 static int
 compare_attribute(gconstpointer a, gconstpointer b)
 {
     int rc;
 
     rc = strcmp((const char *)a, (const char *)b);
 
     return rc;
 }
 
 static void
 create_attr_list(gpointer name, gpointer value, gpointer data)
 {
     int i;
     const char *filt_str[] = FILTER_STR;
 
     CRM_CHECK(name != NULL, return);
 
     /* filtering automatic attributes */
     for (i = 0; filt_str[i] != NULL; i++) {
         if (g_str_has_prefix(name, filt_str[i])) {
             return;
         }
     }
 
     attr_list = g_list_insert_sorted(attr_list, name, compare_attribute);
 }
 
 /* structure for passing multiple user data to g_list_foreach() */
 struct mon_attr_data {
     FILE *stream;
     node_t *node;
 };
 
 static void
 print_node_attribute(gpointer name, gpointer user_data)
 {
     const char *value = NULL;
     struct mon_attr_data *data = (struct mon_attr_data *) user_data;
 
     value = pe_node_attribute_raw(data->node, name);
 
     /* Print attribute name and value */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("    + %-32s\t: %-10s", (char *)name, value);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(data->stream, "   <li>%s: %s",
                     (char *)name, value);
             break;
 
         case mon_output_xml:
             fprintf(data->stream,
                     "            <attribute name=\"%s\" value=\"%s\"",
                     (char *)name, value);
             break;
 
         default:
             break;
     }
 
     /* Print extended information if appropriate */
     print_attr_msg(data->stream, data->node, data->node->details->running_rsc,
                    name, value);
 
     /* Close out the attribute */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(data->stream, "</li>\n");
             break;
 
         case mon_output_xml:
             fprintf(data->stream, " />\n");
             break;
 
         default:
             break;
     }
 }
 
 static void
 print_node_summary(FILE *stream, pe_working_set_t * data_set, gboolean operations)
 {
     xmlNode *node_state = NULL;
     xmlNode *cib_status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);
 
     /* Print heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             if (operations) {
                 print_as("\nOperations:\n");
             } else {
                 print_as("\nMigration Summary:\n");
             }
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             if (operations) {
                 fprintf(stream, " <hr />\n <h2>Operations</h2>\n");
             } else {
                 fprintf(stream, " <hr />\n <h2>Migration Summary</h2>\n");
             }
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <node_history>\n");
             break;
 
         default:
             break;
     }
 
     /* Print each node in the CIB status */
     for (node_state = __xml_first_child(cib_status); node_state != NULL;
          node_state = __xml_next(node_state)) {
         if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
             print_node_history(stream, data_set, node_state, operations);
         }
     }
 
     /* Close section */
     switch (output_format) {
         case mon_output_xml:
             fprintf(stream, "    </node_history>\n");
             break;
 
         default:
             break;
     }
 }
 
 static void
 print_ticket(gpointer name, gpointer value, gpointer data)
 {
     ticket_t *ticket = (ticket_t *) value;
     FILE *stream = (FILE *) data;
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("* %s:\t%s%s", ticket->id,
                      (ticket->granted? "granted" : "revoked"),
                      (ticket->standby? " [standby]" : ""));
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "  <li>%s: %s%s", ticket->id,
                     (ticket->granted? "granted" : "revoked"),
                     (ticket->standby? " [standby]" : ""));
             break;
 
         case mon_output_xml:
             fprintf(stream, "        <ticket id=\"%s\" status=\"%s\" standby=\"%s\"",
                     ticket->id, (ticket->granted? "granted" : "revoked"),
                     (ticket->standby? "true" : "false"));
             break;
 
         default:
             break;
     }
     if (ticket->last_granted > -1) {
         print_nvpair(stdout, "last-granted", NULL, NULL, ticket->last_granted);
     }
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "</li>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, " />\n");
             break;
 
         default:
             break;
     }
 }
 
 static void
 print_cluster_tickets(FILE *stream, pe_working_set_t * data_set)
 {
     /* Print section heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\nTickets:\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <hr />\n <h2>Tickets</h2>\n <ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <tickets>\n");
             break;
 
         default:
             break;
     }
 
     /* Print each ticket */
     g_hash_table_foreach(data_set->tickets, print_ticket, stream);
 
     /* Close section */
     switch (output_format) {
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " </ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    </tickets>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Return human-friendly string representing node name
  *
  * The returned string will be in the format
  *    uname[@hostUname] [(nodeID)]
  * "@hostUname" will be printed if the node is a guest node.
  * "(nodeID)" will be printed if the node ID is different from the node uname,
  *  and detailed output has been requested.
  *
  * \param[in] node  Node to represent
  * \return Newly allocated string with representation of node name
  * \note It is the caller's responsibility to free the result with free().
  */
 static char *
 get_node_display_name(node_t *node)
 {
     char *node_name;
     const char *node_host = NULL;
     const char *node_id = NULL;
     int name_len;
 
     CRM_ASSERT((node != NULL) && (node->details != NULL) && (node->details->uname != NULL));
 
     /* Host is displayed only if this is a guest node */
     if (is_container_remote_node(node)) {
         if (node->details->remote_rsc->running_on) {
             /* running_on is a list, but guest nodes will have exactly one entry
              * unless they are in the process of migrating, in which case they
              * will have two; either way, we can use the first item in the list
              */
             node_t *host_node = (node_t *) node->details->remote_rsc->running_on->data;
 
             if (host_node && host_node->details) {
                 node_host = host_node->details->uname;
             }
         }
         if (node_host == NULL) {
             node_host = ""; /* so we at least get "uname@" to indicate guest */
         }
     }
 
     /* Node ID is displayed if different from uname and detail is requested */
     if (print_clone_detail && safe_str_neq(node->details->uname, node->details->id)) {
         node_id = node->details->id;
     }
 
     /* Determine name length */
     name_len = strlen(node->details->uname) + 1;
     if (node_host) {
         name_len += strlen(node_host) + 1; /* "@node_host" */
     }
     if (node_id) {
         name_len += strlen(node_id) + 3; /* + " (node_id)" */
     }
 
     /* Allocate and populate display name */
     node_name = malloc(name_len);
     CRM_ASSERT(node_name != NULL);
     strcpy(node_name, node->details->uname);
     if (node_host) {
         strcat(node_name, "@");
         strcat(node_name, node_host);
     }
     if (node_id) {
         strcat(node_name, " (");
         strcat(node_name, node_id);
         strcat(node_name, ")");
     }
     return node_name;
 }
 
 /*!
  * \internal
  * \brief Print a negative location constraint
  *
  * \param[in] stream     File stream to display output to
  * \param[in] node       Node affected by constraint
  * \param[in] location   Constraint to print
  */
 static void print_ban(FILE *stream, node_t *node, rsc_to_node_t *location)
 {
     char *node_name = NULL;
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             node_name = get_node_display_name(node);
             print_as(" %s\tprevents %s from running %son %s\n",
                      location->id, location->rsc_lh->id,
                      ((location->role_filter == RSC_ROLE_MASTER)? "as Master " : ""),
                      node_name);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             node_name = get_node_display_name(node);
             fprintf(stream, "  <li>%s prevents %s from running %son %s</li>\n",
                      location->id, location->rsc_lh->id,
                      ((location->role_filter == RSC_ROLE_MASTER)? "as Master " : ""),
                      node_name);
             break;
 
         case mon_output_xml:
             fprintf(stream,
                     "        <ban id=\"%s\" resource=\"%s\" node=\"%s\" weight=\"%d\" master_only=\"%s\" />\n",
                     location->id, location->rsc_lh->id, node->details->uname, node->weight,
                     ((location->role_filter == RSC_ROLE_MASTER)? "true" : "false"));
             break;
 
         default:
             break;
     }
     free(node_name);
 }
 
 /*!
  * \internal
  * \brief Print section for negative location constraints
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set corresponding to CIB status to display
  */
 static void print_neg_locations(FILE *stream, pe_working_set_t *data_set)
 {
     GListPtr gIter, gIter2;
 
     /* Print section heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\nNegative Location Constraints:\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <hr />\n <h2>Negative Location Constraints</h2>\n <ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <bans>\n");
             break;
 
         default:
             break;
     }
 
     /* Print each ban */
     for (gIter = data_set->placement_constraints; gIter != NULL; gIter = gIter->next) {
         rsc_to_node_t *location = (rsc_to_node_t *) gIter->data;
         if (!g_str_has_prefix(location->id, print_neg_location_prefix))
             continue;
         for (gIter2 = location->node_list_rh; gIter2 != NULL; gIter2 = gIter2->next) {
             node_t *node = (node_t *) gIter2->data;
 
             if (node->weight < 0) {
                 print_ban(stream, node, location);
             }
         }
     }
 
     /* Close section */
     switch (output_format) {
         case mon_output_cgi:
         case mon_output_html:
             fprintf(stream, " </ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    </bans>\n");
             break;
 
         default:
             break;
     }
 }
 
 static void
 crm_mon_get_parameters(resource_t *rsc, pe_working_set_t * data_set)
 {
     get_rsc_attributes(rsc->parameters, rsc, NULL, data_set);
     crm_trace("Beekhof: unpacked params for %s (%d)", rsc->id, g_hash_table_size(rsc->parameters));
     if(rsc->children) {
         GListPtr gIter = NULL;
 
         for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
             crm_mon_get_parameters(gIter->data, data_set);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Print node attributes section
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_node_attributes(FILE *stream, pe_working_set_t *data_set)
 {
     GListPtr gIter = NULL;
 
     /* Print section heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\nNode Attributes:\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <hr />\n <h2>Node Attributes</h2>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <node_attributes>\n");
             break;
 
         default:
             break;
     }
 
     /* Unpack all resource parameters (it would be more efficient to do this
      * only when needed for the first time in print_attr_msg())
      */
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         crm_mon_get_parameters(gIter->data, data_set);
     }
 
     /* Display each node's attributes */
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         struct mon_attr_data data;
 
         data.stream = stream;
         data.node = (node_t *) gIter->data;
 
         if (data.node && data.node->details && data.node->details->online) {
             print_node_start(stream, data.node);
             g_hash_table_foreach(data.node->details->attrs, create_attr_list, NULL);
             g_list_foreach(attr_list, print_node_attribute, &data);
             g_list_free(attr_list);
             attr_list = NULL;
             print_node_end(stream);
         }
     }
 
     /* Print section footer */
     switch (output_format) {
         case mon_output_xml:
             fprintf(stream, "    </node_attributes>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Return resource display options corresponding to command-line choices
  *
  * \return Bitmask of pe_print_options suitable for resource print functions
  */
 static int
 get_resource_display_options(void)
 {
     int print_opts;
 
     /* Determine basic output format */
     switch (output_format) {
         case mon_output_console:
             print_opts = pe_print_ncurses;
             break;
         case mon_output_html:
         case mon_output_cgi:
             print_opts = pe_print_html;
             break;
         case mon_output_xml:
             print_opts = pe_print_xml;
             break;
         default:
             print_opts = pe_print_printf;
             break;
     }
 
     /* Add optional display elements */
     if (print_pending) {
         print_opts |= pe_print_pending;
     }
     if (print_clone_detail) {
         print_opts |= pe_print_clone_details;
     }
     if (!inactive_resources) {
         print_opts |= pe_print_clone_active;
     }
     if (print_brief) {
         print_opts |= pe_print_brief;
     }
     return print_opts;
 }
 
 /*!
  * \internal
  * \brief Return human-friendly string representing current time
  *
  * \return Current time as string (as by ctime() but without newline) on success
  *         or "Could not determine current time" on error
  * \note The return value points to a statically allocated string which might be
  *       overwritten by subsequent calls to any of the C library date and time functions.
  */
 static const char *
 crm_now_string(void)
 {
     time_t a_time = time(NULL);
     char *since_epoch = ctime(&a_time);
 
     if ((a_time == (time_t) -1) || (since_epoch == NULL)) {
         return "Could not determine current time";
     }
     since_epoch[strlen(since_epoch) - 1] = EOS; /* trim newline */
     return (since_epoch);
 }
 
 /*!
  * \internal
  * \brief Print header for cluster summary if needed
  *
  * \param[in] stream     File stream to display output to
  */
 static void
 print_cluster_summary_header(FILE *stream)
 {
     switch (output_format) {
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <h2>Cluster Summary</h2>\n <p>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <summary>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print footer for cluster summary if needed
  *
  * \param[in] stream     File stream to display output to
  */
 static void
 print_cluster_summary_footer(FILE *stream)
 {
     switch (output_format) {
         case mon_output_cgi:
         case mon_output_html:
             fprintf(stream, " </p>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    </summary>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print times the display was last updated and CIB last changed
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_cluster_times(FILE *stream, pe_working_set_t *data_set)
 {
     const char *last_written = crm_element_value(data_set->input, XML_CIB_ATTR_WRITTEN);
     const char *user = crm_element_value(data_set->input, XML_ATTR_UPDATE_USER);
     const char *client = crm_element_value(data_set->input, XML_ATTR_UPDATE_CLIENT);
     const char *origin = crm_element_value(data_set->input, XML_ATTR_UPDATE_ORIG);
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("Last updated: %s", crm_now_string());
             print_as((user || client || origin)? "\n" : "\t\t");
             print_as("Last change: %s", last_written ? last_written : "");
             if (user) {
                 print_as(" by %s", user);
             }
             if (client) {
                 print_as(" via %s", client);
             }
             if (origin) {
                 print_as(" on %s", origin);
             }
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <b>Last updated:</b> %s<br/>\n", crm_now_string());
             fprintf(stream, " <b>Last change:</b> %s", last_written ? last_written : "");
             if (user) {
                 fprintf(stream, " by %s", user);
             }
             if (client) {
                 fprintf(stream, " via %s", client);
             }
             if (origin) {
                 fprintf(stream, " on %s", origin);
             }
             fprintf(stream, "<br/>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "        <last_update time=\"%s\" />\n", crm_now_string());
             fprintf(stream, "        <last_change time=\"%s\" user=\"%s\" client=\"%s\" origin=\"%s\" />\n",
                     last_written ? last_written : "", user ? user : "",
                     client ? client : "", origin ? origin : "");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print cluster stack
  *
  * \param[in] stream     File stream to display output to
  * \param[in] stack_s    Stack name
  */
 static void
 print_cluster_stack(FILE *stream, const char *stack_s)
 {
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("Stack: %s\n", stack_s);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <b>Stack:</b> %s<br/>\n", stack_s);
             break;
 
         case mon_output_xml:
             fprintf(stream, "        <stack type=\"%s\" />\n", stack_s);
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print current DC and its version
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_cluster_dc(FILE *stream, pe_working_set_t *data_set)
 {
     node_t *dc = data_set->dc_node;
     xmlNode *dc_version = get_xpath_object("//nvpair[@name='dc-version']",
                                            data_set->input, LOG_DEBUG);
     const char *dc_version_s = dc_version?
                                crm_element_value(dc_version, XML_NVPAIR_ATTR_VALUE)
                                : NULL;
     const char *quorum = crm_element_value(data_set->input, XML_ATTR_HAVE_QUORUM);
     char *dc_name = dc? get_node_display_name(dc) : NULL;
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("Current DC: ");
             if (dc) {
                 print_as("%s (version %s) - partition %s quorum\n",
                          dc_name, (dc_version_s? dc_version_s : "unknown"),
                          (crm_is_true(quorum) ? "with" : "WITHOUT"));
             } else {
                 print_as("NONE\n");
             }
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <b>Current DC:</b> ");
             if (dc) {
                 fprintf(stream, "%s (version %s) - partition %s quorum",
                         dc_name, (dc_version_s? dc_version_s : "unknown"),
                         (crm_is_true(quorum)? "with" : "<font color=\"red\"><b>WITHOUT</b></font>"));
             } else {
                 fprintf(stream, "<font color=\"red\"><b>NONE</b></font>");
             }
             fprintf(stream, "<br/>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream,  "        <current_dc ");
             if (dc) {
                 fprintf(stream,
                         "present=\"true\" version=\"%s\" name=\"%s\" id=\"%s\" with_quorum=\"%s\"",
                         (dc_version_s? dc_version_s : ""), dc->details->uname, dc->details->id,
                         (crm_is_true(quorum) ? "true" : "false"));
             } else {
                 fprintf(stream, "present=\"false\"");
             }
             fprintf(stream, " />\n");
             break;
 
         default:
             break;
     }
     free(dc_name);
 }
 
 /*!
  * \internal
  * \brief Print counts of configured nodes and resources
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  * \param[in] stack_s    Stack name
  */
 static void
 print_cluster_counts(FILE *stream, pe_working_set_t *data_set, const char *stack_s)
 {
     int nnodes = g_list_length(data_set->nodes);
     int nresources = count_resources(data_set, NULL);
     xmlNode *quorum_node = get_xpath_object("//nvpair[@name='" XML_ATTR_EXPECTED_VOTES "']",
                                             data_set->input, LOG_DEBUG);
     const char *quorum_votes = quorum_node?
                                crm_element_value(quorum_node, XML_NVPAIR_ATTR_VALUE)
                                : "unknown";
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
 
             print_as("\n%d node%s configured", nnodes, s_if_plural(nnodes));
             if (stack_s && strstr(stack_s, "classic openais") != NULL) {
                 print_as(" (%s expected votes)", quorum_votes);
             }
             print_as("\n");
 
             print_as("%d resource%s configured",
                      nresources, s_if_plural(nresources));
             if(data_set->disabled_resources || data_set->blocked_resources) {
                 print_as(" (");
                 if (data_set->disabled_resources) {
                     print_as("%d DISABLED", data_set->disabled_resources);
                 }
                 if (data_set->disabled_resources && data_set->blocked_resources) {
                     print_as(", ");
                 }
                 if (data_set->blocked_resources) {
                     print_as("%d BLOCKED from starting due to failure",
                              data_set->blocked_resources);
                 }
                 print_as(")");
             }
             print_as("\n");
 
             break;
 
         case mon_output_html:
         case mon_output_cgi:
 
             fprintf(stream, " %d node%s configured", nnodes, s_if_plural(nnodes));
             if (stack_s && strstr(stack_s, "classic openais") != NULL) {
                 fprintf(stream, " (%s expected votes)", quorum_votes);
             }
             fprintf(stream, "<br/>\n");
 
             fprintf(stream, " %d resource%s configured",
                     nresources, s_if_plural(nresources));
             if (data_set->disabled_resources || data_set->blocked_resources) {
                 fprintf(stream, " (");
                 if (data_set->disabled_resources) {
                     fprintf(stream, "%d <strong>DISABLED</strong>",
                             data_set->disabled_resources);
                 }
                 if (data_set->disabled_resources && data_set->blocked_resources) {
                     fprintf(stream, ", ");
                 }
                 if (data_set->blocked_resources) {
                     fprintf(stream,
                             "%d <strong>BLOCKED</strong> from starting due to failure",
                             data_set->blocked_resources);
                 }
                 fprintf(stream, ")");
             }
             fprintf(stream, "<br/>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream,
                     "        <nodes_configured number=\"%d\" expected_votes=\"%s\" />\n",
                     g_list_length(data_set->nodes), quorum_votes);
             fprintf(stream,
                     "        <resources_configured number=\"%d\" disabled=\"%d\" blocked=\"%d\" />\n",
                     count_resources(data_set, NULL),
                     data_set->disabled_resources, data_set->blocked_resources);
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print cluster-wide options
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  *
  * \note Currently this is only implemented for HTML and XML output, and
  *       prints only a few options. If there is demand, more could be added.
  */
 static void
 print_cluster_options(FILE *stream, pe_working_set_t *data_set)
 {
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
                 print_as("\n              *** Resource management is DISABLED ***");
                 print_as("\n  The cluster will not attempt to start, stop or recover services");
                 print_as("\n");
             }
             break;
 
         case mon_output_html:
             fprintf(stream, " </p>\n <h3>Config Options</h3>\n");
             fprintf(stream, " <table>\n");
             fprintf(stream, "  <tr><th>STONITH of failed nodes</th><td>%s</td></tr>\n",
                     is_set(data_set->flags, pe_flag_stonith_enabled)? "enabled" : "disabled");
 
             fprintf(stream, "  <tr><th>Cluster is</th><td>%ssymmetric</td></tr>\n",
                     is_set(data_set->flags, pe_flag_symmetric_cluster)? "" : "a");
 
             fprintf(stream, "  <tr><th>No Quorum Policy</th><td>");
             switch (data_set->no_quorum_policy) {
                 case no_quorum_freeze:
                     fprintf(stream, "Freeze resources");
                     break;
                 case no_quorum_stop:
                     fprintf(stream, "Stop ALL resources");
                     break;
                 case no_quorum_ignore:
                     fprintf(stream, "Ignore");
                     break;
                 case no_quorum_suicide:
                     fprintf(stream, "Suicide");
                     break;
             }
             fprintf(stream, "</td></tr>\n");
 
             fprintf(stream, "  <tr><th>Resource management</th><td>");
             if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
                 fprintf(stream, "<strong>DISABLED</strong> (the cluster will "
                                 "not attempt to start, stop or recover services)");
             } else {
                 fprintf(stream, "enabled");
             }
             fprintf(stream, "</td></tr>\n");
 
             fprintf(stream, "</table>\n <p>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "        <cluster_options");
             fprintf(stream, " stonith-enabled=\"%s\"",
                     is_set(data_set->flags, pe_flag_stonith_enabled)?
                     "true" : "false");
             fprintf(stream, " symmetric-cluster=\"%s\"",
                     is_set(data_set->flags, pe_flag_symmetric_cluster)?
                     "true" : "false");
             fprintf(stream, " no-quorum-policy=\"");
             switch (data_set->no_quorum_policy) {
                 case no_quorum_freeze:
                     fprintf(stream, "freeze");
                     break;
                 case no_quorum_stop:
                     fprintf(stream, "stop");
                     break;
                 case no_quorum_ignore:
                     fprintf(stream, "ignore");
                     break;
                 case no_quorum_suicide:
                     fprintf(stream, "suicide");
                     break;
             }
             fprintf(stream, "\"");
             fprintf(stream, " maintenance-mode=\"%s\"",
                     is_set(data_set->flags, pe_flag_maintenance_mode)?
                     "true" : "false");
             fprintf(stream, " />\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Get the name of the stack in use (or "unknown" if not available)
  *
  * \param[in] data_set   Working set of CIB state
  *
  * \return String representing stack name
  */
 static const char *
 get_cluster_stack(pe_working_set_t *data_set)
 {
     xmlNode *stack = get_xpath_object("//nvpair[@name='cluster-infrastructure']",
                                       data_set->input, LOG_DEBUG);
     return stack? crm_element_value(stack, XML_NVPAIR_ATTR_VALUE) : "unknown";
 }
 
 /*!
  * \internal
  * \brief Print a summary of cluster-wide information
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_cluster_summary(FILE *stream, pe_working_set_t *data_set)
 {
     const char *stack_s = get_cluster_stack(data_set);
     gboolean header_printed = FALSE;
 
     if (show & mon_show_stack) {
         if (header_printed == FALSE) {
             print_cluster_summary_header(stream);
             header_printed = TRUE;
         }
         print_cluster_stack(stream, stack_s);
     }
 
     /* Always print DC if none, even if not requested */
     if ((data_set->dc_node == NULL) || (show & mon_show_dc)) {
         if (header_printed == FALSE) {
             print_cluster_summary_header(stream);
             header_printed = TRUE;
         }
         print_cluster_dc(stream, data_set);
     }
 
     if (show & mon_show_times) {
         if (header_printed == FALSE) {
             print_cluster_summary_header(stream);
             header_printed = TRUE;
         }
         print_cluster_times(stream, data_set);
     }
 
     if (is_set(data_set->flags, pe_flag_maintenance_mode)
         || data_set->disabled_resources
         || data_set->blocked_resources
         || is_set(show, mon_show_count)) {
         if (header_printed == FALSE) {
             print_cluster_summary_header(stream);
             header_printed = TRUE;
         }
         print_cluster_counts(stream, data_set, stack_s);
     }
 
     /* There is not a separate option for showing cluster options, so show with
      * stack for now; a separate option could be added if there is demand
      */
     if (show & mon_show_stack) {
         print_cluster_options(stream, data_set);
     }
 
     if (header_printed) {
         print_cluster_summary_footer(stream);
     }
 }
 
 /*!
  * \internal
  * \brief Print a failed action
  *
  * \param[in] stream     File stream to display output to
  * \param[in] xml_op     Root of XML tree describing failed action
  */
 static void
 print_failed_action(FILE *stream, xmlNode *xml_op)
 {
     const char *op_key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
     const char *op_key_attr = "op_key";
     const char *last = crm_element_value(xml_op, XML_RSC_OP_LAST_CHANGE);
     const char *node = crm_element_value(xml_op, XML_ATTR_UNAME);
     const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
     const char *exit_reason = crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON);
     int rc = crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_RC), "0");
     int status = crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS), "0");
     char *exit_reason_cleaned;
 
     /* If no op_key was given, use id instead */
     if (op_key == NULL) {
         op_key = ID(xml_op);
         op_key_attr = "id";
     }
 
     /* If no exit reason was given, use "none" */
     if (exit_reason == NULL) {
         exit_reason = "none";
     }
 
     /* Print common action information */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("* %s on %s '%s' (%d): call=%s, status=%s, exitreason='%s'",
                      op_key, node, services_ocf_exitcode_str(rc), rc,
                      call, services_lrm_status_str(status), exit_reason);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "  <li>%s on %s '%s' (%d): call=%s, status=%s, exitreason='%s'",
                      op_key, node, services_ocf_exitcode_str(rc), rc,
                      call, services_lrm_status_str(status), exit_reason);
             break;
 
         case mon_output_xml:
             exit_reason_cleaned = crm_xml_escape(exit_reason);
             fprintf(stream, "        <failure %s=\"%s\" node=\"%s\"",
                     op_key_attr, op_key, node);
             fprintf(stream, " exitstatus=\"%s\" exitreason=\"%s\" exitcode=\"%d\"",
                     services_ocf_exitcode_str(rc), exit_reason_cleaned, rc);
             fprintf(stream, " call=\"%s\" status=\"%s\"",
                     call, services_lrm_status_str(status));
             free(exit_reason_cleaned);
             break;
 
         default:
             break;
     }
 
     /* If last change was given, print timing information as well */
     if (last) {
         time_t run_at = crm_parse_int(last, "0");
         char *run_at_s = ctime(&run_at);
 
         if (run_at_s) {
             run_at_s[24] = 0; /* Overwrite the newline */
         }
 
         switch (output_format) {
             case mon_output_plain:
             case mon_output_console:
                 print_as(",\n    last-rc-change='%s', queued=%sms, exec=%sms",
                          run_at_s? run_at_s : "",
                          crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
                          crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
                 break;
 
             case mon_output_html:
             case mon_output_cgi:
                 fprintf(stream, " last-rc-change='%s', queued=%sms, exec=%sms",
                         run_at_s? run_at_s : "",
                         crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
                         crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
                 break;
 
             case mon_output_xml:
                 fprintf(stream,
                         " last-rc-change=\"%s\" queued=\"%s\" exec=\"%s\" interval=\"%d\" task=\"%s\"",
                         run_at_s? run_at_s : "",
                         crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
                         crm_element_value(xml_op, XML_RSC_OP_T_EXEC),
                         crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL), "0"),
                         crm_element_value(xml_op, XML_LRM_ATTR_TASK));
                 break;
 
             default:
                 break;
         }
     }
 
     /* End the action listing */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "</li>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, " />\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print a section for failed actions
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_failed_actions(FILE *stream, pe_working_set_t *data_set)
 {
     xmlNode *xml_op = NULL;
 
     /* Print section heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\nFailed Actions:\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <hr />\n <h2>Failed Actions</h2>\n <ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <failures>\n");
             break;
 
         default:
             break;
     }
 
     /* Print each failed action */
     for (xml_op = __xml_first_child(data_set->failed); xml_op != NULL;
          xml_op = __xml_next(xml_op)) {
         print_failed_action(stream, xml_op);
     }
 
     /* End section */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " </ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    </failures>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print cluster status to screen
  *
  * This uses the global display preferences set by command-line options
  * to display cluster status in a human-friendly way.
  *
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_status(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     int print_opts = get_resource_display_options();
 
     /* space-separated lists of node names */
     char *online_nodes = NULL;
     char *online_remote_nodes = NULL;
     char *online_guest_nodes = NULL;
     char *offline_nodes = NULL;
     char *offline_remote_nodes = NULL;
 
     if (output_format == mon_output_console) {
         blank_screen();
     }
     print_cluster_summary(stdout, data_set);
     print_as("\n");
 
     /* Gather node information (and print if in bad state or grouping by node) */
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
         const char *node_mode = NULL;
         char *node_name = get_node_display_name(node);
 
         /* Get node mode */
         if (node->details->unclean) {
             if (node->details->online) {
                 node_mode = "UNCLEAN (online)";
 
             } else if (node->details->pending) {
                 node_mode = "UNCLEAN (pending)";
 
             } else {
                 node_mode = "UNCLEAN (offline)";
             }
 
         } else if (node->details->pending) {
             node_mode = "pending";
 
         } else if (node->details->standby_onfail && node->details->online) {
             node_mode = "standby (on-fail)";
 
         } else if (node->details->standby) {
             if (node->details->online) {
                 node_mode = "standby";
             } else {
                 node_mode = "OFFLINE (standby)";
             }
 
         } else if (node->details->maintenance) {
             if (node->details->online) {
                 node_mode = "maintenance";
             } else {
                 node_mode = "OFFLINE (maintenance)";
             }
 
         } else if (node->details->online) {
             node_mode = "online";
             if (group_by_node == FALSE) {
                 if (is_container_remote_node(node)) {
                     online_guest_nodes = add_list_element(online_guest_nodes, node_name);
                 } else if (is_baremetal_remote_node(node)) {
                     online_remote_nodes = add_list_element(online_remote_nodes, node_name);
                 } else {
                     online_nodes = add_list_element(online_nodes, node_name);
                 }
                 free(node_name);
                 continue;
             }
         } else {
             node_mode = "OFFLINE";
             if (group_by_node == FALSE) {
                 if (is_baremetal_remote_node(node)) {
                     offline_remote_nodes = add_list_element(offline_remote_nodes, node_name);
                 } else if (is_container_remote_node(node)) {
                     /* ignore offline guest nodes */
                 } else {
                     offline_nodes = add_list_element(offline_nodes, node_name);
                 }
                 free(node_name);
                 continue;
             }
         }
 
         /* If we get here, node is in bad state, or we're grouping by node */
 
         /* Print the node name and status */
         if (is_container_remote_node(node)) {
             print_as("Guest");
         } else if (is_baremetal_remote_node(node)) {
             print_as("Remote");
         }
         print_as("Node %s: %s\n", node_name, node_mode);
 
         /* If we're grouping by node, print its resources */
         if (group_by_node) {
             if (print_brief) {
                 print_rscs_brief(node->details->running_rsc, "\t", print_opts | pe_print_rsconly,
                                  stdout, FALSE);
             } else {
                 GListPtr gIter2 = NULL;
 
                 for (gIter2 = node->details->running_rsc; gIter2 != NULL; gIter2 = gIter2->next) {
                     resource_t *rsc = (resource_t *) gIter2->data;
 
                     rsc->fns->print(rsc, "\t", print_opts | pe_print_rsconly, stdout);
                 }
             }
         }
         free(node_name);
     }
 
     /* If we're not grouping by node, summarize nodes by status */
     if (online_nodes) {
         print_as("Online: [%s ]\n", online_nodes);
         free(online_nodes);
     }
     if (offline_nodes) {
         print_as("OFFLINE: [%s ]\n", offline_nodes);
         free(offline_nodes);
     }
     if (online_remote_nodes) {
         print_as("RemoteOnline: [%s ]\n", online_remote_nodes);
         free(online_remote_nodes);
     }
     if (offline_remote_nodes) {
         print_as("RemoteOFFLINE: [%s ]\n", offline_remote_nodes);
         free(offline_remote_nodes);
     }
     if (online_guest_nodes) {
         print_as("GuestOnline: [%s ]\n", online_guest_nodes);
         free(online_guest_nodes);
     }
 
     /* Print resources section, if needed */
     print_resources(stdout, data_set, print_opts);
 
     /* print Node Attributes section if requested */
     if (show & mon_show_attributes) {
         print_node_attributes(stdout, data_set);
     }
 
     /* If requested, print resource operations (which includes failcounts)
      * or just failcounts
      */
     if (show & (mon_show_operations | mon_show_failcounts)) {
         print_node_summary(stdout, data_set,
                            ((show & mon_show_operations)? TRUE : FALSE));
     }
 
     /* If there were any failed actions, print them */
     if (xml_has_children(data_set->failed)) {
         print_failed_actions(stdout, data_set);
     }
 
     /* Print tickets if requested */
     if (show & mon_show_tickets) {
         print_cluster_tickets(stdout, data_set);
     }
 
     /* Print negative location constraints if requested */
     if (show & mon_show_bans) {
         print_neg_locations(stdout, data_set);
     }
 
 #if CURSES_ENABLED
     if (output_format == mon_output_console) {
         refresh();
     }
 #endif
 }
 
 /*!
  * \internal
  * \brief Print cluster status in XML format
  *
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_xml_status(pe_working_set_t * data_set)
 {
     FILE *stream = stdout;
     GListPtr gIter = NULL;
     int print_opts = get_resource_display_options();
 
     fprintf(stream, "<?xml version=\"1.0\"?>\n");
     fprintf(stream, "<crm_mon version=\"%s\">\n", VERSION);
 
     print_cluster_summary(stream, data_set);
 
     /*** NODES ***/
     fprintf(stream, "    <nodes>\n");
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
         const char *node_type = "unknown";
 
         switch (node->details->type) {
             case node_member:
                 node_type = "member";
                 break;
             case node_remote:
                 node_type = "remote";
                 break;
             case node_ping:
                 node_type = "ping";
                 break;
         }
 
         fprintf(stream, "        <node name=\"%s\" ", node->details->uname);
         fprintf(stream, "id=\"%s\" ", node->details->id);
         fprintf(stream, "online=\"%s\" ", node->details->online ? "true" : "false");
         fprintf(stream, "standby=\"%s\" ", node->details->standby ? "true" : "false");
         fprintf(stream, "standby_onfail=\"%s\" ", node->details->standby_onfail ? "true" : "false");
         fprintf(stream, "maintenance=\"%s\" ", node->details->maintenance ? "true" : "false");
         fprintf(stream, "pending=\"%s\" ", node->details->pending ? "true" : "false");
         fprintf(stream, "unclean=\"%s\" ", node->details->unclean ? "true" : "false");
         fprintf(stream, "shutdown=\"%s\" ", node->details->shutdown ? "true" : "false");
         fprintf(stream, "expected_up=\"%s\" ", node->details->expected_up ? "true" : "false");
         fprintf(stream, "is_dc=\"%s\" ", node->details->is_dc ? "true" : "false");
         fprintf(stream, "resources_running=\"%d\" ", g_list_length(node->details->running_rsc));
         fprintf(stream, "type=\"%s\" ", node_type);
         if (is_container_remote_node(node)) {
             fprintf(stream, "id_as_resource=\"%s\" ", node->details->remote_rsc->container->id);
         }
 
         if (group_by_node) {
             GListPtr lpc2 = NULL;
 
             fprintf(stream, ">\n");
             for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
                 resource_t *rsc = (resource_t *) lpc2->data;
 
                 rsc->fns->print(rsc, "            ", print_opts | pe_print_rsconly, stream);
             }
             fprintf(stream, "        </node>\n");
         } else {
             fprintf(stream, "/>\n");
         }
     }
     fprintf(stream, "    </nodes>\n");
 
     /* Print resources section, if needed */
     print_resources(stream, data_set, print_opts);
 
     /* print Node Attributes section if requested */
     if (show & mon_show_attributes) {
         print_node_attributes(stream, data_set);
     }
 
     /* If requested, print resource operations (which includes failcounts)
      * or just failcounts
      */
     if (show & (mon_show_operations | mon_show_failcounts)) {
         print_node_summary(stream, data_set,
                            ((show & mon_show_operations)? TRUE : FALSE));
     }
 
     /* If there were any failed actions, print them */
     if (xml_has_children(data_set->failed)) {
         print_failed_actions(stream, data_set);
     }
 
     /* Print tickets if requested */
     if (show & mon_show_tickets) {
         print_cluster_tickets(stream, data_set);
     }
 
     /* Print negative location constraints if requested */
     if (show & mon_show_bans) {
         print_neg_locations(stream, data_set);
     }
 
     fprintf(stream, "</crm_mon>\n");
     fflush(stream);
     fclose(stream);
 }
 
 /*!
  * \internal
  * \brief Print cluster status in HTML format (with HTTP headers if CGI)
  *
  * \param[in] data_set   Working set of CIB state
  * \param[in] filename   Name of file to write HTML to (ignored if CGI)
  *
  * \return 0 on success, -1 on error
  */
 static int
 print_html_status(pe_working_set_t * data_set, const char *filename)
 {
     FILE *stream;
     GListPtr gIter = NULL;
     char *filename_tmp = NULL;
     int print_opts = get_resource_display_options();
 
     if (output_format == mon_output_cgi) {
         stream = stdout;
         fprintf(stream, "Content-Type: text/html\n\n");
 
     } else {
         filename_tmp = crm_concat(filename, "tmp", '.');
         stream = fopen(filename_tmp, "w");
         if (stream == NULL) {
             crm_perror(LOG_ERR, "Cannot open %s for writing", filename_tmp);
             free(filename_tmp);
             return -1;
         }
     }
 
     fprintf(stream, "<html>\n");
     fprintf(stream, " <head>\n");
     fprintf(stream, "  <title>Cluster status</title>\n");
     fprintf(stream, "  <meta http-equiv=\"refresh\" content=\"%d\">\n", reconnect_msec / 1000);
     fprintf(stream, " </head>\n");
     fprintf(stream, "<body>\n");
 
     print_cluster_summary(stream, data_set);
 
     /*** NODE LIST ***/
 
     fprintf(stream, " <hr />\n <h2>Node List</h2>\n");
     fprintf(stream, "<ul>\n");
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
         char *node_name = get_node_display_name(node);
 
         fprintf(stream, "<li>Node: %s: ", node_name);
         if (node->details->standby_onfail && node->details->online) {
             fprintf(stream, "<font color=\"orange\">standby (on-fail)</font>\n");
         } else if (node->details->standby && node->details->online) {
             fprintf(stream, "<font color=\"orange\">standby</font>\n");
         } else if (node->details->standby) {
             fprintf(stream, "<font color=\"red\">OFFLINE (standby)</font>\n");
         } else if (node->details->maintenance && node->details->online) {
             fprintf(stream, "<font color=\"blue\">maintenance</font>\n");
         } else if (node->details->maintenance) {
             fprintf(stream, "<font color=\"red\">OFFLINE (maintenance)</font>\n");
         } else if (node->details->online) {
             fprintf(stream, "<font color=\"green\">online</font>\n");
         } else {
             fprintf(stream, "<font color=\"red\">OFFLINE</font>\n");
         }
         if (print_brief && group_by_node) {
             fprintf(stream, "<ul>\n");
             print_rscs_brief(node->details->running_rsc, NULL, print_opts | pe_print_rsconly,
                              stream, FALSE);
             fprintf(stream, "</ul>\n");
 
         } else if (group_by_node) {
             GListPtr lpc2 = NULL;
 
             fprintf(stream, "<ul>\n");
             for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
                 resource_t *rsc = (resource_t *) lpc2->data;
 
                 fprintf(stream, "<li>");
                 rsc->fns->print(rsc, NULL, print_opts | pe_print_rsconly, stream);
                 fprintf(stream, "</li>\n");
             }
             fprintf(stream, "</ul>\n");
         }
         fprintf(stream, "</li>\n");
         free(node_name);
     }
     fprintf(stream, "</ul>\n");
 
     /* Print resources section, if needed */
     print_resources(stream, data_set, print_opts);
 
     /* print Node Attributes section if requested */
     if (show & mon_show_attributes) {
         print_node_attributes(stream, data_set);
     }
 
     /* If requested, print resource operations (which includes failcounts)
      * or just failcounts
      */
     if (show & (mon_show_operations | mon_show_failcounts)) {
         print_node_summary(stream, data_set,
                            ((show & mon_show_operations)? TRUE : FALSE));
     }
 
     /* If there were any failed actions, print them */
     if (xml_has_children(data_set->failed)) {
         print_failed_actions(stream, data_set);
     }
 
     /* Print tickets if requested */
     if (show & mon_show_tickets) {
         print_cluster_tickets(stream, data_set);
     }
 
     /* Print negative location constraints if requested */
     if (show & mon_show_bans) {
         print_neg_locations(stream, data_set);
     }
 
     fprintf(stream, "</body>\n");
     fprintf(stream, "</html>\n");
     fflush(stream);
     fclose(stream);
 
     if (output_format != mon_output_cgi) {
         if (rename(filename_tmp, filename) != 0) {
             crm_perror(LOG_ERR, "Unable to rename %s->%s", filename_tmp, filename);
         }
         free(filename_tmp);
     }
     return 0;
 }
 
 #if ENABLE_SNMP
 #  include <net-snmp/net-snmp-config.h>
 #  include <net-snmp/snmpv3_api.h>
 #  include <net-snmp/agent/agent_trap.h>
 #  include <net-snmp/library/snmp_client.h>
 #  include <net-snmp/library/mib.h>
 #  include <net-snmp/library/snmp_debug.h>
 
 #  define add_snmp_field(list, oid_string, value) do {			\
 	oid name[MAX_OID_LEN];						\
         size_t name_length = MAX_OID_LEN;				\
 	if (snmp_parse_oid(oid_string, name, &name_length)) {		\
 	    int s_rc = snmp_add_var(list, name, name_length, 's', (value)); \
 	    if(s_rc != 0) {						\
 		crm_err("Could not add %s=%s rc=%d", oid_string, value, s_rc); \
 	    } else {							\
 		crm_trace("Added %s=%s", oid_string, value);		\
 	    }								\
 	} else {							\
 	    crm_err("Could not parse OID: %s", oid_string);		\
 	}								\
     } while(0)								\
 
 #  define add_snmp_field_int(list, oid_string, value) do {		\
 	oid name[MAX_OID_LEN];						\
         size_t name_length = MAX_OID_LEN;				\
 	if (snmp_parse_oid(oid_string, name, &name_length)) {		\
 	    if(NULL == snmp_pdu_add_variable(				\
 		   list, name, name_length, ASN_INTEGER,		\
 		   (u_char *) & value, sizeof(value))) {		\
 		crm_err("Could not add %s=%d", oid_string, value);	\
 	    } else {							\
 		crm_trace("Added %s=%d", oid_string, value);		\
 	    }								\
 	} else {							\
 	    crm_err("Could not parse OID: %s", oid_string);		\
 	}								\
     } while(0)								\
 
 static int
 snmp_input(int operation, netsnmp_session * session, int reqid, netsnmp_pdu * pdu, void *magic)
 {
     return 1;
 }
 
 static netsnmp_session *
 crm_snmp_init(const char *target, char *community)
 {
     static netsnmp_session *session = NULL;
 
 #  ifdef NETSNMPV53
     char target53[128];
 
     snprintf(target53, sizeof(target53), "%s:162", target);
 #  endif
 
     if (session) {
         return session;
     }
 
     if (target == NULL) {
         return NULL;
     }
 
     if (get_crm_log_level() > LOG_INFO) {
         char *debug_tokens = strdup("run:shell,snmptrap,tdomain");
 
         debug_register_tokens(debug_tokens);
         snmp_set_do_debugging(1);
     }
 
     session = calloc(1, sizeof(netsnmp_session));
     snmp_sess_init(session);
     session->version = SNMP_VERSION_2c;
     session->callback = snmp_input;
     session->callback_magic = NULL;
 
     if (community) {
         session->community_len = strlen(community);
         session->community = (unsigned char *)community;
     }
 
     session = snmp_add(session,
 #  ifdef NETSNMPV53
                        netsnmp_tdomain_transport(target53, 0, "udp"),
 #  else
                        netsnmp_transport_open_client("snmptrap", target),
 #  endif
                        NULL, NULL);
 
     if (session == NULL) {
         snmp_sess_perror("Could not create snmp transport", session);
     }
     return session;
 }
 
 #endif
 
 static int
 send_snmp_trap(const char *node, const char *rsc, const char *task, int target_rc, int rc,
                int status, const char *desc)
 {
     int ret = 1;
 
 #if ENABLE_SNMP
     static oid snmptrap_oid[] = { 1, 3, 6, 1, 6, 3, 1, 1, 4, 1, 0 };
     static oid sysuptime_oid[] = { 1, 3, 6, 1, 2, 1, 1, 3, 0 };
 
     netsnmp_pdu *trap_pdu;
     netsnmp_session *session = crm_snmp_init(snmp_target, snmp_community);
 
     trap_pdu = snmp_pdu_create(SNMP_MSG_TRAP2);
     if (!trap_pdu) {
         crm_err("Failed to create SNMP notification");
         return SNMPERR_GENERR;
     }
 
     if (1) {
         /* send uptime */
         char csysuptime[20];
         time_t now = time(NULL);
 
         sprintf(csysuptime, "%lld", (long long) now);
         snmp_add_var(trap_pdu, sysuptime_oid, sizeof(sysuptime_oid) / sizeof(oid), 't', csysuptime);
     }
 
     /* Indicate what the trap is by setting snmpTrapOid.0 */
     ret =
         snmp_add_var(trap_pdu, snmptrap_oid, sizeof(snmptrap_oid) / sizeof(oid), 'o',
                      snmp_crm_trap_oid);
     if (ret != 0) {
         crm_err("Failed set snmpTrapOid.0=%s", snmp_crm_trap_oid);
         return ret;
     }
 
     /* Add extries to the trap */
     if (rsc) {
         add_snmp_field(trap_pdu, snmp_crm_oid_rsc, rsc);
     }
     add_snmp_field(trap_pdu, snmp_crm_oid_node, node);
     add_snmp_field(trap_pdu, snmp_crm_oid_task, task);
     add_snmp_field(trap_pdu, snmp_crm_oid_desc, desc);
 
     add_snmp_field_int(trap_pdu, snmp_crm_oid_rc, rc);
     add_snmp_field_int(trap_pdu, snmp_crm_oid_trc, target_rc);
     add_snmp_field_int(trap_pdu, snmp_crm_oid_status, status);
 
     /* Send and cleanup */
     ret = snmp_send(session, trap_pdu);
     if (ret == 0) {
         /* error */
         snmp_sess_perror("Could not send SNMP trap", session);
         snmp_free_pdu(trap_pdu);
         ret = SNMPERR_GENERR;
     } else {
         ret = SNMPERR_SUCCESS;
     }
 #else
     crm_err("Sending SNMP traps is not supported by this installation");
 #endif
     return ret;
 }
 
 #if ENABLE_ESMTP
 #  include <auth-client.h>
 #  include <libesmtp.h>
 
 static void
 print_recipient_status(smtp_recipient_t recipient, const char *mailbox, void *arg)
 {
     const smtp_status_t *status;
 
     status = smtp_recipient_status(recipient);
     printf("%s: %d %s", mailbox, status->code, status->text);
 }
 
 static void
 event_cb(smtp_session_t session, int event_no, void *arg, ...)
 {
     int *ok;
     va_list alist;
 
     va_start(alist, arg);
     switch (event_no) {
         case SMTP_EV_CONNECT:
         case SMTP_EV_MAILSTATUS:
         case SMTP_EV_RCPTSTATUS:
         case SMTP_EV_MESSAGEDATA:
         case SMTP_EV_MESSAGESENT:
         case SMTP_EV_DISCONNECT:
             break;
 
         case SMTP_EV_WEAK_CIPHER:{
                 int bits = va_arg(alist, long);
                 ok = va_arg(alist, int *);
 
                 crm_debug("SMTP_EV_WEAK_CIPHER, bits=%d - accepted.", bits);
                 *ok = 1;
                 break;
             }
         case SMTP_EV_STARTTLS_OK:
             crm_debug("SMTP_EV_STARTTLS_OK - TLS started here.");
             break;
 
         case SMTP_EV_INVALID_PEER_CERTIFICATE:{
                 long vfy_result = va_arg(alist, long);
                 ok = va_arg(alist, int *);
 
                 /* There is a table in handle_invalid_peer_certificate() of mail-file.c */
                 crm_err("SMTP_EV_INVALID_PEER_CERTIFICATE: %ld", vfy_result);
                 *ok = 1;
                 break;
             }
         case SMTP_EV_NO_PEER_CERTIFICATE:
             ok = va_arg(alist, int *);
 
             crm_debug("SMTP_EV_NO_PEER_CERTIFICATE - accepted.");
             *ok = 1;
             break;
         case SMTP_EV_WRONG_PEER_CERTIFICATE:
             ok = va_arg(alist, int *);
 
             crm_debug("SMTP_EV_WRONG_PEER_CERTIFICATE - accepted.");
             *ok = 1;
             break;
         case SMTP_EV_NO_CLIENT_CERTIFICATE:
             ok = va_arg(alist, int *);
 
             crm_debug("SMTP_EV_NO_CLIENT_CERTIFICATE - accepted.");
             *ok = 1;
             break;
         default:
             crm_debug("Got event: %d - ignored.", event_no);
     }
     va_end(alist);
 }
 #endif
 
 #define BODY_MAX 2048
 
 #if ENABLE_ESMTP
 static void
 crm_smtp_debug(const char *buf, int buflen, int writing, void *arg)
 {
     char type = 0;
     int lpc = 0, last = 0, level = *(int *)arg;
 
     if (writing == SMTP_CB_HEADERS) {
         type = 'H';
     } else if (writing) {
         type = 'C';
     } else {
         type = 'S';
     }
 
     for (; lpc < buflen; lpc++) {
         switch (buf[lpc]) {
             case 0:
             case '\n':
                 if (last > 0) {
                     do_crm_log(level, "   %.*s", lpc - last, buf + last);
                 } else {
                     do_crm_log(level, "%c: %.*s", type, lpc - last, buf + last);
                 }
                 last = lpc + 1;
                 break;
         }
     }
 }
 #endif
 
 static int
 send_custom_trap(const char *node, const char *rsc, const char *task, int target_rc, int rc,
                  int status, const char *desc)
 {
     pid_t pid;
 
     /*setenv needs chars, these are ints */
     char *rc_s = crm_itoa(rc);
     char *status_s = crm_itoa(status);
     char *target_rc_s = crm_itoa(target_rc);
 
     crm_debug("Sending external notification to '%s' via '%s'", external_recipient, external_agent);
 
     if(rsc) {
         setenv("CRM_notify_rsc", rsc, 1);
     }
     if (external_recipient) {
         setenv("CRM_notify_recipient", external_recipient, 1);
     }
     setenv("CRM_notify_node", node, 1);
     setenv("CRM_notify_task", task, 1);
     setenv("CRM_notify_desc", desc, 1);
     setenv("CRM_notify_rc", rc_s, 1);
     setenv("CRM_notify_target_rc", target_rc_s, 1);
     setenv("CRM_notify_status", status_s, 1);
 
     pid = fork();
     if (pid == -1) {
         crm_perror(LOG_ERR, "notification fork() failed.");
     }
     if (pid == 0) {
         /* crm_debug("notification: I am the child. Executing the nofitication program."); */
         execl(external_agent, external_agent, NULL);
         exit(EXIT_FAILURE);
     }
 
     crm_trace("Finished running custom notification program '%s'.", external_agent);
     free(target_rc_s);
     free(status_s);
     free(rc_s);
     return 0;
 }
 
 static int
 send_smtp_trap(const char *node, const char *rsc, const char *task, int target_rc, int rc,
                int status, const char *desc)
 {
 #if ENABLE_ESMTP
     smtp_session_t session;
     smtp_message_t message;
     auth_context_t authctx;
     struct sigaction sa;
 
     int len = 25; /* Note: Check extra padding on the Subject line below */
     int noauth = 1;
     int smtp_debug = LOG_DEBUG;
     char crm_mail_body[BODY_MAX];
     char *crm_mail_subject = NULL;
 
     memset(&sa, 0, sizeof(struct sigaction));
 
     if (node == NULL) {
         node = "-";
     }
     if (rsc == NULL) {
         rsc = "-";
     }
     if (desc == NULL) {
         desc = "-";
     }
 
     if (crm_mail_to == NULL) {
         return 1;
     }
 
     if (crm_mail_host == NULL) {
         crm_mail_host = "localhost:25";
     }
 
     if (crm_mail_prefix == NULL) {
         crm_mail_prefix = "Cluster notification";
     }
 
     crm_debug("Sending '%s' mail to %s via %s", crm_mail_prefix, crm_mail_to, crm_mail_host);
 
     len += strlen(crm_mail_prefix);
     len += strlen(task);
     len += strlen(rsc);
     len += strlen(node);
     len += strlen(desc);
     len++;
 
     crm_mail_subject = calloc(1, len);
     /* If you edit this line, ensure you allocate enough memory for it by altering 'len' above */
     snprintf(crm_mail_subject, len, "%s - %s event for %s on %s: %s\r\n", crm_mail_prefix, task,
              rsc, node, desc);
 
     len = 0;
     len += snprintf(crm_mail_body + len, BODY_MAX - len, "\r\n%s\r\n", crm_mail_prefix);
     len += snprintf(crm_mail_body + len, BODY_MAX - len, "====\r\n\r\n");
     if (rc == target_rc) {
         len += snprintf(crm_mail_body + len, BODY_MAX - len,
                         "Completed operation %s for resource %s on %s\r\n", task, rsc, node);
     } else {
         len += snprintf(crm_mail_body + len, BODY_MAX - len,
                         "Operation %s for resource %s on %s failed: %s\r\n", task, rsc, node, desc);
     }
 
     len += snprintf(crm_mail_body + len, BODY_MAX - len, "\r\nDetails:\r\n");
     len += snprintf(crm_mail_body + len, BODY_MAX - len,
                     "\toperation status: (%d) %s\r\n", status, services_lrm_status_str(status));
     if (status == PCMK_LRM_OP_DONE) {
         len += snprintf(crm_mail_body + len, BODY_MAX - len,
                         "\tscript returned: (%d) %s\r\n", rc, services_ocf_exitcode_str(rc));
         len += snprintf(crm_mail_body + len, BODY_MAX - len,
                         "\texpected return value: (%d) %s\r\n", target_rc,
                         services_ocf_exitcode_str(target_rc));
     }
 
     auth_client_init();
     session = smtp_create_session();
     message = smtp_add_message(session);
 
     smtp_starttls_enable(session, Starttls_ENABLED);
 
     sa.sa_handler = SIG_IGN;
     sigemptyset(&sa.sa_mask);
     sa.sa_flags = 0;
     sigaction(SIGPIPE, &sa, NULL);
 
     smtp_set_server(session, crm_mail_host);
 
     authctx = auth_create_context();
     auth_set_mechanism_flags(authctx, AUTH_PLUGIN_PLAIN, 0);
 
     smtp_set_eventcb(session, event_cb, NULL);
 
     /* Now tell libESMTP it can use the SMTP AUTH extension.
      */
     if (!noauth) {
         crm_debug("Adding authentication context");
         smtp_auth_set_context(session, authctx);
     }
 
     if (crm_mail_from == NULL) {
         struct utsname us;
         char auto_from[BODY_MAX];
 
         CRM_ASSERT(uname(&us) == 0);
         snprintf(auto_from, BODY_MAX, "crm_mon@%s", us.nodename);
         smtp_set_reverse_path(message, auto_from);
 
     } else {
         /* NULL is ok */
         smtp_set_reverse_path(message, crm_mail_from);
     }
 
     smtp_set_header(message, "To", NULL /*phrase */ , NULL /*addr */ ); /* "Phrase" <addr> */
     smtp_add_recipient(message, crm_mail_to);
 
     /* Set the Subject: header and override any subject line in the message headers. */
     smtp_set_header(message, "Subject", crm_mail_subject);
     smtp_set_header_option(message, "Subject", Hdr_OVERRIDE, 1);
 
     smtp_set_message_str(message, crm_mail_body);
     smtp_set_monitorcb(session, crm_smtp_debug, &smtp_debug, 1);
 
     if (smtp_start_session(session)) {
         char buf[128];
         int rc = smtp_errno();
 
         crm_err("SMTP server problem: %s (%d)", smtp_strerror(rc, buf, sizeof buf), rc);
 
     } else {
         char buf[128];
         int rc = smtp_errno();
         const smtp_status_t *smtp_status = smtp_message_transfer_status(message);
 
         if (rc != 0) {
             crm_err("SMTP server problem: %s (%d)", smtp_strerror(rc, buf, sizeof buf), rc);
         }
         crm_info("Send status: %d %s", smtp_status->code, crm_str(smtp_status->text));
         smtp_enumerate_recipients(message, print_recipient_status, NULL);
     }
 
     smtp_destroy_session(session);
     auth_destroy_context(authctx);
     auth_client_exit();
 #endif
     return 0;
 }
 
 static void
 handle_rsc_op(xmlNode * xml, const char *node_id)
 {
     int rc = -1;
     int status = -1;
     int action = -1;
     int interval = 0;
     int target_rc = -1;
     int transition_num = -1;
     gboolean notify = TRUE;
 
     char *rsc = NULL;
     char *task = NULL;
     const char *desc = NULL;
     const char *magic = NULL;
     const char *id = NULL;
     char *update_te_uuid = NULL;
     const char *node = NULL;
 
     xmlNode *n = xml;
     xmlNode * rsc_op = xml;
 
     if(strcmp((const char*)xml->name, XML_LRM_TAG_RSC_OP) != 0) {
         xmlNode *cIter;
 
         for(cIter = xml->children; cIter; cIter = cIter->next) {
             handle_rsc_op(cIter, node_id);
         }
 
         return;
     }
 
     id = crm_element_value(rsc_op, XML_LRM_ATTR_TASK_KEY);
     if (id == NULL) {
         /* Compatibility with <= 1.1.5 */
         id = ID(rsc_op);
     }
 
     magic = crm_element_value(rsc_op, XML_ATTR_TRANSITION_MAGIC);
     if (magic == NULL) {
         /* non-change */
         return;
     }
 
     if (FALSE == decode_transition_magic(magic, &update_te_uuid, &transition_num, &action,
                                          &status, &rc, &target_rc)) {
         crm_err("Invalid event %s detected for %s", magic, id);
         return;
     }
 
     if (parse_op_key(id, &rsc, &task, &interval) == FALSE) {
         crm_err("Invalid event detected for %s", id);
         goto bail;
     }
 
     node = crm_element_value(rsc_op, XML_LRM_ATTR_TARGET);
 
     while (n != NULL && safe_str_neq(XML_CIB_TAG_STATE, TYPE(n))) {
         n = n->parent;
     }
 
     if(node == NULL && n) {
         node = crm_element_value(n, XML_ATTR_UNAME);
     }
 
     if (node == NULL && n) {
         node = ID(n);
     }
 
     if (node == NULL) {
         node = node_id;
     }
 
     if (node == NULL) {
         crm_err("No node detected for event %s (%s)", magic, id);
         goto bail;
     }
 
     /* look up where we expected it to be? */
     desc = pcmk_strerror(pcmk_ok);
     if (status == PCMK_LRM_OP_DONE && target_rc == rc) {
         crm_notice("%s of %s on %s completed: %s", task, rsc, node, desc);
         if (rc == PCMK_OCF_NOT_RUNNING) {
             notify = FALSE;
         }
 
     } else if (status == PCMK_LRM_OP_DONE) {
         desc = services_ocf_exitcode_str(rc);
         crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
 
     } else {
         desc = services_lrm_status_str(status);
         crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
     }
 
     if (notify && snmp_target) {
         send_snmp_trap(node, rsc, task, target_rc, rc, status, desc);
     }
     if (notify && crm_mail_to) {
         send_smtp_trap(node, rsc, task, target_rc, rc, status, desc);
     }
     if (notify && external_agent) {
         send_custom_trap(node, rsc, task, target_rc, rc, status, desc);
     }
   bail:
     free(update_te_uuid);
     free(rsc);
     free(task);
 }
 
 static gboolean
 mon_trigger_refresh(gpointer user_data)
 {
     mainloop_set_trigger(refresh_trigger);
     return FALSE;
 }
 
 #define NODE_PATT "/lrm[@id="
 static char *get_node_from_xpath(const char *xpath) 
 {
     char *nodeid = NULL;
     char *tmp = strstr(xpath, NODE_PATT);
 
     if(tmp) {
         tmp += strlen(NODE_PATT);
         tmp += 1;
 
         nodeid = strdup(tmp);
         tmp = strstr(nodeid, "\'");
         CRM_ASSERT(tmp);
         tmp[0] = 0;
     }
     return nodeid;
 }
 
 static void crm_diff_update_v2(const char *event, xmlNode * msg) 
 {
     xmlNode *change = NULL;
     xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
 
     for (change = __xml_first_child(diff); change != NULL; change = __xml_next(change)) {
         const char *name = NULL;
         const char *op = crm_element_value(change, XML_DIFF_OP);
         const char *xpath = crm_element_value(change, XML_DIFF_PATH);
         xmlNode *match = NULL;
         const char *node = NULL;
 
         if(op == NULL) {
             continue;
 
         } else if(strcmp(op, "create") == 0) {
             match = change->children;
 
         } else if(strcmp(op, "move") == 0) {
             continue;
 
         } else if(strcmp(op, "delete") == 0) {
             continue;
 
         } else if(strcmp(op, "modify") == 0) {
             match = first_named_child(change, XML_DIFF_RESULT);
             if(match) {
                 match = match->children;
             }
         }
 
         if(match) {
             name = (const char *)match->name;
         }
 
         crm_trace("Handling %s operation for %s %p, %s", op, xpath, match, name);
         if(xpath == NULL) {
             /* Version field, ignore */
 
         } else if(name == NULL) {
             crm_debug("No result for %s operation to %s", op, xpath);
             CRM_ASSERT(strcmp(op, "delete") == 0 || strcmp(op, "move") == 0);
 
         } else if(strcmp(name, XML_TAG_CIB) == 0) {
             xmlNode *state = NULL;
             xmlNode *status = first_named_child(match, XML_CIB_TAG_STATUS);
 
             for (state = __xml_first_child(status); state != NULL; state = __xml_next(state)) {
                 node = crm_element_value(state, XML_ATTR_UNAME);
                 if (node == NULL) {
                     node = ID(state);
                 }
                 handle_rsc_op(state, node);
             }
 
         } else if(strcmp(name, XML_CIB_TAG_STATUS) == 0) {
             xmlNode *state = NULL;
 
             for (state = __xml_first_child(match); state != NULL; state = __xml_next(state)) {
                 node = crm_element_value(state, XML_ATTR_UNAME);
                 if (node == NULL) {
                     node = ID(state);
                 }
                 handle_rsc_op(state, node);
             }
 
         } else if(strcmp(name, XML_CIB_TAG_STATE) == 0) {
             node = crm_element_value(match, XML_ATTR_UNAME);
             if (node == NULL) {
                 node = ID(match);
             }
             handle_rsc_op(match, node);
 
         } else if(strcmp(name, XML_CIB_TAG_LRM) == 0) {
             node = ID(match);
             handle_rsc_op(match, node);
 
         } else if(strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
             char *local_node = get_node_from_xpath(xpath);
 
             handle_rsc_op(match, local_node);
             free(local_node);
 
         } else if(strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
             char *local_node = get_node_from_xpath(xpath);
 
             handle_rsc_op(match, local_node);
             free(local_node);
 
         } else if(strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
             char *local_node = get_node_from_xpath(xpath);
 
             handle_rsc_op(match, local_node);
             free(local_node);
 
         } else {
             crm_trace("Ignoring %s operation for %s %p, %s", op, xpath, match, name);
         }
     }
 }
 
 static void crm_diff_update_v1(const char *event, xmlNode * msg) 
 {
     /* Process operation updates */
     xmlXPathObject *xpathObj = xpath_search(msg,
                                             "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED
                                             "//" XML_LRM_TAG_RSC_OP);
     int lpc = 0, max = numXpathResults(xpathObj);
 
     for (lpc = 0; lpc < max; lpc++) {
         xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
 
         handle_rsc_op(rsc_op, NULL);
     }
     freeXpathObject(xpathObj);
 }
 
 void
 crm_diff_update(const char *event, xmlNode * msg)
 {
     int rc = -1;
     long now = time(NULL);
     static bool stale = FALSE;
     static int updates = 0;
     static mainloop_timer_t *refresh_timer = NULL;
     xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
 
     print_dot();
 
     if(refresh_timer == NULL) {
         refresh_timer = mainloop_timer_add("refresh", 2000, FALSE, mon_trigger_refresh, NULL);
     }
 
     if (current_cib != NULL) {
         rc = xml_apply_patchset(current_cib, diff, TRUE);
 
         switch (rc) {
             case -pcmk_err_diff_resync:
             case -pcmk_err_diff_failed:
                 crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
                 free_xml(current_cib); current_cib = NULL;
                 break;
             case pcmk_ok:
                 updates++;
                 break;
             default:
                 crm_notice("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
                 free_xml(current_cib); current_cib = NULL;
         }
     }
 
     if (current_cib == NULL) {
         crm_trace("Re-requesting the full cib");
         cib->cmds->query(cib, NULL, &current_cib, cib_scope_local | cib_sync_call);
     }
 
     if (crm_mail_to || snmp_target || external_agent) {
         int format = 0;
         crm_element_value_int(diff, "format", &format);
         switch(format) {
             case 1:
                 crm_diff_update_v1(event, msg);
                 break;
             case 2:
                 crm_diff_update_v2(event, msg);
                 break;
             default:
                 crm_err("Unknown patch format: %d", format);
         }
     }
 
     if (current_cib == NULL) {
         if(!stale) {
             print_as("--- Stale data ---");
         }
         stale = TRUE;
         return;
     }
 
     stale = FALSE;
     /* Refresh
      * - immediately if the last update was more than 5s ago
      * - every 10 updates
      * - at most 2s after the last update
      */
     if ((now - last_refresh) > (reconnect_msec / 1000)) {
         mainloop_set_trigger(refresh_trigger);
         mainloop_timer_stop(refresh_timer);
         updates = 0;
 
     } else if(updates > 10) {
         mainloop_set_trigger(refresh_trigger);
         mainloop_timer_stop(refresh_timer);
         updates = 0;
 
     } else {
         mainloop_timer_start(refresh_timer);
     }
 }
 
 gboolean
 mon_refresh_display(gpointer user_data)
 {
     xmlNode *cib_copy = copy_xml(current_cib);
     pe_working_set_t data_set;
 
     last_refresh = time(NULL);
 
     if (cli_config_update(&cib_copy, NULL, FALSE) == FALSE) {
         if (cib) {
             cib->cmds->signoff(cib);
         }
         print_as("Upgrade failed: %s", pcmk_strerror(-pcmk_err_schema_validation));
         if (output_format == mon_output_console) {
             sleep(2);
         }
         clean_up(EX_USAGE);
         return FALSE;
     }
 
     set_working_set_defaults(&data_set);
     data_set.input = cib_copy;
     cluster_status(&data_set);
 
     /* Unpack constraints if any section will need them
      * (tickets may be referenced in constraints but not granted yet,
      * and bans need negative location constraints) */
     if (show & (mon_show_bans | mon_show_tickets)) {
         xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set.input);
         unpack_constraints(cib_constraints, &data_set);
     }
 
     switch (output_format) {
         case mon_output_html:
         case mon_output_cgi:
             if (print_html_status(&data_set, output_filename) != 0) {
                 fprintf(stderr, "Critical: Unable to output html file\n");
                 clean_up(EX_USAGE);
             }
             break;
 
         case mon_output_xml:
             print_xml_status(&data_set);
             break;
 
         case mon_output_monitor:
             print_simple_status(&data_set);
             if (has_warnings) {
                 clean_up(MON_STATUS_WARN);
             }
             break;
 
         case mon_output_plain:
         case mon_output_console:
             print_status(&data_set);
             break;
 
         case mon_output_none:
             break;
     }
 
     cleanup_alloc_calculations(&data_set);
     return TRUE;
 }
 
 void
 mon_st_callback(stonith_t * st, stonith_event_t * e)
 {
     char *desc = crm_strdup_printf("Operation %s requested by %s for peer %s: %s (ref=%s)",
                                  e->operation, e->origin, e->target, pcmk_strerror(e->result),
                                  e->id);
 
     if (snmp_target) {
         send_snmp_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
     }
     if (crm_mail_to) {
         send_smtp_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
     }
     if (external_agent) {
         send_custom_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
     }
     free(desc);
 }
 
 /*
  * De-init ncurses, signoff from the CIB and deallocate memory.
  */
 void
 clean_up(int rc)
 {
 #if ENABLE_SNMP
     netsnmp_session *session = crm_snmp_init(NULL, NULL);
 
     if (session) {
         snmp_close(session);
         snmp_shutdown("snmpapp");
     }
 #endif
 
 #if CURSES_ENABLED
     if (output_format == mon_output_console) {
         output_format = mon_output_plain;
         echo();
         nocbreak();
         endwin();
     }
 #endif
 
     if (cib != NULL) {
         cib->cmds->signoff(cib);
         cib_delete(cib);
         cib = NULL;
     }
 
     free(output_filename);
     free(xml_file);
     free(pid_file);
 
     if (rc >= 0) {
         crm_exit(rc);
     }
     return;
 }