diff --git a/extra/resources/ifspeed b/extra/resources/ifspeed
index 8609b50f1b..cf3cdbb2cb 100644
--- a/extra/resources/ifspeed
+++ b/extra/resources/ifspeed
@@ -1,536 +1,541 @@
 #!/bin/bash
 #
 # OCF resource agent which monitors state of network interface and records it
 # as a node attribute in the CIB based on the sum of speeds of its active (up,
 # link detected, not blocked) underlying interfaces.
 #
 # Copyright (c) 2011 Vladislav Bogdanov <bubble@hoster-ok.com>
 # Partially based on 'ping' RA by Andrew Beekhof
 #
 # Change on 2017 by Tomer Azran <tomerazran@gmail.com>:
 #     Add "ip" parameter to detect network interface name by ip address:
 #     http://lists.clusterlabs.org/pipermail/users/2017-August/006224.html
 #
 # OCF instance parameters:
 #    OCF_RESKEY_name:         name of attribute to set in CIB
 #    OCF_RESKEY_ip            ip address to check
 #    OCF_RESKEY_iface:        network interface to monitor
 #    OCF_RESKEY_bridge_ports: if not null and OCF_RESKEY_iface is a bridge, list of
 #                             bridge ports to consider.
 #                             Default is all ports which have designated_bridge=root_id 
 #    OCF_RESKEY_weight_base:  Relative weight of 1Gbps. This can be used to tune
 #                             value of resulting CIB attribute.
 #
 # Initialization:
 
 : ${OCF_FUNCTIONS=${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs}
-. ${OCF_FUNCTIONS}
+
+# If these aren't available, we can still show help,
+# which is all that is needed to build the man pages.
+[ -r "${OCF_FUNCTIONS}" ] && . "${OCF_FUNCTIONS}"
+[ -r "${OCF_FUNCTIONS_DIR}/findif.sh" ] && . "${OCF_FUNCTIONS_DIR}/findif.sh"
+: ${OCF_SUCCESS=0}
+
 : ${__OCF_ACTION=$1}
 
-. ${OCF_FUNCTIONS_DIR}/findif.sh
 FINDIF=findif
 
 # Defaults
 OCF_RESKEY_name_default="ifspeed"
 OCF_RESKEY_bridge_ports_default="detect"
 OCF_RESKEY_weight_base_default=1000
 OCF_RESKEY_dampen_default=5
 
 : ${OCF_RESKEY_name=${OCF_RESKEY_name_default}}
 : ${OCF_RESKEY_bridge_ports=${OCF_RESKEY_bridge_ports_default}}
 : ${OCF_RESKEY_weight_base=${OCF_RESKEY_weight_base_default}}
 : ${OCF_RESKEY_dampen=${OCF_RESKEY_dampen_default}}
 
 meta_data() {
         cat <<END
 <?xml version="1.0"?>
 <!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
 <resource-agent name="ifspeed" version="1.0">
 <version>1.0</version>
 
 <longdesc lang="en">
 Every time the monitor action is run, this resource agent records (in the CIB)
 (relative) speed of network interface it monitors.
 
 This RA can monitor physical interfaces, bonds, bridges, vlans and (hopefully)
 any combination of them.
 
 Examples:
 *) Bridge on top of one 10Gbps interface (eth2) and 802.3ad bonding (bond0) built
    on two 1Gbps interfaces (eth0 and eth1).
 *) Active-backup bonding built on top of one physical interface and one vlan on
    another interface.
 
 For STP-enabled bridges this RA tries to some-how guess network topology and by
 default looks only on ports which are connected to upstream switch. This can be
 overriden by 'bridge_ports' parameter. Active interfaces in this case are those
 in "forwarding" state.
 
 For balancing bonds this RA summs speeds of underlying "up" slave interfaces
 (and applies coefficient 0.8 to result).
 
 For non-balancing bonds ('active-backup' and probably 'broadcast'), only the
 speed of the currently active slave is used. 
 </longdesc>
 <shortdesc lang="en">Network interface speed monitor</shortdesc>
 
 <parameters>
 
 <parameter name="name" unique="1">
 <longdesc lang="en">
 The name of the attribute to set.  This is the name to be used in the constraints.
 </longdesc>
 <shortdesc lang="en">Attribute name</shortdesc>
 <content type="string" default="${OCF_RESKEY_name_default}"/>
 </parameter>
 
 <parameter name="iface" unique="0" required="1">
 <longdesc lang="en">
 Network interface to monitor.
 </longdesc>
 <shortdesc lang="en">Network interface</shortdesc>
 <content type="string" default=""/>
 </parameter>
 
 <parameter name="ip" unique="0" required="0">
 <longdesc lang="en">
 Try to detect interface name by detecting the interface that holds the IP address.
 The IPv4 (dotted quad notation) or IPv6 address (colon hexadecimal notation)
 example IPv4 "192.168.1.1".
 example IPv6 "2001:db8:DC28:0:0:FC57:D4C8:1FFF".
 </longdesc>
 <shortdesc lang="en">IPv4 or IPv6 address</shortdesc>
 <content type="string" default="" />
 </parameter>
 
 <parameter name="bridge_ports" unique="0">
 <longdesc lang="en">
 If not null and OCF_RESKEY_iface is a bridge, list of bridge ports to consider.
 Default is all ports which have designated_bridge=root_id.
 </longdesc>
 <shortdesc lang="en">Bridge ports</shortdesc>
 <content type="string" default="${OCF_RESKEY_bridge_ports_default}"/>
 </parameter>
 
 <parameter name="weight_base" unique="0">
 <longdesc lang="en">
 Relative weight of 1Gbps in interface speed.
 Can be used to tune how big attribute value will be.
 </longdesc>
 <shortdesc lang="en">Weight of 1Gbps</shortdesc>
 <content type="integer" default="${OCF_RESKEY_weight_base_default}"/>
 </parameter>
 
 <parameter name="dampen" unique="0">
 <longdesc lang="en">
 The time to wait (dampening) for further changes to occur.
 </longdesc>
 <shortdesc lang="en">Dampening interval</shortdesc>
 <content type="integer" default="${OCF_RESKEY_dampen_default}"/>
 </parameter>
 
 <parameter name="debug" unique="0">
 <longdesc lang="en">
 Log what have been done more verbosely.
 </longdesc>
 <shortdesc lang="en">Verbose logging</shortdesc>
 <content type="string" default="false"/>
 </parameter>
 
 </parameters>
 
 <actions>
 <action name="start"   timeout="30" />
 <action name="stop"    timeout="30" />
 <action name="monitor" depth="0"  timeout="30" interval="10"/>
 <action name="meta-data"  timeout="5" />
 <action name="validate-all"  timeout="30" />
 </actions>
 </resource-agent>
 END
 }
 
 usage() {
     cat <<END
 usage: $0 {start|stop|monitor|validate-all|meta-data}
 
 Expects to have a fully populated OCF RA-compliant environment set.
 END
 }
 
 start() {
     monitor
     if [ $? -eq $OCF_SUCCESS ]; then
         return $OCF_SUCCESS
     fi
     ha_pseudo_resource ${ha_pseudo_resource_name} start
     update
     return $?
 }
 
 stop() {
     ha_pseudo_resource ${ha_pseudo_resource_name} stop
     attrd_updater -D -n ${OCF_RESKEY_name} -d ${OCF_RESKEY_dampen} ${attrd_options}
     return $OCF_SUCCESS
 }
 
 monitor() {
     local ret
     ha_pseudo_resource ${ha_pseudo_resource_name} monitor
     ret=$?
     if [ ${ret} -eq $OCF_SUCCESS ] ; then
         update
     fi
     return ${ret}
 }
 
 # This function tries to guess nic interface by IP
 get_nic_name_by_ip(){
     # $FINDIF takes its parameters from the environment.
     # Its output is as follows:
     # [NIC_NAME] netmask [NETMASK] broadcast [BROADCAST}
     NICINFO=$( ${FINDIF} )
     rc=$?
     if [ $rc -eq 0 ];then
         # Get NIC_NAME part of findif function output.
         echo "${NICINFO%% *}"
     else
         echo ""
     fi
 }
 
 validate() {
     # Check the interface parameter
     if [ -z "${OCF_RESKEY_iface}" ]; then
         if [ -z "${OCF_RESKEY_ip}" ]; then
             ocf_log err "Empty iface and ip parameters. Please specify either an interface name or valid ip address."
             exit $OCF_ERR_CONFIGURED
         else
             ipcheck_ipv4 "${OCF_RESKEY_ip}"
             if [ $? -eq 1 ] ; then
                 ipcheck_ipv6 "${OCF_RESKEY_ip}"
                 if [ $? -eq 1 ] ; then
                   ocf_exit_reason "ip parameter [${OCF_RESKEY_ip}] is not a valid ip address."
                   exit $OCF_ERR_CONFIGURED
                 fi
             fi
         fi
     fi
     return $OCF_SUCCESS
 }
 
 iface_get_speed() {
     local iface=$1
     local operstate
     local carrier
     local bridge_iface_speed
     local bond_iface_speed
     local vlan_iface_speed
     local speed
 
     if [ ! -e "/sys/class/net/${iface}" ] ; then
         echo "0"
     elif iface_is_bridge ${iface} ; then # bridges do not have operstate
         read carrier < "/sys/class/net/${iface}/carrier"
 
         if [ "${carrier}" != "1" ] ; then
             echo "0"
         else
             bridge_get_speed ${iface}
         fi
     else
         read operstate < "/sys/class/net/${iface}/operstate"
         read carrier < "/sys/class/net/${iface}/carrier"
 
         if [ "${operstate}" != "up" ] || [ "${carrier}" != "1" ] ; then
             echo "0"
         elif iface_is_bond ${iface} ; then
             bond_get_speed ${iface}
         elif iface_is_vlan ${iface} ; then
             iface_get_speed $( vlan_get_phy ${iface} )
         elif iface_is_hfi1 "${iface}" ; then
             hfi1_get_speed "${iface}"
         else
             read speed < "/sys/class/net/${iface}/speed"
             echo ${speed}
         fi
     fi
 }
 
 iface_is_vlan() {
     local iface=$1
     [ -e "/proc/net/vlan/${iface}" ] && return 0 || return 1
 }
 
 iface_is_bridge() {
     local iface=$1
     [ -e "/sys/class/net/${iface}/bridge" ] && return 0 || return 1
 }
 
 iface_is_bond() {
     local iface=$1
     [ -e "/sys/class/net/${iface}/bonding" ] && return 0 || return 1
 }
 
 iface_is_hfi1() {
     local iface=$1
     driver=$(readlink /sys/class/net/${iface}/device/driver)
     [[ $(basename ${driver}) =~ "hfi1" ]] && return 0 || return 1
 }
 
 vlan_get_phy() {
     local iface=$1
     sed -ne "s/^${iface} .*| *//p" < /proc/net/vlan/config
 }
 
 bridge_is_stp_enabled() {
     local iface=$1
     local stp
     read stp < "/sys/class/net/${iface}/bridge/stp_state"
     [ "${stp}" = "1" ] && return 0 || return 1
 }
 
 bridge_get_root_ports() {
     local bridge=$1
     local root_id
     local root_ports=""
     local bridge_id
 
     read root_id < "/sys/class/net/${bridge}/bridge/root_id"
 
     for port in /sys/class/net/${bridge}/brif/* ; do
         read bridge_id < "${port}/designated_bridge"
         if [ "${bridge_id}" = "${root_id}" ] ; then
             root_ports="${root_ports} ${port##*/}"
         fi
     done
 
     root_ports=${root_ports# }
 
     if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell. 
         eval $2=\${root_ports}
     else # Expect sub-shell
         echo ${root_ports}
     fi
 }
 
 # From /inlude/linux/if_bridge.h:
 #define BR_STATE_DISABLED 0
 #define BR_STATE_LISTENING 1
 #define BR_STATE_LEARNING 2
 #define BR_STATE_FORWARDING 3
 #define BR_STATE_BLOCKING 4
 
 bridge_get_active_ports() {
     local bridge=$1
     shift 1
     local ports="$*"
     local active_ports=""
     local port_state
     local stp_state
     local warn=0
 
     bridge_is_stp_enabled ${bridge}
     stp_state=$?
 
     if [ -z "${ports}" ] || [ "${ports}" = "detect" ] ; then
         bridge_get_root_ports ${bridge} ports
     fi
 
     for port in $ports ; do
         if [ ! -e "/sys/class/net/${bridge}/brif/${port}" ] ; then
             ocf_log warning "Port ${port} doesn't belong to bridge ${bridge}"
             continue
         fi
         read port_state < "/sys/class/net/${bridge}/brif/${port}/state"
         if [ "${port_state}" = "3" ] ; then
             if [ -n "${active_ports}" ] && ${stp_state} ; then
                 warn=1
             fi
             active_ports="${active_ports} ${port}"
         fi
     done
     if [ ${warn} -eq 1 ] ; then
         ocf_log warning "More then one upstream port in bridge '${bridge}' is in forwarding state while STP is enabled: ${active_ports}" 
     fi
     echo "${active_ports# }"
 }
 
 bridge_get_speed() {
     local iface=$1
     local bridge_port_speed
     local aggregate_speed=0
 
     if ! iface_is_bridge ${iface} ; then
         echo 0
         return
     fi
 
     local ports=$( bridge_get_active_ports ${iface} ${OCF_RESKEY_bridge_ports} )
     for port in ${ports} ; do
         : $(( aggregate_speed += $( iface_get_speed ${port} ) ))
     done
     if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell. 
         eval $2=\${aggregate_speed}
     else # Expect sub-shell
         echo ${aggregate_speed}
     fi
 }
 
 hfi1_get_speed() {
     local iface=$1
     local hfi1_speed
     local hfi1_value
     local hfi1_desc
 
     # Currently (9/14/2017 Intel Omni Path v10.5.0.0.155) Intel doesn't have Dual/Multiple ports Host Channel Adapters
     # and it's save to use such method to get a speed.
     # Example of output:
     # [root@es-host0 ~]# cat /sys/class/net/ib0/device/infiniband/*/ports/*/rate
     # 100 Gb/sec (4X EDR)
     read hfi1_speed hfi1_value hfi1_desc < /sys/class/net/${iface}/device/infiniband/*/ports/*/rate
     
     # hfi1_value always in Gb/sec, so we need to convert hfi1_speed in Mb/sec
     echo $(( hfi1_speed * 1000 ))
 }
 
 bond_get_slaves() {
     local iface=$1
     local slaves
     read slaves < "/sys/class/net/${iface}/bonding/slaves"
     if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell. 
         eval $2=\${slaves}
     else # Expect sub-shell
         echo ${slaves}
     fi
 }
 
 bond_get_active_iface() {
     local iface=$1
     local active
     read active < "/sys/class/net/${iface}/bonding/active_slave"
     if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell. 
         eval $2=\${active}
     else # Expect sub-shell
         echo ${active}
     fi
 }
 
 bond_is_balancing() {
     local iface=$1
     read mode mode_index < "/sys/class/net/${iface}/bonding/mode"
     case ${mode} in
         "balance-rr"|"balance-xor"|"802.3ad"|"balance-tlb"|"balance-alb")
             return 0
             ;;
         *)
             return 1
             ;;
     esac
 }
 
 bond_get_speed() {
     local iface=$1
     local aggregate_speed=0
     local bond_slave_speed
     local active_iface
     local bond_slaves
 
     if ! iface_is_bond ${iface} ; then
         echo 0
         return
     fi
 
     bond_get_slaves ${iface} bond_slaves
 
     if bond_is_balancing ${iface} ; then
         for slave in ${bond_slaves} ; do
             : $(( aggregate_speed += $( iface_get_speed ${slave} ) ))
         done
         # Bonding is unable to get speed*n
         : $(( aggregate_speed = aggregate_speed * 8 / 10 ))
     else
         bond_get_active_iface ${iface} active_iface
         aggregate_speed=$( iface_get_speed $active_iface )
     fi
     if [ -n "$2" ] ; then # Record value in specified var. This expects we were called not in a sub-shell. 
         eval $2=\${aggregate_speed}
     else # Expect sub-shell
         echo ${aggregate_speed}
     fi
 }
 
 update() {
     local speed;
     local nic=${OCF_RESKEY_iface};
     if [ -z "${OCF_RESKEY_iface}" ]; then
         nic=$( get_nic_name_by_ip )
         if [ -z "${nic}" ];then
             ocf_log err "Could not retrieve network interface name from ip address (${OCF_RESKEY_ip})"
             exit $OCF_ERR_GENERIC
         fi
     fi
     speed=$( iface_get_speed ${nic} )
 
     : $(( score = speed * ${OCF_RESKEY_weight_base} / 1000 ))
     if [ "$__OCF_ACTION" = "start" ] ; then
         attrd_updater -n ${OCF_RESKEY_name} -B ${score} -d ${OCF_RESKEY_dampen} ${attrd_options}
     else 
         attrd_updater -n ${OCF_RESKEY_name} -v ${score} -d ${OCF_RESKEY_dampen} ${attrd_options}
     fi
     rc=$?
     case ${rc} in
         0)
             ocf_is_true ${OCF_RESKEY_debug} && ocf_log debug "Updated ${OCF_RESKEY_name} = ${score}"
             ;;
         *)
             ocf_log warn "Could not update ${OCF_RESKEY_name} = ${score}: rc=${rc}"
             ;;
     esac
     return ${rc}
 }
 
 case $__OCF_ACTION in
     meta-data)
         meta_data
         exit $OCF_SUCCESS
         ;;
     usage|help)
         usage
         exit $OCF_SUCCESS
         ;;
 esac
 
 if [ `uname` != "Linux" ] ; then
     ocf_log err "This RA works only on linux."
     exit $OCF_ERR_INSTALLED
 fi
 
 : ${ha_pseudo_resource_name:="ifspeed-${OCF_RESOURCE_INSTANCE}"}
 
 attrd_options='-q'
 if ocf_is_true ${OCF_RESKEY_debug} ; then
     attrd_options=''
 fi
 
 validate || exit $?
 
 case $__OCF_ACTION in
     start)
         start
         ;;
     stop)
         stop
         ;;
     monitor)
         monitor
         ;;
     validate-all)
         ;;
     *)
         usage
         exit $OCF_ERR_UNIMPLEMENTED
         ;;
 esac
 
 exit $?
diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h
index 5df598acf6..9ffb22bc6b 100644
--- a/include/crm/pengine/internal.h
+++ b/include/crm/pengine/internal.h
@@ -1,307 +1,312 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 #ifndef PE_INTERNAL__H
 #  define PE_INTERNAL__H
 #  include <crm/pengine/status.h>
 #  include <crm/pengine/remote.h>
 
 #  define pe_rsc_info(rsc, fmt, args...)  crm_log_tag(LOG_INFO,  rsc ? rsc->id : "<NULL>", fmt, ##args)
 #  define pe_rsc_debug(rsc, fmt, args...) crm_log_tag(LOG_DEBUG, rsc ? rsc->id : "<NULL>", fmt, ##args)
 #  define pe_rsc_trace(rsc, fmt, args...) crm_log_tag(LOG_TRACE, rsc ? rsc->id : "<NULL>", fmt, ##args)
 
 #  define pe_err(fmt...) { was_processing_error = TRUE; crm_config_error = TRUE; crm_err(fmt); }
 #  define pe_warn(fmt...) { was_processing_warning = TRUE; crm_config_warning = TRUE; crm_warn(fmt); }
 #  define pe_proc_err(fmt...) { was_processing_error = TRUE; crm_err(fmt); }
 #  define pe_proc_warn(fmt...) { was_processing_warning = TRUE; crm_warn(fmt); }
 #  define pe_set_action_bit(action, bit) action->flags = crm_set_bit(__FUNCTION__, __LINE__, action->uuid, action->flags, bit)
 #  define pe_clear_action_bit(action, bit) action->flags = crm_clear_bit(__FUNCTION__, __LINE__, action->uuid, action->flags, bit)
 
 typedef struct notify_data_s {
     GHashTable *keys;
 
     const char *action;
 
     action_t *pre;
     action_t *post;
     action_t *pre_done;
     action_t *post_done;
 
     GListPtr active;            /* notify_entry_t*  */
     GListPtr inactive;          /* notify_entry_t*  */
     GListPtr start;             /* notify_entry_t*  */
     GListPtr stop;              /* notify_entry_t*  */
     GListPtr demote;            /* notify_entry_t*  */
     GListPtr promote;           /* notify_entry_t*  */
     GListPtr master;            /* notify_entry_t*  */
     GListPtr slave;             /* notify_entry_t*  */
     GHashTable *allowed_nodes;
 
 } notify_data_t;
 
 bool pe_can_fence(pe_working_set_t *data_set, node_t *node);
 
 int merge_weights(int w1, int w2);
 void add_hash_param(GHashTable * hash, const char *name, const char *value);
 void append_hashtable(gpointer key, gpointer value, gpointer user_data);
 
 char *native_parameter(resource_t * rsc, node_t * node, gboolean create, const char *name,
                        pe_working_set_t * data_set);
 node_t *native_location(resource_t * rsc, GListPtr * list, gboolean current);
 
 void pe_metadata(void);
 void verify_pe_options(GHashTable * options);
 
 void common_update_score(resource_t * rsc, const char *id, int score);
 void native_add_running(resource_t * rsc, node_t * node, pe_working_set_t * data_set);
 node_t *rsc_known_on(resource_t * rsc, GListPtr * list);
 
 gboolean native_unpack(resource_t * rsc, pe_working_set_t * data_set);
 gboolean group_unpack(resource_t * rsc, pe_working_set_t * data_set);
 gboolean clone_unpack(resource_t * rsc, pe_working_set_t * data_set);
 gboolean master_unpack(resource_t * rsc, pe_working_set_t * data_set);
 gboolean container_unpack(resource_t * rsc, pe_working_set_t * data_set);
 
 resource_t *native_find_rsc(resource_t * rsc, const char *id, node_t * node, int flags);
 
 gboolean native_active(resource_t * rsc, gboolean all);
 gboolean group_active(resource_t * rsc, gboolean all);
 gboolean clone_active(resource_t * rsc, gboolean all);
 gboolean master_active(resource_t * rsc, gboolean all);
 gboolean container_active(resource_t * rsc, gboolean all);
 
 void native_print(resource_t * rsc, const char *pre_text, long options, void *print_data);
 void group_print(resource_t * rsc, const char *pre_text, long options, void *print_data);
 void clone_print(resource_t * rsc, const char *pre_text, long options, void *print_data);
 void master_print(resource_t * rsc, const char *pre_text, long options, void *print_data);
 void container_print(resource_t * rsc, const char *pre_text, long options, void *print_data);
 
 void native_free(resource_t * rsc);
 void group_free(resource_t * rsc);
 void clone_free(resource_t * rsc);
 void master_free(resource_t * rsc);
 void container_free(resource_t * rsc);
 
 enum rsc_role_e native_resource_state(const resource_t * rsc, gboolean current);
 enum rsc_role_e group_resource_state(const resource_t * rsc, gboolean current);
 enum rsc_role_e clone_resource_state(const resource_t * rsc, gboolean current);
 enum rsc_role_e master_resource_state(const resource_t * rsc, gboolean current);
 enum rsc_role_e container_resource_state(const resource_t * rsc, gboolean current);
 
 gboolean common_unpack(xmlNode * xml_obj, resource_t ** rsc, resource_t * parent,
                        pe_working_set_t * data_set);
 void common_free(resource_t * rsc);
 
 extern pe_working_set_t *pe_dataset;
 
 extern node_t *node_copy(const node_t *this_node);
 extern time_t get_effective_time(pe_working_set_t * data_set);
 
 /* Failure handling utilities (from failcounts.c) */
-extern int get_failcount(node_t * node, resource_t * rsc, time_t *last_failure,
-                         pe_working_set_t * data_set);
-extern int get_failcount_full(node_t * node, resource_t * rsc, time_t *last_failure,
-                              bool effective, xmlNode * xml_op, pe_working_set_t * data_set);
-extern int get_failcount_all(node_t * node, resource_t * rsc, time_t *last_failure,
-                             pe_working_set_t * data_set);
+
+// bit flags for fail count handling options
+enum pe_fc_flags_e {
+    pe_fc_default   = 0x00,
+    pe_fc_effective = 0x01, // don't count expired failures
+    pe_fc_fillers   = 0x02, // if container, include filler failures in count
+};
+
+int pe_get_failcount(node_t *node, resource_t *rsc, time_t *last_failure,
+                     uint32_t flags, xmlNode *xml_op,
+                     pe_working_set_t *data_set);
 
 /* Binary like operators for lists of nodes */
 extern void node_list_exclude(GHashTable * list, GListPtr list2, gboolean merge_scores);
 extern GListPtr node_list_dup(GListPtr list, gboolean reset, gboolean filter);
 extern GListPtr node_list_from_hash(GHashTable * hash, gboolean reset, gboolean filter);
 
 extern GHashTable *node_hash_from_list(GListPtr list);
 static inline gpointer
 pe_hash_table_lookup(GHashTable * hash, gconstpointer key)
 {
     if (hash) {
         return g_hash_table_lookup(hash, key);
     }
     return NULL;
 }
 
 extern action_t *get_pseudo_op(const char *name, pe_working_set_t * data_set);
 extern gboolean order_actions(action_t * lh_action, action_t * rh_action, enum pe_ordering order);
 
 GHashTable *node_hash_dup(GHashTable * hash);
 extern GListPtr node_list_and(GListPtr list1, GListPtr list2, gboolean filter);
 
 extern GListPtr node_list_xor(GListPtr list1, GListPtr list2, gboolean filter);
 
 extern GListPtr node_list_minus(GListPtr list1, GListPtr list2, gboolean filter);
 
 extern void pe_free_shallow(GListPtr alist);
 extern void pe_free_shallow_adv(GListPtr alist, gboolean with_data);
 
 /* Printing functions for debug */
 extern void print_node(const char *pre_text, node_t * node, gboolean details);
 
 extern void print_resource(int log_level, const char *pre_text, resource_t * rsc, gboolean details);
 
 extern void dump_node_scores_worker(int level, const char *file, const char *function, int line,
                                     resource_t * rsc, const char *comment, GHashTable * nodes);
 
 extern void dump_node_capacity(int level, const char *comment, node_t * node);
 extern void dump_rsc_utilization(int level, const char *comment, resource_t * rsc, node_t * node);
 
 #  define dump_node_scores(level, rsc, text, nodes) do {		\
         dump_node_scores_worker(level, __FILE__, __FUNCTION__, __LINE__, rsc, text, nodes); \
     } while(0)
 
 /* Sorting functions */
 extern gint sort_rsc_priority(gconstpointer a, gconstpointer b);
 extern gint sort_rsc_index(gconstpointer a, gconstpointer b);
 
 extern xmlNode *find_rsc_op_entry(resource_t * rsc, const char *key);
 
 extern action_t *custom_action(resource_t * rsc, char *key, const char *task, node_t * on_node,
                                gboolean optional, gboolean foo, pe_working_set_t * data_set);
 
 #  define delete_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_DELETE, 0)
 #  define delete_action(rsc, node, optional) custom_action(		\
 		rsc, delete_key(rsc), CRMD_ACTION_DELETE, node,		\
 		optional, TRUE, data_set);
 
 #  define stopped_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_STOPPED, 0)
 #  define stopped_action(rsc, node, optional) custom_action(		\
 		rsc, stopped_key(rsc), CRMD_ACTION_STOPPED, node,	\
 		optional, TRUE, data_set);
 
 #  define stop_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_STOP, 0)
 #  define stop_action(rsc, node, optional) custom_action(			\
 		rsc, stop_key(rsc), CRMD_ACTION_STOP, node,		\
 		optional, TRUE, data_set);
 
 #  define reload_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_RELOAD, 0)
 #  define start_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_START, 0)
 #  define start_action(rsc, node, optional) custom_action(		\
 		rsc, start_key(rsc), CRMD_ACTION_START, node,		\
 		optional, TRUE, data_set)
 
 #  define started_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_STARTED, 0)
 #  define started_action(rsc, node, optional) custom_action(		\
 		rsc, started_key(rsc), CRMD_ACTION_STARTED, node,	\
 		optional, TRUE, data_set)
 
 #  define promote_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_PROMOTE, 0)
 #  define promote_action(rsc, node, optional) custom_action(		\
 		rsc, promote_key(rsc), CRMD_ACTION_PROMOTE, node,	\
 		optional, TRUE, data_set)
 
 #  define promoted_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_PROMOTED, 0)
 #  define promoted_action(rsc, node, optional) custom_action(		\
 		rsc, promoted_key(rsc), CRMD_ACTION_PROMOTED, node,	\
 		optional, TRUE, data_set)
 
 #  define demote_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_DEMOTE, 0)
 #  define demote_action(rsc, node, optional) custom_action(		\
 		rsc, demote_key(rsc), CRMD_ACTION_DEMOTE, node,		\
 		optional, TRUE, data_set)
 
 #  define demoted_key(rsc) generate_op_key(rsc->id, CRMD_ACTION_DEMOTED, 0)
 #  define demoted_action(rsc, node, optional) custom_action(		\
 		rsc, demoted_key(rsc), CRMD_ACTION_DEMOTED, node,	\
 		optional, TRUE, data_set)
 
 extern int pe_get_configured_timeout(resource_t *rsc, const char *action,
                                      pe_working_set_t *data_set);
 
 extern action_t *find_first_action(GListPtr input, const char *uuid, const char *task,
                                    node_t * on_node);
 extern enum action_tasks get_complex_task(resource_t * rsc, const char *name,
                                           gboolean allow_non_atomic);
 
 extern GListPtr find_actions(GListPtr input, const char *key, const node_t *on_node);
 extern GListPtr find_actions_exact(GListPtr input, const char *key, node_t * on_node);
 extern GListPtr find_recurring_actions(GListPtr input, node_t * not_on_node);
 
 extern void pe_free_action(action_t * action);
 
 extern void resource_location(resource_t * rsc, node_t * node, int score, const char *tag,
                               pe_working_set_t * data_set);
 
 extern gint sort_op_by_callid(gconstpointer a, gconstpointer b);
 extern gboolean get_target_role(resource_t * rsc, enum rsc_role_e *role);
 
 extern resource_t *find_clone_instance(resource_t * rsc, const char *sub_id,
                                        pe_working_set_t * data_set);
 
 extern void destroy_ticket(gpointer data);
 extern ticket_t *ticket_new(const char *ticket_id, pe_working_set_t * data_set);
 
 char *clone_strip(const char *last_rsc_id);
 char *clone_zero(const char *last_rsc_id);
 
 int get_target_rc(xmlNode * xml_op);
 
 gint sort_node_uname(gconstpointer a, gconstpointer b);
 bool is_set_recursive(resource_t * rsc, long long flag, bool any);
 
 enum rsc_digest_cmp_val {
     /*! Digests are the same */
     RSC_DIGEST_MATCH = 0,
     /*! Params that require a restart changed */
     RSC_DIGEST_RESTART,
     /*! Some parameter changed.  */
     RSC_DIGEST_ALL,
     /*! rsc op didn't have a digest associated with it, so
      *  it is unknown if parameters changed or not. */
     RSC_DIGEST_UNKNOWN,
 };
 
 typedef struct op_digest_cache_s {
     enum rsc_digest_cmp_val rc;
     xmlNode *params_all;
     xmlNode *params_secure;
     xmlNode *params_restart;
     char *digest_all_calc;
     char *digest_secure_calc;
     char *digest_restart_calc;
 } op_digest_cache_t;
 
 op_digest_cache_t *rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node,
                                          pe_working_set_t * data_set);
 
 action_t *pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe_working_set_t * data_set);
 void trigger_unfencing(
     resource_t * rsc, node_t *node, const char *reason, action_t *dependency, pe_working_set_t * data_set);
 
 void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrite);
 void pe_action_set_flag_reason(const char *function, long line, pe_action_t *action, pe_action_t *reason, const char *text, enum pe_action_flags flags, bool overwrite);
 
 #define pe_action_required(action, reason, text) pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, reason, text, pe_action_optional, FALSE)
 #define pe_action_implies(action, reason, flag) pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, reason, NULL, flag, FALSE)
 
 void set_bit_recursive(resource_t * rsc, unsigned long long flag);
 void clear_bit_recursive(resource_t * rsc, unsigned long long flag);
 
 gboolean add_tag_ref(GHashTable * tags, const char * tag_name,  const char * obj_ref);
 
 void print_rscs_brief(GListPtr rsc_list, const char * pre_text, long options,
                       void * print_data, gboolean print_all);
 void pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason);
 
 node_t *pe_create_node(const char *id, const char *uname, const char *type,
                        const char *score, pe_working_set_t * data_set);
 bool remote_id_conflict(const char *remote_name, pe_working_set_t *data);
 void common_print(resource_t * rsc, const char *pre_text, const char *name, node_t *node, long options, void *print_data);
 resource_t *find_container_child(const char *stem, resource_t * rsc, node_t *node);
 bool fix_remote_addr(resource_t * rsc);
 const char *pe_node_attribute_calculated(pe_node_t *node, const char *name, resource_t *rsc);
 const char *pe_node_attribute_raw(pe_node_t *node, const char *name);
 
 #endif
diff --git a/lib/pengine/failcounts.c b/lib/pengine/failcounts.c
index 8ce69eef6c..53c0f142d7 100644
--- a/lib/pengine/failcounts.c
+++ b/lib/pengine/failcounts.c
@@ -1,340 +1,321 @@
 /*
  * Copyright (C) 2008-2017 Andrew Beekhof <andrew@beekhof.net>
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/types.h>
 #include <regex.h>
 #include <glib.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/util.h>
 #include <crm/pengine/internal.h>
 
-int
-get_failcount(node_t *node, resource_t *rsc, time_t *last_failure,
-              pe_working_set_t *data_set)
-{
-    return get_failcount_full(node, rsc, last_failure, TRUE, NULL, data_set);
-}
-
 static gboolean
 is_matched_failure(const char *rsc_id, xmlNode *conf_op_xml,
                    xmlNode *lrm_op_xml)
 {
     gboolean matched = FALSE;
     const char *conf_op_name = NULL;
     int conf_op_interval = 0;
     const char *lrm_op_task = NULL;
     int lrm_op_interval = 0;
     const char *lrm_op_id = NULL;
     char *last_failure_key = NULL;
 
     if (rsc_id == NULL || conf_op_xml == NULL || lrm_op_xml == NULL) {
         return FALSE;
     }
 
     conf_op_name = crm_element_value(conf_op_xml, "name");
     conf_op_interval = crm_get_msec(crm_element_value(conf_op_xml, "interval"));
     lrm_op_task = crm_element_value(lrm_op_xml, XML_LRM_ATTR_TASK);
     crm_element_value_int(lrm_op_xml, XML_LRM_ATTR_INTERVAL, &lrm_op_interval);
 
     if (safe_str_eq(conf_op_name, lrm_op_task) == FALSE
         || conf_op_interval != lrm_op_interval) {
         return FALSE;
     }
 
     lrm_op_id = ID(lrm_op_xml);
     last_failure_key = generate_op_key(rsc_id, "last_failure", 0);
 
     if (safe_str_eq(last_failure_key, lrm_op_id)) {
         matched = TRUE;
 
     } else {
         char *expected_op_key = generate_op_key(rsc_id, conf_op_name,
                                                 conf_op_interval);
 
         if (safe_str_eq(expected_op_key, lrm_op_id)) {
             int rc = 0;
             int target_rc = get_target_rc(lrm_op_xml);
 
             crm_element_value_int(lrm_op_xml, XML_LRM_ATTR_RC, &rc);
             if (rc != target_rc) {
                 matched = TRUE;
             }
         }
         free(expected_op_key);
     }
 
     free(last_failure_key);
     return matched;
 }
 
 static gboolean
 block_failure(node_t *node, resource_t *rsc, xmlNode *xml_op,
               pe_working_set_t *data_set)
 {
     char *xml_name = clone_strip(rsc->id);
     char *xpath = crm_strdup_printf("//primitive[@id='%s']//op[@on-fail='block']",
                                     xml_name);
     xmlXPathObject *xpathObj = xpath_search(rsc->xml, xpath);
     gboolean should_block = FALSE;
 
     free(xpath);
 
 #if 0
     /* A good idea? */
     if (rsc->container == NULL && is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
         /* In this case, stop on-fail defaults to block in unpack_operation() */
         return TRUE;
     }
 #endif
 
     if (xpathObj) {
         int max = numXpathResults(xpathObj);
         int lpc = 0;
 
         for (lpc = 0; lpc < max; lpc++) {
             xmlNode *pref = getXpathResult(xpathObj, lpc);
 
             if (xml_op) {
                 should_block = is_matched_failure(xml_name, pref, xml_op);
                 if (should_block) {
                     break;
                 }
 
             } else {
                 const char *conf_op_name = NULL;
                 int conf_op_interval = 0;
                 char *lrm_op_xpath = NULL;
                 xmlXPathObject *lrm_op_xpathObj = NULL;
 
                 conf_op_name = crm_element_value(pref, "name");
                 conf_op_interval = crm_get_msec(crm_element_value(pref, "interval"));
 
                 lrm_op_xpath = crm_strdup_printf("//node_state[@uname='%s']"
                                                "//lrm_resource[@id='%s']"
                                                "/lrm_rsc_op[@operation='%s'][@interval='%d']",
                                                node->details->uname, xml_name,
                                                conf_op_name, conf_op_interval);
                 lrm_op_xpathObj = xpath_search(data_set->input, lrm_op_xpath);
 
                 free(lrm_op_xpath);
 
                 if (lrm_op_xpathObj) {
                     int max2 = numXpathResults(lrm_op_xpathObj);
                     int lpc2 = 0;
 
                     for (lpc2 = 0; lpc2 < max2; lpc2++) {
                         xmlNode *lrm_op_xml = getXpathResult(lrm_op_xpathObj,
                                                              lpc2);
 
                         should_block = is_matched_failure(xml_name, pref,
                                                           lrm_op_xml);
                         if (should_block) {
                             break;
                         }
                     }
                 }
                 freeXpathObject(lrm_op_xpathObj);
 
                 if (should_block) {
                     break;
                 }
             }
         }
     }
 
     free(xml_name);
     freeXpathObject(xpathObj);
 
     return should_block;
 }
 
 /*!
  * \internal
  * \brief Get resource name as used in failure-related node attributes
  *
  * \param[in] rsc  Resource to check
  *
  * \return Newly allocated string containing resource's fail name
  * \note The caller is responsible for freeing the result.
  */
 static inline char *
 rsc_fail_name(resource_t *rsc)
 {
     const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);
 
     return is_set(rsc->flags, pe_rsc_unique)? strdup(name) : clone_strip(name);
 }
 
 /*!
  * \internal
  * \brief Compile regular expression to match a failure-related node attribute
  *
  * \param[in]  prefix    Attribute prefix to match
  * \param[in]  rsc_name  Resource name to match as used in failure attributes
  * \param[in]  is_legacy Whether DC uses per-resource fail counts
  * \param[in]  is_unique Whether the resource is a globally unique clone
  * \param[out] re        Where to store resulting regular expression
  *
  * \note Fail attributes are named like PREFIX-RESOURCE#OP_INTERVAL.
  *       The caller is responsible for freeing re with regfree().
  */
 static void
 generate_fail_regex(const char *prefix, const char *rsc_name,
                     gboolean is_legacy, gboolean is_unique, regex_t *re)
 {
     char *pattern;
 
     /* @COMPAT DC < 1.1.17: Fail counts used to be per-resource rather than
      * per-operation.
      */
     const char *op_pattern = (is_legacy? "" : "#.+_[0-9]+");
 
     /* Ignore instance numbers for anything other than globally unique clones.
      * Anonymous clone fail counts could contain an instance number if the
      * clone was initially unique, failed, then was converted to anonymous.
      * @COMPAT Also, before 1.1.8, anonymous clone fail counts always contained
      * clone instance numbers.
      */
     const char *instance_pattern = (is_unique? "" : "(:[0-9]+)?");
 
     pattern = crm_strdup_printf("^%s-%s%s%s$", prefix, rsc_name,
                                 instance_pattern, op_pattern);
     CRM_LOG_ASSERT(regcomp(re, pattern, REG_EXTENDED|REG_NOSUB) == 0);
     free(pattern);
 }
 
 /*!
  * \internal
  * \brief Compile regular expressions to match failure-related node attributes
  *
  * \param[in]  rsc             Resource being checked for failures
  * \param[in]  data_set        Data set (for CRM feature set version)
  * \param[out] failcount_re    Storage for regular expression for fail count
  * \param[out] lastfailure_re  Storage for regular expression for last failure
  *
  * \note The caller is responsible for freeing the expressions with regfree().
  */
 static void
 generate_fail_regexes(resource_t *rsc, pe_working_set_t *data_set,
                       regex_t *failcount_re, regex_t *lastfailure_re)
 {
     char *rsc_name = rsc_fail_name(rsc);
     const char *version = crm_element_value(data_set->input, XML_ATTR_CRM_VERSION);
     gboolean is_legacy = (compare_version(version, "3.0.13") < 0);
 
     generate_fail_regex(CRM_FAIL_COUNT_PREFIX, rsc_name, is_legacy,
                         is_set(rsc->flags, pe_rsc_unique), failcount_re);
 
     generate_fail_regex(CRM_LAST_FAILURE_PREFIX, rsc_name, is_legacy,
                         is_set(rsc->flags, pe_rsc_unique), lastfailure_re);
 
     free(rsc_name);
 }
 
 int
-get_failcount_full(node_t *node, resource_t *rsc, time_t *last_failure,
-                   bool effective, xmlNode *xml_op, pe_working_set_t *data_set)
+pe_get_failcount(node_t *node, resource_t *rsc, time_t *last_failure,
+                 uint32_t flags, xmlNode *xml_op, pe_working_set_t *data_set)
 {
     char *key = NULL;
     const char *value = NULL;
     regex_t failcount_re, lastfailure_re;
     int failcount = 0;
     time_t last = 0;
     GHashTableIter iter;
 
     generate_fail_regexes(rsc, data_set, &failcount_re, &lastfailure_re);
 
     /* Resource fail count is sum of all matching operation fail counts */
     g_hash_table_iter_init(&iter, node->details->attrs);
     while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
         if (regexec(&failcount_re, key, 0, NULL, 0) == 0) {
             failcount = merge_weights(failcount, char2score(value));
         } else if (regexec(&lastfailure_re, key, 0, NULL, 0) == 0) {
             last = QB_MAX(last, crm_int_helper(value, NULL));
         }
     }
 
     regfree(&failcount_re);
     regfree(&lastfailure_re);
 
     if ((failcount > 0) && (last > 0) && (last_failure != NULL)) {
         *last_failure = last;
     }
 
     /* If failure blocks the resource, disregard any failure timeout */
     if ((failcount > 0) && rsc->failure_timeout
         && block_failure(node, rsc, xml_op, data_set)) {
 
         pe_warn("Ignoring failure timeout %d for %s because it conflicts with on-fail=block",
                 rsc->id, rsc->failure_timeout);
         rsc->failure_timeout = 0;
     }
 
     /* If all failures have expired, ignore fail count */
-    if (effective && (failcount > 0) && (last > 0) && rsc->failure_timeout) {
+    if (is_set(flags, pe_fc_effective) && (failcount > 0) && (last > 0)
+        && rsc->failure_timeout) {
+
         time_t now = get_effective_time(data_set);
 
         if (now > (last + rsc->failure_timeout)) {
             crm_debug("Failcount for %s on %s expired after %ds",
                       rsc->id, node->details->uname, rsc->failure_timeout);
             failcount = 0;
         }
     }
 
-    if (failcount > 0) {
-        char *score = score2char(failcount);
-
-        crm_info("%s has failed %s times on %s",
-                 rsc->id, score, node->details->uname);
-        free(score);
-    }
-
-    return failcount;
-}
-
-/* If it's a resource container, get its failcount plus all the failcounts of
- * the resources within it
- */
-int
-get_failcount_all(node_t *node, resource_t *rsc, time_t *last_failure,
-                  pe_working_set_t *data_set)
-{
-    int failcount_all = 0;
-
-    failcount_all = get_failcount(node, rsc, last_failure, data_set);
-
-    if (rsc->fillers) {
+    if (is_set(flags, pe_fc_fillers) && rsc->fillers) {
         GListPtr gIter = NULL;
 
         for (gIter = rsc->fillers; gIter != NULL; gIter = gIter->next) {
             resource_t *filler = (resource_t *) gIter->data;
             time_t filler_last_failure = 0;
 
-            failcount_all += get_failcount(node, filler, &filler_last_failure,
-                                           data_set);
+            failcount += pe_get_failcount(node, filler, &filler_last_failure,
+                                          flags, xml_op, data_set);
 
             if (last_failure && filler_last_failure > *last_failure) {
                 *last_failure = filler_last_failure;
             }
         }
 
-        if (failcount_all != 0) {
-            char *score = score2char(failcount_all);
+        if (failcount > 0) {
+            char *score = score2char(failcount);
 
             crm_info("Container %s and the resources within it have failed %s times on %s",
                      rsc->id, score, node->details->uname);
             free(score);
         }
+
+    } else if (failcount > 0) {
+        char *score = score2char(failcount);
+
+        crm_info("%s has failed %s times on %s",
+                 rsc->id, score, node->details->uname);
+        free(score);
     }
 
-    return failcount_all;
+
+    return failcount;
 }
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index 6436cb84e1..95c3c83555 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -1,3453 +1,3455 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 #include <crm_internal.h>
 
 #include <glib.h>
 
 #include <crm/crm.h>
 #include <crm/services.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crm/common/util.h>
 #include <crm/pengine/rules.h>
 #include <crm/pengine/internal.h>
 #include <unpack.h>
 
 CRM_TRACE_INIT_DATA(pe_status);
 
 #define set_config_flag(data_set, option, flag) do {			\
 	const char *tmp = pe_pref(data_set->config_hash, option);	\
 	if(tmp) {							\
 	    if(crm_is_true(tmp)) {					\
 		set_bit(data_set->flags, flag);			\
 	    } else {							\
 		clear_bit(data_set->flags, flag);		\
 	    }								\
 	}								\
     } while(0)
 
 gboolean unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, xmlNode ** last_failure,
                        enum action_fail_response *failed, pe_working_set_t * data_set);
 static gboolean determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node);
 
 // Bitmask for warnings we only want to print once
 uint32_t pe_wo = 0;
 
 static gboolean
 is_dangling_container_remote_node(node_t *node)
 {
     /* we are looking for a remote-node that was supposed to be mapped to a
      * container resource, but all traces of that container have disappeared 
      * from both the config and the status section. */
     if (is_remote_node(node) &&
         node->details->remote_rsc &&
         node->details->remote_rsc->container == NULL &&
         is_set(node->details->remote_rsc->flags, pe_rsc_orphan_container_filler)) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 
 /*!
  * \brief Schedule a fence action for a node
  *
  * \param[in,out] data_set  Current working set of cluster
  * \param[in,out] node      Node to fence
  * \param[in]     reason    Text description of why fencing is needed
  */
 void
 pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason)
 {
     CRM_CHECK(node, return);
 
     /* A guest node is fenced by marking its container as failed */
     if (is_container_remote_node(node)) {
         resource_t *rsc = node->details->remote_rsc->container;
 
         if (is_set(rsc->flags, pe_rsc_failed) == FALSE) {
             if (!is_set(rsc->flags, pe_rsc_managed)) {
                 crm_notice("Not fencing guest node %s "
                            "(otherwise would because %s): "
                            "its guest resource %s is unmanaged",
                            node->details->uname, reason, rsc->id);
             } else {
                 crm_warn("Guest node %s will be fenced "
                          "(by recovering its guest resource %s): %s",
                          node->details->uname, rsc->id, reason);
 
                 /* We don't mark the node as unclean because that would prevent the
                  * node from running resources. We want to allow it to run resources
                  * in this transition if the recovery succeeds.
                  */
                 node->details->remote_requires_reset = TRUE;
                 set_bit(rsc->flags, pe_rsc_failed);
             }
         }
 
     } else if (is_dangling_container_remote_node(node)) {
         crm_info("Cleaning up dangling connection for guest node %s: "
                  "fencing was already done because %s, "
                  "and guest resource no longer exists",
                  node->details->uname, reason);
         set_bit(node->details->remote_rsc->flags, pe_rsc_failed);
 
     } else if (is_baremetal_remote_node(node)) {
         resource_t *rsc = node->details->remote_rsc;
 
         if (rsc && (!is_set(rsc->flags, pe_rsc_managed))) {
             crm_notice("Not fencing remote node %s "
                        "(otherwise would because %s): connection is unmanaged",
                        node->details->uname, reason);
         } else if(node->details->remote_requires_reset == FALSE) {
             node->details->remote_requires_reset = TRUE;
             crm_warn("Remote node %s %s: %s",
                      node->details->uname,
                      pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
                      reason);
         }
         node->details->unclean = TRUE;
         pe_fence_op(node, NULL, TRUE, reason, data_set);
 
     } else if (node->details->unclean) {
         crm_trace("Cluster node %s %s because %s",
                   node->details->uname,
                   pe_can_fence(data_set, node)? "would also be fenced" : "also is unclean",
                   reason);
 
     } else {
         crm_warn("Cluster node %s %s: %s",
                  node->details->uname,
                  pe_can_fence(data_set, node)? "will be fenced" : "is unclean",
                  reason);
         node->details->unclean = TRUE;
         pe_fence_op(node, NULL, TRUE, reason, data_set);
     }
 }
 
 // @TODO xpaths can't handle templates, rules, or id-refs
 
 // nvpair with provides or requires set to unfencing
 #define XPATH_UNFENCING_NVPAIR XML_CIB_TAG_NVPAIR                \
     "[(@" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_PROVIDES "'"    \
     "or @" XML_NVPAIR_ATTR_NAME "='" XML_RSC_ATTR_REQUIRES "') " \
     "and @" XML_NVPAIR_ATTR_VALUE "='unfencing']"
 
 // unfencing in rsc_defaults or any resource
 #define XPATH_ENABLE_UNFENCING \
     "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RESOURCES   \
     "//" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR                                               \
     "|/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_RSCCONFIG  \
     "/" XML_TAG_META_SETS "/" XPATH_UNFENCING_NVPAIR
 
 static
 void set_if_xpath(unsigned long long flag, const char *xpath,
                   pe_working_set_t *data_set)
 {
     xmlXPathObjectPtr result = NULL;
 
     if (is_not_set(data_set->flags, flag)) {
         result = xpath_search(data_set->input, xpath);
         if (result && (numXpathResults(result) > 0)) {
             set_bit(data_set->flags, flag);
         }
         freeXpathObject(result);
     }
 }
 
 gboolean
 unpack_config(xmlNode * config, pe_working_set_t * data_set)
 {
     const char *value = NULL;
     GHashTable *config_hash = crm_str_table_new();
 
     data_set->config_hash = config_hash;
 
     unpack_instance_attributes(data_set->input, config, XML_CIB_TAG_PROPSET, NULL, config_hash,
                                CIB_OPTIONS_FIRST, FALSE, data_set->now);
 
     verify_pe_options(data_set->config_hash);
 
     set_config_flag(data_set, "enable-startup-probes", pe_flag_startup_probes);
     if(is_not_set(data_set->flags, pe_flag_startup_probes)) {
         crm_info("Startup probes: disabled (dangerous)");
     }
 
     value = pe_pref(data_set->config_hash, XML_ATTR_HAVE_WATCHDOG);
     if (value && crm_is_true(value)) {
         crm_notice("Watchdog will be used via SBD if fencing is required");
         set_bit(data_set->flags, pe_flag_have_stonith_resource);
     }
 
     /* Set certain flags via xpath here, so they can be used before the relevant
      * configuration sections are unpacked.
      */
     set_if_xpath(pe_flag_enable_unfencing, XPATH_ENABLE_UNFENCING, data_set);
 
     value = pe_pref(data_set->config_hash, "stonith-timeout");
     data_set->stonith_timeout = crm_get_msec(value);
     crm_debug("STONITH timeout: %d", data_set->stonith_timeout);
 
     set_config_flag(data_set, "stonith-enabled", pe_flag_stonith_enabled);
     crm_debug("STONITH of failed nodes is %s",
               is_set(data_set->flags, pe_flag_stonith_enabled) ? "enabled" : "disabled");
 
     data_set->stonith_action = pe_pref(data_set->config_hash, "stonith-action");
     crm_trace("STONITH will %s nodes", data_set->stonith_action);
 
     set_config_flag(data_set, "concurrent-fencing", pe_flag_concurrent_fencing);
     crm_debug("Concurrent fencing is %s",
               is_set(data_set->flags, pe_flag_concurrent_fencing) ? "enabled" : "disabled");
 
     set_config_flag(data_set, "stop-all-resources", pe_flag_stop_everything);
     crm_debug("Stop all active resources: %s",
               is_set(data_set->flags, pe_flag_stop_everything) ? "true" : "false");
 
     set_config_flag(data_set, "symmetric-cluster", pe_flag_symmetric_cluster);
     if (is_set(data_set->flags, pe_flag_symmetric_cluster)) {
         crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
     }
 
     value = pe_pref(data_set->config_hash, "default-resource-stickiness");
     if (value) {
         pe_warn_once(pe_wo_default_stick,
                      "Support for 'default-resource-stickiness' cluster property"
                      " is deprecated and will be removed in a future release"
                      " (use resource-stickiness in rsc_defaults instead)");
     }
     data_set->default_resource_stickiness = char2score(value);
     crm_debug("Default stickiness: %d", data_set->default_resource_stickiness);
 
     value = pe_pref(data_set->config_hash, "no-quorum-policy");
 
     if (safe_str_eq(value, "ignore")) {
         data_set->no_quorum_policy = no_quorum_ignore;
 
     } else if (safe_str_eq(value, "freeze")) {
         data_set->no_quorum_policy = no_quorum_freeze;
 
     } else if (safe_str_eq(value, "suicide")) {
         if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
             int do_panic = 0;
 
             crm_element_value_int(data_set->input, XML_ATTR_QUORUM_PANIC,
                                   &do_panic);
             if (do_panic || is_set(data_set->flags, pe_flag_have_quorum)) {
                 data_set->no_quorum_policy = no_quorum_suicide;
             } else {
                 crm_notice("Resetting no-quorum-policy to 'stop': cluster has never had quorum");
                 data_set->no_quorum_policy = no_quorum_stop;
             }
         } else {
             crm_config_err("Resetting no-quorum-policy to 'stop': stonith is not configured");
             data_set->no_quorum_policy = no_quorum_stop;
         }
 
     } else {
         data_set->no_quorum_policy = no_quorum_stop;
     }
 
     switch (data_set->no_quorum_policy) {
         case no_quorum_freeze:
             crm_debug("On loss of CCM Quorum: Freeze resources");
             break;
         case no_quorum_stop:
             crm_debug("On loss of CCM Quorum: Stop ALL resources");
             break;
         case no_quorum_suicide:
             crm_notice("On loss of CCM Quorum: Fence all remaining nodes");
             break;
         case no_quorum_ignore:
             crm_notice("On loss of CCM Quorum: Ignore");
             break;
     }
 
     set_config_flag(data_set, "stop-orphan-resources", pe_flag_stop_rsc_orphans);
     crm_trace("Orphan resources are %s",
               is_set(data_set->flags, pe_flag_stop_rsc_orphans) ? "stopped" : "ignored");
 
     set_config_flag(data_set, "stop-orphan-actions", pe_flag_stop_action_orphans);
     crm_trace("Orphan resource actions are %s",
               is_set(data_set->flags, pe_flag_stop_action_orphans) ? "stopped" : "ignored");
 
     set_config_flag(data_set, "remove-after-stop", pe_flag_remove_after_stop);
     crm_trace("Stopped resources are removed from the status section: %s",
               is_set(data_set->flags, pe_flag_remove_after_stop) ? "true" : "false");
 
     set_config_flag(data_set, "maintenance-mode", pe_flag_maintenance_mode);
     crm_trace("Maintenance mode: %s",
               is_set(data_set->flags, pe_flag_maintenance_mode) ? "true" : "false");
 
     if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
         clear_bit(data_set->flags, pe_flag_is_managed_default);
     } else if (pe_pref(data_set->config_hash, "is-managed-default")) {
         set_config_flag(data_set, "is-managed-default", pe_flag_is_managed_default);
         pe_warn_once(pe_wo_default_isman,
                      "Support for 'is-managed-default' cluster property"
                      " is deprecated and will be removed in a future release"
                      " (use is-managed in rsc_defaults instead)");
     }
     crm_trace("By default resources are %smanaged",
               is_set(data_set->flags, pe_flag_is_managed_default) ? "" : "not ");
 
     set_config_flag(data_set, "start-failure-is-fatal", pe_flag_start_failure_fatal);
     crm_trace("Start failures are %s",
               is_set(data_set->flags,
                      pe_flag_start_failure_fatal) ? "always fatal" : "handled by failcount");
 
     if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
         set_config_flag(data_set, "startup-fencing", pe_flag_startup_fencing);
     }
     if (is_set(data_set->flags, pe_flag_startup_fencing)) {
         crm_trace("Unseen nodes will be fenced");
     } else {
         pe_warn_once(pe_wo_blind, "Blind faith: not fencing unseen nodes");
     }
 
     node_score_red = char2score(pe_pref(data_set->config_hash, "node-health-red"));
     node_score_green = char2score(pe_pref(data_set->config_hash, "node-health-green"));
     node_score_yellow = char2score(pe_pref(data_set->config_hash, "node-health-yellow"));
 
     crm_debug("Node scores: 'red' = %s, 'yellow' = %s, 'green' = %s",
              pe_pref(data_set->config_hash, "node-health-red"),
              pe_pref(data_set->config_hash, "node-health-yellow"),
              pe_pref(data_set->config_hash, "node-health-green"));
 
     data_set->placement_strategy = pe_pref(data_set->config_hash, "placement-strategy");
     crm_trace("Placement strategy: %s", data_set->placement_strategy);
 
     return TRUE;
 }
 
 static void
 destroy_digest_cache(gpointer ptr)
 {
     op_digest_cache_t *data = ptr;
 
     free_xml(data->params_all);
     free_xml(data->params_secure);
     free_xml(data->params_restart);
 
     free(data->digest_all_calc);
     free(data->digest_restart_calc);
     free(data->digest_secure_calc);
 
     free(data);
 }
 
 node_t *
 pe_create_node(const char *id, const char *uname, const char *type,
                const char *score, pe_working_set_t * data_set)
 {
     node_t *new_node = NULL;
 
     if (pe_find_node(data_set->nodes, uname) != NULL) {
         crm_config_warn("Detected multiple node entries with uname=%s"
                         " - this is rarely intended", uname);
     }
 
     new_node = calloc(1, sizeof(node_t));
     if (new_node == NULL) {
         return NULL;
     }
 
     new_node->weight = char2score(score);
     new_node->fixed = FALSE;
     new_node->details = calloc(1, sizeof(struct node_shared_s));
 
     if (new_node->details == NULL) {
         free(new_node);
         return NULL;
     }
 
     crm_trace("Creating node for entry %s/%s", uname, id);
     new_node->details->id = id;
     new_node->details->uname = uname;
     new_node->details->online = FALSE;
     new_node->details->shutdown = FALSE;
     new_node->details->rsc_discovery_enabled = TRUE;
     new_node->details->running_rsc = NULL;
     new_node->details->type = node_ping;
 
     if (safe_str_eq(type, "remote")) {
         new_node->details->type = node_remote;
         set_bit(data_set->flags, pe_flag_have_remote_nodes);
     } else if (type == NULL || safe_str_eq(type, "member")
         || safe_str_eq(type, NORMALNODE)) {
         new_node->details->type = node_member;
     }
 
     new_node->details->attrs = crm_str_table_new();
 
     if (is_remote_node(new_node)) {
         g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
                             strdup("remote"));
     } else {
         g_hash_table_insert(new_node->details->attrs, strdup(CRM_ATTR_KIND),
                             strdup("cluster"));
     }
 
     new_node->details->utilization = crm_str_table_new();
 
     new_node->details->digest_cache =
         g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
                               destroy_digest_cache);
 
     data_set->nodes = g_list_insert_sorted(data_set->nodes, new_node, sort_node_uname);
     return new_node;
 }
 
 bool
 remote_id_conflict(const char *remote_name, pe_working_set_t *data) 
 {
     bool match = FALSE;
 #if 1
     pe_find_resource(data->resources, remote_name);
 #else
     if (data->name_check == NULL) {
         data->name_check = g_hash_table_new(crm_str_hash, g_str_equal);
         for (xml_rsc = __xml_first_child(parent); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) {
             const char *id = ID(xml_rsc);
 
             /* avoiding heap allocation here because we know the duration of this hashtable allows us to */
             g_hash_table_insert(data->name_check, (char *) id, (char *) id);
         }
     }
     if (g_hash_table_lookup(data->name_check, remote_name)) {
         match = TRUE;
     }
 #endif
     if (match) {
         crm_err("Invalid remote-node name, a resource called '%s' already exists.", remote_name);
         return NULL;
     }
 
     return match;
 }
 
 
 static const char *
 expand_remote_rsc_meta(xmlNode *xml_obj, xmlNode *parent, pe_working_set_t *data)
 {
     xmlNode *attr_set = NULL;
     xmlNode *attr = NULL;
 
     const char *container_id = ID(xml_obj);
     const char *remote_name = NULL;
     const char *remote_server = NULL;
     const char *remote_port = NULL;
     const char *connect_timeout = "60s";
     const char *remote_allow_migrate=NULL;
     const char *container_managed = NULL;
 
     for (attr_set = __xml_first_child(xml_obj); attr_set != NULL; attr_set = __xml_next_element(attr_set)) {
         if (safe_str_neq((const char *)attr_set->name, XML_TAG_META_SETS)) {
             continue;
         }
 
         for (attr = __xml_first_child(attr_set); attr != NULL; attr = __xml_next_element(attr)) {
             const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
             const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
 
             if (safe_str_eq(name, XML_RSC_ATTR_REMOTE_NODE)) {
                 remote_name = value;
             } else if (safe_str_eq(name, "remote-addr")) {
                 remote_server = value;
             } else if (safe_str_eq(name, "remote-port")) {
                 remote_port = value;
             } else if (safe_str_eq(name, "remote-connect-timeout")) {
                 connect_timeout = value;
             } else if (safe_str_eq(name, "remote-allow-migrate")) {
                 remote_allow_migrate=value;
             } else if (safe_str_eq(name, XML_RSC_ATTR_MANAGED)) {
                 container_managed = value;
             }
         }
     }
 
     if (remote_name == NULL) {
         return NULL;
     }
 
     if (remote_id_conflict(remote_name, data)) {
         return NULL;
     }
 
     pe_create_remote_xml(parent, remote_name, container_id,
                          remote_allow_migrate, container_managed, "30s", "30s",
                          connect_timeout, remote_server, remote_port);
     return remote_name;
 }
 
 static void
 handle_startup_fencing(pe_working_set_t *data_set, node_t *new_node)
 {
     if ((new_node->details->type == node_remote) && (new_node->details->remote_rsc == NULL)) {
         /* Ignore fencing for remote nodes that don't have a connection resource
          * associated with them. This happens when remote node entries get left
          * in the nodes section after the connection resource is removed.
          */
         return;
     }
 
     if (is_set(data_set->flags, pe_flag_startup_fencing)) {
         // All nodes are unclean until we've seen their status entry
         new_node->details->unclean = TRUE;
 
     } else {
         // Blind faith ...
         new_node->details->unclean = FALSE;
     }
 
     /* We need to be able to determine if a node's status section
      * exists or not separate from whether the node is unclean. */
     new_node->details->unseen = TRUE;
 }
 
 gboolean
 unpack_nodes(xmlNode * xml_nodes, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
     node_t *new_node = NULL;
     const char *id = NULL;
     const char *uname = NULL;
     const char *type = NULL;
     const char *score = NULL;
 
     for (xml_obj = __xml_first_child(xml_nodes); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, TRUE)) {
             new_node = NULL;
 
             id = crm_element_value(xml_obj, XML_ATTR_ID);
             uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
             type = crm_element_value(xml_obj, XML_ATTR_TYPE);
             score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
             crm_trace("Processing node %s/%s", uname, id);
 
             if (id == NULL) {
                 crm_config_err("Must specify id tag in <node>");
                 continue;
             }
             new_node = pe_create_node(id, uname, type, score, data_set);
 
             if (new_node == NULL) {
                 return FALSE;
             }
 
 /* 		if(data_set->have_quorum == FALSE */
 /* 		   && data_set->no_quorum_policy == no_quorum_stop) { */
 /* 			/\* start shutting resources down *\/ */
 /* 			new_node->weight = -INFINITY; */
 /* 		} */
 
             handle_startup_fencing(data_set, new_node);
 
             add_node_attrs(xml_obj, new_node, FALSE, data_set);
             unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_UTILIZATION, NULL,
                                        new_node->details->utilization, NULL, FALSE, data_set->now);
 
             crm_trace("Done with node %s", crm_element_value(xml_obj, XML_ATTR_UNAME));
         }
     }
 
     if (data_set->localhost && pe_find_node(data_set->nodes, data_set->localhost) == NULL) {
         crm_info("Creating a fake local node");
         pe_create_node(data_set->localhost, data_set->localhost, NULL, 0,
                        data_set);
     }
 
     return TRUE;
 }
 
 static void
 setup_container(resource_t * rsc, pe_working_set_t * data_set)
 {
     const char *container_id = NULL;
 
     if (rsc->children) {
         GListPtr gIter = rsc->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             resource_t *child_rsc = (resource_t *) gIter->data;
 
             setup_container(child_rsc, data_set);
         }
         return;
     }
 
     container_id = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_CONTAINER);
     if (container_id && safe_str_neq(container_id, rsc->id)) {
         resource_t *container = pe_find_resource(data_set->resources, container_id);
 
         if (container) {
             rsc->container = container;
             container->fillers = g_list_append(container->fillers, rsc);
             pe_rsc_trace(rsc, "Resource %s's container is %s", rsc->id, container_id);
         } else {
             pe_err("Resource %s: Unknown resource container (%s)", rsc->id, container_id);
         }
     }
 }
 
 gboolean
 unpack_remote_nodes(xmlNode * xml_resources, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
 
     /* generate remote nodes from resource config before unpacking resources */
     for (xml_obj = __xml_first_child(xml_resources); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
         const char *new_node_id = NULL;
 
         /* first check if this is a bare metal remote node. Bare metal remote nodes
          * are defined as a resource primitive only. */
         if (xml_contains_remote_node(xml_obj)) {
             new_node_id = ID(xml_obj);
             /* The "pe_find_node" check is here to make sure we don't iterate over
              * an expanded node that has already been added to the node list. */
             if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
                 crm_trace("Found baremetal remote node %s in container resource %s", new_node_id, ID(xml_obj));
                 pe_create_node(new_node_id, new_node_id, "remote", NULL,
                                data_set);
             }
             continue;
         }
 
         /* Now check for guest remote nodes.
          * guest remote nodes are defined within a resource primitive.
          * Example1: a vm resource might be configured as a remote node.
          * Example2: a vm resource might be configured within a group to be a remote node.
          * Note: right now we only support guest remote nodes in as a standalone primitive
          * or a primitive within a group. No cloned primitives can be a guest remote node
          * right now */
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RESOURCE, TRUE)) {
             /* expands a metadata defined remote resource into the xml config
              * as an actual rsc primitive to be unpacked later. */
             new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources, data_set);
 
             if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
                 crm_trace("Found guest remote node %s in container resource %s", new_node_id, ID(xml_obj));
                 pe_create_node(new_node_id, new_node_id, "remote", NULL,
                                data_set);
             }
             continue;
 
         } else if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, TRUE)) {
             xmlNode *xml_obj2 = NULL;
             /* search through a group to see if any of the primitive contain a remote node. */
             for (xml_obj2 = __xml_first_child(xml_obj); xml_obj2 != NULL; xml_obj2 = __xml_next_element(xml_obj2)) {
 
                 new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources, data_set);
 
                 if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
                     crm_trace("Found guest remote node %s in container resource %s which is in group %s", new_node_id, ID(xml_obj2), ID(xml_obj));
                     pe_create_node(new_node_id, new_node_id, "remote", NULL,
                                    data_set);
                 }
             }
         }
     }
     return TRUE;
 }
 
 
 /* Call this after all the nodes and resources have been
  * unpacked, but before the status section is read.
  *
  * A remote node's online status is reflected by the state
  * of the remote node's connection resource. We need to link
  * the remote node to this connection resource so we can have
  * easy access to the connection resource during the PE calculations.
  */
 static void
 link_rsc2remotenode(pe_working_set_t *data_set, resource_t *new_rsc)
 {
     node_t *remote_node = NULL;
 
     if (new_rsc->is_remote_node == FALSE) {
         return;
     }
 
     if (is_set(data_set->flags, pe_flag_quick_location)) {
         /* remote_nodes and remote_resources are not linked in quick location calculations */
         return;
     }
 
     print_resource(LOG_DEBUG_3, "Linking remote-node connection resource, ", new_rsc, FALSE);
 
     remote_node = pe_find_node(data_set->nodes, new_rsc->id);
     CRM_CHECK(remote_node != NULL, return;);
 
     remote_node->details->remote_rsc = new_rsc;
     /* If this is a baremetal remote-node (no container resource
      * associated with it) then we need to handle startup fencing the same way
      * as cluster nodes. */
     if (new_rsc->container == NULL) {
         handle_startup_fencing(data_set, remote_node);
     } else {
         /* At this point we know if the remote node is a container or baremetal
          * remote node, update the #kind attribute if a container is involved */
         g_hash_table_replace(remote_node->details->attrs, strdup(CRM_ATTR_KIND),
                              strdup("container"));
     }
 }
 
 static void
 destroy_tag(gpointer data)
 {
     tag_t *tag = data;
 
     if (tag) {
         free(tag->id);
         g_list_free_full(tag->refs, free);
         free(tag);
     }
 }
 
 /*!
  * \internal
  * \brief Parse configuration XML for resource information
  *
  * \param[in]     xml_resources  Top of resource configuration XML
  * \param[in,out] data_set       Where to put resource information
  *
  * \return TRUE
  *
  * \note unpack_remote_nodes() MUST be called before this, so that the nodes can
  *       be used when common_unpack() calls resource_location()
  */
 gboolean
 unpack_resources(xmlNode * xml_resources, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
     GListPtr gIter = NULL;
 
     data_set->template_rsc_sets =
         g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
                               destroy_tag);
 
     for (xml_obj = __xml_first_child(xml_resources); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
         resource_t *new_rsc = NULL;
 
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE, TRUE)) {
             const char *template_id = ID(xml_obj);
 
             if (template_id && g_hash_table_lookup_extended(data_set->template_rsc_sets,
                                                             template_id, NULL, NULL) == FALSE) {
                 /* Record the template's ID for the knowledge of its existence anyway. */
                 g_hash_table_insert(data_set->template_rsc_sets, strdup(template_id), NULL);
             }
             continue;
         }
 
         crm_trace("Beginning unpack... <%s id=%s... >", crm_element_name(xml_obj), ID(xml_obj));
         if (common_unpack(xml_obj, &new_rsc, NULL, data_set)) {
             data_set->resources = g_list_append(data_set->resources, new_rsc);
             print_resource(LOG_DEBUG_3, "Added ", new_rsc, FALSE);
 
         } else {
             crm_config_err("Failed unpacking %s %s",
                            crm_element_name(xml_obj), crm_element_value(xml_obj, XML_ATTR_ID));
             if (new_rsc != NULL && new_rsc->fns != NULL) {
                 new_rsc->fns->free(new_rsc);
             }
         }
     }
 
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         setup_container(rsc, data_set);
         link_rsc2remotenode(data_set, rsc);
     }
 
     data_set->resources = g_list_sort(data_set->resources, sort_rsc_priority);
     if (is_set(data_set->flags, pe_flag_quick_location)) {
         /* Ignore */
 
     } else if (is_set(data_set->flags, pe_flag_stonith_enabled)
                && is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {
 
         crm_config_err("Resource start-up disabled since no STONITH resources have been defined");
         crm_config_err("Either configure some or disable STONITH with the stonith-enabled option");
         crm_config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
     }
 
     return TRUE;
 }
 
 gboolean
 unpack_tags(xmlNode * xml_tags, pe_working_set_t * data_set)
 {
     xmlNode *xml_tag = NULL;
 
     data_set->tags =
         g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, destroy_tag);
 
     for (xml_tag = __xml_first_child(xml_tags); xml_tag != NULL; xml_tag = __xml_next_element(xml_tag)) {
         xmlNode *xml_obj_ref = NULL;
         const char *tag_id = ID(xml_tag);
 
         if (crm_str_eq((const char *)xml_tag->name, XML_CIB_TAG_TAG, TRUE) == FALSE) {
             continue;
         }
 
         if (tag_id == NULL) {
             crm_config_err("Failed unpacking %s: %s should be specified",
                            crm_element_name(xml_tag), XML_ATTR_ID);
             continue;
         }
 
         for (xml_obj_ref = __xml_first_child(xml_tag); xml_obj_ref != NULL; xml_obj_ref = __xml_next_element(xml_obj_ref)) {
             const char *obj_ref = ID(xml_obj_ref);
 
             if (crm_str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, TRUE) == FALSE) {
                 continue;
             }
 
             if (obj_ref == NULL) {
                 crm_config_err("Failed unpacking %s for tag %s: %s should be specified",
                                crm_element_name(xml_obj_ref), tag_id, XML_ATTR_ID);
                 continue;
             }
 
             if (add_tag_ref(data_set->tags, tag_id, obj_ref) == FALSE) {
                 return FALSE;
             }
         }
     }
 
     return TRUE;
 }
 
 /* The ticket state section:
  * "/cib/status/tickets/ticket_state" */
 static gboolean
 unpack_ticket_state(xmlNode * xml_ticket, pe_working_set_t * data_set)
 {
     const char *ticket_id = NULL;
     const char *granted = NULL;
     const char *last_granted = NULL;
     const char *standby = NULL;
     xmlAttrPtr xIter = NULL;
 
     ticket_t *ticket = NULL;
 
     ticket_id = ID(xml_ticket);
     if (ticket_id == NULL || strlen(ticket_id) == 0) {
         return FALSE;
     }
 
     crm_trace("Processing ticket state for %s", ticket_id);
 
     ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
     if (ticket == NULL) {
         ticket = ticket_new(ticket_id, data_set);
         if (ticket == NULL) {
             return FALSE;
         }
     }
 
     for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
         const char *prop_name = (const char *)xIter->name;
         const char *prop_value = crm_element_value(xml_ticket, prop_name);
 
         if (crm_str_eq(prop_name, XML_ATTR_ID, TRUE)) {
             continue;
         }
         g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
     }
 
     granted = g_hash_table_lookup(ticket->state, "granted");
     if (granted && crm_is_true(granted)) {
         ticket->granted = TRUE;
         crm_info("We have ticket '%s'", ticket->id);
     } else {
         ticket->granted = FALSE;
         crm_info("We do not have ticket '%s'", ticket->id);
     }
 
     last_granted = g_hash_table_lookup(ticket->state, "last-granted");
     if (last_granted) {
         ticket->last_granted = crm_parse_int(last_granted, 0);
     }
 
     standby = g_hash_table_lookup(ticket->state, "standby");
     if (standby && crm_is_true(standby)) {
         ticket->standby = TRUE;
         if (ticket->granted) {
             crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
         }
     } else {
         ticket->standby = FALSE;
     }
 
     crm_trace("Done with ticket state for %s", ticket_id);
 
     return TRUE;
 }
 
 static gboolean
 unpack_tickets_state(xmlNode * xml_tickets, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
 
     for (xml_obj = __xml_first_child(xml_tickets); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, TRUE) == FALSE) {
             continue;
         }
         unpack_ticket_state(xml_obj, data_set);
     }
 
     return TRUE;
 }
 
 /* @COMPAT DC < 1.1.7: Compatibility with the deprecated ticket state section:
  * "/cib/status/tickets/instance_attributes" */
 static void
 get_ticket_state_legacy(gpointer key, gpointer value, gpointer user_data)
 {
     const char *long_key = key;
     char *state_key = NULL;
 
     const char *granted_prefix = "granted-ticket-";
     const char *last_granted_prefix = "last-granted-";
     static int granted_prefix_strlen = 0;
     static int last_granted_prefix_strlen = 0;
 
     const char *ticket_id = NULL;
     const char *is_granted = NULL;
     const char *last_granted = NULL;
     const char *sep = NULL;
 
     ticket_t *ticket = NULL;
     pe_working_set_t *data_set = user_data;
 
     if (granted_prefix_strlen == 0) {
         granted_prefix_strlen = strlen(granted_prefix);
     }
 
     if (last_granted_prefix_strlen == 0) {
         last_granted_prefix_strlen = strlen(last_granted_prefix);
     }
 
     if (strstr(long_key, granted_prefix) == long_key) {
         ticket_id = long_key + granted_prefix_strlen;
         if (strlen(ticket_id)) {
             state_key = strdup("granted");
             is_granted = value;
         }
     } else if (strstr(long_key, last_granted_prefix) == long_key) {
         ticket_id = long_key + last_granted_prefix_strlen;
         if (strlen(ticket_id)) {
             state_key = strdup("last-granted");
             last_granted = value;
         }
     } else if ((sep = strrchr(long_key, '-'))) {
         ticket_id = sep + 1;
         state_key = strndup(long_key, strlen(long_key) - strlen(sep));
     }
 
     if (ticket_id == NULL || strlen(ticket_id) == 0) {
         free(state_key);
         return;
     }
 
     if (state_key == NULL || strlen(state_key) == 0) {
         free(state_key);
         return;
     }
 
     ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
     if (ticket == NULL) {
         ticket = ticket_new(ticket_id, data_set);
         if (ticket == NULL) {
             free(state_key);
             return;
         }
     }
 
     g_hash_table_replace(ticket->state, state_key, strdup(value));
 
     if (is_granted) {
         if (crm_is_true(is_granted)) {
             ticket->granted = TRUE;
             crm_info("We have ticket '%s'", ticket->id);
         } else {
             ticket->granted = FALSE;
             crm_info("We do not have ticket '%s'", ticket->id);
         }
 
     } else if (last_granted) {
         ticket->last_granted = crm_parse_int(last_granted, 0);
     }
 }
 
 static void
 unpack_handle_remote_attrs(node_t *this_node, xmlNode *state, pe_working_set_t * data_set) 
 {
     const char *resource_discovery_enabled = NULL;
     xmlNode *attrs = NULL;
     resource_t *rsc = NULL;
     const char *shutdown = NULL;
 
     if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
         return;
     }
 
     if ((this_node == NULL) || (is_remote_node(this_node) == FALSE)) {
         return;
     }
     crm_trace("Processing remote node id=%s, uname=%s", this_node->details->id, this_node->details->uname);
 
     this_node->details->remote_maintenance =
         crm_atoi(crm_element_value(state, XML_NODE_IS_MAINTENANCE), "0");
 
     rsc = this_node->details->remote_rsc;
     if (this_node->details->remote_requires_reset == FALSE) {
         this_node->details->unclean = FALSE;
         this_node->details->unseen = FALSE;
     }
     attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
     add_node_attrs(attrs, this_node, TRUE, data_set);
 
     shutdown = pe_node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN);
     if (shutdown != NULL && safe_str_neq("0", shutdown)) {
         crm_info("Node %s is shutting down", this_node->details->uname);
         this_node->details->shutdown = TRUE;
         if (rsc) {
             rsc->next_role = RSC_ROLE_STOPPED;
         }
     }
  
     if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
         crm_info("Node %s is in standby-mode", this_node->details->uname);
         this_node->details->standby = TRUE;
     }
 
     if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance")) ||
         (rsc && !is_set(rsc->flags, pe_rsc_managed))) {
         crm_info("Node %s is in maintenance-mode", this_node->details->uname);
         this_node->details->maintenance = TRUE;
     }
 
     resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
     if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
         if (is_baremetal_remote_node(this_node) && is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
             crm_warn("ignoring %s attribute on baremetal remote node %s, disabling resource discovery requires stonith to be enabled.",
                      XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
         } else {
             /* if we're here, this is either a baremetal node and fencing is enabled,
              * or this is a container node which we don't care if fencing is enabled 
              * or not on. container nodes are 'fenced' by recovering the container resource
              * regardless of whether fencing is enabled. */
             crm_info("Node %s has resource discovery disabled", this_node->details->uname);
             this_node->details->rsc_discovery_enabled = FALSE;
         }
     }
 }
 
 static bool
 unpack_node_loop(xmlNode * status, bool fence, pe_working_set_t * data_set) 
 {
     bool changed = false;
     xmlNode *lrm_rsc = NULL;
 
     for (xmlNode *state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
         const char *id = NULL;
         const char *uname = NULL;
         node_t *this_node = NULL;
         bool process = FALSE;
 
         if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
             continue;
         }
 
         id = crm_element_value(state, XML_ATTR_ID);
         uname = crm_element_value(state, XML_ATTR_UNAME);
         this_node = pe_find_node_any(data_set->nodes, id, uname);
 
         if (this_node == NULL) {
             crm_info("Node %s is unknown", id);
             continue;
 
         } else if (this_node->details->unpacked) {
             crm_info("Node %s is already processed", id);
             continue;
 
         } else if (is_remote_node(this_node) == FALSE && is_set(data_set->flags, pe_flag_stonith_enabled)) {
             // A redundant test, but preserves the order for regression tests
             process = TRUE;
 
         } else if (is_remote_node(this_node)) {
             resource_t *rsc = this_node->details->remote_rsc;
 
             if (fence || (rsc && rsc->role == RSC_ROLE_STARTED)) {
                 determine_remote_online_status(data_set, this_node);
                 unpack_handle_remote_attrs(this_node, state, data_set);
                 process = TRUE;
             }
 
         } else if (this_node->details->online) {
             process = TRUE;
 
         } else if (fence) {
             process = TRUE;
         }
 
         if(process) {
             crm_trace("Processing lrm resource entries on %shealthy%s node: %s",
                       fence?"un":"", is_remote_node(this_node)?" remote":"",
                       this_node->details->uname);
             changed = TRUE;
             this_node->details->unpacked = TRUE;
 
             lrm_rsc = find_xml_node(state, XML_CIB_TAG_LRM, FALSE);
             lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
             unpack_lrm_resources(this_node, lrm_rsc, data_set);
         }
     }
     return changed;
 }
 
 /* remove nodes that are down, stopping */
 /* create +ve rsc_to_node constraints between resources and the nodes they are running on */
 /* anything else? */
 gboolean
 unpack_status(xmlNode * status, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     const char *uname = NULL;
 
     xmlNode *state = NULL;
     node_t *this_node = NULL;
 
     crm_trace("Beginning unpack");
 
     if (data_set->tickets == NULL) {
         data_set->tickets =
             g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, destroy_ticket);
     }
 
     for (state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
         if (crm_str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, TRUE)) {
             xmlNode *xml_tickets = state;
             GHashTable *state_hash = NULL;
 
             /* @COMPAT DC < 1.1.7: Compatibility with the deprecated ticket state section:
              * Unpack the attributes in the deprecated "/cib/status/tickets/instance_attributes" if it exists. */
             state_hash = crm_str_table_new();
 
             unpack_instance_attributes(data_set->input, xml_tickets, XML_TAG_ATTR_SETS, NULL,
                                        state_hash, NULL, TRUE, data_set->now);
 
             g_hash_table_foreach(state_hash, get_ticket_state_legacy, data_set);
 
             if (state_hash) {
                 g_hash_table_destroy(state_hash);
             }
 
             /* Unpack the new "/cib/status/tickets/ticket_state"s */
             unpack_tickets_state(xml_tickets, data_set);
         }
 
         if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE)) {
             xmlNode *attrs = NULL;
             const char *resource_discovery_enabled = NULL;
 
             id = crm_element_value(state, XML_ATTR_ID);
             uname = crm_element_value(state, XML_ATTR_UNAME);
             this_node = pe_find_node_any(data_set->nodes, id, uname);
 
             if (uname == NULL) {
                 /* error */
                 continue;
 
             } else if (this_node == NULL) {
                 crm_config_warn("Node %s in status section no longer exists", uname);
                 continue;
 
             } else if (is_remote_node(this_node)) {
                 /* online state for remote nodes is determined by the
                  * rsc state after all the unpacking is done. we do however
                  * need to mark whether or not the node has been fenced as this plays
                  * a role during unpacking cluster node resource state */
                 this_node->details->remote_was_fenced = 
                     crm_atoi(crm_element_value(state, XML_NODE_IS_FENCED), "0");
                 continue;
             }
 
             crm_trace("Processing node id=%s, uname=%s", id, uname);
 
             /* Mark the node as provisionally clean
              * - at least we have seen it in the current cluster's lifetime
              */
             this_node->details->unclean = FALSE;
             this_node->details->unseen = FALSE;
             attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
             add_node_attrs(attrs, this_node, TRUE, data_set);
 
             if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) {
                 crm_info("Node %s is in standby-mode", this_node->details->uname);
                 this_node->details->standby = TRUE;
             }
 
             if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance"))) {
                 crm_info("Node %s is in maintenance-mode", this_node->details->uname);
                 this_node->details->maintenance = TRUE;
             }
 
             resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY);
             if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
                 crm_warn("ignoring %s attribute on node %s, disabling resource discovery is not allowed on cluster nodes",
                     XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
             }
 
             crm_trace("determining node state");
             determine_online_status(state, this_node, data_set);
 
             if (is_not_set(data_set->flags, pe_flag_have_quorum)
                 && this_node->details->online
                 && (data_set->no_quorum_policy == no_quorum_suicide)) {
                 /* Everything else should flow from this automatically
                  * At least until the PE becomes able to migrate off healthy resources
                  */
                 pe_fence_node(data_set, this_node, "cluster does not have quorum");
             }
         }
     }
 
 
     while(unpack_node_loop(status, FALSE, data_set)) {
         crm_trace("Start another loop");
     }
 
     // Now catch any nodes we didn't see
     unpack_node_loop(status, is_set(data_set->flags, pe_flag_stonith_enabled), data_set);
 
     for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *this_node = gIter->data;
 
         if (this_node == NULL) {
             continue;
         } else if(is_remote_node(this_node) == FALSE) {
             continue;
         } else if(this_node->details->unpacked) {
             continue;
         }
         determine_remote_online_status(data_set, this_node);
     }
 
     return TRUE;
 }
 
 static gboolean
 determine_online_status_no_fencing(pe_working_set_t * data_set, xmlNode * node_state,
                                    node_t * this_node)
 {
     gboolean online = FALSE;
     const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
     const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
     const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
 
     if (!crm_is_true(in_cluster)) {
         crm_trace("Node is down: in_cluster=%s", crm_str(in_cluster));
 
     } else if (safe_str_eq(is_peer, ONLINESTATUS)) {
         if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {
             online = TRUE;
         } else {
             crm_debug("Node is not ready to run resources: %s", join);
         }
 
     } else if (this_node->details->expected_up == FALSE) {
         crm_trace("CRMd is down: in_cluster=%s", crm_str(in_cluster));
         crm_trace("\tis_peer=%s, join=%s, expected=%s",
                   crm_str(is_peer), crm_str(join), crm_str(exp_state));
 
     } else {
         /* mark it unclean */
         pe_fence_node(data_set, this_node, "peer is unexpectedly down");
         crm_info("\tin_cluster=%s, is_peer=%s, join=%s, expected=%s",
                  crm_str(in_cluster), crm_str(is_peer), crm_str(join), crm_str(exp_state));
     }
     return online;
 }
 
 static gboolean
 determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_state,
                                 node_t * this_node)
 {
     gboolean online = FALSE;
     gboolean do_terminate = FALSE;
     const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
     const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
     const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
     const char *terminate = pe_node_attribute_raw(this_node, "terminate");
 
 /*
   - XML_NODE_IN_CLUSTER    ::= true|false
   - XML_NODE_IS_PEER       ::= true|false|online|offline
   - XML_NODE_JOIN_STATE    ::= member|down|pending|banned
   - XML_NODE_EXPECTED      ::= member|down
 */
 
     if (crm_is_true(terminate)) {
         do_terminate = TRUE;
 
     } else if (terminate != NULL && strlen(terminate) > 0) {
         /* could be a time() value */
         char t = terminate[0];
 
         if (t != '0' && isdigit(t)) {
             do_terminate = TRUE;
         }
     }
 
     crm_trace("%s: in_cluster=%s, is_peer=%s, join=%s, expected=%s, term=%d",
               this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
               crm_str(join), crm_str(exp_state), do_terminate);
 
     online = crm_is_true(in_cluster);
     if (safe_str_eq(is_peer, ONLINESTATUS)) {
         is_peer = XML_BOOLEAN_YES;
     }
     if (exp_state == NULL) {
         exp_state = CRMD_JOINSTATE_DOWN;
     }
 
     if (this_node->details->shutdown) {
         crm_debug("%s is shutting down", this_node->details->uname);
 
         /* Slightly different criteria since we can't shut down a dead peer */
         online = crm_is_true(is_peer);
 
     } else if (in_cluster == NULL) {
         pe_fence_node(data_set, this_node, "peer has not been seen by the cluster");
 
     } else if (safe_str_eq(join, CRMD_JOINSTATE_NACK)) {
         pe_fence_node(data_set, this_node, "peer failed the pacemaker membership criteria");
 
     } else if (do_terminate == FALSE && safe_str_eq(exp_state, CRMD_JOINSTATE_DOWN)) {
 
         if (crm_is_true(in_cluster) || crm_is_true(is_peer)) {
             crm_info("- Node %s is not ready to run resources", this_node->details->uname);
             this_node->details->standby = TRUE;
             this_node->details->pending = TRUE;
 
         } else {
             crm_trace("%s is down or still coming up", this_node->details->uname);
         }
 
     } else if (do_terminate && safe_str_eq(join, CRMD_JOINSTATE_DOWN)
                && crm_is_true(in_cluster) == FALSE && crm_is_true(is_peer) == FALSE) {
         crm_info("Node %s was just shot", this_node->details->uname);
         online = FALSE;
 
     } else if (crm_is_true(in_cluster) == FALSE) {
         pe_fence_node(data_set, this_node, "peer is no longer part of the cluster");
 
     } else if (crm_is_true(is_peer) == FALSE) {
         pe_fence_node(data_set, this_node, "peer process is no longer available");
 
         /* Everything is running at this point, now check join state */
     } else if (do_terminate) {
         pe_fence_node(data_set, this_node, "termination was requested");
 
     } else if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {
         crm_info("Node %s is active", this_node->details->uname);
 
     } else if (safe_str_eq(join, CRMD_JOINSTATE_PENDING)
                || safe_str_eq(join, CRMD_JOINSTATE_DOWN)) {
         crm_info("Node %s is not ready to run resources", this_node->details->uname);
         this_node->details->standby = TRUE;
         this_node->details->pending = TRUE;
 
     } else {
         pe_fence_node(data_set, this_node, "peer was in an unknown state");
         crm_warn("%s: in-cluster=%s, is-peer=%s, join=%s, expected=%s, term=%d, shutdown=%d",
                  this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
                  crm_str(join), crm_str(exp_state), do_terminate, this_node->details->shutdown);
     }
 
     return online;
 }
 
 static gboolean
 determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node)
 {
     resource_t *rsc = this_node->details->remote_rsc;
     resource_t *container = NULL;
     pe_node_t *host = NULL;
 
     /* If there is a node state entry for a (former) Pacemaker Remote node
      * but no resource creating that node, the node's connection resource will
      * be NULL. Consider it an offline remote node in that case.
      */
     if (rsc == NULL) {
         this_node->details->online = FALSE;
         goto remote_online_done;
     }
 
     container = rsc->container;
 
     if (container && (g_list_length(rsc->running_on) == 1)) {
         host = rsc->running_on->data;
     }
 
     /* If the resource is currently started, mark it online. */
     if (rsc->role == RSC_ROLE_STARTED) {
         crm_trace("%s node %s presumed ONLINE because connection resource is started",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->online = TRUE;
     }
 
     /* consider this node shutting down if transitioning start->stop */
     if (rsc->role == RSC_ROLE_STARTED && rsc->next_role == RSC_ROLE_STOPPED) {
         crm_trace("%s node %s shutting down because connection resource is stopping",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->shutdown = TRUE;
     }
 
     /* Now check all the failure conditions. */
     if(container && is_set(container->flags, pe_rsc_failed)) {
         crm_trace("Guest node %s UNCLEAN because guest resource failed",
                   this_node->details->id);
         this_node->details->online = FALSE;
         this_node->details->remote_requires_reset = TRUE;
 
     } else if(is_set(rsc->flags, pe_rsc_failed)) {
         crm_trace("%s node %s OFFLINE because connection resource failed",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->online = FALSE;
 
     } else if (rsc->role == RSC_ROLE_STOPPED
         || (container && container->role == RSC_ROLE_STOPPED)) {
 
         crm_trace("%s node %s OFFLINE because its resource is stopped",
                   (container? "Guest" : "Remote"), this_node->details->id);
         this_node->details->online = FALSE;
         this_node->details->remote_requires_reset = FALSE;
 
     } else if (host && (host->details->online == FALSE)
                && host->details->unclean) {
         crm_trace("Guest node %s UNCLEAN because host is unclean",
                   this_node->details->id);
         this_node->details->online = FALSE;
         this_node->details->remote_requires_reset = TRUE;
     }
 
 remote_online_done:
     crm_trace("Remote node %s online=%s",
         this_node->details->id, this_node->details->online ? "TRUE" : "FALSE");
     return this_node->details->online;
 }
 
 gboolean
 determine_online_status(xmlNode * node_state, node_t * this_node, pe_working_set_t * data_set)
 {
     gboolean online = FALSE;
     const char *shutdown = NULL;
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
 
     if (this_node == NULL) {
         crm_config_err("No node to check");
         return online;
     }
 
     this_node->details->shutdown = FALSE;
     this_node->details->expected_up = FALSE;
     shutdown = pe_node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN);
 
     if (shutdown != NULL && safe_str_neq("0", shutdown)) {
         this_node->details->shutdown = TRUE;
 
     } else if (safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) {
         this_node->details->expected_up = TRUE;
     }
 
     if (this_node->details->type == node_ping) {
         this_node->details->unclean = FALSE;
         online = FALSE;         /* As far as resource management is concerned,
                                  * the node is safely offline.
                                  * Anyone caught abusing this logic will be shot
                                  */
 
     } else if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
         online = determine_online_status_no_fencing(data_set, node_state, this_node);
 
     } else {
         online = determine_online_status_fencing(data_set, node_state, this_node);
     }
 
     if (online) {
         this_node->details->online = TRUE;
 
     } else {
         /* remove node from contention */
         this_node->fixed = TRUE;
         this_node->weight = -INFINITY;
     }
 
     if (online && this_node->details->shutdown) {
         /* don't run resources here */
         this_node->fixed = TRUE;
         this_node->weight = -INFINITY;
     }
 
     if (this_node->details->type == node_ping) {
         crm_info("Node %s is not a pacemaker node", this_node->details->uname);
 
     } else if (this_node->details->unclean) {
         pe_proc_warn("Node %s is unclean", this_node->details->uname);
 
     } else if (this_node->details->online) {
         crm_info("Node %s is %s", this_node->details->uname,
                  this_node->details->shutdown ? "shutting down" :
                  this_node->details->pending ? "pending" :
                  this_node->details->standby ? "standby" :
                  this_node->details->maintenance ? "maintenance" : "online");
 
     } else {
         crm_trace("Node %s is offline", this_node->details->uname);
     }
 
     return online;
 }
 
 char *
 clone_strip(const char *last_rsc_id)
 {
     int lpc = 0;
     char *zero = NULL;
 
     CRM_CHECK(last_rsc_id != NULL, return NULL);
     lpc = strlen(last_rsc_id);
     while (--lpc > 0) {
         switch (last_rsc_id[lpc]) {
             case 0:
                 crm_err("Empty string: %s", last_rsc_id);
                 return NULL;
                 break;
             case '0':
             case '1':
             case '2':
             case '3':
             case '4':
             case '5':
             case '6':
             case '7':
             case '8':
             case '9':
                 break;
             case ':':
                 zero = calloc(1, lpc + 1);
                 memcpy(zero, last_rsc_id, lpc);
                 zero[lpc] = 0;
                 return zero;
             default:
                 goto done;
         }
     }
   done:
     zero = strdup(last_rsc_id);
     return zero;
 }
 
 char *
 clone_zero(const char *last_rsc_id)
 {
     int lpc = 0;
     char *zero = NULL;
 
     CRM_CHECK(last_rsc_id != NULL, return NULL);
     if (last_rsc_id != NULL) {
         lpc = strlen(last_rsc_id);
     }
 
     while (--lpc > 0) {
         switch (last_rsc_id[lpc]) {
             case 0:
                 return NULL;
                 break;
             case '0':
             case '1':
             case '2':
             case '3':
             case '4':
             case '5':
             case '6':
             case '7':
             case '8':
             case '9':
                 break;
             case ':':
                 zero = calloc(1, lpc + 3);
                 memcpy(zero, last_rsc_id, lpc);
                 zero[lpc] = ':';
                 zero[lpc + 1] = '0';
                 zero[lpc + 2] = 0;
                 return zero;
             default:
                 goto done;
         }
     }
   done:
     lpc = strlen(last_rsc_id);
     zero = calloc(1, lpc + 3);
     memcpy(zero, last_rsc_id, lpc);
     zero[lpc] = ':';
     zero[lpc + 1] = '0';
     zero[lpc + 2] = 0;
     crm_trace("%s -> %s", last_rsc_id, zero);
     return zero;
 }
 
 static resource_t *
 create_fake_resource(const char *rsc_id, xmlNode * rsc_entry, pe_working_set_t * data_set)
 {
     resource_t *rsc = NULL;
     xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
 
     copy_in_properties(xml_rsc, rsc_entry);
     crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
     crm_log_xml_debug(xml_rsc, "Orphan resource");
 
     if (!common_unpack(xml_rsc, &rsc, NULL, data_set)) {
         return NULL;
     }
 
     if (xml_contains_remote_node(xml_rsc)) {
         node_t *node;
 
         crm_debug("Detected orphaned remote node %s", rsc_id);
         node = pe_find_node(data_set->nodes, rsc_id);
         if (node == NULL) {
 	        node = pe_create_node(rsc_id, rsc_id, "remote", NULL, data_set);
         }
         link_rsc2remotenode(data_set, rsc);
 
         if (node) {
             crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
             node->details->shutdown = TRUE;
         }
     }
 
     if (crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER)) {
         /* This orphaned rsc needs to be mapped to a container. */
         crm_trace("Detected orphaned container filler %s", rsc_id);
         set_bit(rsc->flags, pe_rsc_orphan_container_filler);
     }
     set_bit(rsc->flags, pe_rsc_orphan);
     data_set->resources = g_list_append(data_set->resources, rsc);
     return rsc;
 }
 
 extern resource_t *create_child_clone(resource_t * rsc, int sub_id, pe_working_set_t * data_set);
 
 static resource_t *
 find_anonymous_clone(pe_working_set_t * data_set, node_t * node, resource_t * parent,
                      const char *rsc_id)
 {
     GListPtr rIter = NULL;
     resource_t *rsc = NULL;
     gboolean skip_inactive = FALSE;
 
     CRM_ASSERT(parent != NULL);
     CRM_ASSERT(pe_rsc_is_clone(parent));
     CRM_ASSERT(is_not_set(parent->flags, pe_rsc_unique));
 
     /* Find an instance active (or partially active for grouped clones) on the specified node */
     pe_rsc_trace(parent, "Looking for %s on %s in %s", rsc_id, node->details->uname, parent->id);
     for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
         GListPtr nIter = NULL;
         GListPtr locations = NULL;
         resource_t *child = rIter->data;
 
         child->fns->location(child, &locations, TRUE);
         if (locations == NULL) {
             pe_rsc_trace(child, "Resource %s, skip inactive", child->id);
             continue;
         }
 
         for (nIter = locations; nIter && rsc == NULL; nIter = nIter->next) {
             node_t *childnode = nIter->data;
 
             if (childnode->details == node->details) {
                 /* ->find_rsc() because we might be a cloned group */
                 rsc = parent->fns->find_rsc(child, rsc_id, NULL, pe_find_clone);
                 if(rsc) {
                     pe_rsc_trace(rsc, "Resource %s, active", rsc->id);
                 }
             }
 
             /* Keep this block, it means we'll do the right thing if
              * anyone toggles the unique flag to 'off'
              */
             if (rsc && rsc->running_on) {
                 crm_notice("/Anonymous/ clone %s is already running on %s",
                            parent->id, node->details->uname);
                 skip_inactive = TRUE;
                 rsc = NULL;
             }
         }
 
         g_list_free(locations);
     }
 
     /* Find an inactive instance */
     if (skip_inactive == FALSE) {
         pe_rsc_trace(parent, "Looking for %s anywhere", rsc_id);
         for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
             GListPtr locations = NULL;
             resource_t *child = rIter->data;
 
             if (is_set(child->flags, pe_rsc_block)) {
                 pe_rsc_trace(child, "Skip: blocked in stopped state");
                 continue;
             }
 
             child->fns->location(child, &locations, TRUE);
             if (locations == NULL) {
                 /* ->find_rsc() because we might be a cloned group */
                 rsc = parent->fns->find_rsc(child, rsc_id, NULL, pe_find_clone);
                 pe_rsc_trace(parent, "Resource %s, empty slot", rsc->id);
             }
             g_list_free(locations);
         }
     }
 
     if (rsc == NULL) {
         /* Create an extra orphan */
         resource_t *top = create_child_clone(parent, -1, data_set);
 
         /* ->find_rsc() because we might be a cloned group */
         rsc = top->fns->find_rsc(top, rsc_id, NULL, pe_find_clone);
         CRM_ASSERT(rsc != NULL);
 
         pe_rsc_debug(parent, "Created orphan %s for %s: %s on %s", top->id, parent->id, rsc_id,
                      node->details->uname);
     }
 
     if (safe_str_neq(rsc_id, rsc->id)) {
         pe_rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
                     rsc_id, node->details->uname, rsc->id,
                     is_set(rsc->flags, pe_rsc_orphan) ? " (ORPHAN)" : "");
     }
 
     return rsc;
 }
 
 static resource_t *
 unpack_find_resource(pe_working_set_t * data_set, node_t * node, const char *rsc_id,
                      xmlNode * rsc_entry)
 {
     resource_t *rsc = NULL;
     resource_t *parent = NULL;
 
     crm_trace("looking for %s", rsc_id);
     rsc = pe_find_resource(data_set->resources, rsc_id);
 
     /* no match */
     if (rsc == NULL) {
         /* Even when clone-max=0, we still create a single :0 orphan to match against */
         char *tmp = clone_zero(rsc_id);
         resource_t *clone0 = pe_find_resource(data_set->resources, tmp);
 
         if (clone0 && is_not_set(clone0->flags, pe_rsc_unique)) {
             rsc = clone0;
         } else {
             crm_trace("%s is not known as %s either", rsc_id, tmp);
         }
 
         parent = uber_parent(clone0);
         free(tmp);
 
         crm_trace("%s not found: %s", rsc_id, parent ? parent->id : "orphan");
 
     } else if (rsc->variant > pe_native) {
         crm_trace("%s is no longer a primitive resource, the lrm_resource entry is obsolete",
                   rsc_id);
         return NULL;
 
     } else {
         parent = uber_parent(rsc);
     }
 
     if(parent && parent->parent) {
         rsc = find_container_child(rsc_id, rsc, node);
 
     } else if (pe_rsc_is_clone(parent)) {
         if (is_not_set(parent->flags, pe_rsc_unique)) {
             char *base = clone_strip(rsc_id);
 
             rsc = find_anonymous_clone(data_set, node, parent, base);
             CRM_ASSERT(rsc != NULL);
             free(base);
         }
 
         if (rsc && safe_str_neq(rsc_id, rsc->id)) {
             free(rsc->clone_name);
             rsc->clone_name = strdup(rsc_id);
         }
     }
 
     return rsc;
 }
 
 static resource_t *
 process_orphan_resource(xmlNode * rsc_entry, node_t * node, pe_working_set_t * data_set)
 {
     resource_t *rsc = NULL;
     const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
 
     crm_debug("Detected orphan resource %s on %s", rsc_id, node->details->uname);
     rsc = create_fake_resource(rsc_id, rsc_entry, data_set);
 
     if (is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
         clear_bit(rsc->flags, pe_rsc_managed);
 
     } else {
         print_resource(LOG_DEBUG_3, "Added orphan", rsc, FALSE);
 
         CRM_CHECK(rsc != NULL, return NULL);
         resource_location(rsc, NULL, -INFINITY, "__orphan_dont_run__", data_set);
     }
     return rsc;
 }
 
 static void
 process_rsc_state(resource_t * rsc, node_t * node,
                   enum action_fail_response on_fail,
                   xmlNode * migrate_op, pe_working_set_t * data_set)
 {
     node_t *tmpnode = NULL;
     char *reason = NULL;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
                  rsc->id, role2text(rsc->role), node->details->uname, fail2text(on_fail));
 
     /* process current state */
     if (rsc->role != RSC_ROLE_UNKNOWN) {
         resource_t *iter = rsc;
 
         while (iter) {
             if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
                 node_t *n = node_copy(node);
 
                 pe_rsc_trace(rsc, "%s (aka. %s) known on %s", rsc->id, rsc->clone_name,
                              n->details->uname);
                 g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
             }
             if (is_set(iter->flags, pe_rsc_unique)) {
                 break;
             }
             iter = iter->parent;
         }
     }
 
     /* If a managed resource is believed to be running, but node is down ... */
     if (rsc->role > RSC_ROLE_STOPPED
         && node->details->online == FALSE
         && node->details->maintenance == FALSE
         && is_set(rsc->flags, pe_rsc_managed)) {
 
         gboolean should_fence = FALSE;
 
         /* If this is a guest node, fence it (regardless of whether fencing is
          * enabled, because guest node fencing is done by recovery of the
          * container resource rather than by stonithd). Mark the resource
          * we're processing as failed. When the guest comes back up, its
          * operation history in the CIB will be cleared, freeing the affected
          * resource to run again once we are sure we know its state.
          */
         if (is_container_remote_node(node)) {
             set_bit(rsc->flags, pe_rsc_failed);
             should_fence = TRUE;
 
         } else if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
             if (is_baremetal_remote_node(node) && node->details->remote_rsc
                 && is_not_set(node->details->remote_rsc->flags, pe_rsc_failed)) {
 
                 /* setting unseen = true means that fencing of the remote node will
                  * only occur if the connection resource is not going to start somewhere.
                  * This allows connection resources on a failed cluster-node to move to
                  * another node without requiring the baremetal remote nodes to be fenced
                  * as well. */
                 node->details->unseen = TRUE;
                 reason = crm_strdup_printf("%s is active there (fencing will be"
                                            " revoked if remote connection can "
                                            "be re-established elsewhere)",
                                            rsc->id);
             }
             should_fence = TRUE;
         }
 
         if (should_fence) {
             if (reason == NULL) {
                reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
             }
             pe_fence_node(data_set, node, reason);
         }
         free(reason);
     }
 
     if (node->details->unclean) {
         /* No extra processing needed
          * Also allows resources to be started again after a node is shot
          */
         on_fail = action_fail_ignore;
     }
 
     switch (on_fail) {
         case action_fail_ignore:
             /* nothing to do */
             break;
 
         case action_fail_fence:
             /* treat it as if it is still running
              * but also mark the node as unclean
              */
             reason = crm_strdup_printf("%s failed there", rsc->id);
             pe_fence_node(data_set, node, reason);
             free(reason);
             break;
 
         case action_fail_standby:
             node->details->standby = TRUE;
             node->details->standby_onfail = TRUE;
             break;
 
         case action_fail_block:
             /* is_managed == FALSE will prevent any
              * actions being sent for the resource
              */
             clear_bit(rsc->flags, pe_rsc_managed);
             set_bit(rsc->flags, pe_rsc_block);
             break;
 
         case action_fail_migrate:
             /* make sure it comes up somewhere else
              * or not at all
              */
             resource_location(rsc, node, -INFINITY, "__action_migration_auto__", data_set);
             break;
 
         case action_fail_stop:
             rsc->next_role = RSC_ROLE_STOPPED;
             break;
 
         case action_fail_recover:
             if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
                 set_bit(rsc->flags, pe_rsc_failed);
                 stop_action(rsc, node, FALSE);
             }
             break;
 
         case action_fail_restart_container:
             set_bit(rsc->flags, pe_rsc_failed);
 
             if (rsc->container) {
                 stop_action(rsc->container, node, FALSE);
             } else if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
                 stop_action(rsc, node, FALSE);
             }
             break;
 
         case action_fail_reset_remote:
             set_bit(rsc->flags, pe_rsc_failed);
             if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
                 tmpnode = NULL;
                 if (rsc->is_remote_node) {
                     tmpnode = pe_find_node(data_set->nodes, rsc->id);
                 }
                 if (tmpnode &&
                     is_baremetal_remote_node(tmpnode) &&
                     tmpnode->details->remote_was_fenced == 0) {
 
                     /* connection resource to baremetal resource failed in a way that
                      * should result in fencing the remote-node. */
                     pe_fence_node(data_set, tmpnode,
                                   "remote connection is unrecoverable");
                 }
             }
 
             /* require the stop action regardless if fencing is occurring or not. */
             if (rsc->role > RSC_ROLE_STOPPED) {
                 stop_action(rsc, node, FALSE);
             }
 
             /* if reconnect delay is in use, prevent the connection from exiting the
              * "STOPPED" role until the failure is cleared by the delay timeout. */
             if (rsc->remote_reconnect_interval) {
                 rsc->next_role = RSC_ROLE_STOPPED;
             }
             break;
     }
 
     /* ensure a remote-node connection failure forces an unclean remote-node
      * to be fenced. By setting unseen = FALSE, the remote-node failure will
      * result in a fencing operation regardless if we're going to attempt to 
      * reconnect to the remote-node in this transition or not. */
     if (is_set(rsc->flags, pe_rsc_failed) && rsc->is_remote_node) {
         tmpnode = pe_find_node(data_set->nodes, rsc->id);
         if (tmpnode && tmpnode->details->unclean) {
             tmpnode->details->unseen = FALSE;
         }
     }
 
     if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
         if (is_set(rsc->flags, pe_rsc_orphan)) {
             if (is_set(rsc->flags, pe_rsc_managed)) {
                 crm_config_warn("Detected active orphan %s running on %s",
                                 rsc->id, node->details->uname);
             } else {
                 crm_config_warn("Cluster configured not to stop active orphans."
                                 " %s must be stopped manually on %s",
                                 rsc->id, node->details->uname);
             }
         }
 
         native_add_running(rsc, node, data_set);
         if (on_fail != action_fail_ignore) {
             set_bit(rsc->flags, pe_rsc_failed);
         }
 
     } else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
         /* Only do this for older status sections that included instance numbers
          * Otherwise stopped instances will appear as orphans
          */
         pe_rsc_trace(rsc, "Resetting clone_name %s for %s (stopped)", rsc->clone_name, rsc->id);
         free(rsc->clone_name);
         rsc->clone_name = NULL;
 
     } else {
         char *key = stop_key(rsc);
         GListPtr possible_matches = find_actions(rsc->actions, key, node);
         GListPtr gIter = possible_matches;
 
         for (; gIter != NULL; gIter = gIter->next) {
             action_t *stop = (action_t *) gIter->data;
 
             stop->flags |= pe_action_optional;
         }
 
         g_list_free(possible_matches);
         free(key);
     }
 }
 
 /* create active recurring operations as optional */
 static void
 process_recurring(node_t * node, resource_t * rsc,
                   int start_index, int stop_index,
                   GListPtr sorted_op_list, pe_working_set_t * data_set)
 {
     int counter = -1;
     const char *task = NULL;
     const char *status = NULL;
     GListPtr gIter = sorted_op_list;
 
     CRM_ASSERT(rsc);
     pe_rsc_trace(rsc, "%s: Start index %d, stop index = %d", rsc->id, start_index, stop_index);
 
     for (; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         int interval = 0;
         char *key = NULL;
         const char *id = ID(rsc_op);
         const char *interval_s = NULL;
 
         counter++;
 
         if (node->details->online == FALSE) {
             pe_rsc_trace(rsc, "Skipping %s/%s: node is offline", rsc->id, node->details->uname);
             break;
 
             /* Need to check if there's a monitor for role="Stopped" */
         } else if (start_index < stop_index && counter <= stop_index) {
             pe_rsc_trace(rsc, "Skipping %s/%s: resource is not active", id, node->details->uname);
             continue;
 
         } else if (counter < start_index) {
             pe_rsc_trace(rsc, "Skipping %s/%s: old %d", id, node->details->uname, counter);
             continue;
         }
 
         interval_s = crm_element_value(rsc_op, XML_LRM_ATTR_INTERVAL);
         interval = crm_parse_int(interval_s, "0");
         if (interval == 0) {
             pe_rsc_trace(rsc, "Skipping %s/%s: non-recurring", id, node->details->uname);
             continue;
         }
 
         status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
         if (safe_str_eq(status, "-1")) {
             pe_rsc_trace(rsc, "Skipping %s/%s: status", id, node->details->uname);
             continue;
         }
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         /* create the action */
         key = generate_op_key(rsc->id, task, interval);
         pe_rsc_trace(rsc, "Creating %s/%s", key, node->details->uname);
         custom_action(rsc, key, task, node, TRUE, TRUE, data_set);
     }
 }
 
 void
 calculate_active_ops(GListPtr sorted_op_list, int *start_index, int *stop_index)
 {
     int counter = -1;
     int implied_monitor_start = -1;
     int implied_master_start = -1;
     const char *task = NULL;
     const char *status = NULL;
     GListPtr gIter = sorted_op_list;
 
     *stop_index = -1;
     *start_index = -1;
 
     for (; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         counter++;
 
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
 
         if (safe_str_eq(task, CRMD_ACTION_STOP)
             && safe_str_eq(status, "0")) {
             *stop_index = counter;
 
         } else if (safe_str_eq(task, CRMD_ACTION_START) || safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
             *start_index = counter;
 
         } else if ((implied_monitor_start <= *stop_index) && safe_str_eq(task, CRMD_ACTION_STATUS)) {
             const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);
 
             if (safe_str_eq(rc, "0") || safe_str_eq(rc, "8")) {
                 implied_monitor_start = counter;
             }
         } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE) || safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
             implied_master_start = counter;
         }
     }
 
     if (*start_index == -1) {
         if (implied_master_start != -1) {
             *start_index = implied_master_start;
         } else if (implied_monitor_start != -1) {
             *start_index = implied_monitor_start;
         }
     }
 }
 
 static resource_t *
 unpack_lrm_rsc_state(node_t * node, xmlNode * rsc_entry, pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     int stop_index = -1;
     int start_index = -1;
     enum rsc_role_e req_role = RSC_ROLE_UNKNOWN;
 
     const char *task = NULL;
     const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
 
     resource_t *rsc = NULL;
     GListPtr op_list = NULL;
     GListPtr sorted_op_list = NULL;
 
     xmlNode *migrate_op = NULL;
     xmlNode *rsc_op = NULL;
     xmlNode *last_failure = NULL;
 
     enum action_fail_response on_fail = FALSE;
     enum rsc_role_e saved_role = RSC_ROLE_UNKNOWN;
 
     crm_trace("[%s] Processing %s on %s",
               crm_element_name(rsc_entry), rsc_id, node->details->uname);
 
     /* extract operations */
     op_list = NULL;
     sorted_op_list = NULL;
 
     for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             op_list = g_list_prepend(op_list, rsc_op);
         }
     }
 
     if (op_list == NULL) {
         /* if there are no operations, there is nothing to do */
         return NULL;
     }
 
     /* find the resource */
     rsc = unpack_find_resource(data_set, node, rsc_id, rsc_entry);
     if (rsc == NULL) {
         rsc = process_orphan_resource(rsc_entry, node, data_set);
     }
     CRM_ASSERT(rsc != NULL);
 
     /* process operations */
     saved_role = rsc->role;
     on_fail = action_fail_ignore;
     rsc->role = RSC_ROLE_UNKNOWN;
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
             migrate_op = rsc_op;
         }
 
         unpack_rsc_op(rsc, node, rsc_op, &last_failure, &on_fail, data_set);
     }
 
     /* create active recurring operations as optional */
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
     process_recurring(node, rsc, start_index, stop_index, sorted_op_list, data_set);
 
     /* no need to free the contents */
     g_list_free(sorted_op_list);
 
     process_rsc_state(rsc, node, on_fail, migrate_op, data_set);
 
     if (get_target_role(rsc, &req_role)) {
         if (rsc->next_role == RSC_ROLE_UNKNOWN || req_role < rsc->next_role) {
             pe_rsc_debug(rsc, "%s: Overwriting calculated next role %s"
                          " with requested next role %s",
                          rsc->id, role2text(rsc->next_role), role2text(req_role));
             rsc->next_role = req_role;
 
         } else if (req_role > rsc->next_role) {
             pe_rsc_info(rsc, "%s: Not overwriting calculated next role %s"
                         " with requested next role %s",
                         rsc->id, role2text(rsc->next_role), role2text(req_role));
         }
     }
 
     if (saved_role > rsc->role) {
         rsc->role = saved_role;
     }
 
     return rsc;
 }
 
 static void
 handle_orphaned_container_fillers(xmlNode * lrm_rsc_list, pe_working_set_t * data_set)
 {
     xmlNode *rsc_entry = NULL;
     for (rsc_entry = __xml_first_child(lrm_rsc_list); rsc_entry != NULL;
         rsc_entry = __xml_next_element(rsc_entry)) {
 
         resource_t *rsc;
         resource_t *container;
         const char *rsc_id;
         const char *container_id;
 
         if (safe_str_neq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE)) {
             continue;
         }
 
         container_id = crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER);
         rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
         if (container_id == NULL || rsc_id == NULL) {
             continue;
         }
 
         container = pe_find_resource(data_set->resources, container_id);
         if (container == NULL) {
             continue;
         }
 
         rsc = pe_find_resource(data_set->resources, rsc_id);
         if (rsc == NULL ||
             is_set(rsc->flags, pe_rsc_orphan_container_filler) == FALSE ||
             rsc->container != NULL) {
             continue;
         }
 
         pe_rsc_trace(rsc, "Mapped orphaned rsc %s's container to  %s", rsc->id, container_id);
         rsc->container = container;
         container->fillers = g_list_append(container->fillers, rsc);
     }
 }
 
 gboolean
 unpack_lrm_resources(node_t * node, xmlNode * lrm_rsc_list, pe_working_set_t * data_set)
 {
     xmlNode *rsc_entry = NULL;
     gboolean found_orphaned_container_filler = FALSE;
 
     CRM_CHECK(node != NULL, return FALSE);
 
     crm_trace("Unpacking resources on %s", node->details->uname);
 
     for (rsc_entry = __xml_first_child(lrm_rsc_list); rsc_entry != NULL;
          rsc_entry = __xml_next_element(rsc_entry)) {
 
         if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
             resource_t *rsc = unpack_lrm_rsc_state(node, rsc_entry, data_set);
             if (!rsc) {
                 continue;
             }
             if (is_set(rsc->flags, pe_rsc_orphan_container_filler)) {
                 found_orphaned_container_filler = TRUE;
             }
         }
     }
 
     /* now that all the resource state has been unpacked for this node
      * we have to go back and map any orphaned container fillers to their
      * container resource */
     if (found_orphaned_container_filler) {
         handle_orphaned_container_fillers(lrm_rsc_list, data_set);
     }
     return TRUE;
 }
 
 static void
 set_active(resource_t * rsc)
 {
     resource_t *top = uber_parent(rsc);
 
     if (top && top->variant == pe_master) {
         rsc->role = RSC_ROLE_SLAVE;
     } else {
         rsc->role = RSC_ROLE_STARTED;
     }
 }
 
 static void
 set_node_score(gpointer key, gpointer value, gpointer user_data)
 {
     node_t *node = value;
     int *score = user_data;
 
     node->weight = *score;
 }
 
 #define STATUS_PATH_MAX 1024
 static xmlNode *
 find_lrm_op(const char *resource, const char *op, const char *node, const char *source,
             pe_working_set_t * data_set)
 {
     int offset = 0;
     char xpath[STATUS_PATH_MAX];
 
     offset += snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//node_state[@uname='%s']", node);
     offset +=
         snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//" XML_LRM_TAG_RESOURCE "[@id='%s']",
                  resource);
 
     /* Need to check against transition_magic too? */
     if (source && safe_str_eq(op, CRMD_ACTION_MIGRATE)) {
         offset +=
             snprintf(xpath + offset, STATUS_PATH_MAX - offset,
                      "/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_target='%s']", op,
                      source);
     } else if (source && safe_str_eq(op, CRMD_ACTION_MIGRATED)) {
         offset +=
             snprintf(xpath + offset, STATUS_PATH_MAX - offset,
                      "/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_source='%s']", op,
                      source);
     } else {
         offset +=
             snprintf(xpath + offset, STATUS_PATH_MAX - offset,
                      "/" XML_LRM_TAG_RSC_OP "[@operation='%s']", op);
     }
 
     CRM_LOG_ASSERT(offset > 0);
     return get_xpath_object(xpath, data_set->input, LOG_DEBUG);
 }
 
 static void
 unpack_rsc_migration(resource_t *rsc, node_t *node, xmlNode *xml_op, pe_working_set_t * data_set)
 {
 
     /*
      * The normal sequence is (now): migrate_to(Src) -> migrate_from(Tgt) -> stop(Src)
      *
      * So if a migrate_to is followed by a stop, then we don't need to care what
      * happened on the target node
      *
      * Without the stop, we need to look for a successful migrate_from.
      * This would also imply we're no longer running on the source
      *
      * Without the stop, and without a migrate_from op we make sure the resource
      * gets stopped on both source and target (assuming the target is up)
      *
      */
     int stop_id = 0;
     int task_id = 0;
     xmlNode *stop_op =
         find_lrm_op(rsc->id, CRMD_ACTION_STOP, node->details->id, NULL, data_set);
 
     if (stop_op) {
         crm_element_value_int(stop_op, XML_LRM_ATTR_CALLID, &stop_id);
     }
 
     crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &task_id);
 
     if (stop_op == NULL || stop_id < task_id) {
         int from_rc = 0, from_status = 0;
         const char *migrate_source =
             crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
         const char *migrate_target =
             crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
         node_t *target = pe_find_node(data_set->nodes, migrate_target);
         node_t *source = pe_find_node(data_set->nodes, migrate_source);
         xmlNode *migrate_from =
             find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, migrate_target, migrate_source,
                         data_set);
 
         rsc->role = RSC_ROLE_STARTED;       /* can be master? */
         if (migrate_from) {
             crm_element_value_int(migrate_from, XML_LRM_ATTR_RC, &from_rc);
             crm_element_value_int(migrate_from, XML_LRM_ATTR_OPSTATUS, &from_status);
             pe_rsc_trace(rsc, "%s op on %s exited with status=%d, rc=%d",
                          ID(migrate_from), migrate_target, from_status, from_rc);
         }
 
         if (migrate_from && from_rc == PCMK_OCF_OK
             && from_status == PCMK_LRM_OP_DONE) {
             pe_rsc_trace(rsc, "Detected dangling migration op: %s on %s", ID(xml_op),
                          migrate_source);
 
             /* all good
              * just need to arrange for the stop action to get sent
              * but _without_ affecting the target somehow
              */
             rsc->role = RSC_ROLE_STOPPED;
             rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);
 
         } else if (migrate_from) {  /* Failed */
             if (target && target->details->online) {
                 pe_rsc_trace(rsc, "Marking active on %s %p %d", migrate_target, target,
                              target->details->online);
                 native_add_running(rsc, target, data_set);
             }
 
         } else {    /* Pending or complete but erased */
             if (target && target->details->online) {
                 pe_rsc_trace(rsc, "Marking active on %s %p %d", migrate_target, target,
                              target->details->online);
 
                 native_add_running(rsc, target, data_set);
                 if (source && source->details->online) {
                     /* If we make it here we have a partial migration.  The migrate_to
                      * has completed but the migrate_from on the target has not. Hold on
                      * to the target and source on the resource. Later on if we detect that
                      * the resource is still going to run on that target, we may continue
                      * the migration */
                     rsc->partial_migration_target = target;
                     rsc->partial_migration_source = source;
                 }
             } else {
                 /* Consider it failed here - forces a restart, prevents migration */
                 set_bit(rsc->flags, pe_rsc_failed);
                 clear_bit(rsc->flags, pe_rsc_allow_migrate);
             }
         }
     }
 }
 
 static void
 unpack_rsc_migration_failure(resource_t *rsc, node_t *node, xmlNode *xml_op, pe_working_set_t * data_set) 
 {
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
 
     CRM_ASSERT(rsc);
     if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
         int stop_id = 0;
         int migrate_id = 0;
         const char *migrate_source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
         const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
         xmlNode *stop_op =
             find_lrm_op(rsc->id, CRMD_ACTION_STOP, migrate_source, NULL, data_set);
         xmlNode *migrate_op =
             find_lrm_op(rsc->id, CRMD_ACTION_MIGRATE, migrate_source, migrate_target,
                         data_set);
 
         if (stop_op) {
             crm_element_value_int(stop_op, XML_LRM_ATTR_CALLID, &stop_id);
         }
         if (migrate_op) {
             crm_element_value_int(migrate_op, XML_LRM_ATTR_CALLID, &migrate_id);
         }
 
         /* Get our state right */
         rsc->role = RSC_ROLE_STARTED;   /* can be master? */
 
         if (stop_op == NULL || stop_id < migrate_id) {
             node_t *source = pe_find_node(data_set->nodes, migrate_source);
 
             if (source && source->details->online) {
                 native_add_running(rsc, source, data_set);
             }
         }
 
     } else if (safe_str_eq(task, CRMD_ACTION_MIGRATE)) {
         int stop_id = 0;
         int migrate_id = 0;
         const char *migrate_source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
         const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
         xmlNode *stop_op =
             find_lrm_op(rsc->id, CRMD_ACTION_STOP, migrate_target, NULL, data_set);
         xmlNode *migrate_op =
             find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, migrate_target, migrate_source,
                         data_set);
 
         if (stop_op) {
             crm_element_value_int(stop_op, XML_LRM_ATTR_CALLID, &stop_id);
         }
         if (migrate_op) {
             crm_element_value_int(migrate_op, XML_LRM_ATTR_CALLID, &migrate_id);
         }
 
         /* Get our state right */
         rsc->role = RSC_ROLE_STARTED;   /* can be master? */
 
         if (stop_op == NULL || stop_id < migrate_id) {
             node_t *target = pe_find_node(data_set->nodes, migrate_target);
 
             pe_rsc_trace(rsc, "Stop: %p %d, Migrated: %p %d", stop_op, stop_id, migrate_op,
                          migrate_id);
             if (target && target->details->online) {
                 native_add_running(rsc, target, data_set);
             }
 
         } else if (migrate_op == NULL) {
             /* Make sure it gets cleaned up, the stop may pre-date the migrate_from */
             rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);
         }
     }
 }
 
 static void
 record_failed_op(xmlNode *op, node_t* node, pe_working_set_t * data_set)
 {
     xmlNode *xIter = NULL;
     const char *op_key = crm_element_value(op, XML_LRM_ATTR_TASK_KEY);
 
     if (node->details->online == FALSE) {
         return;
     }
 
     for (xIter = data_set->failed->children; xIter; xIter = xIter->next) {
         const char *key = crm_element_value(xIter, XML_LRM_ATTR_TASK_KEY);
         const char *uname = crm_element_value(xIter, XML_ATTR_UNAME);
 
         if(safe_str_eq(op_key, key) && safe_str_eq(uname, node->details->uname)) {
             crm_trace("Skipping duplicate entry %s on %s", op_key, node->details->uname);
             return;
         }
     }
 
     crm_trace("Adding entry %s on %s", op_key, node->details->uname);
     crm_xml_add(op, XML_ATTR_UNAME, node->details->uname);
     add_node_copy(data_set->failed, op);
 }
 
 static const char *get_op_key(xmlNode *xml_op)
 {
     const char *key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
     if(key == NULL) {
         key = ID(xml_op);
     }
     return key;
 }
 
 static void
 unpack_rsc_op_failure(resource_t * rsc, node_t * node, int rc, xmlNode * xml_op, xmlNode ** last_failure,
                       enum action_fail_response * on_fail, pe_working_set_t * data_set)
 {
     int interval = 0;
     bool is_probe = FALSE;
     action_t *action = NULL;
 
     const char *key = get_op_key(xml_op);
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     const char *op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
 
     CRM_ASSERT(rsc);
 
     *last_failure = xml_op;
 
     crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);
     if(interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
         is_probe = TRUE;
         pe_rsc_trace(rsc, "is a probe: %s", key);
     }
 
     if (rc != PCMK_OCF_NOT_INSTALLED || is_set(data_set->flags, pe_flag_symmetric_cluster)) {
         crm_warn("Processing failed op %s for %s on %s: %s (%d)",
                  task, rsc->id, node->details->uname, services_ocf_exitcode_str(rc),
                  rc);
 
         record_failed_op(xml_op, node, data_set);
 
     } else {
         crm_trace("Processing failed op %s for %s on %s: %s (%d)",
                  task, rsc->id, node->details->uname, services_ocf_exitcode_str(rc),
                  rc);
     }
 
     action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);
     if ((action->on_fail <= action_fail_fence && *on_fail < action->on_fail) ||
         (action->on_fail == action_fail_reset_remote && *on_fail <= action_fail_recover) ||
         (action->on_fail == action_fail_restart_container && *on_fail <= action_fail_recover) ||
         (*on_fail == action_fail_restart_container && action->on_fail >= action_fail_migrate)) {
         pe_rsc_trace(rsc, "on-fail %s -> %s for %s (%s)", fail2text(*on_fail),
                      fail2text(action->on_fail), action->uuid, key);
         *on_fail = action->on_fail;
     }
 
     if (safe_str_eq(task, CRMD_ACTION_STOP)) {
         resource_location(rsc, node, -INFINITY, "__stop_fail__", data_set);
 
     } else if (safe_str_eq(task, CRMD_ACTION_MIGRATE) || safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
         unpack_rsc_migration_failure(rsc, node, xml_op, data_set);
 
     } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
         rsc->role = RSC_ROLE_MASTER;
 
     } else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
         /*
          * staying in role=master ends up putting the PE/TE into a loop
          * setting role=slave is not dangerous because no master will be
          * promoted until the failed resource has been fully stopped
          */
         if (action->on_fail == action_fail_block) {
             rsc->role = RSC_ROLE_MASTER;
             rsc->next_role = RSC_ROLE_STOPPED;
 
         } else if(rc == PCMK_OCF_NOT_RUNNING) {
             rsc->role = RSC_ROLE_STOPPED;
 
         } else {
             crm_warn("Forcing %s to stop after a failed demote action", rsc->id);
             rsc->role = RSC_ROLE_SLAVE;
             rsc->next_role = RSC_ROLE_STOPPED;
         }
 
     } else if (compare_version("2.0", op_version) > 0 && safe_str_eq(task, CRMD_ACTION_START)) {
         crm_warn("Compatibility handling for failed op %s on %s", key, node->details->uname);
         resource_location(rsc, node, -INFINITY, "__legacy_start__", data_set);
     }
 
     if(is_probe && rc == PCMK_OCF_NOT_INSTALLED) {
         /* leave stopped */
         pe_rsc_trace(rsc, "Leaving %s stopped", rsc->id);
         rsc->role = RSC_ROLE_STOPPED;
 
     } else if (rsc->role < RSC_ROLE_STARTED) {
         pe_rsc_trace(rsc, "Setting %s active", rsc->id);
         set_active(rsc);
     }
 
     pe_rsc_trace(rsc, "Resource %s: role=%s, unclean=%s, on_fail=%s, fail_role=%s",
                  rsc->id, role2text(rsc->role),
                  node->details->unclean ? "true" : "false",
                  fail2text(action->on_fail), role2text(action->fail_role));
 
     if (action->fail_role != RSC_ROLE_STARTED && rsc->next_role < action->fail_role) {
         rsc->next_role = action->fail_role;
     }
 
     if (action->fail_role == RSC_ROLE_STOPPED) {
         int score = -INFINITY;
 
         resource_t *fail_rsc = rsc;
 
         if (fail_rsc->parent) {
             resource_t *parent = uber_parent(fail_rsc);
 
             if (pe_rsc_is_clone(parent)
                 && is_not_set(parent->flags, pe_rsc_unique)) {
                 /* for clone and master resources, if a child fails on an operation
                  * with on-fail = stop, all the resources fail.  Do this by preventing
                  * the parent from coming up again. */
                 fail_rsc = parent;
             }
         }
         crm_warn("Making sure %s doesn't come up again", fail_rsc->id);
         /* make sure it doesn't come up again */
         g_hash_table_destroy(fail_rsc->allowed_nodes);
         fail_rsc->allowed_nodes = node_hash_from_list(data_set->nodes);
         g_hash_table_foreach(fail_rsc->allowed_nodes, set_node_score, &score);
     }
 
     pe_free_action(action);
 }
 
 static int
 determine_op_status(
     resource_t *rsc, int rc, int target_rc, node_t * node, xmlNode * xml_op, enum action_fail_response * on_fail, pe_working_set_t * data_set) 
 {
     int interval = 0;
     int result = PCMK_LRM_OP_DONE;
 
     const char *key = get_op_key(xml_op);
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
 
     bool is_probe = FALSE;
 
     CRM_ASSERT(rsc);
     crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);
     if (interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
         is_probe = TRUE;
     }
 
     if (target_rc >= 0 && target_rc != rc) {
         result = PCMK_LRM_OP_ERROR;
         pe_rsc_debug(rsc, "%s on %s returned '%s' (%d) instead of the expected value: '%s' (%d)",
                      key, node->details->uname,
                      services_ocf_exitcode_str(rc), rc,
                      services_ocf_exitcode_str(target_rc), target_rc);
     }
 
     /* we could clean this up significantly except for old LRMs and CRMs that
      * didn't include target_rc and liked to remap status
      */
     switch (rc) {
         case PCMK_OCF_OK:
             if (is_probe && target_rc == 7) {
                 result = PCMK_LRM_OP_DONE;
                 pe_rsc_info(rsc, "Operation %s found resource %s active on %s",
                             task, rsc->id, node->details->uname);
 
                 /* legacy code for pre-0.6.5 operations */
             } else if (target_rc < 0 && interval > 0 && rsc->role == RSC_ROLE_MASTER) {
                 /* catch status ops that return 0 instead of 8 while they
                  *   are supposed to be in master mode
                  */
                 result = PCMK_LRM_OP_ERROR;
             }
             break;
 
         case PCMK_OCF_NOT_RUNNING:
             if (is_probe || target_rc == rc || is_not_set(rsc->flags, pe_rsc_managed)) {
                 result = PCMK_LRM_OP_DONE;
                 rsc->role = RSC_ROLE_STOPPED;
 
                 /* clear any previous failure actions */
                 *on_fail = action_fail_ignore;
                 rsc->next_role = RSC_ROLE_UNKNOWN;
 
             } else if (safe_str_neq(task, CRMD_ACTION_STOP)) {
                 result = PCMK_LRM_OP_ERROR;
             }
             break;
 
         case PCMK_OCF_RUNNING_MASTER:
             if (is_probe) {
                 result = PCMK_LRM_OP_DONE;
                 pe_rsc_info(rsc, "Operation %s found resource %s active in master mode on %s",
                             task, rsc->id, node->details->uname);
 
             } else if (target_rc == rc) {
                 /* nothing to do */
 
             } else if (target_rc >= 0) {
                 result = PCMK_LRM_OP_ERROR;
 
                 /* legacy code for pre-0.6.5 operations */
             } else if (safe_str_neq(task, CRMD_ACTION_STATUS)
                        || rsc->role != RSC_ROLE_MASTER) {
                 result = PCMK_LRM_OP_ERROR;
                 if (rsc->role != RSC_ROLE_MASTER) {
                     crm_err("%s reported %s in master mode on %s",
                             key, rsc->id, node->details->uname);
                 }
             }
             rsc->role = RSC_ROLE_MASTER;
             break;
 
         case PCMK_OCF_DEGRADED_MASTER:
         case PCMK_OCF_FAILED_MASTER:
             rsc->role = RSC_ROLE_MASTER;
             result = PCMK_LRM_OP_ERROR;
             break;
 
         case PCMK_OCF_NOT_CONFIGURED:
             result = PCMK_LRM_OP_ERROR_FATAL;
             break;
 
         case PCMK_OCF_NOT_INSTALLED:
         case PCMK_OCF_INVALID_PARAM:
         case PCMK_OCF_INSUFFICIENT_PRIV:
         case PCMK_OCF_UNIMPLEMENT_FEATURE:
             if (rc == PCMK_OCF_UNIMPLEMENT_FEATURE && interval > 0) {
                 result = PCMK_LRM_OP_NOTSUPPORTED;
                 break;
 
             } else if (pe_can_fence(data_set, node) == FALSE
                && safe_str_eq(task, CRMD_ACTION_STOP)) {
                 /* If a stop fails and we can't fence, there's nothing else we can do */
                 pe_proc_err("No further recovery can be attempted for %s: %s action failed with '%s' (%d)",
                             rsc->id, task, services_ocf_exitcode_str(rc), rc);
                 clear_bit(rsc->flags, pe_rsc_managed);
                 set_bit(rsc->flags, pe_rsc_block);
             }
             result = PCMK_LRM_OP_ERROR_HARD;
             break;
 
         default:
             if (result == PCMK_LRM_OP_DONE) {
                 crm_info("Treating %s (rc=%d) on %s as an ERROR",
                          key, rc, node->details->uname);
                 result = PCMK_LRM_OP_ERROR;
             }
     }
 
     return result;
 }
 
 static bool check_operation_expiry(resource_t *rsc, node_t *node, int rc, xmlNode *xml_op, pe_working_set_t * data_set)
 {
     bool expired = FALSE;
     time_t last_failure = 0;
     int interval = 0;
     int failure_timeout = rsc->failure_timeout;
     const char *key = get_op_key(xml_op);
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     const char *clear_reason = NULL;
 
     /* clearing recurring monitor operation failures automatically
      * needs to be carefully considered */
     if (safe_str_eq(crm_element_value(xml_op, XML_LRM_ATTR_TASK), "monitor") &&
         safe_str_neq(crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL), "0")) {
 
         /* TODO, in the future we should consider not clearing recurring monitor
          * op failures unless the last action for a resource was a "stop" action.
          * otherwise it is possible that clearing the monitor failure will result
          * in the resource being in an undeterministic state.
          *
          * For now we handle this potential undeterministic condition for remote
          * node connection resources by not clearing a recurring monitor op failure
          * until after the node has been fenced. */
 
         if (is_set(data_set->flags, pe_flag_stonith_enabled) &&
             (rsc->remote_reconnect_interval)) {
 
             node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);
             if (remote_node && remote_node->details->remote_was_fenced == 0) {
                 if (strstr(ID(xml_op), "last_failure")) {
                     crm_info("Waiting to clear monitor failure for remote node %s until fencing has occurred", rsc->id); 
                 }
                 /* disabling failure timeout for this operation because we believe
                  * fencing of the remote node should occur first. */ 
                 failure_timeout = 0;
             }
         }
     }
 
     if (failure_timeout > 0) {
         int last_run = 0;
 
         if (crm_element_value_int(xml_op, XML_RSC_OP_LAST_CHANGE, &last_run) == 0) {
             time_t now = get_effective_time(data_set);
 
             if (now > (last_run + failure_timeout)) {
                 expired = TRUE;
             }
         }
     }
 
     if (expired) {
         if (failure_timeout > 0) {
-            int fc = get_failcount_full(node, rsc, &last_failure, FALSE, xml_op, data_set);
-            if(fc) {
-                if (get_failcount_full(node, rsc, &last_failure, TRUE, xml_op, data_set) == 0) {
-                    clear_reason = "it expired";
+            if (pe_get_failcount(node, rsc, &last_failure, pe_fc_default,
+                                 xml_op, data_set)) {
 
+                if (pe_get_failcount(node, rsc, &last_failure, pe_fc_effective,
+                                     xml_op, data_set) == 0) {
+                    clear_reason = "it expired";
                 } else {
                     expired = FALSE;
                 }
+
             } else if (rsc->remote_reconnect_interval && strstr(ID(xml_op), "last_failure")) {
                 /* always clear last failure when reconnect interval is set */
                 clear_reason = "reconnect interval is set";
             }
         }
 
     } else if (strstr(ID(xml_op), "last_failure") &&
                ((strcmp(task, "start") == 0) || (strcmp(task, "monitor") == 0))) {
 
         op_digest_cache_t *digest_data = NULL;
 
         digest_data = rsc_action_digest_cmp(rsc, xml_op, node, data_set);
 
         if (digest_data->rc == RSC_DIGEST_UNKNOWN) {
             crm_trace("rsc op %s/%s on node %s does not have a op digest to compare against", rsc->id,
                       key, node->details->id);
         } else if (digest_data->rc != RSC_DIGEST_MATCH) {
             clear_reason = "resource parameters have changed";
         }
     }
 
     if (clear_reason != NULL) {
         char *key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
         action_t *clear_op = custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT,
                                            node, FALSE, TRUE, data_set);
 
         add_hash_param(clear_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
 
         crm_notice("Clearing failure of %s on %s because %s " CRM_XS " %s",
                    rsc->id, node->details->uname, clear_reason, clear_op->uuid);
     }
 
     crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);
     if(expired && interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
         switch(rc) {
             case PCMK_OCF_OK:
             case PCMK_OCF_NOT_RUNNING:
             case PCMK_OCF_RUNNING_MASTER:
             case PCMK_OCF_DEGRADED:
             case PCMK_OCF_DEGRADED_MASTER:
                 /* Don't expire probes that return these values */ 
                 expired = FALSE;
                 break;
         }
     }
     
     return expired;
 }
 
 int get_target_rc(xmlNode *xml_op)
 {
     int dummy = 0;
     int target_rc = 0;
     char *dummy_string = NULL;
     const char *key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);
     if (key == NULL) {
         return -1;
     }
 
     decode_transition_key(key, &dummy_string, &dummy, &dummy, &target_rc);
     free(dummy_string);
 
     return target_rc;
 }
 
 static enum action_fail_response
 get_action_on_fail(resource_t *rsc, const char *key, const char *task, pe_working_set_t * data_set) 
 {
     int result = action_fail_recover;
     action_t *action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);
 
     result = action->on_fail;
     pe_free_action(action);
 
     return result;
 }
 
 static void
 update_resource_state(resource_t * rsc, node_t * node, xmlNode * xml_op, const char * task, int rc,
                       xmlNode * last_failure, enum action_fail_response * on_fail, pe_working_set_t * data_set)
 {
     gboolean clear_past_failure = FALSE;
 
     CRM_ASSERT(rsc);
     CRM_ASSERT(xml_op);
 
     if (rc == PCMK_OCF_NOT_RUNNING) {
         clear_past_failure = TRUE;
 
     } else if (rc == PCMK_OCF_NOT_INSTALLED) {
         rsc->role = RSC_ROLE_STOPPED;
 
     } else if (safe_str_eq(task, CRMD_ACTION_STATUS)) {
         if (last_failure) {
             const char *op_key = get_op_key(xml_op);
             const char *last_failure_key = get_op_key(last_failure);
 
             if (safe_str_eq(op_key, last_failure_key)) {
                 clear_past_failure = TRUE;
             }
         }
 
         if (rsc->role < RSC_ROLE_STARTED) {
             set_active(rsc);
         }
 
     } else if (safe_str_eq(task, CRMD_ACTION_START)) {
         rsc->role = RSC_ROLE_STARTED;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_STOP)) {
         rsc->role = RSC_ROLE_STOPPED;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
         rsc->role = RSC_ROLE_MASTER;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
         /* Demote from Master does not clear an error */
         rsc->role = RSC_ROLE_SLAVE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
         rsc->role = RSC_ROLE_STARTED;
         clear_past_failure = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_MIGRATE)) {
         unpack_rsc_migration(rsc, node, xml_op, data_set);
 
     } else if (rsc->role < RSC_ROLE_STARTED) {
         pe_rsc_trace(rsc, "%s active on %s", rsc->id, node->details->uname);
         set_active(rsc);
     }
 
     /* clear any previous failure actions */
     if (clear_past_failure) {
         switch (*on_fail) {
             case action_fail_stop:
             case action_fail_fence:
             case action_fail_migrate:
             case action_fail_standby:
                 pe_rsc_trace(rsc, "%s.%s is not cleared by a completed stop",
                              rsc->id, fail2text(*on_fail));
                 break;
 
             case action_fail_block:
             case action_fail_ignore:
             case action_fail_recover:
             case action_fail_restart_container:
                 *on_fail = action_fail_ignore;
                 rsc->next_role = RSC_ROLE_UNKNOWN;
                 break;
             case action_fail_reset_remote:
                 if (rsc->remote_reconnect_interval == 0) {
                     /* when reconnect delay is not in use, the connection is allowed
                      * to start again after the remote node is fenced and completely
                      * stopped. Otherwise, with reconnect delay we wait for the failure
                      * to be cleared entirely before reconnected can be attempted. */ 
                     *on_fail = action_fail_ignore;
                     rsc->next_role = RSC_ROLE_UNKNOWN;
                 }
                 break;
         }
     }
 }
 
 
 gboolean
 unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, xmlNode ** last_failure,
               enum action_fail_response * on_fail, pe_working_set_t * data_set)
 {
     int task_id = 0;
 
     const char *key = NULL;
     const char *task = NULL;
     const char *task_key = NULL;
 
     int rc = 0;
     int status = PCMK_LRM_OP_PENDING-1;
     int target_rc = get_target_rc(xml_op);
     int interval = 0;
 
     gboolean expired = FALSE;
     resource_t *parent = rsc;
     enum action_fail_response failure_strategy = action_fail_recover;
 
     CRM_CHECK(rsc != NULL, return FALSE);
     CRM_CHECK(node != NULL, return FALSE);
     CRM_CHECK(xml_op != NULL, return FALSE);
 
     task_key = get_op_key(xml_op);
 
     task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);
 
     crm_element_value_int(xml_op, XML_LRM_ATTR_RC, &rc);
     crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &task_id);
     crm_element_value_int(xml_op, XML_LRM_ATTR_OPSTATUS, &status);
     crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);
 
     CRM_CHECK(task != NULL, return FALSE);
     CRM_CHECK(status <= PCMK_LRM_OP_NOT_INSTALLED, return FALSE);
     CRM_CHECK(status >= PCMK_LRM_OP_PENDING, return FALSE);
 
     if (safe_str_eq(task, CRMD_ACTION_NOTIFY) ||
         safe_str_eq(task, CRMD_ACTION_METADATA)) {
         /* safe to ignore these */
         return TRUE;
     }
 
     if (is_not_set(rsc->flags, pe_rsc_unique)) {
         parent = uber_parent(rsc);
     }
 
     pe_rsc_trace(rsc, "Unpacking task %s/%s (call_id=%d, status=%d, rc=%d) on %s (role=%s)",
                  task_key, task, task_id, status, rc, node->details->uname, role2text(rsc->role));
 
     if (node->details->unclean) {
         pe_rsc_trace(rsc, "Node %s (where %s is running) is unclean."
                      " Further action depends on the value of the stop's on-fail attribute",
                      node->details->uname, rsc->id);
     }
 
     if (status == PCMK_LRM_OP_ERROR) {
         /* Older versions set this if rc != 0 but it's up to us to decide */
         status = PCMK_LRM_OP_DONE;
     }
 
     if(status != PCMK_LRM_OP_NOT_INSTALLED) {
         expired = check_operation_expiry(rsc, node, rc, xml_op, data_set);
     }
 
     /* Degraded results are informational only, re-map them to their error-free equivalents */
     if (rc == PCMK_OCF_DEGRADED && safe_str_eq(task, CRMD_ACTION_STATUS)) {
         rc = PCMK_OCF_OK;
 
         /* Add them to the failed list to highlight them for the user */
         if ((node->details->shutdown == FALSE) || (node->details->online == TRUE)) {
             crm_trace("Remapping %d to %d", PCMK_OCF_DEGRADED, PCMK_OCF_OK);
             record_failed_op(xml_op, node, data_set);
         }
 
     } else if (rc == PCMK_OCF_DEGRADED_MASTER && safe_str_eq(task, CRMD_ACTION_STATUS)) {
         rc = PCMK_OCF_RUNNING_MASTER;
 
         /* Add them to the failed list to highlight them for the user */
         if ((node->details->shutdown == FALSE) || (node->details->online == TRUE)) {
             crm_trace("Remapping %d to %d", PCMK_OCF_DEGRADED_MASTER, PCMK_OCF_RUNNING_MASTER);
             record_failed_op(xml_op, node, data_set);
         }
     }
 
     if (expired && target_rc != rc) {
         const char *magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);
 
         pe_rsc_debug(rsc, "Expired operation '%s' on %s returned '%s' (%d) instead of the expected value: '%s' (%d)",
                      key, node->details->uname,
                      services_ocf_exitcode_str(rc), rc,
                      services_ocf_exitcode_str(target_rc), target_rc);
 
         if(interval == 0) {
             crm_notice("Ignoring expired calculated failure %s (rc=%d, magic=%s) on %s",
                        task_key, rc, magic, node->details->uname);
             goto done;
 
         } else if(node->details->online && node->details->unclean == FALSE) {
             crm_notice("Re-initiated expired calculated failure %s (rc=%d, magic=%s) on %s",
                        task_key, rc, magic, node->details->uname);
             /* This is SO horrible, but we don't have access to CancelXmlOp() yet */
             crm_xml_add(xml_op, XML_LRM_ATTR_RESTART_DIGEST, "calculated-failure-timeout");
             goto done;
         }
     }
 
     if(status == PCMK_LRM_OP_DONE || status == PCMK_LRM_OP_ERROR) {
         status = determine_op_status(rsc, rc, target_rc, node, xml_op, on_fail, data_set);
     }
 
     pe_rsc_trace(rsc, "Handling status: %d", status);
     switch (status) {
         case PCMK_LRM_OP_CANCELLED:
             /* do nothing?? */
             pe_err("Don't know what to do for cancelled ops yet");
             break;
 
         case PCMK_LRM_OP_PENDING:
             if (safe_str_eq(task, CRMD_ACTION_START)) {
                 set_bit(rsc->flags, pe_rsc_start_pending);
                 set_active(rsc);
 
             } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
                 rsc->role = RSC_ROLE_MASTER;
 
             } else if (safe_str_eq(task, CRMD_ACTION_MIGRATE) && node->details->unclean) {
                 /* If a pending migrate_to action is out on a unclean node,
                  * we have to force the stop action on the target. */
                 const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
                 node_t *target = pe_find_node(data_set->nodes, migrate_target);
                 if (target) {
                     stop_action(rsc, target, FALSE);
                 }
             }
 
             if (rsc->pending_task == NULL) {
                 if (safe_str_eq(task, CRMD_ACTION_STATUS) && interval == 0) {
                     /* Pending probes are not printed, even if pending
                      * operations are requested. If someone ever requests that
                      * behavior, uncomment this and the corresponding part of
                      * native.c:native_pending_task().
                      */
                     /*rsc->pending_task = strdup("probe");*/
 
                 } else {
                     rsc->pending_task = strdup(task);
                 }
             }
             break;
 
         case PCMK_LRM_OP_DONE:
             pe_rsc_trace(rsc, "%s/%s completed on %s", rsc->id, task, node->details->uname);
             update_resource_state(rsc, node, xml_op, task, rc, *last_failure, on_fail, data_set);
             break;
 
         case PCMK_LRM_OP_NOT_INSTALLED:
             failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
             if (failure_strategy == action_fail_ignore) {
                 crm_warn("Cannot ignore failed %s (status=%d, rc=%d) on %s: "
                          "Resource agent doesn't exist",
                          task_key, status, rc, node->details->uname);
                 /* Also for printing it as "FAILED" by marking it as pe_rsc_failed later */
                 *on_fail = action_fail_migrate;
             }
             resource_location(parent, node, -INFINITY, "hard-error", data_set);
             unpack_rsc_op_failure(rsc, node, rc, xml_op, last_failure, on_fail, data_set);
             break;
 
         case PCMK_LRM_OP_ERROR:
         case PCMK_LRM_OP_ERROR_HARD:
         case PCMK_LRM_OP_ERROR_FATAL:
         case PCMK_LRM_OP_TIMEOUT:
         case PCMK_LRM_OP_NOTSUPPORTED:
 
             failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
             if ((failure_strategy == action_fail_ignore)
                 || (failure_strategy == action_fail_restart_container
                     && safe_str_eq(task, CRMD_ACTION_STOP))) {
 
                 crm_warn("Pretending the failure of %s (rc=%d) on %s succeeded",
                          task_key, rc, node->details->uname);
 
                 update_resource_state(rsc, node, xml_op, task, target_rc, *last_failure, on_fail, data_set);
                 crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
                 set_bit(rsc->flags, pe_rsc_failure_ignored);
 
                 record_failed_op(xml_op, node, data_set);
 
                 if (failure_strategy == action_fail_restart_container && *on_fail <= action_fail_recover) {
                     *on_fail = failure_strategy;
                 }
 
             } else {
                 unpack_rsc_op_failure(rsc, node, rc, xml_op, last_failure, on_fail, data_set);
 
                 if(status == PCMK_LRM_OP_ERROR_HARD) {
                     do_crm_log(rc != PCMK_OCF_NOT_INSTALLED?LOG_ERR:LOG_NOTICE,
                                "Preventing %s from re-starting on %s: operation %s failed '%s' (%d)",
                                parent->id, node->details->uname,
                                task, services_ocf_exitcode_str(rc), rc);
 
                     resource_location(parent, node, -INFINITY, "hard-error", data_set);
 
                 } else if(status == PCMK_LRM_OP_ERROR_FATAL) {
                     crm_err("Preventing %s from re-starting anywhere: operation %s failed '%s' (%d)",
                             parent->id, task, services_ocf_exitcode_str(rc), rc);
 
                     resource_location(parent, NULL, -INFINITY, "fatal-error", data_set);
                 }
             }
             break;
     }
 
   done:
     pe_rsc_trace(rsc, "Resource %s after %s: role=%s, next=%s", rsc->id, task, role2text(rsc->role), role2text(rsc->next_role));
     return TRUE;
 }
 
 gboolean
 add_node_attrs(xmlNode * xml_obj, node_t * node, gboolean overwrite, pe_working_set_t * data_set)
 {
     const char *cluster_name = NULL;
 
     g_hash_table_insert(node->details->attrs,
                         strdup(CRM_ATTR_UNAME), strdup(node->details->uname));
 
     g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_ID),
                         strdup(node->details->id));
     if (safe_str_eq(node->details->id, data_set->dc_uuid)) {
         data_set->dc_node = node;
         node->details->is_dc = TRUE;
         g_hash_table_insert(node->details->attrs,
                             strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_TRUE));
     } else {
         g_hash_table_insert(node->details->attrs,
                             strdup(CRM_ATTR_IS_DC), strdup(XML_BOOLEAN_FALSE));
     }
 
     cluster_name = g_hash_table_lookup(data_set->config_hash, "cluster-name");
     if (cluster_name) {
         g_hash_table_insert(node->details->attrs, strdup(CRM_ATTR_CLUSTER_NAME),
                             strdup(cluster_name));
     }
 
     unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_ATTR_SETS, NULL,
                                node->details->attrs, NULL, overwrite, data_set->now);
 
     if (pe_node_attribute_raw(node, CRM_ATTR_SITE_NAME) == NULL) {
         const char *site_name = pe_node_attribute_raw(node, "site-name");
 
         if (site_name) {
             g_hash_table_insert(node->details->attrs,
                                 strdup(CRM_ATTR_SITE_NAME),
                                 strdup(site_name));
 
         } else if (cluster_name) {
             /* Default to cluster-name if unset */
             g_hash_table_insert(node->details->attrs,
                                 strdup(CRM_ATTR_SITE_NAME),
                                 strdup(cluster_name));
         }
     }
     return TRUE;
 }
 
 static GListPtr
 extract_operations(const char *node, const char *rsc, xmlNode * rsc_entry, gboolean active_filter)
 {
     int counter = -1;
     int stop_index = -1;
     int start_index = -1;
 
     xmlNode *rsc_op = NULL;
 
     GListPtr gIter = NULL;
     GListPtr op_list = NULL;
     GListPtr sorted_op_list = NULL;
 
     /* extract operations */
     op_list = NULL;
     sorted_op_list = NULL;
 
     for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             crm_xml_add(rsc_op, "resource", rsc);
             crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
             op_list = g_list_prepend(op_list, rsc_op);
         }
     }
 
     if (op_list == NULL) {
         /* if there are no operations, there is nothing to do */
         return NULL;
     }
 
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 
     /* create active recurring operations as optional */
     if (active_filter == FALSE) {
         return sorted_op_list;
     }
 
     op_list = NULL;
 
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         counter++;
 
         if (start_index < stop_index) {
             crm_trace("Skipping %s: not active", ID(rsc_entry));
             break;
 
         } else if (counter < start_index) {
             crm_trace("Skipping %s: old", ID(rsc_op));
             continue;
         }
         op_list = g_list_append(op_list, rsc_op);
     }
 
     g_list_free(sorted_op_list);
     return op_list;
 }
 
 GListPtr
 find_operations(const char *rsc, const char *node, gboolean active_filter,
                 pe_working_set_t * data_set)
 {
     GListPtr output = NULL;
     GListPtr intermediate = NULL;
 
     xmlNode *tmp = NULL;
     xmlNode *status = find_xml_node(data_set->input, XML_CIB_TAG_STATUS, TRUE);
 
     node_t *this_node = NULL;
 
     xmlNode *node_state = NULL;
 
     for (node_state = __xml_first_child(status); node_state != NULL;
          node_state = __xml_next_element(node_state)) {
 
         if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
             const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);
 
             if (node != NULL && safe_str_neq(uname, node)) {
                 continue;
             }
 
             this_node = pe_find_node(data_set->nodes, uname);
             if(this_node == NULL) {
                 CRM_LOG_ASSERT(this_node != NULL);
                 continue;
 
             } else if (is_remote_node(this_node)) {
                 determine_remote_online_status(data_set, this_node);
 
             } else {
                 determine_online_status(node_state, this_node, data_set);
             }
 
             if (this_node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) {
                 /* offline nodes run no resources...
                  * unless stonith is enabled in which case we need to
                  *   make sure rsc start events happen after the stonith
                  */
                 xmlNode *lrm_rsc = NULL;
 
                 tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
                 tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);
 
                 for (lrm_rsc = __xml_first_child(tmp); lrm_rsc != NULL;
                      lrm_rsc = __xml_next_element(lrm_rsc)) {
                     if (crm_str_eq((const char *)lrm_rsc->name, XML_LRM_TAG_RESOURCE, TRUE)) {
 
                         const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID);
 
                         if (rsc != NULL && safe_str_neq(rsc_id, rsc)) {
                             continue;
                         }
 
                         intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
                         output = g_list_concat(output, intermediate);
                     }
                 }
             }
         }
     }
 
     return output;
 }
diff --git a/pengine/allocate.c b/pengine/allocate.c
index 33f4ff22e0..0c7d33d822 100644
--- a/pengine/allocate.c
+++ b/pengine/allocate.c
@@ -1,2548 +1,2560 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <glib.h>
 
 #include <crm/pengine/status.h>
 #include <pengine.h>
 #include <allocate.h>
 #include <utils.h>
 
 CRM_TRACE_INIT_DATA(pe_allocate);
 
 void set_alloc_actions(pe_working_set_t * data_set);
 extern void ReloadRsc(resource_t * rsc, node_t *node, pe_working_set_t * data_set);
 extern gboolean DeleteRsc(resource_t * rsc, node_t * node, gboolean optional, pe_working_set_t * data_set);
 static void apply_remote_node_ordering(pe_working_set_t *data_set);
 static enum remote_connection_state get_remote_node_state(pe_node_t *node);
 
 enum remote_connection_state {
     remote_state_unknown = 0,
     remote_state_alive = 1,
     remote_state_resting = 2,
     remote_state_failed = 3,
     remote_state_stopped = 4
 };
 
 resource_alloc_functions_t resource_class_alloc_functions[] = {
     {
      native_merge_weights,
      native_color,
      native_create_actions,
      native_create_probe,
      native_internal_constraints,
      native_rsc_colocation_lh,
      native_rsc_colocation_rh,
      native_rsc_location,
      native_action_flags,
      native_update_actions,
      native_expand,
      native_append_meta,
      },
     {
      group_merge_weights,
      group_color,
      group_create_actions,
      native_create_probe,
      group_internal_constraints,
      group_rsc_colocation_lh,
      group_rsc_colocation_rh,
      group_rsc_location,
      group_action_flags,
      group_update_actions,
      group_expand,
      group_append_meta,
      },
     {
      clone_merge_weights,
      clone_color,
      clone_create_actions,
      clone_create_probe,
      clone_internal_constraints,
      clone_rsc_colocation_lh,
      clone_rsc_colocation_rh,
      clone_rsc_location,
      clone_action_flags,
      container_update_actions,
      clone_expand,
      clone_append_meta,
      },
     {
      master_merge_weights,
      master_color,
      master_create_actions,
      clone_create_probe,
      master_internal_constraints,
      clone_rsc_colocation_lh,
      master_rsc_colocation_rh,
      clone_rsc_location,
      clone_action_flags,
      container_update_actions,
      clone_expand,
      master_append_meta,
      },
     {
      container_merge_weights,
      container_color,
      container_create_actions,
      container_create_probe,
      container_internal_constraints,
      container_rsc_colocation_lh,
      container_rsc_colocation_rh,
      container_rsc_location,
      container_action_flags,
      container_update_actions,
      container_expand,
      container_append_meta,
      }
 };
 
 gboolean
 update_action_flags(action_t * action, enum pe_action_flags flags, const char *source, int line)
 {
     static unsigned long calls = 0;
     gboolean changed = FALSE;
     gboolean clear = is_set(flags, pe_action_clear);
     enum pe_action_flags last = action->flags;
 
     if (clear) {
         action->flags = crm_clear_bit(source, line, action->uuid, action->flags, flags);
     } else {
         action->flags = crm_set_bit(source, line, action->uuid, action->flags, flags);
     }
 
     if (last != action->flags) {
         calls++;
         changed = TRUE;
         /* Useful for tracking down _who_ changed a specific flag */
         /* CRM_ASSERT(calls != 534); */
         clear_bit(flags, pe_action_clear);
         crm_trace("%s on %s: %sset flags 0x%.6x (was 0x%.6x, now 0x%.6x, %lu, %s)",
                   action->uuid, action->node ? action->node->details->uname : "[none]",
                   clear ? "un-" : "", flags, last, action->flags, calls, source);
     }
 
     return changed;
 }
 
 static gboolean
 check_rsc_parameters(resource_t * rsc, node_t * node, xmlNode * rsc_entry,
                      gboolean active_here, pe_working_set_t * data_set)
 {
     int attr_lpc = 0;
     gboolean force_restart = FALSE;
     gboolean delete_resource = FALSE;
     gboolean changed = FALSE;
 
     const char *value = NULL;
     const char *old_value = NULL;
 
     const char *attr_list[] = {
         XML_ATTR_TYPE,
         XML_AGENT_ATTR_CLASS,
         XML_AGENT_ATTR_PROVIDER
     };
 
     for (; attr_lpc < DIMOF(attr_list); attr_lpc++) {
         value = crm_element_value(rsc->xml, attr_list[attr_lpc]);
         old_value = crm_element_value(rsc_entry, attr_list[attr_lpc]);
         if (value == old_value  /* i.e. NULL */
             || crm_str_eq(value, old_value, TRUE)) {
             continue;
         }
 
         changed = TRUE;
         trigger_unfencing(rsc, node, "Device definition changed", NULL, data_set);
         if (active_here) {
             force_restart = TRUE;
             crm_notice("Forcing restart of %s on %s, %s changed: %s -> %s",
                        rsc->id, node->details->uname, attr_list[attr_lpc],
                        crm_str(old_value), crm_str(value));
         }
     }
     if (force_restart) {
         /* make sure the restart happens */
         stop_action(rsc, node, FALSE);
         set_bit(rsc->flags, pe_rsc_start_pending);
         delete_resource = TRUE;
 
     } else if (changed) {
         delete_resource = TRUE;
     }
     return delete_resource;
 }
 
 static void
 CancelXmlOp(resource_t * rsc, xmlNode * xml_op, node_t * active_node,
             const char *reason, pe_working_set_t * data_set)
 {
     int interval = 0;
     action_t *cancel = NULL;
 
     char *key = NULL;
     const char *task = NULL;
     const char *call_id = NULL;
     const char *interval_s = NULL;
 
     CRM_CHECK(xml_op != NULL, return);
     CRM_CHECK(active_node != NULL, return);
 
     task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
     interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
 
     interval = crm_parse_int(interval_s, "0");
 
     /* we need to reconstruct the key because of the way we used to construct resource IDs */
     key = generate_op_key(rsc->id, task, interval);
 
     crm_info("Action %s on %s will be stopped: %s",
              key, active_node->details->uname, reason ? reason : "unknown");
 
     /* TODO: This looks highly dangerous if we ever try to schedule 'key' too */
     cancel = custom_action(rsc, strdup(key), RSC_CANCEL, active_node, FALSE, TRUE, data_set);
 
     free(cancel->task);
     free(cancel->cancel_task);
     cancel->task = strdup(RSC_CANCEL);
     cancel->cancel_task = strdup(task);
 
     add_hash_param(cancel->meta, XML_LRM_ATTR_TASK, task);
     add_hash_param(cancel->meta, XML_LRM_ATTR_CALLID, call_id);
     add_hash_param(cancel->meta, XML_LRM_ATTR_INTERVAL, interval_s);
 
     custom_action_order(rsc, stop_key(rsc), NULL, rsc, NULL, cancel, pe_order_optional, data_set);
     free(key);
     key = NULL;
 }
 
 static gboolean
 check_action_definition(resource_t * rsc, node_t * active_node, xmlNode * xml_op,
                         pe_working_set_t * data_set)
 {
     char *key = NULL;
     int interval = 0;
     const char *interval_s = NULL;
     const op_digest_cache_t *digest_data = NULL;
     gboolean did_change = FALSE;
 
     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     const char *op_version;
     const char *digest_secure = NULL;
 
     CRM_CHECK(active_node != NULL, return FALSE);
     if (safe_str_eq(task, RSC_STOP)) {
         return FALSE;
     }
 
     interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
     interval = crm_parse_int(interval_s, "0");
 
     if (interval > 0) {
         xmlNode *op_match = NULL;
 
         /* we need to reconstruct the key because of the way we used to construct resource IDs */
         key = generate_op_key(rsc->id, task, interval);
 
         pe_rsc_trace(rsc, "Checking parameters for %s", key);
         op_match = find_rsc_op_entry(rsc, key);
 
         if (op_match == NULL && is_set(data_set->flags, pe_flag_stop_action_orphans)) {
             CancelXmlOp(rsc, xml_op, active_node, "orphan", data_set);
             free(key);
             return TRUE;
 
         } else if (op_match == NULL) {
             pe_rsc_debug(rsc, "Orphan action detected: %s on %s", key, active_node->details->uname);
             free(key);
             return TRUE;
         }
         free(key);
         key = NULL;
     }
 
     crm_trace("Testing %s_%s_%d on %s",
               rsc->id, task, interval, active_node->details->uname);
     if (interval == 0 && safe_str_eq(task, RSC_STATUS)) {
         /* Reload based on the start action not a probe */
         task = RSC_START;
 
     } else if (interval == 0 && safe_str_eq(task, RSC_MIGRATED)) {
         /* Reload based on the start action not a migrate */
         task = RSC_START;
     } else if (interval == 0 && safe_str_eq(task, RSC_PROMOTE)) {
         /* Reload based on the start action not a promote */
         task = RSC_START;
     }
 
     op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
     digest_data = rsc_action_digest_cmp(rsc, xml_op, active_node, data_set);
 
     if(is_set(data_set->flags, pe_flag_sanitized)) {
         digest_secure = crm_element_value(xml_op, XML_LRM_ATTR_SECURE_DIGEST);
     }
 
     if(digest_data->rc != RSC_DIGEST_MATCH
        && digest_secure
        && digest_data->digest_secure_calc
        && strcmp(digest_data->digest_secure_calc, digest_secure) == 0) {
         if (is_set(data_set->flags, pe_flag_sanitized)) {
             printf("Only 'private' parameters to %s_%s_%d on %s changed: %s\n",
                    rsc->id, task, interval, active_node->details->uname,
                    crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
         }
 
     } else if (digest_data->rc == RSC_DIGEST_RESTART) {
         /* Changes that force a restart */
         const char *digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);
 
         did_change = TRUE;
         key = generate_op_key(rsc->id, task, interval);
         crm_log_xml_info(digest_data->params_restart, "params:restart");
         pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (restart:%s) %s",
                  key, active_node->details->uname,
                  crm_str(digest_restart), digest_data->digest_restart_calc,
                  op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
 
         custom_action(rsc, key, task, NULL, FALSE, TRUE, data_set);
         trigger_unfencing(rsc, active_node, "Device parameters changed", NULL, data_set);
 
     } else if ((digest_data->rc == RSC_DIGEST_ALL) || (digest_data->rc == RSC_DIGEST_UNKNOWN)) {
         /* Changes that can potentially be handled by a reload */
         const char *digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);
         const char *digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
 
         did_change = TRUE;
         trigger_unfencing(rsc, active_node, "Device parameters changed (reload)", NULL, data_set);
         crm_log_xml_info(digest_data->params_all, "params:reload");
         key = generate_op_key(rsc->id, task, interval);
         pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (reload:%s) %s",
                  key, active_node->details->uname,
                  crm_str(digest_all), digest_data->digest_all_calc, op_version,
                  crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
 
         if (interval > 0) {
             action_t *op = NULL;
 
 #if 0
             /* Always reload/restart the entire resource */
             ReloadRsc(rsc, active_node, data_set);
 #else
             /* Re-sending the recurring op is sufficient - the old one will be cancelled automatically */
             op = custom_action(rsc, key, task, active_node, TRUE, TRUE, data_set);
             set_bit(op->flags, pe_action_reschedule);
 #endif
 
         } else if (digest_restart && rsc->isolation_wrapper == NULL && (uber_parent(rsc))->isolation_wrapper == NULL) {
             pe_rsc_trace(rsc, "Reloading '%s' action for resource %s", task, rsc->id);
 
             /* Reload this resource */
             ReloadRsc(rsc, active_node, data_set);
             free(key);
 
         } else {
             pe_rsc_trace(rsc, "Resource %s doesn't know how to reload", rsc->id);
 
             /* Re-send the start/demote/promote op
              * Recurring ops will be detected independently
              */
             custom_action(rsc, key, task, NULL, FALSE, TRUE, data_set);
         }
     }
 
     return did_change;
 }
 
 
 static void
 check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     int offset = -1;
     int interval = 0;
     int stop_index = 0;
     int start_index = 0;
 
     const char *task = NULL;
     const char *interval_s = NULL;
 
     xmlNode *rsc_op = NULL;
     GListPtr op_list = NULL;
     GListPtr sorted_op_list = NULL;
     gboolean is_probe = FALSE;
     gboolean did_change = FALSE;
 
     CRM_CHECK(node != NULL, return);
 
     if (is_set(rsc->flags, pe_rsc_orphan)) {
         resource_t *parent = uber_parent(rsc);
         if(parent == NULL
            || pe_rsc_is_clone(parent) == FALSE
            || is_set(parent->flags, pe_rsc_unique)) {
             pe_rsc_trace(rsc, "Skipping param check for %s and deleting: orphan", rsc->id);
             DeleteRsc(rsc, node, FALSE, data_set);
         } else {
             pe_rsc_trace(rsc, "Skipping param check for %s (orphan clone)", rsc->id);
         }
         return;
 
     } else if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
         if (check_rsc_parameters(rsc, node, rsc_entry, FALSE, data_set)) {
             DeleteRsc(rsc, node, FALSE, data_set);
         }
         pe_rsc_trace(rsc, "Skipping param check for %s: no longer active on %s",
                      rsc->id, node->details->uname);
         return;
     }
 
     pe_rsc_trace(rsc, "Processing %s on %s", rsc->id, node->details->uname);
 
     if (check_rsc_parameters(rsc, node, rsc_entry, TRUE, data_set)) {
         DeleteRsc(rsc, node, FALSE, data_set);
     }
 
     for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             op_list = g_list_prepend(op_list, rsc_op);
         }
     }
 
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         offset++;
 
         if (start_index < stop_index) {
             /* stopped */
             continue;
         } else if (offset < start_index) {
             /* action occurred prior to a start */
             continue;
         }
 
         is_probe = FALSE;
         did_change = FALSE;
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
 
         interval_s = crm_element_value(rsc_op, XML_LRM_ATTR_INTERVAL);
         interval = crm_parse_int(interval_s, "0");
 
         if (interval == 0 && safe_str_eq(task, RSC_STATUS)) {
             is_probe = TRUE;
         }
 
         if (interval > 0 &&
             (is_set(rsc->flags, pe_rsc_maintenance) || node->details->maintenance)) {
             CancelXmlOp(rsc, rsc_op, node, "maintenance mode", data_set);
 
         } else if (is_probe || safe_str_eq(task, RSC_START) || safe_str_eq(task, RSC_PROMOTE) || interval > 0
                    || safe_str_eq(task, RSC_MIGRATED)) {
             did_change = check_action_definition(rsc, node, rsc_op, data_set);
         }
 
-        if (did_change && get_failcount(node, rsc, NULL, data_set)) {
+        if (did_change && pe_get_failcount(node, rsc, NULL, pe_fc_effective,
+                                           NULL, data_set)) {
+
             char *key = NULL;
             action_t *action_clear = NULL;
 
             key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
             action_clear =
                 custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE, data_set);
             set_bit(action_clear->flags, pe_action_runnable);
 
             crm_notice("Clearing failure of %s on %s "
                        "because action definition changed " CRM_XS " %s",
                        rsc->id, node->details->uname, action_clear->uuid);
         }
     }
 
     g_list_free(sorted_op_list);
 
 }
 
 static GListPtr
 find_rsc_list(GListPtr result, resource_t * rsc, const char *id, gboolean renamed_clones,
               gboolean partial, pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     gboolean match = FALSE;
 
     if (id == NULL) {
         return NULL;
 
     } else if (rsc == NULL && data_set) {
 
         for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
             resource_t *child = (resource_t *) gIter->data;
 
             result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
         }
 
         return result;
 
     } else if (rsc == NULL) {
         return NULL;
     }
 
     if (partial) {
         if (strstr(rsc->id, id)) {
             match = TRUE;
 
         } else if (renamed_clones && rsc->clone_name && strstr(rsc->clone_name, id)) {
             match = TRUE;
         }
 
     } else {
         if (strcmp(rsc->id, id) == 0) {
             match = TRUE;
 
         } else if (renamed_clones && rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
             match = TRUE;
         }
     }
 
     if (match) {
         result = g_list_prepend(result, rsc);
     }
 
     if (rsc->children) {
         gIter = rsc->children;
         for (; gIter != NULL; gIter = gIter->next) {
             resource_t *child = (resource_t *) gIter->data;
 
             result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
         }
     }
 
     return result;
 }
 
 static void
 check_actions(pe_working_set_t * data_set)
 {
     const char *id = NULL;
     node_t *node = NULL;
     xmlNode *lrm_rscs = NULL;
     xmlNode *status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);
 
     xmlNode *node_state = NULL;
 
     for (node_state = __xml_first_child(status); node_state != NULL;
          node_state = __xml_next_element(node_state)) {
         if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
             id = crm_element_value(node_state, XML_ATTR_ID);
             lrm_rscs = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
             lrm_rscs = find_xml_node(lrm_rscs, XML_LRM_TAG_RESOURCES, FALSE);
 
             node = pe_find_node_id(data_set->nodes, id);
 
             if (node == NULL) {
                 continue;
 
             /* Still need to check actions for a maintenance node to cancel existing monitor operations */
             } else if (can_run_resources(node) == FALSE && node->details->maintenance == FALSE) {
                 crm_trace("Skipping param check for %s: can't run resources",
                           node->details->uname);
                 continue;
             }
 
             crm_trace("Processing node %s", node->details->uname);
             if (node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) {
                 xmlNode *rsc_entry = NULL;
 
                 for (rsc_entry = __xml_first_child(lrm_rscs); rsc_entry != NULL;
                      rsc_entry = __xml_next_element(rsc_entry)) {
                     if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
 
                         if (xml_has_children(rsc_entry)) {
                             GListPtr gIter = NULL;
                             GListPtr result = NULL;
                             const char *rsc_id = ID(rsc_entry);
 
                             CRM_CHECK(rsc_id != NULL, return);
 
                             result = find_rsc_list(NULL, NULL, rsc_id, TRUE, FALSE, data_set);
                             for (gIter = result; gIter != NULL; gIter = gIter->next) {
                                 resource_t *rsc = (resource_t *) gIter->data;
 
                                 if (rsc->variant != pe_native) {
                                     continue;
                                 }
                                 check_actions_for(rsc_entry, rsc, node, data_set);
                             }
                             g_list_free(result);
                         }
                     }
                 }
             }
         }
     }
 }
 
 static gboolean
 apply_placement_constraints(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
 
     crm_trace("Applying constraints...");
 
     for (gIter = data_set->placement_constraints; gIter != NULL; gIter = gIter->next) {
         rsc_to_node_t *cons = (rsc_to_node_t *) gIter->data;
 
         cons->rsc_lh->cmds->rsc_location(cons->rsc_lh, cons);
     }
 
     return TRUE;
 
 }
 
 static gboolean
 failcount_clear_action_exists(node_t * node, resource_t * rsc)
 {
     gboolean rc = FALSE;
     char *key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
     GListPtr list = find_actions_exact(rsc->actions, key, node);
 
     if (list) {
         rc = TRUE;
     }
     g_list_free(list);
     free(key);
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Force resource away if failures hit migration threshold
  *
  * \param[in,out] rsc       Resource to check for failures
  * \param[in,out] node      Node to check for failures
  * \param[in,out] data_set  Cluster working set to update
  */
 static void
 check_migration_threshold(resource_t *rsc, node_t *node,
                           pe_working_set_t *data_set)
 {
     int fail_count, countdown;
     resource_t *failed;
 
     /* Migration threshold of 0 means never force away */
     if (rsc->migration_threshold == 0) {
         return;
     }
 
+    // If we're ignoring failures, also ignore the migration threshold
+    if (is_set(rsc->flags, pe_rsc_failure_ignored)) {
+        return;
+    }
+
     /* If there are no failures, there's no need to force away */
-    fail_count = get_failcount_all(node, rsc, NULL, data_set);
+    fail_count = pe_get_failcount(node, rsc, NULL,
+                                  pe_fc_effective|pe_fc_fillers, NULL,
+                                  data_set);
     if (fail_count <= 0) {
         return;
     }
 
     /* How many more times recovery will be tried on this node */
     countdown = QB_MAX(rsc->migration_threshold - fail_count, 0);
 
     /* If failed resource has a parent, we'll force the parent away */
     failed = rsc;
     if (is_not_set(rsc->flags, pe_rsc_unique)) {
         failed = uber_parent(rsc);
     }
 
     if (countdown == 0) {
         resource_location(failed, node, -INFINITY, "__fail_limit__", data_set);
         crm_warn("Forcing %s away from %s after %d failures (max=%d)",
                  failed->id, node->details->uname, fail_count,
                  rsc->migration_threshold);
     } else {
         crm_info("%s can fail %d more times on %s before being forced off",
                  failed->id, countdown, node->details->uname);
     }
 }
 
 static void
 common_apply_stickiness(resource_t * rsc, node_t * node, pe_working_set_t * data_set)
 {
     if (rsc->children) {
         GListPtr gIter = rsc->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             resource_t *child_rsc = (resource_t *) gIter->data;
 
             common_apply_stickiness(child_rsc, node, data_set);
         }
         return;
     }
 
     if (is_set(rsc->flags, pe_rsc_managed)
         && rsc->stickiness != 0 && g_list_length(rsc->running_on) == 1) {
         node_t *current = pe_find_node_id(rsc->running_on, node->details->id);
         node_t *match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
 
         if (current == NULL) {
 
         } else if (match != NULL || is_set(data_set->flags, pe_flag_symmetric_cluster)) {
             resource_t *sticky_rsc = rsc;
 
             resource_location(sticky_rsc, node, rsc->stickiness, "stickiness", data_set);
             pe_rsc_debug(sticky_rsc, "Resource %s: preferring current location"
                          " (node=%s, weight=%d)", sticky_rsc->id,
                          node->details->uname, rsc->stickiness);
         } else {
             GHashTableIter iter;
             node_t *nIter = NULL;
 
             pe_rsc_debug(rsc, "Ignoring stickiness for %s: the cluster is asymmetric"
                          " and node %s is not explicitly allowed", rsc->id, node->details->uname);
             g_hash_table_iter_init(&iter, rsc->allowed_nodes);
             while (g_hash_table_iter_next(&iter, NULL, (void **)&nIter)) {
                 crm_err("%s[%s] = %d", rsc->id, nIter->details->uname, nIter->weight);
             }
         }
     }
 
     /* Check the migration threshold only if a failcount clear action
      * has not already been placed for this resource on the node.
      * There is no sense in potentially forcing the resource from this
      * node if the failcount is being reset anyway. */
     if (failcount_clear_action_exists(node, rsc) == FALSE) {
         check_migration_threshold(rsc, node, data_set);
     }
 }
 
 void
 complex_set_cmds(resource_t * rsc)
 {
     GListPtr gIter = rsc->children;
 
     rsc->cmds = &resource_class_alloc_functions[rsc->variant];
 
     for (; gIter != NULL; gIter = gIter->next) {
         resource_t *child_rsc = (resource_t *) gIter->data;
 
         complex_set_cmds(child_rsc);
     }
 }
 
 void
 set_alloc_actions(pe_working_set_t * data_set)
 {
 
     GListPtr gIter = data_set->resources;
 
     for (; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         complex_set_cmds(rsc);
     }
 }
 
 static void
 calculate_system_health(gpointer gKey, gpointer gValue, gpointer user_data)
 {
     const char *key = (const char *)gKey;
     const char *value = (const char *)gValue;
     int *system_health = (int *)user_data;
 
     if (!gKey || !gValue || !user_data) {
         return;
     }
 
     if (crm_starts_with(key, "#health")) {
         int score;
 
         /* Convert the value into an integer */
         score = char2score(value);
 
         /* Add it to the running total */
         *system_health = merge_weights(score, *system_health);
     }
 }
 
 static gboolean
 apply_system_health(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     const char *health_strategy = pe_pref(data_set->config_hash, "node-health-strategy");
     int base_health = 0;
 
     if (health_strategy == NULL || safe_str_eq(health_strategy, "none")) {
         /* Prevent any accidental health -> score translation */
         node_score_red = 0;
         node_score_yellow = 0;
         node_score_green = 0;
         return TRUE;
 
     } else if (safe_str_eq(health_strategy, "migrate-on-red")) {
 
         /* Resources on nodes which have health values of red are
          * weighted away from that node.
          */
         node_score_red = -INFINITY;
         node_score_yellow = 0;
         node_score_green = 0;
 
     } else if (safe_str_eq(health_strategy, "only-green")) {
 
         /* Resources on nodes which have health values of red or yellow
          * are forced away from that node.
          */
         node_score_red = -INFINITY;
         node_score_yellow = -INFINITY;
         node_score_green = 0;
 
     } else if (safe_str_eq(health_strategy, "progressive")) {
         /* Same as the above, but use the r/y/g scores provided by the user
          * Defaults are provided by the pe_prefs table
          * Also, custom health "base score" can be used
          */
         base_health = crm_parse_int(pe_pref(data_set->config_hash, "node-health-base"), "0");
 
     } else if (safe_str_eq(health_strategy, "custom")) {
 
         /* Requires the admin to configure the rsc_location constaints for
          * processing the stored health scores
          */
         /* TODO: Check for the existence of appropriate node health constraints */
         return TRUE;
 
     } else {
         crm_err("Unknown node health strategy: %s", health_strategy);
         return FALSE;
     }
 
     crm_info("Applying automated node health strategy: %s", health_strategy);
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         int system_health = base_health;
         node_t *node = (node_t *) gIter->data;
 
         /* Search through the node hash table for system health entries. */
         g_hash_table_foreach(node->details->attrs, calculate_system_health, &system_health);
 
         crm_info(" Node %s has an combined system health of %d",
                  node->details->uname, system_health);
 
         /* If the health is non-zero, then create a new rsc2node so that the
          * weight will be added later on.
          */
         if (system_health != 0) {
 
             GListPtr gIter2 = data_set->resources;
 
             for (; gIter2 != NULL; gIter2 = gIter2->next) {
                 resource_t *rsc = (resource_t *) gIter2->data;
 
                 rsc2node_new(health_strategy, rsc, system_health, NULL, node, data_set);
             }
         }
     }
 
     return TRUE;
 }
 
 gboolean
 stage0(pe_working_set_t * data_set)
 {
     xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);
 
     if (data_set->input == NULL) {
         return FALSE;
     }
 
     if (is_set(data_set->flags, pe_flag_have_status) == FALSE) {
         crm_trace("Calculating status");
         cluster_status(data_set);
     }
 
     set_alloc_actions(data_set);
     apply_system_health(data_set);
     unpack_constraints(cib_constraints, data_set);
 
     return TRUE;
 }
 
 /*
  * Check nodes for resources started outside of the LRM
  */
 gboolean
 probe_resources(pe_working_set_t * data_set)
 {
     action_t *probe_node_complete = NULL;
 
     for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
         const char *probed = pe_node_attribute_raw(node, CRM_OP_PROBED);
 
         if (node->details->online == FALSE) {
 
             if (is_baremetal_remote_node(node) && node->details->remote_rsc
                 && (get_remote_node_state(node) == remote_state_failed)) {
 
                 pe_fence_node(data_set, node, "the connection is unrecoverable");
             }
             continue;
 
         } else if (node->details->unclean) {
             continue;
 
         } else if (node->details->rsc_discovery_enabled == FALSE) {
             /* resource discovery is disabled for this node */
             continue;
         }
 
         if (probed != NULL && crm_is_true(probed) == FALSE) {
             action_t *probe_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_REPROBE, node->details->uname),
                                                CRM_OP_REPROBE, node, FALSE, TRUE, data_set);
 
             add_hash_param(probe_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
             continue;
         }
 
         for (GListPtr gIter2 = data_set->resources; gIter2 != NULL; gIter2 = gIter2->next) {
             resource_t *rsc = (resource_t *) gIter2->data;
 
             rsc->cmds->create_probe(rsc, node, probe_node_complete, FALSE, data_set);
         }
     }
     return TRUE;
 }
 
 static void
 rsc_discover_filter(resource_t *rsc, node_t *node)
 {
     GListPtr gIter = rsc->children;
     resource_t *top = uber_parent(rsc);
     node_t *match;
 
     if (rsc->exclusive_discover == FALSE && top->exclusive_discover == FALSE) {
         return;
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         resource_t *child_rsc = (resource_t *) gIter->data;
         rsc_discover_filter(child_rsc, node);
     }
 
     match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
     if (match && match->rsc_discover_mode != pe_discover_exclusive) {
         match->weight = -INFINITY;
     }
 }
 
 /*
  * Count how many valid nodes we have (so we know the maximum number of
  *  colors we can resolve).
  *
  * Apply node constraints (i.e. filter the "allowed_nodes" part of resources)
  */
 gboolean
 stage2(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
 
     crm_trace("Applying placement constraints");
 
     gIter = data_set->nodes;
     for (; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
         if (node == NULL) {
             /* error */
 
         } else if (node->weight >= 0.0  /* global weight */
                    && node->details->online && node->details->type != node_ping) {
             data_set->max_valid_nodes++;
         }
     }
 
     apply_placement_constraints(data_set);
 
     gIter = data_set->nodes;
     for (; gIter != NULL; gIter = gIter->next) {
         GListPtr gIter2 = NULL;
         node_t *node = (node_t *) gIter->data;
 
         gIter2 = data_set->resources;
         for (; gIter2 != NULL; gIter2 = gIter2->next) {
             resource_t *rsc = (resource_t *) gIter2->data;
 
             common_apply_stickiness(rsc, node, data_set);
             rsc_discover_filter(rsc, node);
         }
     }
 
     return TRUE;
 }
 
 /*
  * Create internal resource constraints before allocation
  */
 gboolean
 stage3(pe_working_set_t * data_set)
 {
 
     GListPtr gIter = data_set->resources;
 
     for (; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         rsc->cmds->internal_constraints(rsc, data_set);
     }
 
     return TRUE;
 }
 
 /*
  * Check for orphaned or redefined actions
  */
 gboolean
 stage4(pe_working_set_t * data_set)
 {
     check_actions(data_set);
     return TRUE;
 }
 
 static gint
 sort_rsc_process_order(gconstpointer a, gconstpointer b, gpointer data)
 {
     int rc = 0;
     int r1_weight = -INFINITY;
     int r2_weight = -INFINITY;
 
     const char *reason = "existence";
 
     const GListPtr nodes = (GListPtr) data;
     resource_t *resource1 = (resource_t *) convert_const_pointer(a);
     resource_t *resource2 = (resource_t *) convert_const_pointer(b);
 
     node_t *r1_node = NULL;
     node_t *r2_node = NULL;
     GListPtr gIter = NULL;
     GHashTable *r1_nodes = NULL;
     GHashTable *r2_nodes = NULL;
 
     if (a == NULL && b == NULL) {
         goto done;
     }
     if (a == NULL) {
         return 1;
     }
     if (b == NULL) {
         return -1;
     }
 
     reason = "priority";
     r1_weight = resource1->priority;
     r2_weight = resource2->priority;
 
     if (r1_weight > r2_weight) {
         rc = -1;
         goto done;
     }
 
     if (r1_weight < r2_weight) {
         rc = 1;
         goto done;
     }
 
     reason = "no node list";
     if (nodes == NULL) {
         goto done;
     }
 
     r1_nodes =
         rsc_merge_weights(resource1, resource1->id, NULL, NULL, 1,
                           pe_weights_forward | pe_weights_init);
     dump_node_scores(LOG_TRACE, NULL, resource1->id, r1_nodes);
     r2_nodes =
         rsc_merge_weights(resource2, resource2->id, NULL, NULL, 1,
                           pe_weights_forward | pe_weights_init);
     dump_node_scores(LOG_TRACE, NULL, resource2->id, r2_nodes);
 
     /* Current location score */
     reason = "current location";
     r1_weight = -INFINITY;
     r2_weight = -INFINITY;
 
     if (resource1->running_on) {
         r1_node = g_list_nth_data(resource1->running_on, 0);
         r1_node = g_hash_table_lookup(r1_nodes, r1_node->details->id);
         if (r1_node != NULL) {
             r1_weight = r1_node->weight;
         }
     }
     if (resource2->running_on) {
         r2_node = g_list_nth_data(resource2->running_on, 0);
         r2_node = g_hash_table_lookup(r2_nodes, r2_node->details->id);
         if (r2_node != NULL) {
             r2_weight = r2_node->weight;
         }
     }
 
     if (r1_weight > r2_weight) {
         rc = -1;
         goto done;
     }
 
     if (r1_weight < r2_weight) {
         rc = 1;
         goto done;
     }
 
     reason = "score";
     for (gIter = nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
         r1_node = NULL;
         r2_node = NULL;
 
         r1_weight = -INFINITY;
         if (r1_nodes) {
             r1_node = g_hash_table_lookup(r1_nodes, node->details->id);
         }
         if (r1_node) {
             r1_weight = r1_node->weight;
         }
 
         r2_weight = -INFINITY;
         if (r2_nodes) {
             r2_node = g_hash_table_lookup(r2_nodes, node->details->id);
         }
         if (r2_node) {
             r2_weight = r2_node->weight;
         }
 
         if (r1_weight > r2_weight) {
             rc = -1;
             goto done;
         }
 
         if (r1_weight < r2_weight) {
             rc = 1;
             goto done;
         }
     }
 
   done:
     crm_trace("%s (%d) on %s %c %s (%d) on %s: %s",
               resource1->id, r1_weight, r1_node ? r1_node->details->id : "n/a",
               rc < 0 ? '>' : rc > 0 ? '<' : '=',
               resource2->id, r2_weight, r2_node ? r2_node->details->id : "n/a", reason);
 
     if (r1_nodes) {
         g_hash_table_destroy(r1_nodes);
     }
     if (r2_nodes) {
         g_hash_table_destroy(r2_nodes);
     }
 
     return rc;
 }
 
 static void
 allocate_resources(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
 
     if (is_set(data_set->flags, pe_flag_have_remote_nodes)) {
         /* Force remote connection resources to be allocated first. This
          * also forces any colocation dependencies to be allocated as well */
         for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
             resource_t *rsc = (resource_t *) gIter->data;
             if (rsc->is_remote_node == FALSE) {
                 continue;
             }
             pe_rsc_trace(rsc, "Allocating: %s", rsc->id);
             /* For remote node connection resources, always prefer the partial
              * migration target during resource allocation, if the rsc is in the
              * middle of a migration.
              */
             rsc->cmds->allocate(rsc, rsc->partial_migration_target, data_set);
         }
     }
 
     /* now do the rest of the resources */
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
         if (rsc->is_remote_node == TRUE) {
             continue;
         }
         pe_rsc_trace(rsc, "Allocating: %s", rsc->id);
         rsc->cmds->allocate(rsc, NULL, data_set);
     }
 }
 
 /* We always use pe_order_preserve with these convenience functions to exempt
  * internally generated constraints from the prohibition of user constraints
  * involving remote connection resources.
  *
  * The start ordering additionally uses pe_order_runnable_left so that the
  * specified action is not runnable if the start is not runnable.
  */
 
 static inline void
 order_start_then_action(resource_t *lh_rsc, action_t *rh_action,
                         enum pe_ordering extra, pe_working_set_t *data_set)
 {
     if (lh_rsc && rh_action && data_set) {
         custom_action_order(lh_rsc, start_key(lh_rsc), NULL,
                             rh_action->rsc, NULL, rh_action,
                             pe_order_preserve | pe_order_runnable_left | extra,
                             data_set);
     }
 }
 
 static inline void
 order_action_then_stop(action_t *lh_action, resource_t *rh_rsc,
                        enum pe_ordering extra, pe_working_set_t *data_set)
 {
     if (lh_action && rh_rsc && data_set) {
         custom_action_order(lh_action->rsc, NULL, lh_action,
                             rh_rsc, stop_key(rh_rsc), NULL,
                             pe_order_preserve | extra, data_set);
     }
 }
 
 static void
 cleanup_orphans(resource_t * rsc, pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
 
     if (is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
         return;
     }
 
     /* Don't recurse into ->children, those are just unallocated clone instances */
     if(is_not_set(rsc->flags, pe_rsc_orphan)) {
         return;
     }
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
-        if (node->details->online && get_failcount(node, rsc, NULL, data_set)) {
+        if (node->details->online
+            && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
+                                data_set)) {
+
             char *key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
             action_t *clear_op = custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT,
                                                node, FALSE, TRUE, data_set);
 
             add_hash_param(clear_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
 
             pe_rsc_info(rsc,
                         "Clearing failure of %s on %s because it is orphaned "
                         CRM_XS " %s",
                         rsc->id, node->details->uname, clear_op->uuid);
 
             /* We can't use order_action_then_stop() here because its
              * pe_order_preserve breaks things
              */
             custom_action_order(clear_op->rsc, NULL, clear_op,
                                 rsc, stop_key(rsc), NULL,
                                 pe_order_optional, data_set);
         }
     }
 }
 
 gboolean
 stage5(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
 
     if (safe_str_neq(data_set->placement_strategy, "default")) {
         GListPtr nodes = g_list_copy(data_set->nodes);
 
         nodes = g_list_sort_with_data(nodes, sort_node_weight, NULL);
 
         data_set->resources =
             g_list_sort_with_data(data_set->resources, sort_rsc_process_order, nodes);
 
         g_list_free(nodes);
     }
 
     gIter = data_set->nodes;
     for (; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
         dump_node_capacity(show_utilization ? 0 : utilization_log_level, "Original", node);
     }
 
     crm_trace("Allocating services");
     /* Take (next) highest resource, assign it and create its actions */
 
     allocate_resources(data_set);
 
     gIter = data_set->nodes;
     for (; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
         dump_node_capacity(show_utilization ? 0 : utilization_log_level, "Remaining", node);
     }
 
     if (is_set(data_set->flags, pe_flag_startup_probes)) {
         crm_trace("Calculating needed probes");
         /* This code probably needs optimization
          * ptest -x with 100 nodes, 100 clones and clone-max=100:
 
          With probes:
 
          ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
          ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
          ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
          ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:292 Check actions
          ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
          ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: stage5: allocate.c:881 Allocating services
          ptest[14781]: 2010/09/27_17:56:49 notice: TRACE: stage5: allocate.c:894 Calculating needed probes
          ptest[14781]: 2010/09/27_17:56:51 notice: TRACE: stage5: allocate.c:899 Creating actions
          ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: stage5: allocate.c:905 Creating done
          ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
          ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
          36s
          ptest[14781]: 2010/09/27_17:57:28 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
 
          Without probes:
 
          ptest[14637]: 2010/09/27_17:56:21 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
          ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
          ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
          ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:292 Check actions
          ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
          ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: stage5: allocate.c:881 Allocating services
          ptest[14637]: 2010/09/27_17:56:24 notice: TRACE: stage5: allocate.c:899 Creating actions
          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: stage5: allocate.c:905 Creating done
          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
         */
 
         probe_resources(data_set);
     }
 
     crm_trace("Handle orphans");
 
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
         cleanup_orphans(rsc, data_set);
     }
 
     crm_trace("Creating actions");
 
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         rsc->cmds->create_actions(rsc, data_set);
     }
 
     crm_trace("Creating done");
     return TRUE;
 }
 
 static gboolean
 is_managed(const resource_t * rsc)
 {
     GListPtr gIter = rsc->children;
 
     if (is_set(rsc->flags, pe_rsc_managed)) {
         return TRUE;
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         resource_t *child_rsc = (resource_t *) gIter->data;
 
         if (is_managed(child_rsc)) {
             return TRUE;
         }
     }
 
     return FALSE;
 }
 
 static gboolean
 any_managed_resources(pe_working_set_t * data_set)
 {
 
     GListPtr gIter = data_set->resources;
 
     for (; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         if (is_managed(rsc)) {
             return TRUE;
         }
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Create pseudo-op for guest node fence, and order relative to it
  *
  * \param[in] node      Guest node to fence
  * \param[in] done      STONITH_DONE operation
  * \param[in] data_set  Working set of CIB state
  */
 static void
 fence_guest(pe_node_t *node, pe_action_t *done, pe_working_set_t *data_set)
 {
     resource_t *container = node->details->remote_rsc->container;
     pe_action_t *stop = NULL;
     pe_action_t *stonith_op = NULL;
 
     /* The fence action is just a label; we don't do anything differently for
      * off vs. reboot. We specify it explicitly, rather than let it default to
      * cluster's default action, because we are not _initiating_ fencing -- we
      * are creating a pseudo-event to describe fencing that is already occurring
      * by other means (container recovery).
      */
     const char *fence_action = "off";
 
     /* Check whether guest's container resource is has any explicit stop or
      * start (the stop may be implied by fencing of the guest's host).
      */
     if (container) {
         stop = find_first_action(container->actions, NULL, CRMD_ACTION_STOP, NULL);
 
         if (find_first_action(container->actions, NULL, CRMD_ACTION_START, NULL)) {
             fence_action = "reboot";
         }
     }
 
     /* Create a fence pseudo-event, so we have an event to order actions
      * against, and crmd can always detect it.
      */
     stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean", data_set);
     update_action_flags(stonith_op, pe_action_pseudo | pe_action_runnable,
                         __FUNCTION__, __LINE__);
 
     /* We want to imply stops/demotes after the guest is stopped, not wait until
      * it is restarted, so we always order pseudo-fencing after stop, not start
      * (even though start might be closer to what is done for a real reboot).
      */
     if(stop && is_set(stop->flags, pe_action_pseudo)) {
         pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE, NULL, data_set);
         crm_info("Implying guest node %s is down (action %d) after %s fencing",
                  node->details->uname, stonith_op->id, stop->node->details->uname);
         order_actions(parent_stonith_op, stonith_op,
                       pe_order_runnable_left|pe_order_implies_then);
 
     } else if (stop) {
         order_actions(stop, stonith_op,
                       pe_order_runnable_left|pe_order_implies_then);
         crm_info("Implying guest node %s is down (action %d) "
                  "after container %s is stopped (action %d)",
                  node->details->uname, stonith_op->id,
                  container->id, stop->id);
     } else {
         crm_info("Implying guest node %s is down (action %d) ",
                  node->details->uname, stonith_op->id);
     }
 
     /* @TODO: Order pseudo-fence after any (optional) fence of guest's host */
 
     /* Order/imply other actions relative to pseudo-fence as with real fence */
     stonith_constraints(node, stonith_op, data_set);
     order_actions(stonith_op, done, pe_order_implies_then);
 }
 
 /*
  * Create dependencies for stonith and shutdown operations
  */
 gboolean
 stage6(pe_working_set_t * data_set)
 {
     action_t *dc_down = NULL;
     action_t *dc_fence = NULL;
     action_t *stonith_op = NULL;
     action_t *last_stonith = NULL;
     gboolean integrity_lost = FALSE;
     action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);
     action_t *done = get_pseudo_op(STONITH_DONE, data_set);
     gboolean need_stonith = TRUE;
     GListPtr gIter;
     GListPtr stonith_ops = NULL;
 
     /* Remote ordering constraints need to happen prior to calculate
      * fencing because it is one more place we will mark the node as
      * dirty.
      *
      * A nice side-effect of doing it first is that we can remove a
      * bunch of special logic from apply_*_ordering() because its
      * already part of pe_fence_node()
      */
     crm_trace("Creating remote ordering constraints");
     apply_remote_node_ordering(data_set);
 
     crm_trace("Processing fencing and shutdown cases");
     if (any_managed_resources(data_set) == FALSE) {
         crm_notice("Delaying fencing operations until there are resources to manage");
         need_stonith = FALSE;
     }
 
     /* Check each node for stonith/shutdown */
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
         /* Guest nodes are "fenced" by recovering their container resource,
          * so handle them separately.
          */
         if (is_container_remote_node(node)) {
             if (node->details->remote_requires_reset && need_stonith) {
                 fence_guest(node, done, data_set);
             }
             continue;
         }
 
         stonith_op = NULL;
 
         if (node->details->unclean
             && need_stonith && pe_can_fence(data_set, node)) {
 
             stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", data_set);
             pe_warn("Scheduling Node %s for STONITH", node->details->uname);
 
             stonith_constraints(node, stonith_op, data_set);
 
             if (node->details->is_dc) {
                 dc_down = stonith_op;
                 dc_fence = stonith_op;
 
             } else if (is_set(data_set->flags, pe_flag_concurrent_fencing) == FALSE) {
                 if (last_stonith) {
                     order_actions(last_stonith, stonith_op, pe_order_optional);
                 }
                 last_stonith = stonith_op;
 
             } else {
                 order_actions(stonith_op, done, pe_order_implies_then);
                 stonith_ops = g_list_append(stonith_ops, stonith_op);
             }
 
         } else if (node->details->online && node->details->shutdown &&
                 /* TODO define what a shutdown op means for a remote node.
                  * For now we do not send shutdown operations for remote nodes, but
                  * if we can come up with a good use for this in the future, we will. */
                     is_remote_node(node) == FALSE) {
 
             action_t *down_op = NULL;
 
             crm_notice("Scheduling Node %s for shutdown", node->details->uname);
 
             down_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_SHUTDOWN, node->details->uname),
                                     CRM_OP_SHUTDOWN, node, FALSE, TRUE, data_set);
 
             shutdown_constraints(node, down_op, data_set);
             add_hash_param(down_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
 
             if (node->details->is_dc) {
                 dc_down = down_op;
             }
         }
 
         if (node->details->unclean && stonith_op == NULL) {
             integrity_lost = TRUE;
             pe_warn("Node %s is unclean!", node->details->uname);
         }
     }
 
     if (integrity_lost) {
         if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
             pe_warn("YOUR RESOURCES ARE NOW LIKELY COMPROMISED");
             pe_err("ENABLE STONITH TO KEEP YOUR RESOURCES SAFE");
 
         } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE) {
             crm_notice("Cannot fence unclean nodes until quorum is"
                        " attained (or no-quorum-policy is set to ignore)");
         }
     }
 
     if (dc_down != NULL) {
         GListPtr gIter = NULL;
 
         crm_trace("Ordering shutdowns before %s on %s (DC)",
                   dc_down->task, dc_down->node->details->uname);
 
         add_hash_param(dc_down->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
 
         for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
             action_t *node_stop = (action_t *) gIter->data;
 
             if (safe_str_neq(CRM_OP_SHUTDOWN, node_stop->task)) {
                 continue;
             } else if (node_stop->node->details->is_dc) {
                 continue;
             }
 
             crm_debug("Ordering shutdown on %s before %s on %s",
                       node_stop->node->details->uname,
                       dc_down->task, dc_down->node->details->uname);
 
             order_actions(node_stop, dc_down, pe_order_optional);
         }
 
         if (last_stonith) {
             if (dc_down != last_stonith) {
                 order_actions(last_stonith, dc_down, pe_order_optional);
             }
 
         } else {
             GListPtr gIter2 = NULL;
 
             for (gIter2 = stonith_ops; gIter2 != NULL; gIter2 = gIter2->next) {
                 stonith_op = (action_t *) gIter2->data;
 
                 if (dc_down != stonith_op) {
                     order_actions(stonith_op, dc_down, pe_order_optional);
                 }
             }
         }
     }
 
 
     if (dc_fence) {
         order_actions(dc_down, done, pe_order_implies_then);
 
     } else if (last_stonith) {
         order_actions(last_stonith, done, pe_order_implies_then);
     }
 
     order_actions(done, all_stopped, pe_order_implies_then);
 
     g_list_free(stonith_ops);
     return TRUE;
 }
 
 /*
  * Determine the sets of independent actions and the correct order for the
  *  actions in each set.
  *
  * Mark dependencies of un-runnable actions un-runnable
  *
  */
 static GListPtr
 find_actions_by_task(GListPtr actions, resource_t * rsc, const char *original_key)
 {
     GListPtr list = NULL;
 
     list = find_actions(actions, original_key, NULL);
     if (list == NULL) {
         /* we're potentially searching a child of the original resource */
         char *key = NULL;
         char *tmp = NULL;
         char *task = NULL;
         int interval = 0;
 
         if (parse_op_key(original_key, &tmp, &task, &interval)) {
             key = generate_op_key(rsc->id, task, interval);
             /* crm_err("looking up %s instead of %s", key, original_key); */
             /* slist_iter(action, action_t, actions, lpc, */
             /*         crm_err("  - %s", action->uuid)); */
             list = find_actions(actions, key, NULL);
 
         } else {
             crm_err("search key: %s", original_key);
         }
 
         free(key);
         free(tmp);
         free(task);
     }
 
     return list;
 }
 
 static void
 rsc_order_then(action_t * lh_action, resource_t * rsc, order_constraint_t * order)
 {
     GListPtr gIter = NULL;
     GListPtr rh_actions = NULL;
     action_t *rh_action = NULL;
     enum pe_ordering type = order->type;
 
     CRM_CHECK(rsc != NULL, return);
     CRM_CHECK(order != NULL, return);
 
     rh_action = order->rh_action;
     crm_trace("Processing RH of ordering constraint %d", order->id);
 
     if (rh_action != NULL) {
         rh_actions = g_list_prepend(NULL, rh_action);
 
     } else if (rsc != NULL) {
         rh_actions = find_actions_by_task(rsc->actions, rsc, order->rh_action_task);
     }
 
     if (rh_actions == NULL) {
         pe_rsc_trace(rsc, "No RH-Side (%s/%s) found for constraint..."
                      " ignoring", rsc->id, order->rh_action_task);
         if (lh_action) {
             pe_rsc_trace(rsc, "LH-Side was: %s", lh_action->uuid);
         }
         return;
     }
 
     if (lh_action && lh_action->rsc == rsc && is_set(lh_action->flags, pe_action_dangle)) {
         pe_rsc_trace(rsc, "Detected dangling operation %s -> %s", lh_action->uuid,
                      order->rh_action_task);
         clear_bit(type, pe_order_implies_then);
     }
 
     gIter = rh_actions;
     for (; gIter != NULL; gIter = gIter->next) {
         action_t *rh_action_iter = (action_t *) gIter->data;
 
         if (lh_action) {
             order_actions(lh_action, rh_action_iter, type);
 
         } else if (type & pe_order_implies_then) {
             update_action_flags(rh_action_iter, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
             crm_warn("Unrunnable %s 0x%.6x", rh_action_iter->uuid, type);
         } else {
             crm_warn("neither %s 0x%.6x", rh_action_iter->uuid, type);
         }
     }
 
     g_list_free(rh_actions);
 }
 
 static void
 rsc_order_first(resource_t * lh_rsc, order_constraint_t * order, pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     GListPtr lh_actions = NULL;
     action_t *lh_action = order->lh_action;
     resource_t *rh_rsc = order->rh_rsc;
 
     crm_trace("Processing LH of ordering constraint %d", order->id);
     CRM_ASSERT(lh_rsc != NULL);
 
     if (lh_action != NULL) {
         lh_actions = g_list_prepend(NULL, lh_action);
 
     } else if (lh_action == NULL) {
         lh_actions = find_actions_by_task(lh_rsc->actions, lh_rsc, order->lh_action_task);
     }
 
     if (lh_actions == NULL && lh_rsc != rh_rsc) {
         char *key = NULL;
         char *rsc_id = NULL;
         char *op_type = NULL;
         int interval = 0;
 
         parse_op_key(order->lh_action_task, &rsc_id, &op_type, &interval);
         key = generate_op_key(lh_rsc->id, op_type, interval);
 
         if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_STOPPED && safe_str_eq(op_type, RSC_STOP)) {
             free(key);
             pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
                          lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
 
         } else if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_SLAVE && safe_str_eq(op_type, RSC_DEMOTE)) {
             free(key);
             pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
                          lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
 
         } else {
             pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - creating",
                          lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
             lh_action = custom_action(lh_rsc, key, op_type, NULL, TRUE, TRUE, data_set);
             lh_actions = g_list_prepend(NULL, lh_action);
         }
 
         free(op_type);
         free(rsc_id);
     }
 
     gIter = lh_actions;
     for (; gIter != NULL; gIter = gIter->next) {
         action_t *lh_action_iter = (action_t *) gIter->data;
 
         if (rh_rsc == NULL && order->rh_action) {
             rh_rsc = order->rh_action->rsc;
         }
         if (rh_rsc) {
             rsc_order_then(lh_action_iter, rh_rsc, order);
 
         } else if (order->rh_action) {
             order_actions(lh_action_iter, order->rh_action, order->type);
         }
     }
 
     g_list_free(lh_actions);
 }
 
 extern gboolean update_action(action_t * action);
 extern void update_colo_start_chain(action_t * action);
 
 static int
 is_recurring_action(action_t *action) 
 {
     const char *interval_s = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL);
     int interval = crm_parse_int(interval_s, "0");
     if(interval > 0) {
         return TRUE;
     }
     return FALSE;
 }
 
 static void
 apply_container_ordering(action_t *action, pe_working_set_t *data_set)
 {
     /* VMs are also classified as containers for these purposes... in
      * that they both involve a 'thing' running on a real or remote
      * cluster node.
      *
      * This allows us to be smarter about the type and extent of
      * recovery actions required in various scenarios
      */
     resource_t *remote_rsc = NULL;
     resource_t *container = NULL;
     enum action_tasks task = text2task(action->task);
 
     CRM_ASSERT(action->rsc);
     CRM_ASSERT(action->node);
     CRM_ASSERT(is_remote_node(action->node));
 
     remote_rsc = action->node->details->remote_rsc;
     CRM_ASSERT(remote_rsc);
 
     container = remote_rsc->container;
     CRM_ASSERT(container);
 
     if(is_set(container->flags, pe_rsc_failed)) {
         pe_fence_node(data_set, action->node, "container failed");
     }
 
     crm_trace("Order %s action %s relative to %s%s for %s%s",
               action->task, action->uuid,
               is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
               remote_rsc->id,
               is_set(container->flags, pe_rsc_failed)? "failed " : "",
               container->id);
 
     if (safe_str_eq(action->task, CRMD_ACTION_MIGRATE)
         || safe_str_eq(action->task, CRMD_ACTION_MIGRATE)) {
         /* Migration ops map to "no_action", but we need to apply the same
          * ordering as for stop or demote (see get_router_node()).
          */
         task = stop_rsc;
     }
 
     switch (task) {
         case start_rsc:
         case action_promote:
             /* Force resource recovery if the container is recovered */
             order_start_then_action(container, action, pe_order_implies_then,
                                     data_set);
 
             /* Wait for the connection resource to be up too */
             order_start_then_action(remote_rsc, action, pe_order_none,
                                     data_set);
             break;
 
         case stop_rsc:
         case action_demote:
             if (is_set(container->flags, pe_rsc_failed)) {
                 /* When the container representing a guest node fails, any stop
                  * or demote actions for resources running on the guest node
                  * are implied by the container stopping. This is similar to
                  * how fencing operations work for cluster nodes and remote
                  * nodes.
                  */
             } else {
                 /* Ensure the operation happens before the connection is brought
                  * down.
                  *
                  * If we really wanted to, we could order these after the
                  * connection start, IFF the container's current role was
                  * stopped (otherwise we re-introduce an ordering loop when the
                  * connection is restarting).
                  */
                 order_action_then_stop(action, remote_rsc, pe_order_none,
                                        data_set);
             }
             break;
 
         default:
             /* Wait for the connection resource to be up */
             if (is_recurring_action(action)) {
                 /* In case we ever get the recovery logic wrong, force
                  * recurring monitors to be restarted, even if just
                  * the connection was re-established
                  */
                 if(task != no_action) {
                     order_start_then_action(remote_rsc, action,
                                             pe_order_implies_then, data_set);
                 }
             } else {
                 order_start_then_action(remote_rsc, action, pe_order_none,
                                         data_set);
             }
             break;
     }
 }
 
 static enum remote_connection_state
 get_remote_node_state(pe_node_t *node) 
 {
     resource_t *remote_rsc = NULL;
     node_t *cluster_node = NULL;
 
     CRM_ASSERT(node);
 
     remote_rsc = node->details->remote_rsc;
     CRM_ASSERT(remote_rsc);
 
     if(remote_rsc->running_on) {
         cluster_node = remote_rsc->running_on->data;
     }
 
 
     /* If the cluster node the remote connection resource resides on
      * is unclean or went offline, we can't process any operations
      * on that remote node until after it starts elsewhere.
      */
     if(remote_rsc->next_role == RSC_ROLE_STOPPED || remote_rsc->allocated_to == NULL) {
         /* The connection resource is not going to run anywhere */
 
         if (cluster_node && cluster_node->details->unclean) {
             /* The remote connection is failed because its resource is on a
              * failed node and can't be recovered elsewhere, so we must fence.
              */
             return remote_state_failed;
         }
 
         if (is_not_set(remote_rsc->flags, pe_rsc_failed)) {
             /* Connection resource is cleanly stopped */
             return remote_state_stopped;
         }
 
         /* Connection resource is failed */
 
         if ((remote_rsc->next_role == RSC_ROLE_STOPPED)
             && remote_rsc->remote_reconnect_interval
             && node->details->remote_was_fenced) {
 
             /* We won't know whether the connection is recoverable until the
              * reconnect interval expires and we reattempt connection.
              */
             return remote_state_unknown;
         }
 
         /* The remote connection is in a failed state. If there are any
          * resources known to be active on it (stop) or in an unknown state
          * (probe), we must assume the worst and fence it.
          */
         return remote_state_failed;
 
     } else if (cluster_node == NULL) {
         /* Connection is recoverable but not currently running anywhere, see if we can recover it first */
         return remote_state_unknown;
 
     } else if(cluster_node->details->unclean == TRUE
               || cluster_node->details->online == FALSE) {
         /* Connection is running on a dead node, see if we can recover it first */
         return remote_state_resting;
 
     } else if (g_list_length(remote_rsc->running_on) > 1
                && remote_rsc->partial_migration_source
                && remote_rsc->partial_migration_target) {
         /* We're in the middle of migrating a connection resource,
          * wait until after the resource migrates before performing
          * any actions.
          */
         return remote_state_resting;
 
     }
     return remote_state_alive;
 }
 
 static void
 apply_remote_ordering(action_t *action, pe_working_set_t *data_set)
 {
     resource_t *remote_rsc = NULL;
     node_t *cluster_node = NULL;
     enum action_tasks task = text2task(action->task);
     enum remote_connection_state state = get_remote_node_state(action->node);
 
     enum pe_ordering order_opts = pe_order_none;
 
     if (action->rsc == NULL) {
         return;
     }
 
     CRM_ASSERT(action->node);
     CRM_ASSERT(is_remote_node(action->node));
 
     remote_rsc = action->node->details->remote_rsc;
     CRM_ASSERT(remote_rsc);
 
     if(remote_rsc->running_on) {
         cluster_node = remote_rsc->running_on->data;
     }
 
     crm_trace("Order %s action %s relative to %s%s (state %d)",
               action->task, action->uuid,
               is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
               remote_rsc->id, state);
 
     if (safe_str_eq(action->task, CRMD_ACTION_MIGRATE)
         || safe_str_eq(action->task, CRMD_ACTION_MIGRATE)) {
         /* Migration ops map to "no_action", but we need to apply the same
          * ordering as for stop or demote (see get_router_node()).
          */
         task = stop_rsc;
     }
 
     switch (task) {
         case start_rsc:
         case action_promote:
             order_opts = pe_order_none;
 
             if (state == remote_state_failed) {
                 /* Force recovery, by making this action required */
                 order_opts |= pe_order_implies_then;
             }
 
             /* Ensure connection is up before running this action */
             order_start_then_action(remote_rsc, action, order_opts, data_set);
             break;
 
         case stop_rsc:
             /* Handle special case with remote node where stop actions need to be
              * ordered after the connection resource starts somewhere else.
              */
             if(state == remote_state_resting) {
                 /* Wait for the connection resource to be up and assume everything is as we left it */
                 order_start_then_action(remote_rsc, action, pe_order_none,
                                         data_set);
 
             } else {
                 if(state == remote_state_failed) {
                     /* We would only be here if the resource is
                      * running on the remote node.  Since we have no
                      * way to stop it, it is necessary to fence the
                      * node.
                      */
                     pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable");
                 }
 
                 order_action_then_stop(action, remote_rsc,
                                        pe_order_implies_first, data_set);
             }
             break;
 
         case action_demote:
             /* Only order this demote relative to the connection start if the
              * connection isn't being torn down. Otherwise, the demote would be
              * blocked because the connection start would not be allowed.
              */
             if(state == remote_state_resting || state == remote_state_unknown) {
                 order_start_then_action(remote_rsc, action, pe_order_none,
                                         data_set);
             } /* Otherwise we can rely on the stop ordering */
             break;
 
         default:
             /* Wait for the connection resource to be up */
             if (is_recurring_action(action)) {
                 /* In case we ever get the recovery logic wrong, force
                  * recurring monitors to be restarted, even if just
                  * the connection was re-established
                  */
                 order_start_then_action(remote_rsc, action,
                                         pe_order_implies_then, data_set);
 
             } else {
                 if(task == monitor_rsc && state == remote_state_failed) {
                     /* We would only be here if we do not know the
                      * state of the resource on the remote node.
                      * Since we have no way to find out, it is
                      * necessary to fence the node.
                      */
                     pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable");
                 }
 
                 if(cluster_node && state == remote_state_stopped) {
                     /* The connection is currently up, but is going
                      * down permanently.
                      *
                      * Make sure we check services are actually
                      * stopped _before_ we let the connection get
                      * closed
                      */
                     order_action_then_stop(action, remote_rsc,
                                            pe_order_runnable_left, data_set);
 
                 } else {
                     order_start_then_action(remote_rsc, action, pe_order_none,
                                             data_set);
                 }
             }
             break;
     }
 }
 
 static void
 apply_remote_node_ordering(pe_working_set_t *data_set)
 {
     if (is_set(data_set->flags, pe_flag_have_remote_nodes) == FALSE) {
         return;
     }
 
     for (GListPtr gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         action_t *action = (action_t *) gIter->data;
         resource_t *remote = NULL;
 
         // We are only interested in resource actions
         if (action->rsc == NULL) {
             continue;
         }
 
         /* Special case: If we are clearing the failcount of an actual
          * remote connection resource, then make sure this happens before
          * any start of the resource in this transition.
          */
         if (action->rsc->is_remote_node &&
             safe_str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT)) {
 
             custom_action_order(action->rsc,
                 NULL,
                 action,
                 action->rsc,
                 generate_op_key(action->rsc->id, RSC_START, 0),
                 NULL,
                 pe_order_optional,
                 data_set);
 
             continue;
         }
 
         // We are only interested in actions allocated to a node
         if (action->node == NULL) {
             continue;
         }
 
         if (is_remote_node(action->node) == FALSE) {
             continue;
         }
 
         /* We are only interested in real actions.
          *
          * @TODO This is probably wrong; pseudo-actions might be converted to
          * real actions and vice versa later in update_actions() at the end of
          * stage7().
          */
         if (is_set(action->flags, pe_action_pseudo)) {
             continue;
         }
 
         remote = action->node->details->remote_rsc;
         if (remote == NULL) {
             // Orphaned
             continue;
         }
 
         /* The action occurs across a remote connection, so create
          * ordering constraints that guarantee the action occurs while the node
          * is active (after start, before stop ... things like that).
          *
          * This is somewhat brittle in that we need to make sure the results of
          * this ordering are compatible with the result of get_router_node().
          * It would probably be better to add XML_LRM_ATTR_ROUTER_NODE as part
          * of this logic rather than action2xml().
          */
         if (remote->container) {
             crm_trace("Container ordering for %s", action->uuid);
             apply_container_ordering(action, data_set);
 
         } else {
             crm_trace("Remote ordering for %s", action->uuid);
             apply_remote_ordering(action, data_set);
         }
     }
 }
 
 static void
 order_probes(pe_working_set_t * data_set) 
 {
 #if 0
     GListPtr gIter = NULL;
 
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         /* Given "A then B", we would prefer to wait for A to be
          * started before probing B.
          *
          * If A was a filesystem on which the binaries and data for B
          * lived, it would have been useful if the author of B's agent
          * could assume that A is running before B.monitor will be
          * called.
          *
          * However we can't _only_ probe once A is running, otherwise
          * we'd not detect the state of B if A could not be started
          * for some reason.
          *
          * In practice however, we cannot even do an opportunistic
          * version of this because B may be moving:
          *
          *   B.probe -> B.start
          *   B.probe -> B.stop
          *   B.stop -> B.start
          *   A.stop -> A.start
          *   A.start -> B.probe
          *
          * So far so good, but if we add the result of this code:
          *
          *   B.stop -> A.stop
          *
          * Then we get a loop:
          *
          *   B.probe -> B.stop -> A.stop -> A.start -> B.probe
          *
          * We could kill the 'B.probe -> B.stop' dependency, but that
          * could mean stopping B "too" soon, because B.start must wait
          * for the probes to complete.
          *
          * Another option is to allow it only if A is a non-unique
          * clone with clone-max == node-max (since we'll never be
          * moving it).  However, we could still be stopping one
          * instance at the same time as starting another.
 
          * The complexity of checking for allowed conditions combined
          * with the ever narrowing usecase suggests that this code
          * should remain disabled until someone gets smarter.
          */
         action_t *start = NULL;
         GListPtr actions = NULL;
         GListPtr probes = NULL;
         char *key = NULL;
 
         key = start_key(rsc);
         actions = find_actions(rsc->actions, key, NULL);
         free(key);
 
         if (actions) {
             start = actions->data;
             g_list_free(actions);
         }
 
         if(start == NULL) {
             crm_err("No start action for %s", rsc->id);
             continue;
         }
 
         key = generate_op_key(rsc->id, CRMD_ACTION_STATUS, 0);
         probes = find_actions(rsc->actions, key, NULL);
         free(key);
 
         for (actions = start->actions_before; actions != NULL; actions = actions->next) {
             action_wrapper_t *before = (action_wrapper_t *) actions->data;
 
             GListPtr pIter = NULL;
             action_t *first = before->action;
             resource_t *first_rsc = first->rsc;
 
             if(first->required_runnable_before) {
                 GListPtr clone_actions = NULL;
                 for (clone_actions = first->actions_before; clone_actions != NULL; clone_actions = clone_actions->next) {
                     before = (action_wrapper_t *) clone_actions->data;
 
                     crm_trace("Testing %s -> %s (%p) for %s", first->uuid, before->action->uuid, before->action->rsc, start->uuid);
 
                     CRM_ASSERT(before->action->rsc);
                     first_rsc = before->action->rsc;
                     break;
                 }
 
             } else if(safe_str_neq(first->task, RSC_START)) {
                 crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
             }
 
             if(first_rsc == NULL) {
                 continue;
 
             } else if(uber_parent(first_rsc) == uber_parent(start->rsc)) {
                 crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
                 continue;
 
             } else if(FALSE && pe_rsc_is_clone(uber_parent(first_rsc)) == FALSE) {
                 crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
                 continue;
             }
 
             crm_err("Applying %s before %s %d", first->uuid, start->uuid, uber_parent(first_rsc)->variant);
 
             for (pIter = probes; pIter != NULL; pIter = pIter->next) {
                 action_t *probe = (action_t *) pIter->data;
 
                 crm_err("Ordering %s before %s", first->uuid, probe->uuid);
                 order_actions(first, probe, pe_order_optional);
             }
         }
     }
 #endif
 }
 
 gboolean
 stage7(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
 
     crm_trace("Applying ordering constraints");
 
     /* Don't ask me why, but apparently they need to be processed in
      * the order they were created in... go figure
      *
      * Also g_list_append() has horrendous performance characteristics
      * So we need to use g_list_prepend() and then reverse the list here
      */
     data_set->ordering_constraints = g_list_reverse(data_set->ordering_constraints);
 
     for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
         order_constraint_t *order = (order_constraint_t *) gIter->data;
         resource_t *rsc = order->lh_rsc;
 
         crm_trace("Applying ordering constraint: %d", order->id);
 
         if (rsc != NULL) {
             crm_trace("rsc_action-to-*");
             rsc_order_first(rsc, order, data_set);
             continue;
         }
 
         rsc = order->rh_rsc;
         if (rsc != NULL) {
             crm_trace("action-to-rsc_action");
             rsc_order_then(order->lh_action, rsc, order);
 
         } else {
             crm_trace("action-to-action");
             order_actions(order->lh_action, order->rh_action, order->type);
         }
     }
 
     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         action_t *action = (action_t *) gIter->data;
 
         update_colo_start_chain(action);
     }
 
     crm_trace("Ordering probes");
     order_probes(data_set);
 
     crm_trace("Updating %d actions", g_list_length(data_set->actions));
     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         action_t *action = (action_t *) gIter->data;
 
         update_action(action);
     }
 
     LogNodeActions(data_set, FALSE);
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         LogActions(rsc, data_set, FALSE);
     }
     return TRUE;
 }
 
 int transition_id = -1;
 
 /*
  * Create a dependency graph to send to the transitioner (via the CRMd)
  */
 gboolean
 stage8(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     const char *value = NULL;
 
     transition_id++;
     crm_trace("Creating transition graph %d.", transition_id);
 
     data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);
 
     value = pe_pref(data_set->config_hash, "cluster-delay");
     crm_xml_add(data_set->graph, "cluster-delay", value);
 
     value = pe_pref(data_set->config_hash, "stonith-timeout");
     crm_xml_add(data_set->graph, "stonith-timeout", value);
 
     crm_xml_add(data_set->graph, "failed-stop-offset", "INFINITY");
 
     if (is_set(data_set->flags, pe_flag_start_failure_fatal)) {
         crm_xml_add(data_set->graph, "failed-start-offset", "INFINITY");
     } else {
         crm_xml_add(data_set->graph, "failed-start-offset", "1");
     }
 
     value = pe_pref(data_set->config_hash, "batch-limit");
     crm_xml_add(data_set->graph, "batch-limit", value);
 
     crm_xml_add_int(data_set->graph, "transition_id", transition_id);
 
     value = pe_pref(data_set->config_hash, "migration-limit");
     if (crm_int_helper(value, NULL) > 0) {
         crm_xml_add(data_set->graph, "migration-limit", value);
     }
 
 /* errors...
    slist_iter(action, action_t, action_list, lpc,
    if(action->optional == FALSE && action->runnable == FALSE) {
    print_action("Ignoring", action, TRUE);
    }
    );
 */
 
     gIter = data_set->resources;
     for (; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         pe_rsc_trace(rsc, "processing actions for rsc=%s", rsc->id);
         rsc->cmds->expand(rsc, data_set);
     }
 
     crm_log_xml_trace(data_set->graph, "created resource-driven action list");
 
     /* pseudo action to distribute list of nodes with maintenance state update */
     add_maintenance_update(data_set);
 
     /* catch any non-resource specific actions */
     crm_trace("processing non-resource actions");
 
     gIter = data_set->actions;
     for (; gIter != NULL; gIter = gIter->next) {
         action_t *action = (action_t *) gIter->data;
 
         if (action->rsc
             && action->node
             && action->node->details->shutdown
             && is_not_set(action->rsc->flags, pe_rsc_maintenance)
             && is_not_set(action->flags, pe_action_optional)
             && is_not_set(action->flags, pe_action_runnable)
             && crm_str_eq(action->task, RSC_STOP, TRUE)
             ) {
             /* Eventually we should just ignore the 'fence' case
              * But for now it's the best way to detect (in CTS) when
              * CIB resource updates are being lost
              */
             if (is_set(data_set->flags, pe_flag_have_quorum)
                 || data_set->no_quorum_policy == no_quorum_ignore) {
                 crm_crit("Cannot %s node '%s' because of %s:%s%s (%s)",
                          action->node->details->unclean ? "fence" : "shut down",
                          action->node->details->uname, action->rsc->id,
                          is_not_set(action->rsc->flags, pe_rsc_managed) ? " unmanaged" : " blocked",
                          is_set(action->rsc->flags, pe_rsc_failed) ? " failed" : "",
                          action->uuid);
             }
         }
 
         graph_element_from_action(action, data_set);
     }
 
     crm_log_xml_trace(data_set->graph, "created generic action list");
     crm_trace("Created transition graph %d.", transition_id);
 
     return TRUE;
 }
 
 void
 LogNodeActions(pe_working_set_t * data_set, gboolean terminal)
 {
     GListPtr gIter = NULL;
 
     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         char *node_name = NULL;
         char *task = NULL;
         action_t *action = (action_t *) gIter->data;
 
         if (action->rsc != NULL) {
             continue;
         } else if (is_set(action->flags, pe_action_optional)) {
             continue;
         }
 
         if (is_container_remote_node(action->node)) {
             node_name = crm_strdup_printf("%s (resource: %s)", action->node->details->uname, action->node->details->remote_rsc->container->id);
         } else if(action->node) {
             node_name = crm_strdup_printf("%s", action->node->details->uname);
         }
 
 
         if (safe_str_eq(action->task, CRM_OP_SHUTDOWN)) {
             task = strdup("Shutdown");
         } else if (safe_str_eq(action->task, CRM_OP_FENCE)) {
             const char *op = g_hash_table_lookup(action->meta, "stonith_action");
             task = crm_strdup_printf("Fence (%s)", op);
         }
 
         if(task == NULL) {
             /* Nothing to report */
         } else if(terminal && action->reason) {
             printf(" * %s %s '%s'\n", task, node_name, action->reason);
         } else if(terminal) {
             printf(" * %s %s\n", task, node_name);
         } else if(action->reason) {
             crm_notice(" * %s %s '%s'\n", task, node_name, action->reason);
         } else {
             crm_notice(" * %s %s\n", task, node_name);
         }
 
         free(node_name);
         free(task);
     }
 }
 
 void
 cleanup_alloc_calculations(pe_working_set_t * data_set)
 {
     if (data_set == NULL) {
         return;
     }
 
     crm_trace("deleting %d order cons: %p",
               g_list_length(data_set->ordering_constraints), data_set->ordering_constraints);
     pe_free_ordering(data_set->ordering_constraints);
     data_set->ordering_constraints = NULL;
 
     crm_trace("deleting %d node cons: %p",
               g_list_length(data_set->placement_constraints), data_set->placement_constraints);
     pe_free_rsc_to_node(data_set->placement_constraints);
     data_set->placement_constraints = NULL;
 
     crm_trace("deleting %d inter-resource cons: %p",
               g_list_length(data_set->colocation_constraints), data_set->colocation_constraints);
     g_list_free_full(data_set->colocation_constraints, free);
     data_set->colocation_constraints = NULL;
 
     crm_trace("deleting %d ticket deps: %p",
               g_list_length(data_set->ticket_constraints), data_set->ticket_constraints);
     g_list_free_full(data_set->ticket_constraints, free);
     data_set->ticket_constraints = NULL;
 
     cleanup_calculations(data_set);
 }
diff --git a/pengine/container.c b/pengine/container.c
index a205c5368d..a3be33bc06 100644
--- a/pengine/container.c
+++ b/pengine/container.c
@@ -1,932 +1,939 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <crm/msg_xml.h>
 #include <allocate.h>
 #include <notif.h>
 #include <utils.h>
 
 #define VARIANT_CONTAINER 1
 #include <lib/pengine/variant.h>
 
 static bool
 is_child_container_node(container_variant_data_t *data, pe_node_t *node)
 {
     for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) {
         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
         if(node->details == tuple->node->details) {
             return TRUE;
         }
     }
     return FALSE;
 }
 
 gint sort_clone_instance(gconstpointer a, gconstpointer b, gpointer data_set);
 void distribute_children(resource_t *rsc, GListPtr children, GListPtr nodes,
                          int max, int per_host_max, pe_working_set_t * data_set);
 
 static GListPtr get_container_list(resource_t *rsc) 
 {
     GListPtr containers = NULL;
     container_variant_data_t *data = NULL;
 
     if(rsc->variant == pe_container) {
         get_container_variant_data(data, rsc);
         for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) {
             container_grouping_t *tuple = (container_grouping_t *)gIter->data;
             containers = g_list_append(containers, tuple->docker);
         }
     }
     return containers;
 }
 
 static GListPtr get_containers_or_children(resource_t *rsc) 
 {
     GListPtr containers = NULL;
     container_variant_data_t *data = NULL;
 
     if(rsc->variant == pe_container) {
         get_container_variant_data(data, rsc);
         for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) {
             container_grouping_t *tuple = (container_grouping_t *)gIter->data;
             containers = g_list_append(containers, tuple->docker);
         }
         return containers;
     } else {
         return rsc->children;
     }
 }
 
 static bool
 migration_threshold_reached(resource_t *rsc, node_t *node,
                             pe_working_set_t *data_set)
 {
     int fail_count, countdown;
 
     /* Migration threshold of 0 means never force away */
     if (rsc->migration_threshold == 0) {
         return FALSE;
     }
 
+    // If we're ignoring failures, also ignore the migration threshold
+    if (is_set(rsc->flags, pe_rsc_failure_ignored)) {
+        return FALSE;
+    }
+
     /* If there are no failures, there's no need to force away */
-    fail_count = get_failcount_all(node, rsc, NULL, data_set);
+    fail_count = pe_get_failcount(node, rsc, NULL,
+                                  pe_fc_effective|pe_fc_fillers, NULL,
+                                  data_set);
     if (fail_count <= 0) {
         return FALSE;
     }
 
     /* How many more times recovery will be tried on this node */
     countdown = QB_MAX(rsc->migration_threshold - fail_count, 0);
 
     if (countdown == 0) {
         crm_warn("Forcing %s away from %s after %d failures (max=%d)",
                  rsc->id, node->details->uname, fail_count,
                  rsc->migration_threshold);
         return TRUE;
     }
 
     crm_info("%s can fail %d more times on %s before being forced off",
              rsc->id, countdown, node->details->uname);
     return FALSE;
 }
 
 node_t *
 container_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
 {
     GListPtr containers = NULL;
     GListPtr nodes = NULL;
     container_variant_data_t *container_data = NULL;
 
     CRM_CHECK(rsc != NULL, return NULL);
 
     get_container_variant_data(container_data, rsc);
 
     set_bit(rsc->flags, pe_rsc_allocating);
     containers = get_container_list(rsc);
 
     dump_node_scores(show_scores ? 0 : scores_log_level, rsc, __FUNCTION__, rsc->allowed_nodes);
 
     nodes = g_hash_table_get_values(rsc->allowed_nodes);
     nodes = g_list_sort_with_data(nodes, sort_node_weight, NULL);
     containers = g_list_sort_with_data(containers, sort_clone_instance, data_set);
     distribute_children(rsc, containers, nodes,
                         container_data->replicas, container_data->replicas_per_host, data_set);
     g_list_free(nodes);
     g_list_free(containers);
 
     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
         pe_node_t *docker_host = tuple->docker->allocated_to;
 
         CRM_ASSERT(tuple);
         if(tuple->ip) {
             tuple->ip->cmds->allocate(tuple->ip, prefer, data_set);
         }
 
         if(tuple->remote && is_remote_node(docker_host)) {
             /* We need 'nested' connection resources to be on the same
              * host because pacemaker-remoted only supports a single
              * active connection
              */
             rsc_colocation_new("child-remote-with-docker-remote", NULL,
                                INFINITY, tuple->remote, docker_host->details->remote_rsc, NULL, NULL, data_set);
         }
 
         if(tuple->remote) {
             tuple->remote->cmds->allocate(tuple->remote, prefer, data_set);
         }
 
         // Explicitly allocate tuple->child before the container->child
         if(tuple->child) {
             pe_node_t *node = NULL;
             GHashTableIter iter;
             g_hash_table_iter_init(&iter, tuple->child->allowed_nodes);
             while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
                 if(node->details != tuple->node->details) {
                     node->weight = -INFINITY;
                 } else if(migration_threshold_reached(tuple->child, node, data_set) == FALSE) {
                     node->weight = INFINITY;
                 }
             }
 
             set_bit(tuple->child->parent->flags, pe_rsc_allocating);
             tuple->child->cmds->allocate(tuple->child, tuple->node, data_set);
             clear_bit(tuple->child->parent->flags, pe_rsc_allocating);
         }
     }
 
     if(container_data->child) {
         pe_node_t *node = NULL;
         GHashTableIter iter;
         g_hash_table_iter_init(&iter, container_data->child->allowed_nodes);
         while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
             if(is_child_container_node(container_data, node)) {
                 node->weight = 0;
             } else {
                 node->weight = -INFINITY;
             }
         }
         container_data->child->cmds->allocate(container_data->child, prefer, data_set);
     }
 
     clear_bit(rsc->flags, pe_rsc_allocating);
     clear_bit(rsc->flags, pe_rsc_provisional);
     return NULL;
 }
 
 
 void
 container_create_actions(resource_t * rsc, pe_working_set_t * data_set)
 {
     pe_action_t *action = NULL;
     GListPtr containers = NULL;
     container_variant_data_t *container_data = NULL;
 
     CRM_CHECK(rsc != NULL, return);
 
     containers = get_container_list(rsc);
     get_container_variant_data(container_data, rsc);
     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 
         CRM_ASSERT(tuple);
         if(tuple->ip) {
             tuple->ip->cmds->create_actions(tuple->ip, data_set);
         }
         if(tuple->docker) {
             tuple->docker->cmds->create_actions(tuple->docker, data_set);
         }
         if(tuple->remote) {
             tuple->remote->cmds->create_actions(tuple->remote, data_set);
         }
     }
 
     clone_create_pseudo_actions(rsc, containers, NULL, NULL,  data_set);
 
     if(container_data->child) {
         container_data->child->cmds->create_actions(container_data->child, data_set);
 
         if(container_data->child->variant == pe_master) {
             /* promote */
             action = create_pseudo_resource_op(rsc, RSC_PROMOTE, TRUE, TRUE, data_set);
             action = create_pseudo_resource_op(rsc, RSC_PROMOTED, TRUE, TRUE, data_set);
             action->priority = INFINITY;
 
             /* demote */
             action = create_pseudo_resource_op(rsc, RSC_DEMOTE, TRUE, TRUE, data_set);
             action = create_pseudo_resource_op(rsc, RSC_DEMOTED, TRUE, TRUE, data_set);
             action->priority = INFINITY;
         }
     }
 
     g_list_free(containers);
 }
 
 void
 container_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
 {
     container_variant_data_t *container_data = NULL;
 
     CRM_CHECK(rsc != NULL, return);
 
     get_container_variant_data(container_data, rsc);
 
     if(container_data->child) {
         new_rsc_order(rsc, RSC_START, container_data->child, RSC_START, pe_order_implies_first_printed, data_set);
         new_rsc_order(rsc, RSC_STOP, container_data->child, RSC_STOP, pe_order_implies_first_printed, data_set);
 
         if(container_data->child->children) {
             new_rsc_order(container_data->child, RSC_STARTED, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set);
             new_rsc_order(container_data->child, RSC_STOPPED, rsc, RSC_STOPPED, pe_order_implies_then_printed, data_set);
         } else {
             new_rsc_order(container_data->child, RSC_START, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set);
             new_rsc_order(container_data->child, RSC_STOP, rsc, RSC_STOPPED, pe_order_implies_then_printed, data_set);
         }
     }
 
     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 
         CRM_ASSERT(tuple);
         CRM_ASSERT(tuple->docker);
 
         tuple->docker->cmds->internal_constraints(tuple->docker, data_set);
 
         order_start_start(rsc, tuple->docker, pe_order_runnable_left | pe_order_implies_first_printed);
 
         if(tuple->child) {
             order_stop_stop(rsc, tuple->child, pe_order_implies_first_printed);
         }
         order_stop_stop(rsc, tuple->docker, pe_order_implies_first_printed);
         new_rsc_order(tuple->docker, RSC_START, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set);
         new_rsc_order(tuple->docker, RSC_STOP, rsc, RSC_STOPPED, pe_order_implies_then_printed, data_set);
 
         if(tuple->ip) {
             tuple->ip->cmds->internal_constraints(tuple->ip, data_set);
 
             // Start ip then docker
             new_rsc_order(tuple->ip, RSC_START, tuple->docker, RSC_START,
                           pe_order_runnable_left|pe_order_preserve, data_set);
             new_rsc_order(tuple->docker, RSC_STOP, tuple->ip, RSC_STOP,
                           pe_order_implies_first|pe_order_preserve, data_set);
 
             rsc_colocation_new("ip-with-docker", NULL, INFINITY, tuple->ip, tuple->docker, NULL, NULL, data_set);
         }
 
         if(tuple->remote) {
             /* This handles ordering and colocating remote relative to docker
              * (via "resource-with-container"). Since IP is also ordered and
              * colocated relative to docker, we don't need to do anything
              * explicit here with IP.
              */
             tuple->remote->cmds->internal_constraints(tuple->remote, data_set);
         }
 
         if(tuple->child) {
             CRM_ASSERT(tuple->remote);
 
             // Start of the remote then child is implicit in the PE's remote logic
         }
 
     }
 
     if(container_data->child) {
         container_data->child->cmds->internal_constraints(container_data->child, data_set);
         if(container_data->child->variant == pe_master) {
             master_promotion_constraints(rsc, data_set);
 
             /* child demoted before global demoted */
             new_rsc_order(container_data->child, RSC_DEMOTED, rsc, RSC_DEMOTED, pe_order_implies_then_printed, data_set);
 
             /* global demote before child demote */
             new_rsc_order(rsc, RSC_DEMOTE, container_data->child, RSC_DEMOTE, pe_order_implies_first_printed, data_set);
 
             /* child promoted before global promoted */
             new_rsc_order(container_data->child, RSC_PROMOTED, rsc, RSC_PROMOTED, pe_order_implies_then_printed, data_set);
 
             /* global promote before child promote */
             new_rsc_order(rsc, RSC_PROMOTE, container_data->child, RSC_PROMOTE, pe_order_implies_first_printed, data_set);
         }
 
     } else {
 //    int type = pe_order_optional | pe_order_implies_then | pe_order_restart;
 //        custom_action_order(rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL,
 //                            rsc, generate_op_key(rsc->id, RSC_START, 0), NULL, pe_order_optional, data_set);
     }
 }
 
 
 
 static resource_t *
 find_compatible_tuple_by_node(resource_t * rsc_lh, node_t * candidate, resource_t * rsc,
                               enum rsc_role_e filter, gboolean current)
 {
     container_variant_data_t *container_data = NULL;
 
     CRM_CHECK(candidate != NULL, return NULL);
     get_container_variant_data(container_data, rsc);
 
     crm_trace("Looking for compatible child from %s for %s on %s",
               rsc_lh->id, rsc->id, candidate->details->uname);
 
     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 
         if(is_child_compatible(tuple->docker, candidate, filter, current)) {
             crm_trace("Pairing %s with %s on %s",
                       rsc_lh->id, tuple->docker->id, candidate->details->uname);
             return tuple->docker;
         }
     }
 
     crm_trace("Can't pair %s with %s", rsc_lh->id, rsc->id);
     return NULL;
 }
 
 static resource_t *
 find_compatible_tuple(resource_t *rsc_lh, resource_t * rsc, enum rsc_role_e filter,
                       gboolean current)
 {
     GListPtr scratch = NULL;
     resource_t *pair = NULL;
     node_t *active_node_lh = NULL;
 
     active_node_lh = rsc_lh->fns->location(rsc_lh, NULL, current);
     if (active_node_lh) {
         return find_compatible_tuple_by_node(rsc_lh, active_node_lh, rsc, filter, current);
     }
 
     scratch = g_hash_table_get_values(rsc_lh->allowed_nodes);
     scratch = g_list_sort_with_data(scratch, sort_node_weight, NULL);
 
     for (GListPtr gIter = scratch; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
         pair = find_compatible_tuple_by_node(rsc_lh, node, rsc, filter, current);
         if (pair) {
             goto done;
         }
     }
 
     pe_rsc_debug(rsc, "Can't pair %s with %s", rsc_lh->id, rsc->id);
   done:
     g_list_free(scratch);
     return pair;
 }
 
 void
 container_rsc_colocation_lh(resource_t * rsc, resource_t * rsc_rh, rsc_colocation_t * constraint)
 {
     /* -- Never called --
      *
      * Instead we add the colocation constraints to the child and call from there
      */
     CRM_ASSERT(FALSE);
 }
 
 int copies_per_node(resource_t * rsc) 
 {
     /* Strictly speaking, there should be a 'copies_per_node' addition
      * to the resource function table and each case would be a
      * function.  However that would be serious overkill to return an
      * int.  In fact, it seems to me that both function tables
      * could/should be replaced by resources.{c,h} full of
      * rsc_{some_operation} functions containing a switch as below
      * which calls out to functions named {variant}_{some_operation}
      * as needed.
      */
     switch(rsc->variant) {
         case pe_unknown:
             return 0;
         case pe_native:
         case pe_group:
             return 1;
         case pe_clone:
         case pe_master:
             {
                 const char *max_clones_node = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION_NODEMAX);
                 return crm_parse_int(max_clones_node, "1");
             }
         case pe_container:
             {
                 container_variant_data_t *data = NULL;
                 get_container_variant_data(data, rsc);
                 return data->replicas_per_host;
             }
     }
     return 0;
 }
 
 void
 container_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc, rsc_colocation_t * constraint)
 {
     GListPtr allocated_rhs = NULL;
     container_variant_data_t *container_data = NULL;
 
     CRM_CHECK(constraint != NULL, return);
     CRM_CHECK(rsc_lh != NULL, pe_err("rsc_lh was NULL for %s", constraint->id); return);
     CRM_CHECK(rsc != NULL, pe_err("rsc was NULL for %s", constraint->id); return);
     CRM_ASSERT(rsc_lh->variant == pe_native);
 
     if (is_set(rsc->flags, pe_rsc_provisional)) {
         pe_rsc_trace(rsc, "%s is still provisional", rsc->id);
         return;
 
     } else if(constraint->rsc_lh->variant > pe_group) {
         resource_t *rh_child = find_compatible_tuple(rsc_lh, rsc, RSC_ROLE_UNKNOWN, FALSE);
 
         if (rh_child) {
             pe_rsc_debug(rsc, "Pairing %s with %s", rsc_lh->id, rh_child->id);
             rsc_lh->cmds->rsc_colocation_lh(rsc_lh, rh_child, constraint);
 
         } else if (constraint->score >= INFINITY) {
             crm_notice("Cannot pair %s with instance of %s", rsc_lh->id, rsc->id);
             assign_node(rsc_lh, NULL, TRUE);
 
         } else {
             pe_rsc_debug(rsc, "Cannot pair %s with instance of %s", rsc_lh->id, rsc->id);
         }
 
         return;
     }
 
     get_container_variant_data(container_data, rsc);
     pe_rsc_trace(rsc, "Processing constraint %s: %s -> %s %d",
                  constraint->id, rsc_lh->id, rsc->id, constraint->score);
 
     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 
         if (constraint->score < INFINITY) {
             tuple->docker->cmds->rsc_colocation_rh(rsc_lh, tuple->docker, constraint);
 
         } else {
             node_t *chosen = tuple->docker->fns->location(tuple->docker, NULL, FALSE);
 
             if (chosen != NULL && is_set_recursive(tuple->docker, pe_rsc_block, TRUE) == FALSE) {
                 pe_rsc_trace(rsc, "Allowing %s: %s %d", constraint->id, chosen->details->uname, chosen->weight);
                 allocated_rhs = g_list_prepend(allocated_rhs, chosen);
             }
         }
     }
 
     if (constraint->score >= INFINITY) {
         node_list_exclude(rsc_lh->allowed_nodes, allocated_rhs, FALSE);
     }
     g_list_free(allocated_rhs);
 }
 
 enum pe_action_flags
 container_action_flags(action_t * action, node_t * node)
 {
     enum pe_action_flags flags = 0;
     container_variant_data_t *data = NULL;
 
     get_container_variant_data(data, action->rsc);
     if(data->child) {
         flags = summary_action_flags(action, data->child->children, node);
 
     } else {
         GListPtr containers = get_container_list(action->rsc);
         flags = summary_action_flags(action, containers, node);
         g_list_free(containers);
     }
     return flags;
 }
 
 resource_t *
 find_compatible_child_by_node(resource_t * local_child, node_t * local_node, resource_t * rsc,
                               enum rsc_role_e filter, gboolean current)
 {
     GListPtr gIter = NULL;
     GListPtr children = NULL;
 
     if (local_node == NULL) {
         crm_err("Can't colocate unrunnable child %s with %s", local_child->id, rsc->id);
         return NULL;
     }
 
     crm_trace("Looking for compatible child from %s for %s on %s",
               local_child->id, rsc->id, local_node->details->uname);
 
     children = get_containers_or_children(rsc);
     for (gIter = children; gIter != NULL; gIter = gIter->next) {
         resource_t *child_rsc = (resource_t *) gIter->data;
 
         if(is_child_compatible(child_rsc, local_node, filter, current)) {
             crm_trace("Pairing %s with %s on %s",
                       local_child->id, child_rsc->id, local_node->details->uname);
             return child_rsc;
         }
     }
 
     crm_trace("Can't pair %s with %s", local_child->id, rsc->id);
     if(children != rsc->children) {
         g_list_free(children);
     }
     return NULL;
 }
 
 static container_grouping_t *
 tuple_for_docker(resource_t *rsc, resource_t *docker, node_t *node)
 {
     if(rsc->variant == pe_container) {
         container_variant_data_t *data = NULL;
         get_container_variant_data(data, rsc);
         for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) {
             container_grouping_t *tuple = (container_grouping_t *)gIter->data;
             if(tuple->child
                && docker == tuple->docker
                && node->details == tuple->node->details) {
                 return tuple;
             }
         }
     }
     return NULL;
 }
 
 static enum pe_graph_flags
 container_update_interleave_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
                      enum pe_action_flags filter, enum pe_ordering type)
 {
     GListPtr gIter = NULL;
     GListPtr children = NULL;
     gboolean current = FALSE;
     enum pe_graph_flags changed = pe_graph_none;
 
     /* Fix this - lazy */
     if (crm_ends_with(first->uuid, "_stopped_0")
         || crm_ends_with(first->uuid, "_demoted_0")) {
         current = TRUE;
     }
 
     children = get_containers_or_children(then->rsc);
     for (gIter = children; gIter != NULL; gIter = gIter->next) {
         resource_t *then_child = (resource_t *) gIter->data;
         resource_t *first_child = find_compatible_child(then_child, first->rsc, RSC_ROLE_UNKNOWN, current);
         if (first_child == NULL && current) {
             crm_trace("Ignore");
 
         } else if (first_child == NULL) {
             crm_debug("No match found for %s (%d / %s / %s)", then_child->id, current, first->uuid, then->uuid);
 
             /* Me no like this hack - but what else can we do?
              *
              * If there is no-one active or about to be active
              *   on the same node as then_child, then they must
              *   not be allowed to start
              */
             if (type & (pe_order_runnable_left | pe_order_implies_then) /* Mandatory */ ) {
                 pe_rsc_info(then->rsc, "Inhibiting %s from being active", then_child->id);
                 if(assign_node(then_child, NULL, TRUE)) {
                     changed |= pe_graph_updated_then;
                 }
             }
 
         } else {
             pe_action_t *first_action = NULL;
             pe_action_t *then_action = NULL;
 
             enum action_tasks task = clone_child_action(first);
             const char *first_task = task2text(task);
 
             container_grouping_t *first_tuple = tuple_for_docker(first->rsc, first_child, node);
             container_grouping_t *then_tuple = tuple_for_docker(then->rsc, then_child, node);
 
             if(strstr(first->task, "stop") && first_tuple && first_tuple->child) {
                 /* Except for 'stopped' we should be looking at the
                  * in-container resource, actions for the child will
                  * happen later and are therefor more likely to align
                  * with the user's intent.
                  */
                 first_action = find_first_action(first_tuple->child->actions, NULL, task2text(task), node);
             } else {
                 first_action = find_first_action(first_child->actions, NULL, task2text(task), node);
             }
 
             if(strstr(then->task, "mote") && then_tuple && then_tuple->child) {
                 /* Promote/demote actions will never be found for the
                  * docker resource, look in the child instead
                  *
                  * Alternatively treat:
                  *  'XXXX then promote YYYY' as 'XXXX then start container for YYYY', and
                  *  'demote XXXX then stop YYYY' as 'stop container for XXXX then stop YYYY'
                  */
                 then_action = find_first_action(then_tuple->child->actions, NULL, then->task, node);
             } else {
                 then_action = find_first_action(then_child->actions, NULL, then->task, node);
             }
 
             if (first_action == NULL) {
                 if (is_not_set(first_child->flags, pe_rsc_orphan)
                     && crm_str_eq(first_task, RSC_STOP, TRUE) == FALSE
                     && crm_str_eq(first_task, RSC_DEMOTE, TRUE) == FALSE) {
                     crm_err("Internal error: No action found for %s in %s (first)",
                             first_task, first_child->id);
 
                 } else {
                     crm_trace("No action found for %s in %s%s (first)",
                               first_task, first_child->id,
                               is_set(first_child->flags, pe_rsc_orphan) ? " (ORPHAN)" : "");
                 }
                 continue;
             }
 
             /* We're only interested if 'then' is neither stopping nor being demoted */ 
             if (then_action == NULL) {
                 if (is_not_set(then_child->flags, pe_rsc_orphan)
                     && crm_str_eq(then->task, RSC_STOP, TRUE) == FALSE
                     && crm_str_eq(then->task, RSC_DEMOTE, TRUE) == FALSE) {
                     crm_err("Internal error: No action found for %s in %s (then)",
                             then->task, then_child->id);
 
                 } else {
                     crm_trace("No action found for %s in %s%s (then)",
                               then->task, then_child->id,
                               is_set(then_child->flags, pe_rsc_orphan) ? " (ORPHAN)" : "");
                 }
                 continue;
             }
 
             if (order_actions(first_action, then_action, type)) {
                 crm_debug("Created constraint for %s (%d) -> %s (%d) %.6x",
                           first_action->uuid, is_set(first_action->flags, pe_action_optional),
                           then_action->uuid, is_set(then_action->flags, pe_action_optional), type);
                 changed |= (pe_graph_updated_first | pe_graph_updated_then);
             }
             if(first_action && then_action) {
                 changed |= then_child->cmds->update_actions(first_action, then_action, node,
                                                             first_child->cmds->action_flags(first_action, node),
                                                             filter, type);
             } else {
                 crm_err("Nothing found either for %s (%p) or %s (%p) %s",
                         first_child->id, first_action,
                         then_child->id, then_action, task2text(task));
             }
         }
     }
 
     if(children != then->rsc->children) {
         g_list_free(children);
     }
     return changed;
 }
 
 bool can_interleave_actions(pe_action_t *first, pe_action_t *then) 
 {
     bool interleave = FALSE;
     resource_t *rsc = NULL;
     const char *interleave_s = NULL;
 
     if(first->rsc == NULL || then->rsc == NULL) {
         crm_trace("Not interleaving %s with %s (both must be resources)", first->uuid, then->uuid);
         return FALSE;
     } else if(first->rsc == then->rsc) {
         crm_trace("Not interleaving %s with %s (must belong to different resources)", first->uuid, then->uuid);
         return FALSE;
     } else if(first->rsc->variant < pe_clone || then->rsc->variant < pe_clone) {
         crm_trace("Not interleaving %s with %s (both sides must be clones, masters, or bundles)", first->uuid, then->uuid);
         return FALSE;
     }
 
     if (crm_ends_with(then->uuid, "_stop_0") || crm_ends_with(then->uuid, "_demote_0")) {
         rsc = first->rsc;
     } else {
         rsc = then->rsc;
     }
 
     interleave_s = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERLEAVE);
     interleave = crm_is_true(interleave_s);
     crm_trace("Interleave %s -> %s: %s (based on %s)",
               first->uuid, then->uuid, interleave ? "yes" : "no", rsc->id);
 
     return interleave;
 }
 
 enum pe_graph_flags
 container_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
                      enum pe_action_flags filter, enum pe_ordering type)
 {
     enum pe_graph_flags changed = pe_graph_none;
 
     crm_trace("%s -> %s", first->uuid, then->uuid);
 
     if(can_interleave_actions(first, then)) {
         changed = container_update_interleave_actions(first, then, node, flags, filter, type);
 
     } else if(then->rsc) {
         GListPtr gIter = NULL;
         GListPtr children = NULL;
 
         // Handle the 'primitive' ordering case
         changed |= native_update_actions(first, then, node, flags, filter, type);
 
         // Now any children (or containers in the case of a bundle)
         children = get_containers_or_children(then->rsc);
         for (gIter = children; gIter != NULL; gIter = gIter->next) {
             resource_t *then_child = (resource_t *) gIter->data;
             enum pe_graph_flags then_child_changed = pe_graph_none;
             action_t *then_child_action = find_first_action(then_child->actions, NULL, then->task, node);
 
             if (then_child_action) {
                 enum pe_action_flags then_child_flags = then_child->cmds->action_flags(then_child_action, node);
 
                 if (is_set(then_child_flags, pe_action_runnable)) {
                     then_child_changed |=
                         then_child->cmds->update_actions(first, then_child_action, node, flags, filter, type);
                 }
                 changed |= then_child_changed;
                 if (then_child_changed & pe_graph_updated_then) {
                     for (GListPtr lpc = then_child_action->actions_after; lpc != NULL; lpc = lpc->next) {
                         action_wrapper_t *next = (action_wrapper_t *) lpc->data;
                         update_action(next->action);
                     }
                 }
             }
         }
 
         if(children != then->rsc->children) {
             g_list_free(children);
         }
     }
     return changed;
 }
 
 void
 container_rsc_location(resource_t * rsc, rsc_to_node_t * constraint)
 {
     container_variant_data_t *container_data = NULL;
     get_container_variant_data(container_data, rsc);
 
     pe_rsc_trace(rsc, "Processing location constraint %s for %s", constraint->id, rsc->id);
 
     native_rsc_location(rsc, constraint);
 
     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 
         if (tuple->docker) {
             tuple->docker->cmds->rsc_location(tuple->docker, constraint);
         }
         if(tuple->ip) {
             tuple->ip->cmds->rsc_location(tuple->ip, constraint);
         }
     }
 
     if(container_data->child && (constraint->role_filter == RSC_ROLE_SLAVE || constraint->role_filter == RSC_ROLE_MASTER)) {
         container_data->child->cmds->rsc_location(container_data->child, constraint);
         container_data->child->rsc_location = g_list_prepend(container_data->child->rsc_location, constraint);
     }
 }
 
 void
 container_expand(resource_t * rsc, pe_working_set_t * data_set)
 {
     container_variant_data_t *container_data = NULL;
 
     CRM_CHECK(rsc != NULL, return);
 
     get_container_variant_data(container_data, rsc);
 
     if(container_data->child) {
         container_data->child->cmds->expand(container_data->child, data_set);
     }
 
     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 
 
         CRM_ASSERT(tuple);
         if (tuple->docker && tuple->remote && tuple->docker->allocated_to
             && fix_remote_addr(tuple->remote)) {
 
             // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
             xmlNode *nvpair = get_xpath_object("//nvpair[@name='addr']", tuple->remote->xml, LOG_ERR);
 
             g_hash_table_replace(tuple->remote->parameters, strdup("addr"), strdup(tuple->docker->allocated_to->details->uname));
             crm_xml_add(nvpair, "value", tuple->docker->allocated_to->details->uname);
         }
         if(tuple->ip) {
             tuple->ip->cmds->expand(tuple->ip, data_set);
         }
         if(tuple->docker) {
             tuple->docker->cmds->expand(tuple->docker, data_set);
         }
         if(tuple->remote) {
             tuple->remote->cmds->expand(tuple->remote, data_set);
         }
     }
 }
 
 gboolean
 container_create_probe(resource_t * rsc, node_t * node, action_t * complete,
                    gboolean force, pe_working_set_t * data_set)
 {
     bool any_created = FALSE;
     container_variant_data_t *container_data = NULL;
 
     CRM_CHECK(rsc != NULL, return FALSE);
 
     get_container_variant_data(container_data, rsc);
     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 
         CRM_ASSERT(tuple);
         if(tuple->ip) {
             any_created |= tuple->ip->cmds->create_probe(tuple->ip, node, complete, force, data_set);
         }
         if(tuple->child && node->details == tuple->node->details) {
             any_created |= tuple->child->cmds->create_probe(tuple->child, node, complete, force, data_set);
         }
         if(tuple->docker) {
             bool created = tuple->docker->cmds->create_probe(tuple->docker, node, complete, force, data_set);
 
             if(created) {
                 any_created = TRUE;
                 /* If we're limited to one replica per host (due to
                  * the lack of an IP range probably), then we don't
                  * want any of our peer containers starting until
                  * we've established that no other copies are already
                  * running.
                  *
                  * Partly this is to ensure that replicas_per_host is
                  * observed, but also to ensure that the containers
                  * don't fail to start because the necessary port
                  * mappings (which won't include an IP for uniqueness)
                  * are already taken
                  */
 
                 for (GListPtr tIter = container_data->tuples; tIter != NULL && container_data->replicas_per_host == 1; tIter = tIter->next) {
                     container_grouping_t *other = (container_grouping_t *)tIter->data;
 
                     if ((other != tuple) && (other != NULL)
                         && (other->docker != NULL)) {
 
                         custom_action_order(tuple->docker, generate_op_key(tuple->docker->id, RSC_STATUS, 0), NULL,
                                             other->docker, generate_op_key(other->docker->id, RSC_START, 0), NULL,
                                             pe_order_optional, data_set);
                     }
                 }
             }
         }
         if(tuple->remote) {
             any_created |= tuple->remote->cmds->create_probe(tuple->remote, node, complete, force, data_set);
         }
     }
     return any_created;
 }
 
 void
 container_append_meta(resource_t * rsc, xmlNode * xml)
 {
 }
 
 GHashTable *
 container_merge_weights(resource_t * rsc, const char *rhs, GHashTable * nodes, const char *attr,
                     float factor, enum pe_weights flags)
 {
     return rsc_merge_weights(rsc, rhs, nodes, attr, factor, flags);
 }
 
 void container_LogActions(
     resource_t * rsc, pe_working_set_t * data_set, gboolean terminal)
 {
     container_variant_data_t *container_data = NULL;
 
     CRM_CHECK(rsc != NULL, return);
 
     get_container_variant_data(container_data, rsc);
     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 
         CRM_ASSERT(tuple);
         if(tuple->ip) {
             LogActions(tuple->ip, data_set, terminal);
         }
         if(tuple->docker) {
             LogActions(tuple->docker, data_set, terminal);
         }
         if(tuple->remote) {
             LogActions(tuple->remote, data_set, terminal);
         }
         if(tuple->child) {
             LogActions(tuple->child, data_set, terminal);
         }
     }
 }
diff --git a/pengine/regression.sh b/pengine/regression.sh
index 1832ce9a63..3a874640b7 100755
--- a/pengine/regression.sh
+++ b/pengine/regression.sh
@@ -1,896 +1,897 @@
 #!/bin/bash
 
  # Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  #
  # This program is free software; you can redistribute it and/or
  # modify it under the terms of the GNU General Public
  # License as published by the Free Software Foundation; either
  # version 2 of the License, or (at your option) any later version.
  #
  # This software is distributed in the hope that it will be useful,
  # but WITHOUT ANY WARRANTY; without even the implied warranty of
  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  # General Public License for more details.
  #
  # You should have received a copy of the GNU General Public
  # License along with this library; if not, write to the Free Software
  # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  #
 
 core=`dirname $0`
 . $core/regression.core.sh || exit 1
 
 DO_VERSIONED_TESTS=0
 
 create_mode="true"
 info Generating test outputs for these tests...
 # do_test file description
 info Done.
 echo ""
 
 info Performing the following tests from $io_dir
 create_mode="false"
 echo ""
 
 do_test simple1 "Offline     "
 do_test simple2 "Start       "
 do_test simple3 "Start 2     "
 do_test simple4 "Start Failed"
 do_test simple6 "Stop Start  "
 do_test simple7 "Shutdown    "
 #do_test simple8 "Stonith	"
 #do_test simple9 "Lower version"
 #do_test simple10 "Higher version"
 do_test simple11 "Priority (ne)"
 do_test simple12 "Priority (eq)"
 do_test simple8 "Stickiness"
 
 echo ""
 do_test group1 "Group		"
 do_test group2 "Group + Native	"
 do_test group3 "Group + Group	"
 do_test group4 "Group + Native (nothing)"
 do_test group5 "Group + Native (move)   "
 do_test group6 "Group + Group (move)    "
 do_test group7 "Group colocation"
 do_test group13 "Group colocation (cant run)"
 do_test group8 "Group anti-colocation"
 do_test group9 "Group recovery"
 do_test group10 "Group partial recovery"
 do_test group11 "Group target_role"
 do_test group14 "Group stop (graph terminated)"
 do_test group15 "-ve group colocation"
 do_test bug-1573 "Partial stop of a group with two children"
 do_test bug-1718 "Mandatory group ordering - Stop group_FUN"
 do_test bug-lf-2613 "Move group on failure"
 do_test bug-lf-2619 "Move group on clone failure"
 do_test group-fail "Ensure stop order is preserved for partially active groups"
 do_test group-unmanaged "No need to restart r115 because r114 is unmanaged"
 do_test group-unmanaged-stopped "Make sure r115 is stopped when r114 fails"
 do_test group-dependents "Account for the location preferences of things colocated with a group"
 
 echo ""
 do_test rsc_dep1 "Must not     "
 do_test rsc_dep3 "Must         "
 do_test rsc_dep5 "Must not 3   "
 do_test rsc_dep7 "Must 3       "
 do_test rsc_dep10 "Must (but cant)"
 do_test rsc_dep2  "Must (running) "
 do_test rsc_dep8  "Must (running : alt) "
 do_test rsc_dep4  "Must (running + move)"
 do_test asymmetric "Asymmetric - require explicit location constraints"
 
 echo ""
 do_test orphan-0 "Orphan ignore"
 do_test orphan-1 "Orphan stop"
 do_test orphan-2 "Orphan stop, remove failcount"
 
 echo ""
 do_test params-0 "Params: No change"
 do_test params-1 "Params: Changed"
 do_test params-2 "Params: Resource definition"
 do_test params-4 "Params: Reload"
 do_test params-5 "Params: Restart based on probe digest"
 do_test novell-251689 "Resource definition change + target_role=stopped"
 do_test bug-lf-2106 "Restart all anonymous clone instances after config change"
 do_test params-6 "Params: Detect reload in previously migrated resource"
 do_test nvpair-id-ref "Support id-ref in nvpair with optional name"
 do_test not-reschedule-unneeded-monitor "Do not reschedule unneeded monitors while resource definitions have changed"
 do_test reload-becomes-restart "Cancel reload if restart becomes required"
 
 echo ""
 do_test target-0 "Target Role : baseline"
 do_test target-1 "Target Role : master"
 do_test target-2 "Target Role : invalid"
 
 echo ""
 do_test base-score "Set a node's default score for all nodes"
 
 echo ""
 do_test date-1 "Dates" -t "2005-020"
 do_test date-2 "Date Spec - Pass" -t "2005-020T12:30"
 do_test date-3 "Date Spec - Fail" -t "2005-020T11:30"
 do_test origin "Timing of recurring operations" -t "2014-05-07 00:28:00" 
 do_test probe-0 "Probe (anon clone)"
 do_test probe-1 "Pending Probe"
 do_test probe-2 "Correctly re-probe cloned groups"
 do_test probe-3 "Probe (pending node)"
 do_test probe-4 "Probe (pending node + stopped resource)"
 do_test standby "Standby"
 do_test comments "Comments"
 
 echo ""
 do_test one-or-more-0 "Everything starts"
 do_test one-or-more-1 "Nothing starts because of A"
 do_test one-or-more-2 "D can start because of C"
 do_test one-or-more-3 "D cannot start because of B and C"
 do_test one-or-more-4 "D cannot start because of target-role"
 do_test one-or-more-5 "Start A and F even though C and D are stopped"
 do_test one-or-more-6 "Leave A running even though B is stopped"
 do_test one-or-more-7 "Leave A running even though C is stopped"
 do_test bug-5140-require-all-false "Allow basegrp:0 to stop"
 do_test clone-require-all-1 "clone B starts node 3 and 4"
 do_test clone-require-all-2 "clone B remains stopped everywhere"
 do_test clone-require-all-3 "clone B stops everywhere because A stops everywhere"
 do_test clone-require-all-4 "clone B remains on node 3 and 4 with only one instance of A remaining."
 do_test clone-require-all-5 "clone B starts on node 1 3 and 4"
 do_test clone-require-all-6 "clone B remains active after shutting down instances of A"
 do_test clone-require-all-7 "clone A and B both start at the same time. all instances of A start before B."
 do_test clone-require-all-no-interleave-1 "C starts everywhere after A and B"
 do_test clone-require-all-no-interleave-2 "C starts on nodes 1, 2, and 4 with only one active instance of B"
 do_test clone-require-all-no-interleave-3 "C remains active when instance of B is stopped on one node and started on another."
 do_test one-or-more-unrunnable-instances "Avoid dependencies on instances that won't ever be started"
 
 echo ""
 do_test order1 "Order start 1     "
 do_test order2 "Order start 2     "
 do_test order3 "Order stop	  "
 do_test order4 "Order (multiple)  "
 do_test order5 "Order (move)  "
 do_test order6 "Order (move w/ restart)  "
 do_test order7 "Order (mandatory)  "
 do_test order-optional "Order (score=0)  "
 do_test order-required "Order (score=INFINITY)  "
 do_test bug-lf-2171 "Prevent group start when clone is stopped"
 do_test order-clone "Clone ordering should be able to prevent startup of dependent clones"
 do_test order-sets "Ordering for resource sets"
 do_test order-serialize "Serialize resources without inhibiting migration"
 do_test order-serialize-set "Serialize a set of resources without inhibiting migration"
 do_test clone-order-primitive "Order clone start after a primitive"
 do_test clone-order-16instances "Verify ordering of 16 cloned resources"
 do_test order-optional-keyword "Order (optional keyword)"
 do_test order-mandatory "Order (mandatory keyword)"
 do_test bug-lf-2493 "Don't imply colocation requirements when applying ordering constraints with clones"
 do_test ordered-set-basic-startup "Constraint set with default order settings."
 do_test ordered-set-natural "Allow natural set ordering"
 do_test order-wrong-kind "Order (error)"
 
 echo ""
 do_test coloc-loop "Colocation - loop"
 do_test coloc-many-one "Colocation - many-to-one"
 do_test coloc-list "Colocation - many-to-one with list"
 do_test coloc-group "Colocation - groups"
 do_test coloc-slave-anti "Anti-colocation with slave shouldn't prevent master colocation"
 do_test coloc-attr "Colocation based on node attributes"
 do_test coloc-negative-group "Negative colocation with a group"
 do_test coloc-intra-set "Intra-set colocation"
 do_test bug-lf-2435 "Colocation sets with a negative score"
 do_test coloc-clone-stays-active "Ensure clones don't get stopped/demoted because a dependent must stop"
 do_test coloc_fp_logic "Verify floating point calculations in colocation are working"
 do_test colo_master_w_native "cl#5070 - Verify promotion order is affected when colocating master to native rsc."
 do_test colo_slave_w_native  "cl#5070 - Verify promotion order is affected when colocating slave to native rsc."
 do_test anti-colocation-order "cl#5187 - Prevent resources in an anti-colocation from even temporarily running on a same node"
 do_test anti-colocation-master "Organize order of actions for master resources in anti-colocations"
 do_test anti-colocation-slave "Organize order of actions for slave resources in anti-colocations"
 do_test enforce-colo1 "Always enforce B with A INFINITY."
 do_test complex_enforce_colo "Always enforce B with A INFINITY. (make sure heat-engine stops)"
 
 echo ""
 do_test rsc-sets-seq-true "Resource Sets - sequential=false"
 do_test rsc-sets-seq-false "Resource Sets - sequential=true"
 do_test rsc-sets-clone "Resource Sets - Clone"
 do_test rsc-sets-master "Resource Sets - Master"
 do_test rsc-sets-clone-1 "Resource Sets - Clone (lf#2404)"
 
 #echo ""
 #do_test agent1 "version: lt (empty)"
 #do_test agent2 "version: eq	"
 #do_test agent3 "version: gt	"
 
 echo ""
 do_test attrs1 "string: eq (and)     "
 do_test attrs2 "string: lt / gt (and)"
 do_test attrs3 "string: ne (or)      "
 do_test attrs4 "string: exists       "
 do_test attrs5 "string: not_exists   "
 do_test attrs6 "is_dc: true          "
 do_test attrs7 "is_dc: false         "
 do_test attrs8 "score_attribute      "
 do_test per-node-attrs "Per node resource parameters"
 
 echo ""
 do_test mon-rsc-1 "Schedule Monitor - start"
 do_test mon-rsc-2 "Schedule Monitor - move "
 do_test mon-rsc-3 "Schedule Monitor - pending start     "
 do_test mon-rsc-4 "Schedule Monitor - move/pending start"
 
 echo ""
 do_test rec-rsc-0 "Resource Recover - no start     "
 do_test rec-rsc-1 "Resource Recover - start        "
 do_test rec-rsc-2 "Resource Recover - monitor      "
 do_test rec-rsc-3 "Resource Recover - stop - ignore"
 do_test rec-rsc-4 "Resource Recover - stop - block "
 do_test rec-rsc-5 "Resource Recover - stop - fence "
 do_test rec-rsc-6 "Resource Recover - multiple - restart"
 do_test rec-rsc-7 "Resource Recover - multiple - stop   "
 do_test rec-rsc-8 "Resource Recover - multiple - block  "
 do_test rec-rsc-9 "Resource Recover - group/group"
 do_test monitor-recovery "on-fail=block + resource recovery detected by recurring monitor"
 do_test stop-failure-no-quorum "Stop failure without quorum"
 do_test stop-failure-no-fencing "Stop failure without fencing available"
 do_test stop-failure-with-fencing "Stop failure with fencing available"
 do_test multiple-active-block-group "Support of multiple-active=block for resource groups"
 do_test multiple-monitor-one-failed "Consider resource failed if any of the configured monitor operations failed"
 
 echo ""
 do_test quorum-1 "No quorum - ignore"
 do_test quorum-2 "No quorum - freeze"
 do_test quorum-3 "No quorum - stop  "
 do_test quorum-4 "No quorum - start anyway"
 do_test quorum-5 "No quorum - start anyway (group)"
 do_test quorum-6 "No quorum - start anyway (clone)"
 do_test bug-cl-5212 "No promotion with no-quorum-policy=freeze"
 do_test suicide-needed-inquorate "no-quorum-policy=suicide: suicide necessary"
 do_test suicide-not-needed-initial-quorum "no-quorum-policy=suicide: suicide not necessary at initial quorum"
 do_test suicide-not-needed-never-quorate "no-quorum-policy=suicide: suicide not necessary if never quorate"
 do_test suicide-not-needed-quorate "no-quorum-policy=suicide: suicide necessary if quorate"
 
 echo ""
 do_test rec-node-1 "Node Recover - Startup   - no fence"
 do_test rec-node-2 "Node Recover - Startup   - fence   "
 do_test rec-node-3 "Node Recover - HA down   - no fence"
 do_test rec-node-4 "Node Recover - HA down   - fence   "
 do_test rec-node-5 "Node Recover - CRM down  - no fence"
 do_test rec-node-6 "Node Recover - CRM down  - fence   "
 do_test rec-node-7 "Node Recover - no quorum - ignore  "
 do_test rec-node-8 "Node Recover - no quorum - freeze  "
 do_test rec-node-9 "Node Recover - no quorum - stop    "
 do_test rec-node-10 "Node Recover - no quorum - stop w/fence"
 do_test rec-node-11 "Node Recover - CRM down w/ group - fence   "
 do_test rec-node-12 "Node Recover - nothing active - fence   "
 do_test rec-node-13 "Node Recover - failed resource + shutdown - fence   "
 do_test rec-node-15 "Node Recover - unknown lrm section"
 do_test rec-node-14 "Serialize all stonith's"
 
 echo ""
 do_test multi1 "Multiple Active (stop/start)"
 
 echo ""
 do_test migrate-begin     "Normal migration"
 do_test migrate-success   "Completed migration"
 do_test migrate-partial-1 "Completed migration, missing stop on source"
 do_test migrate-partial-2 "Successful migrate_to only"
 do_test migrate-partial-3 "Successful migrate_to only, target down"
 do_test migrate-partial-4 "Migrate from the correct host after migrate_to+migrate_from"
 do_test bug-5186-partial-migrate "Handle partial migration when src node loses membership"
 
 do_test migrate-fail-2 "Failed migrate_from"
 do_test migrate-fail-3 "Failed migrate_from + stop on source"
 do_test migrate-fail-4 "Failed migrate_from + stop on target - ideally we wouldn't need to re-stop on target"
 do_test migrate-fail-5 "Failed migrate_from + stop on source and target"
 
 do_test migrate-fail-6 "Failed migrate_to"
 do_test migrate-fail-7 "Failed migrate_to + stop on source"
 do_test migrate-fail-8 "Failed migrate_to + stop on target - ideally we wouldn't need to re-stop on target"
 do_test migrate-fail-9 "Failed migrate_to + stop on source and target"
 
 do_test migrate-stop "Migration in a stopping stack"
 do_test migrate-start "Migration in a starting stack"
 do_test migrate-stop_start "Migration in a restarting stack"
 do_test migrate-stop-complex "Migration in a complex stopping stack"
 do_test migrate-start-complex "Migration in a complex starting stack"
 do_test migrate-stop-start-complex "Migration in a complex moving stack"
 do_test migrate-shutdown "Order the post-migration 'stop' before node shutdown"
 
 do_test migrate-1 "Migrate (migrate)"
 do_test migrate-2 "Migrate (stable)"
 do_test migrate-3 "Migrate (failed migrate_to)"
 do_test migrate-4 "Migrate (failed migrate_from)"
 do_test novell-252693 "Migration in a stopping stack"
 do_test novell-252693-2 "Migration in a starting stack"
 do_test novell-252693-3 "Non-Migration in a starting and stopping stack"
 do_test bug-1820 "Migration in a group"
 do_test bug-1820-1 "Non-migration in a group"
 do_test migrate-5 "Primitive migration with a clone"
 do_test migrate-fencing "Migration after Fencing"
 do_test migrate-both-vms "Migrate two VMs that have no colocation"
 do_test migration-behind-migrating-remote "Migrate resource behind migrating remote connection"
 
 do_test 1-a-then-bm-move-b "Advanced migrate logic. A then B. migrate B."
 do_test 2-am-then-b-move-a "Advanced migrate logic, A then B, migrate A without stopping B"
 do_test 3-am-then-bm-both-migrate "Advanced migrate logic. A then B. migrate both"
 do_test 4-am-then-bm-b-not-migratable "Advanced migrate logic, A then B, B not migratable"
 do_test 5-am-then-bm-a-not-migratable "Advanced migrate logic. A then B. move both, a not migratable"
 do_test 6-migrate-group "Advanced migrate logic, migrate a group"
 do_test 7-migrate-group-one-unmigratable "Advanced migrate logic, migrate group mixed with allow-migrate true/false"
 do_test 8-am-then-bm-a-migrating-b-stopping "Advanced migrate logic, A then B, A migrating, B stopping"
 do_test 9-am-then-bm-b-migrating-a-stopping "Advanced migrate logic, A then B, B migrate, A stopping"
 do_test 10-a-then-bm-b-move-a-clone "Advanced migrate logic, A clone then B, migrate B while stopping A"
 do_test 11-a-then-bm-b-move-a-clone-starting "Advanced migrate logic, A clone then B, B moving while A is start/stopping"
 
 do_test a-promote-then-b-migrate "A promote then B start. migrate B"
 do_test a-demote-then-b-migrate "A demote then B stop. migrate B"
 
 if [ $DO_VERSIONED_TESTS -eq 1 ]; then
 	do_test migrate-versioned "Disable migration for versioned resources"
 fi
 
 #echo ""
 #do_test complex1 "Complex	"
 
 do_test bug-lf-2422 "Dependency on partially active group - stop ocfs:*"
 
 echo ""
 do_test clone-anon-probe-1 "Probe the correct (anonymous) clone instance for each node"
 do_test clone-anon-probe-2 "Avoid needless re-probing of anonymous clones"
 do_test clone-anon-failcount "Merge failcounts for anonymous clones"
 do_test inc0 "Incarnation start"
 do_test inc1 "Incarnation start order"
 do_test inc2 "Incarnation silent restart, stop, move"
 do_test inc3 "Inter-incarnation ordering, silent restart, stop, move"
 do_test inc4 "Inter-incarnation ordering, silent restart, stop, move (ordered)"
 do_test inc5 "Inter-incarnation ordering, silent restart, stop, move (restart 1)"
 do_test inc6 "Inter-incarnation ordering, silent restart, stop, move (restart 2)"
 do_test inc7 "Clone colocation"
 do_test inc8 "Clone anti-colocation"
 do_test inc9 "Non-unique clone"
 do_test inc10 "Non-unique clone (stop)"
 do_test inc11 "Primitive colocation with clones"
 do_test inc12 "Clone shutdown"
 do_test cloned-group "Make sure only the correct number of cloned groups are started"
 do_test cloned-group-stop "Ensure stopping qpidd also stops glance and cinder"
 do_test clone-no-shuffle "Don't prioritize allocation of instances that must be moved"
 do_test clone-max-zero "Orphan processing with clone-max=0"
 do_test clone-anon-dup "Bug LF#2087 - Correctly parse the state of anonymous clones that are active more than once per node"
 do_test bug-lf-2160 "Don't shuffle clones due to colocation"
 do_test bug-lf-2213 "clone-node-max enforcement for cloned groups"
 do_test bug-lf-2153 "Clone ordering constraints"
 do_test bug-lf-2361 "Ensure clones observe mandatory ordering constraints if the LHS is unrunnable"
 do_test bug-lf-2317 "Avoid needless restart of primitive depending on a clone"
 do_test clone-colocate-instance-1 "Colocation with a specific clone instance (negative example)"
 do_test clone-colocate-instance-2 "Colocation with a specific clone instance"
 do_test clone-order-instance "Ordering with specific clone instances"
 do_test bug-lf-2453 "Enforce mandatory clone ordering without colocation"
 do_test bug-lf-2508 "Correctly reconstruct the status of anonymous cloned groups"
 do_test bug-lf-2544 "Balanced clone placement"
 do_test bug-lf-2445 "Redistribute clones with node-max > 1 and stickiness = 0"
 do_test bug-lf-2574 "Avoid clone shuffle"
 do_test bug-lf-2581 "Avoid group restart due to unrelated clone (re)start"
 do_test bug-cl-5168 "Don't shuffle clones"
 do_test bug-cl-5170 "Prevent clone from starting with on-fail=block"
 do_test clone-fail-block-colocation "Move colocated group when failed clone has on-fail=block"
 do_test clone-interleave-1 "Clone-3 cannot start on pcmk-1 due to interleaved ordering (no colocation)"
 do_test clone-interleave-2 "Clone-3 must stop on pcmk-1 due to interleaved ordering (no colocation)"
 do_test clone-interleave-3 "Clone-3 must be recovered on pcmk-1 due to interleaved ordering (no colocation)"
 do_test rebalance-unique-clones "Rebalance unique clone instances with no stickiness"
 
 echo ""
 do_test cloned_start_one  "order first clone then clone... first clone_min=2"
 do_test cloned_start_two  "order first clone then clone... first clone_min=2"
 do_test cloned_stop_one   "order first clone then clone... first clone_min=2"
 do_test cloned_stop_two   "order first clone then clone... first clone_min=2"
 do_test clone_min_interleave_start_one "order first clone then clone... first clone_min=2 and then has interleave=true"
 do_test clone_min_interleave_start_two "order first clone then clone... first clone_min=2 and then has interleave=true"
 do_test clone_min_interleave_stop_one  "order first clone then clone... first clone_min=2 and then has interleave=true"
 do_test clone_min_interleave_stop_two  "order first clone then clone... first clone_min=2 and then has interleave=true"
 do_test clone_min_start_one "order first clone then primitive... first clone_min=2"
 do_test clone_min_start_two "order first clone then primitive... first clone_min=2"
 do_test clone_min_stop_all  "order first clone then primitive... first clone_min=2"
 do_test clone_min_stop_one  "order first clone then primitive... first clone_min=2"
 do_test clone_min_stop_two  "order first clone then primitive... first clone_min=2"
 
 echo ""
 do_test unfence-startup "Clean unfencing"
 do_test unfence-definition "Unfencing when the agent changes"
 do_test unfence-parameters "Unfencing when the agent parameters changes"
 
 echo ""
 do_test master-0 "Stopped -> Slave"
 do_test master-1 "Stopped -> Promote"
 do_test master-2 "Stopped -> Promote : notify"
 do_test master-3 "Stopped -> Promote : master location"
 do_test master-4 "Started -> Promote : master location"
 do_test master-5 "Promoted -> Promoted"
 do_test master-6 "Promoted -> Promoted (2)"
 do_test master-7 "Promoted -> Fenced"
 do_test master-8 "Promoted -> Fenced -> Moved"
 do_test master-9 "Stopped + Promotable + No quorum"
 do_test master-10 "Stopped -> Promotable : notify with monitor"
 do_test master-11 "Stopped -> Promote : colocation"
 do_test novell-239082 "Demote/Promote ordering"
 do_test novell-239087 "Stable master placement"
 do_test master-12 "Promotion based solely on rsc_location constraints"
 do_test master-13 "Include preferences of colocated resources when placing master"
 do_test master-demote "Ordering when actions depends on demoting a slave resource"
 do_test master-ordering "Prevent resources from starting that need a master"
 do_test bug-1765 "Master-Master Colocation (dont stop the slaves)"
 do_test master-group "Promotion of cloned groups"
 do_test bug-lf-1852 "Don't shuffle master/slave instances unnecessarily"
 do_test master-failed-demote "Don't retry failed demote actions"
 do_test master-failed-demote-2 "Don't retry failed demote actions (notify=false)"
 do_test master-depend "Ensure resources that depend on the master don't get allocated until the master does"
 do_test master-reattach "Re-attach to a running master"
 do_test master-allow-start "Don't include master score if it would prevent allocation"
 do_test master-colocation "Allow master instances placemaker to be influenced by colocation constraints"
 do_test master-pseudo "Make sure promote/demote pseudo actions are created correctly"
 do_test master-role "Prevent target-role from promoting more than master-max instances"
 do_test bug-lf-2358 "Master-Master anti-colocation"
 do_test master-promotion-constraint "Mandatory master colocation constraints"
 do_test unmanaged-master "Ensure role is preserved for unmanaged resources"
 do_test master-unmanaged-monitor "Start the correct monitor operation for unmanaged masters"
 do_test master-demote-2 "Demote does not clear past failure"
 do_test master-move "Move master based on failure of colocated group"
 do_test master-probed-score "Observe the promotion score of probed resources"
 do_test colocation_constraint_stops_master "cl#5054 - Ensure master is demoted when stopped by colocation constraint"
 do_test colocation_constraint_stops_slave  "cl#5054 - Ensure slave is not demoted when stopped by colocation constraint"
 do_test order_constraint_stops_master      "cl#5054 - Ensure master is demoted when stopped by order constraint"
 do_test order_constraint_stops_slave       "cl#5054 - Ensure slave is not demoted when stopped by order constraint"
 do_test master_monitor_restart "cl#5072 - Ensure master monitor operation will start after promotion."
 do_test bug-rh-880249 "Handle replacement of an m/s resource with a primitive"
 do_test bug-5143-ms-shuffle "Prevent master shuffling due to promotion score"
 do_test master-demote-block "Block promotion if demote fails with on-fail=block"
 do_test master-dependent-ban "Don't stop instances from being active because a dependent is banned from that host"
 do_test master-stop "Stop instances due to location constraint with role=Started"
 do_test master-partially-demoted-group "Allow partially demoted group to finish demoting"
 do_test bug-cl-5213 "Ensure role colocation with -INFINITY is enforced"
 do_test bug-cl-5219 "Allow unrelated resources with a common colocation target to remain promoted"
 do_test master-asymmetrical-order "Fix the behaviors of multi-state resources with asymmetrical ordering"
 do_test master-notify "Master promotion with notifies"
 do_test master-score-startup "Use permanent master scores without LRM history"
 
 echo ""
 do_test history-1 "Correctly parse stateful-1 resource state"
 
 echo ""
 do_test managed-0 "Managed (reference)"
 do_test managed-1 "Not managed - down "
 do_test managed-2 "Not managed - up   "
 do_test bug-5028 "Shutdown should block if anything depends on an unmanaged resource"
 do_test bug-5028-detach "Ensure detach still works"
 do_test bug-5028-bottom "Ensure shutdown still blocks if the blocked resource is at the bottom of the stack"
 do_test unmanaged-stop-1 "cl#5155 - Block the stop of resources if any depending resource is unmanaged "
 do_test unmanaged-stop-2 "cl#5155 - Block the stop of resources if the first resource in a mandatory stop order is unmanaged "
 do_test unmanaged-stop-3 "cl#5155 - Block the stop of resources if any depending resource in a group is unmanaged "
 do_test unmanaged-stop-4 "cl#5155 - Block the stop of resources if any depending resource in the middle of a group is unmanaged "
 do_test unmanaged-block-restart "Block restart of resources if any dependent resource in a group is unmanaged"
 
 echo ""
 do_test interleave-0 "Interleave (reference)"
 do_test interleave-1 "coloc - not interleaved"
 do_test interleave-2 "coloc - interleaved   "
 do_test interleave-3 "coloc - interleaved (2)"
 do_test interleave-pseudo-stop "Interleaved clone during stonith"
 do_test interleave-stop "Interleaved clone during stop"
 do_test interleave-restart "Interleaved clone during dependency restart"
 
 echo ""
 do_test notify-0 "Notify reference"
 do_test notify-1 "Notify simple"
 do_test notify-2 "Notify simple, confirm"
 do_test notify-3 "Notify move, confirm"
 do_test novell-239079 "Notification priority"
 #do_test notify-2 "Notify - 764"
 
 echo ""
 do_test 594 "OSDL #594 - Unrunnable actions scheduled in transition"
 do_test 662 "OSDL #662 - Two resources start on one node when incarnation_node_max = 1"
 do_test 696 "OSDL #696 - CRM starts stonith RA without monitor"
 do_test 726 "OSDL #726 - Attempting to schedule rsc_posic041_monitor_5000 _after_ a stop"
 do_test 735 "OSDL #735 - Correctly detect that rsc_hadev1 is stopped on hadev3"
 do_test 764 "OSDL #764 - Missing monitor op for DoFencing:child_DoFencing:1"
 do_test 797 "OSDL #797 - Assert triggered: task_id_i > max_call_id"
 do_test 829 "OSDL #829"
 do_test 994 "OSDL #994 - Stopping the last resource in a resource group causes the entire group to be restarted"
 do_test 994-2 "OSDL #994 - with a dependent resource"
 do_test 1360 "OSDL #1360 - Clone stickiness"
 do_test 1484 "OSDL #1484 - on_fail=stop"
 do_test 1494 "OSDL #1494 - Clone stability"
 do_test unrunnable-1 "Unrunnable"
 do_test unrunnable-2 "Unrunnable 2"
 do_test stonith-0 "Stonith loop - 1"
 do_test stonith-1 "Stonith loop - 2"
 do_test stonith-2 "Stonith loop - 3"
 do_test stonith-3 "Stonith startup"
 do_test stonith-4 "Stonith node state"
 do_test bug-1572-1 "Recovery of groups depending on master/slave"
 do_test bug-1572-2 "Recovery of groups depending on master/slave when the master is never re-promoted"
 do_test bug-1685 "Depends-on-master ordering"
 do_test bug-1822 "Don't promote partially active groups"
 do_test bug-pm-11 "New resource added to a m/s group"
 do_test bug-pm-12 "Recover only the failed portion of a cloned group"
 do_test bug-n-387749 "Don't shuffle clone instances"
 do_test bug-n-385265 "Don't ignore the failure stickiness of group children - resource_idvscommon should stay stopped"
 do_test bug-n-385265-2 "Ensure groups are migrated instead of remaining partially active on the current node"
 do_test bug-lf-1920 "Correctly handle probes that find active resources"
 do_test bnc-515172 "Location constraint with multiple expressions"
 do_test colocate-primitive-with-clone "Optional colocation with a clone"
 do_test use-after-free-merge "Use-after-free in native_merge_weights"
 do_test bug-lf-2551 "STONITH ordering for stop"
 do_test bug-lf-2606 "Stonith implies demote"
 do_test bug-lf-2474 "Ensure resource op timeout takes precedence over op_defaults"
 do_test bug-suse-707150 "Prevent vm-01 from starting due to colocation/ordering"
 do_test bug-5014-A-start-B-start "Verify when A starts B starts using symmetrical=false"
 do_test bug-5014-A-stop-B-started "Verify when A stops B does not stop if it has already started using symmetric=false"
 do_test bug-5014-A-stopped-B-stopped "Verify when A is stopped and B has not started, B does not start before A using symmetric=false"
 do_test bug-5014-CthenAthenB-C-stopped "Verify when C then A is symmetrical=true, A then B is symmetric=false, and C is stopped that nothing starts."
 do_test bug-5014-CLONE-A-start-B-start "Verify when A starts B starts using clone resources with symmetric=false"
 do_test bug-5014-CLONE-A-stop-B-started "Verify when A stops B does not stop if it has already started using clone resources with symmetric=false."
 do_test bug-5014-GROUP-A-start-B-start "Verify when A starts B starts when using group resources with symmetric=false."
 do_test bug-5014-GROUP-A-stopped-B-started "Verify when A stops B does not stop if it has already started using group resources with symmetric=false."
 do_test bug-5014-GROUP-A-stopped-B-stopped "Verify when A is stopped and B has not started, B does not start before A using group resources with symmetric=false."
 do_test bug-5014-ordered-set-symmetrical-false "Verify ordered sets work with symmetrical=false"
 do_test bug-5014-ordered-set-symmetrical-true "Verify ordered sets work with symmetrical=true"
 do_test bug-5007-masterslave_colocation "Verify use of colocation scores other than INFINITY and -INFINITY work on multi-state resources."
 do_test bug-5038 "Prevent restart of anonymous clones when clone-max decreases"
 do_test bug-5025-1 "Automatically clean up failcount after resource config change with reload"
 do_test bug-5025-2 "Make sure clear failcount action isn't set when config does not change."
 do_test bug-5025-3 "Automatically clean up failcount after resource config change with restart"
 do_test bug-5025-4 "Clear failcount when last failure is a start op and rsc attributes changed."
 do_test failcount "Ensure failcounts are correctly expired"
 do_test failcount-block "Ensure failcounts are not expired when on-fail=block is present"
 do_test per-op-failcount "Ensure per-operation failcount is handled and not passed to fence agent"
+do_test on-fail-ignore "Ensure on-fail=ignore works even beyond migration-threshold"
 do_test monitor-onfail-restart "bug-5058 - Monitor failure with on-fail set to restart"
 do_test monitor-onfail-stop    "bug-5058 - Monitor failure wiht on-fail set to stop"
 do_test bug-5059 "No need to restart p_stateful1:*"
 do_test bug-5069-op-enabled  "Test on-fail=ignore with failure when monitor is enabled."
 do_test bug-5069-op-disabled "Test on-fail-ignore with failure when monitor is disabled."
 do_test obsolete-lrm-resource "cl#5115 - Do not use obsolete lrm_resource sections"
 do_test expire-non-blocked-failure "Ignore failure-timeout only if the failed operation has on-fail=block"
 do_test asymmetrical-order-move "Respect asymmetrical ordering when trying to move resources"
 do_test start-then-stop-with-unfence "Avoid graph loop with start-then-stop constraint plus unfencing"
 
 do_test ignore_stonith_rsc_order1 "cl#5056- Ignore order constraint between stonith and non-stonith rsc."
 do_test ignore_stonith_rsc_order2 "cl#5056- Ignore order constraint with group rsc containing mixed stonith and non-stonith."
 do_test ignore_stonith_rsc_order3 "cl#5056- Ignore order constraint, stonith clone and mixed group"
 do_test ignore_stonith_rsc_order4 "cl#5056- Ignore order constraint, stonith clone and clone with nested mixed group"
 do_test honor_stonith_rsc_order1 "cl#5056- Honor order constraint, stonith clone and pure stonith group(single rsc)."
 do_test honor_stonith_rsc_order2 "cl#5056- Honor order constraint, stonith clone and pure stonith group(multiple rsc)"
 do_test honor_stonith_rsc_order3 "cl#5056- Honor order constraint, stonith clones with nested pure stonith group."
 do_test honor_stonith_rsc_order4 "cl#5056- Honor order constraint, between two native stonith rscs."
 do_test probe-timeout "cl#5099 - Default probe timeout"
 
 do_test concurrent-fencing "Allow performing fencing operations in parallel"
 
 echo ""
 do_test systemhealth1  "System Health ()               #1"
 do_test systemhealth2  "System Health ()               #2"
 do_test systemhealth3  "System Health ()               #3"
 do_test systemhealthn1 "System Health (None)           #1"
 do_test systemhealthn2 "System Health (None)           #2"
 do_test systemhealthn3 "System Health (None)           #3"
 do_test systemhealthm1 "System Health (Migrate On Red) #1"
 do_test systemhealthm2 "System Health (Migrate On Red) #2"
 do_test systemhealthm3 "System Health (Migrate On Red) #3"
 do_test systemhealtho1 "System Health (Only Green)     #1"
 do_test systemhealtho2 "System Health (Only Green)     #2"
 do_test systemhealtho3 "System Health (Only Green)     #3"
 do_test systemhealthp1 "System Health (Progessive)     #1"
 do_test systemhealthp2 "System Health (Progessive)     #2"
 do_test systemhealthp3 "System Health (Progessive)     #3"
 
 echo ""
 do_test utilization "Placement Strategy - utilization"
 do_test minimal     "Placement Strategy - minimal"
 do_test balanced    "Placement Strategy - balanced"
 
 echo ""
 do_test placement-stickiness "Optimized Placement Strategy - stickiness"
 do_test placement-priority   "Optimized Placement Strategy - priority"
 do_test placement-location   "Optimized Placement Strategy - location"
 do_test placement-capacity   "Optimized Placement Strategy - capacity"
 
 echo ""
 do_test utilization-order1 "Utilization Order - Simple"
 do_test utilization-order2 "Utilization Order - Complex"
 do_test utilization-order3 "Utilization Order - Migrate"
 do_test utilization-order4 "Utilization Order - Live Mirgration (bnc#695440)"
 do_test utilization-shuffle "Don't displace prmExPostgreSQLDB2 on act2, Start prmExPostgreSQLDB1 on act3"
 do_test load-stopped-loop "Avoid transition loop due to load_stopped (cl#5044)"
 do_test load-stopped-loop-2 "cl#5235 - Prevent graph loops that can be introduced by load_stopped -> migrate_to ordering"
 
 echo ""
 do_test colocated-utilization-primitive-1 "Colocated Utilization - Primitive"
 do_test colocated-utilization-primitive-2 "Colocated Utilization - Choose the most capable node"
 do_test colocated-utilization-group "Colocated Utilization - Group"
 do_test colocated-utilization-clone "Colocated Utilization - Clone"
 
 do_test utilization-check-allowed-nodes "Only check the capacities of the nodes that can run the resource"
 
 echo ""
 do_test reprobe-target_rc "Ensure correct target_rc for reprobe of inactive resources"
 do_test node-maintenance-1 "cl#5128 - Node maintenance"
 do_test node-maintenance-2 "cl#5128 - Node maintenance (coming out of maintenance mode)"
 do_test shutdown-maintenance-node "Do not fence a maintenance node if it shuts down cleanly"
 
 do_test rsc-maintenance "Per-resource maintenance"
 
 echo ""
 do_test not-installed-agent "The resource agent is missing"
 do_test not-installed-tools "Something the resource agent needs is missing"
 
 echo ""
 do_test stopped-monitor-00	"Stopped Monitor - initial start"
 do_test stopped-monitor-01	"Stopped Monitor - failed started"
 do_test stopped-monitor-02	"Stopped Monitor - started multi-up"
 do_test stopped-monitor-03	"Stopped Monitor - stop started"
 do_test stopped-monitor-04	"Stopped Monitor - failed stop"
 do_test stopped-monitor-05	"Stopped Monitor - start unmanaged"
 do_test stopped-monitor-06	"Stopped Monitor - unmanaged multi-up"
 do_test stopped-monitor-07	"Stopped Monitor - start unmanaged multi-up"
 do_test stopped-monitor-08	"Stopped Monitor - migrate"
 do_test stopped-monitor-09	"Stopped Monitor - unmanage started"
 do_test stopped-monitor-10	"Stopped Monitor - unmanaged started multi-up"
 do_test stopped-monitor-11	"Stopped Monitor - stop unmanaged started"
 do_test stopped-monitor-12	"Stopped Monitor - unmanaged started multi-up (targer-role="Stopped")"
 do_test stopped-monitor-20	"Stopped Monitor - initial stop"
 do_test stopped-monitor-21	"Stopped Monitor - stopped single-up"
 do_test stopped-monitor-22	"Stopped Monitor - stopped multi-up"
 do_test stopped-monitor-23	"Stopped Monitor - start stopped"
 do_test stopped-monitor-24	"Stopped Monitor - unmanage stopped"
 do_test stopped-monitor-25	"Stopped Monitor - unmanaged stopped multi-up"
 do_test stopped-monitor-26	"Stopped Monitor - start unmanaged stopped"
 do_test stopped-monitor-27	"Stopped Monitor - unmanaged stopped multi-up (target-role="Started")"
 do_test stopped-monitor-30	"Stopped Monitor - new node started"
 do_test stopped-monitor-31	"Stopped Monitor - new node stopped"
 
 echo""
 do_test ticket-primitive-1 "Ticket - Primitive (loss-policy=stop, initial)"
 do_test ticket-primitive-2 "Ticket - Primitive (loss-policy=stop, granted)"
 do_test ticket-primitive-3 "Ticket - Primitive (loss-policy-stop, revoked)"
 do_test ticket-primitive-4 "Ticket - Primitive (loss-policy=demote, initial)"
 do_test ticket-primitive-5 "Ticket - Primitive (loss-policy=demote, granted)"
 do_test ticket-primitive-6 "Ticket - Primitive (loss-policy=demote, revoked)"
 do_test ticket-primitive-7 "Ticket - Primitive (loss-policy=fence, initial)"
 do_test ticket-primitive-8 "Ticket - Primitive (loss-policy=fence, granted)"
 do_test ticket-primitive-9 "Ticket - Primitive (loss-policy=fence, revoked)"
 do_test ticket-primitive-10 "Ticket - Primitive (loss-policy=freeze, initial)"
 do_test ticket-primitive-11 "Ticket - Primitive (loss-policy=freeze, granted)"
 do_test ticket-primitive-12 "Ticket - Primitive (loss-policy=freeze, revoked)"
 
 do_test ticket-primitive-13 "Ticket - Primitive (loss-policy=stop, standby, granted)"
 do_test ticket-primitive-14 "Ticket - Primitive (loss-policy=stop, granted, standby)"
 do_test ticket-primitive-15 "Ticket - Primitive (loss-policy=stop, standby, revoked)"
 do_test ticket-primitive-16 "Ticket - Primitive (loss-policy=demote, standby, granted)"
 do_test ticket-primitive-17 "Ticket - Primitive (loss-policy=demote, granted, standby)"
 do_test ticket-primitive-18 "Ticket - Primitive (loss-policy=demote, standby, revoked)"
 do_test ticket-primitive-19 "Ticket - Primitive (loss-policy=fence, standby, granted)"
 do_test ticket-primitive-20 "Ticket - Primitive (loss-policy=fence, granted, standby)"
 do_test ticket-primitive-21 "Ticket - Primitive (loss-policy=fence, standby, revoked)"
 do_test ticket-primitive-22 "Ticket - Primitive (loss-policy=freeze, standby, granted)"
 do_test ticket-primitive-23 "Ticket - Primitive (loss-policy=freeze, granted, standby)"
 do_test ticket-primitive-24 "Ticket - Primitive (loss-policy=freeze, standby, revoked)"
 
 echo""
 do_test ticket-group-1 "Ticket - Group (loss-policy=stop, initial)"
 do_test ticket-group-2 "Ticket - Group (loss-policy=stop, granted)"
 do_test ticket-group-3 "Ticket - Group (loss-policy-stop, revoked)"
 do_test ticket-group-4 "Ticket - Group (loss-policy=demote, initial)"
 do_test ticket-group-5 "Ticket - Group (loss-policy=demote, granted)"
 do_test ticket-group-6 "Ticket - Group (loss-policy=demote, revoked)"
 do_test ticket-group-7 "Ticket - Group (loss-policy=fence, initial)"
 do_test ticket-group-8 "Ticket - Group (loss-policy=fence, granted)"
 do_test ticket-group-9 "Ticket - Group (loss-policy=fence, revoked)"
 do_test ticket-group-10 "Ticket - Group (loss-policy=freeze, initial)"
 do_test ticket-group-11 "Ticket - Group (loss-policy=freeze, granted)"
 do_test ticket-group-12 "Ticket - Group (loss-policy=freeze, revoked)"
 
 do_test ticket-group-13 "Ticket - Group (loss-policy=stop, standby, granted)"
 do_test ticket-group-14 "Ticket - Group (loss-policy=stop, granted, standby)"
 do_test ticket-group-15 "Ticket - Group (loss-policy=stop, standby, revoked)"
 do_test ticket-group-16 "Ticket - Group (loss-policy=demote, standby, granted)"
 do_test ticket-group-17 "Ticket - Group (loss-policy=demote, granted, standby)"
 do_test ticket-group-18 "Ticket - Group (loss-policy=demote, standby, revoked)"
 do_test ticket-group-19 "Ticket - Group (loss-policy=fence, standby, granted)"
 do_test ticket-group-20 "Ticket - Group (loss-policy=fence, granted, standby)"
 do_test ticket-group-21 "Ticket - Group (loss-policy=fence, standby, revoked)"
 do_test ticket-group-22 "Ticket - Group (loss-policy=freeze, standby, granted)"
 do_test ticket-group-23 "Ticket - Group (loss-policy=freeze, granted, standby)"
 do_test ticket-group-24 "Ticket - Group (loss-policy=freeze, standby, revoked)"
 
 echo""
 do_test ticket-clone-1 "Ticket - Clone (loss-policy=stop, initial)"
 do_test ticket-clone-2 "Ticket - Clone (loss-policy=stop, granted)"
 do_test ticket-clone-3 "Ticket - Clone (loss-policy-stop, revoked)"
 do_test ticket-clone-4 "Ticket - Clone (loss-policy=demote, initial)"
 do_test ticket-clone-5 "Ticket - Clone (loss-policy=demote, granted)"
 do_test ticket-clone-6 "Ticket - Clone (loss-policy=demote, revoked)"
 do_test ticket-clone-7 "Ticket - Clone (loss-policy=fence, initial)"
 do_test ticket-clone-8 "Ticket - Clone (loss-policy=fence, granted)"
 do_test ticket-clone-9 "Ticket - Clone (loss-policy=fence, revoked)"
 do_test ticket-clone-10 "Ticket - Clone (loss-policy=freeze, initial)"
 do_test ticket-clone-11 "Ticket - Clone (loss-policy=freeze, granted)"
 do_test ticket-clone-12 "Ticket - Clone (loss-policy=freeze, revoked)"
 
 do_test ticket-clone-13 "Ticket - Clone (loss-policy=stop, standby, granted)"
 do_test ticket-clone-14 "Ticket - Clone (loss-policy=stop, granted, standby)"
 do_test ticket-clone-15 "Ticket - Clone (loss-policy=stop, standby, revoked)"
 do_test ticket-clone-16 "Ticket - Clone (loss-policy=demote, standby, granted)"
 do_test ticket-clone-17 "Ticket - Clone (loss-policy=demote, granted, standby)"
 do_test ticket-clone-18 "Ticket - Clone (loss-policy=demote, standby, revoked)"
 do_test ticket-clone-19 "Ticket - Clone (loss-policy=fence, standby, granted)"
 do_test ticket-clone-20 "Ticket - Clone (loss-policy=fence, granted, standby)"
 do_test ticket-clone-21 "Ticket - Clone (loss-policy=fence, standby, revoked)"
 do_test ticket-clone-22 "Ticket - Clone (loss-policy=freeze, standby, granted)"
 do_test ticket-clone-23 "Ticket - Clone (loss-policy=freeze, granted, standby)"
 do_test ticket-clone-24 "Ticket - Clone (loss-policy=freeze, standby, revoked)"
 
 echo""
 do_test ticket-master-1 "Ticket - Master (loss-policy=stop, initial)"
 do_test ticket-master-2 "Ticket - Master (loss-policy=stop, granted)"
 do_test ticket-master-3 "Ticket - Master (loss-policy-stop, revoked)"
 do_test ticket-master-4 "Ticket - Master (loss-policy=demote, initial)"
 do_test ticket-master-5 "Ticket - Master (loss-policy=demote, granted)"
 do_test ticket-master-6 "Ticket - Master (loss-policy=demote, revoked)"
 do_test ticket-master-7 "Ticket - Master (loss-policy=fence, initial)"
 do_test ticket-master-8 "Ticket - Master (loss-policy=fence, granted)"
 do_test ticket-master-9 "Ticket - Master (loss-policy=fence, revoked)"
 do_test ticket-master-10 "Ticket - Master (loss-policy=freeze, initial)"
 do_test ticket-master-11 "Ticket - Master (loss-policy=freeze, granted)"
 do_test ticket-master-12 "Ticket - Master (loss-policy=freeze, revoked)"
 
 do_test ticket-master-13 "Ticket - Master (loss-policy=stop, standby, granted)"
 do_test ticket-master-14 "Ticket - Master (loss-policy=stop, granted, standby)"
 do_test ticket-master-15 "Ticket - Master (loss-policy=stop, standby, revoked)"
 do_test ticket-master-16 "Ticket - Master (loss-policy=demote, standby, granted)"
 do_test ticket-master-17 "Ticket - Master (loss-policy=demote, granted, standby)"
 do_test ticket-master-18 "Ticket - Master (loss-policy=demote, standby, revoked)"
 do_test ticket-master-19 "Ticket - Master (loss-policy=fence, standby, granted)"
 do_test ticket-master-20 "Ticket - Master (loss-policy=fence, granted, standby)"
 do_test ticket-master-21 "Ticket - Master (loss-policy=fence, standby, revoked)"
 do_test ticket-master-22 "Ticket - Master (loss-policy=freeze, standby, granted)"
 do_test ticket-master-23 "Ticket - Master (loss-policy=freeze, granted, standby)"
 do_test ticket-master-24 "Ticket - Master (loss-policy=freeze, standby, revoked)"
 
 echo ""
 do_test ticket-rsc-sets-1 "Ticket - Resource sets (1 ticket, initial)"
 do_test ticket-rsc-sets-2 "Ticket - Resource sets (1 ticket, granted)"
 do_test ticket-rsc-sets-3 "Ticket - Resource sets (1 ticket, revoked)"
 do_test ticket-rsc-sets-4 "Ticket - Resource sets (2 tickets, initial)"
 do_test ticket-rsc-sets-5 "Ticket - Resource sets (2 tickets, granted)"
 do_test ticket-rsc-sets-6 "Ticket - Resource sets (2 tickets, granted)"
 do_test ticket-rsc-sets-7 "Ticket - Resource sets (2 tickets, revoked)"
 
 do_test ticket-rsc-sets-8 "Ticket - Resource sets (1 ticket, standby, granted)"
 do_test ticket-rsc-sets-9 "Ticket - Resource sets (1 ticket, granted, standby)"
 do_test ticket-rsc-sets-10 "Ticket - Resource sets (1 ticket, standby, revoked)"
 do_test ticket-rsc-sets-11 "Ticket - Resource sets (2 tickets, standby, granted)"
 do_test ticket-rsc-sets-12 "Ticket - Resource sets (2 tickets, standby, granted)"
 do_test ticket-rsc-sets-13 "Ticket - Resource sets (2 tickets, granted, standby)"
 do_test ticket-rsc-sets-14 "Ticket - Resource sets (2 tickets, standby, revoked)"
 
 do_test cluster-specific-params "Cluster-specific instance attributes based on rules"
 do_test site-specific-params "Site-specific instance attributes based on rules"
 
 echo ""
 do_test template-1 "Template - 1"
 do_test template-2 "Template - 2"
 do_test template-3 "Template - 3 (merge operations)"
 
 do_test template-coloc-1 "Template - Colocation 1"
 do_test template-coloc-2 "Template - Colocation 2"
 do_test template-coloc-3 "Template - Colocation 3"
 do_test template-order-1 "Template - Order 1"
 do_test template-order-2 "Template - Order 2"
 do_test template-order-3 "Template - Order 3"
 do_test template-ticket  "Template - Ticket"
 
 do_test template-rsc-sets-1  "Template - Resource Sets 1"
 do_test template-rsc-sets-2  "Template - Resource Sets 2"
 do_test template-rsc-sets-3  "Template - Resource Sets 3"
 do_test template-rsc-sets-4  "Template - Resource Sets 4"
 
 do_test template-clone-primitive "Cloned primitive from template"
 do_test template-clone-group     "Cloned group from template"
 
 do_test location-sets-templates "Resource sets and templates - Location"
 
 do_test tags-coloc-order-1 "Tags - Colocation and Order (Simple)"
 do_test tags-coloc-order-2 "Tags - Colocation and Order (Resource Sets with Templates)"
 do_test tags-location      "Tags - Location"
 do_test tags-ticket        "Tags - Ticket"
 
 echo ""
 do_test container-1 "Container - initial"
 do_test container-2 "Container - monitor failed"
 do_test container-3 "Container - stop failed"
 do_test container-4 "Container - reached migration-threshold"
 do_test container-group-1 "Container in group - initial"
 do_test container-group-2 "Container in group - monitor failed"
 do_test container-group-3 "Container in group - stop failed"
 do_test container-group-4 "Container in group - reached migration-threshold"
 do_test container-is-remote-node "Place resource within container when container is remote-node"
 do_test bug-rh-1097457 "Kill user defined container/contents ordering"
 do_test bug-cl-5247 "Graph loop when recovering m/s resource in a container"
 
 do_test bundle-order-startup "Bundle startup ordering"
 do_test bundle-order-partial-start "Bundle startup ordering when some dependancies are already running"
 do_test bundle-order-partial-start-2 "Bundle startup ordering when some dependancies and the container are already running"
 do_test bundle-order-stop    "Bundle stop ordering"
 do_test bundle-order-partial-stop "Bundle startup ordering when some dependancies are already stopped"
 
 do_test bundle-order-startup-clone "Prevent startup because bundle isn't promoted"
 do_test bundle-order-startup-clone-2 "Bundle startup with clones"
 do_test bundle-order-stop-clone "Stop bundle because clone is stopping"
 do_test bundle-nested-colocation "Colocation of nested connection resources"
 
 do_test bundle-order-fencing "Order pseudo bundle fencing after parent node fencing if both are happening"
 
 echo ""
 do_test whitebox-fail1 "Fail whitebox container rsc."
 do_test whitebox-fail2 "Fail whitebox container rsc lrmd connection."
 do_test whitebox-fail3 "Failed containers should not run nested on remote nodes."
 do_test whitebox-start "Start whitebox container with resources assigned to it"
 do_test whitebox-stop "Stop whitebox container with resources assigned to it"
 do_test whitebox-move "Move whitebox container with resources assigned to it"
 do_test whitebox-asymmetric "Verify connection rsc opts-in based on container resource"
 do_test whitebox-ms-ordering "Verify promote/demote can not occur before connection is established"
 do_test whitebox-ms-ordering-move "Stop/Start cycle within a moving container"
 do_test whitebox-orphaned    "Properly shutdown orphaned whitebox container"
 do_test whitebox-orphan-ms   "Properly tear down orphan ms resources on remote-nodes"
 do_test whitebox-unexpectedly-running "Recover container nodes the cluster did not start."
 do_test whitebox-migrate1 "Migrate both container and connection resource"
 do_test whitebox-imply-stop-on-fence "imply stop action on container node rsc when host node is fenced"
 do_test whitebox-nested-group "Verify guest remote-node works nested in a group"
 do_test guest-node-host-dies "Verify guest node is recovered if host goes away"
 
 echo ""
 do_test remote-startup-probes  "Baremetal remote-node startup probes"
 do_test remote-startup         "Startup a newly discovered remote-nodes with no status."
 do_test remote-fence-unclean   "Fence unclean baremetal remote-node"
 do_test remote-fence-unclean2  "Fence baremetal remote-node after cluster node fails and connection can not be recovered"
 do_test remote-fence-unclean-3 "Probe failed remote nodes (triggers fencing)"
 do_test remote-move            "Move remote-node connection resource"
 do_test remote-disable         "Disable a baremetal remote-node"
 do_test remote-probe-disable   "Probe then stop a baremetal remote-node"
 do_test remote-orphaned        "Properly shutdown orphaned connection resource"
 do_test remote-orphaned2       "verify we can handle orphaned remote connections with active resources on the remote"
 do_test remote-recover         "Recover connection resource after cluster-node fails."
 do_test remote-stale-node-entry "Make sure we properly handle leftover remote-node entries in the node section"
 do_test remote-partial-migrate  "Make sure partial migrations are handled before ops on the remote node."
 do_test remote-partial-migrate2 "Make sure partial migration target is prefered for remote connection."
 do_test remote-recover-fail     "Make sure start failure causes fencing if rsc are active on remote."
 do_test remote-start-fail       "Make sure a start failure does not result in fencing if no active resources are on remote."
 do_test remote-unclean2         "Make monitor failure always results in fencing, even if no rsc are active on remote."
 do_test remote-fence-before-reconnect "Fence before clearing recurring monitor failure"
 do_test remote-recovery		"Recover remote connections before attempting demotion"
 do_test remote-recover-connection "Optimistically recovery of only the connection"
 do_test remote-recover-all        "Fencing when the connection has no home"
 do_test remote-recover-no-resources   "Fencing when the connection has no home and no active resources"
 do_test remote-recover-unknown        "Fencing when the connection has no home and the remote has no operation history"
 do_test remote-reconnect-delay        "Waiting for remote reconnect interval to expire"
 do_test remote-connection-unrecoverable  "Remote connection host must be fenced, with connection unrecoverable"
 
 echo ""
 do_test resource-discovery      "Exercises resource-discovery location constraint option."
 do_test rsc-discovery-per-node  "Disable resource discovery per node"
 
 echo ""
 do_test isolation-start-all   "Start docker isolated resources."
 do_test isolation-restart-all "Restart docker isolated resources."
 do_test isolation-clone       "Cloned isolated primitive."
 
 if [ $DO_VERSIONED_TESTS -eq 1 ]; then
 	echo ""
 	do_test versioned-resources     "Start resources with #ra-version rules"
 	do_test restart-versioned       "Restart resources on #ra-version change"
 	do_test reload-versioned        "Reload resources on #ra-version change"
 
 	echo ""
 	do_test versioned-operations-1  "Use #ra-version to configure operations of native resources"
 	do_test versioned-operations-2  "Use #ra-version to configure operations of stonith resources"
 	do_test versioned-operations-3  "Use #ra-version to configure operations of master/slave resources"
 	do_test versioned-operations-4  "Use #ra-version to configure operations of groups of the resources"
 fi
 
 echo ""
 test_results
diff --git a/pengine/test10/on-fail-ignore.dot b/pengine/test10/on-fail-ignore.dot
new file mode 100644
index 0000000000..b90f07180b
--- /dev/null
+++ b/pengine/test10/on-fail-ignore.dot
@@ -0,0 +1,15 @@
+digraph "g" {
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+"fence_db1_monitor_60000 407892-db2" [ style=bold color="green" fontcolor="black"]
+"fence_db1_start_0 407892-db2" -> "fence_db1_monitor_60000 407892-db2" [ style = bold]
+"fence_db1_start_0 407892-db2" [ style=bold color="green" fontcolor="black"]
+"fence_db1_stop_0 407892-db2" -> "all_stopped" [ style = bold]
+"fence_db1_stop_0 407892-db2" -> "fence_db1_start_0 407892-db2" [ style = bold]
+"fence_db1_stop_0 407892-db2" [ style=bold color="green" fontcolor="black"]
+"fence_db2_monitor_60000 407888-db1" [ style=bold color="green" fontcolor="black"]
+"fence_db2_start_0 407888-db1" -> "fence_db2_monitor_60000 407888-db1" [ style = bold]
+"fence_db2_start_0 407888-db1" [ style=bold color="green" fontcolor="black"]
+"fence_db2_stop_0 407888-db1" -> "all_stopped" [ style = bold]
+"fence_db2_stop_0 407888-db1" -> "fence_db2_start_0 407888-db1" [ style = bold]
+"fence_db2_stop_0 407888-db1" [ style=bold color="green" fontcolor="black"]
+}
diff --git a/pengine/test10/on-fail-ignore.exp b/pengine/test10/on-fail-ignore.exp
new file mode 100644
index 0000000000..20e5c9a805
--- /dev/null
+++ b/pengine/test10/on-fail-ignore.exp
@@ -0,0 +1,87 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
+  <synapse id="0">
+    <action_set>
+      <rsc_op id="7" operation="stop" operation_key="fence_db1_stop_0" on_node="407892-db2" on_node_uuid="2">
+        <primitive id="fence_db1" class="stonith" type="fence_ipmilan"/>
+        <attributes CRM_meta_on_node="407892-db2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000"  delay="4" ipaddr="192.168.103.3" lanplus="1" login="root" method="onoff" passwd="****" pcmk_host_check="static-list" pcmk_host_list="407888-db1" power_wait="4"/>
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="1">
+    <action_set>
+      <rsc_op id="6" operation="start" operation_key="fence_db1_start_0" on_node="407892-db2" on_node_uuid="2">
+        <primitive id="fence_db1" class="stonith" type="fence_ipmilan"/>
+        <attributes CRM_meta_on_node="407892-db2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000"  delay="4" ipaddr="192.168.103.3" lanplus="1" login="root" method="onoff" passwd="****" pcmk_host_check="static-list" pcmk_host_list="407888-db1" power_wait="4"/>
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="7" operation="stop" operation_key="fence_db1_stop_0" on_node="407892-db2" on_node_uuid="2"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="2">
+    <action_set>
+      <rsc_op id="3" operation="monitor" operation_key="fence_db1_monitor_60000" on_node="407892-db2" on_node_uuid="2">
+        <primitive id="fence_db1" class="stonith" type="fence_ipmilan"/>
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="407892-db2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000"  delay="4" ipaddr="192.168.103.3" lanplus="1" login="root" method="onoff" passwd="****" pcmk_host_check="static-list" pcmk_host_list="407888-db1" power_wait="4"/>
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="6" operation="start" operation_key="fence_db1_start_0" on_node="407892-db2" on_node_uuid="2"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="3">
+    <action_set>
+      <rsc_op id="8" operation="stop" operation_key="fence_db2_stop_0" on_node="407888-db1" on_node_uuid="1">
+        <primitive id="fence_db2" class="stonith" type="fence_ipmilan"/>
+        <attributes CRM_meta_on_node="407888-db1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000"  ipaddr="192.168.103.4" lanplus="1" login="root" method="onoff" passwd="****" pcmk_host_check="static-list" pcmk_host_list="407892-db2" power_wait="4"/>
+      </rsc_op>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="4">
+    <action_set>
+      <rsc_op id="5" operation="start" operation_key="fence_db2_start_0" on_node="407888-db1" on_node_uuid="1">
+        <primitive id="fence_db2" class="stonith" type="fence_ipmilan"/>
+        <attributes CRM_meta_on_node="407888-db1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000"  ipaddr="192.168.103.4" lanplus="1" login="root" method="onoff" passwd="****" pcmk_host_check="static-list" pcmk_host_list="407892-db2" power_wait="4"/>
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="8" operation="stop" operation_key="fence_db2_stop_0" on_node="407888-db1" on_node_uuid="1"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="5">
+    <action_set>
+      <rsc_op id="1" operation="monitor" operation_key="fence_db2_monitor_60000" on_node="407888-db1" on_node_uuid="1">
+        <primitive id="fence_db2" class="stonith" type="fence_ipmilan"/>
+        <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="407888-db1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000"  ipaddr="192.168.103.4" lanplus="1" login="root" method="onoff" passwd="****" pcmk_host_check="static-list" pcmk_host_list="407892-db2" power_wait="4"/>
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="5" operation="start" operation_key="fence_db2_start_0" on_node="407888-db1" on_node_uuid="1"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="6">
+    <action_set>
+      <pseudo_event id="4" operation="all_stopped" operation_key="all_stopped">
+        <attributes />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="7" operation="stop" operation_key="fence_db1_stop_0" on_node="407892-db2" on_node_uuid="2"/>
+      </trigger>
+      <trigger>
+        <rsc_op id="8" operation="stop" operation_key="fence_db2_stop_0" on_node="407888-db1" on_node_uuid="1"/>
+      </trigger>
+    </inputs>
+  </synapse>
+</transition_graph>
diff --git a/pengine/test10/on-fail-ignore.scores b/pengine/test10/on-fail-ignore.scores
new file mode 100644
index 0000000000..41e8b75a44
--- /dev/null
+++ b/pengine/test10/on-fail-ignore.scores
@@ -0,0 +1,8 @@
+Allocation scores:
+Using the original execution date of: 2017-10-26 14:23:50Z
+native_color: fence_db1 allocation score on 407888-db1: 0
+native_color: fence_db1 allocation score on 407892-db2: 100
+native_color: fence_db2 allocation score on 407888-db1: 100
+native_color: fence_db2 allocation score on 407892-db2: 0
+native_color: nfs_snet_ip allocation score on 407888-db1: 100
+native_color: nfs_snet_ip allocation score on 407892-db2: 0
diff --git a/pengine/test10/on-fail-ignore.summary b/pengine/test10/on-fail-ignore.summary
new file mode 100644
index 0000000000..9ee3b31483
--- /dev/null
+++ b/pengine/test10/on-fail-ignore.summary
@@ -0,0 +1,30 @@
+Using the original execution date of: 2017-10-26 14:23:50Z
+
+Current cluster status:
+Online: [ 407888-db1 407892-db2 ]
+
+ fence_db1	(stonith:fence_ipmilan):	Started 407892-db2
+ fence_db2	(stonith:fence_ipmilan):	Started 407888-db1
+ nfs_snet_ip	(ocf::heartbeat:IPaddr2):	Started 407888-db1 (failure ignored)
+
+Transition Summary:
+ * Restart    fence_db1     ( 407892-db2 )  
+ * Restart    fence_db2     ( 407888-db1 )  
+
+Executing cluster transition:
+ * Resource action: fence_db1       stop on 407892-db2
+ * Resource action: fence_db1       start on 407892-db2
+ * Resource action: fence_db1       monitor=60000 on 407892-db2
+ * Resource action: fence_db2       stop on 407888-db1
+ * Resource action: fence_db2       start on 407888-db1
+ * Resource action: fence_db2       monitor=60000 on 407888-db1
+ * Pseudo action:   all_stopped
+Using the original execution date of: 2017-10-26 14:23:50Z
+
+Revised cluster status:
+Online: [ 407888-db1 407892-db2 ]
+
+ fence_db1	(stonith:fence_ipmilan):	Started 407892-db2
+ fence_db2	(stonith:fence_ipmilan):	Started 407888-db1
+ nfs_snet_ip	(ocf::heartbeat:IPaddr2):	Started 407888-db1 (failure ignored)
+
diff --git a/pengine/test10/on-fail-ignore.xml b/pengine/test10/on-fail-ignore.xml
new file mode 100644
index 0000000000..43747d87a2
--- /dev/null
+++ b/pengine/test10/on-fail-ignore.xml
@@ -0,0 +1,113 @@
+<cib crm_feature_set="3.0.12" validate-with="pacemaker-2.8" epoch="40" num_updates="16" admin_epoch="0" cib-last-written="Sat Oct 21 10:28:39 2017" update-origin="407888-db1" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="1" execution-date="1509027830">
+  <configuration>
+    <crm_config>
+      <cluster_property_set id="cib-bootstrap-options">
+        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.16-12.el7_4.2-94ff4df"/>
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="clus_mysqlc"/>
+        <nvpair id="cib-bootstrap-options-default-resource-stickiness" name="default-resource-stickiness" value="100"/>
+      </cluster_property_set>
+    </crm_config>
+    <nodes>
+      <node id="1" uname="407888-db1"/>
+      <node id="2" uname="407892-db2"/>
+    </nodes>
+    <resources>
+      <primitive class="stonith" id="fence_db1" type="fence_ipmilan">
+        <instance_attributes id="fence_db1-instance_attributes">
+          <nvpair id="fence_db1-instance_attributes-delay" name="delay" value="4"/>
+          <nvpair id="fence_db1-instance_attributes-ipaddr" name="ipaddr" value="192.168.103.3"/>
+          <nvpair id="fence_db1-instance_attributes-lanplus" name="lanplus" value="1"/>
+          <nvpair id="fence_db1-instance_attributes-login" name="login" value="root"/>
+          <nvpair id="fence_db1-instance_attributes-method" name="method" value="onoff"/>
+          <nvpair id="fence_db1-instance_attributes-passwd" name="passwd" value="****"/>
+          <nvpair id="fence_db1-instance_attributes-pcmk_host_check" name="pcmk_host_check" value="static-list"/>
+          <nvpair id="fence_db1-instance_attributes-pcmk_host_list" name="pcmk_host_list" value="407888-db1"/>
+          <nvpair id="fence_db1-instance_attributes-power_wait" name="power_wait" value="4"/>
+        </instance_attributes>
+        <operations>
+          <op id="fence_db1-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+      <primitive class="stonith" id="fence_db2" type="fence_ipmilan">
+        <instance_attributes id="fence_db2-instance_attributes">
+          <nvpair id="fence_db2-instance_attributes-ipaddr" name="ipaddr" value="192.168.103.4"/>
+          <nvpair id="fence_db2-instance_attributes-lanplus" name="lanplus" value="1"/>
+          <nvpair id="fence_db2-instance_attributes-login" name="login" value="root"/>
+          <nvpair id="fence_db2-instance_attributes-method" name="method" value="onoff"/>
+          <nvpair id="fence_db2-instance_attributes-passwd" name="passwd" value="****"/>
+          <nvpair id="fence_db2-instance_attributes-pcmk_host_check" name="pcmk_host_check" value="static-list"/>
+          <nvpair id="fence_db2-instance_attributes-pcmk_host_list" name="pcmk_host_list" value="407892-db2"/>
+          <nvpair id="fence_db2-instance_attributes-power_wait" name="power_wait" value="4"/>
+        </instance_attributes>
+        <operations>
+          <op id="fence_db2-monitor-interval-60s" interval="60s" name="monitor"/>
+        </operations>
+      </primitive>
+        <primitive class="ocf" id="nfs_snet_ip" provider="heartbeat" type="IPaddr2">
+          <instance_attributes id="nfs_snet_ip-instance_attributes">
+            <nvpair id="nfs_snet_ip-instance_attributes-ip" name="ip" value="10.136.132.228"/>
+          </instance_attributes>
+          <meta_attributes id="nfs_snet_ip-meta_attributes">
+            <nvpair id="nfs_snet_ip-meta_attributes-migration-threshold" name="migration-threshold" value="2"/>
+          </meta_attributes>
+          <operations>
+            <op id="nfs_snet_ip-monitor-interval-10s" interval="10s" name="monitor" on-fail="ignore" timeout="10s"/>
+            <op id="nfs_snet_ip-start-interval-0s" interval="0s" name="start" timeout="20s"/>
+            <op id="nfs_snet_ip-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
+          </operations>
+        </primitive>
+    </resources>
+    <constraints/>
+  </configuration>
+  <status>
+    <node_state id="1" uname="407888-db1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <lrm id="1">
+        <lrm_resources>
+          <lrm_resource id="fence_db1" type="fence_ipmilan" class="stonith">
+            <lrm_rsc_op id="fence_db1_last_0" operation_key="fence_db1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="3:3:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" transition-magic="0:0;3:3:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" on_node="407888-db1" call-id="9" rc-code="0" op-status="0" interval="0" last-run="1508577153" last-rc-change="1508577153" exec-time="0" queue-time="0" op-digest="5397ff9100b6061844743be1480cacb6" op-secure-params=" passwd  password " op-secure-digest="d7ead554cd349871ab13dad142fdd3f9"/>
+            <lrm_rsc_op id="fence_db1_monitor_60000" operation_key="fence_db1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="5:2:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" transition-magic="0:0;5:2:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" on_node="407888-db1" call-id="7" rc-code="0" op-status="0" interval="60000" last-rc-change="1508577153" exec-time="121" queue-time="0" op-digest="0f2ec8ca9089ba249057558bf057a7e8" op-secure-params=" passwd  password " op-secure-digest="d7ead554cd349871ab13dad142fdd3f9"/>
+          </lrm_resource>
+          <lrm_resource id="fence_db2" type="fence_ipmilan" class="stonith">
+            <lrm_rsc_op id="fence_db2_last_0" operation_key="fence_db2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="7:4:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" transition-magic="0:0;7:4:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" on_node="407888-db1" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1508577161" last-rc-change="1508577161" exec-time="121" queue-time="0" op-digest="16a37bd94b2a5efbc774fb6ac0ed7a4c" op-secure-params=" passwd  password " op-secure-digest="6cecd340b584b96981bffee3e535ff98"/>
+            <lrm_rsc_op id="fence_db2_monitor_60000" operation_key="fence_db2_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="8:4:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" transition-magic="0:0;8:4:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" on_node="407888-db1" call-id="15" rc-code="0" op-status="0" interval="60000" last-rc-change="1508577162" exec-time="117" queue-time="0" op-digest="48e93a15224b9a84a6af5e82a7eab9e1" op-secure-params=" passwd  password " op-secure-digest="6cecd340b584b96981bffee3e535ff98"/>
+          </lrm_resource>
+          <lrm_resource id="nfs_snet_ip" type="IPaddr2" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="nfs_snet_ip_last_0" operation_key="nfs_snet_ip_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="55:20:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" transition-magic="0:0;55:20:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" on_node="407888-db1" call-id="67" rc-code="0" op-status="0" interval="0" last-run="1508577516" last-rc-change="1508577516" exec-time="4075" queue-time="0" op-digest="3318a8a5986a3f9535736f92159c8037"/>
+            <lrm_rsc_op id="nfs_snet_ip_monitor_10000" operation_key="nfs_snet_ip_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="56:20:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" transition-magic="0:0;56:20:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" on_node="407888-db1" call-id="68" rc-code="0" op-status="0" interval="10000" last-rc-change="1508577520" exec-time="38" queue-time="0" op-digest="93e01a8306279f10eb7583786235ab86"/>
+            <lrm_rsc_op id="nfs_snet_ip_last_failure_0" operation_key="nfs_snet_ip_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="56:20:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" transition-magic="0:7;56:20:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" exit-reason="Unable to find nic or netmask." on_node="407888-db1" call-id="68" rc-code="7" op-status="0" interval="10000" last-rc-change="1509027829" exec-time="0" queue-time="0" op-digest="93e01a8306279f10eb7583786235ab86"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+      <transient_attributes id="1">
+        <instance_attributes id="status-1">
+          <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
+          <nvpair id="status-1-fail-count-nfs_snet_ip" name="fail-count-nfs_snet_ip" value="2"/>
+          <nvpair id="status-1-last-failure-nfs_snet_ip" name="last-failure-nfs_snet_ip" value="1509027829"/>
+        </instance_attributes>
+      </transient_attributes>
+    </node_state>
+    <node_state id="2" uname="407892-db2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <lrm id="2">
+        <lrm_resources>
+          <lrm_resource id="fence_db1" type="fence_ipmilan" class="stonith">
+            <lrm_rsc_op id="fence_db1_last_0" operation_key="fence_db1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="4:3:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" transition-magic="0:0;4:3:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" on_node="407892-db2" call-id="6" rc-code="0" op-status="0" interval="0" last-run="1508577153" last-rc-change="1508577153" exec-time="116" queue-time="0" op-digest="5397ff9100b6061844743be1480cacb6" op-secure-params=" passwd  password " op-secure-digest="d7ead554cd349871ab13dad142fdd3f9"/>
+            <lrm_rsc_op id="fence_db1_monitor_60000" operation_key="fence_db1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="5:3:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" transition-magic="0:0;5:3:0:dcc10f17-10a1-4e5d-89af-ef7ff033520d" on_node="407892-db2" call-id="7" rc-code="0" op-status="0" interval="60000" last-rc-change="1508577154" exec-time="116" queue-time="0" op-digest="0f2ec8ca9089ba249057558bf057a7e8" op-secure-params=" passwd  password " op-secure-digest="d7ead554cd349871ab13dad142fdd3f9"/>
+          </lrm_resource>
+          <lrm_resource id="fence_db2" type="fence_ipmilan" class="stonith">
+            <lrm_rsc_op id="fence_db2_last_0" operation_key="fence_db2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="4:4:7:dcc10f17-10a1-4e5d-89af-ef7ff033520d" transition-magic="0:7;4:4:7:dcc10f17-10a1-4e5d-89af-ef7ff033520d" on_node="407892-db2" call-id="11" rc-code="7" op-status="0" interval="0" last-run="1508577161" last-rc-change="1508577161" exec-time="0" queue-time="0" op-digest="16a37bd94b2a5efbc774fb6ac0ed7a4c" op-secure-params=" passwd  password " op-secure-digest="6cecd340b584b96981bffee3e535ff98"/>
+          </lrm_resource>
+          <lrm_resource id="nfs_snet_ip" type="IPaddr2" class="ocf" provider="heartbeat">
+            <lrm_rsc_op id="nfs_snet_ip_last_0" operation_key="nfs_snet_ip_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="9:14:7:dcc10f17-10a1-4e5d-89af-ef7ff033520d" transition-magic="0:7;9:14:7:dcc10f17-10a1-4e5d-89af-ef7ff033520d" on_node="407892-db2" call-id="44" rc-code="7" op-status="0" interval="0" last-run="1508577451" last-rc-change="1508577451" exec-time="39" queue-time="0" op-digest="3318a8a5986a3f9535736f92159c8037"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+      <transient_attributes id="2">
+        <instance_attributes id="status-2">
+          <nvpair id="status-2-shutdown" name="shutdown" value="0"/>
+        </instance_attributes>
+      </transient_attributes>
+    </node_state>
+  </status>
+</cib>
diff --git a/tools/crm_mon.c b/tools/crm_mon.c
index 711e623bc1..401a0d403d 100644
--- a/tools/crm_mon.c
+++ b/tools/crm_mon.c
@@ -1,4244 +1,4247 @@
 /*
  * Copyright (C) 2004-2015 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <libgen.h>
 #include <sys/utsname.h>
 
 #include <crm/msg_xml.h>
 #include <crm/services.h>
 #include <crm/lrmd.h>
 #include <crm/common/internal.h>  /* crm_ends_with_ext */
 #include <crm/common/ipc.h>
 #include <crm/common/mainloop.h>
 #include <crm/common/util.h>
 #include <crm/common/xml.h>
 
 #include <crm/cib/internal.h>
 #include <crm/pengine/status.h>
 #include <crm/pengine/internal.h>
 #include <../lib/pengine/unpack.h>
 #include <../pengine/pengine.h>
 #include <crm/stonith-ng.h>
 
 extern void cleanup_alloc_calculations(pe_working_set_t * data_set);
 
 void clean_up(int rc);
 void crm_diff_update(const char *event, xmlNode * msg);
 gboolean mon_refresh_display(gpointer user_data);
 int cib_connect(gboolean full);
 void mon_st_callback(stonith_t * st, stonith_event_t * e);
 static char *get_node_display_name(node_t *node);
 
 /*
  * Definitions indicating which items to print
  */
 
 #define mon_show_times      (0x0001U)
 #define mon_show_stack      (0x0002U)
 #define mon_show_dc         (0x0004U)
 #define mon_show_count      (0x0008U)
 #define mon_show_nodes      (0x0010U)
 #define mon_show_resources  (0x0020U)
 #define mon_show_attributes (0x0040U)
 #define mon_show_failcounts (0x0080U)
 #define mon_show_operations (0x0100U)
 #define mon_show_tickets    (0x0200U)
 #define mon_show_bans       (0x0400U)
 
 #define mon_show_headers    (mon_show_times | mon_show_stack | mon_show_dc | mon_show_count)
 #define mon_show_default    (mon_show_headers | mon_show_nodes | mon_show_resources)
 #define mon_show_all        (mon_show_default | mon_show_attributes | mon_show_failcounts \
                      | mon_show_operations | mon_show_tickets | mon_show_bans)
 
 unsigned int show = mon_show_default;
 
 /*
  * Definitions indicating how to output
  */
 
 enum mon_output_format_e {
     mon_output_none,
     mon_output_monitor,
     mon_output_plain,
     mon_output_console,
     mon_output_xml,
     mon_output_html,
     mon_output_cgi
 } output_format = mon_output_console;
 
 char *output_filename = NULL;   /* if sending output to a file, its name */
 
 /* other globals */
 char *xml_file = NULL;
 char *pid_file = NULL;
 char *snmp_target = NULL;
 char *snmp_community = NULL;
 
 gboolean group_by_node = FALSE;
 gboolean inactive_resources = FALSE;
 int reconnect_msec = 5000;
 gboolean daemonize = FALSE;
 GMainLoop *mainloop = NULL;
 guint timer_id = 0;
 GList *attr_list = NULL;
 
 const char *crm_mail_host = NULL;
 const char *crm_mail_prefix = NULL;
 const char *crm_mail_from = NULL;
 const char *crm_mail_to = NULL;
 const char *external_agent = NULL;
 const char *external_recipient = NULL;
 
 cib_t *cib = NULL;
 stonith_t *st = NULL;
 xmlNode *current_cib = NULL;
 
 gboolean one_shot = FALSE;
 gboolean has_warnings = FALSE;
 gboolean print_timing = FALSE;
 gboolean watch_fencing = FALSE;
 gboolean print_brief = FALSE;
 gboolean print_pending = TRUE;
 gboolean print_clone_detail = FALSE;
 
 /* FIXME allow, detect, and correctly interpret glob pattern or regex? */
 const char *print_neg_location_prefix = "";
 
 /* Never display node attributes whose name starts with one of these prefixes */
 #define FILTER_STR { CRM_FAIL_COUNT_PREFIX, CRM_LAST_FAILURE_PREFIX,       \
                      "shutdown", "terminate", "standby", "probe_complete", \
                      "#", NULL }
 
 long last_refresh = 0;
 crm_trigger_t *refresh_trigger = NULL;
 
 /*
  * 1.3.6.1.4.1.32723 has been assigned to the project by IANA
  * http://www.iana.org/assignments/enterprise-numbers
  */
 #define PACEMAKER_PREFIX "1.3.6.1.4.1.32723"
 #define PACEMAKER_TRAP_PREFIX PACEMAKER_PREFIX ".1"
 
 #define snmp_crm_trap_oid   PACEMAKER_TRAP_PREFIX
 #define snmp_crm_oid_node   PACEMAKER_TRAP_PREFIX ".1"
 #define snmp_crm_oid_rsc    PACEMAKER_TRAP_PREFIX ".2"
 #define snmp_crm_oid_task   PACEMAKER_TRAP_PREFIX ".3"
 #define snmp_crm_oid_desc   PACEMAKER_TRAP_PREFIX ".4"
 #define snmp_crm_oid_status PACEMAKER_TRAP_PREFIX ".5"
 #define snmp_crm_oid_rc     PACEMAKER_TRAP_PREFIX ".6"
 #define snmp_crm_oid_trc    PACEMAKER_TRAP_PREFIX ".7"
 
 /* Define exit codes for monitoring-compatible output */
 #define MON_STATUS_OK   (0)
 #define MON_STATUS_WARN (1)
 
 /* Convenience macro for prettifying output (e.g. "node" vs "nodes") */
 #define s_if_plural(i) (((i) == 1)? "" : "s")
 
 #if CURSES_ENABLED
 #  define print_dot() if (output_format == mon_output_console) { \
 	printw(".");				\
 	clrtoeol();				\
 	refresh();				\
     } else {					\
 	fprintf(stdout, ".");			\
     }
 #else
 #  define print_dot() fprintf(stdout, ".");
 #endif
 
 #if CURSES_ENABLED
 #  define print_as(fmt, args...) if (output_format == mon_output_console) { \
 	printw(fmt, ##args);				\
 	clrtoeol();					\
 	refresh();					\
     } else {						\
 	fprintf(stdout, fmt, ##args);			\
     }
 #else
 #  define print_as(fmt, args...) fprintf(stdout, fmt, ##args);
 #endif
 
 static void
 blank_screen(void)
 {
 #if CURSES_ENABLED
     int lpc = 0;
 
     for (lpc = 0; lpc < LINES; lpc++) {
         move(lpc, 0);
         clrtoeol();
     }
     move(0, 0);
     refresh();
 #endif
 }
 
 static gboolean
 mon_timer_popped(gpointer data)
 {
     int rc = pcmk_ok;
 
 #if CURSES_ENABLED
     if (output_format == mon_output_console) {
         clear();
         refresh();
     }
 #endif
 
     if (timer_id > 0) {
         g_source_remove(timer_id);
     }
 
     print_as("Reconnecting...\n");
     rc = cib_connect(TRUE);
 
     if (rc != pcmk_ok) {
         timer_id = g_timeout_add(reconnect_msec, mon_timer_popped, NULL);
     }
     return FALSE;
 }
 
 static void
 mon_cib_connection_destroy(gpointer user_data)
 {
     print_as("Connection to the CIB terminated\n");
     if (cib) {
         cib->cmds->signoff(cib);
         timer_id = g_timeout_add(reconnect_msec, mon_timer_popped, NULL);
     }
     return;
 }
 
 /*
  * Mainloop signal handler.
  */
 static void
 mon_shutdown(int nsig)
 {
     clean_up(EX_OK);
 }
 
 #if ON_DARWIN
 #  define sighandler_t sig_t
 #endif
 
 #if CURSES_ENABLED
 #  ifndef HAVE_SIGHANDLER_T
 typedef void (*sighandler_t) (int);
 #  endif
 static sighandler_t ncurses_winch_handler;
 static void
 mon_winresize(int nsig)
 {
     static int not_done;
     int lines = 0, cols = 0;
 
     if (!not_done++) {
         if (ncurses_winch_handler)
             /* the original ncurses WINCH signal handler does the
              * magic of retrieving the new window size;
              * otherwise, we'd have to use ioctl or tgetent */
             (*ncurses_winch_handler) (SIGWINCH);
         getmaxyx(stdscr, lines, cols);
         resizeterm(lines, cols);
         mainloop_set_trigger(refresh_trigger);
     }
     not_done--;
 }
 #endif
 
 int
 cib_connect(gboolean full)
 {
     int rc = pcmk_ok;
     static gboolean need_pass = TRUE;
 
     CRM_CHECK(cib != NULL, return -EINVAL);
 
     if (getenv("CIB_passwd") != NULL) {
         need_pass = FALSE;
     }
 
     if (watch_fencing && st == NULL) {
         st = stonith_api_new();
     }
 
     if (watch_fencing && st->state == stonith_disconnected) {
         crm_trace("Connecting to stonith");
         rc = st->cmds->connect(st, crm_system_name, NULL);
         if (rc == pcmk_ok) {
             crm_trace("Setting up stonith callbacks");
             st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, mon_st_callback);
         }
     }
 
     if (cib->state != cib_connected_query && cib->state != cib_connected_command) {
         crm_trace("Connecting to the CIB");
         if ((output_format == mon_output_console) && need_pass && (cib->variant == cib_remote)) {
             need_pass = FALSE;
             print_as("Password:");
         }
 
         rc = cib->cmds->signon(cib, crm_system_name, cib_query);
 
         if (rc != pcmk_ok) {
             return rc;
         }
 
         rc = cib->cmds->query(cib, NULL, &current_cib, cib_scope_local | cib_sync_call);
         if (rc == pcmk_ok) {
             mon_refresh_display(NULL);
         }
 
         if (rc == pcmk_ok && full) {
             if (rc == pcmk_ok) {
                 rc = cib->cmds->set_connection_dnotify(cib, mon_cib_connection_destroy);
                 if (rc == -EPROTONOSUPPORT) {
                     print_as
                         ("Notification setup not supported, won't be able to reconnect after failure");
                     if (output_format == mon_output_console) {
                         sleep(2);
                     }
                     rc = pcmk_ok;
                 }
 
             }
 
             if (rc == pcmk_ok) {
                 cib->cmds->del_notify_callback(cib, T_CIB_DIFF_NOTIFY, crm_diff_update);
                 rc = cib->cmds->add_notify_callback(cib, T_CIB_DIFF_NOTIFY, crm_diff_update);
             }
 
             if (rc != pcmk_ok) {
                 print_as("Notification setup failed, could not monitor CIB actions");
                 if (output_format == mon_output_console) {
                     sleep(2);
                 }
                 clean_up(-rc);
             }
         }
     }
     return rc;
 }
 
 /* *INDENT-OFF* */
 static struct crm_option long_options[] = {
     /* Top-level Options */
     {"help",           0, 0, '?', "\tThis text"},
     {"version",        0, 0, '$', "\tVersion information"  },
     {"verbose",        0, 0, 'V', "\tIncrease debug output"},
     {"quiet",          0, 0, 'Q', "\tDisplay only essential output" },
 
     {"-spacer-",	1, 0, '-', "\nModes (mutually exclusive):"},
     {"as-html",        1, 0, 'h', "\tWrite cluster status to the named html file"},
     {"as-xml",         0, 0, 'X', "\t\tWrite cluster status as xml to stdout. This will enable one-shot mode."},
     {"web-cgi",        0, 0, 'w', "\t\tWeb mode with output suitable for CGI (preselected when run as *.cgi)"},
     {"simple-status",  0, 0, 's', "\tDisplay the cluster status once as a simple one line output (suitable for nagios)"},
     {"snmp-traps",     1, 0, 'S', "\tSend SNMP traps to this station", !ENABLE_SNMP},
     {"snmp-community", 1, 0, 'C', "Specify community for SNMP traps(default is NULL)", !ENABLE_SNMP},
     {"mail-to",        1, 0, 'T', "\tSend Mail alerts to this user.  See also --mail-from, --mail-host, --mail-prefix", !ENABLE_ESMTP},
 
     {"-spacer-",	1, 0, '-', "\nDisplay Options:"},
     {"group-by-node",  0, 0, 'n', "\tGroup resources by node"     },
     {"inactive",       0, 0, 'r', "\t\tDisplay inactive resources"  },
     {"failcounts",     0, 0, 'f', "\tDisplay resource fail counts"},
     {"operations",     0, 0, 'o', "\tDisplay resource operation history" },
     {"timing-details", 0, 0, 't', "\tDisplay resource operation history with timing details" },
     {"tickets",        0, 0, 'c', "\t\tDisplay cluster tickets"},
     {"watch-fencing",  0, 0, 'W', "\tListen for fencing events. For use with --external-agent, --mail-to and/or --snmp-traps where supported"},
     {"neg-locations",  2, 0, 'L', "Display negative location constraints [optionally filtered by id prefix]"},
     {"show-node-attributes", 0, 0, 'A', "Display node attributes" },
     {"hide-headers",   0, 0, 'D', "\tHide all headers" },
     {"show-detail",    0, 0, 'R', "\tShow more details (node IDs, individual clone instances)" },
     {"brief",          0, 0, 'b', "\t\tBrief output" },
     {"pending",        0, 0, 'j', "\t\tDisplay pending state if 'record-pending' is enabled", pcmk_option_hidden},
 
     {"-spacer-",	1, 0, '-', "\nAdditional Options:"},
     {"interval",       1, 0, 'i', "\tUpdate frequency in seconds" },
     {"one-shot",       0, 0, '1', "\t\tDisplay the cluster status once on the console and exit"},
     {"disable-ncurses",0, 0, 'N', "\tDisable the use of ncurses", !CURSES_ENABLED},
     {"daemonize",      0, 0, 'd', "\tRun in the background as a daemon"},
     {"pid-file",       1, 0, 'p', "\t(Advanced) Daemon pid file location"},
     {"mail-from",      1, 0, 'F', "\tMail alerts should come from the named user", !ENABLE_ESMTP},
     {"mail-host",      1, 0, 'H', "\tMail alerts should be sent via the named host", !ENABLE_ESMTP},
     {"mail-prefix",    1, 0, 'P', "Subjects for mail alerts should start with this string", !ENABLE_ESMTP},
     {"external-agent",    1, 0, 'E', "A program to run when resource operations take place."},
     {"external-recipient",1, 0, 'e', "A recipient for your program (assuming you want the program to send something to someone)."},
 
 
     {"xml-file",       1, 0, 'x', NULL, pcmk_option_hidden},
 
     {"-spacer-",	1, 0, '-', "\nExamples:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', "Display the cluster status on the console with updates as they occur:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_mon", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Display the cluster status on the console just once then exit:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_mon -1", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Display your cluster status, group resources by node, and include inactive resources in the list:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_mon --group-by-node --inactive", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Start crm_mon as a background daemon and have it write the cluster status to an HTML file:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_mon --daemonize --as-html /path/to/docroot/filename.html", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Start crm_mon and export the current cluster status as xml to stdout, then exit.:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_mon --as-xml", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Start crm_mon as a background daemon and have it send email alerts:", pcmk_option_paragraph|!ENABLE_ESMTP},
     {"-spacer-",	1, 0, '-', " crm_mon --daemonize --mail-to user@example.com --mail-host mail.example.com", pcmk_option_example|!ENABLE_ESMTP},
     {"-spacer-",	1, 0, '-', "Start crm_mon as a background daemon and have it send SNMP alerts:", pcmk_option_paragraph|!ENABLE_SNMP},
     {"-spacer-",	1, 0, '-', " crm_mon --daemonize --snmp-traps snmptrapd.example.com", pcmk_option_example|!ENABLE_SNMP},
 
     {NULL, 0, 0, 0}
 };
 /* *INDENT-ON* */
 
 #if CURSES_ENABLED
 static const char *
 get_option_desc(char c)
 {
     int lpc;
 
     for (lpc = 0; long_options[lpc].name != NULL; lpc++) {
 
         if (long_options[lpc].name[0] == '-')
             continue;
 
         if (long_options[lpc].val == c) {
             const char * tab = NULL;
             tab = strrchr(long_options[lpc].desc, '\t');
             return tab ? ++tab : long_options[lpc].desc;
         }
     }
 
     return NULL;
 }
 
 #define print_option_help(option, condition) \
     print_as("%c %c: \t%s\n", ((condition)? '*': ' '), option, get_option_desc(option));
 
 static gboolean
 detect_user_input(GIOChannel *channel, GIOCondition condition, gpointer unused)
 {
     int c;
     gboolean config_mode = FALSE;
 
     while (1) {
 
         /* Get user input */
         c = getchar();
 
         switch (c) {
             case 'c':
                 show ^= mon_show_tickets;
                 break;
             case 'f':
                 show ^= mon_show_failcounts;
                 break;
             case 'n':
                 group_by_node = ! group_by_node;
                 break;
             case 'o':
                 show ^= mon_show_operations;
                 if ((show & mon_show_operations) == 0) {
                     print_timing = 0;
                 }
                 break;
             case 'r':
                 inactive_resources = ! inactive_resources;
                 break;
             case 'R':
                 print_clone_detail = ! print_clone_detail;
                 break;
             case 't':
                 print_timing = ! print_timing;
                 if (print_timing) {
                     show |= mon_show_operations;
                 }
                 break;
             case 'A':
                 show ^= mon_show_attributes;
                 break;
             case 'L':
                 show ^= mon_show_bans;
                 break;
             case 'D':
                 /* If any header is shown, clear them all, otherwise set them all */
                 if (show & mon_show_headers) {
                     show &= ~mon_show_headers;
                 } else {
                     show |= mon_show_headers;
                 }
                 break;
             case 'b':
                 print_brief = ! print_brief;
                 break;
             case 'j':
                 print_pending = ! print_pending;
                 break;
             case '?':
                 config_mode = TRUE;
                 break;
             default:
                 goto refresh;
         }
 
         if (!config_mode)
             goto refresh;
 
         blank_screen();
 
         print_as("Display option change mode\n");
         print_as("\n");
         print_option_help('c', show & mon_show_tickets);
         print_option_help('f', show & mon_show_failcounts);
         print_option_help('n', group_by_node);
         print_option_help('o', show & mon_show_operations);
         print_option_help('r', inactive_resources);
         print_option_help('t', print_timing);
         print_option_help('A', show & mon_show_attributes);
         print_option_help('L', show & mon_show_bans);
         print_option_help('D', (show & mon_show_headers) == 0);
         print_option_help('R', print_clone_detail);
         print_option_help('b', print_brief);
         print_option_help('j', print_pending);
         print_as("\n");
         print_as("Toggle fields via field letter, type any other key to return");
     }
 
 refresh:
     mon_refresh_display(NULL);
     return TRUE;
 }
 #endif
 
 int
 main(int argc, char **argv)
 {
     int flag;
     int argerr = 0;
     int exit_code = 0;
     int option_index = 0;
 
     pid_file = strdup("/tmp/ClusterMon.pid");
     crm_log_cli_init("crm_mon");
     crm_set_options(NULL, "mode [options]", long_options,
                     "Provides a summary of cluster's current state."
                     "\n\nOutputs varying levels of detail in a number of different formats.\n");
 
 #if !defined (ON_DARWIN) && !defined (ON_BSD)
     /* prevent zombies */
     signal(SIGCLD, SIG_IGN);
 #endif
 
     if (crm_ends_with_ext(argv[0], ".cgi") == TRUE) {
         output_format = mon_output_cgi;
         one_shot = TRUE;
     }
 
     while (1) {
         flag = crm_get_option(argc, argv, &option_index);
         if (flag == -1)
             break;
 
         switch (flag) {
             case 'V':
                 crm_bump_log_level(argc, argv);
                 break;
             case 'Q':
                 show &= ~mon_show_times;
                 break;
             case 'i':
                 reconnect_msec = crm_get_msec(optarg);
                 break;
             case 'n':
                 group_by_node = TRUE;
                 break;
             case 'r':
                 inactive_resources = TRUE;
                 break;
             case 'W':
                 watch_fencing = TRUE;
                 break;
             case 'd':
                 daemonize = TRUE;
                 break;
             case 't':
                 print_timing = TRUE;
                 show |= mon_show_operations;
                 break;
             case 'o':
                 show |= mon_show_operations;
                 break;
             case 'f':
                 show |= mon_show_failcounts;
                 break;
             case 'A':
                 show |= mon_show_attributes;
                 break;
             case 'L':
                 show |= mon_show_bans;
                 print_neg_location_prefix = optarg? optarg : "";
                 break;
             case 'D':
                 show &= ~mon_show_headers;
                 break;
             case 'b':
                 print_brief = TRUE;
                 break;
             case 'j':
                 print_pending = TRUE;
                 break;
             case 'R':
                 print_clone_detail = TRUE;
                 break;
             case 'c':
                 show |= mon_show_tickets;
                 break;
             case 'p':
                 free(pid_file);
                 if(optarg == NULL) {
                     return crm_help(flag, EX_USAGE);
                 }
                 pid_file = strdup(optarg);
                 break;
             case 'x':
                 if(optarg == NULL) {
                     return crm_help(flag, EX_USAGE);
                 }
                 xml_file = strdup(optarg);
                 one_shot = TRUE;
                 break;
             case 'h':
                 if(optarg == NULL) {
                     return crm_help(flag, EX_USAGE);
                 }
                 argerr += (output_format != mon_output_console);
                 output_format = mon_output_html;
                 output_filename = strdup(optarg);
                 umask(S_IWGRP | S_IWOTH);
                 break;
             case 'X':
                 argerr += (output_format != mon_output_console);
                 output_format = mon_output_xml;
                 one_shot = TRUE;
                 break;
             case 'w':
                 /* do not allow argv[0] and argv[1...] redundancy */
                 argerr += (output_format != mon_output_console);
                 output_format = mon_output_cgi;
                 one_shot = TRUE;
                 break;
             case 's':
                 argerr += (output_format != mon_output_console);
                 output_format = mon_output_monitor;
                 one_shot = TRUE;
                 break;
             case 'S':
                 snmp_target = optarg;
                 break;
             case 'T':
                 crm_mail_to = optarg;
                 break;
             case 'F':
                 crm_mail_from = optarg;
                 break;
             case 'H':
                 crm_mail_host = optarg;
                 break;
             case 'P':
                 crm_mail_prefix = optarg;
                 break;
             case 'E':
                 external_agent = optarg;
                 break;
             case 'e':
                 external_recipient = optarg;
                 break;
             case '1':
                 one_shot = TRUE;
                 break;
             case 'N':
                 if (output_format == mon_output_console) {
                     output_format = mon_output_plain;
                 }
                 break;
             case 'C':
                 snmp_community = optarg;
                 break;
             case '$':
             case '?':
                 return crm_help(flag, EX_OK);
                 break;
             default:
                 printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
                 ++argerr;
                 break;
         }
     }
 
     /* Extra sanity checks when in CGI mode */
     if (output_format == mon_output_cgi) {
         argerr += (optind < argc);
         argerr += (output_filename != NULL);
         argerr += (xml_file != NULL);
         argerr += (snmp_target != NULL);
         argerr += (crm_mail_to != NULL);
         argerr += (external_agent != NULL);
         argerr += (daemonize == TRUE);  /* paranoia */
 
     } else if (optind < argc) {
         printf("non-option ARGV-elements: ");
         while (optind < argc)
             printf("%s ", argv[optind++]);
         printf("\n");
     }
 
     if (argerr) {
         if (output_format == mon_output_cgi) {
             fprintf(stdout, "Content-Type: text/plain\n"
                             "Status: 500\n\n");
             return EX_USAGE;
         }
         return crm_help('?', EX_USAGE);
     }
 
     /* XML output always prints everything */
     if (output_format == mon_output_xml) {
         show = mon_show_all;
         print_timing = TRUE;
     }
 
     if (one_shot) {
         if (output_format == mon_output_console) {
             output_format = mon_output_plain;
         }
 
     } else if (daemonize) {
         if ((output_format == mon_output_console) || (output_format == mon_output_plain)) {
             output_format = mon_output_none;
         }
         crm_enable_stderr(FALSE);
 
         if ((output_format != mon_output_html) && (output_format != mon_output_xml)
             && !snmp_target && !crm_mail_to && !external_agent) {
             printf
                 ("Looks like you forgot to specify one or more of: --as-html, --as-xml, --mail-to, --snmp-target, --external-agent\n");
             return crm_help('?', EX_USAGE);
         }
 
         crm_make_daemon(crm_system_name, TRUE, pid_file);
 
     } else if (output_format == mon_output_console) {
 #if CURSES_ENABLED
         initscr();
         cbreak();
         noecho();
         crm_enable_stderr(FALSE);
 #else
         one_shot = TRUE;
         output_format = mon_output_plain;
         printf("Defaulting to one-shot mode\n");
         printf("You need to have curses available at compile time to enable console mode\n");
 #endif
     }
 
     crm_info("Starting %s", crm_system_name);
     if (xml_file != NULL) {
         current_cib = filename2xml(xml_file);
         mon_refresh_display(NULL);
         return exit_code;
     }
 
     if (current_cib == NULL) {
         cib = cib_new();
 
         do {
             if (!one_shot) {
                 print_as("Attempting connection to the cluster...\n");
             }
             exit_code = cib_connect(!one_shot);
 
             if (one_shot) {
                 break;
 
             } else if (exit_code != pcmk_ok) {
                 sleep(reconnect_msec / 1000);
 #if CURSES_ENABLED
                 if (output_format == mon_output_console) {
                     clear();
                     refresh();
                 }
 #endif
             }
 
         } while (exit_code == -ENOTCONN);
 
         if (exit_code != pcmk_ok) {
             if (output_format == mon_output_monitor) {
                 printf("CLUSTER WARN: Connection to cluster failed: %s\n", pcmk_strerror(exit_code));
                 clean_up(MON_STATUS_WARN);
             } else {
                 print_as("\nConnection to cluster failed: %s\n", pcmk_strerror(exit_code));
             }
             if (output_format == mon_output_console) {
                 sleep(2);
             }
             clean_up(-exit_code);
         }
     }
 
     if (one_shot) {
         return exit_code;
     }
 
     mainloop = g_main_new(FALSE);
 
     mainloop_add_signal(SIGTERM, mon_shutdown);
     mainloop_add_signal(SIGINT, mon_shutdown);
 #if CURSES_ENABLED
     if (output_format == mon_output_console) {
         ncurses_winch_handler = signal(SIGWINCH, mon_winresize);
         if (ncurses_winch_handler == SIG_DFL ||
             ncurses_winch_handler == SIG_IGN || ncurses_winch_handler == SIG_ERR)
             ncurses_winch_handler = NULL;
         g_io_add_watch(g_io_channel_unix_new(STDIN_FILENO), G_IO_IN, detect_user_input, NULL);
     }
 #endif
     refresh_trigger = mainloop_add_trigger(G_PRIORITY_LOW, mon_refresh_display, NULL);
 
     g_main_run(mainloop);
     g_main_destroy(mainloop);
 
     crm_info("Exiting %s", crm_system_name);
 
     clean_up(0);
     return 0;                   /* never reached */
 }
 
 #define mon_warn(fmt...) do {			\
 	if (!has_warnings) {			\
 	    print_as("CLUSTER WARN:");		\
 	} else {				\
 	    print_as(",");			\
 	}					\
 	print_as(fmt);				\
 	has_warnings = TRUE;			\
     } while(0)
 
 static int
 count_resources(pe_working_set_t * data_set, resource_t * rsc)
 {
     int count = 0;
     GListPtr gIter = NULL;
 
     if (rsc == NULL) {
         gIter = data_set->resources;
     } else if (rsc->children) {
         gIter = rsc->children;
     } else {
         return is_not_set(rsc->flags, pe_rsc_orphan);
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         count += count_resources(data_set, gIter->data);
     }
     return count;
 }
 
 /*!
  * \internal
  * \brief Print one-line status suitable for use with monitoring software
  *
  * \param[in] data_set  Working set of CIB state
  *
  * \note This function's output (and the return code when the program exits)
  *       should conform to https://www.monitoring-plugins.org/doc/guidelines.html
  */
 static void
 print_simple_status(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     int nodes_online = 0;
     int nodes_standby = 0;
     int nodes_maintenance = 0;
 
     if (data_set->dc_node == NULL) {
         mon_warn(" No DC");
     }
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
         if (node->details->standby && node->details->online) {
             nodes_standby++;
         } else if (node->details->maintenance && node->details->online) {
             nodes_maintenance++;
         } else if (node->details->online) {
             nodes_online++;
         } else {
             mon_warn(" offline node: %s", node->details->uname);
         }
     }
 
     if (!has_warnings) {
         int nresources = count_resources(data_set, NULL);
 
         print_as("CLUSTER OK: %d node%s online", nodes_online, s_if_plural(nodes_online));
         if (nodes_standby > 0) {
             print_as(", %d standby node%s", nodes_standby, s_if_plural(nodes_standby));
         }
         if (nodes_maintenance > 0) {
             print_as(", %d maintenance node%s", nodes_maintenance, s_if_plural(nodes_maintenance));
         }
         print_as(", %d resource%s configured", nresources, s_if_plural(nresources));
     }
 
     print_as("\n");
 }
 
 /*!
  * \internal
  * \brief Print a [name]=[value][units] pair, optionally using time string
  *
  * \param[in] stream      File stream to display output to
  * \param[in] name        Name to display
  * \param[in] value       Value to display (or NULL to convert time instead)
  * \param[in] units       Units to display (or NULL for no units)
  * \param[in] epoch_time  Epoch time to convert if value is NULL
  */
 static void
 print_nvpair(FILE *stream, const char *name, const char *value,
              const char *units, time_t epoch_time)
 {
     /* print name= */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as(" %s=", name);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
         case mon_output_xml:
             fprintf(stream, " %s=", name);
             break;
 
         default:
             break;
     }
 
     /* If we have a value (and optionally units), print it */
     if (value) {
         switch (output_format) {
             case mon_output_plain:
             case mon_output_console:
                 print_as("%s%s", value, (units? units : ""));
                 break;
 
             case mon_output_html:
             case mon_output_cgi:
                 fprintf(stream, "%s%s", value, (units? units : ""));
                 break;
 
             case mon_output_xml:
                 fprintf(stream, "\"%s%s\"", value, (units? units : ""));
                 break;
 
             default:
                 break;
         }
 
     /* Otherwise print user-friendly time string */
     } else {
         static char empty_str[] = "";
         char *c, *date_str = asctime(localtime(&epoch_time));
 
         for (c = (date_str != NULL) ? date_str : empty_str; *c != '\0'; ++c) {
             if (*c == '\n') {
                 *c = '\0';
                 break;
             }
         }
         switch (output_format) {
             case mon_output_plain:
             case mon_output_console:
                 print_as("'%s'", date_str);
                 break;
 
             case mon_output_html:
             case mon_output_cgi:
             case mon_output_xml:
                 fprintf(stream, "\"%s\"", date_str);
                 break;
 
             default:
                 break;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Print whatever is needed to start a node section
  *
  * \param[in] stream     File stream to display output to
  * \param[in] node       Node to print
  */
 static void
 print_node_start(FILE *stream, node_t *node)
 {
     char *node_name;
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             node_name = get_node_display_name(node);
             print_as("* Node %s:\n", node_name);
             free(node_name);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             node_name = get_node_display_name(node);
             fprintf(stream, "  <h3>Node: %s</h3>\n  <ul>\n", node_name);
             free(node_name);
             break;
 
         case mon_output_xml:
             fprintf(stream, "        <node name=\"%s\">\n", node->details->uname);
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print whatever is needed to end a node section
  *
  * \param[in] stream     File stream to display output to
  */
 static void
 print_node_end(FILE *stream)
 {
     switch (output_format) {
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "  </ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "        </node>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print resources section heading appropriate to options
  *
  * \param[in] stream      File stream to display output to
  */
 static void
 print_resources_heading(FILE *stream)
 {
     const char *heading;
 
     if (group_by_node) {
 
         /* Active resources have already been printed by node */
         heading = (inactive_resources? "Inactive resources" : NULL);
 
     } else if (inactive_resources) {
         heading = "Full list of resources";
 
     } else {
         heading = "Active resources";
     }
 
     /* Print section heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n%s:\n\n", heading);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <hr />\n <h2>%s</h2>\n", heading);
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <resources>\n");
             break;
 
         default:
             break;
     }
 
 }
 
 /*!
  * \internal
  * \brief Print whatever resource section closing is appropriate
  *
  * \param[in] stream     File stream to display output to
  */
 static void
 print_resources_closing(FILE *stream, gboolean printed_heading)
 {
     const char *heading;
 
     /* What type of resources we did or did not display */
     if (group_by_node) {
         heading = "inactive ";
     } else if (inactive_resources) {
         heading = "";
     } else {
         heading = "active ";
     }
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             if (!printed_heading) {
                 print_as("\nNo %sresources\n\n", heading);
             }
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             if (!printed_heading) {
                 fprintf(stream, " <hr />\n <h2>No %sresources</h2>\n", heading);
             }
             break;
 
         case mon_output_xml:
             fprintf(stream, "    %s\n",
                     (printed_heading? "</resources>" : "<resources/>"));
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print whatever resource section(s) are appropriate
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Cluster state to display
  * \param[in] print_opts  Bitmask of pe_print_options
  */
 static void
 print_resources(FILE *stream, pe_working_set_t *data_set, int print_opts)
 {
     GListPtr rsc_iter;
     const char *prefix = NULL;
     gboolean printed_heading = FALSE;
     gboolean brief_output = print_brief;
 
     /* If we already showed active resources by node, and
      * we're not showing inactive resources, we have nothing to do
      */
     if (group_by_node && !inactive_resources) {
         return;
     }
 
     /* XML uses an indent, and ignores brief option for resources */
     if (output_format == mon_output_xml) {
         prefix = "        ";
         brief_output = FALSE;
     }
 
     /* If we haven't already printed resources grouped by node,
      * and brief output was requested, print resource summary */
     if (brief_output && !group_by_node) {
         print_resources_heading(stream);
         printed_heading = TRUE;
         print_rscs_brief(data_set->resources, NULL, print_opts, stream,
                          inactive_resources);
     }
 
     /* For each resource, display it if appropriate */
     for (rsc_iter = data_set->resources; rsc_iter != NULL; rsc_iter = rsc_iter->next) {
         resource_t *rsc = (resource_t *) rsc_iter->data;
 
         /* Complex resources may have some sub-resources active and some inactive */
         gboolean is_active = rsc->fns->active(rsc, TRUE);
         gboolean partially_active = rsc->fns->active(rsc, FALSE);
 
         /* Skip inactive orphans (deleted but still in CIB) */
         if (is_set(rsc->flags, pe_rsc_orphan) && !is_active) {
             continue;
 
         /* Skip active resources if we already displayed them by node */
         } else if (group_by_node) {
             if (is_active) {
                 continue;
             }
 
         /* Skip primitives already counted in a brief summary */
         } else if (brief_output && (rsc->variant == pe_native)) {
             continue;
 
         /* Skip resources that aren't at least partially active,
          * unless we're displaying inactive resources
          */
         } else if (!partially_active && !inactive_resources) {
             continue;
         }
 
         /* Print this resource */
         if (printed_heading == FALSE) {
             print_resources_heading(stream);
             printed_heading = TRUE;
         }
         rsc->fns->print(rsc, prefix, print_opts, stream);
     }
 
     print_resources_closing(stream, printed_heading);
 }
 
 /*!
  * \internal
  * \brief Print heading for resource history
  *
  * \param[in] stream      File stream to display output to
  * \param[in] data_set    Current state of CIB
  * \param[in] node        Node that ran this resource
  * \param[in] rsc         Resource to print
  * \param[in] rsc_id      ID of resource to print
  * \param[in] all         Whether to print every resource or just failed ones
  */
 static void
 print_rsc_history_start(FILE *stream, pe_working_set_t *data_set, node_t *node,
                         resource_t *rsc, const char *rsc_id, gboolean all)
 {
     time_t last_failure = 0;
-    int failcount = rsc? get_failcount_full(node, rsc, &last_failure, FALSE, NULL, data_set) : 0;
+    int failcount = rsc?
+                    pe_get_failcount(node, rsc, &last_failure, pe_fc_default,
+                                     NULL, data_set)
+                    : 0;
 
     if (!all && !failcount && (last_failure <= 0)) {
         return;
     }
 
     /* Print resource ID */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("   %s:", rsc_id);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "   <li>%s:", rsc_id);
             break;
 
         case mon_output_xml:
             fprintf(stream, "            <resource_history id=\"%s\"", rsc_id);
             break;
 
         default:
             break;
     }
 
     /* If resource is an orphan, that's all we can say about it */
     if (rsc == NULL) {
         switch (output_format) {
             case mon_output_plain:
             case mon_output_console:
                 print_as(" orphan");
                 break;
 
             case mon_output_html:
             case mon_output_cgi:
                 fprintf(stream, " orphan");
                 break;
 
             case mon_output_xml:
                 fprintf(stream, " orphan=\"true\"");
                 break;
 
             default:
                 break;
         }
 
     /* If resource is not an orphan, print some details */
     } else if (all || failcount || (last_failure > 0)) {
 
         /* Print migration threshold */
         switch (output_format) {
             case mon_output_plain:
             case mon_output_console:
                 print_as(" migration-threshold=%d", rsc->migration_threshold);
                 break;
 
             case mon_output_html:
             case mon_output_cgi:
                 fprintf(stream, " migration-threshold=%d", rsc->migration_threshold);
                 break;
 
             case mon_output_xml:
                 fprintf(stream, " orphan=\"false\" migration-threshold=\"%d\"",
                         rsc->migration_threshold);
                 break;
 
             default:
                 break;
         }
 
         /* Print fail count if any */
         if (failcount > 0) {
             switch (output_format) {
                 case mon_output_plain:
                 case mon_output_console:
                     print_as(" " CRM_FAIL_COUNT_PREFIX "=%d", failcount);
                     break;
 
                 case mon_output_html:
                 case mon_output_cgi:
                     fprintf(stream, " " CRM_FAIL_COUNT_PREFIX "=%d", failcount);
                     break;
 
                 case mon_output_xml:
                     fprintf(stream, " " CRM_FAIL_COUNT_PREFIX "=\"%d\"",
                             failcount);
                     break;
 
                 default:
                     break;
             }
         }
 
         /* Print last failure time if any */
         if (last_failure > 0) {
             print_nvpair(stream, CRM_LAST_FAILURE_PREFIX, NULL, NULL,
                          last_failure);
         }
     }
 
     /* End the heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "\n    <ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, ">\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print closing for resource history
  *
  * \param[in] stream      File stream to display output to
  */
 static void
 print_rsc_history_end(FILE *stream)
 {
     switch (output_format) {
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "    </ul>\n   </li>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "            </resource_history>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print operation history
  *
  * \param[in] stream      File stream to display output to
  * \param[in] data_set    Current state of CIB
  * \param[in] node        Node this operation is for
  * \param[in] xml_op      Root of XML tree describing this operation
  * \param[in] task        Task parsed from this operation's XML
  * \param[in] interval    Interval parsed from this operation's XML
  * \param[in] rc          Return code parsed from this operation's XML
  */
 static void
 print_op_history(FILE *stream, pe_working_set_t *data_set, node_t *node,
                  xmlNode *xml_op, const char *task, const char *interval, int rc)
 {
     const char *value = NULL;
     const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
 
     /* Begin the operation description */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("    + (%s) %s:", call, task);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "     <li>(%s) %s:", call, task);
             break;
 
         case mon_output_xml:
             fprintf(stream, "                <operation_history call=\"%s\" task=\"%s\"",
                     call, task);
             break;
 
         default:
             break;
     }
 
     /* Add name=value pairs as appropriate */
     if (safe_str_neq(interval, "0")) {
         print_nvpair(stream, "interval", interval, "ms", 0);
     }
     if (print_timing) {
         int int_value;
         const char *attr;
 
         attr = XML_RSC_OP_LAST_CHANGE;
         value = crm_element_value(xml_op, attr);
         if (value) {
             int_value = crm_parse_int(value, NULL);
             if (int_value > 0) {
                 print_nvpair(stream, attr, NULL, NULL, int_value);
             }
         }
 
         attr = XML_RSC_OP_LAST_RUN;
         value = crm_element_value(xml_op, attr);
         if (value) {
             int_value = crm_parse_int(value, NULL);
             if (int_value > 0) {
                 print_nvpair(stream, attr, NULL, NULL, int_value);
             }
         }
 
         attr = XML_RSC_OP_T_EXEC;
         value = crm_element_value(xml_op, attr);
         if (value) {
             print_nvpair(stream, attr, value, "ms", 0);
         }
 
         attr = XML_RSC_OP_T_QUEUE;
         value = crm_element_value(xml_op, attr);
         if (value) {
             print_nvpair(stream, attr, value, "ms", 0);
         }
     }
 
     /* End the operation description */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as(" rc=%d (%s)\n", rc, services_ocf_exitcode_str(rc));
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " rc=%d (%s)</li>\n", rc, services_ocf_exitcode_str(rc));
             break;
 
         case mon_output_xml:
             fprintf(stream, " rc=\"%d\" rc_text=\"%s\" />\n", rc, services_ocf_exitcode_str(rc));
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print resource operation/failure history
  *
  * \param[in] stream      File stream to display output to
  * \param[in] data_set    Current state of CIB
  * \param[in] node        Node that ran this resource
  * \param[in] rsc_entry   Root of XML tree describing resource status
  * \param[in] operations  Whether to print operations or just failcounts
  */
 static void
 print_rsc_history(FILE *stream, pe_working_set_t *data_set, node_t *node,
                   xmlNode *rsc_entry, gboolean operations)
 {
     GListPtr gIter = NULL;
     GListPtr op_list = NULL;
     gboolean printed = FALSE;
     const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
     resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
     xmlNode *rsc_op = NULL;
 
     /* If we're not showing operations, just print the resource failure summary */
     if (operations == FALSE) {
         print_rsc_history_start(stream, data_set, node, rsc, rsc_id, FALSE);
         print_rsc_history_end(stream);
         return;
     }
 
     /* Create a list of this resource's operations */
     for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next(rsc_op)) {
         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             op_list = g_list_append(op_list, rsc_op);
         }
     }
     op_list = g_list_sort(op_list, sort_op_by_callid);
 
     /* Print each operation */
     for (gIter = op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *xml_op = (xmlNode *) gIter->data;
         const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
         const char *interval = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
         const char *op_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC);
         int rc = crm_parse_int(op_rc, "0");
 
         /* Display 0-interval monitors as "probe" */
         if (safe_str_eq(task, CRMD_ACTION_STATUS) && safe_str_eq(interval, "0")) {
             task = "probe";
         }
 
         /* Ignore notifies and some probes */
         if (safe_str_eq(task, CRMD_ACTION_NOTIFY) || (safe_str_eq(task, "probe") && (rc == 7))) {
             continue;
         }
 
         /* If this is the first printed operation, print heading for resource */
         if (printed == FALSE) {
             printed = TRUE;
             print_rsc_history_start(stream, data_set, node, rsc, rsc_id, TRUE);
         }
 
         /* Print the operation */
         print_op_history(stream, data_set, node, xml_op, task, interval, rc);
     }
 
     /* Free the list we created (no need to free the individual items) */
     g_list_free(op_list);
 
     /* If we printed anything, close the resource */
     if (printed) {
         print_rsc_history_end(stream);
     }
 }
 
 /*!
  * \internal
  * \brief Print node operation/failure history
  *
  * \param[in] stream      File stream to display output to
  * \param[in] data_set    Current state of CIB
  * \param[in] node_state  Root of XML tree describing node status
  * \param[in] operations  Whether to print operations or just failcounts
  */
 static void
 print_node_history(FILE *stream, pe_working_set_t *data_set,
                    xmlNode *node_state, gboolean operations)
 {
     node_t *node = pe_find_node_id(data_set->nodes, ID(node_state));
     xmlNode *lrm_rsc = NULL;
     xmlNode *rsc_entry = NULL;
 
     if (node && node->details && node->details->online) {
         print_node_start(stream, node);
 
         lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
         lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
 
         /* Print history of each of the node's resources */
         for (rsc_entry = __xml_first_child(lrm_rsc); rsc_entry != NULL;
              rsc_entry = __xml_next(rsc_entry)) {
 
             if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
                 print_rsc_history(stream, data_set, node, rsc_entry, operations);
             }
         }
 
         print_node_end(stream);
     }
 }
 
 /*!
  * \internal
  * \brief Print extended information about an attribute if appropriate
  *
  * \param[in] data_set  Working set of CIB state
  *
  * \return TRUE if extended information was printed, FALSE otherwise
  * \note Currently, extended information is only supported for ping/pingd
  *       resources, for which a message will be printed if connectivity is lost
  *       or degraded.
  */
 static gboolean
 print_attr_msg(FILE *stream, node_t * node, GListPtr rsc_list, const char *attrname, const char *attrvalue)
 {
     GListPtr gIter = NULL;
 
     for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
         const char *type = g_hash_table_lookup(rsc->meta, "type");
 
         if (rsc->children != NULL) {
             if (print_attr_msg(stream, node, rsc->children, attrname, attrvalue)) {
                 return TRUE;
             }
         }
 
         if (safe_str_eq(type, "ping") || safe_str_eq(type, "pingd")) {
             const char *name = g_hash_table_lookup(rsc->parameters, "name");
 
             if (name == NULL) {
                 name = "pingd";
             }
 
             /* To identify the resource with the attribute name. */
             if (safe_str_eq(name, attrname)) {
                 int host_list_num = 0;
                 int expected_score = 0;
                 int value = crm_parse_int(attrvalue, "0");
                 const char *hosts = g_hash_table_lookup(rsc->parameters, "host_list");
                 const char *multiplier = g_hash_table_lookup(rsc->parameters, "multiplier");
 
                 if(hosts) {
                     char **host_list = g_strsplit(hosts, " ", 0);
                     host_list_num = g_strv_length(host_list);
                     g_strfreev(host_list);
                 }
 
                 /* pingd multiplier is the same as the default value. */
                 expected_score = host_list_num * crm_parse_int(multiplier, "1");
 
                 switch (output_format) {
                     case mon_output_plain:
                     case mon_output_console:
                         if (value <= 0) {
                             print_as("\t: Connectivity is lost");
                         } else if (value < expected_score) {
                             print_as("\t: Connectivity is degraded (Expected=%d)", expected_score);
                         }
                         break;
 
                     case mon_output_html:
                     case mon_output_cgi:
                         if (value <= 0) {
                             fprintf(stream, " <b>(connectivity is lost)</b>");
                         } else if (value < expected_score) {
                             fprintf(stream, " <b>(connectivity is degraded -- expected %d)</b>",
                                     expected_score);
                         }
                         break;
 
                     case mon_output_xml:
                         fprintf(stream, " expected=\"%d\"", expected_score);
                         break;
 
                     default:
                         break;
                 }
                 return TRUE;
             }
         }
     }
     return FALSE;
 }
 
 static int
 compare_attribute(gconstpointer a, gconstpointer b)
 {
     int rc;
 
     rc = strcmp((const char *)a, (const char *)b);
 
     return rc;
 }
 
 static void
 create_attr_list(gpointer name, gpointer value, gpointer data)
 {
     int i;
     const char *filt_str[] = FILTER_STR;
 
     CRM_CHECK(name != NULL, return);
 
     /* filtering automatic attributes */
     for (i = 0; filt_str[i] != NULL; i++) {
         if (g_str_has_prefix(name, filt_str[i])) {
             return;
         }
     }
 
     attr_list = g_list_insert_sorted(attr_list, name, compare_attribute);
 }
 
 /* structure for passing multiple user data to g_list_foreach() */
 struct mon_attr_data {
     FILE *stream;
     node_t *node;
 };
 
 static void
 print_node_attribute(gpointer name, gpointer user_data)
 {
     const char *value = NULL;
     struct mon_attr_data *data = (struct mon_attr_data *) user_data;
 
     value = pe_node_attribute_raw(data->node, name);
 
     /* Print attribute name and value */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("    + %-32s\t: %-10s", (char *)name, value);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(data->stream, "   <li>%s: %s",
                     (char *)name, value);
             break;
 
         case mon_output_xml:
             fprintf(data->stream,
                     "            <attribute name=\"%s\" value=\"%s\"",
                     (char *)name, value);
             break;
 
         default:
             break;
     }
 
     /* Print extended information if appropriate */
     print_attr_msg(data->stream, data->node, data->node->details->running_rsc,
                    name, value);
 
     /* Close out the attribute */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(data->stream, "</li>\n");
             break;
 
         case mon_output_xml:
             fprintf(data->stream, " />\n");
             break;
 
         default:
             break;
     }
 }
 
 static void
 print_node_summary(FILE *stream, pe_working_set_t * data_set, gboolean operations)
 {
     xmlNode *node_state = NULL;
     xmlNode *cib_status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);
 
     /* Print heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             if (operations) {
                 print_as("\nOperations:\n");
             } else {
                 print_as("\nMigration Summary:\n");
             }
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             if (operations) {
                 fprintf(stream, " <hr />\n <h2>Operations</h2>\n");
             } else {
                 fprintf(stream, " <hr />\n <h2>Migration Summary</h2>\n");
             }
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <node_history>\n");
             break;
 
         default:
             break;
     }
 
     /* Print each node in the CIB status */
     for (node_state = __xml_first_child(cib_status); node_state != NULL;
          node_state = __xml_next(node_state)) {
         if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
             print_node_history(stream, data_set, node_state, operations);
         }
     }
 
     /* Close section */
     switch (output_format) {
         case mon_output_xml:
             fprintf(stream, "    </node_history>\n");
             break;
 
         default:
             break;
     }
 }
 
 static void
 print_ticket(gpointer name, gpointer value, gpointer data)
 {
     ticket_t *ticket = (ticket_t *) value;
     FILE *stream = (FILE *) data;
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("* %s:\t%s%s", ticket->id,
                      (ticket->granted? "granted" : "revoked"),
                      (ticket->standby? " [standby]" : ""));
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "  <li>%s: %s%s", ticket->id,
                     (ticket->granted? "granted" : "revoked"),
                     (ticket->standby? " [standby]" : ""));
             break;
 
         case mon_output_xml:
             fprintf(stream, "        <ticket id=\"%s\" status=\"%s\" standby=\"%s\"",
                     ticket->id, (ticket->granted? "granted" : "revoked"),
                     (ticket->standby? "true" : "false"));
             break;
 
         default:
             break;
     }
     if (ticket->last_granted > -1) {
         print_nvpair(stdout, "last-granted", NULL, NULL, ticket->last_granted);
     }
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "</li>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, " />\n");
             break;
 
         default:
             break;
     }
 }
 
 static void
 print_cluster_tickets(FILE *stream, pe_working_set_t * data_set)
 {
     /* Print section heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\nTickets:\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <hr />\n <h2>Tickets</h2>\n <ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <tickets>\n");
             break;
 
         default:
             break;
     }
 
     /* Print each ticket */
     g_hash_table_foreach(data_set->tickets, print_ticket, stream);
 
     /* Close section */
     switch (output_format) {
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " </ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    </tickets>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Return human-friendly string representing node name
  *
  * The returned string will be in the format
  *    uname[@hostUname] [(nodeID)]
  * "@hostUname" will be printed if the node is a guest node.
  * "(nodeID)" will be printed if the node ID is different from the node uname,
  *  and detailed output has been requested.
  *
  * \param[in] node  Node to represent
  * \return Newly allocated string with representation of node name
  * \note It is the caller's responsibility to free the result with free().
  */
 static char *
 get_node_display_name(node_t *node)
 {
     char *node_name;
     const char *node_host = NULL;
     const char *node_id = NULL;
     int name_len;
 
     CRM_ASSERT((node != NULL) && (node->details != NULL) && (node->details->uname != NULL));
 
     /* Host is displayed only if this is a guest node */
     if (is_container_remote_node(node)) {
         if (node->details->remote_rsc->running_on) {
             /* running_on is a list, but guest nodes will have exactly one entry
              * unless they are in the process of migrating, in which case they
              * will have two; either way, we can use the first item in the list
              */
             node_t *host_node = (node_t *) node->details->remote_rsc->running_on->data;
 
             if (host_node && host_node->details) {
                 node_host = host_node->details->uname;
             }
         }
         if (node_host == NULL) {
             node_host = ""; /* so we at least get "uname@" to indicate guest */
         }
     }
 
     /* Node ID is displayed if different from uname and detail is requested */
     if (print_clone_detail && safe_str_neq(node->details->uname, node->details->id)) {
         node_id = node->details->id;
     }
 
     /* Determine name length */
     name_len = strlen(node->details->uname) + 1;
     if (node_host) {
         name_len += strlen(node_host) + 1; /* "@node_host" */
     }
     if (node_id) {
         name_len += strlen(node_id) + 3; /* + " (node_id)" */
     }
 
     /* Allocate and populate display name */
     node_name = malloc(name_len);
     CRM_ASSERT(node_name != NULL);
     strcpy(node_name, node->details->uname);
     if (node_host) {
         strcat(node_name, "@");
         strcat(node_name, node_host);
     }
     if (node_id) {
         strcat(node_name, " (");
         strcat(node_name, node_id);
         strcat(node_name, ")");
     }
     return node_name;
 }
 
 /*!
  * \internal
  * \brief Print a negative location constraint
  *
  * \param[in] stream     File stream to display output to
  * \param[in] node       Node affected by constraint
  * \param[in] location   Constraint to print
  */
 static void print_ban(FILE *stream, node_t *node, rsc_to_node_t *location)
 {
     char *node_name = NULL;
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             node_name = get_node_display_name(node);
             print_as(" %s\tprevents %s from running %son %s\n",
                      location->id, location->rsc_lh->id,
                      ((location->role_filter == RSC_ROLE_MASTER)? "as Master " : ""),
                      node_name);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             node_name = get_node_display_name(node);
             fprintf(stream, "  <li>%s prevents %s from running %son %s</li>\n",
                      location->id, location->rsc_lh->id,
                      ((location->role_filter == RSC_ROLE_MASTER)? "as Master " : ""),
                      node_name);
             break;
 
         case mon_output_xml:
             fprintf(stream,
                     "        <ban id=\"%s\" resource=\"%s\" node=\"%s\" weight=\"%d\" master_only=\"%s\" />\n",
                     location->id, location->rsc_lh->id, node->details->uname, node->weight,
                     ((location->role_filter == RSC_ROLE_MASTER)? "true" : "false"));
             break;
 
         default:
             break;
     }
     free(node_name);
 }
 
 /*!
  * \internal
  * \brief Print section for negative location constraints
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set corresponding to CIB status to display
  */
 static void print_neg_locations(FILE *stream, pe_working_set_t *data_set)
 {
     GListPtr gIter, gIter2;
 
     /* Print section heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\nNegative Location Constraints:\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <hr />\n <h2>Negative Location Constraints</h2>\n <ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <bans>\n");
             break;
 
         default:
             break;
     }
 
     /* Print each ban */
     for (gIter = data_set->placement_constraints; gIter != NULL; gIter = gIter->next) {
         rsc_to_node_t *location = (rsc_to_node_t *) gIter->data;
         if (!g_str_has_prefix(location->id, print_neg_location_prefix))
             continue;
         for (gIter2 = location->node_list_rh; gIter2 != NULL; gIter2 = gIter2->next) {
             node_t *node = (node_t *) gIter2->data;
 
             if (node->weight < 0) {
                 print_ban(stream, node, location);
             }
         }
     }
 
     /* Close section */
     switch (output_format) {
         case mon_output_cgi:
         case mon_output_html:
             fprintf(stream, " </ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    </bans>\n");
             break;
 
         default:
             break;
     }
 }
 
 static void
 crm_mon_get_parameters(resource_t *rsc, pe_working_set_t * data_set)
 {
     get_rsc_attributes(rsc->parameters, rsc, NULL, data_set);
     crm_trace("Beekhof: unpacked params for %s (%d)", rsc->id, g_hash_table_size(rsc->parameters));
     if(rsc->children) {
         GListPtr gIter = NULL;
 
         for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
             crm_mon_get_parameters(gIter->data, data_set);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Print node attributes section
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_node_attributes(FILE *stream, pe_working_set_t *data_set)
 {
     GListPtr gIter = NULL;
 
     /* Print section heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\nNode Attributes:\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <hr />\n <h2>Node Attributes</h2>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <node_attributes>\n");
             break;
 
         default:
             break;
     }
 
     /* Unpack all resource parameters (it would be more efficient to do this
      * only when needed for the first time in print_attr_msg())
      */
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         crm_mon_get_parameters(gIter->data, data_set);
     }
 
     /* Display each node's attributes */
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         struct mon_attr_data data;
 
         data.stream = stream;
         data.node = (node_t *) gIter->data;
 
         if (data.node && data.node->details && data.node->details->online) {
             print_node_start(stream, data.node);
             g_hash_table_foreach(data.node->details->attrs, create_attr_list, NULL);
             g_list_foreach(attr_list, print_node_attribute, &data);
             g_list_free(attr_list);
             attr_list = NULL;
             print_node_end(stream);
         }
     }
 
     /* Print section footer */
     switch (output_format) {
         case mon_output_xml:
             fprintf(stream, "    </node_attributes>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Return resource display options corresponding to command-line choices
  *
  * \return Bitmask of pe_print_options suitable for resource print functions
  */
 static int
 get_resource_display_options(void)
 {
     int print_opts;
 
     /* Determine basic output format */
     switch (output_format) {
         case mon_output_console:
             print_opts = pe_print_ncurses;
             break;
         case mon_output_html:
         case mon_output_cgi:
             print_opts = pe_print_html;
             break;
         case mon_output_xml:
             print_opts = pe_print_xml;
             break;
         default:
             print_opts = pe_print_printf;
             break;
     }
 
     /* Add optional display elements */
     if (print_pending) {
         print_opts |= pe_print_pending;
     }
     if (print_clone_detail) {
         print_opts |= pe_print_clone_details;
     }
     if (!inactive_resources) {
         print_opts |= pe_print_clone_active;
     }
     if (print_brief) {
         print_opts |= pe_print_brief;
     }
     return print_opts;
 }
 
 /*!
  * \internal
  * \brief Return human-friendly string representing current time
  *
  * \return Current time as string (as by ctime() but without newline) on success
  *         or "Could not determine current time" on error
  * \note The return value points to a statically allocated string which might be
  *       overwritten by subsequent calls to any of the C library date and time functions.
  */
 static const char *
 crm_now_string(void)
 {
     time_t a_time = time(NULL);
     char *since_epoch = ctime(&a_time);
 
     if ((a_time == (time_t) -1) || (since_epoch == NULL)) {
         return "Could not determine current time";
     }
     since_epoch[strlen(since_epoch) - 1] = EOS; /* trim newline */
     return (since_epoch);
 }
 
 /*!
  * \internal
  * \brief Print header for cluster summary if needed
  *
  * \param[in] stream     File stream to display output to
  */
 static void
 print_cluster_summary_header(FILE *stream)
 {
     switch (output_format) {
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <h2>Cluster Summary</h2>\n <p>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <summary>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print footer for cluster summary if needed
  *
  * \param[in] stream     File stream to display output to
  */
 static void
 print_cluster_summary_footer(FILE *stream)
 {
     switch (output_format) {
         case mon_output_cgi:
         case mon_output_html:
             fprintf(stream, " </p>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    </summary>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print times the display was last updated and CIB last changed
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_cluster_times(FILE *stream, pe_working_set_t *data_set)
 {
     const char *last_written = crm_element_value(data_set->input, XML_CIB_ATTR_WRITTEN);
     const char *user = crm_element_value(data_set->input, XML_ATTR_UPDATE_USER);
     const char *client = crm_element_value(data_set->input, XML_ATTR_UPDATE_CLIENT);
     const char *origin = crm_element_value(data_set->input, XML_ATTR_UPDATE_ORIG);
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("Last updated: %s", crm_now_string());
             print_as((user || client || origin)? "\n" : "\t\t");
             print_as("Last change: %s", last_written ? last_written : "");
             if (user) {
                 print_as(" by %s", user);
             }
             if (client) {
                 print_as(" via %s", client);
             }
             if (origin) {
                 print_as(" on %s", origin);
             }
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <b>Last updated:</b> %s<br/>\n", crm_now_string());
             fprintf(stream, " <b>Last change:</b> %s", last_written ? last_written : "");
             if (user) {
                 fprintf(stream, " by %s", user);
             }
             if (client) {
                 fprintf(stream, " via %s", client);
             }
             if (origin) {
                 fprintf(stream, " on %s", origin);
             }
             fprintf(stream, "<br/>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "        <last_update time=\"%s\" />\n", crm_now_string());
             fprintf(stream, "        <last_change time=\"%s\" user=\"%s\" client=\"%s\" origin=\"%s\" />\n",
                     last_written ? last_written : "", user ? user : "",
                     client ? client : "", origin ? origin : "");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print cluster stack
  *
  * \param[in] stream     File stream to display output to
  * \param[in] stack_s    Stack name
  */
 static void
 print_cluster_stack(FILE *stream, const char *stack_s)
 {
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("Stack: %s\n", stack_s);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <b>Stack:</b> %s<br/>\n", stack_s);
             break;
 
         case mon_output_xml:
             fprintf(stream, "        <stack type=\"%s\" />\n", stack_s);
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print current DC and its version
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_cluster_dc(FILE *stream, pe_working_set_t *data_set)
 {
     node_t *dc = data_set->dc_node;
     xmlNode *dc_version = get_xpath_object("//nvpair[@name='dc-version']",
                                            data_set->input, LOG_DEBUG);
     const char *dc_version_s = dc_version?
                                crm_element_value(dc_version, XML_NVPAIR_ATTR_VALUE)
                                : NULL;
     const char *quorum = crm_element_value(data_set->input, XML_ATTR_HAVE_QUORUM);
     char *dc_name = dc? get_node_display_name(dc) : NULL;
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("Current DC: ");
             if (dc) {
                 print_as("%s (version %s) - partition %s quorum\n",
                          dc_name, (dc_version_s? dc_version_s : "unknown"),
                          (crm_is_true(quorum) ? "with" : "WITHOUT"));
             } else {
                 print_as("NONE\n");
             }
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <b>Current DC:</b> ");
             if (dc) {
                 fprintf(stream, "%s (version %s) - partition %s quorum",
                         dc_name, (dc_version_s? dc_version_s : "unknown"),
                         (crm_is_true(quorum)? "with" : "<font color=\"red\"><b>WITHOUT</b></font>"));
             } else {
                 fprintf(stream, "<font color=\"red\"><b>NONE</b></font>");
             }
             fprintf(stream, "<br/>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream,  "        <current_dc ");
             if (dc) {
                 fprintf(stream,
                         "present=\"true\" version=\"%s\" name=\"%s\" id=\"%s\" with_quorum=\"%s\"",
                         (dc_version_s? dc_version_s : ""), dc->details->uname, dc->details->id,
                         (crm_is_true(quorum) ? "true" : "false"));
             } else {
                 fprintf(stream, "present=\"false\"");
             }
             fprintf(stream, " />\n");
             break;
 
         default:
             break;
     }
     free(dc_name);
 }
 
 /*!
  * \internal
  * \brief Print counts of configured nodes and resources
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  * \param[in] stack_s    Stack name
  */
 static void
 print_cluster_counts(FILE *stream, pe_working_set_t *data_set, const char *stack_s)
 {
     int nnodes = g_list_length(data_set->nodes);
     int nresources = count_resources(data_set, NULL);
     xmlNode *quorum_node = get_xpath_object("//nvpair[@name='" XML_ATTR_EXPECTED_VOTES "']",
                                             data_set->input, LOG_DEBUG);
     const char *quorum_votes = quorum_node?
                                crm_element_value(quorum_node, XML_NVPAIR_ATTR_VALUE)
                                : "unknown";
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
 
             print_as("\n%d node%s configured", nnodes, s_if_plural(nnodes));
             if (stack_s && strstr(stack_s, "classic openais") != NULL) {
                 print_as(" (%s expected votes)", quorum_votes);
             }
             print_as("\n");
 
             print_as("%d resource%s configured",
                      nresources, s_if_plural(nresources));
             if(data_set->disabled_resources || data_set->blocked_resources) {
                 print_as(" (");
                 if (data_set->disabled_resources) {
                     print_as("%d DISABLED", data_set->disabled_resources);
                 }
                 if (data_set->disabled_resources && data_set->blocked_resources) {
                     print_as(", ");
                 }
                 if (data_set->blocked_resources) {
                     print_as("%d BLOCKED from starting due to failure",
                              data_set->blocked_resources);
                 }
                 print_as(")");
             }
             print_as("\n");
 
             break;
 
         case mon_output_html:
         case mon_output_cgi:
 
             fprintf(stream, " %d node%s configured", nnodes, s_if_plural(nnodes));
             if (stack_s && strstr(stack_s, "classic openais") != NULL) {
                 fprintf(stream, " (%s expected votes)", quorum_votes);
             }
             fprintf(stream, "<br/>\n");
 
             fprintf(stream, " %d resource%s configured",
                     nresources, s_if_plural(nresources));
             if (data_set->disabled_resources || data_set->blocked_resources) {
                 fprintf(stream, " (");
                 if (data_set->disabled_resources) {
                     fprintf(stream, "%d <strong>DISABLED</strong>",
                             data_set->disabled_resources);
                 }
                 if (data_set->disabled_resources && data_set->blocked_resources) {
                     fprintf(stream, ", ");
                 }
                 if (data_set->blocked_resources) {
                     fprintf(stream,
                             "%d <strong>BLOCKED</strong> from starting due to failure",
                             data_set->blocked_resources);
                 }
                 fprintf(stream, ")");
             }
             fprintf(stream, "<br/>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream,
                     "        <nodes_configured number=\"%d\" expected_votes=\"%s\" />\n",
                     g_list_length(data_set->nodes), quorum_votes);
             fprintf(stream,
                     "        <resources_configured number=\"%d\" disabled=\"%d\" blocked=\"%d\" />\n",
                     count_resources(data_set, NULL),
                     data_set->disabled_resources, data_set->blocked_resources);
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print cluster-wide options
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  *
  * \note Currently this is only implemented for HTML and XML output, and
  *       prints only a few options. If there is demand, more could be added.
  */
 static void
 print_cluster_options(FILE *stream, pe_working_set_t *data_set)
 {
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
                 print_as("\n              *** Resource management is DISABLED ***");
                 print_as("\n  The cluster will not attempt to start, stop or recover services");
                 print_as("\n");
             }
             break;
 
         case mon_output_html:
             fprintf(stream, " </p>\n <h3>Config Options</h3>\n");
             fprintf(stream, " <table>\n");
             fprintf(stream, "  <tr><th>STONITH of failed nodes</th><td>%s</td></tr>\n",
                     is_set(data_set->flags, pe_flag_stonith_enabled)? "enabled" : "disabled");
 
             fprintf(stream, "  <tr><th>Cluster is</th><td>%ssymmetric</td></tr>\n",
                     is_set(data_set->flags, pe_flag_symmetric_cluster)? "" : "a");
 
             fprintf(stream, "  <tr><th>No Quorum Policy</th><td>");
             switch (data_set->no_quorum_policy) {
                 case no_quorum_freeze:
                     fprintf(stream, "Freeze resources");
                     break;
                 case no_quorum_stop:
                     fprintf(stream, "Stop ALL resources");
                     break;
                 case no_quorum_ignore:
                     fprintf(stream, "Ignore");
                     break;
                 case no_quorum_suicide:
                     fprintf(stream, "Suicide");
                     break;
             }
             fprintf(stream, "</td></tr>\n");
 
             fprintf(stream, "  <tr><th>Resource management</th><td>");
             if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
                 fprintf(stream, "<strong>DISABLED</strong> (the cluster will "
                                 "not attempt to start, stop or recover services)");
             } else {
                 fprintf(stream, "enabled");
             }
             fprintf(stream, "</td></tr>\n");
 
             fprintf(stream, "</table>\n <p>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "        <cluster_options");
             fprintf(stream, " stonith-enabled=\"%s\"",
                     is_set(data_set->flags, pe_flag_stonith_enabled)?
                     "true" : "false");
             fprintf(stream, " symmetric-cluster=\"%s\"",
                     is_set(data_set->flags, pe_flag_symmetric_cluster)?
                     "true" : "false");
             fprintf(stream, " no-quorum-policy=\"");
             switch (data_set->no_quorum_policy) {
                 case no_quorum_freeze:
                     fprintf(stream, "freeze");
                     break;
                 case no_quorum_stop:
                     fprintf(stream, "stop");
                     break;
                 case no_quorum_ignore:
                     fprintf(stream, "ignore");
                     break;
                 case no_quorum_suicide:
                     fprintf(stream, "suicide");
                     break;
             }
             fprintf(stream, "\"");
             fprintf(stream, " maintenance-mode=\"%s\"",
                     is_set(data_set->flags, pe_flag_maintenance_mode)?
                     "true" : "false");
             fprintf(stream, " />\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Get the name of the stack in use (or "unknown" if not available)
  *
  * \param[in] data_set   Working set of CIB state
  *
  * \return String representing stack name
  */
 static const char *
 get_cluster_stack(pe_working_set_t *data_set)
 {
     xmlNode *stack = get_xpath_object("//nvpair[@name='cluster-infrastructure']",
                                       data_set->input, LOG_DEBUG);
     return stack? crm_element_value(stack, XML_NVPAIR_ATTR_VALUE) : "unknown";
 }
 
 /*!
  * \internal
  * \brief Print a summary of cluster-wide information
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_cluster_summary(FILE *stream, pe_working_set_t *data_set)
 {
     const char *stack_s = get_cluster_stack(data_set);
     gboolean header_printed = FALSE;
 
     if (show & mon_show_stack) {
         if (header_printed == FALSE) {
             print_cluster_summary_header(stream);
             header_printed = TRUE;
         }
         print_cluster_stack(stream, stack_s);
     }
 
     /* Always print DC if none, even if not requested */
     if ((data_set->dc_node == NULL) || (show & mon_show_dc)) {
         if (header_printed == FALSE) {
             print_cluster_summary_header(stream);
             header_printed = TRUE;
         }
         print_cluster_dc(stream, data_set);
     }
 
     if (show & mon_show_times) {
         if (header_printed == FALSE) {
             print_cluster_summary_header(stream);
             header_printed = TRUE;
         }
         print_cluster_times(stream, data_set);
     }
 
     if (is_set(data_set->flags, pe_flag_maintenance_mode)
         || data_set->disabled_resources
         || data_set->blocked_resources
         || is_set(show, mon_show_count)) {
         if (header_printed == FALSE) {
             print_cluster_summary_header(stream);
             header_printed = TRUE;
         }
         print_cluster_counts(stream, data_set, stack_s);
     }
 
     /* There is not a separate option for showing cluster options, so show with
      * stack for now; a separate option could be added if there is demand
      */
     if (show & mon_show_stack) {
         print_cluster_options(stream, data_set);
     }
 
     if (header_printed) {
         print_cluster_summary_footer(stream);
     }
 }
 
 /*!
  * \internal
  * \brief Print a failed action
  *
  * \param[in] stream     File stream to display output to
  * \param[in] xml_op     Root of XML tree describing failed action
  */
 static void
 print_failed_action(FILE *stream, xmlNode *xml_op)
 {
     const char *op_key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
     const char *op_key_attr = "op_key";
     const char *last = crm_element_value(xml_op, XML_RSC_OP_LAST_CHANGE);
     const char *node = crm_element_value(xml_op, XML_ATTR_UNAME);
     const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
     const char *exit_reason = crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON);
     int rc = crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_RC), "0");
     int status = crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS), "0");
     char *exit_reason_cleaned;
 
     /* If no op_key was given, use id instead */
     if (op_key == NULL) {
         op_key = ID(xml_op);
         op_key_attr = "id";
     }
 
     /* If no exit reason was given, use "none" */
     if (exit_reason == NULL) {
         exit_reason = "none";
     }
 
     /* Print common action information */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("* %s on %s '%s' (%d): call=%s, status=%s, exitreason='%s'",
                      op_key, node, services_ocf_exitcode_str(rc), rc,
                      call, services_lrm_status_str(status), exit_reason);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "  <li>%s on %s '%s' (%d): call=%s, status=%s, exitreason='%s'",
                      op_key, node, services_ocf_exitcode_str(rc), rc,
                      call, services_lrm_status_str(status), exit_reason);
             break;
 
         case mon_output_xml:
             exit_reason_cleaned = crm_xml_escape(exit_reason);
             fprintf(stream, "        <failure %s=\"%s\" node=\"%s\"",
                     op_key_attr, op_key, node);
             fprintf(stream, " exitstatus=\"%s\" exitreason=\"%s\" exitcode=\"%d\"",
                     services_ocf_exitcode_str(rc), exit_reason_cleaned, rc);
             fprintf(stream, " call=\"%s\" status=\"%s\"",
                     call, services_lrm_status_str(status));
             free(exit_reason_cleaned);
             break;
 
         default:
             break;
     }
 
     /* If last change was given, print timing information as well */
     if (last) {
         time_t run_at = crm_parse_int(last, "0");
         char *run_at_s = ctime(&run_at);
 
         if (run_at_s) {
             run_at_s[24] = 0; /* Overwrite the newline */
         }
 
         switch (output_format) {
             case mon_output_plain:
             case mon_output_console:
                 print_as(",\n    last-rc-change='%s', queued=%sms, exec=%sms",
                          run_at_s? run_at_s : "",
                          crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
                          crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
                 break;
 
             case mon_output_html:
             case mon_output_cgi:
                 fprintf(stream, " last-rc-change='%s', queued=%sms, exec=%sms",
                         run_at_s? run_at_s : "",
                         crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
                         crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
                 break;
 
             case mon_output_xml:
                 fprintf(stream,
                         " last-rc-change=\"%s\" queued=\"%s\" exec=\"%s\" interval=\"%d\" task=\"%s\"",
                         run_at_s? run_at_s : "",
                         crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
                         crm_element_value(xml_op, XML_RSC_OP_T_EXEC),
                         crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL), "0"),
                         crm_element_value(xml_op, XML_LRM_ATTR_TASK));
                 break;
 
             default:
                 break;
         }
     }
 
     /* End the action listing */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "</li>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, " />\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print a section for failed actions
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_failed_actions(FILE *stream, pe_working_set_t *data_set)
 {
     xmlNode *xml_op = NULL;
 
     /* Print section heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\nFailed Actions:\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <hr />\n <h2>Failed Actions</h2>\n <ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <failures>\n");
             break;
 
         default:
             break;
     }
 
     /* Print each failed action */
     for (xml_op = __xml_first_child(data_set->failed); xml_op != NULL;
          xml_op = __xml_next(xml_op)) {
         print_failed_action(stream, xml_op);
     }
 
     /* End section */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " </ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    </failures>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print cluster status to screen
  *
  * This uses the global display preferences set by command-line options
  * to display cluster status in a human-friendly way.
  *
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_status(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     int print_opts = get_resource_display_options();
 
     /* space-separated lists of node names */
     char *online_nodes = NULL;
     char *online_remote_nodes = NULL;
     char *online_guest_nodes = NULL;
     char *offline_nodes = NULL;
     char *offline_remote_nodes = NULL;
 
     if (output_format == mon_output_console) {
         blank_screen();
     }
     print_cluster_summary(stdout, data_set);
     print_as("\n");
 
     /* Gather node information (and print if in bad state or grouping by node) */
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
         const char *node_mode = NULL;
         char *node_name = get_node_display_name(node);
 
         /* Get node mode */
         if (node->details->unclean) {
             if (node->details->online) {
                 node_mode = "UNCLEAN (online)";
 
             } else if (node->details->pending) {
                 node_mode = "UNCLEAN (pending)";
 
             } else {
                 node_mode = "UNCLEAN (offline)";
             }
 
         } else if (node->details->pending) {
             node_mode = "pending";
 
         } else if (node->details->standby_onfail && node->details->online) {
             node_mode = "standby (on-fail)";
 
         } else if (node->details->standby) {
             if (node->details->online) {
                 node_mode = "standby";
             } else {
                 node_mode = "OFFLINE (standby)";
             }
 
         } else if (node->details->maintenance) {
             if (node->details->online) {
                 node_mode = "maintenance";
             } else {
                 node_mode = "OFFLINE (maintenance)";
             }
 
         } else if (node->details->online) {
             node_mode = "online";
             if (group_by_node == FALSE) {
                 if (is_container_remote_node(node)) {
                     online_guest_nodes = add_list_element(online_guest_nodes, node_name);
                 } else if (is_baremetal_remote_node(node)) {
                     online_remote_nodes = add_list_element(online_remote_nodes, node_name);
                 } else {
                     online_nodes = add_list_element(online_nodes, node_name);
                 }
                 free(node_name);
                 continue;
             }
         } else {
             node_mode = "OFFLINE";
             if (group_by_node == FALSE) {
                 if (is_baremetal_remote_node(node)) {
                     offline_remote_nodes = add_list_element(offline_remote_nodes, node_name);
                 } else if (is_container_remote_node(node)) {
                     /* ignore offline guest nodes */
                 } else {
                     offline_nodes = add_list_element(offline_nodes, node_name);
                 }
                 free(node_name);
                 continue;
             }
         }
 
         /* If we get here, node is in bad state, or we're grouping by node */
 
         /* Print the node name and status */
         if (is_container_remote_node(node)) {
             print_as("Guest");
         } else if (is_baremetal_remote_node(node)) {
             print_as("Remote");
         }
         print_as("Node %s: %s\n", node_name, node_mode);
 
         /* If we're grouping by node, print its resources */
         if (group_by_node) {
             if (print_brief) {
                 print_rscs_brief(node->details->running_rsc, "\t", print_opts | pe_print_rsconly,
                                  stdout, FALSE);
             } else {
                 GListPtr gIter2 = NULL;
 
                 for (gIter2 = node->details->running_rsc; gIter2 != NULL; gIter2 = gIter2->next) {
                     resource_t *rsc = (resource_t *) gIter2->data;
 
                     rsc->fns->print(rsc, "\t", print_opts | pe_print_rsconly, stdout);
                 }
             }
         }
         free(node_name);
     }
 
     /* If we're not grouping by node, summarize nodes by status */
     if (online_nodes) {
         print_as("Online: [%s ]\n", online_nodes);
         free(online_nodes);
     }
     if (offline_nodes) {
         print_as("OFFLINE: [%s ]\n", offline_nodes);
         free(offline_nodes);
     }
     if (online_remote_nodes) {
         print_as("RemoteOnline: [%s ]\n", online_remote_nodes);
         free(online_remote_nodes);
     }
     if (offline_remote_nodes) {
         print_as("RemoteOFFLINE: [%s ]\n", offline_remote_nodes);
         free(offline_remote_nodes);
     }
     if (online_guest_nodes) {
         print_as("GuestOnline: [%s ]\n", online_guest_nodes);
         free(online_guest_nodes);
     }
 
     /* Print resources section, if needed */
     print_resources(stdout, data_set, print_opts);
 
     /* print Node Attributes section if requested */
     if (show & mon_show_attributes) {
         print_node_attributes(stdout, data_set);
     }
 
     /* If requested, print resource operations (which includes failcounts)
      * or just failcounts
      */
     if (show & (mon_show_operations | mon_show_failcounts)) {
         print_node_summary(stdout, data_set,
                            ((show & mon_show_operations)? TRUE : FALSE));
     }
 
     /* If there were any failed actions, print them */
     if (xml_has_children(data_set->failed)) {
         print_failed_actions(stdout, data_set);
     }
 
     /* Print tickets if requested */
     if (show & mon_show_tickets) {
         print_cluster_tickets(stdout, data_set);
     }
 
     /* Print negative location constraints if requested */
     if (show & mon_show_bans) {
         print_neg_locations(stdout, data_set);
     }
 
 #if CURSES_ENABLED
     if (output_format == mon_output_console) {
         refresh();
     }
 #endif
 }
 
 /*!
  * \internal
  * \brief Print cluster status in XML format
  *
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_xml_status(pe_working_set_t * data_set)
 {
     FILE *stream = stdout;
     GListPtr gIter = NULL;
     int print_opts = get_resource_display_options();
 
     fprintf(stream, "<?xml version=\"1.0\"?>\n");
     fprintf(stream, "<crm_mon version=\"%s\">\n", VERSION);
 
     print_cluster_summary(stream, data_set);
 
     /*** NODES ***/
     fprintf(stream, "    <nodes>\n");
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
         const char *node_type = "unknown";
 
         switch (node->details->type) {
             case node_member:
                 node_type = "member";
                 break;
             case node_remote:
                 node_type = "remote";
                 break;
             case node_ping:
                 node_type = "ping";
                 break;
         }
 
         fprintf(stream, "        <node name=\"%s\" ", node->details->uname);
         fprintf(stream, "id=\"%s\" ", node->details->id);
         fprintf(stream, "online=\"%s\" ", node->details->online ? "true" : "false");
         fprintf(stream, "standby=\"%s\" ", node->details->standby ? "true" : "false");
         fprintf(stream, "standby_onfail=\"%s\" ", node->details->standby_onfail ? "true" : "false");
         fprintf(stream, "maintenance=\"%s\" ", node->details->maintenance ? "true" : "false");
         fprintf(stream, "pending=\"%s\" ", node->details->pending ? "true" : "false");
         fprintf(stream, "unclean=\"%s\" ", node->details->unclean ? "true" : "false");
         fprintf(stream, "shutdown=\"%s\" ", node->details->shutdown ? "true" : "false");
         fprintf(stream, "expected_up=\"%s\" ", node->details->expected_up ? "true" : "false");
         fprintf(stream, "is_dc=\"%s\" ", node->details->is_dc ? "true" : "false");
         fprintf(stream, "resources_running=\"%d\" ", g_list_length(node->details->running_rsc));
         fprintf(stream, "type=\"%s\" ", node_type);
         if (is_container_remote_node(node)) {
             fprintf(stream, "id_as_resource=\"%s\" ", node->details->remote_rsc->container->id);
         }
 
         if (group_by_node) {
             GListPtr lpc2 = NULL;
 
             fprintf(stream, ">\n");
             for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
                 resource_t *rsc = (resource_t *) lpc2->data;
 
                 rsc->fns->print(rsc, "            ", print_opts | pe_print_rsconly, stream);
             }
             fprintf(stream, "        </node>\n");
         } else {
             fprintf(stream, "/>\n");
         }
     }
     fprintf(stream, "    </nodes>\n");
 
     /* Print resources section, if needed */
     print_resources(stream, data_set, print_opts);
 
     /* print Node Attributes section if requested */
     if (show & mon_show_attributes) {
         print_node_attributes(stream, data_set);
     }
 
     /* If requested, print resource operations (which includes failcounts)
      * or just failcounts
      */
     if (show & (mon_show_operations | mon_show_failcounts)) {
         print_node_summary(stream, data_set,
                            ((show & mon_show_operations)? TRUE : FALSE));
     }
 
     /* If there were any failed actions, print them */
     if (xml_has_children(data_set->failed)) {
         print_failed_actions(stream, data_set);
     }
 
     /* Print tickets if requested */
     if (show & mon_show_tickets) {
         print_cluster_tickets(stream, data_set);
     }
 
     /* Print negative location constraints if requested */
     if (show & mon_show_bans) {
         print_neg_locations(stream, data_set);
     }
 
     fprintf(stream, "</crm_mon>\n");
     fflush(stream);
     fclose(stream);
 }
 
 /*!
  * \internal
  * \brief Print cluster status in HTML format (with HTTP headers if CGI)
  *
  * \param[in] data_set   Working set of CIB state
  * \param[in] filename   Name of file to write HTML to (ignored if CGI)
  *
  * \return 0 on success, -1 on error
  */
 static int
 print_html_status(pe_working_set_t * data_set, const char *filename)
 {
     FILE *stream;
     GListPtr gIter = NULL;
     char *filename_tmp = NULL;
     int print_opts = get_resource_display_options();
 
     if (output_format == mon_output_cgi) {
         stream = stdout;
         fprintf(stream, "Content-Type: text/html\n\n");
 
     } else {
         filename_tmp = crm_concat(filename, "tmp", '.');
         stream = fopen(filename_tmp, "w");
         if (stream == NULL) {
             crm_perror(LOG_ERR, "Cannot open %s for writing", filename_tmp);
             free(filename_tmp);
             return -1;
         }
     }
 
     fprintf(stream, "<html>\n");
     fprintf(stream, " <head>\n");
     fprintf(stream, "  <title>Cluster status</title>\n");
     fprintf(stream, "  <meta http-equiv=\"refresh\" content=\"%d\">\n", reconnect_msec / 1000);
     fprintf(stream, " </head>\n");
     fprintf(stream, "<body>\n");
 
     print_cluster_summary(stream, data_set);
 
     /*** NODE LIST ***/
 
     fprintf(stream, " <hr />\n <h2>Node List</h2>\n");
     fprintf(stream, "<ul>\n");
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
         char *node_name = get_node_display_name(node);
 
         fprintf(stream, "<li>Node: %s: ", node_name);
         if (node->details->standby_onfail && node->details->online) {
             fprintf(stream, "<font color=\"orange\">standby (on-fail)</font>\n");
         } else if (node->details->standby && node->details->online) {
             fprintf(stream, "<font color=\"orange\">standby</font>\n");
         } else if (node->details->standby) {
             fprintf(stream, "<font color=\"red\">OFFLINE (standby)</font>\n");
         } else if (node->details->maintenance && node->details->online) {
             fprintf(stream, "<font color=\"blue\">maintenance</font>\n");
         } else if (node->details->maintenance) {
             fprintf(stream, "<font color=\"red\">OFFLINE (maintenance)</font>\n");
         } else if (node->details->online) {
             fprintf(stream, "<font color=\"green\">online</font>\n");
         } else {
             fprintf(stream, "<font color=\"red\">OFFLINE</font>\n");
         }
         if (print_brief && group_by_node) {
             fprintf(stream, "<ul>\n");
             print_rscs_brief(node->details->running_rsc, NULL, print_opts | pe_print_rsconly,
                              stream, FALSE);
             fprintf(stream, "</ul>\n");
 
         } else if (group_by_node) {
             GListPtr lpc2 = NULL;
 
             fprintf(stream, "<ul>\n");
             for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
                 resource_t *rsc = (resource_t *) lpc2->data;
 
                 fprintf(stream, "<li>");
                 rsc->fns->print(rsc, NULL, print_opts | pe_print_rsconly, stream);
                 fprintf(stream, "</li>\n");
             }
             fprintf(stream, "</ul>\n");
         }
         fprintf(stream, "</li>\n");
         free(node_name);
     }
     fprintf(stream, "</ul>\n");
 
     /* Print resources section, if needed */
     print_resources(stream, data_set, print_opts);
 
     /* print Node Attributes section if requested */
     if (show & mon_show_attributes) {
         print_node_attributes(stream, data_set);
     }
 
     /* If requested, print resource operations (which includes failcounts)
      * or just failcounts
      */
     if (show & (mon_show_operations | mon_show_failcounts)) {
         print_node_summary(stream, data_set,
                            ((show & mon_show_operations)? TRUE : FALSE));
     }
 
     /* If there were any failed actions, print them */
     if (xml_has_children(data_set->failed)) {
         print_failed_actions(stream, data_set);
     }
 
     /* Print tickets if requested */
     if (show & mon_show_tickets) {
         print_cluster_tickets(stream, data_set);
     }
 
     /* Print negative location constraints if requested */
     if (show & mon_show_bans) {
         print_neg_locations(stream, data_set);
     }
 
     fprintf(stream, "</body>\n");
     fprintf(stream, "</html>\n");
     fflush(stream);
     fclose(stream);
 
     if (output_format != mon_output_cgi) {
         if (rename(filename_tmp, filename) != 0) {
             crm_perror(LOG_ERR, "Unable to rename %s->%s", filename_tmp, filename);
         }
         free(filename_tmp);
     }
     return 0;
 }
 
 #if ENABLE_SNMP
 #  include <net-snmp/net-snmp-config.h>
 #  include <net-snmp/snmpv3_api.h>
 #  include <net-snmp/agent/agent_trap.h>
 #  include <net-snmp/library/snmp_client.h>
 #  include <net-snmp/library/mib.h>
 #  include <net-snmp/library/snmp_debug.h>
 
 #  define add_snmp_field(list, oid_string, value) do {			\
 	oid name[MAX_OID_LEN];						\
         size_t name_length = MAX_OID_LEN;				\
 	if (snmp_parse_oid(oid_string, name, &name_length)) {		\
 	    int s_rc = snmp_add_var(list, name, name_length, 's', (value)); \
 	    if(s_rc != 0) {						\
 		crm_err("Could not add %s=%s rc=%d", oid_string, value, s_rc); \
 	    } else {							\
 		crm_trace("Added %s=%s", oid_string, value);		\
 	    }								\
 	} else {							\
 	    crm_err("Could not parse OID: %s", oid_string);		\
 	}								\
     } while(0)								\
 
 #  define add_snmp_field_int(list, oid_string, value) do {		\
 	oid name[MAX_OID_LEN];						\
         size_t name_length = MAX_OID_LEN;				\
 	if (snmp_parse_oid(oid_string, name, &name_length)) {		\
 	    if(NULL == snmp_pdu_add_variable(				\
 		   list, name, name_length, ASN_INTEGER,		\
 		   (u_char *) & value, sizeof(value))) {		\
 		crm_err("Could not add %s=%d", oid_string, value);	\
 	    } else {							\
 		crm_trace("Added %s=%d", oid_string, value);		\
 	    }								\
 	} else {							\
 	    crm_err("Could not parse OID: %s", oid_string);		\
 	}								\
     } while(0)								\
 
 static int
 snmp_input(int operation, netsnmp_session * session, int reqid, netsnmp_pdu * pdu, void *magic)
 {
     return 1;
 }
 
 static netsnmp_session *
 crm_snmp_init(const char *target, char *community)
 {
     static netsnmp_session *session = NULL;
 
 #  ifdef NETSNMPV53
     char target53[128];
 
     snprintf(target53, sizeof(target53), "%s:162", target);
 #  endif
 
     if (session) {
         return session;
     }
 
     if (target == NULL) {
         return NULL;
     }
 
     if (get_crm_log_level() > LOG_INFO) {
         char *debug_tokens = strdup("run:shell,snmptrap,tdomain");
 
         debug_register_tokens(debug_tokens);
         snmp_set_do_debugging(1);
     }
 
     session = calloc(1, sizeof(netsnmp_session));
     snmp_sess_init(session);
     session->version = SNMP_VERSION_2c;
     session->callback = snmp_input;
     session->callback_magic = NULL;
 
     if (community) {
         session->community_len = strlen(community);
         session->community = (unsigned char *)community;
     }
 
     session = snmp_add(session,
 #  ifdef NETSNMPV53
                        netsnmp_tdomain_transport(target53, 0, "udp"),
 #  else
                        netsnmp_transport_open_client("snmptrap", target),
 #  endif
                        NULL, NULL);
 
     if (session == NULL) {
         snmp_sess_perror("Could not create snmp transport", session);
     }
     return session;
 }
 
 #endif
 
 static int
 send_snmp_trap(const char *node, const char *rsc, const char *task, int target_rc, int rc,
                int status, const char *desc)
 {
     int ret = 1;
 
 #if ENABLE_SNMP
     static oid snmptrap_oid[] = { 1, 3, 6, 1, 6, 3, 1, 1, 4, 1, 0 };
     static oid sysuptime_oid[] = { 1, 3, 6, 1, 2, 1, 1, 3, 0 };
 
     netsnmp_pdu *trap_pdu;
     netsnmp_session *session = crm_snmp_init(snmp_target, snmp_community);
 
     trap_pdu = snmp_pdu_create(SNMP_MSG_TRAP2);
     if (!trap_pdu) {
         crm_err("Failed to create SNMP notification");
         return SNMPERR_GENERR;
     }
 
     if (1) {
         /* send uptime */
         char csysuptime[20];
         time_t now = time(NULL);
 
         sprintf(csysuptime, "%lld", (long long) now);
         snmp_add_var(trap_pdu, sysuptime_oid, sizeof(sysuptime_oid) / sizeof(oid), 't', csysuptime);
     }
 
     /* Indicate what the trap is by setting snmpTrapOid.0 */
     ret =
         snmp_add_var(trap_pdu, snmptrap_oid, sizeof(snmptrap_oid) / sizeof(oid), 'o',
                      snmp_crm_trap_oid);
     if (ret != 0) {
         crm_err("Failed set snmpTrapOid.0=%s", snmp_crm_trap_oid);
         return ret;
     }
 
     /* Add extries to the trap */
     if (rsc) {
         add_snmp_field(trap_pdu, snmp_crm_oid_rsc, rsc);
     }
     add_snmp_field(trap_pdu, snmp_crm_oid_node, node);
     add_snmp_field(trap_pdu, snmp_crm_oid_task, task);
     add_snmp_field(trap_pdu, snmp_crm_oid_desc, desc);
 
     add_snmp_field_int(trap_pdu, snmp_crm_oid_rc, rc);
     add_snmp_field_int(trap_pdu, snmp_crm_oid_trc, target_rc);
     add_snmp_field_int(trap_pdu, snmp_crm_oid_status, status);
 
     /* Send and cleanup */
     ret = snmp_send(session, trap_pdu);
     if (ret == 0) {
         /* error */
         snmp_sess_perror("Could not send SNMP trap", session);
         snmp_free_pdu(trap_pdu);
         ret = SNMPERR_GENERR;
     } else {
         ret = SNMPERR_SUCCESS;
     }
 #else
     crm_err("Sending SNMP traps is not supported by this installation");
 #endif
     return ret;
 }
 
 #if ENABLE_ESMTP
 #  include <auth-client.h>
 #  include <libesmtp.h>
 
 static void
 print_recipient_status(smtp_recipient_t recipient, const char *mailbox, void *arg)
 {
     const smtp_status_t *status;
 
     status = smtp_recipient_status(recipient);
     printf("%s: %d %s", mailbox, status->code, status->text);
 }
 
 static void
 event_cb(smtp_session_t session, int event_no, void *arg, ...)
 {
     int *ok;
     va_list alist;
 
     va_start(alist, arg);
     switch (event_no) {
         case SMTP_EV_CONNECT:
         case SMTP_EV_MAILSTATUS:
         case SMTP_EV_RCPTSTATUS:
         case SMTP_EV_MESSAGEDATA:
         case SMTP_EV_MESSAGESENT:
         case SMTP_EV_DISCONNECT:
             break;
 
         case SMTP_EV_WEAK_CIPHER:{
                 int bits = va_arg(alist, long);
                 ok = va_arg(alist, int *);
 
                 crm_debug("SMTP_EV_WEAK_CIPHER, bits=%d - accepted.", bits);
                 *ok = 1;
                 break;
             }
         case SMTP_EV_STARTTLS_OK:
             crm_debug("SMTP_EV_STARTTLS_OK - TLS started here.");
             break;
 
         case SMTP_EV_INVALID_PEER_CERTIFICATE:{
                 long vfy_result = va_arg(alist, long);
                 ok = va_arg(alist, int *);
 
                 /* There is a table in handle_invalid_peer_certificate() of mail-file.c */
                 crm_err("SMTP_EV_INVALID_PEER_CERTIFICATE: %ld", vfy_result);
                 *ok = 1;
                 break;
             }
         case SMTP_EV_NO_PEER_CERTIFICATE:
             ok = va_arg(alist, int *);
 
             crm_debug("SMTP_EV_NO_PEER_CERTIFICATE - accepted.");
             *ok = 1;
             break;
         case SMTP_EV_WRONG_PEER_CERTIFICATE:
             ok = va_arg(alist, int *);
 
             crm_debug("SMTP_EV_WRONG_PEER_CERTIFICATE - accepted.");
             *ok = 1;
             break;
         case SMTP_EV_NO_CLIENT_CERTIFICATE:
             ok = va_arg(alist, int *);
 
             crm_debug("SMTP_EV_NO_CLIENT_CERTIFICATE - accepted.");
             *ok = 1;
             break;
         default:
             crm_debug("Got event: %d - ignored.", event_no);
     }
     va_end(alist);
 }
 #endif
 
 #define BODY_MAX 2048
 
 #if ENABLE_ESMTP
 static void
 crm_smtp_debug(const char *buf, int buflen, int writing, void *arg)
 {
     char type = 0;
     int lpc = 0, last = 0, level = *(int *)arg;
 
     if (writing == SMTP_CB_HEADERS) {
         type = 'H';
     } else if (writing) {
         type = 'C';
     } else {
         type = 'S';
     }
 
     for (; lpc < buflen; lpc++) {
         switch (buf[lpc]) {
             case 0:
             case '\n':
                 if (last > 0) {
                     do_crm_log(level, "   %.*s", lpc - last, buf + last);
                 } else {
                     do_crm_log(level, "%c: %.*s", type, lpc - last, buf + last);
                 }
                 last = lpc + 1;
                 break;
         }
     }
 }
 #endif
 
 static int
 send_custom_trap(const char *node, const char *rsc, const char *task, int target_rc, int rc,
                  int status, const char *desc)
 {
     pid_t pid;
 
     /*setenv needs chars, these are ints */
     char *rc_s = crm_itoa(rc);
     char *status_s = crm_itoa(status);
     char *target_rc_s = crm_itoa(target_rc);
 
     crm_debug("Sending external notification to '%s' via '%s'", external_recipient, external_agent);
 
     if(rsc) {
         setenv("CRM_notify_rsc", rsc, 1);
     }
     if (external_recipient) {
         setenv("CRM_notify_recipient", external_recipient, 1);
     }
     setenv("CRM_notify_node", node, 1);
     setenv("CRM_notify_task", task, 1);
     setenv("CRM_notify_desc", desc, 1);
     setenv("CRM_notify_rc", rc_s, 1);
     setenv("CRM_notify_target_rc", target_rc_s, 1);
     setenv("CRM_notify_status", status_s, 1);
 
     pid = fork();
     if (pid == -1) {
         crm_perror(LOG_ERR, "notification fork() failed.");
     }
     if (pid == 0) {
         /* crm_debug("notification: I am the child. Executing the nofitication program."); */
         execl(external_agent, external_agent, NULL);
         exit(EXIT_FAILURE);
     }
 
     crm_trace("Finished running custom notification program '%s'.", external_agent);
     free(target_rc_s);
     free(status_s);
     free(rc_s);
     return 0;
 }
 
 static int
 send_smtp_trap(const char *node, const char *rsc, const char *task, int target_rc, int rc,
                int status, const char *desc)
 {
 #if ENABLE_ESMTP
     smtp_session_t session;
     smtp_message_t message;
     auth_context_t authctx;
     struct sigaction sa;
 
     int len = 25; /* Note: Check extra padding on the Subject line below */
     int noauth = 1;
     int smtp_debug = LOG_DEBUG;
     char crm_mail_body[BODY_MAX];
     char *crm_mail_subject = NULL;
 
     memset(&sa, 0, sizeof(struct sigaction));
 
     if (node == NULL) {
         node = "-";
     }
     if (rsc == NULL) {
         rsc = "-";
     }
     if (desc == NULL) {
         desc = "-";
     }
 
     if (crm_mail_to == NULL) {
         return 1;
     }
 
     if (crm_mail_host == NULL) {
         crm_mail_host = "localhost:25";
     }
 
     if (crm_mail_prefix == NULL) {
         crm_mail_prefix = "Cluster notification";
     }
 
     crm_debug("Sending '%s' mail to %s via %s", crm_mail_prefix, crm_mail_to, crm_mail_host);
 
     len += strlen(crm_mail_prefix);
     len += strlen(task);
     len += strlen(rsc);
     len += strlen(node);
     len += strlen(desc);
     len++;
 
     crm_mail_subject = calloc(1, len);
     /* If you edit this line, ensure you allocate enough memory for it by altering 'len' above */
     snprintf(crm_mail_subject, len, "%s - %s event for %s on %s: %s\r\n", crm_mail_prefix, task,
              rsc, node, desc);
 
     len = 0;
     len += snprintf(crm_mail_body + len, BODY_MAX - len, "\r\n%s\r\n", crm_mail_prefix);
     len += snprintf(crm_mail_body + len, BODY_MAX - len, "====\r\n\r\n");
     if (rc == target_rc) {
         len += snprintf(crm_mail_body + len, BODY_MAX - len,
                         "Completed operation %s for resource %s on %s\r\n", task, rsc, node);
     } else {
         len += snprintf(crm_mail_body + len, BODY_MAX - len,
                         "Operation %s for resource %s on %s failed: %s\r\n", task, rsc, node, desc);
     }
 
     len += snprintf(crm_mail_body + len, BODY_MAX - len, "\r\nDetails:\r\n");
     len += snprintf(crm_mail_body + len, BODY_MAX - len,
                     "\toperation status: (%d) %s\r\n", status, services_lrm_status_str(status));
     if (status == PCMK_LRM_OP_DONE) {
         len += snprintf(crm_mail_body + len, BODY_MAX - len,
                         "\tscript returned: (%d) %s\r\n", rc, services_ocf_exitcode_str(rc));
         len += snprintf(crm_mail_body + len, BODY_MAX - len,
                         "\texpected return value: (%d) %s\r\n", target_rc,
                         services_ocf_exitcode_str(target_rc));
     }
 
     auth_client_init();
     session = smtp_create_session();
     message = smtp_add_message(session);
 
     smtp_starttls_enable(session, Starttls_ENABLED);
 
     sa.sa_handler = SIG_IGN;
     sigemptyset(&sa.sa_mask);
     sa.sa_flags = 0;
     sigaction(SIGPIPE, &sa, NULL);
 
     smtp_set_server(session, crm_mail_host);
 
     authctx = auth_create_context();
     auth_set_mechanism_flags(authctx, AUTH_PLUGIN_PLAIN, 0);
 
     smtp_set_eventcb(session, event_cb, NULL);
 
     /* Now tell libESMTP it can use the SMTP AUTH extension.
      */
     if (!noauth) {
         crm_debug("Adding authentication context");
         smtp_auth_set_context(session, authctx);
     }
 
     if (crm_mail_from == NULL) {
         struct utsname us;
         char auto_from[BODY_MAX];
 
         CRM_ASSERT(uname(&us) == 0);
         snprintf(auto_from, BODY_MAX, "crm_mon@%s", us.nodename);
         smtp_set_reverse_path(message, auto_from);
 
     } else {
         /* NULL is ok */
         smtp_set_reverse_path(message, crm_mail_from);
     }
 
     smtp_set_header(message, "To", NULL /*phrase */ , NULL /*addr */ ); /* "Phrase" <addr> */
     smtp_add_recipient(message, crm_mail_to);
 
     /* Set the Subject: header and override any subject line in the message headers. */
     smtp_set_header(message, "Subject", crm_mail_subject);
     smtp_set_header_option(message, "Subject", Hdr_OVERRIDE, 1);
 
     smtp_set_message_str(message, crm_mail_body);
     smtp_set_monitorcb(session, crm_smtp_debug, &smtp_debug, 1);
 
     if (smtp_start_session(session)) {
         char buf[128];
         int rc = smtp_errno();
 
         crm_err("SMTP server problem: %s (%d)", smtp_strerror(rc, buf, sizeof buf), rc);
 
     } else {
         char buf[128];
         int rc = smtp_errno();
         const smtp_status_t *smtp_status = smtp_message_transfer_status(message);
 
         if (rc != 0) {
             crm_err("SMTP server problem: %s (%d)", smtp_strerror(rc, buf, sizeof buf), rc);
         }
         crm_info("Send status: %d %s", smtp_status->code, crm_str(smtp_status->text));
         smtp_enumerate_recipients(message, print_recipient_status, NULL);
     }
 
     smtp_destroy_session(session);
     auth_destroy_context(authctx);
     auth_client_exit();
 #endif
     return 0;
 }
 
 static void
 handle_rsc_op(xmlNode * xml, const char *node_id)
 {
     int rc = -1;
     int status = -1;
     int action = -1;
     int interval = 0;
     int target_rc = -1;
     int transition_num = -1;
     gboolean notify = TRUE;
 
     char *rsc = NULL;
     char *task = NULL;
     const char *desc = NULL;
     const char *magic = NULL;
     const char *id = NULL;
     char *update_te_uuid = NULL;
     const char *node = NULL;
 
     xmlNode *n = xml;
     xmlNode * rsc_op = xml;
 
     if(strcmp((const char*)xml->name, XML_LRM_TAG_RSC_OP) != 0) {
         xmlNode *cIter;
 
         for(cIter = xml->children; cIter; cIter = cIter->next) {
             handle_rsc_op(cIter, node_id);
         }
 
         return;
     }
 
     id = crm_element_value(rsc_op, XML_LRM_ATTR_TASK_KEY);
     if (id == NULL) {
         /* Compatibility with <= 1.1.5 */
         id = ID(rsc_op);
     }
 
     magic = crm_element_value(rsc_op, XML_ATTR_TRANSITION_MAGIC);
     if (magic == NULL) {
         /* non-change */
         return;
     }
 
     if (FALSE == decode_transition_magic(magic, &update_te_uuid, &transition_num, &action,
                                          &status, &rc, &target_rc)) {
         crm_err("Invalid event %s detected for %s", magic, id);
         return;
     }
 
     if (parse_op_key(id, &rsc, &task, &interval) == FALSE) {
         crm_err("Invalid event detected for %s", id);
         goto bail;
     }
 
     node = crm_element_value(rsc_op, XML_LRM_ATTR_TARGET);
 
     while (n != NULL && safe_str_neq(XML_CIB_TAG_STATE, TYPE(n))) {
         n = n->parent;
     }
 
     if(node == NULL && n) {
         node = crm_element_value(n, XML_ATTR_UNAME);
     }
 
     if (node == NULL && n) {
         node = ID(n);
     }
 
     if (node == NULL) {
         node = node_id;
     }
 
     if (node == NULL) {
         crm_err("No node detected for event %s (%s)", magic, id);
         goto bail;
     }
 
     /* look up where we expected it to be? */
     desc = pcmk_strerror(pcmk_ok);
     if (status == PCMK_LRM_OP_DONE && target_rc == rc) {
         crm_notice("%s of %s on %s completed: %s", task, rsc, node, desc);
         if (rc == PCMK_OCF_NOT_RUNNING) {
             notify = FALSE;
         }
 
     } else if (status == PCMK_LRM_OP_DONE) {
         desc = services_ocf_exitcode_str(rc);
         crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
 
     } else {
         desc = services_lrm_status_str(status);
         crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
     }
 
     if (notify && snmp_target) {
         send_snmp_trap(node, rsc, task, target_rc, rc, status, desc);
     }
     if (notify && crm_mail_to) {
         send_smtp_trap(node, rsc, task, target_rc, rc, status, desc);
     }
     if (notify && external_agent) {
         send_custom_trap(node, rsc, task, target_rc, rc, status, desc);
     }
   bail:
     free(update_te_uuid);
     free(rsc);
     free(task);
 }
 
 static gboolean
 mon_trigger_refresh(gpointer user_data)
 {
     mainloop_set_trigger(refresh_trigger);
     return FALSE;
 }
 
 #define NODE_PATT "/lrm[@id="
 static char *get_node_from_xpath(const char *xpath) 
 {
     char *nodeid = NULL;
     char *tmp = strstr(xpath, NODE_PATT);
 
     if(tmp) {
         tmp += strlen(NODE_PATT);
         tmp += 1;
 
         nodeid = strdup(tmp);
         tmp = strstr(nodeid, "\'");
         CRM_ASSERT(tmp);
         tmp[0] = 0;
     }
     return nodeid;
 }
 
 static void crm_diff_update_v2(const char *event, xmlNode * msg) 
 {
     xmlNode *change = NULL;
     xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
 
     for (change = __xml_first_child(diff); change != NULL; change = __xml_next(change)) {
         const char *name = NULL;
         const char *op = crm_element_value(change, XML_DIFF_OP);
         const char *xpath = crm_element_value(change, XML_DIFF_PATH);
         xmlNode *match = NULL;
         const char *node = NULL;
 
         if(op == NULL) {
             continue;
 
         } else if(strcmp(op, "create") == 0) {
             match = change->children;
 
         } else if(strcmp(op, "move") == 0) {
             continue;
 
         } else if(strcmp(op, "delete") == 0) {
             continue;
 
         } else if(strcmp(op, "modify") == 0) {
             match = first_named_child(change, XML_DIFF_RESULT);
             if(match) {
                 match = match->children;
             }
         }
 
         if(match) {
             name = (const char *)match->name;
         }
 
         crm_trace("Handling %s operation for %s %p, %s", op, xpath, match, name);
         if(xpath == NULL) {
             /* Version field, ignore */
 
         } else if(name == NULL) {
             crm_debug("No result for %s operation to %s", op, xpath);
             CRM_ASSERT(strcmp(op, "delete") == 0 || strcmp(op, "move") == 0);
 
         } else if(strcmp(name, XML_TAG_CIB) == 0) {
             xmlNode *state = NULL;
             xmlNode *status = first_named_child(match, XML_CIB_TAG_STATUS);
 
             for (state = __xml_first_child(status); state != NULL; state = __xml_next(state)) {
                 node = crm_element_value(state, XML_ATTR_UNAME);
                 if (node == NULL) {
                     node = ID(state);
                 }
                 handle_rsc_op(state, node);
             }
 
         } else if(strcmp(name, XML_CIB_TAG_STATUS) == 0) {
             xmlNode *state = NULL;
 
             for (state = __xml_first_child(match); state != NULL; state = __xml_next(state)) {
                 node = crm_element_value(state, XML_ATTR_UNAME);
                 if (node == NULL) {
                     node = ID(state);
                 }
                 handle_rsc_op(state, node);
             }
 
         } else if(strcmp(name, XML_CIB_TAG_STATE) == 0) {
             node = crm_element_value(match, XML_ATTR_UNAME);
             if (node == NULL) {
                 node = ID(match);
             }
             handle_rsc_op(match, node);
 
         } else if(strcmp(name, XML_CIB_TAG_LRM) == 0) {
             node = ID(match);
             handle_rsc_op(match, node);
 
         } else if(strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
             char *local_node = get_node_from_xpath(xpath);
 
             handle_rsc_op(match, local_node);
             free(local_node);
 
         } else if(strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
             char *local_node = get_node_from_xpath(xpath);
 
             handle_rsc_op(match, local_node);
             free(local_node);
 
         } else if(strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
             char *local_node = get_node_from_xpath(xpath);
 
             handle_rsc_op(match, local_node);
             free(local_node);
 
         } else {
             crm_trace("Ignoring %s operation for %s %p, %s", op, xpath, match, name);
         }
     }
 }
 
 static void crm_diff_update_v1(const char *event, xmlNode * msg) 
 {
     /* Process operation updates */
     xmlXPathObject *xpathObj = xpath_search(msg,
                                             "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED
                                             "//" XML_LRM_TAG_RSC_OP);
     int lpc = 0, max = numXpathResults(xpathObj);
 
     for (lpc = 0; lpc < max; lpc++) {
         xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
 
         handle_rsc_op(rsc_op, NULL);
     }
     freeXpathObject(xpathObj);
 }
 
 void
 crm_diff_update(const char *event, xmlNode * msg)
 {
     int rc = -1;
     long now = time(NULL);
     static bool stale = FALSE;
     static int updates = 0;
     static mainloop_timer_t *refresh_timer = NULL;
     xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
 
     print_dot();
 
     if(refresh_timer == NULL) {
         refresh_timer = mainloop_timer_add("refresh", 2000, FALSE, mon_trigger_refresh, NULL);
     }
 
     if (current_cib != NULL) {
         rc = xml_apply_patchset(current_cib, diff, TRUE);
 
         switch (rc) {
             case -pcmk_err_diff_resync:
             case -pcmk_err_diff_failed:
                 crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
                 free_xml(current_cib); current_cib = NULL;
                 break;
             case pcmk_ok:
                 updates++;
                 break;
             default:
                 crm_notice("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
                 free_xml(current_cib); current_cib = NULL;
         }
     }
 
     if (current_cib == NULL) {
         crm_trace("Re-requesting the full cib");
         cib->cmds->query(cib, NULL, &current_cib, cib_scope_local | cib_sync_call);
     }
 
     if (crm_mail_to || snmp_target || external_agent) {
         int format = 0;
         crm_element_value_int(diff, "format", &format);
         switch(format) {
             case 1:
                 crm_diff_update_v1(event, msg);
                 break;
             case 2:
                 crm_diff_update_v2(event, msg);
                 break;
             default:
                 crm_err("Unknown patch format: %d", format);
         }
     }
 
     if (current_cib == NULL) {
         if(!stale) {
             print_as("--- Stale data ---");
         }
         stale = TRUE;
         return;
     }
 
     stale = FALSE;
     /* Refresh
      * - immediately if the last update was more than 5s ago
      * - every 10 updates
      * - at most 2s after the last update
      */
     if ((now - last_refresh) > (reconnect_msec / 1000)) {
         mainloop_set_trigger(refresh_trigger);
         mainloop_timer_stop(refresh_timer);
         updates = 0;
 
     } else if(updates > 10) {
         mainloop_set_trigger(refresh_trigger);
         mainloop_timer_stop(refresh_timer);
         updates = 0;
 
     } else {
         mainloop_timer_start(refresh_timer);
     }
 }
 
 gboolean
 mon_refresh_display(gpointer user_data)
 {
     xmlNode *cib_copy = copy_xml(current_cib);
     pe_working_set_t data_set;
 
     last_refresh = time(NULL);
 
     if (cli_config_update(&cib_copy, NULL, FALSE) == FALSE) {
         if (cib) {
             cib->cmds->signoff(cib);
         }
         print_as("Upgrade failed: %s", pcmk_strerror(-pcmk_err_schema_validation));
         if (output_format == mon_output_console) {
             sleep(2);
         }
         clean_up(EX_USAGE);
         return FALSE;
     }
 
     set_working_set_defaults(&data_set);
     data_set.input = cib_copy;
     cluster_status(&data_set);
 
     /* Unpack constraints if any section will need them
      * (tickets may be referenced in constraints but not granted yet,
      * and bans need negative location constraints) */
     if (show & (mon_show_bans | mon_show_tickets)) {
         xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set.input);
         unpack_constraints(cib_constraints, &data_set);
     }
 
     switch (output_format) {
         case mon_output_html:
         case mon_output_cgi:
             if (print_html_status(&data_set, output_filename) != 0) {
                 fprintf(stderr, "Critical: Unable to output html file\n");
                 clean_up(EX_USAGE);
             }
             break;
 
         case mon_output_xml:
             print_xml_status(&data_set);
             break;
 
         case mon_output_monitor:
             print_simple_status(&data_set);
             if (has_warnings) {
                 clean_up(MON_STATUS_WARN);
             }
             break;
 
         case mon_output_plain:
         case mon_output_console:
             print_status(&data_set);
             break;
 
         case mon_output_none:
             break;
     }
 
     cleanup_alloc_calculations(&data_set);
     return TRUE;
 }
 
 void
 mon_st_callback(stonith_t * st, stonith_event_t * e)
 {
     char *desc = crm_strdup_printf("Operation %s requested by %s for peer %s: %s (ref=%s)",
                                  e->operation, e->origin, e->target, pcmk_strerror(e->result),
                                  e->id);
 
     if (snmp_target) {
         send_snmp_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
     }
     if (crm_mail_to) {
         send_smtp_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
     }
     if (external_agent) {
         send_custom_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
     }
     free(desc);
 }
 
 /*
  * De-init ncurses, signoff from the CIB and deallocate memory.
  */
 void
 clean_up(int rc)
 {
 #if ENABLE_SNMP
     netsnmp_session *session = crm_snmp_init(NULL, NULL);
 
     if (session) {
         snmp_close(session);
         snmp_shutdown("snmpapp");
     }
 #endif
 
 #if CURSES_ENABLED
     if (output_format == mon_output_console) {
         output_format = mon_output_plain;
         echo();
         nocbreak();
         endwin();
     }
 #endif
 
     if (cib != NULL) {
         cib->cmds->signoff(cib);
         cib_delete(cib);
         cib = NULL;
     }
 
     free(output_filename);
     free(xml_file);
     free(pid_file);
 
     if (rc >= 0) {
         crm_exit(rc);
     }
     return;
 }