diff --git a/crmd/callbacks.c b/crmd/callbacks.c
index 3c5669872f..f25c802b57 100644
--- a/crmd/callbacks.c
+++ b/crmd/callbacks.c
@@ -1,238 +1,236 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <string.h>
 #include <crmd_fsa.h>
 
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crm/cluster.h>
 #include <crm/cib.h>
 
 #include <crmd.h>
 #include <crmd_messages.h>
 #include <crmd_callbacks.h>
 #include <crmd_lrm.h>
 #include <tengine.h>
 #include <membership.h>
 
 void crmd_ha_connection_destroy(gpointer user_data);
 
 /* From join_dc... */
 extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
 
 void
 crmd_ha_connection_destroy(gpointer user_data)
 {
     crm_trace("Invoked");
     if (is_set(fsa_input_register, R_HA_DISCONNECTED)) {
         /* we signed out, so this is expected */
         crm_info("Heartbeat disconnection complete");
         return;
     }
 
     crm_crit("Lost connection to heartbeat service!");
     register_fsa_input(C_HA_DISCONNECT, I_ERROR, NULL);
     trigger_fsa(fsa_source);
 }
 
 void
 crmd_ha_msg_filter(xmlNode * msg)
 {
     if (AM_I_DC) {
         const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
 
         if (safe_str_eq(sys_from, CRM_SYSTEM_DC)) {
             const char *from = crm_element_value(msg, F_ORIG);
 
             if (safe_str_neq(from, fsa_our_uname)) {
                 int level = LOG_INFO;
                 const char *op = crm_element_value(msg, F_CRM_TASK);
 
                 /* make sure the election happens NOW */
                 if (fsa_state != S_ELECTION) {
                     ha_msg_input_t new_input;
 
                     level = LOG_WARNING;
                     new_input.msg = msg;
                     register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, &new_input,
                                            __FUNCTION__);
                 }
 
                 do_crm_log(level, "Another DC detected: %s (op=%s)", from, op);
                 goto done;
             }
         }
 
     } else {
         const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
 
         if (safe_str_eq(sys_to, CRM_SYSTEM_DC)) {
             return;
         }
     }
 
     /* crm_log_xml_trace("HA[inbound]", msg); */
     route_message(C_HA_MESSAGE, msg);
 
   done:
     trigger_fsa(fsa_source);
 }
 
 extern gboolean process_lrm_event(lrmd_event_data_t * op);
 
 void
 lrm_op_callback(lrmd_event_data_t * op)
 {
     CRM_CHECK(op != NULL, return);
     process_lrm_event(op);
 }
 
 static void
 crmd_proc_update(crm_node_t * member, enum crm_proc_flag client)
 {
     const char *status = NULL;
 
     CRM_CHECK(member != NULL, return);
     status = (member->processes & client) ? ONLINESTATUS : OFFLINESTATUS;
     crm_notice("Status update: Client %s/%s now has status [%s] (DC=%s)",
                member->uname, peer2text(client), status, AM_I_DC ? "true" : crm_str(fsa_our_dc));
 
     if (is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) {
         return;
     } else if (fsa_state == S_STOPPING) {
         return;
     }
 
     if (safe_str_eq(member->uname, fsa_our_dc) && crm_is_peer_active(member) == FALSE) {
         /* Did the DC leave us? */
         crm_info("Got client status callback - our DC is dead");
         register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL);
 
     } else if (AM_I_DC) {
-        enum crm_proc_flag messaging = crm_proc_plugin | crm_proc_heartbeat | crm_proc_cpg;
         xmlNode *update = NULL;
 
         update = create_node_state(member->uname,
-            (member->processes & messaging) ? ACTIVESTATUS : DEADSTATUS,
             NULL, status, NULL, NULL, FALSE, __FUNCTION__);
 
         fsa_cib_anon_update(XML_CIB_TAG_STATUS, update,
                             cib_scope_local | cib_quorum_override | cib_can_create);
         free_xml(update);
 
         if ((member->processes & client) == 0) {
             erase_node_from_join(member->uname);
             check_join_state(fsa_state, __FUNCTION__);
             fail_incompletable_actions(transition_graph, member->uuid);
 
         } else {
             register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
         }
     }
 
     trigger_fsa(fsa_source);
 }
 
 void
 peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
 {
     gboolean reset_status_entry = FALSE;
     uint32_t old = 0;
 
     set_bit(fsa_input_register, R_PEER_DATA);
     if (node->uname == NULL) {
         return;
     }
 
     switch (type) {
         case crm_status_uname:
             crm_info("status: %s is now %s", node->uname, node->state);
             /* reset_status_entry = TRUE; */
             /* If we've never seen the node, then it also wont be in the status section */
             break;
         case crm_status_nstate:
             crm_info("status: %s is now %s (was %s)", node->uname, node->state, (const char *)data);
             reset_status_entry = TRUE;
             break;
         case crm_status_processes:
             if (data) {
                 old = *(const uint32_t *)data;
             }
 
             if ((node->processes ^ old) & proc_flags) {
                 crmd_proc_update(node, proc_flags);
             }
             break;
     }
 
     /* Can this be removed now that do_cl_join_finalize_respond() does the same thing? */
     if (AM_I_DC && reset_status_entry && safe_str_eq(CRMD_STATE_ACTIVE, node->state)) {
         crm_action_t *down = match_down_event(0, node->uname, NULL);
 
         erase_status_tag(node->uname, XML_CIB_TAG_LRM, cib_scope_local);
         erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
         if (down) {
             const char *task = crm_element_value(down->xml, XML_LRM_ATTR_TASK);
 
             if (safe_str_eq(task, CRM_OP_FENCE)) {
                 crm_info("Node return implies stonith of %s (action %d) completed", node->uname,
                          down->id);
                 down->confirmed = TRUE;
             }
         }
 
         /* TODO: potentially we also want to set XML_NODE_JOIN_STATE and XML_NODE_EXPECTED here */
     }
 }
 
 void
 crmd_cib_connection_destroy(gpointer user_data)
 {
     CRM_CHECK(user_data == fsa_cib_conn,;);
 
     crm_trace("Invoked");
     trigger_fsa(fsa_source);
     fsa_cib_conn->state = cib_disconnected;
 
     if (is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) {
         crm_info("Connection to the CIB terminated...");
         return;
     }
 
     /* eventually this will trigger a reconnect, not a shutdown */
     crm_err("Connection to the CIB terminated...");
     register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
     clear_bit(fsa_input_register, R_CIB_CONNECTED);
 
     return;
 }
 
 gboolean
 crm_fsa_trigger(gpointer user_data)
 {
     crm_trace("Invoked (queue len: %d)", g_list_length(fsa_message_queue));
     s_crmd_fsa(C_FSA_INTERNAL);
     crm_trace("Exited  (queue len: %d)", g_list_length(fsa_message_queue));
     return TRUE;
 }
diff --git a/crmd/crmd_utils.h b/crmd/crmd_utils.h
index a4af537131..ced10497ce 100644
--- a/crmd/crmd_utils.h
+++ b/crmd/crmd_utils.h
@@ -1,95 +1,95 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 #ifndef CRMD_UTILS__H
 #  define CRMD_UTILS__H
 
 #  include <crm/crm.h>
 #  include <crm/common/xml.h>
 
 #  define CLIENT_EXIT_WAIT 30
 #  define FAKE_TE_ID	"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
 
 extern void process_client_disconnect(crmd_client_t * curr_client);
 
 #  define fsa_cib_update(section, data, options, call_id, user_name)	\
 	if(fsa_cib_conn != NULL) {					\
 	    call_id = fsa_cib_conn->cmds->delegated_variant_op(		\
 		fsa_cib_conn, CIB_OP_MODIFY, NULL, section, data,	\
 		NULL, options, user_name);				\
 									\
 	} else {							\
 		crm_err("No CIB connection available");			\
 	}
 
 #  define fsa_cib_anon_update(section, data, options)			\
 	if(fsa_cib_conn != NULL) {					\
 	    fsa_cib_conn->cmds->modify(					\
 		fsa_cib_conn, section, data, options);			\
 									\
 	} else {							\
 		crm_err("No CIB connection available");			\
 	}
 
 extern gboolean fsa_has_quorum;
 extern int last_peer_update;
 
 extern gboolean crm_timer_stop(fsa_timer_t * timer);
 extern gboolean crm_timer_start(fsa_timer_t * timer);
 extern gboolean crm_timer_popped(gpointer data);
 extern gboolean is_timer_started(fsa_timer_t *timer);
 
-extern xmlNode *create_node_state(const char *uname, const char *ha_state, const char *ccm_state,
-                                  const char *crmd_state, const char *join_state,
+extern xmlNode *create_node_state(const char *uname, const char *in_cluster,
+                                  const char *is_peer, const char *join_state,
                                   const char *exp_state, gboolean clear_shutdown, const char *src);
 
 extern void create_node_entry(const char *uuid, const char *uname, const char *type);
 
 extern gboolean stop_subsystem(struct crm_subsystem_s *centry, gboolean force_quit);
 extern gboolean start_subsystem(struct crm_subsystem_s *centry);
 
 extern void fsa_dump_actions(long long action, const char *text);
 extern void fsa_dump_inputs(int log_level, const char *text, long long input_register);
 
 extern gboolean update_dc(xmlNode * msg);
 extern void erase_node_from_join(const char *node);
 extern void populate_cib_nodes(gboolean with_client_status);
 extern void crm_update_quorum(gboolean quorum, gboolean force_update);
 extern void erase_status_tag(const char *uname, const char *tag, int options);
 extern void update_attrd(const char *host, const char *name, const char *value,
                          const char *user_name);
 
 extern const char *get_timer_desc(fsa_timer_t * timer);
 gboolean too_many_st_failures(void);
 
 #  define start_transition(state) do {					\
 	switch(state) {							\
 	    case S_TRANSITION_ENGINE:					\
 		register_fsa_action(A_TE_CANCEL);			\
 		break;							\
 	    case S_POLICY_ENGINE:					\
 	    case S_IDLE:						\
 		register_fsa_input(C_FSA_INTERNAL, I_PE_CALC, NULL);	\
 		break;							\
 	    default:							\
 		crm_debug("NOT starting a new transition in state %s",	\
 			  fsa_state2string(fsa_state));			\
 		break;							\
 	}								\
     } while(0)
 
 #endif
diff --git a/crmd/heartbeat.c b/crmd/heartbeat.c
index d5dfdc1b8a..2e5547e867 100644
--- a/crmd/heartbeat.c
+++ b/crmd/heartbeat.c
@@ -1,553 +1,553 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 /* put these first so that uuid_t is defined without conflicts */
 #include <crm_internal.h>
 
 #include <string.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/cluster.h>
 #include <crmd_messages.h>
 #include <crmd_fsa.h>
 #include <fsa_proto.h>
 #include <crmd_callbacks.h>
 #include <tengine.h>
 #include <membership.h>
 
 #include <ocf/oc_event.h>
 #include <ocf/oc_membership.h>
 
 void oc_ev_special(const oc_ev_t *, oc_ev_class_t, int);
 void ccm_event_detail(const oc_ev_membership_t * oc, oc_ed_t event);
 gboolean crmd_ha_msg_dispatch(ll_cluster_t * cluster_conn, gpointer user_data);
 void crmd_ccm_msg_callback(oc_ed_t event, void *cookie, size_t size, const void *data);
 int ccm_dispatch(gpointer user_data);
 
 #define CCM_EVENT_DETAIL 0
 #define CCM_EVENT_DETAIL_PARTIAL 0
 
 int (*ccm_api_callback_done) (void *cookie) = NULL;
 int (*ccm_api_handle_event) (const oc_ev_t * token) = NULL;
 static gboolean fsa_have_quorum = FALSE;
 
 static oc_ev_t *fsa_ev_token;
 static void *ccm_library = NULL;
 static int num_ccm_register_fails = 0;
 static int max_ccm_register_fails = 30;
 
 static void
 ccm_connection_destroy(void *userdata)
 {
 }
 
 /*	 A_CCM_CONNECT	*/
 void
 do_ccm_control(long long action,
                enum crmd_fsa_cause cause,
                enum crmd_fsa_state cur_state,
                enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     static struct mainloop_fd_callbacks ccm_fd_callbacks = 
         {
             .dispatch = ccm_dispatch,
             .destroy = ccm_connection_destroy,
         };
 
     if (is_heartbeat_cluster()) {
         int (*ccm_api_register) (oc_ev_t ** token) =
             find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_register", 1);
 
         int (*ccm_api_set_callback) (const oc_ev_t * token,
                                      oc_ev_class_t class,
                                      oc_ev_callback_t * fn,
                                      oc_ev_callback_t ** prev_fn) =
             find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_set_callback", 1);
 
         void (*ccm_api_special) (const oc_ev_t *, oc_ev_class_t, int) =
             find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_special", 1);
         int (*ccm_api_activate) (const oc_ev_t * token, int *fd) =
             find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_activate", 1);
         int (*ccm_api_unregister) (oc_ev_t * token) =
             find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_unregister", 1);
 
         if (action & A_CCM_DISCONNECT) {
             set_bit(fsa_input_register, R_CCM_DISCONNECTED);
             (*ccm_api_unregister) (fsa_ev_token);
         }
 
         if (action & A_CCM_CONNECT) {
             int ret;
             int fsa_ev_fd;
             gboolean did_fail = FALSE;
 
             crm_trace("Registering with CCM");
             clear_bit(fsa_input_register, R_CCM_DISCONNECTED);
             ret = (*ccm_api_register) (&fsa_ev_token);
             if (ret != 0) {
                 crm_warn("CCM registration failed");
                 did_fail = TRUE;
             }
 
             if (did_fail == FALSE) {
                 crm_trace("Setting up CCM callbacks");
                 ret = (*ccm_api_set_callback) (fsa_ev_token, OC_EV_MEMB_CLASS,
                                                crmd_ccm_msg_callback, NULL);
                 if (ret != 0) {
                     crm_warn("CCM callback not set");
                     did_fail = TRUE;
                 }
             }
             if (did_fail == FALSE) {
                 (*ccm_api_special) (fsa_ev_token, OC_EV_MEMB_CLASS, 0 /*don't care */ );
 
                 crm_trace("Activating CCM token");
                 ret = (*ccm_api_activate) (fsa_ev_token, &fsa_ev_fd);
                 if (ret != 0) {
                     crm_warn("CCM Activation failed");
                     did_fail = TRUE;
                 }
             }
 
             if (did_fail) {
                 num_ccm_register_fails++;
                 (*ccm_api_unregister) (fsa_ev_token);
 
                 if (num_ccm_register_fails < max_ccm_register_fails) {
                     crm_warn("CCM Connection failed"
                              " %d times (%d max)", num_ccm_register_fails, max_ccm_register_fails);
 
                     crm_timer_start(wait_timer);
                     crmd_fsa_stall(NULL);
                     return;
 
                 } else {
                     crm_err("CCM Activation failed %d (max) times", num_ccm_register_fails);
                     register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
                     return;
                 }
             }
 
             crm_info("CCM connection established... waiting for first callback");
             mainloop_add_fd("heartbeat-ccm", fsa_ev_fd, fsa_ev_token, &ccm_fd_callbacks);
 
         }
     }
 
     if (action & ~(A_CCM_CONNECT | A_CCM_DISCONNECT)) {
         crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__);
     }
 }
 
 void
 ccm_event_detail(const oc_ev_membership_t * oc, oc_ed_t event)
 {
     int lpc;
     gboolean member = FALSE;
 
     member = FALSE;
 
     crm_trace("-----------------------");
     crm_info("%s: trans=%d, nodes=%d, new=%d, lost=%d n_idx=%d, "
              "new_idx=%d, old_idx=%d",
              ccm_event_name(event),
              oc->m_instance,
              oc->m_n_member, oc->m_n_in, oc->m_n_out, oc->m_memb_idx, oc->m_in_idx, oc->m_out_idx);
 
 #  if !CCM_EVENT_DETAIL_PARTIAL
     for (lpc = 0; lpc < oc->m_n_member; lpc++) {
         crm_info("\tCURRENT: %s [nodeid=%d, born=%d]",
                  oc->m_array[oc->m_memb_idx + lpc].node_uname,
                  oc->m_array[oc->m_memb_idx + lpc].node_id,
                  oc->m_array[oc->m_memb_idx + lpc].node_born_on);
 
         if (safe_str_eq(fsa_our_uname, oc->m_array[oc->m_memb_idx + lpc].node_uname)) {
             member = TRUE;
         }
     }
     if (member == FALSE) {
         crm_warn("MY NODE IS NOT IN CCM THE MEMBERSHIP LIST");
     }
 #  endif
     for (lpc = 0; lpc < (int)oc->m_n_in; lpc++) {
         crm_info("\tNEW:     %s [nodeid=%d, born=%d]",
                  oc->m_array[oc->m_in_idx + lpc].node_uname,
                  oc->m_array[oc->m_in_idx + lpc].node_id,
                  oc->m_array[oc->m_in_idx + lpc].node_born_on);
     }
 
     for (lpc = 0; lpc < (int)oc->m_n_out; lpc++) {
         crm_info("\tLOST:    %s [nodeid=%d, born=%d]",
                  oc->m_array[oc->m_out_idx + lpc].node_uname,
                  oc->m_array[oc->m_out_idx + lpc].node_id,
                  oc->m_array[oc->m_out_idx + lpc].node_born_on);
     }
 
     crm_trace("-----------------------");
 
 }
 
 
 /*	 A_CCM_UPDATE_CACHE	*/
 /*
  * Take the opportunity to update the node status in the CIB as well
  */
 void
 do_ccm_update_cache(enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state,
                     oc_ed_t event, const oc_ev_membership_t * oc, xmlNode * xml)
 {
     unsigned long long instance = 0;
     unsigned int lpc = 0;
 
     if (is_heartbeat_cluster()) {
         CRM_ASSERT(oc != NULL);
         instance = oc->m_instance;
     }
 
     CRM_ASSERT(crm_peer_seq <= instance);
 
     switch (cur_state) {
         case S_STOPPING:
         case S_TERMINATE:
         case S_HALT:
             crm_debug("Ignoring %s CCM event %llu, we're in state %s",
                       ccm_event_name(event), instance, fsa_state2string(cur_state));
             return;
         case S_ELECTION:
             register_fsa_action(A_ELECTION_CHECK);
             break;
         default:
             break;
     }
 
     if (is_heartbeat_cluster()) {
         ccm_event_detail(oc, event);
 
         /*--*-- Recently Dead Member Nodes --*--*/
         for (lpc = 0; lpc < oc->m_n_out; lpc++) {
             crm_update_ccm_node(oc, lpc + oc->m_out_idx, CRM_NODE_LOST, instance);
         }
 
             /*--*-- All Member Nodes --*--*/
         for (lpc = 0; lpc < oc->m_n_member; lpc++) {
             crm_update_ccm_node(oc, lpc + oc->m_memb_idx, CRM_NODE_ACTIVE, instance);
         }
     }
 
     if (event == OC_EV_MS_EVICTED) {
         crm_update_peer(__FUNCTION__, 0, 0, 0, -1, 0, fsa_our_uuid, fsa_our_uname, NULL, CRM_NODE_EVICTED);
 
         /* todo: drop back to S_PENDING instead */
         /* get out... NOW!
          *
          * go via the error recovery process so that HA will
          *    restart us if required
          */
         register_fsa_error_adv(cause, I_ERROR, NULL, NULL, __FUNCTION__);
     }
 
     post_cache_update(instance);
     return;
 }
 
 int
 ccm_dispatch(gpointer user_data)
 {
     int rc = 0;
     oc_ev_t *ccm_token = (oc_ev_t *) user_data;
     gboolean was_error = FALSE;
 
     crm_trace("Invoked");
     if (ccm_api_handle_event == NULL) {
         ccm_api_handle_event =
             find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_handle_event", 1);
     }
     rc = (*ccm_api_handle_event) (ccm_token);
 
     if (rc != 0) {
         if (is_set(fsa_input_register, R_CCM_DISCONNECTED) == FALSE) {
             /* we signed out, so this is expected */
             register_fsa_input(C_CCM_CALLBACK, I_ERROR, NULL);
             crm_err("CCM connection appears to have failed: rc=%d.", rc);
         }
         was_error = TRUE;
     }
 
     trigger_fsa(fsa_source);
     if(was_error) {
         return -1;
     }
 
     return 0;
 }
 
 void
 crmd_ccm_msg_callback(oc_ed_t event, void *cookie, size_t size, const void *data)
 {
     gboolean update_cache = FALSE;
     const oc_ev_membership_t *membership = data;
 
     gboolean update_quorum = FALSE;
 
     crm_trace("Invoked");
     CRM_ASSERT(data != NULL);
 
     crm_info("Quorum %s after event=%s (id=%d)",
              ccm_have_quorum(event) ? "(re)attained" : "lost",
              ccm_event_name(event), membership->m_instance);
 
     if (crm_peer_seq > membership->m_instance) {
         crm_err("Membership instance ID went backwards! %llu->%d",
                 crm_peer_seq, membership->m_instance);
         CRM_ASSERT(crm_peer_seq <= membership->m_instance);
         return;
     }
 
     /*
      * OC_EV_MS_NEW_MEMBERSHIP:   membership with quorum
      * OC_EV_MS_MS_INVALID:       membership without quorum
      * OC_EV_MS_NOT_PRIMARY:      previous membership no longer valid
      * OC_EV_MS_PRIMARY_RESTORED: previous membership restored
      * OC_EV_MS_EVICTED:          the client is evicted from ccm.
      */
 
     switch (event) {
         case OC_EV_MS_NEW_MEMBERSHIP:
         case OC_EV_MS_INVALID:
             update_cache = TRUE;
             update_quorum = TRUE;
             break;
         case OC_EV_MS_NOT_PRIMARY:
             break;
         case OC_EV_MS_PRIMARY_RESTORED:
             update_cache = TRUE;
             crm_peer_seq = membership->m_instance;
             break;
         case OC_EV_MS_EVICTED:
             update_quorum = TRUE;
             register_fsa_input(C_FSA_INTERNAL, I_STOP, NULL);
             crm_err("Shutting down after CCM event: %s", ccm_event_name(event));
             break;
         default:
             crm_err("Unknown CCM event: %d", event);
     }
 
     if (update_quorum) {
         crm_have_quorum = ccm_have_quorum(event);
         crm_update_quorum(crm_have_quorum, FALSE);
 
         if (crm_have_quorum == FALSE) {
             /* did we just loose quorum? */
             if (fsa_have_quorum) {
                 crm_info("Quorum lost: %s", ccm_event_name(event));
             }
         }
     }
 
     if (update_cache) {
         crm_trace("Updating cache after event %s", ccm_event_name(event));
         do_ccm_update_cache(C_CCM_CALLBACK, fsa_state, event, data, NULL);
 
     } else if (event != OC_EV_MS_NOT_PRIMARY) {
         crm_peer_seq = membership->m_instance;
         register_fsa_action(A_TE_CANCEL);
     }
 
     if (ccm_api_callback_done == NULL) {
         ccm_api_callback_done =
             find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_callback_done", 1);
     }
     (*ccm_api_callback_done) (cookie);
     return;
 }
 
 
 void
 crmd_ha_status_callback(const char *node, const char *status, void *private)
 {
     xmlNode *update = NULL;
     crm_node_t *peer = NULL;
 
     crm_notice("Status update: Node %s now has status [%s]", node, status);
 
     peer = crm_get_peer(0, node);
     if (safe_str_eq(status, PINGSTATUS)) {
         return;
     }
 
     if (safe_str_eq(status, DEADSTATUS)) {
         /* this node is toast */
         crm_update_peer_proc(__FUNCTION__, peer, crm_proc_heartbeat, OFFLINESTATUS);
         if (AM_I_DC) {
-            update = create_node_state(node, DEADSTATUS, XML_BOOLEAN_NO, OFFLINESTATUS,
+            update = create_node_state(node, XML_BOOLEAN_NO, OFFLINESTATUS,
                                        CRMD_JOINSTATE_DOWN, NULL, TRUE, __FUNCTION__);
         }
 
     } else {
         crm_update_peer_proc(__FUNCTION__, peer, crm_proc_heartbeat, ONLINESTATUS);
         if (AM_I_DC) {
-            update = create_node_state(node, ACTIVESTATUS, NULL, NULL,
+            update = create_node_state(node, NULL, NULL,
                                        CRMD_JOINSTATE_PENDING, NULL, FALSE, __FUNCTION__);
         }
     }
 
     trigger_fsa(fsa_source);
 
     if (update != NULL) {
         fsa_cib_anon_update(XML_CIB_TAG_STATUS, update,
                             cib_scope_local | cib_quorum_override | cib_can_create);
         free_xml(update);
     }
 }
 
 void
 crmd_client_status_callback(const char *node, const char *client, const char *status, void *private)
 {
     const char *join = NULL;
     crm_node_t *peer = NULL;
     gboolean clear_shutdown = FALSE;
 
     crm_trace("Invoked");
     if (safe_str_neq(client, CRM_SYSTEM_CRMD)) {
         return;
     }
 
     if (safe_str_eq(status, JOINSTATUS)) {
         clear_shutdown = TRUE;
         status = ONLINESTATUS;
         join = CRMD_JOINSTATE_PENDING;
 
     } else if (safe_str_eq(status, LEAVESTATUS)) {
         status = OFFLINESTATUS;
         join = CRMD_JOINSTATE_DOWN;
 /* 		clear_shutdown = TRUE; */
     }
 
     set_bit(fsa_input_register, R_PEER_DATA);
 
     crm_notice("Status update: Client %s/%s now has status [%s] (DC=%s)",
                node, client, status, AM_I_DC ? "true" : "false");
 
     if (safe_str_eq(status, ONLINESTATUS)) {
         /* remove the cached value in case it changed */
         crm_trace("Uncaching UUID for %s", node);
         unget_uuid(node);
     }
 
     peer = crm_get_peer(0, node);
 
     if (AM_I_DC) {
         xmlNode *update = NULL;
 
         crm_trace("Got client status callback");
         update =
-            create_node_state(node, NULL, NULL, status, join, NULL, clear_shutdown, __FUNCTION__);
+            create_node_state(node, NULL, status, join, NULL, clear_shutdown, __FUNCTION__);
 
         fsa_cib_anon_update(XML_CIB_TAG_STATUS, update,
                             cib_scope_local | cib_quorum_override | cib_can_create);
         free_xml(update);
     }
     crm_update_peer_proc(__FUNCTION__, peer, crm_proc_crmd, status);
 }
 
 void
 crmd_ha_msg_callback(HA_Message * hamsg, void *private_data)
 {
     int level = LOG_DEBUG;
     crm_node_t *from_node = NULL;
 
     xmlNode *msg = convert_ha_message(NULL, hamsg, __FUNCTION__);
     const char *from = crm_element_value(msg, F_ORIG);
     const char *op = crm_element_value(msg, F_CRM_TASK);
     const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
 
     CRM_CHECK(from != NULL, crm_log_xml_err(msg, "anon"); goto bail);
 
     crm_trace("HA[inbound]: %s from %s", op, from);
 
     if (crm_peer_cache == NULL || crm_active_peers() == 0) {
         crm_debug("Ignoring HA messages until we are"
                   " connected to the CCM (%s op from %s)", op, from);
         crm_log_xml_trace(msg, "HA[inbound]: Ignore (No CCM)");
         goto bail;
     }
 
     from_node = crm_get_peer(0, from);
     if (crm_is_peer_active(from_node) == FALSE) {
         if (safe_str_eq(op, CRM_OP_VOTE)) {
             level = LOG_WARNING;
 
         } else if (AM_I_DC && safe_str_eq(op, CRM_OP_JOIN_ANNOUNCE)) {
             level = LOG_WARNING;
 
         } else if (safe_str_eq(sys_from, CRM_SYSTEM_DC)) {
             level = LOG_WARNING;
         }
         do_crm_log(level,
                    "Ignoring HA message (op=%s) from %s: not in our"
                    " membership list (size=%d)", op, from, crm_active_peers());
 
         crm_log_xml_trace(msg, "HA[inbound]: CCM Discard");
 
     } else {
         crmd_ha_msg_filter(msg);
     }
 
   bail:
     free_xml(msg);
     return;
 }
 
 gboolean
 crmd_ha_msg_dispatch(ll_cluster_t * cluster_conn, gpointer user_data)
 {
     IPC_Channel *channel = NULL;
     gboolean stay_connected = TRUE;
 
     crm_trace("Invoked");
 
     if (cluster_conn != NULL) {
         channel = cluster_conn->llc_ops->ipcchan(cluster_conn);
     }
 
     CRM_CHECK(cluster_conn != NULL,;);
     CRM_CHECK(channel != NULL,;);
 
     if (channel != NULL && IPC_ISRCONN(channel)) {
         if (cluster_conn->llc_ops->msgready(cluster_conn) == 0) {
             crm_trace("no message ready yet");
         }
         /* invoke the callbacks but dont block */
         cluster_conn->llc_ops->rcvmsg(cluster_conn, 0);
     }
 
     if (channel == NULL || channel->ch_status != IPC_CONNECT) {
         if (is_set(fsa_input_register, R_HA_DISCONNECTED) == FALSE) {
             crm_crit("Lost connection to heartbeat service.");
         } else {
             crm_info("Lost connection to heartbeat service.");
         }
         trigger_fsa(fsa_source);
         stay_connected = FALSE;
     }
 
     return stay_connected;
 }
diff --git a/crmd/lrm.c b/crmd/lrm.c
index 07da747eb2..c97d40c0e4 100644
--- a/crmd/lrm.c
+++ b/crmd/lrm.c
@@ -1,2002 +1,2002 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 
 #include <crm/crm.h>
 #include <crm/services.h>
 #include <crm/cib/internal.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crmd.h>
 #include <crmd_fsa.h>
 #include <crmd_messages.h>
 #include <crmd_callbacks.h>
 #include <crmd_lrm.h>
 
 #define START_DELAY_THRESHOLD 5 * 60 * 1000
 
 typedef struct resource_history_s {
     char *id;
     lrmd_rsc_info_t rsc;
     lrmd_event_data_t *last;
     lrmd_event_data_t *failed;
     GList *recurring_op_list;
 
     /* Resources must be stopped using the same
      * parameters they were started with.  This hashtable
      * holds the parameters that should be used for the next stop
      * cmd on this resource. */
     GHashTable *stop_params;
 } rsc_history_t;
 
 struct recurring_op_s {
     char *rsc_id;
     char *op_type;
     char *op_key;
     int call_id;
     int interval;
     int last_rc;
     int last_op_status;
     gboolean remove;
     gboolean cancelled;
 };
 
 struct pending_deletion_op_s {
     char *rsc;
     ha_msg_input_t *input;
 };
 
 struct delete_event_s {
     int rc;
     const char *rsc;
 };
 
 GHashTable *resource_history = NULL;
 GHashTable *pending_ops = NULL;
 GHashTable *deletion_ops = NULL;
 
 int num_lrm_register_fails = 0;
 int max_lrm_register_fails = 30;
 
 gboolean process_lrm_event(lrmd_event_data_t * op);
 gboolean is_rsc_active(const char *rsc_id);
 gboolean build_active_RAs(xmlNode * rsc_list);
 static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
 static int delete_rsc_status(const char *rsc_id, int call_options, const char *user_name);
 
 static lrmd_event_data_t *construct_op(xmlNode * rsc_op, const char *rsc_id, const char *operation);
 void do_lrm_rsc_op(lrmd_rsc_info_t * rsc, const char *operation, xmlNode * msg, xmlNode * request);
 
 void send_direct_ack(const char *to_host, const char *to_sys,
                      lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id);
 
 static void
 lrm_connection_destroy(void)
 {
     if (is_set(fsa_input_register, R_LRM_CONNECTED)) {
         crm_crit("LRM Connection failed");
         register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
         clear_bit(fsa_input_register, R_LRM_CONNECTED);
 
     } else {
         crm_info("LRM Connection disconnected");
     }
 
 }
 
 static void
 free_deletion_op(gpointer value)
 {
     struct pending_deletion_op_s *op = value;
 
     free(op->rsc);
     delete_ha_msg_input(op->input);
     free(op);
 }
 
 static void
 free_recurring_op(gpointer value)
 {
     struct recurring_op_s *op = (struct recurring_op_s *)value;
 
     free(op->rsc_id);
     free(op->op_type);
     free(op->op_key);
     free(op);
 }
 
 static char *
 make_stop_id(const char *rsc, int call_id)
 {
     char *op_id = NULL;
 
     op_id = calloc(1, strlen(rsc) + 34);
     if (op_id != NULL) {
         snprintf(op_id, strlen(rsc) + 34, "%s:%d", rsc, call_id);
     }
     return op_id;
 }
 
 static void
 copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
 {
     if (strstr(key, CRM_META "_") == NULL) {
         g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
     }
 }
 
 static void
 copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
 {
     if (strstr(key, CRM_META "_") != NULL) {
         g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
     }
 }
 
 static void
 history_cache_destroy(gpointer data)
 {
     rsc_history_t *entry = data;
 
     if (entry->stop_params) {
         g_hash_table_destroy(entry->stop_params);
     }
 
     free(entry->rsc.type);
     free(entry->rsc.class);
     free(entry->rsc.provider);
 
     lrmd_free_event(entry->failed);
     lrmd_free_event(entry->last);
     free(entry->id);
     free(entry);
 }
 
 static void
 update_history_cache(lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
 {
     int target_rc = 0;
     rsc_history_t *entry = NULL;
 
     if (op->rsc_deleted) {
         crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
         delete_rsc_status(op->rsc_id, cib_quorum_override, NULL);
         return;
     }
 
     if (safe_str_eq(op->op_type, RSC_NOTIFY)) {
         return;
     }
 
     crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);
 
     entry = g_hash_table_lookup(resource_history, op->rsc_id);
     if (entry == NULL && rsc) {
         entry = calloc(1, sizeof(rsc_history_t));
         entry->id = strdup(op->rsc_id);
         g_hash_table_insert(resource_history, entry->id, entry);
 
         entry->rsc.id = entry->id;
         entry->rsc.type = strdup(rsc->type);
         entry->rsc.class = strdup(rsc->class);
         if (rsc->provider) {
             entry->rsc.provider = strdup(rsc->provider);
         } else {
             entry->rsc.provider = NULL;
         }
 
     } else if (entry == NULL) {
         crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
         return;
     }
 
     target_rc = rsc_op_expected_rc(op);
     if (op->op_status == PCMK_LRM_OP_CANCELLED) {
         if (op->interval > 0) {
             GList *gIter, *gIterNext;
 
             crm_trace("Removing cancelled recurring op: %s_%s_%d", op->rsc_id, op->op_type, op->interval);
 
             for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIterNext) {
                 lrmd_event_data_t *existing = gIter->data;
                 gIterNext = gIter->next;
 
                 if (safe_str_eq(op->rsc_id, existing->rsc_id)
                     && safe_str_eq(op->op_type, existing->op_type)
                     && op->interval == existing->interval) {
                     lrmd_free_event(existing);
                     entry->recurring_op_list = g_list_delete_link(entry->recurring_op_list, gIter);
                 }
             }
             return;
 
         } else {
             crm_trace("Skipping %s_%s_%d rc=%d, status=%d", op->rsc_id, op->op_type, op->interval,
                       op->rc, op->op_status);
         }
 
     } else if (did_rsc_op_fail(op, target_rc)) {
         /* We must store failed monitors here
          * - otherwise the block below will cause them to be forgetten them when a stop happens
          */
         if (entry->failed) {
             lrmd_free_event(entry->failed);
         }
         entry->failed = lrmd_copy_event(op);
 
     } else if (op->interval == 0) {
         if (entry->last) {
             lrmd_free_event(entry->last);
         }
         entry->last = lrmd_copy_event(op);
 
         if (op->params &&
             (safe_str_eq(CRMD_ACTION_START, op->op_type) ||
             safe_str_eq(CRMD_ACTION_STATUS, op->op_type))) {
 
             if (entry->stop_params) {
                 g_hash_table_destroy(entry->stop_params);
             }
             entry->stop_params = g_hash_table_new_full(crm_str_hash,
                 g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
 
             g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
         }
     }
 
     if (op->interval > 0) {
         crm_trace("Adding recurring op: %s_%s_%d", op->rsc_id, op->op_type, op->interval);
         entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));
 
     } else if (entry->recurring_op_list && safe_str_eq(op->op_type, RSC_STATUS) == FALSE) {
         GList *gIter = entry->recurring_op_list;
 
         crm_trace("Dropping %d recurring ops because of: %s_%s_%d",
                   g_list_length(gIter), op->rsc_id, op->op_type, op->interval);
         for (; gIter != NULL; gIter = gIter->next) {
             lrmd_free_event(gIter->data);
         }
         g_list_free(entry->recurring_op_list);
         entry->recurring_op_list = NULL;
     }
 }
 
 /*	 A_LRM_CONNECT	*/
 void
 do_lrm_control(long long action,
                enum crmd_fsa_cause cause,
                enum crmd_fsa_state cur_state,
                enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     if (fsa_lrm_conn == NULL) {
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
         return;
     }
 
     if (action & A_LRM_DISCONNECT) {
         if (verify_stopped(cur_state, LOG_INFO) == FALSE) {
             if(action == A_LRM_DISCONNECT) {
                 crmd_fsa_stall(NULL);
                 return;
             }
         }
 
         if (is_set(fsa_input_register, R_LRM_CONNECTED)) {
             clear_bit(fsa_input_register, R_LRM_CONNECTED);
 
             fsa_lrm_conn->cmds->disconnect(fsa_lrm_conn);
             crm_info("Disconnected from the LRM");
         }
 
         g_hash_table_destroy(resource_history);
         resource_history = NULL;
         g_hash_table_destroy(deletion_ops);
         deletion_ops = NULL;
         g_hash_table_destroy(pending_ops);
         pending_ops = NULL;
     }
 
     if (action & A_LRM_CONNECT) {
         int ret = pcmk_ok;
 
         deletion_ops = g_hash_table_new_full(crm_str_hash, g_str_equal,
                                              g_hash_destroy_str, free_deletion_op);
 
         pending_ops = g_hash_table_new_full(crm_str_hash, g_str_equal,
                                             g_hash_destroy_str, free_recurring_op);
 
         resource_history = g_hash_table_new_full(crm_str_hash, g_str_equal,
                                                  NULL, history_cache_destroy);
 
         crm_debug("Connecting to the LRM");
         ret = fsa_lrm_conn->cmds->connect(fsa_lrm_conn, CRM_SYSTEM_CRMD, NULL);
 
         if (ret != pcmk_ok) {
             if (++num_lrm_register_fails < max_lrm_register_fails) {
                 crm_warn("Failed to sign on to the LRM %d"
                          " (%d max) times", num_lrm_register_fails, max_lrm_register_fails);
 
                 crm_timer_start(wait_timer);
                 crmd_fsa_stall(NULL);
                 return;
             }
         }
 
         if (ret == pcmk_ok) {
             crm_trace("LRM: set_lrm_callback...");
             fsa_lrm_conn->cmds->set_callback(fsa_lrm_conn, lrm_op_callback);
         }
 
         if (ret != pcmk_ok) {
             crm_err("Failed to sign on to the LRM %d" " (max) times", num_lrm_register_fails);
             register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
             return;
         }
 
         set_bit(fsa_input_register, R_LRM_CONNECTED);
         crm_debug("LRM connection established");
     }
 
     if (action & ~(A_LRM_CONNECT | A_LRM_DISCONNECT)) {
         crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__);
     }
 }
 
 static void
 ghash_print_pending(gpointer key, gpointer value, gpointer user_data)
 {
     const char *stop_id = key;
     int *log_level = user_data;
     struct recurring_op_s *pending = value;
 
     do_crm_log(*log_level, "Pending action: %s (%s)", stop_id, pending->op_key);
 }
 
 static void
 ghash_print_pending_for_rsc(gpointer key, gpointer value, gpointer user_data)
 {
     const char *stop_id = key;
     char *rsc = user_data;
     struct recurring_op_s *pending = value;
 
     if (safe_str_eq(rsc, pending->rsc_id)) {
         crm_notice("%sction %s (%s) incomplete at shutdown",
                    pending->interval == 0 ? "A" : "Recurring a", stop_id, pending->op_key);
     }
 }
 
 static void
 ghash_count_pending(gpointer key, gpointer value, gpointer user_data)
 {
     int *counter = user_data;
     struct recurring_op_s *pending = value;
 
     if (pending->interval > 0) {
         /* Ignore recurring actions in the shutdown calculations */
         return;
     }
 
     (*counter)++;
 }
 
 gboolean
 verify_stopped(enum crmd_fsa_state cur_state, int log_level)
 {
     int counter = 0;
     gboolean rc = TRUE;
 
     GHashTableIter gIter;
     rsc_history_t *entry = NULL;
 
     crm_debug("Checking for active resources before exit");
 
     if (cur_state == S_TERMINATE) {
         log_level = LOG_ERR;
     }
 
     if (pending_ops) {
         if (is_set(fsa_input_register, R_LRM_CONNECTED)) {
             /* Only log/complain about non-recurring actions */
             g_hash_table_foreach_remove(pending_ops, stop_recurring_actions, NULL);
         }
         g_hash_table_foreach(pending_ops, ghash_count_pending, &counter);
     }
 
     if (counter > 0) {
         rc = FALSE;
         do_crm_log(log_level,
                    "%d pending LRM operations at shutdown%s",
                    counter, cur_state == S_TERMINATE ? "" : "... waiting");
 
         if (cur_state == S_TERMINATE || !is_set(fsa_input_register, R_SENT_RSC_STOP)) {
             g_hash_table_foreach(pending_ops, ghash_print_pending, &log_level);
         }
         goto bail;
     }
 
     if (resource_history == NULL) {
         goto bail;
     }
 
     g_hash_table_iter_init(&gIter, resource_history);
     while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
         if (is_rsc_active(entry->id) == FALSE) {
             continue;
         }
 
         crm_err("Resource %s was active at shutdown."
                 "  You may ignore this error if it is unmanaged.", entry->id);
 
         g_hash_table_foreach(pending_ops, ghash_print_pending_for_rsc, entry->id);
     }
 
   bail:
     set_bit(fsa_input_register, R_SENT_RSC_STOP);
 
     if (cur_state == S_TERMINATE) {
         rc = TRUE;
     }
 
     return rc;
 }
 
 static char *
 get_rsc_metadata(const char *type, const char *class, const char *provider)
 {
     char *metadata = NULL;
 
     CRM_CHECK(type != NULL, return NULL);
     CRM_CHECK(class != NULL, return NULL);
     if (provider == NULL) {
         provider = "heartbeat";
     }
 
     crm_trace("Retreiving metadata for %s::%s:%s", type, class, provider);
     fsa_lrm_conn->cmds->get_metadata(fsa_lrm_conn, class, provider, type, &metadata, 0);
 
     if (metadata) {
         /* copy the metadata because the LRM likes using
          *   g_alloc instead of cl_malloc
          */
         char *m_copy = strdup(metadata);
 
         g_free(metadata);
         metadata = m_copy;
 
     } else {
         crm_warn("No metadata found for %s::%s:%s", type, class, provider);
     }
 
     return metadata;
 }
 
 typedef struct reload_data_s {
     char *key;
     char *metadata;
     time_t last_query;
     gboolean can_reload;
     GListPtr restart_list;
 } reload_data_t;
 
 static void
 g_hash_destroy_reload(gpointer data)
 {
     reload_data_t *reload = data;
 
     free(reload->key);
     free(reload->metadata);
     g_list_free_full(reload->restart_list, free);
     free(reload);
 }
 
 GHashTable *reload_hash = NULL;
 static GListPtr
 get_rsc_restart_list(lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
 {
     int len = 0;
     char *key = NULL;
     char *copy = NULL;
     const char *value = NULL;
     const char *provider = NULL;
 
     xmlNode *param = NULL;
     xmlNode *params = NULL;
     xmlNode *actions = NULL;
     xmlNode *metadata = NULL;
 
     time_t now = time(NULL);
     reload_data_t *reload = NULL;
 
     if (reload_hash == NULL) {
         reload_hash = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, g_hash_destroy_reload);
     }
 
     provider = rsc->provider;
     if (provider == NULL) {
         provider = "heartbeat";
     }
 
     len = strlen(rsc->type) + strlen(rsc->class) + strlen(provider) + 4;
     key = malloc( len);
     snprintf(key, len, "%s::%s:%s", rsc->type, rsc->class, provider);
 
     reload = g_hash_table_lookup(reload_hash, key);
 
     if (reload && ((now - 9) > reload->last_query)
         && safe_str_eq(op->op_type, RSC_START)) {
         reload = NULL;          /* re-query */
     }
 
     if (reload == NULL) {
         xmlNode *action = NULL;
 
         reload = calloc(1, sizeof(reload_data_t));
         g_hash_table_replace(reload_hash, key, reload);
 
         reload->last_query = now;
         reload->key = key;
         key = NULL;
         reload->metadata = get_rsc_metadata(rsc->type, rsc->class, provider);
 
         metadata = string2xml(reload->metadata);
         if (metadata == NULL) {
             crm_err("Metadata for %s::%s:%s is not valid XML",
                     rsc->provider, rsc->class, rsc->type);
             goto cleanup;
         }
 
         actions = find_xml_node(metadata, "actions", TRUE);
 
         for (action = __xml_first_child(actions); action != NULL; action = __xml_next(action)) {
             if (crm_str_eq((const char *)action->name, "action", TRUE)) {
                 value = crm_element_value(action, "name");
                 if (safe_str_eq("reload", value)) {
                     reload->can_reload = TRUE;
                     break;
                 }
             }
         }
 
         if (reload->can_reload == FALSE) {
             goto cleanup;
         }
 
         params = find_xml_node(metadata, "parameters", TRUE);
         for (param = __xml_first_child(params); param != NULL; param = __xml_next(param)) {
             if (crm_str_eq((const char *)param->name, "parameter", TRUE)) {
                 value = crm_element_value(param, "unique");
                 if (crm_is_true(value)) {
                     value = crm_element_value(param, "name");
                     if (value == NULL) {
                         crm_err("%s: NULL param", key);
                         continue;
                     }
                     crm_debug("Attr %s is not reloadable", value);
                     copy = strdup(value);
                     CRM_CHECK(copy != NULL, continue);
                     reload->restart_list = g_list_append(reload->restart_list, copy);
                 }
             }
         }
     }
 
   cleanup:
     free(key);
     free_xml(metadata);
     return reload->restart_list;
 }
 
 static void
 append_restart_list(lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, xmlNode * update, const char *version)
 {
     int len = 0;
     char *list = NULL;
     char *digest = NULL;
     const char *value = NULL;
     gboolean non_empty = FALSE;
     xmlNode *restart = NULL;
     GListPtr restart_list = NULL;
     GListPtr lpc = NULL;
 
     if (op->interval > 0) {
         /* monitors are not reloadable */
         return;
 
     } else if (op->params == NULL) {
         crm_debug("%s has no parameters", ID(update));
         return;
 
     } else if (rsc == NULL) {
         return;
 
     } else if (crm_str_eq(CRMD_ACTION_STOP, op->op_type, TRUE)) {
         /* Stopped resources don't need to be reloaded */
         return;
 
     } else if (compare_version("1.0.8", version) > 0) {
         /* Caller version does not support reloads */
         return;
     }
 
     restart_list = get_rsc_restart_list(rsc, op);
     if (restart_list == NULL) {
         /* Resource does not support reloads */
         return;
     }
 
     restart = create_xml_node(NULL, XML_TAG_PARAMS);
     for (lpc = restart_list; lpc != NULL; lpc = lpc->next) {
         const char *param = (const char *)lpc->data;
 
         int start = len;
 
         CRM_CHECK(param != NULL, continue);
         value = g_hash_table_lookup(op->params, param);
         if (value != NULL) {
             non_empty = TRUE;
             crm_xml_add(restart, param, value);
         }
         len += strlen(param) + 2;
         list = realloc(list, len + 1);
         sprintf(list + start, " %s ", param);
     }
 
     digest = calculate_operation_digest(restart, version);
     crm_xml_add(update, XML_LRM_ATTR_OP_RESTART, list);
     crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);
 
 #if 0
     crm_debug("%s: %s, %s", rsc->id, digest, list);
     if (non_empty) {
         crm_log_xml_debug(restart, "restart digest source");
     }
 #endif
 
     free_xml(restart);
     free(digest);
     free(list);
 }
 
 static gboolean
 build_operation_update(xmlNode * parent, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *src)
 {
     int target_rc = 0;
     xmlNode *xml_op = NULL;
     const char *caller_version = CRM_FEATURE_SET;
 
     if (op == NULL) {
         return FALSE;
 
     } else if (AM_I_DC) {
 
     } else if (fsa_our_dc_version != NULL) {
         caller_version = fsa_our_dc_version;
     } else if (op->params == NULL) {
         caller_version = fsa_our_dc_version;
     } else {
         /* there is a small risk in formerly mixed clusters that
          *   it will be sub-optimal.
          * however with our upgrade policy, the update we send
          *   should still be completely supported anyway
          */
         caller_version = g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION);
         crm_debug("Falling back to operation originator version: %s", caller_version);
     }
 
     target_rc = rsc_op_expected_rc(op);
     xml_op = create_operation_update(parent, op, caller_version, target_rc, src, LOG_DEBUG);
 
     if (xml_op) {
         append_restart_list(rsc, op, xml_op, caller_version);
     }
     return TRUE;
 }
 
 gboolean
 is_rsc_active(const char *rsc_id)
 {
     rsc_history_t *entry = NULL;
 
     crm_trace("Processing lrmd_rsc_info_t entry %s", rsc_id);
 
     entry = g_hash_table_lookup(resource_history, rsc_id);
     if (entry == NULL || entry->last == NULL) {
         return FALSE;
     }
 
     if (entry->last->rc == PCMK_EXECRA_OK && safe_str_eq(entry->last->op_type, CRMD_ACTION_STOP)) {
         return FALSE;
 
     } else if (entry->last->rc == PCMK_EXECRA_OK
                && safe_str_eq(entry->last->op_type, CRMD_ACTION_MIGRATE)) {
         /* a stricter check is too complex...
          * leave that to the PE
          */
         return FALSE;
 
     } else if (entry->last->rc == PCMK_EXECRA_NOT_RUNNING) {
         return FALSE;
 
     } else if (entry->last->interval == 0 && entry->last->rc == PCMK_EXECRA_NOT_CONFIGURED) {
         /* Badly configured resources can't be reliably stopped */
         return FALSE;
     }
 
     return TRUE;
 }
 
 gboolean
 build_active_RAs(xmlNode * rsc_list)
 {
     GHashTableIter iter;
     rsc_history_t *entry = NULL;
 
     g_hash_table_iter_init(&iter, resource_history);
     while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
 
         GList *gIter = NULL;
         xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE);
 
         crm_xml_add(xml_rsc, XML_ATTR_ID, entry->id);
         crm_xml_add(xml_rsc, XML_ATTR_TYPE, entry->rsc.type);
         crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, entry->rsc.class);
         crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, entry->rsc.provider);
 
         build_operation_update(xml_rsc, &(entry->rsc), entry->last, __FUNCTION__);
         build_operation_update(xml_rsc, &(entry->rsc), entry->failed, __FUNCTION__);
         for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
             build_operation_update(xml_rsc, &(entry->rsc), gIter->data, __FUNCTION__);
         }
     }
 
     return FALSE;
 }
 
 xmlNode *
 do_lrm_query(gboolean is_replace)
 {
     gboolean shut_down = FALSE;
     xmlNode *xml_result = NULL;
     xmlNode *xml_state = NULL;
     xmlNode *xml_data = NULL;
     xmlNode *rsc_list = NULL;
     const char *exp_state = CRMD_STATE_ACTIVE;
 
     if (is_set(fsa_input_register, R_SHUTDOWN)) {
         exp_state = CRMD_STATE_INACTIVE;
         shut_down = TRUE;
     }
 
-    xml_state = create_node_state(fsa_our_uname, ACTIVESTATUS, XML_BOOLEAN_TRUE,
+    xml_state = create_node_state(fsa_our_uname, XML_BOOLEAN_TRUE,
                                   ONLINESTATUS, CRMD_JOINSTATE_MEMBER, exp_state,
                                   !shut_down, __FUNCTION__);
 
     xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM);
     crm_xml_add(xml_data, XML_ATTR_ID, fsa_our_uuid);
     rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES);
 
     /* Build a list of active (not always running) resources */
     build_active_RAs(rsc_list);
 
     xml_result = create_cib_fragment(xml_state, XML_CIB_TAG_STATUS);
     crm_log_xml_trace(xml_state, "Current state of the LRM");
     free_xml(xml_state);
 
     return xml_result;
 }
 
 static void
 notify_deleted(ha_msg_input_t * input, const char *rsc_id, int rc)
 {
     lrmd_event_data_t *op = NULL;
     const char *from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
     const char *from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
 
     crm_info("Notifying %s on %s that %s was%s deleted",
              from_sys, from_host, rsc_id, rc == pcmk_ok ? "" : " not");
 
     op = construct_op(input->xml, rsc_id, CRMD_ACTION_DELETE);
     CRM_ASSERT(op != NULL);
 
     if (rc == pcmk_ok) {
         op->op_status = PCMK_LRM_OP_DONE;
         op->rc = PCMK_EXECRA_OK;
     } else {
         op->op_status = PCMK_LRM_OP_ERROR;
         op->rc = PCMK_EXECRA_UNKNOWN_ERROR;
     }
 
     send_direct_ack(from_host, from_sys, NULL, op, rsc_id);
     lrmd_free_event(op);
 
     if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
         /* this isn't expected - trigger a new transition */
         time_t now = time(NULL);
         char *now_s = crm_itoa(now);
 
         crm_debug("Triggering a refresh after %s deleted %s from the LRM", from_sys, rsc_id);
 
         update_attr_delegate(
             fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
             "last-lrm-refresh", now_s, FALSE, NULL);
 
         free(now_s);
     }
 }
 
 static gboolean
 lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
 {
     struct delete_event_s *event = user_data;
     struct pending_deletion_op_s *op = value;
 
     if (safe_str_eq(event->rsc, op->rsc)) {
         notify_deleted(op->input, event->rsc, event->rc);
         return TRUE;
     }
     return FALSE;
 }
 
 static gboolean
 lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
 {
     const char *rsc = user_data;
     struct recurring_op_s *pending = value;
 
     if (safe_str_eq(rsc, pending->rsc_id)) {
         crm_info("Removing op %s:%d for deleted resource %s",
                  pending->op_key, pending->call_id, rsc);
         return TRUE;
     }
     return FALSE;
 }
 
 /*
  * Remove the rsc from the CIB
  *
  * Avoids refreshing the entire LRM section of this host
  */
 #define rsc_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']"
 
 static int
 delete_rsc_status(const char *rsc_id, int call_options, const char *user_name)
 {
     char *rsc_xpath = NULL;
     int max = 0;
     int rc = pcmk_ok;
 
     CRM_CHECK(rsc_id != NULL, return -ENXIO);
 
     max = strlen(rsc_template) + strlen(rsc_id) + strlen(fsa_our_uname) + 1;
     rsc_xpath = calloc(1, max);
     snprintf(rsc_xpath, max, rsc_template, fsa_our_uname, rsc_id);
 
     rc = fsa_cib_conn->cmds->delegated_variant_op(fsa_cib_conn, CIB_OP_DELETE, NULL, rsc_xpath,
                                                   NULL, NULL, call_options | cib_xpath, user_name);
 
     free(rsc_xpath);
     return rc;
 }
 
 static void
 delete_rsc_entry(ha_msg_input_t * input, const char *rsc_id, GHashTableIter *rsc_gIter, int rc, const char *user_name)
 {
     struct delete_event_s event;
 
     CRM_CHECK(rsc_id != NULL, return);
 
     if (rc == pcmk_ok) {
         char *rsc_id_copy = strdup(rsc_id);
 
         if (rsc_gIter)
             g_hash_table_iter_remove(rsc_gIter);
         else
             g_hash_table_remove(resource_history, rsc_id_copy);
         crm_debug("sync: Sending delete op for %s", rsc_id_copy);
         delete_rsc_status(rsc_id_copy, cib_quorum_override, user_name);
 
         g_hash_table_foreach_remove(pending_ops, lrm_remove_deleted_op, rsc_id_copy);
 
         free(rsc_id_copy);
     }
 
     if (input) {
         notify_deleted(input, rsc_id, rc);
     }
 
     event.rc = rc;
     event.rsc = rsc_id;
     g_hash_table_foreach_remove(deletion_ops, lrm_remove_deleted_rsc, &event);
 }
 
 /*
  * Remove the op from the CIB
  *
  * Avoids refreshing the entire LRM section of this host
  */
 
 #define op_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s']"
 #define op_call_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s' and @"XML_LRM_ATTR_CALLID"='%d']"
 
 static void
 delete_op_entry(lrmd_event_data_t * op, const char *rsc_id, const char *key, int call_id)
 {
     xmlNode *xml_top = NULL;
 
     if (op != NULL) {
         xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
         crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
         crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);
 
         if(op->interval > 0) {
             char *op_id = generate_op_key(op->rsc_id, op->op_type, op->interval);
             /* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
             crm_xml_add(xml_top, XML_ATTR_ID, op_id);
             free(op_id);
         }
 
         crm_debug("async: Sending delete op for %s_%s_%d (call=%d)",
                   op->rsc_id, op->op_type, op->interval, op->call_id);
 
         fsa_cib_conn->cmds->delete(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top, cib_quorum_override);
 
     } else if (rsc_id != NULL && key != NULL) {
         int max = 0;
         char *op_xpath = NULL;
 
         if (call_id > 0) {
             max =
                 strlen(op_call_template) + strlen(rsc_id) + strlen(fsa_our_uname) + strlen(key) +
                 10;
             op_xpath = calloc(1, max);
             snprintf(op_xpath, max, op_call_template, fsa_our_uname, rsc_id, key, call_id);
 
         } else {
             max = strlen(op_template) + strlen(rsc_id) + strlen(fsa_our_uname) + strlen(key) + 1;
             op_xpath = calloc(1, max);
             snprintf(op_xpath, max, op_template, fsa_our_uname, rsc_id, key);
         }
 
         crm_debug("sync: Sending delete op for %s (call=%d)", rsc_id, call_id);
         fsa_cib_conn->cmds->delete(fsa_cib_conn, op_xpath, NULL, cib_quorum_override | cib_xpath);
 
         free(op_xpath);
 
     } else {
         crm_err("Not enough information to delete op entry: rsc=%p key=%p", rsc_id, key);
         return;
     }
 
     crm_log_xml_trace(xml_top, "op:cancel");
     free_xml(xml_top);
 }
 
 void lrm_clear_last_failure(const char *rsc_id)
 {
     char *attr = NULL;
     GHashTableIter iter;
     rsc_history_t *entry = NULL;
 
     attr = generate_op_key(rsc_id, "last_failure", 0);
     delete_op_entry(NULL, rsc_id, attr, 0);
     free(attr);
 
     if (!resource_history) {
         return;
     }
 
     g_hash_table_iter_init(&iter, resource_history);
     while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
         if (safe_str_eq(rsc_id, entry->id)) {
             lrmd_free_event(entry->failed);
             entry->failed = NULL;
         }
     }
 }
 
 static gboolean
 cancel_op(const char *rsc_id, const char *key, int op, gboolean remove)
 {
     int rc = pcmk_ok;
     struct recurring_op_s *pending = NULL;
 
     CRM_CHECK(op != 0, return FALSE);
     CRM_CHECK(rsc_id != NULL, return FALSE);
     if (key == NULL) {
         key = make_stop_id(rsc_id, op);
     }
     pending = g_hash_table_lookup(pending_ops, key);
 
     if (pending) {
         if (remove && pending->remove == FALSE) {
             pending->remove = TRUE;
             crm_debug("Scheduling %s for removal", key);
         }
 
         if (pending->cancelled) {
             crm_debug("Operation %s already cancelled", key);
             return TRUE;
         }
 
         pending->cancelled = TRUE;
 
     } else {
         crm_info("No pending op found for %s", key);
         return TRUE;
     }
 
     crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);
 
     rc = fsa_lrm_conn->cmds->cancel(fsa_lrm_conn,
         pending->rsc_id,
         pending->op_type,
         pending->interval);
 
     if (rc == pcmk_ok) {
         crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
     } else {
         crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
         /* The caller needs to make sure the entry is
          * removed from the pending_ops list
          *
          * Usually by returning TRUE inside the worker function
          * supplied to g_hash_table_foreach_remove()
          *
          * Not removing the entry from pending_ops will block
          * the node from shutting down
          */
         return FALSE;
     }
 
     return TRUE;
 }
 
 struct cancel_data {
     gboolean done;
     gboolean remove;
     const char *key;
     lrmd_rsc_info_t *rsc;
 };
 
 static gboolean
 cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
 {
     struct cancel_data *data = user_data;
     struct recurring_op_s *op = (struct recurring_op_s *)value;
 
     if (safe_str_eq(op->op_key, data->key)) {
         data->done = TRUE;
         if (cancel_op(data->rsc->id, key, op->call_id, data->remove) == FALSE) {
             return TRUE;
         }
     }
     return FALSE;
 }
 
 static gboolean
 cancel_op_key(lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
 {
     struct cancel_data data;
 
     CRM_CHECK(rsc != NULL, return FALSE);
     CRM_CHECK(key != NULL, return FALSE);
 
     data.key = key;
     data.rsc = rsc;
     data.done = FALSE;
     data.remove = remove;
 
     g_hash_table_foreach_remove(pending_ops, cancel_action_by_key, &data);
     return data.done;
 }
 
 static lrmd_rsc_info_t *
 get_lrm_resource(xmlNode * resource, xmlNode * op_msg, gboolean do_create)
 {
     lrmd_rsc_info_t *rsc = NULL;
     const char *id = ID(resource);
     const char *type = crm_element_value(resource, XML_ATTR_TYPE);
     const char *class = crm_element_value(resource, XML_AGENT_ATTR_CLASS);
     const char *provider = crm_element_value(resource, XML_AGENT_ATTR_PROVIDER);
     const char *long_id = crm_element_value(resource, XML_ATTR_ID_LONG);
 
     crm_trace("Retrieving %s from the LRM.", id);
     CRM_CHECK(id != NULL, return NULL);
 
     rsc = fsa_lrm_conn->cmds->get_rsc_info(fsa_lrm_conn, id, 0);
     if (!rsc && long_id) {
         rsc = fsa_lrm_conn->cmds->get_rsc_info(fsa_lrm_conn, long_id, 0);
     }
 
     if (!rsc && do_create) {
         CRM_CHECK(class != NULL, return NULL);
         CRM_CHECK(type != NULL, return NULL);
 
         crm_trace("Adding rsc %s before operation", id);
 
         fsa_lrm_conn->cmds->register_rsc(fsa_lrm_conn,
             id, class, provider, type, 0);
 
         rsc = fsa_lrm_conn->cmds->get_rsc_info(fsa_lrm_conn, id, 0);
 
         if (!rsc) {
             fsa_data_t *msg_data = NULL;
 
             crm_err("Could not add resource %s to LRM", id);
             register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
         }
     }
 
     return rsc;
 }
 
 static void
 delete_resource(const char *id, lrmd_rsc_info_t * rsc, GHashTableIter *gIter,
                 const char *sys, const char *host, const char *user, ha_msg_input_t * request)
 {
     int rc = pcmk_ok;
 
     crm_info("Removing resource %s for %s (%s) on %s", id, sys, user ? user : "internal", host);
 
     if (rsc) {
         rc = fsa_lrm_conn->cmds->unregister_rsc(fsa_lrm_conn, id, 0);
     }
 
     if (rc == pcmk_ok) {
         crm_trace("Resource '%s' deleted", id);
     } else if (rc == -EINPROGRESS) {
         crm_info("Deletion of resource '%s' pending", id);
         if (request) {
             struct pending_deletion_op_s *op = NULL;
             char *ref = crm_element_value_copy(request->msg, XML_ATTR_REFERENCE);
 
             op = calloc(1, sizeof(struct pending_deletion_op_s));
             op->rsc = strdup(rsc->id);
             op->input = copy_ha_msg_input(request);
             g_hash_table_insert(deletion_ops, ref, op);
         }
         return;
     } else {
         crm_warn("Deletion of resource '%s' for %s (%s) on %s failed: %d",
                  id, sys, user ? user : "internal", host, rc);
     }
 
     delete_rsc_entry(request, id, gIter, rc, user);
 }
 
 /*	 A_LRM_INVOKE	*/
 void
 do_lrm_invoke(long long action,
               enum crmd_fsa_cause cause,
               enum crmd_fsa_state cur_state,
               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     gboolean done = FALSE;
     gboolean create_rsc = TRUE;
     const char *crm_op = NULL;
     const char *from_sys = NULL;
     const char *from_host = NULL;
     const char *operation = NULL;
     ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
     const char *user_name = NULL;
 
 #if ENABLE_ACL
     user_name = crm_element_value(input->msg, F_CRM_USER);
     crm_trace("LRM command from user '%s'", user_name);
 #endif
 
     crm_op = crm_element_value(input->msg, F_CRM_TASK);
     from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
     if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
         from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
     }
 
     crm_trace("LRM command from: %s", from_sys);
 
     if (safe_str_eq(crm_op, CRM_OP_LRM_DELETE)) {
         operation = CRMD_ACTION_DELETE;
 
     } else if (safe_str_eq(operation, CRM_OP_LRM_REFRESH)) {
         crm_op = CRM_OP_LRM_REFRESH;
 
     } else if (safe_str_eq(crm_op, CRM_OP_LRM_FAIL)) {
         rsc_history_t *entry = NULL;
         lrmd_event_data_t *op = NULL;
         lrmd_rsc_info_t *rsc = NULL;
         xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);
 
         CRM_CHECK(xml_rsc != NULL, return);
 
         /* The lrmd can not fail a resource, it does not understand the
          * concept of success or failure in relation to a resource, it simply
          * executes operations and reports the results. We determine what a failure is.
          * Becaues of this, if we want to fail a resource we have to fake what we
          * understand a failure to look like.
          *
          * To do this we create a fake lrmd operation event for the resource
          * we want to fail.  We then pass that event to the lrmd client callback
          * so it will be processed as if it actually came from the lrmd. */
         op = construct_op(input->xml, ID(xml_rsc), "asyncmon");
         free((char *) op->user_data);
         op->user_data = NULL;
         entry = g_hash_table_lookup(resource_history, op->rsc_id);
         /* Make sure the call id is greater than the last successful operation,
          * otherwise the failure will not result in a possible recovery of the resource
          * as it could appear the failure occurred before the successful start */
         if (entry && entry->last) {
             op->call_id = entry->last->call_id + 1;
             if (op->call_id < 0) {
                 op->call_id = 1;
             }
         }
         op->interval = 0;
         op->op_status = PCMK_LRM_OP_DONE;
         op->rc = PCMK_EXECRA_UNKNOWN_ERROR;
         CRM_ASSERT(op != NULL);
 
 #  if ENABLE_ACL
         if (user_name && is_privileged(user_name) == FALSE) {
             crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc));
             send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
             lrmd_free_event(op);
             return;
         }
 #  endif
 
         rsc = get_lrm_resource(xml_rsc, input->xml, create_rsc);
         if (rsc) {
             crm_info("Failing resource %s...", rsc->id);
             process_lrm_event(op);
             op->op_status = PCMK_LRM_OP_DONE;
             op->rc = PCMK_EXECRA_OK;
             lrmd_free_rsc_info(rsc);
         } else {
             crm_info("Cannot find/create resource in order to fail it...");
             crm_log_xml_warn(input->msg, "bad input");
         }
 
         send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
         lrmd_free_event(op);
         return;
 
     } else if (input->xml != NULL) {
         operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK);
     }
 
     if (safe_str_eq(crm_op, CRM_OP_LRM_REFRESH)) {
         int rc = pcmk_ok;
         xmlNode *fragment = do_lrm_query(TRUE);
 
         crm_info("Forcing a local LRM refresh");
 
         fsa_cib_update(XML_CIB_TAG_STATUS, fragment, cib_quorum_override, rc, user_name);
         free_xml(fragment);
 
     } else if (safe_str_eq(crm_op, CRM_OP_LRM_QUERY)) {
         xmlNode *data = do_lrm_query(FALSE);
         xmlNode *reply = create_reply(input->msg, data);
 
         if (relay_message(reply, TRUE) == FALSE) {
             crm_err("Unable to route reply");
             crm_log_xml_err(reply, "reply");
         }
         free_xml(reply);
         free_xml(data);
 
     } else if (safe_str_eq(operation, CRM_OP_PROBED)) {
         update_attrd(NULL, CRM_OP_PROBED, XML_BOOLEAN_TRUE, user_name);
 
     } else if (safe_str_eq(crm_op, CRM_OP_REPROBE)) {
         GHashTableIter gIter;
         rsc_history_t *entry = NULL;
 
         crm_notice("Forcing the status of all resources to be redetected");
 
         g_hash_table_iter_init(&gIter, resource_history);
         while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
             delete_resource(entry->id, &entry->rsc, &gIter, from_sys, from_host, user_name, NULL);
         }
 
         /* Now delete the copy in the CIB */
         erase_status_tag(fsa_our_uname, XML_CIB_TAG_LRM, cib_scope_local);
 
         /* And finally, _delete_ the value in attrd
          * Setting it to FALSE results in the PE sending us back here again
          */
         update_attrd(NULL, CRM_OP_PROBED, NULL, user_name);
 
     } else if (operation != NULL) {
         lrmd_rsc_info_t *rsc = NULL;
         xmlNode *params = NULL;
         xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);
 
         CRM_CHECK(xml_rsc != NULL, return);
 
         /* only the first 16 chars are used by the LRM */
         params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE);
 
         if (safe_str_eq(operation, CRMD_ACTION_DELETE)) {
             create_rsc = FALSE;
         }
 
         rsc = get_lrm_resource(xml_rsc, input->xml, create_rsc);
 
         if (rsc == NULL && create_rsc) {
             crm_err("Invalid resource definition");
             crm_log_xml_warn(input->msg, "bad input");
 
         } else if (rsc == NULL) {
             lrmd_event_data_t *op = NULL;
 
             crm_notice("Not creating resource for a %s event: %s", operation, ID(input->xml));
             delete_rsc_entry(input, ID(xml_rsc), NULL, pcmk_ok, user_name);
 
             op = construct_op(input->xml, ID(xml_rsc), operation);
             op->op_status = PCMK_LRM_OP_DONE;
             op->rc = PCMK_EXECRA_OK;
             CRM_ASSERT(op != NULL);
             send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
             lrmd_free_event(op);
 
         } else if (safe_str_eq(operation, CRMD_ACTION_CANCEL)) {
             lrmd_event_data_t *op = NULL;
             char *op_key = NULL;
             char *meta_key = NULL;
             int call = 0;
             const char *call_id = NULL;
             const char *op_task = NULL;
             const char *op_interval = NULL;
 
             CRM_CHECK(params != NULL, crm_log_xml_warn(input->xml, "Bad command");
                       return);
 
             meta_key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
             op_interval = crm_element_value(params, meta_key);
             free(meta_key);
 
             meta_key = crm_meta_name(XML_LRM_ATTR_TASK);
             op_task = crm_element_value(params, meta_key);
             free(meta_key);
 
             meta_key = crm_meta_name(XML_LRM_ATTR_CALLID);
             call_id = crm_element_value(params, meta_key);
             free(meta_key);
 
             CRM_CHECK(op_task != NULL, crm_log_xml_warn(input->xml, "Bad command");
                       return);
             CRM_CHECK(op_interval != NULL, crm_log_xml_warn(input->xml, "Bad command");
                       return);
 
             op = construct_op(input->xml, rsc->id, op_task);
             CRM_ASSERT(op != NULL);
             op_key = generate_op_key(rsc->id, op_task, crm_parse_int(op_interval, "0"));
 
             crm_debug("PE requested op %s (call=%s) be cancelled",
                       op_key, call_id ? call_id : "NA");
             call = crm_parse_int(call_id, "0");
             if (call == 0) {
                 /* the normal case when the PE cancels a recurring op */
                 done = cancel_op_key(rsc, op_key, TRUE);
 
             } else {
                 /* the normal case when the PE cancels an orphan op */
                 done = cancel_op(rsc->id, NULL, call, TRUE);
             }
 
             if (done == FALSE) {
                 crm_debug("Nothing known about operation %d for %s", call, op_key);
                 delete_op_entry(NULL, rsc->id, op_key, call);
 
                 /* needed?? surely not otherwise the cancel_op_(_key) wouldn't
                  * have failed in the first place
                  */
                 g_hash_table_remove(pending_ops, op_key);
             }
 
             op->rc = PCMK_EXECRA_OK;
             op->op_status = PCMK_LRM_OP_DONE;
             send_direct_ack(from_host, from_sys, rsc, op, rsc->id);
 
             free(op_key);
             lrmd_free_event(op);
 
         } else if (safe_str_eq(operation, CRMD_ACTION_DELETE)) {
             int cib_rc = pcmk_ok;
 
             CRM_ASSERT(rsc != NULL);
 
             cib_rc = delete_rsc_status(rsc->id, cib_dryrun | cib_sync_call, user_name);
             if (cib_rc != pcmk_ok) {
                 lrmd_event_data_t *op = NULL;
 
                 crm_err
                     ("Attempt of deleting resource status '%s' from CIB for %s (user=%s) on %s failed: (rc=%d) %s",
                      rsc->id, from_sys, user_name ? user_name : "unknown", from_host, cib_rc,
                      pcmk_strerror(cib_rc));
 
                 op = construct_op(input->xml, rsc->id, operation);
                 op->op_status = PCMK_LRM_OP_ERROR;
 
                 if (cib_rc == -EACCES) {
                     op->rc = PCMK_EXECRA_INSUFFICIENT_PRIV;
                 } else {
                     op->rc = PCMK_EXECRA_UNKNOWN_ERROR;
                 }
                 send_direct_ack(from_host, from_sys, NULL, op, rsc->id);
                 lrmd_free_event(op);
                 return;
             }
 
             delete_resource(rsc->id, rsc, NULL, from_sys, from_host, user_name, input);
 
         } else if (rsc != NULL) {
             do_lrm_rsc_op(rsc, operation, input->xml, input->msg);
         }
 
         lrmd_free_rsc_info(rsc);
 
     } else {
         crm_err("Operation was neither a lrm_query, nor a rsc op.  %s", crm_str(crm_op));
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
 }
 
 static lrmd_event_data_t *
 construct_op(xmlNode * rsc_op, const char *rsc_id, const char *operation)
 {
     lrmd_event_data_t *op = NULL;
     const char *op_delay = NULL;
     const char *op_timeout = NULL;
     const char *op_interval = NULL;
     GHashTable *params = NULL;
 
     const char *transition = NULL;
 
     CRM_LOG_ASSERT(rsc_id != NULL);
 
     op = calloc(1, sizeof(lrmd_event_data_t));
     op->type = lrmd_event_exec_complete;
     op->op_type = strdup(operation);
     op->op_status = PCMK_LRM_OP_PENDING;
     op->rc = -1;
     op->rsc_id = strdup(rsc_id);
     op->interval = 0;
     op->timeout = 0;
     op->start_delay = 0;
 
     if (rsc_op == NULL) {
         CRM_LOG_ASSERT(safe_str_eq(CRMD_ACTION_STOP, operation));
         op->user_data = NULL;
         /* the stop_all_resources() case
          * by definition there is no DC (or they'd be shutting
          *   us down).
          * So we should put our version here.
          */
         op->params = g_hash_table_new_full(crm_str_hash, g_str_equal,
                                            g_hash_destroy_str, g_hash_destroy_str);
 
         g_hash_table_insert(op->params,
                             strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));
 
         crm_trace("Constructed %s op for %s", operation, rsc_id);
         return op;
     }
 
     params = xml2list(rsc_op);
     g_hash_table_remove(params, CRM_META "_op_target_rc");
 
     op_delay = crm_meta_value(params, XML_OP_ATTR_START_DELAY);
     op_timeout = crm_meta_value(params, XML_ATTR_TIMEOUT);
     op_interval = crm_meta_value(params, XML_LRM_ATTR_INTERVAL);
 
     op->interval = crm_parse_int(op_interval, "0");
     op->timeout = crm_parse_int(op_timeout, "0");
     op->start_delay = crm_parse_int(op_delay, "0");
 
     if (safe_str_neq(operation, RSC_STOP)) {
         op->params = params;
 
     } else {
         rsc_history_t *entry = g_hash_table_lookup(resource_history, rsc_id);
 
         /* If we do not have stop parameters cached, use
          * whatever we are given */
         if (!entry || !entry->stop_params) {
             op->params = params;
         } else {
             /* Copy the cached parameter list so that we stop the resource
              * with the old attributes, not the new ones */
             op->params = g_hash_table_new_full(crm_str_hash, g_str_equal,
                 g_hash_destroy_str, g_hash_destroy_str);
 
             g_hash_table_foreach(params, copy_meta_keys, op->params);
             g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
             g_hash_table_destroy(params);
             params = NULL;
         }
     }
 
     /* sanity */
     if (op->interval < 0) {
         op->interval = 0;
     }
     if (op->timeout <= 0) {
         op->timeout = op->interval;
     }
     if (op->start_delay < 0) {
         op->start_delay = 0;
     }
 
     transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY);
     CRM_CHECK(transition != NULL, return op);
 
     op->user_data = strdup(transition);
 
     if (op->interval != 0) {
         if (safe_str_eq(operation, CRMD_ACTION_START)
             || safe_str_eq(operation, CRMD_ACTION_STOP)) {
             crm_err("Start and Stop actions cannot have an interval: %d", op->interval);
             op->interval = 0;
         }
     }
 
     crm_trace("Constructed %s op for %s: interval=%d", operation, rsc_id, op->interval);
 
     return op;
 }
 
 void
 send_direct_ack(const char *to_host, const char *to_sys,
                 lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id)
 {
     xmlNode *reply = NULL;
     xmlNode *update, *iter;
     xmlNode *fragment;
 
     CRM_CHECK(op != NULL, return);
     if (op->rsc_id == NULL) {
         CRM_LOG_ASSERT(rsc_id != NULL);
         op->rsc_id = strdup(rsc_id);
     }
     if (to_sys == NULL) {
         to_sys = CRM_SYSTEM_TENGINE;
     }
-    update = create_node_state(fsa_our_uname, NULL, NULL, NULL, NULL, NULL, FALSE, __FUNCTION__);
+    update = create_node_state(fsa_our_uname, NULL, NULL, NULL, NULL, FALSE, __FUNCTION__);
 
     iter = create_xml_node(update, XML_CIB_TAG_LRM);
     crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
 
     crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);
 
     build_operation_update(iter, rsc, op, __FUNCTION__);
     fragment = create_cib_fragment(update, XML_CIB_TAG_STATUS);
 
     reply = create_request(CRM_OP_INVOKE_LRM, fragment, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);
 
     crm_log_xml_trace(update, "ACK Update");
 
     crm_debug("ACK'ing resource op %s_%s_%d from %s: %s",
              op->rsc_id, op->op_type, op->interval, op->user_data,
              crm_element_value(reply, XML_ATTR_REFERENCE));
 
     if (relay_message(reply, TRUE) == FALSE) {
         crm_log_xml_err(reply, "Unable to route reply");
     }
 
     free_xml(fragment);
     free_xml(update);
     free_xml(reply);
 }
 
 static gboolean
 stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
 {
     lrmd_rsc_info_t *rsc = user_data;
     struct recurring_op_s *op = (struct recurring_op_s *)value;
 
     if (op->interval != 0 && safe_str_eq(op->rsc_id, rsc->id)) {
         if (cancel_op(rsc->id, key, op->call_id, FALSE) == FALSE) {
             return TRUE;
         }
     }
 
     return FALSE;
 }
 
 static gboolean
 stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
 {
     gboolean remove = FALSE;
     struct recurring_op_s *op = (struct recurring_op_s *)value;
 
     if (op->interval != 0) {
         remove = cancel_op(op->rsc_id, key, op->call_id, FALSE);
     }
 
     return remove;
 }
 
 void
 do_lrm_rsc_op(lrmd_rsc_info_t * rsc, const char *operation, xmlNode * msg, xmlNode * request)
 {
     int call_id = 0;
     char *op_id = NULL;
     lrmd_event_data_t *op = NULL;
     lrmd_key_value_t *params = NULL;
     fsa_data_t *msg_data = NULL;
     const char *transition = NULL;
 
     CRM_CHECK(rsc != NULL, return);
 
     if (msg != NULL) {
         transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
         if (transition == NULL) {
             crm_log_xml_err(msg, "Missing transition number");
         }
     }
 
     op = construct_op(msg, rsc->id, operation);
 
     /* stop the monitor before stopping the resource */
     if (crm_str_eq(operation, CRMD_ACTION_STOP, TRUE)
         || crm_str_eq(operation, CRMD_ACTION_DEMOTE, TRUE)
         || crm_str_eq(operation, CRMD_ACTION_PROMOTE, TRUE)
         || crm_str_eq(operation, CRMD_ACTION_MIGRATE, TRUE)) {
         g_hash_table_foreach_remove(pending_ops, stop_recurring_action_by_rsc, rsc);
     }
 
     /* now do the op */
     crm_debug("Performing key=%s op=%s_%s_%d", transition, rsc->id, operation, op->interval);
 
     if (fsa_state != S_NOT_DC && fsa_state != S_POLICY_ENGINE && fsa_state != S_TRANSITION_ENGINE) {
         if (safe_str_neq(operation, "fail")
             && safe_str_neq(operation, CRMD_ACTION_STOP)) {
             crm_info("Discarding attempt to perform action %s on %s"
                      " in state %s", operation, rsc->id, fsa_state2string(fsa_state));
             op->rc = 99;
             op->op_status = PCMK_LRM_OP_ERROR;
             send_direct_ack(NULL, NULL, rsc, op, rsc->id);
             lrmd_free_event(op);
             free(op_id);
             return;
         }
     }
 
     op_id = generate_op_key(rsc->id, op->op_type, op->interval);
 
     if (op->interval > 0) {
         /* cancel it so we can then restart it without conflict */
         cancel_op_key(rsc, op_id, FALSE);
     }
 
     if (op->params) {
         char *key = NULL;
         char *value = NULL;
         GHashTableIter iter;
 
         g_hash_table_iter_init(&iter, op->params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             params = lrmd_key_value_add(params, key, value);
         }
     }
 
     call_id = fsa_lrm_conn->cmds->exec(fsa_lrm_conn,
         rsc->id,
         op->op_type,
         op->user_data,
         op->interval,
         op->timeout,
         op->start_delay,
         0,
         params);
 
     if (call_id <= 0) {
         crm_err("Operation %s on %s failed: %d", operation, rsc->id, call_id);
         register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
 
     } else {
         /* record all operations so we can wait
          * for them to complete during shutdown
          */
         char *call_id_s = make_stop_id(rsc->id, call_id);
         struct recurring_op_s *pending = NULL;
 
         pending = calloc(1, sizeof(struct recurring_op_s));
         crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);
 
         pending->call_id = call_id;
         pending->interval = op->interval;
         pending->op_type = strdup(operation);
         pending->op_key = strdup(op_id);
         pending->rsc_id = strdup(rsc->id);
         pending->last_rc = -1; /* All rc are positive, -1 indicates the last rc has not been set. */
         pending->last_op_status = -2;
         g_hash_table_replace(pending_ops, call_id_s, pending);
 
         if (op->interval > 0 && op->start_delay > START_DELAY_THRESHOLD) {
             char *uuid = NULL;
             int dummy = 0, target_rc = 0;
 
             crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);
 
             decode_transition_key(op->user_data, &uuid, &dummy, &dummy, &target_rc);
             free(uuid);
 
             op->rc = target_rc;
             op->op_status = PCMK_LRM_OP_DONE;
             send_direct_ack(NULL, NULL, rsc, op, rsc->id);
         }
     }
 
     free(op_id);
     lrmd_free_event(op);
     return;
 }
 
 static void
 cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     switch (rc) {
         case pcmk_ok:
         case -pcmk_err_diff_failed:
         case -pcmk_err_diff_resync:
             crm_trace("Resource update %d complete: rc=%d", call_id, rc);
             break;
         default:
             crm_warn("Resource update %d failed: (rc=%d) %s", call_id, rc, pcmk_strerror(rc));
     }
 }
 
 static int
 do_update_resource(lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
 {
 /*
   <status>
   <nodes_status id=uname>
   <lrm>
   <lrm_resources>
   <lrm_resource id=...>
   </...>
 */
     int rc = pcmk_ok;
     xmlNode *update, *iter = NULL;
     int call_opt = cib_quorum_override;
 
     CRM_CHECK(op != NULL, return 0);
 
     if (fsa_state == S_ELECTION || fsa_state == S_PENDING) {
         crm_info("Sending update to local CIB in state: %s", fsa_state2string(fsa_state));
         call_opt |= cib_scope_local;
     }
 
     iter = create_xml_node(iter, XML_CIB_TAG_STATUS);
     update = iter;
     iter = create_xml_node(iter, XML_CIB_TAG_STATE);
 
     set_uuid(iter, XML_ATTR_UUID, fsa_our_uname);
     crm_xml_add(iter, XML_ATTR_UNAME, fsa_our_uname);
     crm_xml_add(iter, XML_ATTR_ORIGIN, __FUNCTION__);
 
     iter = create_xml_node(iter, XML_CIB_TAG_LRM);
     crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);
 
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
     crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);
 
     build_operation_update(iter, rsc, op, __FUNCTION__);
 
     if (rsc) {
         crm_xml_add(iter, XML_ATTR_TYPE, rsc->type);
         crm_xml_add(iter, XML_AGENT_ATTR_CLASS, rsc->class);
         crm_xml_add(iter, XML_AGENT_ATTR_PROVIDER, rsc->provider);
 
         CRM_CHECK(rsc->type != NULL, crm_err("Resource %s has no value for type", op->rsc_id));
         CRM_CHECK(rsc->class != NULL, crm_err("Resource %s has no value for class", op->rsc_id));
 
     } else {
         crm_warn("Resource %s no longer exists in the lrmd", op->rsc_id);
         goto cleanup;
     }
 
     /* make it an asyncronous call and be done with it
      *
      * Best case:
      *   the resource state will be discovered during
      *   the next signup or election.
      *
      * Bad case:
      *   we are shutting down and there is no DC at the time,
      *   but then why were we shutting down then anyway?
      *   (probably because of an internal error)
      *
      * Worst case:
      *   we get shot for having resources "running" when the really weren't
      *
      * the alternative however means blocking here for too long, which
      * isnt acceptable
      */
     fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, rc, NULL);
 
     /* the return code is a call number, not an error code */
     crm_trace("Sent resource state update message: %d", rc);
     fsa_cib_conn->cmds->register_callback(fsa_cib_conn, rc, 60, FALSE, NULL, "cib_rsc_callback",
                                           cib_rsc_callback);
 
   cleanup:
     free_xml(update);
     return rc;
 }
 
 void
 do_lrm_event(long long action,
              enum crmd_fsa_cause cause,
              enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
 {
     CRM_CHECK(FALSE, return);
 }
 
 gboolean
 process_lrm_event(lrmd_event_data_t * op)
 {
     char *op_id = NULL;
     char *op_key = NULL;
 
     int update_id = 0;
     int log_level = LOG_ERR;
     gboolean removed = FALSE;
     lrmd_rsc_info_t *rsc = NULL;
 
     struct recurring_op_s *pending = NULL;
 
     CRM_CHECK(op != NULL, return FALSE);
 
     if (op->type == lrmd_event_disconnect) {
         lrm_connection_destroy();
         return TRUE;
     } else if (op->type != lrmd_event_exec_complete) {
         return TRUE;
     }
 
     CRM_CHECK(op->rsc_id != NULL, return FALSE);
     op_id = make_stop_id(op->rsc_id, op->call_id);
     pending = g_hash_table_lookup(pending_ops, op_id);
 
     /* ignore recurring ops that have not changed. */
     if (op->interval &&
             pending &&
             (pending->last_rc == op->rc) &&
             (pending->last_op_status == op->op_status)) {
         free(op_id);
         return TRUE;
     }
 
     if (pending) {
         pending->last_rc = op->rc;
         pending->last_op_status = op->op_status;
     }
 
     op_key = generate_op_key(op->rsc_id, op->op_type, op->interval);
     rsc = fsa_lrm_conn->cmds->get_rsc_info(fsa_lrm_conn, op->rsc_id, 0);
 
     switch (op->op_status) {
         case PCMK_LRM_OP_ERROR:
         case PCMK_LRM_OP_PENDING:
         case PCMK_LRM_OP_NOTSUPPORTED:
             break;
         case PCMK_LRM_OP_CANCELLED:
             log_level = LOG_INFO;
             break;
         case PCMK_LRM_OP_DONE:
             log_level = LOG_NOTICE;
             break;
         case PCMK_LRM_OP_TIMEOUT:
             log_level = LOG_DEBUG_3;
             crm_err("LRM operation %s (%d) %s (timeout=%dms)",
                     op_key, op->call_id, services_lrm_status_str(op->op_status), op->timeout);
             break;
         default:
             crm_err("Mapping unknown status (%d) to ERROR", op->op_status);
             op->op_status = PCMK_LRM_OP_ERROR;
     }
 
     if (op->op_status == PCMK_LRM_OP_ERROR
         && (op->rc == PCMK_EXECRA_RUNNING_MASTER || op->rc == PCMK_EXECRA_NOT_RUNNING)) {
         /* Leave it up to the TE/PE to decide if this is an error */
         op->op_status = PCMK_LRM_OP_DONE;
         log_level = LOG_INFO;
     }
 
     if (op->op_status != PCMK_LRM_OP_CANCELLED) {
         if (safe_str_eq(op->op_type, RSC_NOTIFY)) {
             /* Keep notify ops out of the CIB */
             send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);
         } else {
             update_id = do_update_resource(rsc, op);
         }
     } else if (op->interval == 0) {
         /* This will occur when "crm resource cleanup" is called while actions are in-flight */
         crm_err("Op %s (call=%d): Cancelled", op_key, op->call_id);
         send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);
 
     } else if (pending == NULL) {
         /* Operations that are cancelled may safely be removed
          * from the pending op list before the lrmd completion event
          * is received. Only report non-cancelled ops here. */
         if (op->op_status != PCMK_LRM_OP_CANCELLED) {
             crm_err("Op %s (call=%d): No 'pending' entry", op_key, op->call_id);
         }
     } else if (op->user_data == NULL) {
         crm_err("Op %s (call=%d): No user data", op_key, op->call_id);
 
     } else if (pending->remove) {
         delete_op_entry(op, op->rsc_id, op_key, op->call_id);
 
     } else {
         /* Before a stop is called, no need to direct ack */
         crm_trace("Op %s (call=%d): no delete event required", op_key, op->call_id);
     }
 
     if ((op->interval == 0) && g_hash_table_remove(pending_ops, op_id)) {
         removed = TRUE;
         crm_trace("Op %s (call=%d, stop-id=%s): Confirmed", op_key, op->call_id, op_id);
     }
 
     if (op->op_status == PCMK_LRM_OP_DONE) {
         do_crm_log(log_level,
                    "LRM operation %s (call=%d, rc=%d, cib-update=%d, confirmed=%s) %s",
                    op_key, op->call_id, op->rc, update_id, removed ? "true" : "false",
                    lrmd_event_rc2str(op->rc));
     } else {
         do_crm_log(log_level,
                    "LRM operation %s (call=%d, status=%d, cib-update=%d, confirmed=%s) %s",
                    op_key, op->call_id, op->op_status, update_id, removed ? "true" : "false",
                    services_lrm_status_str(op->op_status));
     }
 
     if (op->rc != 0 && op->output != NULL) {
         crm_info("Result: %s", op->output);
     } else if (op->output != NULL) {
         crm_debug("Result: %s", op->output);
     }
 
     if (op->rsc_deleted) {
         crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
         delete_rsc_entry(NULL, op->rsc_id, NULL, pcmk_ok, NULL);
     }
 
     /* If a shutdown was escalated while operations were pending, 
      * then the FSA will be stalled right now... allow it to continue
      */
     mainloop_set_trigger(fsa_source);
     update_history_cache(rsc, op);
 
     lrmd_free_rsc_info(rsc);
     free(op_key);
     free(op_id);
 
     return TRUE;
 }
diff --git a/crmd/membership.c b/crmd/membership.c
index b569f9e825..26c40a8c0b 100644
--- a/crmd/membership.c
+++ b/crmd/membership.c
@@ -1,254 +1,253 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 /* put these first so that uuid_t is defined without conflicts */
 #include <crm_internal.h>
 
 #include <string.h>
 
 #include <crm/crm.h>
 #include <crm/cib/internal.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/cluster/internal.h>
 #include <crmd_messages.h>
 #include <crmd_fsa.h>
 #include <fsa_proto.h>
 #include <crmd_callbacks.h>
 #include <tengine.h>
 #include <membership.h>
 
 gboolean membership_flux_hack = FALSE;
 void post_cache_update(int instance);
 
 void ghash_update_cib_node(gpointer key, gpointer value, gpointer user_data);
 
 int last_peer_update = 0;
 
 extern GHashTable *voted;
 
 struct update_data_s {
     const char *state;
     const char *caller;
     xmlNode *updates;
     gboolean overwrite_join;
 };
 
 extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
 
 static void
 check_dead_member(const char *uname, GHashTable * members)
 {
     CRM_CHECK(uname != NULL, return);
     if (members != NULL && g_hash_table_lookup(members, uname) != NULL) {
         crm_err("%s didnt really leave the membership!", uname);
         return;
     }
     erase_node_from_join(uname);
     if (voted != NULL) {
         g_hash_table_remove(voted, uname);
     }
 
     if (safe_str_eq(fsa_our_uname, uname)) {
         crm_err("We're not part of the cluster anymore");
         register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
 
     } else if (AM_I_DC == FALSE && safe_str_eq(uname, fsa_our_dc)) {
         crm_warn("Our DC node (%s) left the cluster", uname);
         register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
 
     } else if (fsa_state == S_INTEGRATION || fsa_state == S_FINALIZE_JOIN) {
         check_join_state(fsa_state, __FUNCTION__);
     }
 }
 
 
 static void
 reap_dead_nodes(gpointer key, gpointer value, gpointer user_data)
 {
     crm_node_t *node = value;
 
     if (crm_is_peer_active(node) == FALSE) {
         check_dead_member(node->uname, NULL);
         fail_incompletable_actions(transition_graph, node->uuid);
     }
 }
 
 gboolean ever_had_quorum = FALSE;
 
 void
 post_cache_update(int instance)
 {
     xmlNode *no_op = NULL;
 
     crm_peer_seq = instance;
     crm_debug("Updated cache after membership event %d.", instance);
 
     g_hash_table_foreach(crm_peer_cache, reap_dead_nodes, NULL);
     set_bit(fsa_input_register, R_MEMBERSHIP);
 
     if (AM_I_DC) {
         populate_cib_nodes(FALSE);
         do_update_cib_nodes(FALSE, __FUNCTION__);
     }
 
     /*
      * If we lost nodes, we should re-check the election status
      * Safe to call outside of an election
      */
     register_fsa_action(A_ELECTION_CHECK);
 
     /* Membership changed, remind everyone we're here.
      * This will aid detection of duplicate DCs
      */
     no_op = create_request(CRM_OP_NOOP, NULL, NULL, CRM_SYSTEM_CRMD,
                            AM_I_DC ? CRM_SYSTEM_DC : CRM_SYSTEM_CRMD, NULL);
     send_cluster_message(NULL, crm_msg_crmd, no_op, FALSE);
     free_xml(no_op);
 }
 
 static void
 crmd_node_update_complete(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     fsa_data_t *msg_data = NULL;
 
     last_peer_update = 0;
 
     if (rc == pcmk_ok) {
         crm_trace("Node update %d complete", call_id);
 
     } else {
         crm_err("Node update %d failed", call_id);
         crm_log_xml_debug(msg, "failed");
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
 }
 
 void
 ghash_update_cib_node(gpointer key, gpointer value, gpointer user_data)
 {
     xmlNode *tmp1 = NULL;
     const char *join = NULL;
     crm_node_t *node = value;
     struct update_data_s *data = (struct update_data_s *)user_data;
-    enum crm_proc_flag messaging = crm_proc_plugin | crm_proc_heartbeat | crm_proc_cpg;
 
     data->state = XML_BOOLEAN_NO;
     if (safe_str_eq(node->state, CRM_NODE_ACTIVE)) {
         data->state = XML_BOOLEAN_YES;
     }
 
     crm_debug("Updating %s: %s (overwrite=%s) hash_size=%d",
               node->uname, data->state, data->overwrite_join ? "true" : "false",
               g_hash_table_size(confirmed_nodes));
 
     if (data->overwrite_join) {
         if ((node->processes & proc_flags) == FALSE) {
             join = CRMD_JOINSTATE_DOWN;
 
         } else {
             const char *peer_member = g_hash_table_lookup(confirmed_nodes, node->uname);
 
             if (peer_member != NULL) {
                 join = CRMD_JOINSTATE_MEMBER;
             } else {
                 join = CRMD_JOINSTATE_PENDING;
             }
         }
     }
 
     tmp1 =
-        create_node_state(node->uname, (node->processes & messaging) ? ACTIVESTATUS : DEADSTATUS,
+        create_node_state(node->uname,
                           data->state,
                           (node->processes & proc_flags) ? ONLINESTATUS : OFFLINESTATUS, join,
                           NULL, FALSE, data->caller);
 
     add_node_copy(data->updates, tmp1);
     free_xml(tmp1);
 }
 
 void
 do_update_cib_nodes(gboolean overwrite, const char *caller)
 {
     int call_id = 0;
     int call_options = cib_scope_local | cib_quorum_override;
     struct update_data_s update_data;
     xmlNode *fragment = NULL;
 
     if (crm_peer_cache == NULL) {
         /* We got a replace notification before being connected to
          *   the CCM.
          * So there is no need to update the local CIB with our values
          *   - since we have none.
          */
         return;
 
     } else if (AM_I_DC == FALSE) {
         return;
     }
 
     fragment = create_xml_node(NULL, XML_CIB_TAG_STATUS);
 
     update_data.caller = caller;
     update_data.updates = fragment;
     update_data.overwrite_join = overwrite;
 
     g_hash_table_foreach(crm_peer_cache, ghash_update_cib_node, &update_data);
 
     fsa_cib_update(XML_CIB_TAG_STATUS, fragment, call_options, call_id, NULL);
     add_cib_op_callback(fsa_cib_conn, call_id, FALSE, NULL, crmd_node_update_complete);
     last_peer_update = call_id;
 
     free_xml(fragment);
 }
 
 static void
 cib_quorum_update_complete(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     fsa_data_t *msg_data = NULL;
 
     if (rc == pcmk_ok) {
         crm_trace("Quorum update %d complete", call_id);
 
     } else {
         crm_err("Quorum update %d failed", call_id);
         crm_log_xml_debug(msg, "failed");
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
 }
 
 void
 crm_update_quorum(gboolean quorum, gboolean force_update)
 {
     ever_had_quorum |= quorum;
     if (AM_I_DC && (force_update || fsa_has_quorum != quorum)) {
         int call_id = 0;
         xmlNode *update = NULL;
         int call_options = cib_scope_local | cib_quorum_override;
 
         update = create_xml_node(NULL, XML_TAG_CIB);
         crm_xml_add_int(update, XML_ATTR_HAVE_QUORUM, quorum);
         set_uuid(update, XML_ATTR_DC_UUID, fsa_our_uname);
 
         fsa_cib_update(XML_TAG_CIB, update, call_options, call_id, NULL);
         crm_debug("Updating quorum status to %s (call=%d)", quorum ? "true" : "false", call_id);
         add_cib_op_callback(fsa_cib_conn, call_id, FALSE, NULL, cib_quorum_update_complete);
         free_xml(update);
     }
     fsa_has_quorum = quorum;
 }
diff --git a/crmd/messages.c b/crmd/messages.c
index 5181b9c887..035cd5ba4b 100644
--- a/crmd/messages.c
+++ b/crmd/messages.c
@@ -1,984 +1,984 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <string.h>
 #include <time.h>
 #include <crmd_fsa.h>
 
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crm/cluster/internal.h>
 #include <crm/cib.h>
 
 #include <crmd.h>
 #include <crmd_messages.h>
 #include <crmd_lrm.h>
 
 GListPtr fsa_message_queue = NULL;
 extern void crm_shutdown(int nsig);
 
 void handle_response(xmlNode * stored_msg);
 enum crmd_fsa_input handle_request(xmlNode * stored_msg);
 enum crmd_fsa_input handle_shutdown_request(xmlNode * stored_msg);
 
 #ifdef MSG_LOG
 #  define ROUTER_RESULT(x)	crm_trace("Router result: %s", x);	\
     crm_log_xml_trace(msg, "router.log");
 #else
 #  define ROUTER_RESULT(x)	crm_trace("Router result: %s", x)
 #endif
 /* debug only, can wrap all it likes */
 int last_data_id = 0;
 
 void
 register_fsa_error_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
                        fsa_data_t * cur_data, void *new_data, const char *raised_from)
 {
     /* save the current actions if any */
     if (fsa_actions != A_NOTHING) {
         register_fsa_input_adv(cur_data ? cur_data->fsa_cause : C_FSA_INTERNAL,
                                I_NULL, cur_data ? cur_data->data : NULL,
                                fsa_actions, TRUE, __FUNCTION__);
     }
 
     /* reset the action list */
     fsa_actions = A_NOTHING;
 
     /* register the error */
     register_fsa_input_adv(cause, input, new_data, A_NOTHING, TRUE, raised_from);
 }
 
 int
 register_fsa_input_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
                        void *data, long long with_actions,
                        gboolean prepend, const char *raised_from)
 {
     unsigned old_len = g_list_length(fsa_message_queue);
     fsa_data_t *fsa_data = NULL;
 
     last_data_id++;
     CRM_CHECK(raised_from != NULL, raised_from = "<unknown>");
 
     crm_trace("%s %s FSA input %d (%s) (cause=%s) %s data",
                 raised_from, prepend ? "prepended" : "appended", last_data_id,
                 fsa_input2string(input), fsa_cause2string(cause), data ? "with" : "without");
 
     if (input == I_WAIT_FOR_EVENT) {
         do_fsa_stall = TRUE;
         crm_debug("Stalling the FSA pending further input: cause=%s", fsa_cause2string(cause));
         if (old_len > 0) {
             crm_warn("%s stalled the FSA with pending inputs", raised_from);
             fsa_dump_queue(LOG_DEBUG);
         }
         if (data == NULL) {
             set_bit(fsa_actions, with_actions);
             with_actions = A_NOTHING;
             return 0;
         }
         crm_err("%s stalled the FSA with data - this may be broken", raised_from);
     }
 
     if (input == I_NULL && with_actions == A_NOTHING /* && data == NULL */ ) {
         /* no point doing anything */
         crm_err("Cannot add entry to queue: no input and no action");
         return 0;
     }
 
     fsa_data = calloc(1, sizeof(fsa_data_t));
     fsa_data->id = last_data_id;
     fsa_data->fsa_input = input;
     fsa_data->fsa_cause = cause;
     fsa_data->origin = raised_from;
     fsa_data->data = NULL;
     fsa_data->data_type = fsa_dt_none;
     fsa_data->actions = with_actions;
 
     if (with_actions != A_NOTHING) {
         crm_trace("Adding actions %.16llx to input", with_actions);
     }
 
     if (data != NULL) {
         switch (cause) {
             case C_FSA_INTERNAL:
             case C_CRMD_STATUS_CALLBACK:
             case C_IPC_MESSAGE:
             case C_HA_MESSAGE:
                 crm_trace("Copying %s data from %s as a HA msg",
                             fsa_cause2string(cause), raised_from);
                 CRM_CHECK(((ha_msg_input_t *) data)->msg != NULL,
                           crm_err("Bogus data from %s", raised_from));
                 fsa_data->data = copy_ha_msg_input(data);
                 fsa_data->data_type = fsa_dt_ha_msg;
                 break;
 
             case C_LRM_OP_CALLBACK:
                 crm_trace("Copying %s data from %s as lrmd_event_data_t",
                             fsa_cause2string(cause), raised_from);
                 fsa_data->data = lrmd_copy_event((lrmd_event_data_t *) data);
                 fsa_data->data_type = fsa_dt_lrm;
                 break;
 
             case C_CCM_CALLBACK:
             case C_SUBSYSTEM_CONNECT:
             case C_LRM_MONITOR_CALLBACK:
             case C_TIMER_POPPED:
             case C_SHUTDOWN:
             case C_HEARTBEAT_FAILED:
             case C_HA_DISCONNECT:
             case C_ILLEGAL:
             case C_UNKNOWN:
             case C_STARTUP:
                 crm_err("Copying %s data (from %s)"
                         " not yet implemented", fsa_cause2string(cause), raised_from);
                 exit(1);
                 break;
         }
         crm_trace("%s data copied", fsa_cause2string(fsa_data->fsa_cause));
     }
 
     /* make sure to free it properly later */
     if (prepend) {
         crm_trace("Prepending input");
         fsa_message_queue = g_list_prepend(fsa_message_queue, fsa_data);
     } else {
         fsa_message_queue = g_list_append(fsa_message_queue, fsa_data);
     }
 
     crm_trace("Queue len: %d", g_list_length(fsa_message_queue));
 
     fsa_dump_queue(LOG_DEBUG_2);
 
     if (old_len == g_list_length(fsa_message_queue)) {
         crm_err("Couldnt add message to the queue");
     }
 
     if (fsa_source) {
         crm_trace("Triggering FSA: %s", __FUNCTION__);
         mainloop_set_trigger(fsa_source);
     }
     return last_data_id;
 }
 
 void
 fsa_dump_queue(int log_level)
 {
     int offset = 0;
     GListPtr lpc = NULL;
 
     for (lpc = fsa_message_queue; lpc != NULL; lpc = lpc->next) {
         fsa_data_t *data = (fsa_data_t *) lpc->data;
 
         do_crm_log_unlikely(log_level,
                    "queue[%d(%d)]: input %s raised by %s()\t(cause=%s)",
                    offset++, data->id, fsa_input2string(data->fsa_input),
                    data->origin, fsa_cause2string(data->fsa_cause));
     }
 }
 
 ha_msg_input_t *
 copy_ha_msg_input(ha_msg_input_t * orig)
 {
     ha_msg_input_t *copy = NULL;
     xmlNodePtr data = NULL;
 
     if (orig != NULL) {
         crm_trace("Copy msg");
         data = copy_xml(orig->msg);
 
     } else {
         crm_trace("No message to copy");
     }
     copy = new_ha_msg_input(data);
     if (orig && orig->msg != NULL) {
         CRM_CHECK(copy->msg != NULL, crm_err("copy failed"));
     }
     return copy;
 }
 
 void
 delete_fsa_input(fsa_data_t * fsa_data)
 {
     lrmd_event_data_t *op = NULL;
     xmlNode *foo = NULL;
 
     if (fsa_data == NULL) {
         return;
     }
     crm_trace("About to free %s data", fsa_cause2string(fsa_data->fsa_cause));
 
     if (fsa_data->data != NULL) {
         switch (fsa_data->data_type) {
             case fsa_dt_ha_msg:
                 delete_ha_msg_input(fsa_data->data);
                 break;
 
             case fsa_dt_xml:
                 foo = fsa_data->data;
                 free_xml(foo);
                 break;
 
             case fsa_dt_lrm:
                 op = (lrmd_event_data_t *) fsa_data->data;
                 lrmd_free_event(op);
                 break;
 
             case fsa_dt_none:
                 if (fsa_data->data != NULL) {
                     crm_err("Dont know how to free %s data from %s",
                             fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
                     exit(1);
                 }
                 break;
         }
         crm_trace("%s data freed", fsa_cause2string(fsa_data->fsa_cause));
     }
 
     free(fsa_data);
 }
 
 /* returns the next message */
 fsa_data_t *
 get_message(void)
 {
     fsa_data_t *message = g_list_nth_data(fsa_message_queue, 0);
 
     fsa_message_queue = g_list_remove(fsa_message_queue, message);
     crm_trace("Processing input %d", message->id);
     return message;
 }
 
 /* returns the current head of the FIFO queue */
 gboolean
 is_message(void)
 {
     return (g_list_length(fsa_message_queue) > 0);
 }
 
 void *
 fsa_typed_data_adv(fsa_data_t * fsa_data, enum fsa_data_type a_type, const char *caller)
 {
     void *ret_val = NULL;
 
     if (fsa_data == NULL) {
         crm_err("%s: No FSA data available", caller);
 
     } else if (fsa_data->data == NULL) {
         crm_err("%s: No message data available. Origin: %s", caller, fsa_data->origin);
 
     } else if (fsa_data->data_type != a_type) {
         crm_crit(
                    "%s: Message data was the wrong type! %d vs. requested=%d."
                    "  Origin: %s", caller, fsa_data->data_type, a_type, fsa_data->origin);
         CRM_ASSERT(fsa_data->data_type == a_type);
     } else {
         ret_val = fsa_data->data;
     }
 
     return ret_val;
 }
 
 /*	A_MSG_ROUTE	*/
 void
 do_msg_route(long long action,
              enum crmd_fsa_cause cause,
              enum crmd_fsa_state cur_state,
              enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
 
     route_message(msg_data->fsa_cause, input->msg);
 }
 
 void
 route_message(enum crmd_fsa_cause cause, xmlNode * input)
 {
     ha_msg_input_t fsa_input;
     enum crmd_fsa_input result = I_NULL;
 
     fsa_input.msg = input;
     CRM_CHECK(cause == C_IPC_MESSAGE || cause == C_HA_MESSAGE, return);
 
     /* try passing the buck first */
     if (relay_message(input, cause == C_IPC_MESSAGE)) {
         return;
     }
 
     /* handle locally */
     result = handle_message(input);
 
     /* done or process later? */
     switch (result) {
         case I_NULL:
         case I_CIB_OP:
         case I_ROUTER:
         case I_NODE_JOIN:
         case I_JOIN_REQUEST:
         case I_JOIN_RESULT:
             break;
         default:
             /* Defering local processing of message */
             register_fsa_input_later(cause, result, &fsa_input);
             return;
     }
 
     if (result != I_NULL) {
         /* add to the front of the queue */
         register_fsa_input(cause, result, &fsa_input);
     }
 }
 
 gboolean
 relay_message(xmlNode * msg, gboolean originated_locally)
 {
     int dest = 1;
     int is_for_dc = 0;
     int is_for_dcib = 0;
     int is_for_te = 0;
     int is_for_crm = 0;
     int is_for_cib = 0;
     int is_local = 0;
     gboolean processing_complete = FALSE;
     const char *host_to = crm_element_value(msg, F_CRM_HOST_TO);
     const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
     const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
     const char *type = crm_element_value(msg, F_TYPE);
     const char *msg_error = NULL;
 
     crm_trace("Routing message %s", crm_element_value(msg, XML_ATTR_REFERENCE));
 
     if (msg == NULL) {
         msg_error = "Cannot route empty message";
 
     } else if (safe_str_eq(CRM_OP_HELLO, crm_element_value(msg, F_CRM_TASK))) {
         /* quietly ignore */
         processing_complete = TRUE;
 
     } else if (safe_str_neq(type, T_CRM)) {
         msg_error = "Bad message type";
 
     } else if (sys_to == NULL) {
         msg_error = "Bad message destination: no subsystem";
     }
 
     if (msg_error != NULL) {
         processing_complete = TRUE;
         crm_err("%s", msg_error);
         crm_log_xml_warn(msg, "bad msg");
     }
 
     if (processing_complete) {
         return TRUE;
     }
 
     processing_complete = TRUE;
 
     is_for_dc = (strcasecmp(CRM_SYSTEM_DC, sys_to) == 0);
     is_for_dcib = (strcasecmp(CRM_SYSTEM_DCIB, sys_to) == 0);
     is_for_te = (strcasecmp(CRM_SYSTEM_TENGINE, sys_to) == 0);
     is_for_cib = (strcasecmp(CRM_SYSTEM_CIB, sys_to) == 0);
     is_for_crm = (strcasecmp(CRM_SYSTEM_CRMD, sys_to) == 0);
 
     is_local = 0;
     if (host_to == NULL || strlen(host_to) == 0) {
         if (is_for_dc || is_for_te) {
             is_local = 0;
 
         } else if (is_for_crm && originated_locally) {
             is_local = 0;
 
         } else {
             is_local = 1;
         }
 
     } else if (safe_str_eq(fsa_our_uname, host_to)) {
         is_local = 1;
     }
 
     if (is_for_dc || is_for_dcib || is_for_te) {
         if (AM_I_DC && is_for_te) {
             ROUTER_RESULT("Message result: Local relay");
             send_msg_via_ipc(msg, sys_to);
 
         } else if (AM_I_DC) {
             ROUTER_RESULT("Message result: DC/CRMd process");
             processing_complete = FALSE;        /* more to be done by caller */
         } else if (originated_locally && safe_str_neq(sys_from, CRM_SYSTEM_PENGINE)
                    && safe_str_neq(sys_from, CRM_SYSTEM_TENGINE)) {
 
             /* Neither the TE or PE should be sending messages
              *   to DC's on other nodes
              *
              * By definition, if we are no longer the DC, then
              *   the PE or TE's data should be discarded
              */
 
 #if SUPPORT_COROSYNC
             if (is_openais_cluster()) {
                 dest = text2msg_type(sys_to);
             }
 #endif
             ROUTER_RESULT("Message result: External relay to DC");
             send_cluster_message(host_to, dest, msg, TRUE);
 
         } else {
             /* discard */
             ROUTER_RESULT("Message result: Discard, not DC");
         }
 
     } else if (is_local && (is_for_crm || is_for_cib)) {
         ROUTER_RESULT("Message result: CRMd process");
         processing_complete = FALSE;    /* more to be done by caller */
 
     } else if (is_local) {
         ROUTER_RESULT("Message result: Local relay");
         send_msg_via_ipc(msg, sys_to);
 
     } else {
 #if SUPPORT_COROSYNC
         if (is_openais_cluster()) {
             dest = text2msg_type(sys_to);
         }
 #endif
         ROUTER_RESULT("Message result: External relay");
         send_cluster_message(host_to, dest, msg, TRUE);
     }
 
     return processing_complete;
 }
 
 static gboolean
 process_hello_message(xmlNode * hello,
                       char **uuid, char **client_name, char **major_version, char **minor_version)
 {
     const char *local_uuid;
     const char *local_client_name;
     const char *local_major_version;
     const char *local_minor_version;
 
     *uuid = NULL;
     *client_name = NULL;
     *major_version = NULL;
     *minor_version = NULL;
 
     if (hello == NULL) {
         return FALSE;
     }
 
     local_uuid = crm_element_value(hello, "client_uuid");
     local_client_name = crm_element_value(hello, "client_name");
     local_major_version = crm_element_value(hello, "major_version");
     local_minor_version = crm_element_value(hello, "minor_version");
 
     if (local_uuid == NULL || strlen(local_uuid) == 0) {
         crm_err("Hello message was not valid (field %s not found)", "uuid");
         return FALSE;
 
     } else if (local_client_name == NULL || strlen(local_client_name) == 0) {
         crm_err("Hello message was not valid (field %s not found)", "client name");
         return FALSE;
 
     } else if (local_major_version == NULL || strlen(local_major_version) == 0) {
         crm_err("Hello message was not valid (field %s not found)", "major version");
         return FALSE;
 
     } else if (local_minor_version == NULL || strlen(local_minor_version) == 0) {
         crm_err("Hello message was not valid (field %s not found)", "minor version");
         return FALSE;
     }
 
     *uuid = strdup(local_uuid);
     *client_name = strdup(local_client_name);
     *major_version = strdup(local_major_version);
     *minor_version = strdup(local_minor_version);
 
     crm_trace("Hello message ok");
     return TRUE;
 }
 
 gboolean
 crmd_authorize_message(xmlNode * client_msg, crmd_client_t * curr_client)
 {
     /* check the best case first */
     const char *sys_from = crm_element_value(client_msg, F_CRM_SYS_FROM);
     char *uuid = NULL;
     char *client_name = NULL;
     char *major_version = NULL;
     char *minor_version = NULL;
     const char *filtered_from;
     gpointer table_key = NULL;
     gboolean auth_result = FALSE;
     gboolean can_reply = FALSE; /* no-one has registered with this id */
 
     xmlNode *xml = NULL;
     const char *op = crm_element_value(client_msg, F_CRM_TASK);
 
     if (safe_str_neq(CRM_OP_HELLO, op)) {
 
         if (sys_from == NULL) {
             crm_warn("Message [%s] was had no value for %s... discarding",
                      crm_element_value(client_msg, XML_ATTR_REFERENCE), F_CRM_SYS_FROM);
             return FALSE;
         }
 
         filtered_from = sys_from;
 
         /* The CIB can have two names on the DC */
         if (strcasecmp(sys_from, CRM_SYSTEM_DCIB) == 0)
             filtered_from = CRM_SYSTEM_CIB;
 
         if (g_hash_table_lookup(ipc_clients, filtered_from) != NULL) {
             can_reply = TRUE;   /* reply can be routed */
         }
 
         crm_trace("Message reply can%s be routed from %s.", can_reply ? "" : " not", sys_from);
 
         if (can_reply == FALSE) {
             crm_warn("Message [%s] not authorized",
                      crm_element_value(client_msg, XML_ATTR_REFERENCE));
         }
 
         return can_reply;
     }
 
     crm_trace("received client join msg");
     crm_log_xml_trace(client_msg, "join");
     xml = get_message_xml(client_msg, F_CRM_DATA);
     auth_result = process_hello_message(xml, &uuid, &client_name, &major_version, &minor_version);
 
     if (auth_result == TRUE) {
         if (client_name == NULL || uuid == NULL) {
             crm_err("Bad client details (client_name=%s, uuid=%s)",
                     crm_str(client_name), crm_str(uuid));
             auth_result = FALSE;
         }
     }
 
     if (auth_result == TRUE) {
         /* check version */
         int mav = atoi(major_version);
         int miv = atoi(minor_version);
 
         crm_trace("Checking client version number");
         if (mav < 0 || miv < 0) {
             crm_err("Client version (%d:%d) is not acceptable", mav, miv);
             auth_result = FALSE;
         }
     }
 
     table_key = (gpointer) generate_hash_key(client_name, uuid);
 
     if (auth_result == TRUE) {
         xmlNode *xml = NULL;
         crm_trace("Accepted client %s", crm_str(table_key));
 
         curr_client->table_key = table_key;
         curr_client->sub_sys = strdup(client_name);
         curr_client->uuid = strdup(uuid);
 
         g_hash_table_insert(ipc_clients, table_key, curr_client->ipc);
 
         xml = create_hello_message("n/a", CRM_SYSTEM_CRMD, "0", "1");
         crm_ipcs_send(curr_client->ipc, xml, FALSE);
         free_xml(xml);
 
         crm_trace("Updated client list with %s", crm_str(table_key));
 
         crm_trace("Triggering FSA: %s", __FUNCTION__);
         mainloop_set_trigger(fsa_source);
 
     } else {
         free(table_key);
         crm_warn("Rejected client logon request");
         qb_ipcs_disconnect(curr_client->ipc);
     }
 
     free(uuid);
     free(minor_version);
     free(major_version);
     free(client_name);
 
     /* hello messages should never be processed further */
     return FALSE;
 }
 
 enum crmd_fsa_input
 handle_message(xmlNode * msg)
 {
     const char *type = NULL;
 
     CRM_CHECK(msg != NULL, return I_NULL);
 
     type = crm_element_value(msg, F_CRM_MSG_TYPE);
     if (crm_str_eq(type, XML_ATTR_REQUEST, TRUE)) {
         return handle_request(msg);
 
     } else if (crm_str_eq(type, XML_ATTR_RESPONSE, TRUE)) {
         handle_response(msg);
         return I_NULL;
     }
 
     crm_err("Unknown message type: %s", type);
     return I_NULL;
 }
 
 static enum crmd_fsa_input
 handle_failcount_op(xmlNode * stored_msg)
 {
     const char *rsc = NULL;
     xmlNode *xml_rsc = get_xpath_object("//" XML_CIB_TAG_RESOURCE, stored_msg, LOG_ERR);
 
     if (xml_rsc) {
         rsc = ID(xml_rsc);
     }
 
     if (rsc) {
         char *attr = NULL;
 
         crm_info("Removing failcount for %s", rsc);
 
         attr = crm_concat("fail-count", rsc, '-');
         update_attrd(NULL, attr, NULL, NULL);
         free(attr);
 
         attr = crm_concat("last-failure", rsc, '-');
         update_attrd(NULL, attr, NULL, NULL);
         free(attr);
 
         lrm_clear_last_failure(rsc);
     } else {
         crm_log_xml_warn(stored_msg, "invalid failcount op");
     }
 
     return I_NULL;
 }
 
 enum crmd_fsa_input
 handle_request(xmlNode * stored_msg)
 {
     xmlNode *msg = NULL;
     const char *op = crm_element_value(stored_msg, F_CRM_TASK);
 
     /* Optimize this for the DC - it has the most to do */
 
     if (op == NULL) {
         crm_log_xml_err(stored_msg, "Bad message");
         return I_NULL;
     }
 
     /*========== DC-Only Actions ==========*/
     if (AM_I_DC) {
         if (strcmp(op, CRM_OP_JOIN_ANNOUNCE) == 0) {
             return I_NODE_JOIN;
 
         } else if (strcmp(op, CRM_OP_JOIN_REQUEST) == 0) {
             return I_JOIN_REQUEST;
 
         } else if (strcmp(op, CRM_OP_JOIN_CONFIRM) == 0) {
             return I_JOIN_RESULT;
 
         } else if (strcmp(op, CRM_OP_SHUTDOWN) == 0) {
             const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
             gboolean dc_match = safe_str_eq(host_from, fsa_our_dc);
 
             if (is_set(fsa_input_register, R_SHUTDOWN)) {
                 crm_info("Shutting ourselves down (DC)");
                 return I_STOP;
 
             } else if (dc_match) {
                 crm_err("We didnt ask to be shut down, yet our"
                         " TE is telling us too." " Better get out now!");
                 return I_TERMINATE;
 
             } else if (fsa_state != S_STOPPING) {
                 crm_err("Another node is asking us to shutdown" " but we think we're ok.");
                 return I_ELECTION;
             }
 
         } else if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
             /* a slave wants to shut down */
             /* create cib fragment and add to message */
             return handle_shutdown_request(stored_msg);
         }
     }
 
     /*========== common actions ==========*/
     if (strcmp(op, CRM_OP_NOVOTE) == 0) {
         ha_msg_input_t fsa_input;
 
         fsa_input.msg = stored_msg;
         register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
                                A_ELECTION_COUNT | A_ELECTION_CHECK, FALSE, __FUNCTION__);
 
     } else if (strcmp(op, CRM_OP_CLEAR_FAILCOUNT) == 0) {
         return handle_failcount_op(stored_msg);
 
     } else if (strcmp(op, CRM_OP_VOTE) == 0) {
         /* count the vote and decide what to do after that */
         ha_msg_input_t fsa_input;
 
         fsa_input.msg = stored_msg;
         register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
                                A_ELECTION_COUNT | A_ELECTION_CHECK, FALSE, __FUNCTION__);
 
         /* Sometimes we _must_ go into S_ELECTION */
         if (fsa_state == S_HALT) {
             crm_debug("Forcing an election from S_HALT");
             return I_ELECTION;
 #if 0
         } else if (AM_I_DC) {
             /* This is the old way of doing things but what is gained? */
             return I_ELECTION;
 #endif
         }
 
     } else if (strcmp(op, CRM_OP_JOIN_OFFER) == 0) {
         crm_debug("Raising I_JOIN_OFFER: join-%s", crm_element_value(stored_msg, F_CRM_JOIN_ID));
         return I_JOIN_OFFER;
 
     } else if (strcmp(op, CRM_OP_JOIN_ACKNAK) == 0) {
         crm_debug("Raising I_JOIN_RESULT: join-%s", crm_element_value(stored_msg, F_CRM_JOIN_ID));
         return I_JOIN_RESULT;
 
     } else if (strcmp(op, CRM_OP_LRM_DELETE) == 0
                || strcmp(op, CRM_OP_LRM_FAIL) == 0
                || strcmp(op, CRM_OP_LRM_REFRESH) == 0 || strcmp(op, CRM_OP_REPROBE) == 0) {
 
         crm_xml_add(stored_msg, F_CRM_SYS_TO, CRM_SYSTEM_LRMD);
         return I_ROUTER;
 
     } else if (strcmp(op, CRM_OP_NOOP) == 0) {
         return I_NULL;
 
     } else if (strcmp(op, CRM_OP_LOCAL_SHUTDOWN) == 0) {
 
         crm_shutdown(SIGTERM);
         /*return I_SHUTDOWN; */
         return I_NULL;
 
         /*========== (NOT_DC)-Only Actions ==========*/
     } else if (AM_I_DC == FALSE && strcmp(op, CRM_OP_SHUTDOWN) == 0) {
 
         const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
         gboolean dc_match = safe_str_eq(host_from, fsa_our_dc);
 
         if (dc_match || fsa_our_dc == NULL) {
             if (is_set(fsa_input_register, R_SHUTDOWN) == FALSE) {
                 crm_err("We didn't ask to be shut down, yet our" " DC is telling us too.");
                 set_bit(fsa_input_register, R_STAYDOWN);
                 return I_STOP;
             }
             crm_info("Shutting down");
             return I_STOP;
 
         } else {
             crm_warn("Discarding %s op from %s", op, host_from);
         }
 
     } else if (strcmp(op, CRM_OP_PING) == 0) {
         /* eventually do some stuff to figure out
          * if we /are/ ok
          */
         const char *sys_to = crm_element_value(stored_msg, F_CRM_SYS_TO);
         xmlNode *ping = create_xml_node(NULL, XML_CRM_TAG_PING);
 
         crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok");
         crm_xml_add(ping, XML_PING_ATTR_SYSFROM, sys_to);
         crm_xml_add(ping, "crmd_state", fsa_state2string(fsa_state));
 
         /* Ok, so technically not so interesting, but CTS needs to see this */
         crm_notice("Current ping state: %s", fsa_state2string(fsa_state));
 
         msg = create_reply(stored_msg, ping);
         relay_message(msg, TRUE);
 
         free_xml(ping);
         free_xml(msg);
 
     } else if (strcmp(op, CRM_OP_RM_NODE_CACHE) == 0) {
         xmlNode *options = get_xpath_object("//"XML_TAG_OPTIONS, stored_msg, LOG_ERR);
         int id = 0;
 
         if (options) {
            crm_element_value_int(options, XML_ATTR_ID, &id);
         }
         if (id) {
             reap_crm_member(id);
         }
 
     } else {
         crm_err("Unexpected request (%s) sent to %s", op, AM_I_DC ? "the DC" : "non-DC node");
         crm_log_xml_err(stored_msg, "Unexpected");
     }
 
     return I_NULL;
 }
 
 void
 handle_response(xmlNode * stored_msg)
 {
     const char *op = crm_element_value(stored_msg, F_CRM_TASK);
 
     if (op == NULL) {
         crm_log_xml_err(stored_msg, "Bad message");
 
     } else if (AM_I_DC && strcmp(op, CRM_OP_PECALC) == 0) {
         /* Check if the PE answer been superceeded by a subsequent request? */
         const char *msg_ref = crm_element_value(stored_msg, XML_ATTR_REFERENCE);
 
         if (msg_ref == NULL) {
             crm_err("%s - Ignoring calculation with no reference", op);
 
         } else if (safe_str_eq(msg_ref, fsa_pe_ref)) {
             ha_msg_input_t fsa_input;
 
             fsa_input.msg = stored_msg;
             register_fsa_input_later(C_IPC_MESSAGE, I_PE_SUCCESS, &fsa_input);
             crm_trace("Completed: %s...", fsa_pe_ref);
 
         } else {
             crm_info("%s calculation %s is obsolete", op, msg_ref);
         }
 
     } else if (strcmp(op, CRM_OP_VOTE) == 0
                || strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0 || strcmp(op, CRM_OP_SHUTDOWN) == 0) {
 
     } else {
         const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
 
         crm_err("Unexpected response (op=%s, src=%s) sent to the %s",
                 op, host_from, AM_I_DC ? "DC" : "CRMd");
     }
 }
 
 enum crmd_fsa_input
 handle_shutdown_request(xmlNode * stored_msg)
 {
     /* handle here to avoid potential version issues
      *   where the shutdown message/proceedure may have
      *   been changed in later versions.
      *
      * This way the DC is always in control of the shutdown
      */
 
     char *now_s = NULL;
     time_t now = time(NULL);
     xmlNode *node_state = NULL;
     const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
 
     if (host_from == NULL) {
         /* we're shutting down and the DC */
         host_from = fsa_our_uname;
     }
 
     crm_info("Creating shutdown request for %s (state=%s)", host_from, fsa_state2string(fsa_state));
 
     crm_log_xml_trace(stored_msg, "message");
 
-    node_state = create_node_state(host_from, NULL, NULL, NULL, NULL,
+    node_state = create_node_state(host_from, NULL, NULL, NULL,
                                    CRMD_STATE_INACTIVE, FALSE, __FUNCTION__);
 
     fsa_cib_anon_update(XML_CIB_TAG_STATUS, node_state, cib_quorum_override);
     crm_log_xml_trace(node_state, "Shutdown update");
     free_xml(node_state);
 
     now_s = crm_itoa(now);
     update_attrd(host_from, XML_CIB_ATTR_SHUTDOWN, now_s, NULL);
     free(now_s);
 
     /* will be picked up by the TE as long as its running */
     return I_NULL;
 }
 
 /* msg is deleted by the time this returns */
 extern gboolean process_te_message(xmlNode * msg, xmlNode * xml_data);
 
 gboolean
 send_msg_via_ipc(xmlNode * msg, const char *sys)
 {
     gboolean send_ok = TRUE;
     qb_ipcs_connection_t *client_channel;
 
     client_channel = (qb_ipcs_connection_t *) g_hash_table_lookup(ipc_clients, sys);
 
     if (crm_element_value(msg, F_CRM_HOST_FROM) == NULL) {
         crm_xml_add(msg, F_CRM_HOST_FROM, fsa_our_uname);
     }
 
     if (client_channel != NULL) {
         /* Transient clients such as crmadmin */
         send_ok = crm_ipcs_send(client_channel, msg, TRUE);
 
     } else if (sys != NULL && strcmp(sys, CRM_SYSTEM_TENGINE) == 0) {
         xmlNode *data = get_message_xml(msg, F_CRM_DATA);
 
         process_te_message(msg, data);
 
     } else if (sys != NULL && strcmp(sys, CRM_SYSTEM_LRMD) == 0) {
         fsa_data_t fsa_data;
         ha_msg_input_t fsa_input;
 
         fsa_input.msg = msg;
         fsa_input.xml = get_message_xml(msg, F_CRM_DATA);
 
         fsa_data.id = 0;
         fsa_data.actions = 0;
         fsa_data.data = &fsa_input;
         fsa_data.fsa_input = I_MESSAGE;
         fsa_data.fsa_cause = C_IPC_MESSAGE;
         fsa_data.origin = __FUNCTION__;
         fsa_data.data_type = fsa_dt_ha_msg;
 
 #ifdef FSA_TRACE
         crm_trace("Invoking action A_LRM_INVOKE (%.16llx)", A_LRM_INVOKE);
 #endif
         do_lrm_invoke(A_LRM_INVOKE, C_IPC_MESSAGE, fsa_state, I_MESSAGE, &fsa_data);
 
     } else {
         crm_err("Unknown Sub-system (%s)... discarding message.", crm_str(sys));
         send_ok = FALSE;
     }
 
     return send_ok;
 }
 
 ha_msg_input_t *
 new_ha_msg_input(xmlNode * orig)
 {
     ha_msg_input_t *input_copy = NULL;
 
     input_copy = calloc(1, sizeof(ha_msg_input_t));
     input_copy->msg = orig;
     input_copy->xml = get_message_xml(input_copy->msg, F_CRM_DATA);
     return input_copy;
 }
 
 void
 delete_ha_msg_input(ha_msg_input_t * orig)
 {
     if (orig == NULL) {
         return;
     }
     free_xml(orig->msg);
     free(orig);
 }
 
diff --git a/crmd/te_actions.c b/crmd/te_actions.c
index 22322f5851..d8c8a780e7 100644
--- a/crmd/te_actions.c
+++ b/crmd/te_actions.c
@@ -1,516 +1,515 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 
 #include <crm/common/xml.h>
 #include <tengine.h>
 
 #include <crmd_fsa.h>
 #include <crmd_messages.h>
 #include <crm/cluster.h>
 
 char *te_uuid = NULL;
 
 void send_rsc_command(crm_action_t * action);
 
 static void
 te_start_action_timer(crm_graph_t * graph, crm_action_t * action)
 {
     action->timer = calloc(1, sizeof(crm_action_timer_t));
     action->timer->timeout = action->timeout;
     action->timer->reason = timeout_action;
     action->timer->action = action;
     action->timer->source_id = g_timeout_add(action->timer->timeout + graph->network_delay,
                                              action_timer_callback, (void *)action->timer);
 
     CRM_ASSERT(action->timer->source_id != 0);
 }
 
 static gboolean
 te_pseudo_action(crm_graph_t * graph, crm_action_t * pseudo)
 {
     crm_debug("Pseudo action %d fired and confirmed", pseudo->id);
     pseudo->confirmed = TRUE;
     update_graph(graph, pseudo);
     trigger_graph();
     return TRUE;
 }
 
 void
 send_stonith_update(crm_action_t * action, const char *target, const char *uuid)
 {
     int rc = pcmk_ok;
 
     /* zero out the node-status & remove all LRM status info */
     xmlNode *node_state = create_xml_node(NULL, XML_CIB_TAG_STATE);
 
     CRM_CHECK(target != NULL, return);
     CRM_CHECK(uuid != NULL, return);
 
     crm_xml_add(node_state, XML_ATTR_UUID, uuid);
     crm_xml_add(node_state, XML_ATTR_UNAME, target);
-    crm_xml_add(node_state, XML_CIB_ATTR_HASTATE, DEADSTATUS);
     crm_xml_add(node_state, XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO);
     crm_xml_add(node_state, XML_NODE_IS_PEER, OFFLINESTATUS);
     crm_xml_add(node_state, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_DOWN);
     crm_xml_add(node_state, XML_NODE_EXPECTED, CRMD_JOINSTATE_DOWN);
     crm_xml_add(node_state, XML_ATTR_ORIGIN, __FUNCTION__);
 
     rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, node_state,
                                     cib_quorum_override | cib_scope_local | cib_can_create);
 
     /* Delay processing the trigger until the update completes */
     crm_debug("Sending fencing update %d for %s", rc, target);
     add_cib_op_callback(fsa_cib_conn, rc, FALSE, strdup(target), cib_fencing_updated);
 
     /* Make sure it sticks */
     /* fsa_cib_conn->cmds->bump_epoch(fsa_cib_conn, cib_quorum_override|cib_scope_local);    */
 
     erase_status_tag(target, XML_CIB_TAG_LRM, cib_scope_local);
     erase_status_tag(target, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
 
     free_xml(node_state);
 
     /* Make sure the membership cache is accurate */
     crm_update_peer(__FUNCTION__, 0, 0, 0, -1, crm_proc_none, uuid, target, NULL, CRM_NODE_LOST);
 
     return;
 }
 
 static gboolean
 te_fence_node(crm_graph_t * graph, crm_action_t * action)
 {
     int rc = 0;
     const char *id = NULL;
     const char *uuid = NULL;
     const char *target = NULL;
     const char *type = NULL;
     gboolean invalid_action = FALSE;
     enum stonith_call_options options = st_opt_none;
 
     id = ID(action->xml);
     target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
     type = crm_meta_value(action->params, "stonith_action");
 
     CRM_CHECK(id != NULL, invalid_action = TRUE);
     CRM_CHECK(uuid != NULL, invalid_action = TRUE);
     CRM_CHECK(type != NULL, invalid_action = TRUE);
     CRM_CHECK(target != NULL, invalid_action = TRUE);
 
     if (invalid_action) {
         crm_log_xml_warn(action->xml, "BadAction");
         return FALSE;
     }
 
     crm_notice("Executing %s fencing operation (%s) on %s (timeout=%d)",
                type, id, target, transition_graph->stonith_timeout);
 
     /* Passing NULL means block until we can connect... */
     te_connect_stonith(NULL);
 
     if (confirmed_nodes && g_hash_table_size(confirmed_nodes) == 1) {
         options |= st_opt_allow_suicide;
     }
 
     rc = stonith_api->cmds->fence(stonith_api, options, target, type,
                                   transition_graph->stonith_timeout / 1000);
 
     stonith_api->cmds->register_callback(
         stonith_api, rc, transition_graph->stonith_timeout / 1000,
         FALSE, generate_transition_key(transition_graph->id, action->id, 0, te_uuid),
         "tengine_stonith_callback", tengine_stonith_callback);
 
     return TRUE;
 }
 
 static int
 get_target_rc(crm_action_t * action)
 {
     const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);
 
     if (target_rc_s != NULL) {
         return crm_parse_int(target_rc_s, "0");
     }
     return 0;
 }
 
 static gboolean
 te_crm_command(crm_graph_t * graph, crm_action_t * action)
 {
     char *counter = NULL;
     xmlNode *cmd = NULL;
     gboolean is_local = FALSE;
 
     const char *id = NULL;
     const char *task = NULL;
     const char *value = NULL;
     const char *on_node = NULL;
 
     gboolean rc = TRUE;
     gboolean no_wait = FALSE;
 
     id = ID(action->xml);
     task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 
     CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
               crm_err( "Corrupted command (id=%s) %s: no node",
                             crm_str(id), crm_str(task));
               return FALSE);
 
     crm_info( "Executing crm-event (%s): %s on %s%s%s",
                   crm_str(id), crm_str(task), on_node,
                   is_local ? " (local)" : "", no_wait ? " - no waiting" : "");
 
     if (safe_str_eq(on_node, fsa_our_uname)) {
         is_local = TRUE;
     }
 
     value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
     if (crm_is_true(value)) {
         no_wait = TRUE;
     }
 
     if (is_local && safe_str_eq(task, CRM_OP_SHUTDOWN)) {
         /* defer until everything else completes */
         crm_info( "crm-event (%s) is a local shutdown", crm_str(id));
         graph->completion_action = tg_shutdown;
         graph->abort_reason = "local shutdown";
         action->confirmed = TRUE;
         update_graph(graph, action);
         trigger_graph();
         return TRUE;
     }
 
     cmd = create_request(task, action->xml, on_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
 
     counter =
         generate_transition_key(transition_graph->id, action->id, get_target_rc(action), te_uuid);
     crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter);
 
     rc = send_cluster_message(on_node, crm_msg_crmd, cmd, TRUE);
     free(counter);
     free_xml(cmd);
 
     if (rc == FALSE) {
         crm_err("Action %d failed: send", action->id);
         return FALSE;
 
     } else if (no_wait) {
         action->confirmed = TRUE;
         update_graph(graph, action);
         trigger_graph();
 
     } else {
         if (action->timeout <= 0) {
             crm_err("Action %d: %s on %s had an invalid timeout (%dms).  Using %dms instead",
                     action->id, task, on_node, action->timeout, graph->network_delay);
             action->timeout = graph->network_delay;
         }
         te_start_action_timer(graph, action);
     }
 
     return TRUE;
 }
 
 gboolean
 cib_action_update(crm_action_t * action, int status, int op_rc)
 {
     lrmd_event_data_t *op = NULL;
     xmlNode *state = NULL;
     xmlNode *rsc = NULL;
     xmlNode *xml_op = NULL;
     xmlNode *action_rsc = NULL;
 
     int rc = pcmk_ok;
 
     const char *name = NULL;
     const char *value = NULL;
     const char *rsc_id = NULL;
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
     const char *target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
 
     int call_options = cib_quorum_override | cib_scope_local;
     int target_rc = get_target_rc(action);
 
     if (status == PCMK_LRM_OP_PENDING) {
         crm_debug("%s %d: Recording pending operation %s on %s",
                   crm_element_name(action->xml), action->id, task_uuid, target);
     } else {
         crm_warn("%s %d: %s on %s timed out",
                  crm_element_name(action->xml), action->id, task_uuid, target);
     }
 
     action_rsc = find_xml_node(action->xml, XML_CIB_TAG_RESOURCE, TRUE);
     if (action_rsc == NULL) {
         return FALSE;
     }
 
     rsc_id = ID(action_rsc);
     CRM_CHECK(rsc_id != NULL, crm_log_xml_err(action->xml, "Bad:action");
               return FALSE);
 
 /*
   update the CIB
 
 <node_state id="hadev">
       <lrm>
         <lrm_resources>
           <lrm_resource id="rsc2" last_op="start" op_code="0" target="hadev"/>
 */
 
     state = create_xml_node(NULL, XML_CIB_TAG_STATE);
 
     crm_xml_add(state, XML_ATTR_UUID, target_uuid);
     crm_xml_add(state, XML_ATTR_UNAME, target);
 
     rsc = create_xml_node(state, XML_CIB_TAG_LRM);
     crm_xml_add(rsc, XML_ATTR_ID, target_uuid);
 
     rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCES);
     rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCE);
     crm_xml_add(rsc, XML_ATTR_ID, rsc_id);
 
     name = XML_ATTR_TYPE;
     value = crm_element_value(action_rsc, name);
     crm_xml_add(rsc, name, value);
     name = XML_AGENT_ATTR_CLASS;
     value = crm_element_value(action_rsc, name);
     crm_xml_add(rsc, name, value);
     name = XML_AGENT_ATTR_PROVIDER;
     value = crm_element_value(action_rsc, name);
     crm_xml_add(rsc, name, value);
 
     op = convert_graph_action(NULL, action, status, op_rc);
     op->call_id = -1;
     op->user_data = generate_transition_key(transition_graph->id, action->id, target_rc, te_uuid);
 
     xml_op = create_operation_update(rsc, op, CRM_FEATURE_SET, target_rc, __FUNCTION__, LOG_INFO);
     lrmd_free_event(op);
 
     crm_trace("Updating CIB with \"%s\" (%s): %s %s on %s",
                 status < 0 ? "new action" : XML_ATTR_TIMEOUT,
                 crm_element_name(action->xml), crm_str(task), rsc_id, target);
     crm_log_xml_trace(xml_op, "Op");
 
     rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, state, call_options);
 
     crm_trace("Updating CIB with %s action %d: %s on %s (call_id=%d)",
                 services_lrm_status_str(status), action->id, task_uuid, target, rc);
 
     add_cib_op_callback(fsa_cib_conn, rc, FALSE, NULL, cib_action_updated);
     free_xml(state);
 
     action->sent_update = TRUE;
 
     if (rc < pcmk_ok) {
         return FALSE;
     }
 
     return TRUE;
 }
 
 static gboolean
 te_rsc_command(crm_graph_t * graph, crm_action_t * action)
 {
     /* never overwrite stop actions in the CIB with
      *   anything other than completed results
      *
      * Writing pending stops makes it look like the
      *   resource is running again
      */
     xmlNode *cmd = NULL;
     xmlNode *rsc_op = NULL;
 
     gboolean rc = TRUE;
     gboolean no_wait = FALSE;
     gboolean is_local = FALSE;
 
     char *counter = NULL;
     const char *task = NULL;
     const char *value = NULL;
     const char *on_node = NULL;
     const char *task_uuid = NULL;
 
     CRM_ASSERT(action != NULL);
     CRM_ASSERT(action->xml != NULL);
 
     action->executed = FALSE;
     on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 
     CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
               crm_err( "Corrupted command(id=%s) %s: no node",
                             ID(action->xml), crm_str(task));
               return FALSE);
 
     rsc_op = action->xml;
     task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
     task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
     on_node = crm_element_value(rsc_op, XML_LRM_ATTR_TARGET);
     counter =
         generate_transition_key(transition_graph->id, action->id, get_target_rc(action), te_uuid);
     crm_xml_add(rsc_op, XML_ATTR_TRANSITION_KEY, counter);
 
     if (safe_str_eq(on_node, fsa_our_uname)) {
         is_local = TRUE;
     }
 
     value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
     if (crm_is_true(value)) {
         no_wait = TRUE;
     }
 
     crm_info("Initiating action %d: %s %s on %s%s%s",
              action->id, task, task_uuid, on_node,
              is_local ? " (local)" : "", no_wait ? " - no waiting" : "");
 
     cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, on_node,
                          CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);
 
     if (is_local) {
         /* shortcut local resource commands */
         ha_msg_input_t data = {
             .msg = cmd,
             .xml = rsc_op,
         };
 
         fsa_data_t msg = {
             .id = 0,
             .data = &data,
             .data_type = fsa_dt_ha_msg,
             .fsa_input = I_NULL,
             .fsa_cause = C_FSA_INTERNAL,
             .actions = A_LRM_INVOKE,
             .origin = __FUNCTION__,
         };
 
         do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, fsa_state, I_NULL, &msg);
 
     } else {
         rc = send_cluster_message(on_node, crm_msg_lrmd, cmd, TRUE);
     }
 
     free(counter);
     free_xml(cmd);
 
     action->executed = TRUE;
     if (rc == FALSE) {
         crm_err("Action %d failed: send", action->id);
         return FALSE;
 
     } else if (no_wait) {
         action->confirmed = TRUE;
         update_graph(transition_graph, action);
         trigger_graph();
 
     } else {
         if (action->timeout <= 0) {
             crm_err("Action %d: %s %s on %s had an invalid timeout (%dms).  Using %dms instead",
                     action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
             action->timeout = graph->network_delay;
         }
         te_start_action_timer(graph, action);
     }
 
     value = crm_meta_value(action->params, XML_OP_ATTR_PENDING);
     if (crm_is_true(value)) {
         /* write a "pending" entry to the CIB, inhibit notification */
         crm_info("Recording pending op %s in the CIB", task_uuid);
         cib_action_update(action, PCMK_LRM_OP_PENDING, PCMK_EXECRA_STATUS_UNKNOWN);
     }
 
     return TRUE;
 }
 
 crm_graph_functions_t te_graph_fns = {
     te_pseudo_action,
     te_rsc_command,
     te_crm_command,
     te_fence_node
 };
 
 void
 notify_crmd(crm_graph_t * graph)
 {
     const char *type = "unknown";
     enum crmd_fsa_input event = I_NULL;
 
     crm_debug("Processing transition completion in state %s", fsa_state2string(fsa_state));
 
     CRM_CHECK(graph->complete, graph->complete = TRUE);
 
     switch (graph->completion_action) {
         case tg_stop:
             type = "stop";
             /* fall through */
         case tg_done:
             type = "done";
             if (fsa_state == S_TRANSITION_ENGINE) {
                 event = I_TE_SUCCESS;
             }
             break;
 
         case tg_restart:
             type = "restart";
             if (fsa_state == S_TRANSITION_ENGINE) {
                 if (transition_timer->period_ms > 0) {
                     crm_timer_stop(transition_timer);
                     crm_timer_start(transition_timer);
                 } else if(too_many_st_failures() == FALSE) {
                     event = I_PE_CALC;
                 }
 
             } else if (fsa_state == S_POLICY_ENGINE) {
                 register_fsa_action(A_PE_INVOKE);
             }
             break;
 
         case tg_shutdown:
             type = "shutdown";
             if (is_set(fsa_input_register, R_SHUTDOWN)) {
                 event = I_STOP;
 
             } else {
                 crm_err("We didn't ask to be shut down, yet our" " PE is telling us too.");
                 event = I_TERMINATE;
             }
     }
 
     crm_debug( "Transition %d status: %s - %s",
                   graph->id, type, crm_str(graph->abort_reason));
 
     graph->abort_reason = NULL;
     graph->completion_action = tg_done;
     clear_bit(fsa_input_register, R_IN_TRANSITION);
 
     if (event != I_NULL) {
         register_fsa_input(C_FSA_INTERNAL, event, NULL);
 
     } else if (fsa_source) {
         mainloop_set_trigger(fsa_source);
     }
 }
diff --git a/crmd/te_callbacks.c b/crmd/te_callbacks.c
index 41c36d8f02..bff64ae5e5 100644
--- a/crmd/te_callbacks.c
+++ b/crmd/te_callbacks.c
@@ -1,603 +1,601 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/stat.h>
 
 #include <crm/crm.h>
 #include <crm/common/xml.h>
 #include <crm/msg_xml.h>
 #include <crm/cib/internal.h>
 
 #include <tengine.h>
 #include <te_callbacks.h>
 #include <crmd_fsa.h>
 
 #include <crm/cluster.h> /* For ONLINESTATUS etc */
 
 void te_update_confirm(const char *event, xmlNode * msg);
 
 extern char *te_uuid;
 gboolean shuttingdown = FALSE;
 crm_graph_t *transition_graph;
 crm_trigger_t *transition_trigger = NULL;
 
 /* #define rsc_op_template "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_CIB_TAG_STATE"[@uname='%s']"//"XML_LRM_TAG_RSC_OP"[@id='%s]" */
 #define rsc_op_template "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_LRM_TAG_RSC_OP"[@id='%s']"
 
 static const char *
 get_node_id(xmlNode * rsc_op)
 {
     xmlNode *node = rsc_op;
 
     while (node != NULL && safe_str_neq(XML_CIB_TAG_STATE, TYPE(node))) {
         node = node->parent;
     }
 
     CRM_CHECK(node != NULL, return NULL);
     return ID(node);
 }
 
 static void
 process_resource_updates(xmlXPathObject * xpathObj)
 {
 /*
     <status>
        <node_state id="node1" state=CRMD_STATE_ACTIVE exp_state="active">
           <lrm>
              <lrm_resources>
         	<rsc_state id="" rsc_id="rsc4" node_id="node1" rsc_state="stopped"/>
 */
     int lpc = 0, max = xpathObj->nodesetval->nodeNr;
 
     for (lpc = 0; lpc < max; lpc++) {
         xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
         const char *node = get_node_id(rsc_op);
 
         process_graph_event(rsc_op, node);
     }
 }
 
 void
 te_update_diff(const char *event, xmlNode * msg)
 {
     int rc = -1;
     const char *op = NULL;
 
     xmlNode *diff = NULL;
     xmlXPathObject *xpathObj = NULL;
 
     int diff_add_updates = 0;
     int diff_add_epoch = 0;
     int diff_add_admin_epoch = 0;
 
     int diff_del_updates = 0;
     int diff_del_epoch = 0;
     int diff_del_admin_epoch = 0;
 
     CRM_CHECK(msg != NULL, return);
     crm_element_value_int(msg, F_CIB_RC, &rc);
 
     if (transition_graph == NULL) {
         crm_trace("No graph");
         return;
 
     } else if (rc < pcmk_ok) {
         crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc));
         return;
 
     } else if (transition_graph->complete == TRUE
                && fsa_state != S_IDLE
                && fsa_state != S_TRANSITION_ENGINE && fsa_state != S_POLICY_ENGINE) {
         crm_trace("Filter state=%s, complete=%d", fsa_state2string(fsa_state),
                     transition_graph->complete);
         return;
     }
 
     op = crm_element_value(msg, F_CIB_OPERATION);
     diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
 
     cib_diff_version_details(diff,
                              &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
                              &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
 
     crm_debug("Processing diff (%s): %d.%d.%d -> %d.%d.%d (%s)", op,
               diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
               diff_add_admin_epoch, diff_add_epoch, diff_add_updates, fsa_state2string(fsa_state));
     log_cib_diff(LOG_DEBUG_2, diff, op);
 
     if (cib_config_changed(NULL, NULL, &diff)) {
         abort_transition(INFINITY, tg_restart, "Non-status change", diff);
         goto bail;              /* configuration changed */
     }
 
     /* Tickets Attributes - Added/Updated */
     xpathObj =
         xpath_search(diff,
                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_TICKETS);
     if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
         xmlNode *aborted = getXpathResult(xpathObj, 0);
 
         abort_transition(INFINITY, tg_restart, "Ticket attribute: update", aborted);
         goto bail;
 
     } else if (xpathObj) {
         xmlXPathFreeObject(xpathObj);
     }
 
     /* Tickets Attributes - Removed */
     xpathObj =
         xpath_search(diff,
                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_TICKETS);
     if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
         xmlNode *aborted = getXpathResult(xpathObj, 0);
 
         abort_transition(INFINITY, tg_restart, "Ticket attribute: removal", aborted);
         goto bail;
 
     } else if (xpathObj) {
         xmlXPathFreeObject(xpathObj);
     }
 
     /* Transient Attributes - Added/Updated */
     xpathObj =
         xpath_search(diff,
                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//"
                      XML_TAG_TRANSIENT_NODEATTRS "//" XML_CIB_TAG_NVPAIR);
     if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
         int lpc;
 
         for (lpc = 0; lpc < xpathObj->nodesetval->nodeNr; lpc++) {
             xmlNode *attr = getXpathResult(xpathObj, lpc);
             const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
             const char *value = NULL;
 
             if (safe_str_eq(CRM_OP_PROBED, name)) {
                 value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
             }
 
             if (crm_is_true(value) == FALSE) {
                 abort_transition(INFINITY, tg_restart, "Transient attribute: update", attr);
                 crm_log_xml_trace(attr, "Abort");
                 goto bail;
             }
         }
 
     } else if (xpathObj) {
         xmlXPathFreeObject(xpathObj);
     }
 
     /* Transient Attributes - Removed */
     xpathObj =
         xpath_search(diff,
                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//"
                      XML_TAG_TRANSIENT_NODEATTRS);
     if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
         xmlNode *aborted = getXpathResult(xpathObj, 0);
 
         abort_transition(INFINITY, tg_restart, "Transient attribute: removal", aborted);
         goto bail;
 
     } else if (xpathObj) {
         xmlXPathFreeObject(xpathObj);
     }
 
     /* Check for node state updates... possibly from a shutdown we requested */
     xpathObj =
         xpath_search(diff, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_STATE);
     if (xpathObj) {
         int lpc = 0, max = xpathObj->nodesetval->nodeNr;
 
         for (lpc = 0; lpc < max; lpc++) {
             xmlNode *node = getXpathResult(xpathObj, lpc);
             const char *event_node = crm_element_value(node, XML_ATTR_ID);
             const char *ccm_state = crm_element_value(node, XML_NODE_IN_CLUSTER);
-            const char *ha_state = crm_element_value(node, XML_CIB_ATTR_HASTATE);
             const char *shutdown_s = crm_element_value(node, XML_CIB_ATTR_SHUTDOWN);
             const char *crmd_state = crm_element_value(node, XML_NODE_IS_PEER);
 
             if (safe_str_eq(ccm_state, XML_BOOLEAN_FALSE)
-                || safe_str_eq(ha_state, DEADSTATUS)
                 || safe_str_eq(crmd_state, CRMD_JOINSTATE_DOWN)) {
                 crm_action_t *shutdown = match_down_event(0, event_node, NULL);
 
                 if (shutdown != NULL) {
                     const char *task = crm_element_value(shutdown->xml, XML_LRM_ATTR_TASK);
 
                     if (safe_str_neq(task, CRM_OP_FENCE)) {
                         /* Wait for stonithd to tell us it is complete via tengine_stonith_callback() */
                         crm_debug("Confirming %s op %d", task, shutdown->id);
                         /* match->confirmed = TRUE; */
                         stop_te_timer(shutdown->timer);
                         update_graph(transition_graph, shutdown);
                         trigger_graph();
                     }
 
                 } else {
                     crm_info("Stonith/shutdown of %s not matched", event_node);
                     abort_transition(INFINITY, tg_restart, "Node failure", node);
                 }
                 fail_incompletable_actions(transition_graph, event_node);
             }
 
             if (shutdown_s) {
                 int shutdown = crm_parse_int(shutdown_s, NULL);
 
                 if (shutdown > 0) {
                     crm_info("Aborting on " XML_CIB_ATTR_SHUTDOWN " attribute for %s", event_node);
                     abort_transition(INFINITY, tg_restart, "Shutdown request", node);
                 }
             }
         }
         xmlXPathFreeObject(xpathObj);
     }
 
     /*
      * Check for and fast-track the processing of LRM refreshes
      * In large clusters this can result in _huge_ speedups
      *
      * Unfortunately we can only do so when there are no pending actions
      * Otherwise we could miss updates we're waiting for and stall 
      *
      */
     xpathObj = NULL;
     if (transition_graph->pending == 0) {
         xpathObj =
             xpath_search(diff,
                          "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//"
                          XML_LRM_TAG_RESOURCE);
     }
 
     if (xpathObj) {
         int updates = xpathObj->nodesetval->nodeNr;
 
         if (updates > 1) {
             /* Updates by, or in response to, TE actions will never contain updates
              * for more than one resource at a time
              */
             crm_debug("Detected LRM refresh - %d resources updated: Skipping all resource events",
                      updates);
             crm_log_xml_trace(diff, "lrm-refresh");
             abort_transition(INFINITY, tg_restart, "LRM Refresh", NULL);
             goto bail;
         }
         xmlXPathFreeObject(xpathObj);
     }
 
     /* Process operation updates */
     xpathObj =
         xpath_search(diff,
                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_LRM_TAG_RSC_OP);
     if (xpathObj) {
         process_resource_updates(xpathObj);
         xmlXPathFreeObject(xpathObj);
     }
 
     /* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */
     xpathObj = xpath_search(diff, "//" XML_TAG_DIFF_REMOVED "//" XML_LRM_TAG_RSC_OP);
     if (xpathObj) {
         int lpc = 0, max = xpathObj->nodesetval->nodeNr;
 
         for (lpc = 0; lpc < max; lpc++) {
             int max = 0;
             const char *op_id = NULL;
             char *rsc_op_xpath = NULL;
             xmlXPathObject *op_match = NULL;
             xmlNode *match = getXpathResult(xpathObj, lpc);
 
             CRM_CHECK(match != NULL, continue);
 
             op_id = ID(match);
 
             max = strlen(rsc_op_template) + strlen(op_id) + 1;
             rsc_op_xpath = calloc(1, max);
             snprintf(rsc_op_xpath, max, rsc_op_template, op_id);
 
             op_match = xpath_search(diff, rsc_op_xpath);
             if (op_match == NULL || op_match->nodesetval->nodeNr == 0) {
                 /* Prevent false positives by matching cancelations too */
                 const char *node = get_node_id(match);
                 crm_action_t *cancelled = get_cancel_action(op_id, node);
 
                 if (cancelled == NULL) {
                     crm_debug("No match for deleted action %s (%s on %s)", rsc_op_xpath, op_id,
                               node);
                     abort_transition(INFINITY, tg_restart, "Resource op removal", match);
                     if (op_match) {
                         xmlXPathFreeObject(op_match);
                     }
                     free(rsc_op_xpath);
                     goto bail;
 
                 } else {
                     crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d",
                               op_id, node, cancelled->id);
                 }
             }
 
             if (op_match) {
                 xmlXPathFreeObject(op_match);
             }
             free(rsc_op_xpath);
         }
     }
 
   bail:
     if (xpathObj) {
         xmlXPathFreeObject(xpathObj);
     }
 }
 
 gboolean
 process_te_message(xmlNode * msg, xmlNode * xml_data)
 {
     const char *from = crm_element_value(msg, F_ORIG);
     const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
     const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
     const char *ref = crm_element_value(msg, XML_ATTR_REFERENCE);
     const char *op = crm_element_value(msg, F_CRM_TASK);
     const char *type = crm_element_value(msg, F_CRM_MSG_TYPE);
 
     crm_trace("Processing %s (%s) message", op, ref);
     crm_log_xml_trace(msg, "ipc");
 
     if (op == NULL) {
         /* error */
 
     } else if (sys_to == NULL || strcasecmp(sys_to, CRM_SYSTEM_TENGINE) != 0) {
         crm_trace("Bad sys-to %s", crm_str(sys_to));
         return FALSE;
 
     } else if (safe_str_eq(op, CRM_OP_INVOKE_LRM)
                && safe_str_eq(sys_from, CRM_SYSTEM_LRMD)
 /* 		  && safe_str_eq(type, XML_ATTR_RESPONSE) */
         ) {
         xmlXPathObject *xpathObj = NULL;
 
         crm_log_xml_trace(msg, "Processing (N)ACK");
         crm_debug("Processing (N)ACK %s from %s", crm_element_value(msg, XML_ATTR_REFERENCE), from);
 
         xpathObj = xpath_search(xml_data, "//" XML_LRM_TAG_RSC_OP);
         if (xpathObj) {
             process_resource_updates(xpathObj);
             xmlXPathFreeObject(xpathObj);
             xpathObj = NULL;
 
         } else {
             crm_log_xml_err(msg, "Invalid (N)ACK");
             return FALSE;
         }
 
     } else {
         crm_err("Unknown command: %s::%s from %s", type, op, sys_from);
     }
 
     crm_trace("finished processing message");
 
     return TRUE;
 }
 
 GHashTable *stonith_failures = NULL;
 struct st_fail_rec 
 {
         int count;
 };
 
 gboolean too_many_st_failures(void) 
 {
     GHashTableIter iter;
     const char *key = NULL;
     struct st_fail_rec *value = NULL;
 
     if(stonith_failures == NULL) {
         return FALSE;
     }
 
     g_hash_table_iter_init(&iter, stonith_failures);
     while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
         if(value->count > 10) {
             crm_notice("Too many failures to fence %s (%d), giving up",
                        key, value->count);
             return TRUE;
         }
     }
     return FALSE;
 }
 
 
 void
 tengine_stonith_callback(stonith_t * stonith, const xmlNode * msg, int call_id, int rc,
                          xmlNode * output, void *userdata)
 {
     char *uuid = NULL;
     int target_rc = -1;
     int stonith_id = -1;
     int transition_id = -1;
     crm_action_t *action = NULL;
     struct st_fail_rec *rec = NULL;
 
     CRM_CHECK(userdata != NULL, return);
     crm_log_xml_trace(output, "StonithOp");
     crm_notice("Stonith operation %d/%s: %s (%d)", call_id, (char *)userdata,
              pcmk_strerror(rc), rc);
 
     if (AM_I_DC == FALSE) {
         return;
     }
 
     /* crm_info("call=%d, optype=%d, node_name=%s, result=%d, node_list=%s, action=%s", */
     /*       op->call_id, op->optype, op->node_name, op->op_result, */
     /*       (char *)op->node_list, op->private_data); */
 
     /* filter out old STONITH actions */
     CRM_CHECK(decode_transition_key(userdata, &uuid, &transition_id, &stonith_id, &target_rc),
               crm_err("Invalid event detected");
               goto bail;
         );
 
     if (transition_graph->complete || stonith_id < 0 || safe_str_neq(uuid, te_uuid)
         || transition_graph->id != transition_id) {
         crm_info("Ignoring STONITH action initiated outside of the current transition");
         goto bail;
     }
 
     /* this will mark the event complete if a match is found */
     action = get_action(stonith_id, FALSE);
     if (action == NULL) {
         crm_err("Stonith action not matched");
         goto bail;
     }
 
     stop_te_timer(action->timer);
     if(stonith_failures == NULL) {
         stonith_failures = g_hash_table_new_full(
             crm_str_hash, g_str_equal, g_hash_destroy_str, free);
     }
 
     if (rc == pcmk_ok) {
         const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
         const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
 
         crm_debug("Stonith operation %d for %s passed", call_id, target);
         if (action->confirmed == FALSE) {
             action->confirmed = TRUE;
             send_stonith_update(action, target, uuid);
         }
         rec = g_hash_table_lookup(stonith_failures, target);
         if(rec) {
             rec->count = 0;
         }
         
     } else {
         const char *target = crm_element_value_const(action->xml, XML_LRM_ATTR_TARGET);
         const char *allow_fail = crm_meta_value(action->params, XML_ATTR_TE_ALLOWFAIL);
 
         action->failed = TRUE;
         if (crm_is_true(allow_fail) == FALSE) {
             crm_notice("Stonith operation %d for %s failed (%s): aborting transition.", call_id, target, pcmk_strerror(rc));
             abort_transition(INFINITY, tg_restart, "Stonith failed", NULL);
         }
 
         rec = g_hash_table_lookup(stonith_failures, target);
         if(rec) {
             rec->count++;
         } else {
             rec = malloc(sizeof(struct st_fail_rec));
             rec->count = 1;
             g_hash_table_insert(stonith_failures, strdup(target), rec);
         }
     }
 
     update_graph(transition_graph, action);
     trigger_graph();
 
   bail:
     free(userdata);
     free(uuid);
     return;
 }
 
 void
 cib_fencing_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     if (rc < pcmk_ok) {
         crm_err("Fencing update %d for %s: failed - %s (%d)",
                 call_id, (char *)user_data, pcmk_strerror(rc), rc);
         crm_log_xml_warn(msg, "Failed update");
         abort_transition(INFINITY, tg_shutdown, "CIB update failed", NULL);
 
     } else {
         crm_info("Fencing update %d for %s: complete", call_id, (char *)user_data);
     }
     free(user_data);
 }
 
 void
 cib_action_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     if (rc < pcmk_ok) {
         crm_err("Update %d FAILED: %s", call_id, pcmk_strerror(rc));
     }
 }
 
 void
 cib_failcount_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     if (rc < pcmk_ok) {
         crm_err("Update %d FAILED: %s", call_id, pcmk_strerror(rc));
     }
 }
 
 gboolean
 action_timer_callback(gpointer data)
 {
     crm_action_timer_t *timer = NULL;
 
     CRM_CHECK(data != NULL, return FALSE);
 
     timer = (crm_action_timer_t *) data;
     stop_te_timer(timer);
 
     crm_warn("Timer popped (timeout=%d, abort_level=%d, complete=%s)",
              timer->timeout,
              transition_graph->abort_priority, transition_graph->complete ? "true" : "false");
 
     CRM_CHECK(timer->action != NULL, return FALSE);
 
     if (transition_graph->complete) {
         crm_warn("Ignoring timeout while not in transition");
 
     } else if (timer->reason == timeout_action_warn) {
         print_action(LOG_WARNING, "Action missed its timeout: ", timer->action);
 
         /* Don't check the FSA state
          *
          * We might also be in S_INTEGRATION or some other state waiting for this
          * action so we can close the transition and continue
          */
 
     } else {
         /* fail the action */
         gboolean send_update = TRUE;
         const char *task = crm_element_value(timer->action->xml, XML_LRM_ATTR_TASK);
 
         print_action(LOG_ERR, "Aborting transition, action lost: ", timer->action);
 
         timer->action->failed = TRUE;
         timer->action->confirmed = TRUE;
         abort_transition(INFINITY, tg_restart, "Action lost", NULL);
 
         update_graph(transition_graph, timer->action);
         trigger_graph();
 
         if (timer->action->type != action_type_rsc) {
             send_update = FALSE;
         } else if (safe_str_eq(task, "cancel")) {
             /* we dont need to update the CIB with these */
             send_update = FALSE;
         }
 
         if (send_update) {
             /* cib_action_update(timer->action, PCMK_LRM_OP_PENDING, PCMK_EXECRA_STATUS_UNKNOWN); */
             cib_action_update(timer->action, PCMK_LRM_OP_TIMEOUT, PCMK_EXECRA_UNKNOWN_ERROR);
         }
     }
 
     return FALSE;
 }
diff --git a/crmd/utils.c b/crmd/utils.c
index ba0642a236..2b98f2993a 100644
--- a/crmd/utils.c
+++ b/crmd/utils.c
@@ -1,1185 +1,1184 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <signal.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/attrd.h>
 
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crm/cluster.h>
 
 #include <crmd_fsa.h>
 #include <crmd_utils.h>
 #include <crmd_messages.h>
 
 /*	A_DC_TIMER_STOP, A_DC_TIMER_START,
  *	A_FINALIZE_TIMER_STOP, A_FINALIZE_TIMER_START
  *	A_INTEGRATE_TIMER_STOP, A_INTEGRATE_TIMER_START
  */
 void
 do_timer_control(long long action,
                  enum crmd_fsa_cause cause,
                  enum crmd_fsa_state cur_state,
                  enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     gboolean timer_op_ok = TRUE;
 
     if (action & A_DC_TIMER_STOP) {
         timer_op_ok = crm_timer_stop(election_trigger);
 
     } else if (action & A_FINALIZE_TIMER_STOP) {
         timer_op_ok = crm_timer_stop(finalization_timer);
 
     } else if (action & A_INTEGRATE_TIMER_STOP) {
         timer_op_ok = crm_timer_stop(integration_timer);
 
 /* 	} else if(action & A_ELECTION_TIMEOUT_STOP) { */
 /* 		timer_op_ok = crm_timer_stop(election_timeout); */
     }
 
     /* dont start a timer that wasnt already running */
     if (action & A_DC_TIMER_START && timer_op_ok) {
         crm_timer_start(election_trigger);
         if (AM_I_DC) {
             /* there can be only one */
             register_fsa_input(cause, I_ELECTION, NULL);
         }
 
     } else if (action & A_FINALIZE_TIMER_START) {
         crm_timer_start(finalization_timer);
 
     } else if (action & A_INTEGRATE_TIMER_START) {
         crm_timer_start(integration_timer);
 
 /* 	} else if(action & A_ELECTION_TIMEOUT_START) { */
 /* 		crm_timer_start(election_timeout); */
     }
 }
 
 const char *
 get_timer_desc(fsa_timer_t * timer)
 {
     if (timer == election_trigger) {
         return "Election Trigger";
 
     } else if (timer == election_timeout) {
         return "Election Timeout";
 
     } else if (timer == shutdown_escalation_timer) {
         return "Shutdown Escalation";
 
     } else if (timer == integration_timer) {
         return "Integration Timer";
 
     } else if (timer == finalization_timer) {
         return "Finalization Timer";
 
     } else if (timer == transition_timer) {
         return "New Transition Timer";
 
     } else if (timer == wait_timer) {
         return "Wait Timer";
 
     } else if (timer == recheck_timer) {
         return "PEngine Recheck Timer";
 
     }
     return "Unknown Timer";
 }
 
 gboolean
 crm_timer_popped(gpointer data)
 {
     fsa_timer_t *timer = (fsa_timer_t *) data;
 
     if (timer == wait_timer
         || timer == recheck_timer
         || timer == transition_timer || timer == finalization_timer || timer == election_trigger) {
         crm_info("%s (%s) just popped (%dms)",
                  get_timer_desc(timer), fsa_input2string(timer->fsa_input), timer->period_ms);
         timer->counter++;
 
     } else {
         crm_err("%s (%s) just popped in state %s! (%dms)",
                 get_timer_desc(timer), fsa_input2string(timer->fsa_input),
                 fsa_state2string(fsa_state), timer->period_ms);
     }
 
     if(timer == election_trigger && election_trigger->counter > 5) {
         crm_notice("We appear to be in an election loop, something may be wrong");
         election_trigger->counter = 0;
     }
 
     if (timer->repeat == FALSE) {
         crm_timer_stop(timer);  /* make it _not_ go off again */
     }
 
     if (timer->fsa_input == I_INTEGRATED) {
         crm_info("Welcomed: %d, Integrated: %d",
                  g_hash_table_size(welcomed_nodes), g_hash_table_size(integrated_nodes));
         if (g_hash_table_size(welcomed_nodes) == 0) {
             /* If we don't even have ourself, start again */
             register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, NULL, __FUNCTION__);
 
         } else {
             register_fsa_input_before(C_TIMER_POPPED, timer->fsa_input, NULL);
         }
 
     } else if (timer == recheck_timer && fsa_state != S_IDLE) {
         crm_debug("Discarding %s event in state: %s",
                   fsa_input2string(timer->fsa_input), fsa_state2string(fsa_state));
 
     } else if (timer == finalization_timer && fsa_state != S_FINALIZE_JOIN) {
         crm_debug("Discarding %s event in state: %s",
                   fsa_input2string(timer->fsa_input), fsa_state2string(fsa_state));
 
     } else if (timer->fsa_input != I_NULL) {
         register_fsa_input(C_TIMER_POPPED, timer->fsa_input, NULL);
     }
 
     crm_trace("Triggering FSA: %s", __FUNCTION__);
     mainloop_set_trigger(fsa_source);
 
     return TRUE;
 }
 
 gboolean 
 is_timer_started(fsa_timer_t *timer)
 {
     if (timer->period_ms > 0) {
         if(transition_timer->source_id == 0) {
             return FALSE;
         } else {
             return TRUE;
         }
     }
     return FALSE;
 }
 
 gboolean
 crm_timer_start(fsa_timer_t * timer)
 {
     const char *timer_desc = get_timer_desc(timer);
 
     if (timer->source_id == 0 && timer->period_ms > 0) {
         timer->source_id = g_timeout_add(timer->period_ms, timer->callback, (void *)timer);
         CRM_ASSERT(timer->source_id != 0);
         crm_debug("Started %s (%s:%dms), src=%d",
                   timer_desc, fsa_input2string(timer->fsa_input),
                   timer->period_ms, timer->source_id);
 
     } else if (timer->period_ms < 0) {
         crm_err("Tried to start %s (%s:%dms) with a -ve period",
                 timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms);
 
     } else {
         crm_debug("%s (%s:%dms) already running: src=%d",
                   timer_desc, fsa_input2string(timer->fsa_input),
                   timer->period_ms, timer->source_id);
         return FALSE;
     }
     return TRUE;
 }
 
 gboolean
 crm_timer_stop(fsa_timer_t * timer)
 {
     const char *timer_desc = get_timer_desc(timer);
 
     if (timer == NULL) {
         crm_err("Attempted to stop NULL timer");
         return FALSE;
 
     } else if (timer->source_id != 0) {
         crm_trace("Stopping %s (%s:%dms), src=%d",
                     timer_desc, fsa_input2string(timer->fsa_input),
                     timer->period_ms, timer->source_id);
         g_source_remove(timer->source_id);
         timer->source_id = 0;
 
     } else {
         crm_trace("%s (%s:%dms) already stopped",
                     timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms);
         return FALSE;
     }
     return TRUE;
 }
 
 const char *
 fsa_input2string(enum crmd_fsa_input input)
 {
     const char *inputAsText = NULL;
 
     switch (input) {
         case I_NULL:
             inputAsText = "I_NULL";
             break;
         case I_CIB_OP:
             inputAsText = "I_CIB_OP (unused)";
             break;
         case I_CIB_UPDATE:
             inputAsText = "I_CIB_UPDATE";
             break;
         case I_DC_TIMEOUT:
             inputAsText = "I_DC_TIMEOUT";
             break;
         case I_ELECTION:
             inputAsText = "I_ELECTION";
             break;
         case I_PE_CALC:
             inputAsText = "I_PE_CALC";
             break;
         case I_RELEASE_DC:
             inputAsText = "I_RELEASE_DC";
             break;
         case I_ELECTION_DC:
             inputAsText = "I_ELECTION_DC";
             break;
         case I_ERROR:
             inputAsText = "I_ERROR";
             break;
         case I_FAIL:
             inputAsText = "I_FAIL";
             break;
         case I_INTEGRATED:
             inputAsText = "I_INTEGRATED";
             break;
         case I_FINALIZED:
             inputAsText = "I_FINALIZED";
             break;
         case I_NODE_JOIN:
             inputAsText = "I_NODE_JOIN";
             break;
         case I_JOIN_OFFER:
             inputAsText = "I_JOIN_OFFER";
             break;
         case I_JOIN_REQUEST:
             inputAsText = "I_JOIN_REQUEST";
             break;
         case I_JOIN_RESULT:
             inputAsText = "I_JOIN_RESULT";
             break;
         case I_NOT_DC:
             inputAsText = "I_NOT_DC";
             break;
         case I_RECOVERED:
             inputAsText = "I_RECOVERED";
             break;
         case I_RELEASE_FAIL:
             inputAsText = "I_RELEASE_FAIL";
             break;
         case I_RELEASE_SUCCESS:
             inputAsText = "I_RELEASE_SUCCESS";
             break;
         case I_RESTART:
             inputAsText = "I_RESTART";
             break;
         case I_PE_SUCCESS:
             inputAsText = "I_PE_SUCCESS";
             break;
         case I_ROUTER:
             inputAsText = "I_ROUTER";
             break;
         case I_SHUTDOWN:
             inputAsText = "I_SHUTDOWN";
             break;
         case I_STARTUP:
             inputAsText = "I_STARTUP";
             break;
         case I_TE_SUCCESS:
             inputAsText = "I_TE_SUCCESS";
             break;
         case I_STOP:
             inputAsText = "I_STOP";
             break;
         case I_DC_HEARTBEAT:
             inputAsText = "I_DC_HEARTBEAT";
             break;
         case I_WAIT_FOR_EVENT:
             inputAsText = "I_WAIT_FOR_EVENT";
             break;
         case I_LRM_EVENT:
             inputAsText = "I_LRM_EVENT";
             break;
         case I_PENDING:
             inputAsText = "I_PENDING";
             break;
         case I_HALT:
             inputAsText = "I_HALT";
             break;
         case I_TERMINATE:
             inputAsText = "I_TERMINATE";
             break;
         case I_ILLEGAL:
             inputAsText = "I_ILLEGAL";
             break;
     }
 
     if (inputAsText == NULL) {
         crm_err("Input %d is unknown", input);
         inputAsText = "<UNKNOWN_INPUT>";
     }
 
     return inputAsText;
 }
 
 const char *
 fsa_state2string(enum crmd_fsa_state state)
 {
     const char *stateAsText = NULL;
 
     switch (state) {
         case S_IDLE:
             stateAsText = "S_IDLE";
             break;
         case S_ELECTION:
             stateAsText = "S_ELECTION";
             break;
         case S_INTEGRATION:
             stateAsText = "S_INTEGRATION";
             break;
         case S_FINALIZE_JOIN:
             stateAsText = "S_FINALIZE_JOIN";
             break;
         case S_NOT_DC:
             stateAsText = "S_NOT_DC";
             break;
         case S_POLICY_ENGINE:
             stateAsText = "S_POLICY_ENGINE";
             break;
         case S_RECOVERY:
             stateAsText = "S_RECOVERY";
             break;
         case S_RELEASE_DC:
             stateAsText = "S_RELEASE_DC";
             break;
         case S_PENDING:
             stateAsText = "S_PENDING";
             break;
         case S_STOPPING:
             stateAsText = "S_STOPPING";
             break;
         case S_TERMINATE:
             stateAsText = "S_TERMINATE";
             break;
         case S_TRANSITION_ENGINE:
             stateAsText = "S_TRANSITION_ENGINE";
             break;
         case S_STARTING:
             stateAsText = "S_STARTING";
             break;
         case S_HALT:
             stateAsText = "S_HALT";
             break;
         case S_ILLEGAL:
             stateAsText = "S_ILLEGAL";
             break;
     }
 
     if (stateAsText == NULL) {
         crm_err("State %d is unknown", state);
         stateAsText = "<UNKNOWN_STATE>";
     }
 
     return stateAsText;
 }
 
 const char *
 fsa_cause2string(enum crmd_fsa_cause cause)
 {
     const char *causeAsText = NULL;
 
     switch (cause) {
         case C_UNKNOWN:
             causeAsText = "C_UNKNOWN";
             break;
         case C_STARTUP:
             causeAsText = "C_STARTUP";
             break;
         case C_IPC_MESSAGE:
             causeAsText = "C_IPC_MESSAGE";
             break;
         case C_HA_MESSAGE:
             causeAsText = "C_HA_MESSAGE";
             break;
         case C_CCM_CALLBACK:
             causeAsText = "C_CCM_CALLBACK";
             break;
         case C_TIMER_POPPED:
             causeAsText = "C_TIMER_POPPED";
             break;
         case C_SHUTDOWN:
             causeAsText = "C_SHUTDOWN";
             break;
         case C_HEARTBEAT_FAILED:
             causeAsText = "C_HEARTBEAT_FAILED";
             break;
         case C_SUBSYSTEM_CONNECT:
             causeAsText = "C_SUBSYSTEM_CONNECT";
             break;
         case C_LRM_OP_CALLBACK:
             causeAsText = "C_LRM_OP_CALLBACK";
             break;
         case C_LRM_MONITOR_CALLBACK:
             causeAsText = "C_LRM_MONITOR_CALLBACK";
             break;
         case C_CRMD_STATUS_CALLBACK:
             causeAsText = "C_CRMD_STATUS_CALLBACK";
             break;
         case C_HA_DISCONNECT:
             causeAsText = "C_HA_DISCONNECT";
             break;
         case C_FSA_INTERNAL:
             causeAsText = "C_FSA_INTERNAL";
             break;
         case C_ILLEGAL:
             causeAsText = "C_ILLEGAL";
             break;
     }
 
     if (causeAsText == NULL) {
         crm_err("Cause %d is unknown", cause);
         causeAsText = "<UNKNOWN_CAUSE>";
     }
 
     return causeAsText;
 }
 
 const char *
 fsa_action2string(long long action)
 {
     const char *actionAsText = NULL;
 
     switch (action) {
 
         case A_NOTHING:
             actionAsText = "A_NOTHING";
             break;
         case A_ELECTION_START:
             actionAsText = "A_ELECTION_START";
             break;
         case A_DC_JOIN_FINAL:
             actionAsText = "A_DC_JOIN_FINAL";
             break;
         case A_READCONFIG:
             actionAsText = "A_READCONFIG";
             break;
         case O_RELEASE:
             actionAsText = "O_RELEASE";
             break;
         case A_STARTUP:
             actionAsText = "A_STARTUP";
             break;
         case A_STARTED:
             actionAsText = "A_STARTED";
             break;
         case A_HA_CONNECT:
             actionAsText = "A_HA_CONNECT";
             break;
         case A_HA_DISCONNECT:
             actionAsText = "A_HA_DISCONNECT";
             break;
         case A_LRM_CONNECT:
             actionAsText = "A_LRM_CONNECT";
             break;
         case A_LRM_EVENT:
             actionAsText = "A_LRM_EVENT";
             break;
         case A_LRM_INVOKE:
             actionAsText = "A_LRM_INVOKE";
             break;
         case A_LRM_DISCONNECT:
             actionAsText = "A_LRM_DISCONNECT";
             break;
         case O_LRM_RECONNECT:
             actionAsText = "O_LRM_RECONNECT";
             break;
         case A_CL_JOIN_QUERY:
             actionAsText = "A_CL_JOIN_QUERY";
             break;
         case A_DC_TIMER_STOP:
             actionAsText = "A_DC_TIMER_STOP";
             break;
         case A_DC_TIMER_START:
             actionAsText = "A_DC_TIMER_START";
             break;
         case A_INTEGRATE_TIMER_START:
             actionAsText = "A_INTEGRATE_TIMER_START";
             break;
         case A_INTEGRATE_TIMER_STOP:
             actionAsText = "A_INTEGRATE_TIMER_STOP";
             break;
         case A_FINALIZE_TIMER_START:
             actionAsText = "A_FINALIZE_TIMER_START";
             break;
         case A_FINALIZE_TIMER_STOP:
             actionAsText = "A_FINALIZE_TIMER_STOP";
             break;
         case A_ELECTION_COUNT:
             actionAsText = "A_ELECTION_COUNT";
             break;
         case A_ELECTION_VOTE:
             actionAsText = "A_ELECTION_VOTE";
             break;
         case A_ELECTION_CHECK:
             actionAsText = "A_ELECTION_CHECK";
             break;
         case A_CL_JOIN_ANNOUNCE:
             actionAsText = "A_CL_JOIN_ANNOUNCE";
             break;
         case A_CL_JOIN_REQUEST:
             actionAsText = "A_CL_JOIN_REQUEST";
             break;
         case A_CL_JOIN_RESULT:
             actionAsText = "A_CL_JOIN_RESULT";
             break;
         case A_DC_JOIN_OFFER_ALL:
             actionAsText = "A_DC_JOIN_OFFER_ALL";
             break;
         case A_DC_JOIN_OFFER_ONE:
             actionAsText = "A_DC_JOIN_OFFER_ONE";
             break;
         case A_DC_JOIN_PROCESS_REQ:
             actionAsText = "A_DC_JOIN_PROCESS_REQ";
             break;
         case A_DC_JOIN_PROCESS_ACK:
             actionAsText = "A_DC_JOIN_PROCESS_ACK";
             break;
         case A_DC_JOIN_FINALIZE:
             actionAsText = "A_DC_JOIN_FINALIZE";
             break;
         case A_MSG_PROCESS:
             actionAsText = "A_MSG_PROCESS";
             break;
         case A_MSG_ROUTE:
             actionAsText = "A_MSG_ROUTE";
             break;
         case A_RECOVER:
             actionAsText = "A_RECOVER";
             break;
         case A_DC_RELEASE:
             actionAsText = "A_DC_RELEASE";
             break;
         case A_DC_RELEASED:
             actionAsText = "A_DC_RELEASED";
             break;
         case A_DC_TAKEOVER:
             actionAsText = "A_DC_TAKEOVER";
             break;
         case A_SHUTDOWN:
             actionAsText = "A_SHUTDOWN";
             break;
         case A_SHUTDOWN_REQ:
             actionAsText = "A_SHUTDOWN_REQ";
             break;
         case A_STOP:
             actionAsText = "A_STOP  ";
             break;
         case A_EXIT_0:
             actionAsText = "A_EXIT_0";
             break;
         case A_EXIT_1:
             actionAsText = "A_EXIT_1";
             break;
         case A_CCM_CONNECT:
             actionAsText = "A_CCM_CONNECT";
             break;
         case A_CCM_DISCONNECT:
             actionAsText = "A_CCM_DISCONNECT";
             break;
         case O_CIB_RESTART:
             actionAsText = "O_CIB_RESTART";
             break;
         case A_CIB_START:
             actionAsText = "A_CIB_START";
             break;
         case A_CIB_STOP:
             actionAsText = "A_CIB_STOP";
             break;
         case A_TE_INVOKE:
             actionAsText = "A_TE_INVOKE";
             break;
         case O_TE_RESTART:
             actionAsText = "O_TE_RESTART";
             break;
         case A_TE_START:
             actionAsText = "A_TE_START";
             break;
         case A_TE_STOP:
             actionAsText = "A_TE_STOP";
             break;
         case A_TE_HALT:
             actionAsText = "A_TE_HALT";
             break;
         case A_TE_CANCEL:
             actionAsText = "A_TE_CANCEL";
             break;
         case A_PE_INVOKE:
             actionAsText = "A_PE_INVOKE";
             break;
         case O_PE_RESTART:
             actionAsText = "O_PE_RESTART";
             break;
         case A_PE_START:
             actionAsText = "A_PE_START";
             break;
         case A_PE_STOP:
             actionAsText = "A_PE_STOP";
             break;
         case A_NODE_BLOCK:
             actionAsText = "A_NODE_BLOCK";
             break;
         case A_UPDATE_NODESTATUS:
             actionAsText = "A_UPDATE_NODESTATUS";
             break;
         case A_LOG:
             actionAsText = "A_LOG   ";
             break;
         case A_ERROR:
             actionAsText = "A_ERROR ";
             break;
         case A_WARN:
             actionAsText = "A_WARN  ";
             break;
             /* Composite actions */
         case A_DC_TIMER_START | A_CL_JOIN_QUERY:
             actionAsText = "A_DC_TIMER_START|A_CL_JOIN_QUERY";
             break;
     }
 
     if (actionAsText == NULL) {
         crm_err("Action %.16llx is unknown", action);
         actionAsText = "<UNKNOWN_ACTION>";
     }
 
     return actionAsText;
 }
 
 void
 fsa_dump_inputs(int log_level, const char *text, long long input_register)
 {
     if (input_register == A_NOTHING) {
         return;
     }
     if (text == NULL) {
         text = "Input register contents:";
     }
 
     if (is_set(input_register, R_THE_DC)) {
         crm_trace( "%s %.16llx (R_THE_DC)", text, R_THE_DC);
     }
     if (is_set(input_register, R_STARTING)) {
         crm_trace( "%s %.16llx (R_STARTING)", text, R_STARTING);
     }
     if (is_set(input_register, R_SHUTDOWN)) {
         crm_trace( "%s %.16llx (R_SHUTDOWN)", text, R_SHUTDOWN);
     }
     if (is_set(input_register, R_STAYDOWN)) {
         crm_trace( "%s %.16llx (R_STAYDOWN)", text, R_STAYDOWN);
     }
     if (is_set(input_register, R_JOIN_OK)) {
         crm_trace( "%s %.16llx (R_JOIN_OK)", text, R_JOIN_OK);
     }
     if (is_set(input_register, R_READ_CONFIG)) {
         crm_trace( "%s %.16llx (R_READ_CONFIG)", text, R_READ_CONFIG);
     }
     if (is_set(input_register, R_INVOKE_PE)) {
         crm_trace( "%s %.16llx (R_INVOKE_PE)", text, R_INVOKE_PE);
     }
     if (is_set(input_register, R_CIB_CONNECTED)) {
         crm_trace( "%s %.16llx (R_CIB_CONNECTED)", text, R_CIB_CONNECTED);
     }
     if (is_set(input_register, R_PE_CONNECTED)) {
         crm_trace( "%s %.16llx (R_PE_CONNECTED)", text, R_PE_CONNECTED);
     }
     if (is_set(input_register, R_TE_CONNECTED)) {
         crm_trace( "%s %.16llx (R_TE_CONNECTED)", text, R_TE_CONNECTED);
     }
     if (is_set(input_register, R_LRM_CONNECTED)) {
         crm_trace( "%s %.16llx (R_LRM_CONNECTED)", text, R_LRM_CONNECTED);
     }
     if (is_set(input_register, R_CIB_REQUIRED)) {
         crm_trace( "%s %.16llx (R_CIB_REQUIRED)", text, R_CIB_REQUIRED);
     }
     if (is_set(input_register, R_PE_REQUIRED)) {
         crm_trace( "%s %.16llx (R_PE_REQUIRED)", text, R_PE_REQUIRED);
     }
     if (is_set(input_register, R_TE_REQUIRED)) {
         crm_trace( "%s %.16llx (R_TE_REQUIRED)", text, R_TE_REQUIRED);
     }
     if (is_set(input_register, R_REQ_PEND)) {
         crm_trace( "%s %.16llx (R_REQ_PEND)", text, R_REQ_PEND);
     }
     if (is_set(input_register, R_PE_PEND)) {
         crm_trace( "%s %.16llx (R_PE_PEND)", text, R_PE_PEND);
     }
     if (is_set(input_register, R_TE_PEND)) {
         crm_trace( "%s %.16llx (R_TE_PEND)", text, R_TE_PEND);
     }
     if (is_set(input_register, R_RESP_PEND)) {
         crm_trace( "%s %.16llx (R_RESP_PEND)", text, R_RESP_PEND);
     }
     if (is_set(input_register, R_CIB_DONE)) {
         crm_trace( "%s %.16llx (R_CIB_DONE)", text, R_CIB_DONE);
     }
     if (is_set(input_register, R_HAVE_CIB)) {
         crm_trace( "%s %.16llx (R_HAVE_CIB)", text, R_HAVE_CIB);
     }
     if (is_set(input_register, R_CIB_ASKED)) {
         crm_trace( "%s %.16llx (R_CIB_ASKED)", text, R_CIB_ASKED);
     }
     if (is_set(input_register, R_MEMBERSHIP)) {
         crm_trace( "%s %.16llx (R_MEMBERSHIP)", text, R_MEMBERSHIP);
     }
     if (is_set(input_register, R_PEER_DATA)) {
         crm_trace( "%s %.16llx (R_PEER_DATA)", text, R_PEER_DATA);
     }
     if (is_set(input_register, R_IN_RECOVERY)) {
         crm_trace( "%s %.16llx (R_IN_RECOVERY)", text, R_IN_RECOVERY);
     }
 }
 
 void
 fsa_dump_actions(long long action, const char *text)
 {
     if (is_set(action, A_READCONFIG)) {
         crm_trace( "Action %.16llx (A_READCONFIG) %s", A_READCONFIG, text);
     }
     if (is_set(action, A_STARTUP)) {
         crm_trace( "Action %.16llx (A_STARTUP) %s", A_STARTUP, text);
     }
     if (is_set(action, A_STARTED)) {
         crm_trace( "Action %.16llx (A_STARTED) %s", A_STARTED, text);
     }
     if (is_set(action, A_HA_CONNECT)) {
         crm_trace( "Action %.16llx (A_CONNECT) %s", A_HA_CONNECT, text);
     }
     if (is_set(action, A_HA_DISCONNECT)) {
         crm_trace( "Action %.16llx (A_DISCONNECT) %s", A_HA_DISCONNECT, text);
     }
     if (is_set(action, A_LRM_CONNECT)) {
         crm_trace( "Action %.16llx (A_LRM_CONNECT) %s", A_LRM_CONNECT, text);
     }
     if (is_set(action, A_LRM_EVENT)) {
         crm_trace( "Action %.16llx (A_LRM_EVENT) %s", A_LRM_EVENT, text);
     }
     if (is_set(action, A_LRM_INVOKE)) {
         crm_trace( "Action %.16llx (A_LRM_INVOKE) %s", A_LRM_INVOKE, text);
     }
     if (is_set(action, A_LRM_DISCONNECT)) {
         crm_trace(
                             "Action %.16llx (A_LRM_DISCONNECT) %s", A_LRM_DISCONNECT, text);
     }
     if (is_set(action, A_DC_TIMER_STOP)) {
         crm_trace(
                             "Action %.16llx (A_DC_TIMER_STOP) %s", A_DC_TIMER_STOP, text);
     }
     if (is_set(action, A_DC_TIMER_START)) {
         crm_trace(
                             "Action %.16llx (A_DC_TIMER_START) %s", A_DC_TIMER_START, text);
     }
     if (is_set(action, A_INTEGRATE_TIMER_START)) {
         crm_trace(
                             "Action %.16llx (A_INTEGRATE_TIMER_START) %s",
                             A_INTEGRATE_TIMER_START, text);
     }
     if (is_set(action, A_INTEGRATE_TIMER_STOP)) {
         crm_trace(
                             "Action %.16llx (A_INTEGRATE_TIMER_STOP) %s",
                             A_INTEGRATE_TIMER_STOP, text);
     }
     if (is_set(action, A_FINALIZE_TIMER_START)) {
         crm_trace(
                             "Action %.16llx (A_FINALIZE_TIMER_START) %s",
                             A_FINALIZE_TIMER_START, text);
     }
     if (is_set(action, A_FINALIZE_TIMER_STOP)) {
         crm_trace(
                             "Action %.16llx (A_FINALIZE_TIMER_STOP) %s",
                             A_FINALIZE_TIMER_STOP, text);
     }
     if (is_set(action, A_ELECTION_COUNT)) {
         crm_trace(
                             "Action %.16llx (A_ELECTION_COUNT) %s", A_ELECTION_COUNT, text);
     }
     if (is_set(action, A_ELECTION_VOTE)) {
         crm_trace(
                             "Action %.16llx (A_ELECTION_VOTE) %s", A_ELECTION_VOTE, text);
     }
     if (is_set(action, A_ELECTION_CHECK)) {
         crm_trace(
                             "Action %.16llx (A_ELECTION_CHECK) %s", A_ELECTION_CHECK, text);
     }
     if (is_set(action, A_CL_JOIN_ANNOUNCE)) {
         crm_trace(
                             "Action %.16llx (A_CL_JOIN_ANNOUNCE) %s", A_CL_JOIN_ANNOUNCE, text);
     }
     if (is_set(action, A_CL_JOIN_REQUEST)) {
         crm_trace(
                             "Action %.16llx (A_CL_JOIN_REQUEST) %s", A_CL_JOIN_REQUEST, text);
     }
     if (is_set(action, A_CL_JOIN_RESULT)) {
         crm_trace(
                             "Action %.16llx (A_CL_JOIN_RESULT) %s", A_CL_JOIN_RESULT, text);
     }
     if (is_set(action, A_DC_JOIN_OFFER_ALL)) {
         crm_trace(
                             "Action %.16llx (A_DC_JOIN_OFFER_ALL) %s", A_DC_JOIN_OFFER_ALL, text);
     }
     if (is_set(action, A_DC_JOIN_OFFER_ONE)) {
         crm_trace(
                             "Action %.16llx (A_DC_JOIN_OFFER_ONE) %s", A_DC_JOIN_OFFER_ONE, text);
     }
     if (is_set(action, A_DC_JOIN_PROCESS_REQ)) {
         crm_trace(
                             "Action %.16llx (A_DC_JOIN_PROCESS_REQ) %s",
                             A_DC_JOIN_PROCESS_REQ, text);
     }
     if (is_set(action, A_DC_JOIN_PROCESS_ACK)) {
         crm_trace(
                             "Action %.16llx (A_DC_JOIN_PROCESS_ACK) %s",
                             A_DC_JOIN_PROCESS_ACK, text);
     }
     if (is_set(action, A_DC_JOIN_FINALIZE)) {
         crm_trace(
                             "Action %.16llx (A_DC_JOIN_FINALIZE) %s", A_DC_JOIN_FINALIZE, text);
     }
     if (is_set(action, A_MSG_PROCESS)) {
         crm_trace( "Action %.16llx (A_MSG_PROCESS) %s", A_MSG_PROCESS, text);
     }
     if (is_set(action, A_MSG_ROUTE)) {
         crm_trace( "Action %.16llx (A_MSG_ROUTE) %s", A_MSG_ROUTE, text);
     }
     if (is_set(action, A_RECOVER)) {
         crm_trace( "Action %.16llx (A_RECOVER) %s", A_RECOVER, text);
     }
     if (is_set(action, A_DC_RELEASE)) {
         crm_trace( "Action %.16llx (A_DC_RELEASE) %s", A_DC_RELEASE, text);
     }
     if (is_set(action, A_DC_RELEASED)) {
         crm_trace( "Action %.16llx (A_DC_RELEASED) %s", A_DC_RELEASED, text);
     }
     if (is_set(action, A_DC_TAKEOVER)) {
         crm_trace( "Action %.16llx (A_DC_TAKEOVER) %s", A_DC_TAKEOVER, text);
     }
     if (is_set(action, A_SHUTDOWN)) {
         crm_trace( "Action %.16llx (A_SHUTDOWN) %s", A_SHUTDOWN, text);
     }
     if (is_set(action, A_SHUTDOWN_REQ)) {
         crm_trace( "Action %.16llx (A_SHUTDOWN_REQ) %s", A_SHUTDOWN_REQ, text);
     }
     if (is_set(action, A_STOP)) {
         crm_trace( "Action %.16llx (A_STOP  ) %s", A_STOP, text);
     }
     if (is_set(action, A_EXIT_0)) {
         crm_trace( "Action %.16llx (A_EXIT_0) %s", A_EXIT_0, text);
     }
     if (is_set(action, A_EXIT_1)) {
         crm_trace( "Action %.16llx (A_EXIT_1) %s", A_EXIT_1, text);
     }
     if (is_set(action, A_CCM_CONNECT)) {
         crm_trace( "Action %.16llx (A_CCM_CONNECT) %s", A_CCM_CONNECT, text);
     }
     if (is_set(action, A_CCM_DISCONNECT)) {
         crm_trace(
                             "Action %.16llx (A_CCM_DISCONNECT) %s", A_CCM_DISCONNECT, text);
     }
     if (is_set(action, A_CIB_START)) {
         crm_trace( "Action %.16llx (A_CIB_START) %s", A_CIB_START, text);
     }
     if (is_set(action, A_CIB_STOP)) {
         crm_trace( "Action %.16llx (A_CIB_STOP) %s", A_CIB_STOP, text);
     }
     if (is_set(action, A_TE_INVOKE)) {
         crm_trace( "Action %.16llx (A_TE_INVOKE) %s", A_TE_INVOKE, text);
     }
     if (is_set(action, A_TE_START)) {
         crm_trace( "Action %.16llx (A_TE_START) %s", A_TE_START, text);
     }
     if (is_set(action, A_TE_STOP)) {
         crm_trace( "Action %.16llx (A_TE_STOP) %s", A_TE_STOP, text);
     }
     if (is_set(action, A_TE_CANCEL)) {
         crm_trace( "Action %.16llx (A_TE_CANCEL) %s", A_TE_CANCEL, text);
     }
     if (is_set(action, A_PE_INVOKE)) {
         crm_trace( "Action %.16llx (A_PE_INVOKE) %s", A_PE_INVOKE, text);
     }
     if (is_set(action, A_PE_START)) {
         crm_trace( "Action %.16llx (A_PE_START) %s", A_PE_START, text);
     }
     if (is_set(action, A_PE_STOP)) {
         crm_trace( "Action %.16llx (A_PE_STOP) %s", A_PE_STOP, text);
     }
     if (is_set(action, A_NODE_BLOCK)) {
         crm_trace( "Action %.16llx (A_NODE_BLOCK) %s", A_NODE_BLOCK, text);
     }
     if (is_set(action, A_UPDATE_NODESTATUS)) {
         crm_trace(
                             "Action %.16llx (A_UPDATE_NODESTATUS) %s", A_UPDATE_NODESTATUS, text);
     }
     if (is_set(action, A_LOG)) {
         crm_trace( "Action %.16llx (A_LOG   ) %s", A_LOG, text);
     }
     if (is_set(action, A_ERROR)) {
         crm_trace( "Action %.16llx (A_ERROR ) %s", A_ERROR, text);
     }
     if (is_set(action, A_WARN)) {
         crm_trace( "Action %.16llx (A_WARN  ) %s", A_WARN, text);
     }
 }
 
 void
 create_node_entry(const char *uuid, const char *uname, const char *type)
 {
 
     /* make sure a node entry exists for the new node
      *
      * this will add anyone except the first ever node in the cluster
      *   since it will also be the DC which doesnt go through the
      *   join process (with itself).  We can include a special case
      *   later if desired.
      */
     xmlNode *tmp1 = create_xml_node(NULL, XML_CIB_TAG_NODE);
 
     crm_trace("Creating node entry for %s", uname);
     set_uuid(tmp1, XML_ATTR_UUID, uname);
 
     crm_xml_add(tmp1, XML_ATTR_UNAME, uname);
     crm_xml_add(tmp1, XML_ATTR_TYPE, type);
 
     fsa_cib_anon_update(XML_CIB_TAG_NODES, tmp1,
                         cib_scope_local | cib_quorum_override | cib_can_create);
 
     free_xml(tmp1);
 
 }
 
 xmlNode *
-create_node_state(const char *uname, const char *ha_state, const char *ccm_state,
-                  const char *crmd_state, const char *join_state, const char *exp_state,
+create_node_state(const char *uname, const char *in_cluster,
+                  const char *is_peer, const char *join_state, const char *exp_state,
                   gboolean clear_shutdown, const char *src)
 {
     xmlNode *node_state = create_xml_node(NULL, XML_CIB_TAG_STATE);
 
     crm_trace("%s Creating node state entry for %s", src, uname);
     set_uuid(node_state, XML_ATTR_UUID, uname);
 
     if (crm_element_value(node_state, XML_ATTR_UUID) == NULL) {
         crm_debug("Node %s is not a cluster member", uname);
         free_xml(node_state);
         return NULL;
     }
 
     crm_xml_add(node_state, XML_ATTR_UNAME, uname);
-    crm_xml_add(node_state, XML_CIB_ATTR_HASTATE, ha_state);
-    crm_xml_add(node_state, XML_NODE_IN_CLUSTER, ccm_state);
-    crm_xml_add(node_state, XML_NODE_IS_PEER, crmd_state);
+    crm_xml_add(node_state, XML_NODE_IN_CLUSTER, in_cluster);
+    crm_xml_add(node_state, XML_NODE_IS_PEER, is_peer);
     crm_xml_add(node_state, XML_NODE_JOIN_STATE, join_state);
     crm_xml_add(node_state, XML_NODE_EXPECTED, exp_state);
     crm_xml_add(node_state, XML_ATTR_ORIGIN, src);
 
     if (clear_shutdown) {
         crm_xml_add(node_state, XML_CIB_ATTR_SHUTDOWN, "0");
     }
 
     crm_log_xml_trace(node_state, "created");
 
     return node_state;
 }
 
 extern GHashTable *ipc_clients;
 
 void
 process_client_disconnect(crmd_client_t * curr_client)
 {
     struct crm_subsystem_s *the_subsystem = NULL;
 
     CRM_CHECK(curr_client != NULL, return);
     crm_trace("received HUP from %s", curr_client->table_key);
 
     if (curr_client->sub_sys == NULL) {
         crm_trace("Client hadn't registered with us yet");
 
     } else if (strcasecmp(CRM_SYSTEM_PENGINE, curr_client->sub_sys) == 0) {
         the_subsystem = pe_subsystem;
 
     } else if (strcasecmp(CRM_SYSTEM_TENGINE, curr_client->sub_sys) == 0) {
         the_subsystem = te_subsystem;
 
     } else if (strcasecmp(CRM_SYSTEM_CIB, curr_client->sub_sys) == 0) {
         the_subsystem = cib_subsystem;
     }
 
     if (the_subsystem != NULL) {
         the_subsystem->source = NULL;
         the_subsystem->client = NULL;
         crm_info("Received HUP from %s:[%d]", the_subsystem->name, the_subsystem->pid);
 
     } else {
         /* else that was a transient client */
         crm_trace("Received HUP from transient client");
     }
 
     if (curr_client->table_key != NULL) {
         /*
          * Key is destroyed below as:
          *      curr_client->table_key
          * Value is cleaned up by:
          *      crmd_ipc_connection_destroy
          *   which will also call:
          *      G_main_del_IPC_Channel
          */
         g_hash_table_remove(ipc_clients, curr_client->table_key);
     }
 }
 
 gboolean
 update_dc(xmlNode * msg)
 {
     char *last_dc = fsa_our_dc;
     const char *dc_version = NULL;
     const char *welcome_from = NULL;
 
     if (msg != NULL) {
         gboolean invalid = FALSE;
 
         dc_version = crm_element_value(msg, F_CRM_VERSION);
         welcome_from = crm_element_value(msg, F_CRM_HOST_FROM);
 
         CRM_CHECK(dc_version != NULL, return FALSE);
         CRM_CHECK(welcome_from != NULL, return FALSE);
 
         if (AM_I_DC && safe_str_neq(welcome_from, fsa_our_uname)) {
             invalid = TRUE;
 
         } else if (fsa_our_dc && safe_str_neq(welcome_from, fsa_our_dc)) {
             invalid = TRUE;
         }
 
         if (invalid) {
             CRM_CHECK(fsa_our_dc != NULL, crm_err("We have no DC"));
             if (AM_I_DC) {
                 crm_err("Not updating DC to %s (%s): we are also a DC", welcome_from, dc_version);
             } else {
                 crm_warn("New DC %s is not %s", welcome_from, fsa_our_dc);
             }
 
             register_fsa_action(A_CL_JOIN_QUERY | A_DC_TIMER_START);
             return FALSE;
         }
     }
 
     free(fsa_our_dc_version);
     fsa_our_dc_version = NULL;
 
     fsa_our_dc = NULL;          /* Free'd as last_dc */
 
     if (welcome_from != NULL) {
         fsa_our_dc = strdup(welcome_from);
     }
     if (dc_version != NULL) {
         fsa_our_dc_version = strdup(dc_version);
     }
 
     if (safe_str_eq(fsa_our_dc, last_dc)) {
         /* do nothing */
 
     } else if (fsa_our_dc != NULL) {
         crm_info("Set DC to %s (%s)", crm_str(fsa_our_dc), crm_str(fsa_our_dc_version));
 
     } else if (last_dc != NULL) {
         crm_debug("Unset DC. Was %s", crm_str(last_dc));
     }
 
     free(last_dc);
     return TRUE;
 }
 
 #define STATUS_PATH_MAX 512
 static void
 erase_xpath_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     char *xpath = user_data;
 
     do_crm_log_unlikely(rc == 0 ? LOG_DEBUG : LOG_NOTICE,
                "Deletion of \"%s\": %s (rc=%d)", xpath, pcmk_strerror(rc), rc);
     free(xpath);
 }
 
 void
 erase_status_tag(const char *uname, const char *tag, int options)
 {
     int rc = pcmk_ok;
     char xpath[STATUS_PATH_MAX];
     int cib_opts = cib_quorum_override | cib_xpath | options;
 
     if (fsa_cib_conn && uname) {
         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", uname, tag);
         crm_info("Deleting xpath: %s", xpath);
         rc = fsa_cib_conn->cmds->delete(fsa_cib_conn, xpath, NULL, cib_opts);
         add_cib_op_callback(fsa_cib_conn, rc, FALSE, strdup(xpath), erase_xpath_callback);
     }
 }
 
 crm_ipc_t *attrd_ipc = NULL;
 
 void
 update_attrd(const char *host, const char *name, const char *value, const char *user_name)
 {
     gboolean rc;
     int max = 5;
 
     if(attrd_ipc == NULL) {
         attrd_ipc = crm_ipc_new(T_ATTRD, 0);
     }
 
     do {
         if (crm_ipc_connected(attrd_ipc) == FALSE) {
             crm_info("Connecting to cluster... %d retries remaining", max);
             crm_ipc_connect(attrd_ipc);
         }
 
         rc = attrd_update_delegate(attrd_ipc, 'U', host, name, value, XML_CIB_TAG_STATUS, NULL, NULL, user_name);
         if (rc > 0) {
             break;
         }
 
         crm_ipc_close(attrd_ipc);
         sleep(5-max);
 
     } while(max--);
     
     if (rc == FALSE) {
         crm_err("Could not send %s %s %s (%d)", T_ATTRD, name ? "update" : "refresh",
                 name?name:"", is_set(fsa_input_register, R_SHUTDOWN));
 
         if(is_set(fsa_input_register, R_SHUTDOWN)) {
             register_fsa_input(C_FSA_INTERNAL, I_FAIL, NULL);
         }
     }
 }
diff --git a/include/crm/compatibility.h b/include/crm/compatibility.h
index 3f7cc2cbca..283f981bf0 100644
--- a/include/crm/compatibility.h
+++ b/include/crm/compatibility.h
@@ -1,304 +1,305 @@
 /* 
  * Copyright (C) 2012 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 #ifndef CRM_COMPATIBILITY__H
 #  define CRM_COMPATIBILITY__H
 #  define LOG_DEBUG_2  LOG_TRACE
 #  define LOG_DEBUG_3  LOG_TRACE
 #  define LOG_DEBUG_4  LOG_TRACE
 #  define LOG_DEBUG_5  LOG_TRACE
 #  define LOG_DEBUG_6  LOG_TRACE
 
+#  define XML_CIB_ATTR_HASTATE    	"ha"
 #  define XML_CIB_ATTR_JOINSTATE       XML_NODE_JOIN_STATE
 #  define XML_CIB_ATTR_EXPSTATE        XML_NODE_EXPECTED
 #  define XML_CIB_ATTR_INCCM           XML_NODE_IN_CLUSTER
 #  define XML_CIB_ATTR_CRMDSTATE       XML_NODE_IS_PEER
 
 enum cib_errors {
     cib_ok			=  pcmk_ok,
     cib_operation		= -EINVAL,
     cib_create_msg		= -EPROTO,
     cib_not_connected           = -ENOTCONN,
     cib_not_authorized          = -EACCES,
     cib_send_failed		= -ECOMM,
     cib_reply_failed            = -ENOMSG,
     cib_return_code		= -EPROTO,
     cib_output_data		= -ENOMSG,
     cib_connection		= -ENOTCONN,
     cib_authentication  	= -EPROTO,
     cib_missing 		= -EINVAL,
     cib_variant                 = -EPROTONOSUPPORT,
     CIBRES_MISSING_FIELD	= -EINVAL,
     cib_unknown                 = -EINVAL,
     cib_STALE                   = -ENOKEY,
     cib_EXISTS                  = -ENOTUNIQ,
     cib_NOTEXISTS		= -ENXIO,
     cib_ACTIVATION		= -ENODATA,
     cib_NOOBJECT		= -EINVAL,
     cib_NOPARENT		= -EINVAL,
     cib_NOTSUPPORTED            = -EPROTONOSUPPORT,
     cib_registration_msg	= -EPROTO,
     cib_callback_token          = -EPROTO,
     cib_callback_register	= -ECOMM,
     cib_client_gone		= -ECONNRESET,
     cib_not_master		= -EPERM,
     cib_missing_data            = -EINVAL,
     cib_remote_timeout          = -ETIME,
     cib_no_quorum		= -pcmk_err_no_quorum,
     cib_diff_failed		= -pcmk_err_diff_failed,
     cib_diff_resync		= -pcmk_err_diff_resync,
     cib_old_data		= -pcmk_err_old_data,
     cib_dtd_validation  	= -pcmk_err_dtd_validation,
     cib_bad_section		= -EINVAL,
     cib_bad_permissions         = -EACCES,
     cib_invalid_argument	= -EINVAL,
     cib_transform_failed        = -pcmk_err_transform_failed,
     cib_permission_denied	= -EACCES,
 };
 
 enum stonith_errors {
     stonith_ok			=  pcmk_ok,
     stonith_pending		= -EINPROGRESS,
     st_err_generic		= -pcmk_err_generic,
     st_err_internal		= -EPROTO,
     st_err_not_supported	= -EPROTONOSUPPORT,
     st_err_connection		= -ENOTCONN,
     st_err_missing		= -EINVAL,
     st_err_exists		= -ENOTUNIQ,
     st_err_timeout		= -ETIME,
     st_err_ipc			= -ECOMM,
     st_err_peer			= -ENOMSG,
     st_err_unknown_operation	= -EOPNOTSUPP,
     st_err_unknown_device	= -ENODEV,
     st_err_none_available	= -EHOSTUNREACH,
     st_err_signal		= -ECONNABORTED,
     st_err_agent_fork		= -ECHILD,
     st_err_agent_args		= -EREMOTEIO,
     st_err_agent		= -ECONNABORTED,
     st_err_invalid_level	= -EINVAL,
 };
 
 
 enum lrmd_errors {
     lrmd_ok                      =  pcmk_ok,
     lrmd_pending                 = -EINPROGRESS,
     lrmd_err_generic             = -EPROTONOSUPPORT,
     lrmd_err_internal            = -EPROTO,
     lrmd_err_connection          = -ENOTCONN,
     lrmd_err_missing             = -EINVAL,
     lrmd_err_ipc                 = -ECOMM,
     lrmd_err_peer                = -ENOMSG,
     lrmd_err_unknown_operation   = -EOPNOTSUPP,
     lrmd_err_unknown_rsc         = -ENODEV,
     lrmd_err_no_metadata         = -EIO,
     lrmd_err_stonith_connection  = -EUNATCH,
     lrmd_err_provider_required   = -EINVAL,
 };
 
 #define stonith_error2string pcmk_strerror
 #define lrmd_error2string    pcmk_strerror
 #define cib_error2string     pcmk_strerror
 
 static inline void
 slist_basic_destroy(GListPtr list)
 {
     GListPtr gIter = NULL;
 
     for (gIter = list; gIter != NULL; gIter = gIter->next) {
         free(gIter->data);
     }
     g_list_free(list);
 }
 
 #  define crm_strdup strdup
 #  define set_bit_inplace set_bit
 #  define clear_bit_inplace clear_bit
 
 #  define crm_malloc0(malloc_obj, length) do {				\
 	malloc_obj = malloc(length);					\
 	if(malloc_obj == NULL) {					\
 	    crm_err("Failed allocation of %lu bytes", (unsigned long)length); \
 	    CRM_ASSERT(malloc_obj != NULL);				\
 	}								\
 	memset(malloc_obj, 0, length);					\
     } while(0)
 
 #  define crm_malloc(malloc_obj, length) do {				\
 	malloc_obj = malloc(length);					\
 	if(malloc_obj == NULL) {					\
 	    crm_err("Failed allocation of %lu bytes", (unsigned long)length); \
 	    CRM_ASSERT(malloc_obj != NULL);				\
 	}								\
     } while(0)
 
 #  define crm_realloc(realloc_obj, length) do {				\
 	realloc_obj = realloc(realloc_obj, length);			\
 	CRM_ASSERT(realloc_obj != NULL);				\
     } while(0)
 
 #define crm_free(free_obj) do { free(free_obj); free_obj=NULL; } while(0)
 
 /* These two child iterator macros are no longer to be used
  * They exist for compatability reasons and will be removed in a
  * future release
  */
 #  define xml_child_iter(parent, child, code) do {			\
 	if(parent != NULL) {						\
 		xmlNode *child = NULL;					\
 		xmlNode *__crm_xml_iter = parent->children;		\
 		while(__crm_xml_iter != NULL) {				\
 			child = __crm_xml_iter;				\
 			__crm_xml_iter = __crm_xml_iter->next;		\
 			if(child->type == XML_ELEMENT_NODE) {		\
 			    code;					\
 			}						\
 		}							\
 	}								\
     } while(0)
 
 #  define xml_child_iter_filter(parent, child, filter, code) do {	\
 	if(parent != NULL) {						\
 	    xmlNode *child = NULL;					\
 	    xmlNode *__crm_xml_iter = parent->children;			\
 	    while(__crm_xml_iter != NULL) {				\
 		child = __crm_xml_iter;					\
 		__crm_xml_iter = __crm_xml_iter->next;			\
 		if(child->type == XML_ELEMENT_NODE) {			\
 		    if(filter == NULL					\
 		       || crm_str_eq(filter, (const char *)child->name, TRUE)) { \
 			code;						\
 		    }							\
 		}							\
 	    }								\
 	}								\
     } while(0)
 
 #  define xml_prop_iter(parent, prop_name, prop_value, code) do {	\
 	if(parent != NULL) {						\
 	    xmlAttrPtr prop_iter = parent->properties;			\
 	    const char *prop_name = NULL;				\
 	    const char *prop_value = NULL;				\
 	    while(prop_iter != NULL) {					\
 		prop_name = (const char *)prop_iter->name;		\
 		prop_value = crm_element_value(parent, prop_name);	\
 		prop_iter = prop_iter->next;				\
 		if(prop_name) {						\
 		    code;						\
 		}							\
 	    }								\
 	}								\
     } while(0)
 
 #  define xml_prop_name_iter(parent, prop_name, code) do {		\
 	if(parent != NULL) {						\
 	    xmlAttrPtr prop_iter = parent->properties;			\
 	    const char *prop_name = NULL;				\
 	    while(prop_iter != NULL) {					\
 		prop_name = (const char *)prop_iter->name;		\
 		prop_iter = prop_iter->next;				\
 		if(prop_name) {						\
 		    code;						\
 		}							\
 	    }								\
 	}								\
     } while(0)
 
 #  define zap_xml_from_parent(parent, xml_obj) free_xml_from_parent(parent, xml_obj); xml_obj = NULL
 
 /* Use something like this instead of the next macro:
 
     GListPtr gIter = rsc->children;
     for(; gIter != NULL; gIter = gIter->next) {
 	resource_t *child_rsc = (resource_t*)gIter->data;
 	...
     }
  */
 #  define slist_destroy(child_type, child, parent, a) do {		\
 	GListPtr __crm_iter_head = parent;				\
 	child_type *child = NULL;					\
 	while(__crm_iter_head != NULL) {				\
 	    child = (child_type *) __crm_iter_head->data;		\
 	    __crm_iter_head = __crm_iter_head->next;			\
 	    { a; }							\
 	}								\
 	g_list_free(parent);						\
     } while(0)
 
 #  ifdef CRM_ATTRD__H
 static inline gboolean
 attrd_update(crm_ipc_t *cluster, char command, const char *host, const char *name,
              const char *value, const char *section, const char *set, const char *dampen)
 {
     return attrd_update_delegate(cluster, command, host, name, value, section, set, dampen, NULL);
 }
 
 static inline gboolean
 attrd_lazy_update(char command, const char *host, const char *name,
                                   const char *value, const char *section, const char *set,
                                   const char *dampen)
 {
     return attrd_update_delegate(NULL, command, host, name, value, section, set, dampen, NULL);
 }
 
 static inline gboolean
 attrd_update_no_mainloop(int *connection, char command, const char *host,
                          const char *name, const char *value, const char *section,
                          const char *set, const char *dampen)
 {
     return attrd_update_delegate(NULL, command, host, name, value, section, set, dampen, NULL);
 }
 #  endif
 
 # ifdef CIB_UTIL__H
 static inline int
 update_attr(cib_t * the_cib, int call_options,
             const char *section, const char *node_uuid, const char *set_type, const char *set_name,
             const char *attr_id, const char *attr_name, const char *attr_value, gboolean to_console)
 {
     return update_attr_delegate(the_cib, call_options, section, node_uuid, set_type, set_name,
                                 attr_id, attr_name, attr_value, to_console, NULL);
 }
 
 static inline int
 find_nvpair_attr(cib_t * the_cib, const char *attr, const char *section, const char *node_uuid,
                  const char *set_type, const char *set_name, const char *attr_id,
                  const char *attr_name, gboolean to_console, char **value)
 {
     return find_nvpair_attr_delegate(the_cib, attr, section, node_uuid, set_type,
                                      set_name, attr_id, attr_name, to_console, value, NULL);
 }
 
 static inline int
 read_attr(cib_t * the_cib,
           const char *section, const char *node_uuid, const char *set_type, const char *set_name,
           const char *attr_id, const char *attr_name, char **attr_value, gboolean to_console)
 {
     return read_attr_delegate(the_cib, section, node_uuid, set_type, set_name,
                               attr_id, attr_name, attr_value, to_console, NULL);
 }
 
 static inline int
 delete_attr(cib_t * the_cib, int options,
             const char *section, const char *node_uuid, const char *set_type, const char *set_name,
             const char *attr_id, const char *attr_name, const char *attr_value, gboolean to_console)
 {
     return delete_attr_delegate(the_cib, options, section, node_uuid, set_type, set_name,
                                 attr_id, attr_name, attr_value, to_console, NULL);
 }
 #  endif
 
 #endif
diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h
index e356ec6ac6..a40dda5fce 100644
--- a/include/crm/msg_xml.h
+++ b/include/crm/msg_xml.h
@@ -1,350 +1,349 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 #ifndef XML_TAGS__H
 #  define XML_TAGS__H
 
 #ifndef F_ORIG
 #  define F_ORIG    "src"
 #endif
 
 #ifndef F_SEQ
 #  define F_SEQ		"seq"
 #endif
 
 #ifndef F_SUBTYPE
 #  define F_SUBTYPE "subt"
 #endif
 
 #ifndef F_TYPE
 #  define F_TYPE    "t"
 #endif
 
 #ifndef F_CLIENTNAME
 #  define	F_CLIENTNAME	"cn"
 #endif
 
 #ifndef F_XML_TAGNAME
 #  define F_XML_TAGNAME	"__name__"
 #endif
 
 #ifndef T_CRM
 #  define T_CRM     "crmd"
 #endif
 
 #ifndef T_ATTRD
 #  define T_ATTRD     "attrd"
 #endif
 
 #  define CIB_OPTIONS_FIRST "cib-bootstrap-options"
 
 #  define F_CRM_DATA			"crm_xml"
 #  define F_CRM_TASK			"crm_task"
 #  define F_CRM_HOST_TO			"crm_host_to"
 #  define F_CRM_MSG_TYPE		F_SUBTYPE
 #  define F_CRM_SYS_TO			"crm_sys_to"
 #  define F_CRM_SYS_FROM		"crm_sys_from"
 #  define F_CRM_HOST_FROM		F_ORIG
 #  define F_CRM_REFERENCE		XML_ATTR_REFERENCE
 #  define F_CRM_VERSION			XML_ATTR_VERSION
 #  define F_CRM_ORIGIN			"origin"
 #  define F_CRM_USER			"crm_user"
 #  define F_CRM_JOIN_ID			"join_id"
 #  define F_CRM_ELECTION_ID		"election-id"
 #  define F_CRM_ELECTION_AGE_S		"election-age-sec"
 #  define F_CRM_ELECTION_AGE_US		"election-age-nano-sec"
 #  define F_CRM_ELECTION_OWNER		"election-owner"
 #  define F_CRM_TGRAPH			"crm-tgraph"
 #  define F_CRM_TGRAPH_INPUT		"crm-tgraph-in"
 
 /*---- Common tags/attrs */
 #  define XML_DIFF_MARKER		"__crm_diff_marker__"
 #  define XML_ATTR_TAGNAME		F_XML_TAGNAME
 #  define XML_TAG_CIB			"cib"
 #  define XML_TAG_FAILED		"failed"
 
 #  define XML_ATTR_CRM_VERSION		"crm_feature_set"
 #  define XML_ATTR_DIGEST		"digest"
 #  define XML_ATTR_VALIDATION		"validate-with"
 
 #  define XML_ATTR_QUORUM_PANIC		"no-quorum-panic"
 #  define XML_ATTR_HAVE_QUORUM		"have-quorum"
 #  define XML_ATTR_EXPECTED_VOTES	"expected-quorum-votes"
 #  define XML_ATTR_GENERATION		"epoch"
 #  define XML_ATTR_GENERATION_ADMIN	"admin_epoch"
 #  define XML_ATTR_NUMUPDATES		"num_updates"
 #  define XML_ATTR_TIMEOUT		"timeout"
 #  define XML_ATTR_ORIGIN		"crm-debug-origin"
 #  define XML_ATTR_TSTAMP		"crm-timestamp"
 #  define XML_CIB_ATTR_WRITTEN		"cib-last-written"
 #  define XML_ATTR_VERSION		"version"
 #  define XML_ATTR_DESC			"description"
 #  define XML_ATTR_ID			"id"
 #  define XML_ATTR_IDREF			"id-ref"
 #  define XML_ATTR_ID_LONG		"long-id"
 #  define XML_ATTR_TYPE			"type"
 #  define XML_ATTR_FILTER_TYPE		"type-filter"
 #  define XML_ATTR_FILTER_ID		"id-filter"
 #  define XML_ATTR_FILTER_PRIORITY	"priority-filter"
 #  define XML_ATTR_VERBOSE		"verbose"
 #  define XML_ATTR_OP			"op"
 #  define XML_ATTR_DC			"is_dc"
 #  define XML_ATTR_DC_UUID		"dc-uuid"
 #  define XML_ATTR_UPDATE_ORIG		"update-origin"
 #  define XML_ATTR_UPDATE_CLIENT	"update-client"
 #  define XML_ATTR_UPDATE_USER		"update-user"
 
 #  define XML_BOOLEAN_TRUE		"true"
 #  define XML_BOOLEAN_FALSE		"false"
 #  define XML_BOOLEAN_YES		XML_BOOLEAN_TRUE
 #  define XML_BOOLEAN_NO		XML_BOOLEAN_FALSE
 
 #  define XML_TAG_OPTIONS		"options"
 
 /*---- top level tags/attrs */
 #  define XML_MSG_TAG			"crm_message"
 #  define XML_MSG_TAG_DATA		"msg_data"
 #  define XML_ATTR_REQUEST		"request"
 #  define XML_ATTR_RESPONSE		"response"
 
 #  define XML_ATTR_UNAME		"uname"
 #  define XML_ATTR_UUID			"id"
 #  define XML_ATTR_REFERENCE		"reference"
 
 #  define XML_FAIL_TAG_RESOURCE		"failed_resource"
 
 #  define XML_FAILRES_ATTR_RESID	"resource_id"
 #  define XML_FAILRES_ATTR_REASON	"reason"
 #  define XML_FAILRES_ATTR_RESSTATUS	"resource_status"
 
 #  define XML_CRM_TAG_PING		"ping_response"
 #  define XML_PING_ATTR_STATUS		"result"
 #  define XML_PING_ATTR_SYSFROM		"crm_subsystem"
 
 #  define XML_TAG_FRAGMENT		"cib_fragment"
 #  define XML_ATTR_RESULT		"result"
 #  define XML_ATTR_SECTION		"section"
 
 #  define XML_FAIL_TAG_CIB		"failed_update"
 
 #  define XML_FAILCIB_ATTR_ID		"id"
 #  define XML_FAILCIB_ATTR_OBJTYPE	"object_type"
 #  define XML_FAILCIB_ATTR_OP		"operation"
 #  define XML_FAILCIB_ATTR_REASON	"reason"
 
 /*---- CIB specific tags/attrs */
 #  define XML_CIB_TAG_SECTION_ALL	"all"
 #  define XML_CIB_TAG_CONFIGURATION	"configuration"
 #  define XML_CIB_TAG_STATUS       	"status"
 #  define XML_CIB_TAG_RESOURCES		"resources"
 #  define XML_CIB_TAG_NODES         	"nodes"
 #  define XML_CIB_TAG_DOMAINS         	"domains"
 #  define XML_CIB_TAG_CONSTRAINTS   	"constraints"
 #  define XML_CIB_TAG_CRMCONFIG   	"crm_config"
 #  define XML_CIB_TAG_OPCONFIG		"op_defaults"
 #  define XML_CIB_TAG_RSCCONFIG   	"rsc_defaults"
 #  define XML_CIB_TAG_ACLS   		"acls"
 
 #  define XML_CIB_TAG_STATE         	"node_state"
 #  define XML_CIB_TAG_NODE          	"node"
 #  define XML_CIB_TAG_DOMAIN          	"domain"
 #  define XML_CIB_TAG_CONSTRAINT    	"constraint"
 #  define XML_CIB_TAG_NVPAIR        	"nvpair"
 
 #  define XML_CIB_TAG_PROPSET	   	"cluster_property_set"
 #  define XML_TAG_ATTR_SETS	   	"instance_attributes"
 #  define XML_TAG_META_SETS	   	"meta_attributes"
 #  define XML_TAG_ATTRS			"attributes"
 #  define XML_TAG_PARAMS		"parameters"
 #  define XML_TAG_PARAM			"param"
 #  define XML_TAG_UTILIZATION		"utilization"
 
 #  define XML_TAG_RESOURCE_REF		"resource_ref"
 #  define XML_CIB_TAG_RESOURCE	  	"primitive"
 #  define XML_CIB_TAG_GROUP	  	"group"
 #  define XML_CIB_TAG_INCARNATION	"clone"
 #  define XML_CIB_TAG_MASTER		"master"
 
 #  define XML_CIB_TAG_RSC_TEMPLATE	"template"
 
 #  define XML_RSC_ATTR_RESTART	  	"restart-type"
 #  define XML_RSC_ATTR_ORDERED		"ordered"
 #  define XML_RSC_ATTR_INTERLEAVE	"interleave"
 #  define XML_RSC_ATTR_INCARNATION	"clone"
 #  define XML_RSC_ATTR_INCARNATION_MAX	"clone-max"
 #  define XML_RSC_ATTR_INCARNATION_NODEMAX	"clone-node-max"
 #  define XML_RSC_ATTR_MASTER_MAX	"master-max"
 #  define XML_RSC_ATTR_MASTER_NODEMAX	"master-node-max"
 #  define XML_RSC_ATTR_STATE		"clone-state"
 #  define XML_RSC_ATTR_MANAGED		"is-managed"
 #  define XML_RSC_ATTR_TARGET_ROLE	"target-role"
 #  define XML_RSC_ATTR_UNIQUE		"globally-unique"
 #  define XML_RSC_ATTR_NOTIFY		"notify"
 #  define XML_RSC_ATTR_STICKINESS	"resource-stickiness"
 #  define XML_RSC_ATTR_FAIL_STICKINESS	"migration-threshold"
 #  define XML_RSC_ATTR_FAIL_TIMEOUT	"failure-timeout"
 #  define XML_RSC_ATTR_MULTIPLE		"multiple-active"
 #  define XML_RSC_ATTR_PRIORITY		"priority"
 #  define XML_OP_ATTR_ON_FAIL		"on-fail"
 #  define XML_OP_ATTR_START_DELAY	"start-delay"
 #  define XML_OP_ATTR_ALLOW_MIGRATE	"allow-migrate"
 #  define XML_OP_ATTR_ORIGIN		"interval-origin"
 #  define XML_OP_ATTR_PENDING		"record-pending"
 
 #  define XML_CIB_TAG_LRM		"lrm"
 #  define XML_LRM_TAG_RESOURCES     	"lrm_resources"
 #  define XML_LRM_TAG_RESOURCE     	"lrm_resource"
 #  define XML_LRM_TAG_AGENTS	     	"lrm_agents"
 #  define XML_LRM_TAG_AGENT		"lrm_agent"
 #  define XML_LRM_TAG_RSC_OP		"lrm_rsc_op"
 #  define XML_AGENT_ATTR_CLASS		"class"
 #  define XML_AGENT_ATTR_PROVIDER	"provider"
 #  define XML_LRM_TAG_ATTRIBUTES	"attributes"
 
 #  define XML_CIB_ATTR_REPLACE       	"replace"
 #  define XML_CIB_ATTR_SOURCE       	"source"
 
 #  define XML_CIB_ATTR_HEALTH       	"health"
 #  define XML_CIB_ATTR_WEIGHT       	"weight"
 #  define XML_CIB_ATTR_PRIORITY     	"priority"
 #  define XML_CIB_ATTR_CLEAR        	"clear_on"
 #  define XML_CIB_ATTR_SOURCE       	"source"
 
 #  define XML_NODE_JOIN_STATE    	"join"
 #  define XML_NODE_EXPECTED     	"expected"
 #  define XML_NODE_IN_CLUSTER        	"in_ccm"
 #  define XML_NODE_IS_PEER    	"crmd"
-#  define XML_CIB_ATTR_HASTATE    	"ha"
 
 #  define XML_CIB_ATTR_SHUTDOWN       	"shutdown"
 #  define XML_CIB_ATTR_STONITH	    	"stonith"
 
 #  define XML_LRM_ATTR_INTERVAL		"interval"
 #  define XML_LRM_ATTR_TASK		"operation"
 #  define XML_LRM_ATTR_TASK_KEY		"operation_key"
 #  define XML_LRM_ATTR_TARGET		"on_node"
 #  define XML_LRM_ATTR_TARGET_UUID	"on_node_uuid"
 #  define XML_LRM_ATTR_RSCID		"rsc-id"
 #  define XML_LRM_ATTR_OPSTATUS		"op-status"
 #  define XML_LRM_ATTR_RC		"rc-code"
 #  define XML_LRM_ATTR_CALLID		"call-id"
 #  define XML_LRM_ATTR_OP_DIGEST	"op-digest"
 #  define XML_LRM_ATTR_OP_RESTART	"op-force-restart"
 #  define XML_LRM_ATTR_RESTART_DIGEST	"op-restart-digest"
 
 #  define XML_LRM_ATTR_MIGRATE_SOURCE	"migrate_source"
 #  define XML_LRM_ATTR_MIGRATE_TARGET	"migrate_target"
 
 #  define XML_TAG_GRAPH			"transition_graph"
 #  define XML_GRAPH_TAG_RSC_OP		"rsc_op"
 #  define XML_GRAPH_TAG_PSEUDO_EVENT	"pseudo_event"
 #  define XML_GRAPH_TAG_CRM_EVENT	"crm_event"
 
 #  define XML_TAG_RULE			"rule"
 #  define XML_RULE_ATTR_SCORE		"score"
 #  define XML_RULE_ATTR_SCORE_ATTRIBUTE	"score-attribute"
 #  define XML_RULE_ATTR_SCORE_MANGLED	"score-attribute-mangled"
 #  define XML_RULE_ATTR_ROLE		"role"
 #  define XML_RULE_ATTR_RESULT		"result"
 #  define XML_RULE_ATTR_BOOLEAN_OP	"boolean-op"
 
 #  define XML_TAG_EXPRESSION		"expression"
 #  define XML_EXPR_ATTR_ATTRIBUTE	"attribute"
 #  define XML_EXPR_ATTR_OPERATION	"operation"
 #  define XML_EXPR_ATTR_VALUE		"value"
 #  define XML_EXPR_ATTR_TYPE		"type"
 
 #  define XML_CONS_TAG_RSC_DEPEND	"rsc_colocation"
 #  define XML_CONS_TAG_RSC_ORDER	"rsc_order"
 #  define XML_CONS_TAG_RSC_LOCATION	"rsc_location"
 #  define XML_CONS_TAG_RSC_TICKET	"rsc_ticket"
 #  define XML_CONS_TAG_RSC_SET		"resource_set"
 #  define XML_CONS_ATTR_SYMMETRICAL	"symmetrical"
 
 #  define XML_COLOC_ATTR_SOURCE		"rsc"
 #  define XML_COLOC_ATTR_SOURCE_ROLE	"rsc-role"
 #  define XML_COLOC_ATTR_TARGET		"with-rsc"
 #  define XML_COLOC_ATTR_TARGET_ROLE	"with-rsc-role"
 #  define XML_COLOC_ATTR_NODE_ATTR	"node-attribute"
 #  define XML_COLOC_ATTR_SOURCE_INSTANCE	"rsc-instance"
 #  define XML_COLOC_ATTR_TARGET_INSTANCE	"with-rsc-instance"
 
 #  define XML_ORDER_ATTR_FIRST		"first"
 #  define XML_ORDER_ATTR_THEN		"then"
 #  define XML_ORDER_ATTR_FIRST_ACTION	"first-action"
 #  define XML_ORDER_ATTR_THEN_ACTION	"then-action"
 #  define XML_ORDER_ATTR_FIRST_INSTANCE	"first-instance"
 #  define XML_ORDER_ATTR_THEN_INSTANCE	"then-instance"
 #  define XML_ORDER_ATTR_KIND		"kind"
 
 #  define XML_TICKET_ATTR_TICKET	"ticket"
 #  define XML_TICKET_ATTR_LOSS_POLICY	"loss-policy"
 
 #  define XML_NVPAIR_ATTR_NAME        	"name"
 #  define XML_NVPAIR_ATTR_VALUE        	"value"
 
 #  define XML_NODE_ATTR_STATE		"state"
 
 #  define XML_CONFIG_ATTR_DC_DEADTIME	"dc-deadtime"
 #  define XML_CONFIG_ATTR_ELECTION_FAIL	"election-timeout"
 #  define XML_CONFIG_ATTR_FORCE_QUIT	"shutdown-escalation"
 #  define XML_CONFIG_ATTR_RECHECK	"cluster-recheck-interval"
 
 #  define XML_CIB_TAG_GENERATION_TUPPLE	"generation_tuple"
 
 #  define XML_ATTR_TRANSITION_MAGIC	"transition-magic"
 #  define XML_ATTR_TRANSITION_KEY	"transition-key"
 
 #  define XML_ATTR_TE_NOWAIT		"op_no_wait"
 #  define XML_ATTR_TE_TARGET_RC		"op_target_rc"
 #  define XML_ATTR_TE_ALLOWFAIL		"op_allow_fail"
 #  define XML_ATTR_LRM_PROBE		"lrm-is-probe"
 #  define XML_TAG_TRANSIENT_NODEATTRS	"transient_attributes"
 
 #  define XML_TAG_DIFF_ADDED		"diff-added"
 #  define XML_TAG_DIFF_REMOVED		"diff-removed"
 
 #  define XML_ACL_TAG_USER		"acl_user"
 #  define XML_ACL_TAG_ROLE		"acl_role"
 #  define XML_ACL_TAG_ROLE_REF 		"role_ref"
 #  define XML_ACL_TAG_READ		"read"
 #  define XML_ACL_TAG_WRITE		"write"
 #  define XML_ACL_TAG_DENY		"deny"
 #  define XML_ACL_ATTR_REF		"ref"
 #  define XML_ACL_ATTR_TAG		"tag"
 #  define XML_ACL_ATTR_XPATH		"xpath"
 #  define XML_ACL_ATTR_ATTRIBUTE	"attribute"
 
 #  define XML_CIB_TAG_TICKETS		"tickets"
 #  define XML_CIB_TAG_TICKET_STATE	"ticket_state"
 
 #  define XML_TAG_FENCING_TOPOLOGY      "fencing-topology"
 #  define XML_TAG_FENCING_LEVEL         "fencing-level"
 #  define XML_ATTR_STONITH_INDEX        "index"
 #  define XML_ATTR_STONITH_TARGET       "target"
 #  define XML_ATTR_STONITH_DEVICES      "devices"
 
 #  include <crm/common/xml.h>
 
 #  define ID(x) crm_element_value(x, XML_ATTR_ID)
 #  define INSTANCE(x) crm_element_value(x, XML_CIB_ATTR_INSTANCE)
 #  define TSTAMP(x) crm_element_value(x, XML_ATTR_TSTAMP)
 #  define TYPE(x) crm_element_name(x)
 #  define NAME(x) crm_element_value(x, XML_NVPAIR_ATTR_NAME)
 #  define VALUE(x) crm_element_value(x, XML_NVPAIR_ATTR_VALUE)
 
 #endif
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index d7ffe303c1..300cba47a8 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -1,2420 +1,2397 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  * 
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 #include <crm_internal.h>
 
 #include <glib.h>
 
 #include <crm/crm.h>
 #include <crm/services.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crm/common/util.h>
 #include <crm/pengine/rules.h>
 #include <crm/pengine/internal.h>
 #include <unpack.h>
 
 CRM_TRACE_INIT_DATA(pe_status);
 
 #define set_config_flag(data_set, option, flag) do {			\
 	const char *tmp = pe_pref(data_set->config_hash, option);	\
 	if(tmp) {							\
 	    if(crm_is_true(tmp)) {					\
 		set_bit(data_set->flags, flag);			\
 	    } else {							\
 		clear_bit(data_set->flags, flag);		\
 	    }								\
 	}								\
     } while(0)
 
 gboolean unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, GListPtr next,
                        enum action_fail_response *failed, pe_working_set_t * data_set);
 
 static void
 pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason)
 {
     CRM_CHECK(node, return);
     if (node->details->unclean == FALSE) {
         if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
             crm_warn("Node %s will be fenced %s", node->details->uname, reason);
         } else {
             crm_warn("Node %s is unclean %s", node->details->uname, reason);
         }
     }
     node->details->unclean = TRUE;
 }
 
 gboolean
 unpack_config(xmlNode * config, pe_working_set_t * data_set)
 {
     const char *value = NULL;
     GHashTable *config_hash =
         g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
 
     data_set->config_hash = config_hash;
 
     unpack_instance_attributes(data_set->input, config, XML_CIB_TAG_PROPSET, NULL, config_hash,
                                CIB_OPTIONS_FIRST, FALSE, data_set->now);
 
     verify_pe_options(data_set->config_hash);
 
     set_config_flag(data_set, "enable-startup-probes", pe_flag_startup_probes);
     crm_info("Startup probes: %s",
              is_set(data_set->flags, pe_flag_startup_probes) ? "enabled" : "disabled (dangerous)");
 
     value = pe_pref(data_set->config_hash, "stonith-timeout");
     data_set->stonith_timeout = crm_get_msec(value);
     crm_debug("STONITH timeout: %d", data_set->stonith_timeout);
 
     set_config_flag(data_set, "stonith-enabled", pe_flag_stonith_enabled);
     crm_debug("STONITH of failed nodes is %s",
               is_set(data_set->flags, pe_flag_stonith_enabled) ? "enabled" : "disabled");
 
     data_set->stonith_action = pe_pref(data_set->config_hash, "stonith-action");
     crm_trace("STONITH will %s nodes", data_set->stonith_action);
 
     set_config_flag(data_set, "stop-all-resources", pe_flag_stop_everything);
     crm_debug("Stop all active resources: %s",
               is_set(data_set->flags, pe_flag_stop_everything) ? "true" : "false");
 
     set_config_flag(data_set, "symmetric-cluster", pe_flag_symmetric_cluster);
     if (is_set(data_set->flags, pe_flag_symmetric_cluster)) {
         crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
     }
 
     value = pe_pref(data_set->config_hash, "default-resource-stickiness");
     data_set->default_resource_stickiness = char2score(value);
     crm_debug("Default stickiness: %d", data_set->default_resource_stickiness);
 
     value = pe_pref(data_set->config_hash, "no-quorum-policy");
 
     if (safe_str_eq(value, "ignore")) {
         data_set->no_quorum_policy = no_quorum_ignore;
 
     } else if (safe_str_eq(value, "freeze")) {
         data_set->no_quorum_policy = no_quorum_freeze;
 
     } else if (safe_str_eq(value, "suicide")) {
         gboolean do_panic = FALSE;
 
         crm_element_value_int(data_set->input, XML_ATTR_QUORUM_PANIC, &do_panic);
 
         if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
             crm_config_err
                 ("Setting no-quorum-policy=suicide makes no sense if stonith-enabled=false");
         }
 
         if (do_panic && is_set(data_set->flags, pe_flag_stonith_enabled)) {
             data_set->no_quorum_policy = no_quorum_suicide;
 
         } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE && do_panic == FALSE) {
             crm_notice("Resetting no-quorum-policy to 'stop': The cluster has never had quorum");
             data_set->no_quorum_policy = no_quorum_stop;
         }
 
     } else {
         data_set->no_quorum_policy = no_quorum_stop;
     }
 
     switch (data_set->no_quorum_policy) {
         case no_quorum_freeze:
             crm_debug("On loss of CCM Quorum: Freeze resources");
             break;
         case no_quorum_stop:
             crm_debug("On loss of CCM Quorum: Stop ALL resources");
             break;
         case no_quorum_suicide:
             crm_notice("On loss of CCM Quorum: Fence all remaining nodes");
             break;
         case no_quorum_ignore:
             crm_notice("On loss of CCM Quorum: Ignore");
             break;
     }
 
     set_config_flag(data_set, "stop-orphan-resources", pe_flag_stop_rsc_orphans);
     crm_trace("Orphan resources are %s",
               is_set(data_set->flags, pe_flag_stop_rsc_orphans) ? "stopped" : "ignored");
 
     set_config_flag(data_set, "stop-orphan-actions", pe_flag_stop_action_orphans);
     crm_trace("Orphan resource actions are %s",
               is_set(data_set->flags, pe_flag_stop_action_orphans) ? "stopped" : "ignored");
 
     set_config_flag(data_set, "remove-after-stop", pe_flag_remove_after_stop);
     crm_trace("Stopped resources are removed from the status section: %s",
               is_set(data_set->flags, pe_flag_remove_after_stop) ? "true" : "false");
 
     set_config_flag(data_set, "maintenance-mode", pe_flag_maintenance_mode);
     crm_trace("Maintenance mode: %s",
               is_set(data_set->flags, pe_flag_maintenance_mode) ? "true" : "false");
 
     if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
         clear_bit(data_set->flags, pe_flag_is_managed_default);
     } else {
         set_config_flag(data_set, "is-managed-default", pe_flag_is_managed_default);
     }
     crm_trace("By default resources are %smanaged",
               is_set(data_set->flags, pe_flag_is_managed_default) ? "" : "not ");
 
     set_config_flag(data_set, "start-failure-is-fatal", pe_flag_start_failure_fatal);
     crm_trace("Start failures are %s",
               is_set(data_set->flags,
                      pe_flag_start_failure_fatal) ? "always fatal" : "handled by failcount");
 
     node_score_red = char2score(pe_pref(data_set->config_hash, "node-health-red"));
     node_score_green = char2score(pe_pref(data_set->config_hash, "node-health-green"));
     node_score_yellow = char2score(pe_pref(data_set->config_hash, "node-health-yellow"));
 
     crm_info("Node scores: 'red' = %s, 'yellow' = %s, 'green' = %s",
              pe_pref(data_set->config_hash, "node-health-red"),
              pe_pref(data_set->config_hash, "node-health-yellow"),
              pe_pref(data_set->config_hash, "node-health-green"));
 
     data_set->placement_strategy = pe_pref(data_set->config_hash, "placement-strategy");
     crm_trace("Placement strategy: %s", data_set->placement_strategy);
 
     return TRUE;
 }
 
 gboolean
 unpack_nodes(xmlNode * xml_nodes, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
     node_t *new_node = NULL;
     const char *id = NULL;
     const char *uname = NULL;
     const char *type = NULL;
     const char *score = NULL;
     gboolean unseen_are_unclean = TRUE;
     const char *blind_faith = pe_pref(data_set->config_hash, "startup-fencing");
 
     if (crm_is_true(blind_faith) == FALSE) {
         unseen_are_unclean = FALSE;
         crm_warn("Blind faith: not fencing unseen nodes");
     }
 
     for (xml_obj = __xml_first_child(xml_nodes); xml_obj != NULL; xml_obj = __xml_next(xml_obj)) {
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, TRUE)) {
             new_node = NULL;
 
             id = crm_element_value(xml_obj, XML_ATTR_ID);
             uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
             type = crm_element_value(xml_obj, XML_ATTR_TYPE);
             score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
             crm_trace("Processing node %s/%s", uname, id);
 
             if (id == NULL) {
                 crm_config_err("Must specify id tag in <node>");
                 continue;
             }
             if (type == NULL) {
                 crm_config_err("Must specify type tag in <node>");
                 continue;
             }
             if (pe_find_node(data_set->nodes, uname) != NULL) {
                 crm_config_warn("Detected multiple node entries with uname=%s"
                                 " - this is rarely intended", uname);
             }
 
             new_node = calloc(1, sizeof(node_t));
             if (new_node == NULL) {
                 return FALSE;
             }
 
             new_node->weight = char2score(score);
             new_node->fixed = FALSE;
             new_node->details = calloc(1, sizeof(struct node_shared_s));
 
             if (new_node->details == NULL) {
                 free(new_node);
                 return FALSE;
             }
 
             crm_trace("Creaing node for entry %s/%s", uname, id);
             new_node->details->id = id;
             new_node->details->uname = uname;
             new_node->details->type = node_ping;
             new_node->details->online = FALSE;
             new_node->details->shutdown = FALSE;
             new_node->details->running_rsc = NULL;
             new_node->details->attrs = g_hash_table_new_full(crm_str_hash, g_str_equal,
                                                              g_hash_destroy_str,
                                                              g_hash_destroy_str);
             new_node->details->utilization =
                 g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
                                       g_hash_destroy_str);
 
 /* 		if(data_set->have_quorum == FALSE */
 /* 		   && data_set->no_quorum_policy == no_quorum_stop) { */
 /* 			/\* start shutting resources down *\/ */
 /* 			new_node->weight = -INFINITY; */
 /* 		} */
 
             if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE
                 || unseen_are_unclean == FALSE) {
                 /* blind faith... */
                 new_node->details->unclean = FALSE;
 
             } else {
                 /* all nodes are unclean until we've seen their
                  * status entry
                  */
                 new_node->details->unclean = TRUE;
             }
 
             if (type == NULL || safe_str_eq(type, "member")
                 || safe_str_eq(type, NORMALNODE)) {
                 new_node->details->type = node_member;
             }
 
             add_node_attrs(xml_obj, new_node, FALSE, data_set);
             unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_UTILIZATION, NULL,
                                        new_node->details->utilization, NULL, FALSE, data_set->now);
 
             data_set->nodes = g_list_append(data_set->nodes, new_node);
             crm_trace("Done with node %s", crm_element_value(xml_obj, XML_ATTR_UNAME));
         }
     }
 
     return TRUE;
 }
 
 static void
 g_hash_destroy_node_list(gpointer data)
 {
     GListPtr domain = data;
 
     g_list_free_full(domain, free);
 }
 
 gboolean
 unpack_domains(xmlNode * xml_domains, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     GListPtr domain = NULL;
     xmlNode *xml_node = NULL;
     xmlNode *xml_domain = NULL;
 
     crm_info("Unpacking domains");
     data_set->domains =
         g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
                               g_hash_destroy_node_list);
 
     for (xml_domain = __xml_first_child(xml_domains); xml_domain != NULL;
          xml_domain = __xml_next(xml_domain)) {
         if (crm_str_eq((const char *)xml_domain->name, XML_CIB_TAG_DOMAIN, TRUE)) {
             domain = NULL;
             id = crm_element_value(xml_domain, XML_ATTR_ID);
 
             for (xml_node = __xml_first_child(xml_domain); xml_node != NULL;
                  xml_node = __xml_next(xml_node)) {
                 if (crm_str_eq((const char *)xml_node->name, XML_CIB_TAG_NODE, TRUE)) {
                     node_t *copy = NULL;
                     node_t *node = NULL;
                     const char *uname = crm_element_value(xml_node, "name");
                     const char *score = crm_element_value(xml_node, XML_RULE_ATTR_SCORE);
 
                     if (uname == NULL) {
                         crm_config_err("Invalid domain %s: Must specify id tag in <node>", id);
                         continue;
                     }
 
                     node = pe_find_node(data_set->nodes, uname);
                     if (node == NULL) {
                         node = pe_find_node_id(data_set->nodes, uname);
                     }
                     if (node == NULL) {
                         crm_config_warn("Invalid domain %s: Node %s does not exist", id, uname);
                         continue;
                     }
 
                     copy = node_copy(node);
                     copy->weight = char2score(score);
                     crm_debug("Adding %s to domain %s with score %s", node->details->uname, id,
                               score);
 
                     domain = g_list_prepend(domain, copy);
                 }
             }
 
             if (domain) {
                 crm_debug("Created domain %s with %d members", id, g_list_length(domain));
                 g_hash_table_replace(data_set->domains, strdup(id), domain);
             }
         }
     }
 
     return TRUE;
 }
 
 static void
 destroy_template_rsc_set(gpointer data)
 {
     xmlNode *rsc_set = data;
 
     free_xml(rsc_set);
 }
 
 gboolean
 unpack_resources(xmlNode * xml_resources, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
 
     data_set->template_rsc_sets =
         g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
                               destroy_template_rsc_set);
 
     for (xml_obj = __xml_first_child(xml_resources); xml_obj != NULL; xml_obj = __xml_next(xml_obj)) {
         resource_t *new_rsc = NULL;
 
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE, TRUE)) {
             const char *template_id = ID(xml_obj);
 
             if (template_id && g_hash_table_lookup_extended(data_set->template_rsc_sets,
                                                             template_id, NULL, NULL) == FALSE) {
                 /* Record the template's ID for the knowledge of its existence anyway. */
                 g_hash_table_insert(data_set->template_rsc_sets, strdup(template_id), NULL);
             }
             continue;
         }
 
         crm_trace("Beginning unpack... <%s id=%s... >", crm_element_name(xml_obj), ID(xml_obj));
         if (common_unpack(xml_obj, &new_rsc, NULL, data_set)) {
             data_set->resources = g_list_append(data_set->resources, new_rsc);
 
             print_resource(LOG_DEBUG_3, "Added", new_rsc, FALSE);
 
         } else {
             crm_config_err("Failed unpacking %s %s",
                            crm_element_name(xml_obj), crm_element_value(xml_obj, XML_ATTR_ID));
             if (new_rsc != NULL && new_rsc->fns != NULL) {
                 new_rsc->fns->free(new_rsc);
             }
         }
     }
 
     data_set->resources = g_list_sort(data_set->resources, sort_rsc_priority);
 
     if (is_set(data_set->flags, pe_flag_stonith_enabled)
         && is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {
         crm_config_err("Resource start-up disabled since no STONITH resources have been defined");
         crm_config_err("Either configure some or disable STONITH with the stonith-enabled option");
         crm_config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
     }
 
     return TRUE;
 }
 
 /* The ticket state section:
  * "/cib/status/tickets/ticket_state" */
 static gboolean
 unpack_ticket_state(xmlNode * xml_ticket, pe_working_set_t * data_set)
 {
     const char *ticket_id = NULL;
     const char *granted = NULL;
     const char *last_granted = NULL;
     const char *standby = NULL;
     xmlAttrPtr xIter = NULL;
 
     ticket_t *ticket = NULL;
 
     ticket_id = ID(xml_ticket);
     if (ticket_id == NULL || strlen(ticket_id) == 0) {
         return FALSE;
     }
 
     crm_trace("Processing ticket state for %s", ticket_id);
 
     ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
     if (ticket == NULL) {
         ticket = ticket_new(ticket_id, data_set);
         if (ticket == NULL) {
             return FALSE;
         }
     }
 
     for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
         const char *prop_name = (const char *)xIter->name;
         const char *prop_value = crm_element_value(xml_ticket, prop_name);
         
         if(crm_str_eq(prop_name, XML_ATTR_ID, TRUE)) {
             continue;
         }
         g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
     }
 
     granted = g_hash_table_lookup(ticket->state, "granted");
     if (granted && crm_is_true(granted)) {
         ticket->granted = TRUE;
         crm_info("We have ticket '%s'", ticket->id);
     } else {
         ticket->granted = FALSE;
         crm_info("We do not have ticket '%s'", ticket->id);
     }
 
     last_granted = g_hash_table_lookup(ticket->state, "last-granted");
     if (last_granted) {
         ticket->last_granted = crm_parse_int(last_granted, 0);
     }
 
     standby = g_hash_table_lookup(ticket->state, "standby");
     if (standby && crm_is_true(standby)) {
         ticket->standby = TRUE;
         if (ticket->granted) {
             crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
         }
     } else {
         ticket->standby = FALSE;
     }
 
     crm_trace("Done with ticket state for %s", ticket_id);
 
     return TRUE;
 }
 
 static gboolean
 unpack_tickets_state(xmlNode * xml_tickets, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
 
     for (xml_obj = __xml_first_child(xml_tickets); xml_obj != NULL; xml_obj = __xml_next(xml_obj)) {
         if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, TRUE) == FALSE) {
             continue;
         }
         unpack_ticket_state(xml_obj, data_set);
     }
 
     return TRUE;
 }
 
 /* Compatibility with the deprecated ticket state section:
  * "/cib/status/tickets/instance_attributes" */
 static void
 get_ticket_state_legacy(gpointer key, gpointer value, gpointer user_data)
 {
     const char *long_key = key;
     char *state_key = NULL;
 
     const char *granted_prefix = "granted-ticket-";
     const char *last_granted_prefix = "last-granted-";
     static int granted_prefix_strlen = 0;
     static int last_granted_prefix_strlen = 0;
 
     const char *ticket_id = NULL;
     const char *is_granted = NULL;
     const char *last_granted = NULL;
     const char *sep = NULL;
 
     ticket_t *ticket = NULL;
     pe_working_set_t *data_set = user_data;
 
     if (granted_prefix_strlen == 0) {
         granted_prefix_strlen = strlen(granted_prefix);
     }
 
     if (last_granted_prefix_strlen == 0) {
         last_granted_prefix_strlen = strlen(last_granted_prefix);
     }
 
     if (strstr(long_key, granted_prefix) == long_key) {
         ticket_id = long_key + granted_prefix_strlen;
         if (strlen(ticket_id)) {
             state_key = strdup("granted");
             is_granted = value;
         }
     } else if (strstr(long_key, last_granted_prefix) == long_key) {
         ticket_id = long_key + last_granted_prefix_strlen;
         if (strlen(ticket_id)) {
             state_key = strdup("last-granted");
             last_granted = value;
         }
     } else if ((sep = strrchr(long_key, '-'))) {
         ticket_id = sep + 1;
         state_key = strndup(long_key, strlen(long_key) - strlen(sep));
     }
 
     if (ticket_id == NULL || strlen(ticket_id) == 0) {
         free(state_key);
         return;
     }
 
     if (state_key == NULL || strlen(state_key) == 0) {
         free(state_key);
         return;
     }
 
     ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
     if (ticket == NULL) {
         ticket = ticket_new(ticket_id, data_set);
         if (ticket == NULL) {
             free(state_key);
             return;
         }
     }
 
     g_hash_table_replace(ticket->state, state_key, strdup(value));
 
     if (is_granted) {
         if (crm_is_true(is_granted)) {
             ticket->granted = TRUE;
             crm_info("We have ticket '%s'", ticket->id);
         } else {
             ticket->granted = FALSE;
             crm_info("We do not have ticket '%s'", ticket->id);
         }
 
     } else if (last_granted) {
         ticket->last_granted = crm_parse_int(last_granted, 0);
     }
 }
 
 /* remove nodes that are down, stopping */
 /* create +ve rsc_to_node constraints between resources and the nodes they are running on */
 /* anything else? */
 gboolean
 unpack_status(xmlNode * status, pe_working_set_t * data_set)
 {
     const char *id = NULL;
     const char *uname = NULL;
 
     xmlNode *lrm_rsc = NULL;
     xmlNode *attrs = NULL;
     xmlNode *state = NULL;
     xmlNode *node_state = NULL;
     node_t *this_node = NULL;
 
     crm_trace("Beginning unpack");
 
     if (data_set->tickets == NULL)  {
         data_set->tickets =
             g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, destroy_ticket);
     }
 
     for (state = __xml_first_child(status); state != NULL; state = __xml_next(state)) {
         if (crm_str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, TRUE)) {
             xmlNode *xml_tickets = state;
             GHashTable *state_hash = NULL;
 
             /* Compatibility with the deprecated ticket state section:
              * Unpack the attributes in the deprecated "/cib/status/tickets/instance_attributes" if it exists. */
             state_hash = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
                                       g_hash_destroy_str);
 
             unpack_instance_attributes(data_set->input, xml_tickets, XML_TAG_ATTR_SETS, NULL,
                                        state_hash, NULL, TRUE, data_set->now);
 
             g_hash_table_foreach(state_hash, get_ticket_state_legacy, data_set);
 
             if (state_hash) {
                 g_hash_table_destroy(state_hash);
             }
 
             /* Unpack the new "/cib/status/tickets/ticket_state"s */
             unpack_tickets_state(xml_tickets, data_set);
         }
 
         if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE)) {
             node_state = state;
 
             id = crm_element_value(node_state, XML_ATTR_ID);
             uname = crm_element_value(node_state, XML_ATTR_UNAME);
             attrs = find_xml_node(node_state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
 
             crm_trace("Processing node id=%s, uname=%s", id, uname);
             this_node = pe_find_node_id(data_set->nodes, id);
 
             if (uname == NULL) {
                 /* error */
                 continue;
 
             } else if (this_node == NULL) {
                 crm_config_warn("Node %s in status section no longer exists", uname);
                 continue;
             }
 
             /* Mark the node as provisionally clean
              * - at least we have seen it in the current cluster's lifetime
              */
             this_node->details->unclean = FALSE;
             add_node_attrs(attrs, this_node, TRUE, data_set);
 
             if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "standby"))) {
                 crm_info("Node %s is in standby-mode", this_node->details->uname);
                 this_node->details->standby = TRUE;
             }
 
             crm_trace("determining node state");
             determine_online_status(node_state, this_node, data_set);
 
             if (this_node->details->online && data_set->no_quorum_policy == no_quorum_suicide) {
                 /* Everything else should flow from this automatically
                  * At least until the PE becomes able to migrate off healthy resources 
                  */
                 pe_fence_node(data_set, this_node, "because the cluster does not have quorum");
             }
         }
     }
 
     /* Now that we know all node states, we can safely handle migration ops
      * But, for now, only process healthy nodes
      *  - this is necessary for the logic in bug lf#2508 to function correctly 
      */
     for (node_state = __xml_first_child(status); node_state != NULL;
          node_state = __xml_next(node_state)) {
         if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
             continue;
         }
 
         id = crm_element_value(node_state, XML_ATTR_ID);
         this_node = pe_find_node_id(data_set->nodes, id);
 
         if (this_node == NULL) {
             crm_info("Node %s is unknown", id);
             continue;
 
         } else if (this_node->details->online) {
             crm_trace("Processing lrm resource entries on healthy node: %s",
                       this_node->details->uname);
             lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
             lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
             unpack_lrm_resources(this_node, lrm_rsc, data_set);
         }
     }
 
     /* Now handle failed nodes - but only if stonith is enabled
      *
      * By definition, offline nodes run no resources so there is nothing to do.
      * Only when stonith is enabled do we need to know what is on the node to
      * ensure rsc start events happen after the stonith
      */
     for (node_state = __xml_first_child(status);
          node_state != NULL && is_set(data_set->flags, pe_flag_stonith_enabled);
          node_state = __xml_next(node_state)) {
 
         if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
             continue;
         }
 
         id = crm_element_value(node_state, XML_ATTR_ID);
         this_node = pe_find_node_id(data_set->nodes, id);
 
         if (this_node == NULL || this_node->details->online) {
             continue;
 
         } else {
             crm_trace("Processing lrm resource entries on unhealthy node: %s",
                       this_node->details->uname);
             lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
             lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
             unpack_lrm_resources(this_node, lrm_rsc, data_set);
         }
     }
 
     return TRUE;
 }
 
 static gboolean
 determine_online_status_no_fencing(pe_working_set_t * data_set, xmlNode * node_state,
                                    node_t * this_node)
 {
     gboolean online = FALSE;
     const char *join_state = crm_element_value(node_state, XML_NODE_JOIN_STATE);
     const char *crm_state = crm_element_value(node_state, XML_NODE_IS_PEER);
     const char *ccm_state = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
-    const char *ha_state = crm_element_value(node_state, XML_CIB_ATTR_HASTATE);
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
 
-    if (ha_state == NULL) {
-        ha_state = DEADSTATUS;
-    }
-
-    if (!crm_is_true(ccm_state) || safe_str_eq(ha_state, DEADSTATUS)) {
-        crm_trace("Node is down: ha_state=%s, ccm_state=%s", crm_str(ha_state), crm_str(ccm_state));
+    if (!crm_is_true(ccm_state)) {
+        crm_trace("Node is down: ccm_state=%s", crm_str(ccm_state));
 
     } else if (safe_str_eq(crm_state, ONLINESTATUS)) {
         if (safe_str_eq(join_state, CRMD_JOINSTATE_MEMBER)) {
             online = TRUE;
         } else {
             crm_debug("Node is not ready to run resources: %s", join_state);
         }
 
     } else if (this_node->details->expected_up == FALSE) {
-        crm_trace("CRMd is down: ha_state=%s, ccm_state=%s", crm_str(ha_state), crm_str(ccm_state));
+        crm_trace("CRMd is down: ccm_state=%s", crm_str(ccm_state));
         crm_trace("\tcrm_state=%s, join_state=%s, expected=%s",
                   crm_str(crm_state), crm_str(join_state), crm_str(exp_state));
 
     } else {
         /* mark it unclean */
         pe_fence_node(data_set, this_node, "because it is partially and/or un-expectedly down");
-        crm_info("\tha_state=%s, ccm_state=%s,"
-                 " crm_state=%s, join_state=%s, expected=%s",
-                 crm_str(ha_state), crm_str(ccm_state),
-                 crm_str(crm_state), crm_str(join_state), crm_str(exp_state));
+        crm_info("\tccm_state=%s, crm_state=%s, join_state=%s, expected=%s",
+                 crm_str(ccm_state), crm_str(crm_state), crm_str(join_state), crm_str(exp_state));
     }
     return online;
 }
 
 static gboolean
 determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_state,
                                 node_t * this_node)
 {
     gboolean online = FALSE;
     gboolean do_terminate = FALSE;
     const char *join_state = crm_element_value(node_state, XML_NODE_JOIN_STATE);
     const char *crm_state = crm_element_value(node_state, XML_NODE_IS_PEER);
     const char *ccm_state = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
-    const char *ha_state = crm_element_value(node_state, XML_CIB_ATTR_HASTATE);
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
     const char *terminate = g_hash_table_lookup(this_node->details->attrs, "terminate");
 
-    if (ha_state == NULL) {
-        ha_state = DEADSTATUS;
-    }
-
     if (crm_is_true(terminate)) {
         do_terminate = TRUE;
 
     } else if (terminate != NULL && strlen(terminate) > 0) {
         /* could be a time() value */
         char t = terminate[0];
 
         if (t != '0' && isdigit(t)) {
             do_terminate = TRUE;
         }
     }
 
     if (crm_is_true(ccm_state)
-        && safe_str_eq(ha_state, ACTIVESTATUS)
         && safe_str_eq(crm_state, ONLINESTATUS)) {
 
         if (safe_str_eq(join_state, CRMD_JOINSTATE_MEMBER)) {
             online = TRUE;
             if (do_terminate) {
                 pe_fence_node(data_set, this_node, "because termination was requested");
             }
 
         } else if (join_state == exp_state /* == NULL */ ) {
             crm_info("Node %s is coming up", this_node->details->uname);
-            crm_debug("\tha_state=%s, ccm_state=%s,"
-                      " crm_state=%s, join_state=%s, expected=%s",
-                      crm_str(ha_state), crm_str(ccm_state),
-                      crm_str(crm_state), crm_str(join_state), crm_str(exp_state));
+            crm_debug("\tccm_state=%s, crm_state=%s, join_state=%s, expected=%s",
+                      crm_str(ccm_state), crm_str(crm_state), crm_str(join_state), crm_str(exp_state));
 
         } else if (safe_str_eq(join_state, CRMD_JOINSTATE_PENDING)) {
             crm_info("Node %s is not ready to run resources", this_node->details->uname);
             this_node->details->standby = TRUE;
             this_node->details->pending = TRUE;
             online = TRUE;
 
         } else if (safe_str_eq(join_state, CRMD_JOINSTATE_NACK)) {
             crm_warn("Node %s is not part of the cluster", this_node->details->uname);
             this_node->details->standby = TRUE;
             this_node->details->pending = TRUE;
             online = TRUE;
 
         } else if (safe_str_eq(join_state, exp_state)) {
             crm_info("Node %s is still coming up: %s", this_node->details->uname, join_state);
-            crm_info("\tha_state=%s, ccm_state=%s, crm_state=%s",
-                     crm_str(ha_state), crm_str(ccm_state), crm_str(crm_state));
+            crm_info("\tccm_state=%s, crm_state=%s",
+                     crm_str(ccm_state), crm_str(crm_state));
             this_node->details->standby = TRUE;
             this_node->details->pending = TRUE;
             online = TRUE;
 
         } else {
             crm_warn("Node %s (%s) is un-expectedly down",
                      this_node->details->uname, this_node->details->id);
-            crm_info("\tha_state=%s, ccm_state=%s,"
-                     " crm_state=%s, join_state=%s, expected=%s",
-                     crm_str(ha_state), crm_str(ccm_state),
-                     crm_str(crm_state), crm_str(join_state), crm_str(exp_state));
+            crm_info("\tccm_state=%s, crm_state=%s, join_state=%s, expected=%s",
+                     crm_str(ccm_state), crm_str(crm_state), crm_str(join_state), crm_str(exp_state));
             pe_fence_node(data_set, this_node, "because it is un-expectedly down");
         }
 
-    } else if (crm_is_true(ccm_state) == FALSE && safe_str_eq(ha_state, DEADSTATUS)
+    } else if (crm_is_true(ccm_state) == FALSE
                && safe_str_eq(crm_state, OFFLINESTATUS)
                && this_node->details->expected_up == FALSE) {
         crm_debug("Node %s is down: join_state=%s, expected=%s",
                   this_node->details->uname, crm_str(join_state), crm_str(exp_state));
 
 #if 0
         /* While a nice optimization, it causes the cluster to block until the node
          *  comes back online.  Which is a serious problem if the cluster software
          *  is not configured to start at boot or stonith is configured to merely
          *  stop the node instead of restart it.
          * Easily triggered by setting terminate=true for the DC
          */
     } else if (do_terminate) {
         crm_info("Node %s is %s after forced termination",
                  this_node->details->uname, crm_is_true(ccm_state) ? "coming up" : "going down");
-        crm_debug("\tha_state=%s, ccm_state=%s,"
-                  " crm_state=%s, join_state=%s, expected=%s",
-                  crm_str(ha_state), crm_str(ccm_state),
-                  crm_str(crm_state), crm_str(join_state), crm_str(exp_state));
+        crm_debug("\tccm_state=%s, crm_state=%s, join_state=%s, expected=%s",
+                  crm_str(ccm_state), crm_str(crm_state), crm_str(join_state), crm_str(exp_state));
 
         if (crm_is_true(ccm_state) == FALSE) {
             this_node->details->standby = TRUE;
             this_node->details->pending = TRUE;
             online = TRUE;
         }
 #endif
 
     } else if (this_node->details->expected_up) {
         /* mark it unclean */
         pe_fence_node(data_set, this_node, "because it is un-expectedly down");
-        crm_info("\tha_state=%s, ccm_state=%s,"
-                 " crm_state=%s, join_state=%s, expected=%s",
-                 crm_str(ha_state), crm_str(ccm_state),
-                 crm_str(crm_state), crm_str(join_state), crm_str(exp_state));
+        crm_info("\tccm_state=%s, crm_state=%s, join_state=%s, expected=%s",
+                 crm_str(ccm_state), crm_str(crm_state), crm_str(join_state), crm_str(exp_state));
 
     } else {
         crm_info("Node %s is down", this_node->details->uname);
-        crm_debug("\tha_state=%s, ccm_state=%s,"
-                  " crm_state=%s, join_state=%s, expected=%s",
-                  crm_str(ha_state), crm_str(ccm_state),
-                  crm_str(crm_state), crm_str(join_state), crm_str(exp_state));
+        crm_debug("\tccm_state=%s, crm_state=%s, join_state=%s, expected=%s",
+                  crm_str(ccm_state), crm_str(crm_state), crm_str(join_state), crm_str(exp_state));
     }
     return online;
 }
 
 gboolean
 determine_online_status(xmlNode * node_state, node_t * this_node, pe_working_set_t * data_set)
 {
     gboolean online = FALSE;
     const char *shutdown = NULL;
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
 
     if (this_node == NULL) {
         crm_config_err("No node to check");
         return online;
     }
 
     this_node->details->shutdown = FALSE;
     this_node->details->expected_up = FALSE;
     shutdown = g_hash_table_lookup(this_node->details->attrs, XML_CIB_ATTR_SHUTDOWN);
 
     if (shutdown != NULL && safe_str_neq("0", shutdown)) {
         this_node->details->shutdown = TRUE;
 
     } else if (safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) {
         this_node->details->expected_up = TRUE;
     }
 
     if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
         online = determine_online_status_no_fencing(data_set, node_state, this_node);
 
     } else {
         online = determine_online_status_fencing(data_set, node_state, this_node);
     }
 
     if (online) {
         this_node->details->online = TRUE;
 
     } else {
         /* remove node from contention */
         this_node->fixed = TRUE;
         this_node->weight = -INFINITY;
     }
 
     if (online && this_node->details->shutdown) {
         /* dont run resources here */
         this_node->fixed = TRUE;
         this_node->weight = -INFINITY;
     }
 
     if (this_node->details->unclean) {
         pe_proc_warn("Node %s is unclean", this_node->details->uname);
 
     } else if (this_node->details->online) {
         crm_info("Node %s is %s", this_node->details->uname,
                  this_node->details->shutdown ? "shutting down" :
                  this_node->details->pending ? "pending" :
                  this_node->details->standby ? "standby" : "online");
 
     } else {
         crm_trace("Node %s is offline", this_node->details->uname);
     }
 
     return online;
 }
 
 #define set_char(x) last_rsc_id[lpc] = x; complete = TRUE;
 
 char *
 clone_zero(const char *last_rsc_id)
 {
     int lpc = 0;
     char *zero = NULL;
 
     CRM_CHECK(last_rsc_id != NULL, return NULL);
     if (last_rsc_id != NULL) {
         lpc = strlen(last_rsc_id);
     }
 
     while (--lpc > 0) {
         switch (last_rsc_id[lpc]) {
             case 0:
                 return NULL;
                 break;
             case '0':
             case '1':
             case '2':
             case '3':
             case '4':
             case '5':
             case '6':
             case '7':
             case '8':
             case '9':
                 break;
             case ':':
                 zero = calloc(1, lpc + 3);
                 memcpy(zero, last_rsc_id, lpc);
                 zero[lpc] = ':';
                 zero[lpc + 1] = '0';
                 zero[lpc + 2] = 0;
                 return zero;
         }
     }
     return NULL;
 }
 
 char *
 increment_clone(char *last_rsc_id)
 {
     int lpc = 0;
     int len = 0;
     char *tmp = NULL;
     gboolean complete = FALSE;
 
     CRM_CHECK(last_rsc_id != NULL, return NULL);
     if (last_rsc_id != NULL) {
         len = strlen(last_rsc_id);
     }
 
     lpc = len - 1;
     while (complete == FALSE && lpc > 0) {
         switch (last_rsc_id[lpc]) {
             case 0:
                 lpc--;
                 break;
             case '0':
                 set_char('1');
                 break;
             case '1':
                 set_char('2');
                 break;
             case '2':
                 set_char('3');
                 break;
             case '3':
                 set_char('4');
                 break;
             case '4':
                 set_char('5');
                 break;
             case '5':
                 set_char('6');
                 break;
             case '6':
                 set_char('7');
                 break;
             case '7':
                 set_char('8');
                 break;
             case '8':
                 set_char('9');
                 break;
             case '9':
                 last_rsc_id[lpc] = '0';
                 lpc--;
                 break;
             case ':':
                 tmp = last_rsc_id;
                 last_rsc_id = calloc(1, len + 2);
                 memcpy(last_rsc_id, tmp, len);
                 last_rsc_id[++lpc] = '1';
                 last_rsc_id[len] = '0';
                 last_rsc_id[len + 1] = 0;
                 complete = TRUE;
                 free(tmp);
                 break;
             default:
                 crm_err("Unexpected char: %c (%d)", last_rsc_id[lpc], lpc);
                 return NULL;
                 break;
         }
     }
     return last_rsc_id;
 }
 
 static int
 get_clone(char *last_rsc_id)
 {
     int clone = 0;
     int lpc = 0;
     int len = 0;
 
     CRM_CHECK(last_rsc_id != NULL, return -1);
     if (last_rsc_id != NULL) {
         len = strlen(last_rsc_id);
     }
 
     lpc = len - 1;
     while (lpc > 0) {
         switch (last_rsc_id[lpc]) {
             case '0':
             case '1':
             case '2':
             case '3':
             case '4':
             case '5':
             case '6':
             case '7':
             case '8':
             case '9':
                 clone += (int)(last_rsc_id[lpc] - '0') * (len - lpc);
                 lpc--;
                 break;
             case ':':
                 return clone;
                 break;
             default:
                 crm_err("Unexpected char: %d (%c)", lpc, last_rsc_id[lpc]);
                 return clone;
                 break;
         }
     }
     return -1;
 }
 
 static resource_t *
 create_fake_resource(const char *rsc_id, xmlNode * rsc_entry, pe_working_set_t * data_set)
 {
     resource_t *rsc = NULL;
     xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
 
     copy_in_properties(xml_rsc, rsc_entry);
     crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
     crm_log_xml_debug(xml_rsc, "Orphan resource");
 
     if (!common_unpack(xml_rsc, &rsc, NULL, data_set)) {
         return NULL;
     }
 
     set_bit(rsc->flags, pe_rsc_orphan);
     data_set->resources = g_list_append(data_set->resources, rsc);
     return rsc;
 }
 
 extern resource_t *create_child_clone(resource_t * rsc, int sub_id, pe_working_set_t * data_set);
 
 static resource_t *
 find_clone(pe_working_set_t * data_set, node_t * node, resource_t * parent, const char *rsc_id)
 {
     int len = 0;
     resource_t *rsc = NULL;
     char *base = clone_zero(rsc_id);
     char *alt_rsc_id = NULL;
 
     CRM_ASSERT(parent != NULL);
     CRM_ASSERT(parent->variant == pe_clone || parent->variant == pe_master);
 
     if (base) {
         len = strlen(base);
     }
     if (len > 0) {
         base[len - 1] = 0;
     }
 
     crm_trace("Looking for %s on %s in %s %d",
               rsc_id, node->details->uname, parent->id, is_set(parent->flags, pe_rsc_unique));
 
     if (is_set(parent->flags, pe_rsc_unique)) {
         crm_trace("Looking for %s", rsc_id);
         rsc = parent->fns->find_rsc(parent, rsc_id, NULL, pe_find_current);
 
     } else {
         crm_trace("Looking for %s on %s", base, node->details->uname);
         rsc = parent->fns->find_rsc(parent, base, node, pe_find_partial | pe_find_current);
         if (rsc != NULL && rsc->running_on) {
             GListPtr gIter = parent->children;
 
             rsc = NULL;
             crm_trace("Looking for an existing orphan for %s: %s on %s", parent->id, rsc_id,
                       node->details->uname);
 
             /* There is already an instance of this _anonymous_ clone active on "node".
              *
              * If there is a partially active orphan (only applies to clone groups) on
              * the same node, use that.
              * Otherwise create a new (orphaned) instance at "orphan_check:".
              */
 
             for (; gIter != NULL; gIter = gIter->next) {
                 resource_t *child = (resource_t *) gIter->data;
                 node_t *loc = child->fns->location(child, NULL, TRUE);
 
                 if (loc && loc->details == node->details) {
                     resource_t *tmp =
                         child->fns->find_rsc(child, base, NULL, pe_find_partial | pe_find_current);
                     if (tmp && tmp->running_on == NULL) {
                         rsc = tmp;
                         break;
                     }
                 }
             }
 
             goto orphan_check;
 
         } else if (((resource_t *) parent->children->data)->variant == pe_group) {
             /* If we're grouped, we need to look for a peer thats active on $node
              * and use their clone instance number
              */
             resource_t *peer =
                 parent->fns->find_rsc(parent, NULL, node, pe_find_clone | pe_find_current);
             if (peer && peer->running_on) {
                 char buffer[256];
                 int clone_num = get_clone(peer->id);
 
                 snprintf(buffer, 256, "%s%d", base, clone_num);
                 rsc =
                     parent->fns->find_rsc(parent, buffer, node, pe_find_current | pe_find_inactive);
                 if (rsc) {
                     crm_trace("Found someone active: %s on %s, becoming %s", peer->id,
                               ((node_t *) peer->running_on->data)->details->uname, buffer);
                 }
             }
         }
 
         if (parent->fns->find_rsc(parent, rsc_id, NULL, pe_find_current)) {
             alt_rsc_id = strdup(rsc_id);
         } else {
             alt_rsc_id = clone_zero(rsc_id);
         }
 
         while (rsc == NULL) {
             rsc = parent->fns->find_rsc(parent, alt_rsc_id, NULL, pe_find_current);
             if (rsc == NULL) {
                 crm_trace("Unknown resource: %s", alt_rsc_id);
                 break;
             }
 
             if (rsc->running_on == NULL) {
                 crm_trace("Resource %s: just right", alt_rsc_id);
                 break;
             }
 
             crm_trace("Resource %s: already active", alt_rsc_id);
             alt_rsc_id = increment_clone(alt_rsc_id);
             rsc = NULL;
         }
     }
 
   orphan_check:
     if (rsc == NULL) {
         /* Create an extra orphan */
         resource_t *top = create_child_clone(parent, -1, data_set);
 
         crm_debug("Created orphan for %s: %s on %s", parent->id, rsc_id, node->details->uname);
         rsc = top->fns->find_rsc(top, base, NULL, pe_find_current | pe_find_partial);
         CRM_ASSERT(rsc != NULL);
     }
 
     free(rsc->clone_name);
     rsc->clone_name = NULL;
     if (safe_str_neq(rsc_id, rsc->id)) {
         crm_info("Internally renamed %s on %s to %s%s",
                  rsc_id, node->details->uname, rsc->id,
                  is_set(rsc->flags, pe_rsc_orphan) ? " (ORPHAN)" : "");
         rsc->clone_name = strdup(rsc_id);
     }
 
     free(alt_rsc_id);
     free(base);
     return rsc;
 }
 
 static resource_t *
 unpack_find_resource(pe_working_set_t * data_set, node_t * node, const char *rsc_id,
                      xmlNode * rsc_entry)
 {
     resource_t *rsc = NULL;
     resource_t *clone_parent = NULL;
     char *alt_rsc_id = strdup(rsc_id);
 
     crm_trace("looking for %s", rsc_id);
 
     rsc = pe_find_resource(data_set->resources, alt_rsc_id);
     /* no match */
     if (rsc == NULL) {
         /* Even when clone-max=0, we still create a single :0 orphan to match against */
         char *tmp = clone_zero(alt_rsc_id);
         resource_t *clone0 = pe_find_resource(data_set->resources, tmp);
 
         clone_parent = uber_parent(clone0);
         free(tmp);
 
         crm_trace("%s not found: %s", alt_rsc_id, clone_parent ? clone_parent->id : "orphan");
 
     } else {
         clone_parent = uber_parent(rsc);
     }
 
     if (clone_parent && clone_parent->variant > pe_group) {
         rsc = find_clone(data_set, node, clone_parent, rsc_id);
         CRM_ASSERT(rsc != NULL);
     }
 
     free(alt_rsc_id);
     return rsc;
 }
 
 static resource_t *
 process_orphan_resource(xmlNode * rsc_entry, node_t * node, pe_working_set_t * data_set)
 {
     resource_t *rsc = NULL;
     const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
 
     crm_debug("Detected orphan resource %s on %s", rsc_id, node->details->uname);
     rsc = create_fake_resource(rsc_id, rsc_entry, data_set);
 
     if (is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
         clear_bit(rsc->flags, pe_rsc_managed);
 
     } else {
         GListPtr gIter = NULL;
 
         print_resource(LOG_DEBUG_3, "Added orphan", rsc, FALSE);
 
         CRM_CHECK(rsc != NULL, return NULL);
         resource_location(rsc, NULL, -INFINITY, "__orphan_dont_run__", data_set);
 
         for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
             node_t *node = (node_t *) gIter->data;
 
             if (node->details->online && get_failcount(node, rsc, NULL, data_set)) {
                 action_t *clear_op = NULL;
                 action_t *ready = get_pseudo_op(CRM_OP_PROBED, data_set);
 
                 clear_op = custom_action(rsc, crm_concat(rsc->id, CRM_OP_CLEAR_FAILCOUNT, '_'),
                                          CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE, data_set);
 
                 add_hash_param(clear_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
                 crm_info("Clearing failcount (%d) for orphaned resource %s on %s (%s)",
                          get_failcount(node, rsc, NULL, data_set), rsc->id, node->details->uname,
                          clear_op->uuid);
 
                 order_actions(clear_op, ready, pe_order_optional);
             }
         }
     }
     return rsc;
 }
 
 static void
 process_rsc_state(resource_t * rsc, node_t * node,
                   enum action_fail_response on_fail,
                   xmlNode * migrate_op, pe_working_set_t * data_set)
 {
     crm_trace("Resource %s is %s on %s: on_fail=%s",
               rsc->id, role2text(rsc->role), node->details->uname, fail2text(on_fail));
 
     /* process current state */
     if (rsc->role != RSC_ROLE_UNKNOWN) {
         resource_t *iter = rsc;
 
         while (iter) {
             if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
                 node_t *n = node_copy(node);
 
                 g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
             }
             if (is_set(iter->flags, pe_rsc_unique)) {
                 break;
             }
             iter = iter->parent;
         }
     }
 
     if (node->details->unclean) {
         /* No extra processing needed
          * Also allows resources to be started again after a node is shot
          */
         on_fail = action_fail_ignore;
     }
 
     switch (on_fail) {
         case action_fail_ignore:
             /* nothing to do */
             break;
 
         case action_fail_fence:
             /* treat it as if it is still running
              * but also mark the node as unclean
              */
             pe_fence_node(data_set, node, "to recover from resource failure(s)");
             break;
 
         case action_fail_standby:
             node->details->standby = TRUE;
             node->details->standby_onfail = TRUE;
             break;
 
         case action_fail_block:
             /* is_managed == FALSE will prevent any
              * actions being sent for the resource
              */
             clear_bit(rsc->flags, pe_rsc_managed);
             set_bit(rsc->flags, pe_rsc_block);
             break;
 
         case action_fail_migrate:
             /* make sure it comes up somewhere else
              * or not at all
              */
             resource_location(rsc, node, -INFINITY, "__action_migration_auto__", data_set);
             break;
 
         case action_fail_stop:
             rsc->next_role = RSC_ROLE_STOPPED;
             break;
 
         case action_fail_recover:
             if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
                 set_bit(rsc->flags, pe_rsc_failed);
                 stop_action(rsc, node, FALSE);
             }
             break;
     }
 
     if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
         if (is_set(rsc->flags, pe_rsc_orphan)) {
             if (is_set(rsc->flags, pe_rsc_managed)) {
                 crm_config_warn("Detected active orphan %s running on %s",
                                 rsc->id, node->details->uname);
             } else {
                 crm_config_warn("Cluster configured not to stop active orphans."
                                 " %s must be stopped manually on %s",
                                 rsc->id, node->details->uname);
             }
         }
 
         native_add_running(rsc, node, data_set);
         if (on_fail != action_fail_ignore) {
             set_bit(rsc->flags, pe_rsc_failed);
         }
 
     } else if (rsc->clone_name) {
         crm_trace("Resetting clone_name %s for %s (stopped)", rsc->clone_name, rsc->id);
         free(rsc->clone_name);
         rsc->clone_name = NULL;
 
     } else {
         char *key = stop_key(rsc);
         GListPtr possible_matches = find_actions(rsc->actions, key, node);
         GListPtr gIter = possible_matches;
 
         for (; gIter != NULL; gIter = gIter->next) {
             action_t *stop = (action_t *) gIter->data;
 
             stop->flags |= pe_action_optional;
         }
 
         free(key);
     }
 }
 
 /* create active recurring operations as optional */
 static void
 process_recurring(node_t * node, resource_t * rsc,
                   int start_index, int stop_index,
                   GListPtr sorted_op_list, pe_working_set_t * data_set)
 {
     int counter = -1;
     const char *task = NULL;
     const char *status = NULL;
     GListPtr gIter = sorted_op_list;
 
     crm_trace("%s: Start index %d, stop index = %d", rsc->id, start_index, stop_index);
 
     for (; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         int interval = 0;
         char *key = NULL;
         const char *id = ID(rsc_op);
         const char *interval_s = NULL;
 
         counter++;
 
         if (node->details->online == FALSE) {
             crm_trace("Skipping %s/%s: node is offline", rsc->id, node->details->uname);
             break;
 
             /* Need to check if there's a monitor for role="Stopped" */
         } else if (start_index < stop_index && counter <= stop_index) {
             crm_trace("Skipping %s/%s: resource is not active", id, node->details->uname);
             continue;
 
         } else if (counter < start_index) {
             crm_trace("Skipping %s/%s: old %d", id, node->details->uname, counter);
             continue;
         }
 
         interval_s = crm_element_value(rsc_op, XML_LRM_ATTR_INTERVAL);
         interval = crm_parse_int(interval_s, "0");
         if (interval == 0) {
             crm_trace("Skipping %s/%s: non-recurring", id, node->details->uname);
             continue;
         }
 
         status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
         if (safe_str_eq(status, "-1")) {
             crm_trace("Skipping %s/%s: status", id, node->details->uname);
             continue;
         }
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         /* create the action */
         key = generate_op_key(rsc->id, task, interval);
         crm_trace("Creating %s/%s", key, node->details->uname);
         custom_action(rsc, key, task, node, TRUE, TRUE, data_set);
     }
 }
 
 void
 calculate_active_ops(GListPtr sorted_op_list, int *start_index, int *stop_index)
 {
     int counter = -1;
     int implied_monitor_start = -1;
     int implied_master_start = -1;
     const char *task = NULL;
     const char *status = NULL;
     GListPtr gIter = sorted_op_list;
 
     *stop_index = -1;
     *start_index = -1;
 
     for (; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         counter++;
 
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
 
         if (safe_str_eq(task, CRMD_ACTION_STOP)
             && safe_str_eq(status, "0")) {
             *stop_index = counter;
 
         } else if (safe_str_eq(task, CRMD_ACTION_START) ||
             safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
             *start_index = counter;
 
         } else if ((implied_monitor_start <= *stop_index) && safe_str_eq(task, CRMD_ACTION_STATUS)) {
             const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);
 
             if (safe_str_eq(rc, "0") || safe_str_eq(rc, "8")) {
                 implied_monitor_start = counter;
             }
         } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE) || safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
             implied_master_start = counter;
         }
     }
 
     if (*start_index == -1) {
         if (implied_master_start != -1) {
             *start_index = implied_master_start;
         } else if (implied_monitor_start != -1) {
             *start_index = implied_monitor_start;
         }
     }
 }
 
 static void
 unpack_lrm_rsc_state(node_t * node, xmlNode * rsc_entry, pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     int stop_index = -1;
     int start_index = -1;
     enum rsc_role_e req_role = RSC_ROLE_UNKNOWN;
 
     const char *task = NULL;
     const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
 
     resource_t *rsc = NULL;
     GListPtr op_list = NULL;
     GListPtr sorted_op_list = NULL;
 
     xmlNode *migrate_op = NULL;
     xmlNode *rsc_op = NULL;
 
     enum action_fail_response on_fail = FALSE;
     enum rsc_role_e saved_role = RSC_ROLE_UNKNOWN;
 
     crm_trace("[%s] Processing %s on %s",
               crm_element_name(rsc_entry), rsc_id, node->details->uname);
 
     /* extract operations */
     op_list = NULL;
     sorted_op_list = NULL;
 
     for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next(rsc_op)) {
         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             op_list = g_list_prepend(op_list, rsc_op);
         }
     }
 
     if (op_list == NULL) {
         /* if there are no operations, there is nothing to do */
         return;
     }
 
     /* find the resource */
     rsc = unpack_find_resource(data_set, node, rsc_id, rsc_entry);
     if (rsc == NULL) {
         rsc = process_orphan_resource(rsc_entry, node, data_set);
     }
     CRM_ASSERT(rsc != NULL);
 
     /* process operations */
     saved_role = rsc->role;
     on_fail = action_fail_ignore;
     rsc->role = RSC_ROLE_UNKNOWN;
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
         if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
             migrate_op = rsc_op;
         }
 
         unpack_rsc_op(rsc, node, rsc_op, gIter->next, &on_fail, data_set);
     }
 
     /* create active recurring operations as optional */
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
     process_recurring(node, rsc, start_index, stop_index, sorted_op_list, data_set);
 
     /* no need to free the contents */
     g_list_free(sorted_op_list);
 
     process_rsc_state(rsc, node, on_fail, migrate_op, data_set);
 
     if (get_target_role(rsc, &req_role)) {
         if (rsc->next_role == RSC_ROLE_UNKNOWN || req_role < rsc->next_role) {
             crm_debug("%s: Overwriting calculated next role %s"
                       " with requested next role %s",
                       rsc->id, role2text(rsc->next_role), role2text(req_role));
             rsc->next_role = req_role;
 
         } else if (req_role > rsc->next_role) {
             crm_info("%s: Not overwriting calculated next role %s"
                      " with requested next role %s",
                      rsc->id, role2text(rsc->next_role), role2text(req_role));
         }
     }
 
     if (saved_role > rsc->role) {
         rsc->role = saved_role;
     }
 }
 
 gboolean
 unpack_lrm_resources(node_t * node, xmlNode * lrm_rsc_list, pe_working_set_t * data_set)
 {
     xmlNode *rsc_entry = NULL;
 
     CRM_CHECK(node != NULL, return FALSE);
 
     crm_trace("Unpacking resources on %s", node->details->uname);
 
     for (rsc_entry = __xml_first_child(lrm_rsc_list); rsc_entry != NULL;
          rsc_entry = __xml_next(rsc_entry)) {
         if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
             unpack_lrm_rsc_state(node, rsc_entry, data_set);
         }
     }
 
     return TRUE;
 }
 
 static void
 set_active(resource_t * rsc)
 {
     resource_t *top = uber_parent(rsc);
 
     if (top && top->variant == pe_master) {
         rsc->role = RSC_ROLE_SLAVE;
     } else {
         rsc->role = RSC_ROLE_STARTED;
     }
 }
 
 static void
 set_node_score(gpointer key, gpointer value, gpointer user_data)
 {
     node_t *node = value;
     int *score = user_data;
 
     node->weight = *score;
 }
 
 #define STATUS_PATH_MAX 1024
 static xmlNode *
 find_lrm_op(const char *resource, const char *op, const char *node, const char *source,
             pe_working_set_t * data_set)
 {
     int offset = 0;
     char xpath[STATUS_PATH_MAX];
 
     offset += snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//node_state[@uname='%s']", node);
     offset +=
         snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//" XML_LRM_TAG_RESOURCE "[@id='%s']",
                  resource);
 
     /* Need to check against transition_magic too? */
     if (source && safe_str_eq(op, CRMD_ACTION_MIGRATE)) {
         offset +=
             snprintf(xpath + offset, STATUS_PATH_MAX - offset,
                      "/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_target='%s']", op,
                      source);
     } else if (source && safe_str_eq(op, CRMD_ACTION_MIGRATED)) {
         offset +=
             snprintf(xpath + offset, STATUS_PATH_MAX - offset,
                      "/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_source='%s']", op,
                      source);
     } else {
         offset +=
             snprintf(xpath + offset, STATUS_PATH_MAX - offset,
                      "/" XML_LRM_TAG_RSC_OP "[@operation='%s']", op);
     }
 
     return get_xpath_object(xpath, data_set->input, LOG_DEBUG);
 }
 
 gboolean
 unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, GListPtr next,
               enum action_fail_response * on_fail, pe_working_set_t * data_set)
 {
     int task_id = 0;
 
     const char *id = NULL;
     const char *key = NULL;
     const char *task = NULL;
     const char *task_key = NULL;
     const char *magic = NULL;
     const char *actual_rc = NULL;
 
 /* 	const char *target_rc   = NULL;	 */
     const char *task_status = NULL;
     const char *interval_s = NULL;
     const char *op_version = NULL;
 
     int interval = 0;
     int task_status_i = -2;
     int actual_rc_i = 0;
     int target_rc = -1;
     int last_failure = 0;
 
     action_t *action = NULL;
     node_t *effective_node = NULL;
     resource_t *failed = NULL;
 
     gboolean expired = FALSE;
     gboolean is_probe = FALSE;
     gboolean clear_past_failure = FALSE;
 
     CRM_CHECK(rsc != NULL, return FALSE);
     CRM_CHECK(node != NULL, return FALSE);
     CRM_CHECK(xml_op != NULL, return FALSE);
 
     id = ID(xml_op);
     task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     task_key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
     task_status = crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS);
     op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
     magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);
     key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);
 
     crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &task_id);
 
     CRM_CHECK(id != NULL, return FALSE);
     CRM_CHECK(task != NULL, return FALSE);
     CRM_CHECK(task_status != NULL, return FALSE);
 
     task_status_i = crm_parse_int(task_status, NULL);
 
     CRM_CHECK(task_status_i <= PCMK_LRM_OP_ERROR, return FALSE);
     CRM_CHECK(task_status_i >= PCMK_LRM_OP_PENDING, return FALSE);
 
     if (safe_str_eq(task, CRMD_ACTION_NOTIFY)) {
         /* safe to ignore these */
         return TRUE;
     }
 
     if (rsc->failure_timeout > 0) {
         int last_run = 0;
 
         if (crm_element_value_int(xml_op, "last-rc-change", &last_run) == 0) {
             time_t now = get_timet_now(data_set);
 
             if (now > (last_run + rsc->failure_timeout)) {
                 expired = TRUE;
             }
         }
     }
 
     crm_trace("Unpacking task %s/%s (call_id=%d, status=%s) on %s (role=%s)",
               id, task, task_id, task_status, node->details->uname, role2text(rsc->role));
 
     interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
     interval = crm_parse_int(interval_s, "0");
 
     if (interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
         is_probe = TRUE;
     }
 
     if (node->details->unclean) {
         crm_trace("Node %s (where %s is running) is unclean."
                   " Further action depends on the value of the stop's on-fail attribue",
                   node->details->uname, rsc->id);
     }
 
     actual_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC);
     CRM_CHECK(actual_rc != NULL, return FALSE);
     actual_rc_i = crm_parse_int(actual_rc, NULL);
 
     if (key) {
         int dummy = 0;
         char *dummy_string = NULL;
 
         decode_transition_key(key, &dummy_string, &dummy, &dummy, &target_rc);
         free(dummy_string);
     }
 
     if (task_status_i == PCMK_LRM_OP_DONE && target_rc >= 0) {
         if (target_rc == actual_rc_i) {
             task_status_i = PCMK_LRM_OP_DONE;
 
         } else {
             task_status_i = PCMK_LRM_OP_ERROR;
             crm_debug("%s on %s returned %d (%s) instead of the expected value: %d (%s)",
                       id, node->details->uname,
                       actual_rc_i, lrmd_event_rc2str(actual_rc_i),
                       target_rc, lrmd_event_rc2str(target_rc));
         }
 
     } else if (task_status_i == PCMK_LRM_OP_ERROR) {
         /* let us decide that */
         task_status_i = PCMK_LRM_OP_DONE;
     }
 
     if (task_status_i == PCMK_LRM_OP_NOTSUPPORTED) {
         actual_rc_i = PCMK_EXECRA_UNIMPLEMENT_FEATURE;
     }
 
     if (task_status_i != actual_rc_i
         && rsc->failure_timeout > 0 && get_failcount(node, rsc, &last_failure, data_set) == 0) {
         if (last_failure > 0) {
             action_t *clear_op = NULL;
 
             clear_op = custom_action(rsc, crm_concat(rsc->id, CRM_OP_CLEAR_FAILCOUNT, '_'),
                                      CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE, data_set);
 
             add_hash_param(clear_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
             crm_notice("Clearing expired failcount for %s on %s", rsc->id, node->details->uname);
         }
     }
 
     if (expired
         && actual_rc_i != PCMK_EXECRA_NOT_RUNNING
         && actual_rc_i != PCMK_EXECRA_RUNNING_MASTER && actual_rc_i != PCMK_EXECRA_OK) {
         crm_notice("Ignoring expired failure %s (rc=%d, magic=%s) on %s",
                    id, actual_rc_i, magic, node->details->uname);
         goto done;
     }
 
     /* we could clean this up significantly except for old LRMs and CRMs that
      * didnt include target_rc and liked to remap status
      */
     switch (actual_rc_i) {
         case PCMK_EXECRA_NOT_RUNNING:
             if (is_probe || target_rc == actual_rc_i) {
                 task_status_i = PCMK_LRM_OP_DONE;
                 rsc->role = RSC_ROLE_STOPPED;
 
                 /* clear any previous failure actions */
                 *on_fail = action_fail_ignore;
                 rsc->next_role = RSC_ROLE_UNKNOWN;
 
             } else if (safe_str_neq(task, CRMD_ACTION_STOP)) {
                 task_status_i = PCMK_LRM_OP_ERROR;
             }
             break;
 
         case PCMK_EXECRA_RUNNING_MASTER:
             if (is_probe) {
                 task_status_i = PCMK_LRM_OP_DONE;
                 crm_notice("Operation %s found resource %s active in master mode on %s",
                            task, rsc->id, node->details->uname);
 
             } else if (target_rc == actual_rc_i) {
                 /* nothing to do */
 
             } else if (target_rc >= 0) {
                 task_status_i = PCMK_LRM_OP_ERROR;
 
                 /* legacy code for pre-0.6.5 operations */
             } else if (safe_str_neq(task, CRMD_ACTION_STATUS)
                        || rsc->role != RSC_ROLE_MASTER) {
                 task_status_i = PCMK_LRM_OP_ERROR;
                 if (rsc->role != RSC_ROLE_MASTER) {
                     crm_err("%s reported %s in master mode on %s",
                             id, rsc->id, node->details->uname);
                 }
             }
             rsc->role = RSC_ROLE_MASTER;
             break;
 
         case PCMK_EXECRA_FAILED_MASTER:
             rsc->role = RSC_ROLE_MASTER;
             task_status_i = PCMK_LRM_OP_ERROR;
             break;
 
         case PCMK_EXECRA_UNIMPLEMENT_FEATURE:
             if (interval > 0) {
                 task_status_i = PCMK_LRM_OP_NOTSUPPORTED;
                 break;
             }
             /* else: fall through */
         case PCMK_EXECRA_INSUFFICIENT_PRIV:
         case PCMK_EXECRA_NOT_INSTALLED:
         case PCMK_EXECRA_INVALID_PARAM:
             effective_node = node;
             /* fall through */
         case PCMK_EXECRA_NOT_CONFIGURED:
             failed = rsc;
             if (is_not_set(rsc->flags, pe_rsc_unique)) {
                 failed = uber_parent(failed);
             }
 
             do_crm_log(actual_rc_i == PCMK_EXECRA_NOT_INSTALLED ? LOG_NOTICE : LOG_ERR,
                        "Preventing %s from re-starting %s %s: operation %s failed '%s' (rc=%d)",
                        failed->id,
                        effective_node ? "on" : "anywhere in the cluster",
                        effective_node ? effective_node->details->uname : "",
                        task, lrmd_event_rc2str(actual_rc_i), actual_rc_i);
 
             resource_location(failed, effective_node, -INFINITY, "hard-error", data_set);
             if (is_probe) {
                 /* treat these like stops */
                 task = CRMD_ACTION_STOP;
                 task_status_i = PCMK_LRM_OP_DONE;
                 crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
                 if (actual_rc_i != PCMK_EXECRA_NOT_INSTALLED
                     || is_set(data_set->flags, pe_flag_symmetric_cluster)) {
                     if ((node->details->shutdown == FALSE) || (node->details->online == TRUE)) {
                         add_node_copy(data_set->failed, xml_op);
                     }
                 }
             }
             break;
 
         case PCMK_EXECRA_OK:
             if (is_probe && target_rc == 7) {
                 task_status_i = PCMK_LRM_OP_DONE;
                 crm_info("Operation %s found resource %s active on %s",
                          task, rsc->id, node->details->uname);
 
                 /* legacy code for pre-0.6.5 operations */
             } else if (target_rc < 0 && interval > 0 && rsc->role == RSC_ROLE_MASTER) {
                 /* catch status ops that return 0 instead of 8 while they
                  *   are supposed to be in master mode
                  */
                 task_status_i = PCMK_LRM_OP_ERROR;
             }
 
             break;
 
         default:
             if (task_status_i == PCMK_LRM_OP_DONE) {
                 crm_info("Remapping %s (rc=%d) on %s to an ERROR",
                          id, actual_rc_i, node->details->uname);
                 task_status_i = PCMK_LRM_OP_ERROR;
             }
     }
 
     if (task_status_i == PCMK_LRM_OP_ERROR
         || task_status_i == PCMK_LRM_OP_TIMEOUT || task_status_i == PCMK_LRM_OP_NOTSUPPORTED) {
         const char *action_key = task_key ? task_key : id;
         action = custom_action(rsc, strdup(action_key), task, NULL, TRUE, FALSE, data_set);
         if (expired) {
             crm_notice("Ignoring expired failure (calculated) %s (rc=%d, magic=%s) on %s",
                        id, actual_rc_i, magic, node->details->uname);
             goto done;
 
         } else if (action->on_fail == action_fail_ignore) {
             crm_warn("Remapping %s (rc=%d) on %s to DONE: ignore",
                      id, actual_rc_i, node->details->uname);
             task_status_i = PCMK_LRM_OP_DONE;
             set_bit(rsc->flags, pe_rsc_failure_ignored);
 
             crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
             if ((node->details->shutdown == FALSE) || (node->details->online == TRUE)) {
                 add_node_copy(data_set->failed, xml_op);
             }
         }
     }
 
     switch (task_status_i) {
         case PCMK_LRM_OP_PENDING:
             if (safe_str_eq(task, CRMD_ACTION_START)) {
                 set_bit(rsc->flags, pe_rsc_start_pending);
                 set_active(rsc);
 
             } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
                 rsc->role = RSC_ROLE_MASTER;
             }
             /*
              * Intentionally ignoring pending migrate ops here;
              * haven't decided if we need to do anything special
              * with them yet...
              */
             break;
 
         case PCMK_LRM_OP_DONE:
             crm_trace("%s/%s completed on %s", rsc->id, task, node->details->uname);
 
             if (actual_rc_i == PCMK_EXECRA_NOT_RUNNING) {
                 clear_past_failure = TRUE;
 
             } else if (safe_str_eq(task, CRMD_ACTION_START)) {
                 rsc->role = RSC_ROLE_STARTED;
                 clear_past_failure = TRUE;
 
             } else if (safe_str_eq(task, CRMD_ACTION_STOP)) {
                 rsc->role = RSC_ROLE_STOPPED;
                 clear_past_failure = TRUE;
 
             } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
                 rsc->role = RSC_ROLE_MASTER;
                 clear_past_failure = TRUE;
 
             } else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
                 /* Demote from Master does not clear an error */
                 rsc->role = RSC_ROLE_SLAVE;
 
             } else if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
                 rsc->role = RSC_ROLE_STARTED;
                 clear_past_failure = TRUE;
 
             } else if (safe_str_eq(task, CRMD_ACTION_MIGRATE)) {
                 /*
                  * The normal sequence is (now): migrate_to(Src) -> migrate_from(Tgt) -> stop(Src)
                  *
                  * So if a migrate_to is followed by a stop, then we dont need to care what
                  * happended on the target node
                  *
                  * Without the stop, we need to look for a successful migrate_from.
                  * This would also imply we're no longer running on the source
                  *
                  * Without the stop, and without a migrate_from op we make sure the resource
                  * gets stopped on both source and target (assuming the target is up) 
                  *
                  */
                 int stop_id = 0;
                 xmlNode *stop_op =
                     find_lrm_op(rsc->id, CRMD_ACTION_STOP, node->details->id, NULL, data_set);
 
                 if (stop_op) {
                     crm_element_value_int(stop_op, XML_LRM_ATTR_CALLID, &stop_id);
                 }
 
                 if (stop_op == NULL || stop_id < task_id) {
                     int from_rc = 0, from_status = 0;
                     const char *migrate_source =
                         crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
                     const char *migrate_target =
                         crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
                     node_t *target = pe_find_node(data_set->nodes, migrate_target);
                     node_t *source = pe_find_node(data_set->nodes, migrate_source);
                     xmlNode *migrate_from =
                         find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, migrate_target, migrate_source,
                                     data_set);
 
                     rsc->role = RSC_ROLE_STARTED;       /* can be master? */
                     if (migrate_from) {
                         crm_element_value_int(migrate_from, XML_LRM_ATTR_RC, &from_rc);
                         crm_element_value_int(migrate_from, XML_LRM_ATTR_OPSTATUS, &from_status);
                         crm_trace("%s op on %s exited with status=%d, rc=%d",
                                   ID(migrate_from), migrate_target, from_status, from_rc);
                     }
 
                     if (migrate_from && from_rc == PCMK_EXECRA_OK && from_status == PCMK_LRM_OP_DONE) {
                         crm_trace("Detected dangling migration op: %s on %s", ID(xml_op),
                                   migrate_source);
 
                         /* all good
                          * just need to arrange for the stop action to get sent
                          * but _without_ affecting the target somehow
                          */
                         rsc->role = RSC_ROLE_STOPPED;
                         rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);
 
                     } else if (migrate_from) {  /* Failed */
                         crm_trace("Marking active on %s %p %d", migrate_target, target,
                                   target->details->online);
                         if (target && target->details->online) {
                             native_add_running(rsc, target, data_set);
                         }
 
                     } else {    /* Pending or complete but erased */
                         node_t *target = pe_find_node_id(data_set->nodes, migrate_target);
 
                         crm_trace("Marking active on %s %p %d", migrate_target, target,
                                   target->details->online);
                         if (target && target->details->online) {
                             native_add_running(rsc, target, data_set);
                             if (source && source->details->online) {
                                 /* If we make it here we have a partial migration.  The migrate_to
                                  * has completed but the migrate_from on the target has not. Hold on
                                  * to the target and source on the resource. Later on if we detect that
                                  * the resource is still going to run on that target, we may continue
                                  * the migration */
                                 rsc->partial_migration_target = target;
                                 rsc->partial_migration_source = source;
                             }
                         } else {
                             /* Consider it failed here - forces a restart, prevents migration */
                             set_bit(rsc->flags, pe_rsc_failed);
                         }
                     }
                 }
 
             } else if (rsc->role < RSC_ROLE_STARTED) {
                 /* start, migrate_to and migrate_from will land here */
                 crm_trace("%s active on %s", rsc->id, node->details->uname);
                 set_active(rsc);
             }
 
             /* clear any previous failure actions */
             if (clear_past_failure) {
                 switch (*on_fail) {
                     case action_fail_block:
                     case action_fail_stop:
                     case action_fail_fence:
                     case action_fail_migrate:
                     case action_fail_standby:
                         crm_trace("%s.%s is not cleared by a completed stop",
                                   rsc->id, fail2text(*on_fail));
                         break;
 
                     case action_fail_ignore:
                     case action_fail_recover:
                         *on_fail = action_fail_ignore;
                         rsc->next_role = RSC_ROLE_UNKNOWN;
                 }
             }
             break;
 
         case PCMK_LRM_OP_ERROR:
         case PCMK_LRM_OP_TIMEOUT:
         case PCMK_LRM_OP_NOTSUPPORTED:
             crm_warn("Processing failed op %s on %s: %s (%d)",
                      id, node->details->uname, lrmd_event_rc2str(actual_rc_i), actual_rc_i);
             crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
             if ((node->details->shutdown == FALSE) || (node->details->online == TRUE)) {
                 add_node_copy(data_set->failed, xml_op);
             }
 
             if (*on_fail < action->on_fail) {
                 *on_fail = action->on_fail;
             }
 
             if (safe_str_eq(task, CRMD_ACTION_STOP)) {
                 resource_location(rsc, node, -INFINITY, "__stop_fail__", data_set);
 
             } else if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
                 int stop_id = 0;
                 int migrate_id = 0;
                 const char *migrate_source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
                 const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
                 xmlNode *stop_op =
                     find_lrm_op(rsc->id, CRMD_ACTION_STOP, migrate_source, NULL, data_set);
                 xmlNode *migrate_op =
                     find_lrm_op(rsc->id, CRMD_ACTION_MIGRATE, migrate_source, migrate_target,
                                 data_set);
 
                 if (stop_op) {
                     crm_element_value_int(stop_op, XML_LRM_ATTR_CALLID, &stop_id);
                 }
                 if (migrate_op) {
                     crm_element_value_int(migrate_op, XML_LRM_ATTR_CALLID, &migrate_id);
                 }
 
                 /* Get our state right */
                 rsc->role = RSC_ROLE_STARTED;   /* can be master? */
 
                 if (stop_op == NULL || stop_id < migrate_id) {
                     node_t *source = pe_find_node(data_set->nodes, migrate_source);
 
                     if (source && source->details->online) {
                         native_add_running(rsc, source, data_set);
                     }
                 }
 
             } else if (safe_str_eq(task, CRMD_ACTION_MIGRATE)) {
                 int stop_id = 0;
                 int migrate_id = 0;
                 const char *migrate_source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
                 const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
 
                 xmlNode *stop_op =
                     find_lrm_op(rsc->id, CRMD_ACTION_STOP, migrate_target, NULL, data_set);
                 xmlNode *migrate_op =
                     find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, migrate_target, migrate_source,
                                 data_set);
 
                 if (stop_op) {
                     crm_element_value_int(stop_op, XML_LRM_ATTR_CALLID, &stop_id);
                 }
                 if (migrate_op) {
                     crm_element_value_int(migrate_op, XML_LRM_ATTR_CALLID, &migrate_id);
                 }
 
                 /* Get our state right */
                 rsc->role = RSC_ROLE_STARTED;   /* can be master? */
 
                 if (stop_op == NULL || stop_id < migrate_id) {
                     node_t *target = pe_find_node(data_set->nodes, migrate_target);
 
                     crm_trace("Stop: %p %d, Migrated: %p %d", stop_op, stop_id, migrate_op,
                               migrate_id);
                     if (target && target->details->online) {
                         native_add_running(rsc, target, data_set);
                     }
 
                 } else if (migrate_op == NULL) {
                     /* Make sure it gets cleaned up, the stop may pre-date the migrate_from */
                     rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);
                 }
 
             } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
                 rsc->role = RSC_ROLE_MASTER;
 
             } else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
                 /*
                  * staying in role=master ends up putting the PE/TE into a loop
                  * setting role=slave is not dangerous because no master will be
                  * promoted until the failed resource has been fully stopped
                  */
                 crm_warn("Forcing %s to stop after a failed demote action", rsc->id);
                 rsc->next_role = RSC_ROLE_STOPPED;
                 rsc->role = RSC_ROLE_SLAVE;
 
             } else if (compare_version("2.0", op_version) > 0
                        && safe_str_eq(task, CRMD_ACTION_START)) {
                 crm_warn("Compatibility handling for failed op %s on %s", id, node->details->uname);
                 resource_location(rsc, node, -INFINITY, "__legacy_start__", data_set);
             }
 
             if (rsc->role < RSC_ROLE_STARTED) {
                 set_active(rsc);
             }
 
             crm_trace("Resource %s: role=%s, unclean=%s, on_fail=%s, fail_role=%s",
                       rsc->id, role2text(rsc->role),
                       node->details->unclean ? "true" : "false",
                       fail2text(action->on_fail), role2text(action->fail_role));
 
             if (action->fail_role != RSC_ROLE_STARTED && rsc->next_role < action->fail_role) {
                 rsc->next_role = action->fail_role;
             }
 
             if (action->fail_role == RSC_ROLE_STOPPED) {
                 int score = -INFINITY;
 
                 crm_err("Making sure %s doesn't come up again", rsc->id);
                 /* make sure it doesnt come up again */
                 g_hash_table_destroy(rsc->allowed_nodes);
                 rsc->allowed_nodes = node_hash_from_list(data_set->nodes);
                 g_hash_table_foreach(rsc->allowed_nodes, set_node_score, &score);
             }
 
             pe_free_action(action);
             action = NULL;
             break;
         case PCMK_LRM_OP_CANCELLED:
             /* do nothing?? */
             pe_err("Dont know what to do for cancelled ops yet");
             break;
     }
 
   done:
     crm_trace("Resource %s after %s: role=%s", rsc->id, task, role2text(rsc->role));
 
     pe_free_action(action);
 
     return TRUE;
 }
 
 gboolean
 add_node_attrs(xmlNode * xml_obj, node_t * node, gboolean overwrite, pe_working_set_t * data_set)
 {
     g_hash_table_insert(node->details->attrs,
                         strdup("#" XML_ATTR_UNAME), strdup(node->details->uname));
     g_hash_table_insert(node->details->attrs,
                         strdup("#" XML_ATTR_ID), strdup(node->details->id));
     if (safe_str_eq(node->details->id, data_set->dc_uuid)) {
         data_set->dc_node = node;
         node->details->is_dc = TRUE;
         g_hash_table_insert(node->details->attrs,
                             strdup("#" XML_ATTR_DC), strdup(XML_BOOLEAN_TRUE));
     } else {
         g_hash_table_insert(node->details->attrs,
                             strdup("#" XML_ATTR_DC), strdup(XML_BOOLEAN_FALSE));
     }
 
     unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_ATTR_SETS, NULL,
                                node->details->attrs, NULL, overwrite, data_set->now);
 
     return TRUE;
 }
 
 static GListPtr
 extract_operations(const char *node, const char *rsc, xmlNode * rsc_entry, gboolean active_filter)
 {
     int counter = -1;
     int stop_index = -1;
     int start_index = -1;
 
     xmlNode *rsc_op = NULL;
 
     GListPtr gIter = NULL;
     GListPtr op_list = NULL;
     GListPtr sorted_op_list = NULL;
 
     /* extract operations */
     op_list = NULL;
     sorted_op_list = NULL;
 
     for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next(rsc_op)) {
         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             crm_xml_add(rsc_op, "resource", rsc);
             crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
             op_list = g_list_prepend(op_list, rsc_op);
         }
     }
 
     if (op_list == NULL) {
         /* if there are no operations, there is nothing to do */
         return NULL;
     }
 
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 
     /* create active recurring operations as optional */
     if (active_filter == FALSE) {
         return sorted_op_list;
     }
 
     op_list = NULL;
 
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
 
     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *rsc_op = (xmlNode *) gIter->data;
 
         counter++;
 
         if (start_index < stop_index) {
             crm_trace("Skipping %s: not active", ID(rsc_entry));
             break;
 
         } else if (counter < start_index) {
             crm_trace("Skipping %s: old", ID(rsc_op));
             continue;
         }
         op_list = g_list_append(op_list, rsc_op);
     }
 
     g_list_free(sorted_op_list);
     return op_list;
 }
 
 GListPtr
 find_operations(const char *rsc, const char *node, gboolean active_filter,
                 pe_working_set_t * data_set)
 {
     GListPtr output = NULL;
     GListPtr intermediate = NULL;
 
     xmlNode *tmp = NULL;
     xmlNode *status = find_xml_node(data_set->input, XML_CIB_TAG_STATUS, TRUE);
 
     const char *uname = NULL;
     node_t *this_node = NULL;
 
     xmlNode *node_state = NULL;
 
     for (node_state = __xml_first_child(status); node_state != NULL;
          node_state = __xml_next(node_state)) {
         if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
             uname = crm_element_value(node_state, XML_ATTR_UNAME);
             if (node != NULL && safe_str_neq(uname, node)) {
                 continue;
             }
 
             this_node = pe_find_node(data_set->nodes, uname);
             CRM_CHECK(this_node != NULL, continue);
 
             determine_online_status(node_state, this_node, data_set);
 
             if (this_node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) {
                 /* offline nodes run no resources...
                  * unless stonith is enabled in which case we need to
                  *   make sure rsc start events happen after the stonith
                  */
                 xmlNode *lrm_rsc = NULL;
 
                 tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
                 tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);
 
                 for (lrm_rsc = __xml_first_child(tmp); lrm_rsc != NULL;
                      lrm_rsc = __xml_next(lrm_rsc)) {
                     if (crm_str_eq((const char *)lrm_rsc->name, XML_LRM_TAG_RESOURCE, TRUE)) {
 
                         const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID);
 
                         if (rsc != NULL && safe_str_neq(rsc_id, rsc)) {
                             continue;
                         }
 
                         intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
                         output = g_list_concat(output, intermediate);
                     }
                 }
             }
         }
     }
 
     return output;
 }
diff --git a/tools/crm_inject.c b/tools/crm_inject.c
index e83a199bbf..8562f64073 100644
--- a/tools/crm_inject.c
+++ b/tools/crm_inject.c
@@ -1,1489 +1,1487 @@
 /* 
  * Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <unistd.h>
 #include <stdlib.h>
 
 #include <sys/stat.h>
 #include <sys/param.h>
 #include <sys/types.h>
 #include <dirent.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/common/util.h>
 #include <crm/transition.h>
 #include <crm/common/iso8601.h>
 #include <crm/pengine/status.h>
 #include <allocate.h>
 
 cib_t *global_cib = NULL;
 GListPtr op_fail = NULL;
 gboolean quiet = FALSE;
 
 #define new_node_template "//"XML_CIB_TAG_NODE"[@uname='%s']"
 #define node_template "//"XML_CIB_TAG_STATE"[@uname='%s']"
 #define rsc_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']"
 #define op_template  "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s']"
 /* #define op_template  "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s' and @"XML_LRM_ATTR_CALLID"='%d']" */
 
 #define FAKE_TE_ID	"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
 
 #define quiet_log(fmt, args...) do {		\
 	if(quiet == FALSE) {			\
 	    printf(fmt , ##args);		\
 	}					\
     } while(0)
 
 extern void cleanup_alloc_calculations(pe_working_set_t * data_set);
 
 extern xmlNode *do_calculations(pe_working_set_t * data_set, xmlNode * xml_input, ha_time_t * now);
 
 char *use_date = NULL;
 static ha_time_t *
 get_date(void)
 {
     if (use_date) {
         char *date_m = use_date;
 
         return parse_date(&date_m);
     }
     return NULL;
 }
 
 static xmlNode *
 find_resource(xmlNode * cib_node, const char *resource)
 {
     char *xpath = NULL;
     xmlNode *match = NULL;
     const char *node = crm_element_value(cib_node, XML_ATTR_UNAME);
     int max = strlen(rsc_template) + strlen(resource) + strlen(node) + 1;
 
     xpath = calloc(1, max);
 
     snprintf(xpath, max, rsc_template, node, resource);
     match = get_xpath_object(xpath, cib_node, LOG_DEBUG_2);
 
     free(xpath);
     return match;
 }
 
 static void
 create_node_entry(cib_t * cib_conn, char *node)
 {
     int rc = pcmk_ok;
     int max = strlen(new_node_template) + strlen(node) + 1;
     char *xpath = NULL;
 
     xpath = calloc(1, max);
 
     snprintf(xpath, max, new_node_template, node);
     rc = cib_conn->cmds->query(cib_conn, xpath, NULL, cib_xpath | cib_sync_call | cib_scope_local);
 
     if (rc == -ENXIO) {
         xmlNode *cib_object = create_xml_node(NULL, XML_CIB_TAG_NODE);
 
         /* Using node uname as uuid ala corosync/openais */
         crm_xml_add(cib_object, XML_ATTR_ID, node);
         crm_xml_add(cib_object, XML_ATTR_UNAME, node);
         crm_xml_add(cib_object, XML_ATTR_TYPE, NORMALNODE);
         cib_conn->cmds->create(cib_conn, XML_CIB_TAG_NODES, cib_object,
                                cib_sync_call | cib_scope_local);
         /* Not bothering with subsequent query to see if it exists,
            we'll bomb out later in the call to determine_host... */
 
         free_xml(cib_object);
     }
 
     free(xpath);
 }
 
 static xmlNode *
 inject_node_state(cib_t * cib_conn, char *node)
 {
     int rc = pcmk_ok;
     int max = strlen(rsc_template) + strlen(node) + 1;
     char *xpath = NULL;
     xmlNode *cib_object = NULL;
 
     xpath = calloc(1, max);
 
     create_node_entry(cib_conn, node);
 
     snprintf(xpath, max, node_template, node);
     rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
                                cib_xpath | cib_sync_call | cib_scope_local);
 
     if(cib_object && ID(cib_object) == NULL) {
         crm_err("Detected multiple node_state entries for xpath=%s, bailing", xpath);
         crm_log_xml_warn(cib_object, "Duplicates");
         exit(1);
     }
 
     if (rc == -ENXIO) {
         char *uuid = NULL;
 
         cib_object = create_xml_node(NULL, XML_CIB_TAG_STATE);
         determine_host(cib_conn, &node, &uuid);
         crm_xml_add(cib_object, XML_ATTR_UUID, uuid);
         crm_xml_add(cib_object, XML_ATTR_UNAME, node);
         cib_conn->cmds->create(cib_conn, XML_CIB_TAG_STATUS, cib_object,
                                cib_sync_call | cib_scope_local);
         free_xml(cib_object);
         free(uuid);
 
         rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
                                    cib_xpath | cib_sync_call | cib_scope_local);
     }
 
     free(xpath);
     CRM_ASSERT(rc == pcmk_ok);
     return cib_object;
 }
 
 static xmlNode *
 modify_node(cib_t * cib_conn, char *node, gboolean up)
 {
     xmlNode *cib_node = inject_node_state(cib_conn, node);
 
     if (up) {
-        crm_xml_add(cib_node, XML_CIB_ATTR_HASTATE, ACTIVESTATUS);
         crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_YES);
         crm_xml_add(cib_node, XML_NODE_IS_PEER, ONLINESTATUS);
         crm_xml_add(cib_node, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_MEMBER);
         crm_xml_add(cib_node, XML_NODE_EXPECTED, CRMD_JOINSTATE_MEMBER);
 
     } else {
-        crm_xml_add(cib_node, XML_CIB_ATTR_HASTATE, DEADSTATUS);
         crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO);
         crm_xml_add(cib_node, XML_NODE_IS_PEER, OFFLINESTATUS);
         crm_xml_add(cib_node, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_DOWN);
         crm_xml_add(cib_node, XML_NODE_EXPECTED, CRMD_JOINSTATE_DOWN);
     }
 
     crm_xml_add(cib_node, XML_ATTR_ORIGIN, crm_system_name);
     return cib_node;
 }
 
 static void
 inject_transient_attr(xmlNode * cib_node, const char *name, const char *value)
 {
     xmlNode *attrs = NULL;
     xmlNode *container = NULL;
     xmlNode *nvp = NULL;
     const char *node_uuid = ID(cib_node);
     char *nvp_id = crm_concat(name, node_uuid, '-');
 
     crm_info("Injecting attribute %s=%s into %s '%s'", name, value, xmlGetNodePath(cib_node),
              ID(cib_node));
 
     attrs = first_named_child(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
     if (attrs == NULL) {
         attrs = create_xml_node(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
         crm_xml_add(attrs, XML_ATTR_ID, node_uuid);
     }
 
     container = first_named_child(attrs, XML_TAG_ATTR_SETS);
     if (container == NULL) {
         container = create_xml_node(attrs, XML_TAG_ATTR_SETS);
         crm_xml_add(container, XML_ATTR_ID, node_uuid);
     }
 
     nvp = create_xml_node(container, XML_CIB_TAG_NVPAIR);
     crm_xml_add(nvp, XML_ATTR_ID, nvp_id);
     crm_xml_add(nvp, XML_NVPAIR_ATTR_NAME, name);
     crm_xml_add(nvp, XML_NVPAIR_ATTR_VALUE, value);
 
     free(nvp_id);
 }
 
 static xmlNode *
 inject_resource(xmlNode * cib_node, const char *resource, const char *rclass, const char *rtype,
                 const char *rprovider)
 {
     xmlNode *lrm = NULL;
     xmlNode *container = NULL;
     xmlNode *cib_resource = NULL;
     char *xpath = NULL;
 
     cib_resource = find_resource(cib_node, resource);
     if (cib_resource != NULL) {
         return cib_resource;
     }
 
     /* One day, add query for class, provider, type */
 
     if (rclass == NULL || rtype == NULL) {
         fprintf(stderr, "Resource %s not found in the status section of %s."
                 "  Please supply the class and type to continue\n", resource, ID(cib_node));
         return NULL;
 
     } else if (safe_str_neq(rclass, "ocf")
                && safe_str_neq(rclass, "stonith")
                && safe_str_neq(rclass, "heartbeat")
                && safe_str_neq(rclass, "lsb")) {
         fprintf(stderr, "Invalid class for %s: %s\n", resource, rclass);
         return NULL;
 
     } else if (safe_str_eq(rclass, "ocf") && rprovider == NULL) {
         fprintf(stderr, "Please specify the provider for resource %s\n", resource);
         return NULL;
     }
 
     xpath = (char *)xmlGetNodePath(cib_node);
     crm_info("Injecting new resource %s into %s '%s'", resource, xpath, ID(cib_node));
     free(xpath);
 
     lrm = first_named_child(cib_node, XML_CIB_TAG_LRM);
     if (lrm == NULL) {
         const char *node_uuid = ID(cib_node);
 
         lrm = create_xml_node(cib_node, XML_CIB_TAG_LRM);
         crm_xml_add(lrm, XML_ATTR_ID, node_uuid);
     }
 
     container = first_named_child(lrm, XML_LRM_TAG_RESOURCES);
     if (container == NULL) {
         container = create_xml_node(lrm, XML_LRM_TAG_RESOURCES);
     }
 
     cib_resource = create_xml_node(container, XML_LRM_TAG_RESOURCE);
     crm_xml_add(cib_resource, XML_ATTR_ID, resource);
 
     crm_xml_add(cib_resource, XML_AGENT_ATTR_CLASS, rclass);
     crm_xml_add(cib_resource, XML_AGENT_ATTR_PROVIDER, rprovider);
     crm_xml_add(cib_resource, XML_ATTR_TYPE, rtype);
 
     return cib_resource;
 }
 
 static lrmd_event_data_t *
 create_op(xmlNode * cib_resource, const char *task, int interval, int outcome)
 {
     lrmd_event_data_t *op = NULL;
     xmlNode *xop = NULL;
 
     op = calloc(1, sizeof(lrmd_event_data_t));
 
     op->rsc_id = strdup(ID(cib_resource));
     op->interval = interval;
     op->op_type = strdup(task);
 
     op->rc = outcome;
     op->op_status = 0;
     op->params = NULL;          /* TODO: Fill me in */
 
     op->call_id = 0;
     for (xop = __xml_first_child(cib_resource); xop != NULL; xop = __xml_next(xop)) {
         int tmp = 0;
 
         crm_element_value_int(xop, XML_LRM_ATTR_CALLID, &tmp);
         if (tmp > op->call_id) {
             op->call_id = tmp;
         }
     }
     op->call_id++;
 
     return op;
 }
 
 static xmlNode *
 inject_op(xmlNode * cib_resource, lrmd_event_data_t * op, int target_rc)
 {
     return create_operation_update(cib_resource, op, CRM_FEATURE_SET, target_rc, crm_system_name,
                                    LOG_DEBUG_2);
 }
 
 static void
 update_failcounts(xmlNode * cib_node, const char *resource, int interval, int rc)
 {
     if (rc == 0) {
         return;
 
     } else if (rc == 7 && interval == 0) {
         return;
 
     } else {
         char *name = NULL;
         char *now = crm_itoa(time(NULL));
 
         name = crm_concat("fail-count", resource, '-');
         inject_transient_attr(cib_node, name, "value++");
 
         name = crm_concat("last-failure", resource, '-');
         inject_transient_attr(cib_node, name, now);
 
         free(name);
         free(now);
     }
 }
 
 static gboolean
 exec_pseudo_action(crm_graph_t * graph, crm_action_t * action)
 {
     const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
     action->confirmed = TRUE;
 
     quiet_log(" * Pseudo action:   %s%s%s\n", task, node?" on ":"", node?node:"");
     update_graph(graph, action);
     return TRUE;
 }
 
 static gboolean
 exec_rsc_action(crm_graph_t * graph, crm_action_t * action)
 {
     int rc = 0;
     GListPtr gIter = NULL;
     lrmd_event_data_t *op = NULL;
     int target_outcome = 0;
 
     const char *rtype = NULL;
     const char *rclass = NULL;
     const char *resource = NULL;
     const char *rprovider = NULL;
     const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);
 
     xmlNode *cib_node = NULL;
     xmlNode *cib_resource = NULL;
     xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
 
     char *node = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
 
     if (safe_str_eq(crm_element_value(action->xml, "operation"), "probe_complete")) {
         crm_info("Skipping %s op for %s\n", crm_element_value(action->xml, "operation"), node);
         goto done;
     }
 
     if (action_rsc == NULL) {
         crm_log_xml_err(action->xml, "Bad");
         free(node);
         return FALSE;
     }
 
     resource = ID(action_rsc);
     rclass = crm_element_value(action_rsc, XML_AGENT_ATTR_CLASS);
     rtype = crm_element_value(action_rsc, XML_ATTR_TYPE);
     rprovider = crm_element_value(action_rsc, XML_AGENT_ATTR_PROVIDER);
 
     if (target_rc_s != NULL) {
         target_outcome = crm_parse_int(target_rc_s, "0");
     }
 
     CRM_ASSERT(global_cib->cmds->query(global_cib, NULL, NULL, cib_sync_call | cib_scope_local) ==
                pcmk_ok);
 
     cib_node = inject_node_state(global_cib, node);
     CRM_ASSERT(cib_node != NULL);
 
     cib_resource = inject_resource(cib_node, resource, rclass, rtype, rprovider);
     CRM_ASSERT(cib_resource != NULL);
 
     op = convert_graph_action(cib_resource, action, 0, target_outcome);
     if(op->interval) {
         quiet_log(" * Resource action: %-15s %s=%d on %s\n", resource, op->op_type, op->interval, node);
     } else {
         quiet_log(" * Resource action: %-15s %s on %s\n", resource, op->op_type, node);
     }
 
     for (gIter = op_fail; gIter != NULL; gIter = gIter->next) {
         char *spec = (char *)gIter->data;
         char *key = NULL;
 
         key = calloc(1, 1 + strlen(spec));
         snprintf(key, strlen(spec), "%s_%s_%d@%s=", resource, op->op_type, op->interval, node);
 
         if (strncasecmp(key, spec, strlen(key)) == 0) {
             rc = sscanf(spec, "%*[^=]=%d", (int *) &op->rc);
 
             action->failed = TRUE;
             graph->abort_priority = INFINITY;
             printf("\tPretending action %d failed with rc=%d\n", action->id, op->rc);
             update_failcounts(cib_node, resource, op->interval, op->rc);
             free(key);
             break;
         }
         free(key);
     }
 
     inject_op(cib_resource, op, target_outcome);
     lrmd_free_event(op);
 
     rc = global_cib->cmds->modify(global_cib, XML_CIB_TAG_STATUS, cib_node,
                                   cib_sync_call | cib_scope_local);
     CRM_ASSERT(rc == pcmk_ok);
 
   done:
     free(node);
     free_xml(cib_node);
     action->confirmed = TRUE;
     update_graph(graph, action);
     return TRUE;
 }
 
 static gboolean
 exec_crmd_action(crm_graph_t * graph, crm_action_t * action)
 {
     const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     action->confirmed = TRUE;
 
     quiet_log(" * Cluster action:  %s on %s\n", task, node);
     update_graph(graph, action);
     return TRUE;
 }
 
 #define STATUS_PATH_MAX 512
 static gboolean
 exec_stonith_action(crm_graph_t * graph, crm_action_t * action)
 {
     int rc = 0;
     char xpath[STATUS_PATH_MAX];
     char *target = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
     xmlNode *cib_node = modify_node(global_cib, target, FALSE);
 
     crm_xml_add(cib_node, XML_ATTR_ORIGIN, __FUNCTION__);
     CRM_ASSERT(cib_node != NULL);
 
     quiet_log(" * Fencing %s\n", target);
     rc = global_cib->cmds->replace(global_cib, XML_CIB_TAG_STATUS, cib_node,
                                    cib_sync_call | cib_scope_local);
     CRM_ASSERT(rc == pcmk_ok);
 
     snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target, XML_CIB_TAG_LRM);
     rc = global_cib->cmds->delete(global_cib, xpath, NULL,
                                   cib_xpath | cib_sync_call | cib_scope_local);
 
     snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target,
              XML_TAG_TRANSIENT_NODEATTRS);
     rc = global_cib->cmds->delete(global_cib, xpath, NULL,
                                   cib_xpath | cib_sync_call | cib_scope_local);
 
     action->confirmed = TRUE;
     update_graph(graph, action);
     free_xml(cib_node);
     free(target);
     return TRUE;
 }
 
 static char *
 add_list_element(char *list, const char *value)
 {
     int len = 0;
     int last = 0;
 
     if (value == NULL) {
         return list;
     }
     if (list) {
         last = strlen(list);
     }
     len = last + 2;             /* +1 space, +1 EOS */
     len += strlen(value);
     list = realloc(list, len);
     sprintf(list + last, " %s", value);
     return list;
 }
 
 static void
 print_cluster_status(pe_working_set_t * data_set)
 {
     char *online_nodes = NULL;
     char *offline_nodes = NULL;
 
     GListPtr gIter = NULL;
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
         const char *node_mode = NULL;
 
         if (node->details->unclean) {
             if (node->details->online && node->details->unclean) {
                 node_mode = "UNCLEAN (online)";
 
             } else if (node->details->pending) {
                 node_mode = "UNCLEAN (pending)";
 
             } else {
                 node_mode = "UNCLEAN (offline)";
             }
 
         } else if (node->details->pending) {
             node_mode = "pending";
 
         } else if (node->details->standby_onfail && node->details->online) {
             node_mode = "standby (on-fail)";
 
         } else if (node->details->standby) {
             if (node->details->online) {
                 node_mode = "standby";
             } else {
                 node_mode = "OFFLINE (standby)";
             }
 
         } else if (node->details->online) {
             node_mode = "online";
             online_nodes = add_list_element(online_nodes, node->details->uname);
             continue;
 
         } else {
             node_mode = "OFFLINE";
             offline_nodes = add_list_element(offline_nodes, node->details->uname);
             continue;
         }
 
         if (safe_str_eq(node->details->uname, node->details->id)) {
             printf("Node %s: %s\n", node->details->uname, node_mode);
         } else {
             printf("Node %s (%s): %s\n", node->details->uname, node->details->id, node_mode);
         }
     }
 
     if (online_nodes) {
         printf("Online: [%s ]\n", online_nodes);
         free(online_nodes);
     }
     if (offline_nodes) {
         printf("OFFLINE: [%s ]\n", offline_nodes);
         free(offline_nodes);
     }
 
     fprintf(stdout, "\n");
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
 
         if (is_set(rsc->flags, pe_rsc_orphan)
             && rsc->role == RSC_ROLE_STOPPED) {
             continue;
         }
         rsc->fns->print(rsc, NULL, pe_print_printf, stdout);
     }
     fprintf(stdout, "\n");
 }
 
 static int
 run_simulation(pe_working_set_t * data_set)
 {
     crm_graph_t *transition = NULL;
     enum transition_status graph_rc = -1;
 
     crm_graph_functions_t exec_fns = {
         exec_pseudo_action,
         exec_rsc_action,
         exec_crmd_action,
         exec_stonith_action,
     };
 
     set_graph_functions(&exec_fns);
 
     quiet_log("\nExecuting cluster transition:\n");
     transition = unpack_graph(data_set->graph, crm_system_name);
     print_graph(LOG_DEBUG, transition);
 
     do {
         graph_rc = run_graph(transition);
 
     } while (graph_rc == transition_active);
 
     if (graph_rc != transition_complete) {
         fprintf(stdout, "Transition failed: %s\n", transition_status(graph_rc));
         print_graph(LOG_ERR, transition);
     }
     destroy_graph(transition);
     if (graph_rc != transition_complete) {
         fprintf(stdout, "An invalid transition was produced\n");
     }
 
     if (quiet == FALSE) {
         xmlNode *cib_object = NULL;
         int rc =
             global_cib->cmds->query(global_cib, NULL, &cib_object, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
         quiet_log("\nRevised cluster status:\n");
         cleanup_alloc_calculations(data_set);
         data_set->input = cib_object;
         data_set->now = get_date();
 
         cluster_status(data_set);
         print_cluster_status(data_set);
     }
 
     if (graph_rc != transition_complete) {
         return graph_rc;
     }
     return 0;
 }
 
 static char *
 create_action_name(action_t * action)
 {
     char *action_name = NULL;
     const char *action_host = NULL;
 
     if (action->node) {
         action_host = action->node->details->uname;
         action_name = crm_concat(action->uuid, action_host, ' ');
 
     } else if (is_set(action->flags, pe_action_pseudo)) {
         action_name = strdup(action->uuid);
 
     } else {
         action_host = "<none>";
         action_name = crm_concat(action->uuid, action_host, ' ');
     }
     if (safe_str_eq(action->task, RSC_CANCEL)) {
         char *tmp_action_name = action_name;
 
         action_name = crm_concat("Cancel", tmp_action_name, ' ');
         free(tmp_action_name);
     }
 
     return action_name;
 }
 
 static void
 create_dotfile(pe_working_set_t * data_set, const char *dot_file, gboolean all_actions)
 {
     GListPtr gIter = NULL;
     FILE *dot_strm = fopen(dot_file, "w");
 
     if (dot_strm == NULL) {
         crm_perror(LOG_ERR, "Could not open %s for writing", dot_file);
         return;
     }
 
     fprintf(dot_strm, " digraph \"g\" {\n");
     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         action_t *action = (action_t *) gIter->data;
         const char *style = "dashed";
         const char *font = "black";
         const char *color = "black";
         char *action_name = create_action_name(action);
 
         crm_trace("Action %d: %p", action->id, action);
 
         if (is_set(action->flags, pe_action_pseudo)) {
             font = "orange";
         }
 
         if (is_set(action->flags, pe_action_dumped)) {
             style = "bold";
             color = "green";
 
         } else if (action->rsc != NULL && is_not_set(action->rsc->flags, pe_rsc_managed)) {
             color = "red";
             font = "purple";
             if (all_actions == FALSE) {
                 goto dont_write;
             }
 
         } else if (is_set(action->flags, pe_action_optional)) {
             color = "blue";
             if (all_actions == FALSE) {
                 goto dont_write;
             }
 
         } else {
             color = "red";
             CRM_CHECK(is_set(action->flags, pe_action_runnable) == FALSE,;
                 );
         }
 
         set_bit(action->flags, pe_action_dumped);
         fprintf(dot_strm, "\"%s\" [ style=%s color=\"%s\" fontcolor=\"%s\"]\n",
                 action_name, style, color, font);
   dont_write:
         free(action_name);
     }
 
     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         action_t *action = (action_t *) gIter->data;
 
         GListPtr gIter2 = NULL;
 
         for (gIter2 = action->actions_before; gIter2 != NULL; gIter2 = gIter2->next) {
             action_wrapper_t *before = (action_wrapper_t *) gIter2->data;
 
             char *before_name = NULL;
             char *after_name = NULL;
             const char *style = "dashed";
             gboolean optional = TRUE;
 
             if (before->state == pe_link_dumped) {
                 optional = FALSE;
                 style = "bold";
             } else if (is_set(action->flags, pe_action_pseudo)
                        && (before->type & pe_order_stonith_stop)) {
                 continue;
             } else if (before->state == pe_link_dup) {
                 continue;
             } else if (before->type == pe_order_none) {
                 continue;
             } else if (is_set(before->action->flags, pe_action_dumped)
                        && is_set(action->flags, pe_action_dumped)
                        && before->type != pe_order_load) {
                 optional = FALSE;
             }
 
             if (all_actions || optional == FALSE) {
                 before_name = create_action_name(before->action);
                 after_name = create_action_name(action);
                 fprintf(dot_strm, "\"%s\" -> \"%s\" [ style = %s]\n",
                         before_name, after_name, style);
                 free(before_name);
                 free(after_name);
             }
         }
     }
 
     fprintf(dot_strm, "}\n");
     if (dot_strm != NULL) {
         fflush(dot_strm);
         fclose(dot_strm);
     }
 }
 
 static int
 find_ticket_state(cib_t * the_cib, const char * ticket_id, xmlNode ** ticket_state_xml)
 {
     int offset = 0;
     static int xpath_max = 1024;
     int rc = pcmk_ok;
     xmlNode *xml_search = NULL;
 
     char *xpath_string = NULL;
 
     CRM_ASSERT(ticket_state_xml != NULL);
     *ticket_state_xml = NULL;
 
     xpath_string = calloc(1, xpath_max);
     offset +=
         snprintf(xpath_string + offset, xpath_max - offset, "%s", "/cib/status/tickets");
 
     if (ticket_id) {
         offset += snprintf(xpath_string + offset, xpath_max - offset, "/%s[@id=\"%s\"]",
                        XML_CIB_TAG_TICKET_STATE, ticket_id);
     }
 
     rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
                               cib_sync_call | cib_scope_local | cib_xpath);
 
     if (rc != pcmk_ok) {
         goto bail;
     }
 
     crm_log_xml_debug(xml_search, "Match");
     if (xml_has_children(xml_search)) {
         if (ticket_id) {
             fprintf(stdout, "Multiple ticket_states match ticket_id=%s\n", ticket_id);
         }
         *ticket_state_xml = xml_search;
     } else {
         *ticket_state_xml = xml_search;
     }
 
   bail:
     free(xpath_string);
     return rc;
 }
 
 static int
 set_ticket_state_attr(const char *ticket_id, const char *attr_name,
                       const char *attr_value, cib_t * cib, int cib_options)
 {
     int rc = pcmk_ok;
     xmlNode *xml_top = NULL;
     xmlNode *ticket_state_xml = NULL;
 
     rc = find_ticket_state(cib, ticket_id, &ticket_state_xml);
     if (rc == pcmk_ok) {
         crm_debug("Found a match state for ticket: id=%s", ticket_id);
         xml_top = ticket_state_xml;
 
     } else if (rc != -ENXIO) {
         return rc;
 
     } else {
         xmlNode *xml_obj = NULL;
 
         xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
         xml_obj = create_xml_node(xml_top, XML_CIB_TAG_TICKETS);
         ticket_state_xml = create_xml_node(xml_obj, XML_CIB_TAG_TICKET_STATE);
         crm_xml_add(ticket_state_xml, XML_ATTR_ID, ticket_id);
     }
 
     crm_xml_add(ticket_state_xml, attr_name, attr_value);
 
     crm_log_xml_debug(xml_top, "Update");
 
     rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, xml_top, cib_options);
 
     free_xml(xml_top);
 
     return rc;
 }
 
 static void
 modify_configuration(pe_working_set_t * data_set,
                      const char *quorum, GListPtr node_up, GListPtr node_down, GListPtr node_fail,
                      GListPtr op_inject, GListPtr ticket_grant, GListPtr ticket_revoke,
                      GListPtr ticket_standby, GListPtr ticket_activate)
 {
     int rc = pcmk_ok;
     GListPtr gIter = NULL;
 
     xmlNode *cib_op = NULL;
     xmlNode *cib_node = NULL;
     xmlNode *cib_resource = NULL;
 
     lrmd_event_data_t *op = NULL;
 
     if (quorum) {
         xmlNode *top = create_xml_node(NULL, XML_TAG_CIB);
 
         quiet_log(" + Setting quorum: %s\n", quorum);
         /* crm_xml_add(top, XML_ATTR_DC_UUID, dc_uuid);      */
         crm_xml_add(top, XML_ATTR_HAVE_QUORUM, quorum);
 
         rc = global_cib->cmds->modify(global_cib, NULL, top, cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = node_up; gIter != NULL; gIter = gIter->next) {
         char *node = (char *)gIter->data;
 
         quiet_log(" + Bringing node %s online\n", node);
         cib_node = modify_node(global_cib, node, TRUE);
         CRM_ASSERT(cib_node != NULL);
 
         rc = global_cib->cmds->modify(global_cib, XML_CIB_TAG_STATUS, cib_node,
                                       cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = node_down; gIter != NULL; gIter = gIter->next) {
         char *node = (char *)gIter->data;
 
         quiet_log(" + Taking node %s offline\n", node);
         cib_node = modify_node(global_cib, node, FALSE);
         CRM_ASSERT(cib_node != NULL);
 
         rc = global_cib->cmds->modify(global_cib, XML_CIB_TAG_STATUS, cib_node,
                                       cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = node_fail; gIter != NULL; gIter = gIter->next) {
         char *node = (char *)gIter->data;
 
         quiet_log(" + Failing node %s\n", node);
         cib_node = modify_node(global_cib, node, TRUE);
         crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO);
         CRM_ASSERT(cib_node != NULL);
 
         rc = global_cib->cmds->modify(global_cib, XML_CIB_TAG_STATUS, cib_node,
                                       cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = ticket_grant; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         quiet_log(" + Granting ticket %s\n", ticket_id);
         rc = set_ticket_state_attr(ticket_id, "granted", "true",
                                   global_cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = ticket_revoke; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         quiet_log(" + Revoking ticket %s\n", ticket_id);
         rc = set_ticket_state_attr(ticket_id, "granted", "false",
                                   global_cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = ticket_standby; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         quiet_log(" + Making ticket %s standby\n", ticket_id);
         rc = set_ticket_state_attr(ticket_id, "standby", "true",
                                   global_cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = ticket_activate; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         quiet_log(" + Activating ticket %s\n", ticket_id);
         rc = set_ticket_state_attr(ticket_id, "standby", "false",
                                   global_cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = op_inject; gIter != NULL; gIter = gIter->next) {
         char *spec = (char *)gIter->data;
 
         int rc = 0;
         int outcome = 0;
         int interval = 0;
 
         char *key = NULL;
         char *node = NULL;
         char *task = NULL;
         char *resource = NULL;
 
         const char *rtype = NULL;
         const char *rclass = NULL;
         const char *rprovider = NULL;
 
         resource_t *rsc = NULL;
 
         quiet_log(" + Injecting %s into the configuration\n", spec);
 
         key = calloc(1, strlen(spec) + 1);
         node = calloc(1, strlen(spec) + 1);
         rc = sscanf(spec, "%[^@]@%[^=]=%d", key, node, &outcome);
         CRM_CHECK(rc == 3,
                   fprintf(stderr, "Invalid operation spec: %s.  Only found %d fields\n", spec, rc);
                   continue);
 
         parse_op_key(key, &resource, &task, &interval);
 
         rsc = pe_find_resource(data_set->resources, resource);
         if (rsc == NULL) {
             fprintf(stderr, " - Invalid resource name: %s\n", resource);
         } else {
             rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
             rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
             rprovider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
 
             cib_node = inject_node_state(global_cib, node);
             CRM_ASSERT(cib_node != NULL);
 
             update_failcounts(cib_node, resource, interval, outcome);
 
             cib_resource = inject_resource(cib_node, resource, rclass, rtype, rprovider);
             CRM_ASSERT(cib_resource != NULL);
 
             op = create_op(cib_resource, task, interval, outcome);
             CRM_ASSERT(op != NULL);
 
             cib_op = inject_op(cib_resource, op, 0);
             CRM_ASSERT(cib_op != NULL);
             lrmd_free_event(op);
 
             rc = global_cib->cmds->modify(global_cib, XML_CIB_TAG_STATUS, cib_node,
                                           cib_sync_call | cib_scope_local);
             CRM_ASSERT(rc == pcmk_ok);
         }
         free(task);
         free(node);
         free(key);
     }
 }
 
 static void
 setup_input(const char *input, const char *output)
 {
     int rc = pcmk_ok;
     cib_t *cib_conn = NULL;
     xmlNode *cib_object = NULL;
     char *local_output = NULL;
 
     if (input == NULL) {
         /* Use live CIB */
         cib_conn = cib_new();
         rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
 
         if (rc == pcmk_ok) {
             cib_object = get_cib_copy(cib_conn);
         }
 
         cib_conn->cmds->signoff(cib_conn);
         cib_delete(cib_conn);
         cib_conn = NULL;
 
         if (cib_object == NULL) {
             fprintf(stderr, "Live CIB query failed: empty result\n");
             exit(3);
         }
 
     } else if (safe_str_eq(input, "-")) {
         cib_object = filename2xml(NULL);
 
     } else {
         cib_object = filename2xml(input);
     }
 
     if (get_object_root(XML_CIB_TAG_STATUS, cib_object) == NULL) {
         create_xml_node(cib_object, XML_CIB_TAG_STATUS);
     }
 
     if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
         free_xml(cib_object);
         exit(-ENOKEY);
     }
 
     if (validate_xml(cib_object, NULL, FALSE) != TRUE) {
         free_xml(cib_object);
         exit(-pcmk_err_dtd_validation);
     }
 
     if (output == NULL) {
         char *pid = crm_itoa(getpid());
 
         local_output = get_shadow_file(pid);
         output = local_output;
         free(pid);
     }
 
     rc = write_xml_file(cib_object, output, FALSE);
     free_xml(cib_object);
     cib_object = NULL;
 
     if (rc < 0) {
         fprintf(stderr, "Could not create '%s': %s\n", output, strerror(errno));
         exit(rc);
     }
     setenv("CIB_file", output, 1);
     free(local_output);
 }
 
 
 /* *INDENT-OFF* */
 static struct crm_option long_options[] = {
     /* Top-level Options */
     {"help",    0, 0, '?', "\tThis text"},
     {"version", 0, 0, '$', "\tVersion information"  },
     {"quiet",   0, 0, 'Q', "\tDisplay only essentialoutput"},
     {"verbose", 0, 0, 'V', "\tIncrease debug output"},
 
     {"-spacer-",      0, 0, '-', "\nOperations:"},
     {"run",           0, 0, 'R', "\tDetermine the cluster's response to the given configuration and status"},
     {"simulate",      0, 0, 'S', "Simulate the transition's execution and display the resulting cluster status"},
     {"in-place",      0, 0, 'X', "Simulate the transition's execution and store the result back to the input file"},
     {"show-scores",   0, 0, 's', "Show allocation scores"},
     {"show-utilization",   0, 0, 'U', "Show utilization information"},
     {"profile",       1, 0, 'P', "Run all tests in the named directory to create profiling data"},
 
     {"-spacer-",     0, 0, '-', "\nSynthetic Cluster Events:"},
     {"node-up",      1, 0, 'u', "\tBring a node online"},
     {"node-down",    1, 0, 'd', "\tTake a node offline"},
     {"node-fail",    1, 0, 'f', "\tMark a node as failed"},
     {"op-inject",    1, 0, 'i', "\t$rsc_$task_$interval@$node=$rc - Inject the specified task before running the simulation"},
     {"op-fail",      1, 0, 'F', "\t$rsc_$task_$interval@$node=$rc - Fail the specified task while running the simulation"},
     {"set-datetime", 1, 0, 't', "Set date/time"},
     {"quorum",       1, 0, 'q', "\tSpecify a value for quorum"},
     {"ticket-grant",     1, 0, 'g', "Grant a ticket"},
     {"ticket-revoke",    1, 0, 'r', "Revoke a ticket"},
     {"ticket-standby",   1, 0, 'b', "Make a ticket standby"},
     {"ticket-activate",  1, 0, 'e', "Activate a ticket"},
 
     {"-spacer-",     0, 0, '-', "\nOutput Options:"},
     
     {"save-input",   1, 0, 'I', "\tSave the input configuration to the named file"},
     {"save-output",  1, 0, 'O', "Save the output configuration to the named file"},
     {"save-graph",   1, 0, 'G', "\tSave the transition graph (XML format) to the named file"},
     {"save-dotfile", 1, 0, 'D', "Save the transition graph (DOT format) to the named file"},
     {"all-actions",  0, 0, 'a', "\tDisplay all possible actions in the DOT graph - even ones not part of the transition"},
     
     {"-spacer-",    0, 0, '-', "\nData Source:"},
     {"live-check",  0, 0, 'L', "\tConnect to the CIB and use the current contents as input"},
     {"xml-file",    1, 0, 'x', "\tRetrieve XML from the named file"},
     {"xml-pipe",    0, 0, 'p', "\tRetrieve XML from stdin"},
     
     {0, 0, 0, 0}
 };
 /* *INDENT-ON* */
 
 static void
 profile_one(const char *xml_file)
 {
     xmlNode *cib_object = NULL;
     pe_working_set_t data_set;
 
     printf("* Testing %s\n", xml_file);
     cib_object = filename2xml(xml_file);
     if (get_object_root(XML_CIB_TAG_STATUS, cib_object) == NULL) {
         create_xml_node(cib_object, XML_CIB_TAG_STATUS);
     }
 
     if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
         free_xml(cib_object);
         return;
     }
 
     if (validate_xml(cib_object, NULL, FALSE) != TRUE) {
         free_xml(cib_object);
         return;
     }
 
     set_working_set_defaults(&data_set);
 
     data_set.input = cib_object;
     data_set.now = get_date();
     do_calculations(&data_set, cib_object, NULL);
 
     cleanup_alloc_calculations(&data_set);
 }
 
 #ifndef FILENAME_MAX
 #  define FILENAME_MAX 512
 #endif
 
 static int
 profile_all(const char *dir)
 {
     struct dirent **namelist;
 
     int lpc = 0;
     int file_num = scandir(dir, &namelist, 0, alphasort);
 
     if (file_num > 0) {
 	struct stat prop;
 	char buffer[FILENAME_MAX + 1];
 
 	while (file_num--) {
 	    if ('.' == namelist[file_num]->d_name[0]) {
 		free(namelist[file_num]);
 		continue;
 
 	    } else if (strstr(namelist[file_num]->d_name, ".xml") == NULL) {
 		free(namelist[file_num]);
 		continue;
 	    }
 
 	    lpc++;
 	    snprintf(buffer, FILENAME_MAX, "%s/%s", dir, namelist[file_num]->d_name);
 	    if (stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) {
 		profile_one(buffer);
 	    }
 	    free(namelist[file_num]);
 	}
 	free(namelist);
     }
 
     return lpc;
 }
 
 int
 main(int argc, char **argv)
 {
     int rc = 0;
     guint modified = 0;
 
     gboolean store = FALSE;
     gboolean process = FALSE;
     gboolean verbose = FALSE;
     gboolean simulate = FALSE;
     gboolean all_actions = FALSE;
     gboolean have_stdout = FALSE;
 
     pe_working_set_t data_set;
 
     const char *xml_file = "-";
     const char *quorum = NULL;
     const char *test_dir = NULL;
     const char *dot_file = NULL;
     const char *graph_file = NULL;
     const char *input_file = NULL;
     const char *output_file = NULL;
 
     int flag = 0;
     int index = 0;
     int argerr = 0;
 
     GListPtr node_up = NULL;
     GListPtr node_down = NULL;
     GListPtr node_fail = NULL;
     GListPtr op_inject = NULL;
     GListPtr ticket_grant = NULL;
     GListPtr ticket_revoke = NULL;
     GListPtr ticket_standby = NULL;
     GListPtr ticket_activate = NULL;
 
     xmlNode *input = NULL;
 
     crm_log_cli_init("crm_simulate");
     crm_set_options(NULL, "datasource operation [additional options]",
                     long_options, "Tool for simulating the cluster's response to events");
 
     if (argc < 2) {
         crm_help('?', EX_USAGE);
     }
 
     while (1) {
         flag = crm_get_option(argc, argv, &index);
         if (flag == -1)
             break;
 
         switch (flag) {
             case 'V':
                 verbose = TRUE;
 
                 if(have_stdout == FALSE) {
                     /* Redirect stderr to stdout so we can grep the output */
                     have_stdout = TRUE;
                     close(STDERR_FILENO);
                     dup2(STDOUT_FILENO, STDERR_FILENO);
                 }
                 
                 crm_bump_log_level();
                 break;
             case '?':
             case '$':
                 crm_help(flag, EX_OK);
                 break;
             case 'p':
                 xml_file = "-";
                 break;
             case 'Q':
                 quiet = TRUE;
                 break;
             case 'L':
                 xml_file = NULL;
                 break;
             case 'x':
                 xml_file = optarg;
                 break;
             case 'u':
                 modified++;
                 node_up = g_list_append(node_up, optarg);
                 break;
             case 'd':
                 modified++;
                 node_down = g_list_append(node_down, optarg);
                 break;
             case 'f':
                 modified++;
                 node_fail = g_list_append(node_fail, optarg);
                 break;
             case 't':
                 use_date = strdup(optarg);
                 break;
             case 'i':
                 modified++;
                 op_inject = g_list_append(op_inject, optarg);
                 break;
             case 'F':
                 process = TRUE;
                 simulate = TRUE;
                 op_fail = g_list_append(op_fail, optarg);
                 break;
             case 'q':
                 modified++;
                 quorum = optarg;
                 break;
             case 'g':
                 modified++;
                 ticket_grant = g_list_append(ticket_grant, optarg);
                 break;
             case 'r':
                 modified++;
                 ticket_revoke = g_list_append(ticket_revoke, optarg);
                 break;
             case 'b':
                 modified++;
                 ticket_standby = g_list_append(ticket_standby, optarg);
                 break;
             case 'e':
                 modified++;
                 ticket_activate = g_list_append(ticket_activate, optarg);
                 break;
             case 'a':
                 all_actions = TRUE;
                 break;
             case 's':
                 process = TRUE;
                 show_scores = TRUE;
                 break;
             case 'U':
                 process = TRUE;
                 show_utilization = TRUE;
                 break;
             case 'S':
                 process = TRUE;
                 simulate = TRUE;
                 break;
             case 'X':
                 store = TRUE;
                 process = TRUE;
                 simulate = TRUE;
                 break;
             case 'R':
                 process = TRUE;
                 break;
             case 'D':
                 process = TRUE;
                 dot_file = optarg;
                 break;
             case 'G':
                 process = TRUE;
                 graph_file = optarg;
                 break;
             case 'I':
                 input_file = optarg;
                 break;
             case 'O':
                 output_file = optarg;
                 break;
             case 'P':
                 test_dir = optarg;
                 break;
             default:
                 ++argerr;
                 break;
         }
     }
 
     if (optind > argc) {
         ++argerr;
     }
 
     if (argerr) {
         crm_help('?', EX_USAGE);
     }
 
     if(test_dir != NULL) {
 	return profile_all(test_dir);
     }
 
     setup_input(xml_file, store ? xml_file : output_file);
 
     global_cib = cib_new();
     global_cib->cmds->signon(global_cib, crm_system_name, cib_command);
 
     set_working_set_defaults(&data_set);
 
     if (data_set.now != NULL) {
         quiet_log(" + Setting effective cluster time: %s", use_date);
         log_date(LOG_WARNING, "Set fake 'now' to", data_set.now, ha_log_date | ha_log_time);
     }
 
     rc = global_cib->cmds->query(global_cib, NULL, &input, cib_sync_call | cib_scope_local);
     CRM_ASSERT(rc == pcmk_ok);
 
     data_set.input = input;
     data_set.now = get_date();
     cluster_status(&data_set);
 
     if (quiet == FALSE) {
         quiet_log("\nCurrent cluster status:\n");
         print_cluster_status(&data_set);
     }
 
     if (modified) {
         quiet_log("Performing requested modifications\n");
         modify_configuration(&data_set, quorum, node_up, node_down, node_fail, op_inject,
                              ticket_grant, ticket_revoke, ticket_standby, ticket_activate);
 
         rc = global_cib->cmds->query(global_cib, NULL, &input, cib_sync_call);
         if (rc != pcmk_ok) {
             fprintf(stderr, "Could not connect to the CIB for input: %s\n", pcmk_strerror(rc));
             goto done;
         }
 
         cleanup_alloc_calculations(&data_set);
         data_set.now = get_date();
         data_set.input = input;
     }
 
     if (input_file != NULL) {
         rc = write_xml_file(input, input_file, FALSE);
         if (rc < 0) {
             fprintf(stderr, "Could not create '%s': %s\n", input_file, strerror(errno));
             goto done;
         }
     }
 
     rc = 0;
     if (process || simulate) {
         ha_time_t *local_date = NULL;
 
         if (show_scores && show_utilization) {
             printf("Allocation scores and utilization information:\n");
         } else if (show_scores) {
             fprintf(stdout, "Allocation scores:\n");
         } else if (show_utilization) {
             printf("Utilization information:\n");
         }
 
         do_calculations(&data_set, input, local_date);
         input = NULL;           /* Don't try and free it twice */
 
         if (graph_file != NULL) {
             char *msg_buffer = dump_xml_formatted(data_set.graph);
             FILE *graph_strm = fopen(graph_file, "w");
 
             if (graph_strm == NULL) {
                 crm_perror(LOG_ERR, "Could not open %s for writing", graph_file);
 
             } else {
                 if (fprintf(graph_strm, "%s\n\n", msg_buffer) < 0) {
                     crm_perror(LOG_ERR, "Write to %s failed", graph_file);
                 }
                 fflush(graph_strm);
                 fclose(graph_strm);
             }
             free(msg_buffer);
         }
 
         if (dot_file != NULL) {
             create_dotfile(&data_set, dot_file, all_actions);
         }
 
         if (quiet == FALSE) {
             GListPtr gIter = NULL;
 
             quiet_log("%sTransition Summary:\n", show_scores || show_utilization
                       || modified ? "\n" : "");
             fflush(stdout);
 
             for (gIter = data_set.resources; gIter != NULL; gIter = gIter->next) {
                 resource_t *rsc = (resource_t *) gIter->data;
 
                 LogActions(rsc, &data_set, TRUE);
             }
         }
     }
 
     if (simulate) {
         rc = run_simulation(&data_set);
     }
 
   done:
     cleanup_alloc_calculations(&data_set);
 
     global_cib->cmds->signoff(global_cib);
     cib_delete(global_cib);
     free(use_date);
     crm_xml_cleanup();
     fflush(stderr);
     qb_log_fini();
     return rc;
 }
diff --git a/tools/crmadmin.c b/tools/crmadmin.c
index f9253f665a..4d8334d8ee 100644
--- a/tools/crmadmin.c
+++ b/tools/crmadmin.c
@@ -1,616 +1,614 @@
 
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <libgen.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crm/common/mainloop.h>
 
 #include <crm/cib.h>
 
 int message_timer_id = -1;
 int message_timeout_ms = 30 * 1000;
 
 GMainLoop *mainloop = NULL;
 crm_ipc_t *crmd_channel = NULL;
 char *admin_uuid = NULL;
 
 void usage(const char *cmd, int exit_status);
 gboolean do_init(void);
 int do_work(void);
 void crmadmin_ipc_connection_destroy(gpointer user_data);
 
 int admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata);
 char *pluralSection(const char *a_section);
 xmlNode *handleCibMod(void);
 int do_find_node_list(xmlNode * xml_node);
 gboolean admin_message_timeout(gpointer data);
 gboolean is_node_online(xmlNode * node_state);
 
 enum debug {
     debug_none,
     debug_dec,
     debug_inc
 };
 
 gboolean BE_VERBOSE = FALSE;
 int expected_responses = 1;
 
 gboolean BASH_EXPORT = FALSE;
 gboolean DO_HEALTH = FALSE;
 gboolean DO_RESET = FALSE;
 gboolean DO_RESOURCE = FALSE;
 gboolean DO_ELECT_DC = FALSE;
 gboolean DO_WHOIS_DC = FALSE;
 gboolean DO_NODE_LIST = FALSE;
 gboolean BE_SILENT = FALSE;
 gboolean DO_RESOURCE_LIST = FALSE;
 enum debug DO_DEBUG = debug_none;
 const char *crmd_operation = NULL;
 
 xmlNode *msg_options = NULL;
 
 const char *standby_on_off = "on";
 const char *admin_verbose = XML_BOOLEAN_FALSE;
 char *id = NULL;
 char *disconnect = NULL;
 char *dest_node = NULL;
 char *rsc_name = NULL;
 char *crm_option = NULL;
 
 int operation_status = 0;
 const char *sys_to = NULL;
 
 /* *INDENT-OFF* */
 static struct crm_option long_options[] = {
     /* Top-level Options */
     {"help",    0, 0, '?', "\tThis text"},
     {"version", 0, 0, '$', "\tVersion information"  },
     {"quiet",   0, 0, 'q', "\tDisplay only the essential query information"},
     {"verbose", 0, 0, 'V', "\tIncrease debug output"},
     
     {"-spacer-",	1, 0, '-', "\nCommands:"},
     /* daemon options */
     {"debug_inc", 1, 0, 'i', "Increase the crmd's debug level on the specified host"},
     {"debug_dec", 1, 0, 'd', "Decrease the crmd's debug level on the specified host"},
     {"status",    1, 0, 'S', "Display the status of the specified node." },
     {"-spacer-",  1, 0, '-', "\n\tResult is the node's internal FSM state which can be useful for debugging\n"},
     {"dc_lookup", 0, 0, 'D', "Display the uname of the node co-ordinating the cluster."},
     {"-spacer-",  1, 0, '-', "\n\tThis is an internal detail and is rarely useful to administrators except when deciding on which node to examine the logs.\n"},
     {"nodes",     0, 0, 'N', "\tDisplay the uname of all member nodes"},
     {"election",  0, 0, 'E', "(Advanced) Start an election for the cluster co-ordinator"},
     {"kill",      1, 0, 'K', "(Advanced) Shut down the crmd (not the rest of the clusterstack ) on the specified node"},
     {"health",    0, 0, 'H', NULL, 1},
     
     {"-spacer-",	1, 0, '-', "\nAdditional Options:"},
     {XML_ATTR_TIMEOUT, 1, 0, 't', "Time (in milliseconds) to wait before declaring the operation failed"},
     {"bash-export", 0, 0, 'B', "Create Bash export entries of the form 'export uname=uuid'\n"},
 
     {"-spacer-",  1, 0, '-', "Notes:"},
     {"-spacer-",  1, 0, '-', " The -i,-d,-K and -E commands are rarely used and may be removed in future versions."},
 
     {0, 0, 0, 0}
 };
 /* *INDENT-ON* */
 
 int
 main(int argc, char **argv)
 {
     int option_index = 0;
     int argerr = 0;
     int flag;
 
     crm_log_cli_init("crmadmin");
     crm_set_options(NULL, "command [options]", long_options,
                     "Development tool for performing some crmd-specific commands."
                     "\n  Likely to be replaced by crm_node in the future");
     if (argc < 2) {
         crm_help('?', EX_USAGE);
     }
 
     while (1) {
         flag = crm_get_option(argc, argv, &option_index);
         if (flag == -1)
             break;
 
         switch (flag) {
             case 'V':
                 BE_VERBOSE = TRUE;
                 admin_verbose = XML_BOOLEAN_TRUE;
                 crm_bump_log_level();
                 break;
             case 't':
                 message_timeout_ms = atoi(optarg);
                 if (message_timeout_ms < 1) {
                     message_timeout_ms = 30 * 1000;
                 }
                 break;
 
             case '$':
             case '?':
                 crm_help(flag, EX_OK);
                 break;
             case 'D':
                 DO_WHOIS_DC = TRUE;
                 break;
             case 'B':
                 BASH_EXPORT = TRUE;
                 break;
             case 'K':
                 DO_RESET = TRUE;
                 crm_trace("Option %c => %s", flag, optarg);
                 dest_node = strdup(optarg);
                 crmd_operation = CRM_OP_LOCAL_SHUTDOWN;
                 break;
             case 'q':
                 BE_SILENT = TRUE;
                 break;
             case 'i':
                 DO_DEBUG = debug_inc;
                 crm_trace("Option %c => %s", flag, optarg);
                 dest_node = strdup(optarg);
                 break;
             case 'd':
                 DO_DEBUG = debug_dec;
                 crm_trace("Option %c => %s", flag, optarg);
                 dest_node = strdup(optarg);
                 break;
             case 'S':
                 DO_HEALTH = TRUE;
                 crm_trace("Option %c => %s", flag, optarg);
                 dest_node = strdup(optarg);
                 break;
             case 'E':
                 DO_ELECT_DC = TRUE;
                 break;
             case 'N':
                 DO_NODE_LIST = TRUE;
                 break;
             case 'H':
                 DO_HEALTH = TRUE;
                 break;
             default:
                 printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
                 ++argerr;
                 break;
         }
     }
 
     if (optind < argc) {
         printf("non-option ARGV-elements: ");
         while (optind < argc)
             printf("%s ", argv[optind++]);
         printf("\n");
     }
 
     if (optind > argc) {
         ++argerr;
     }
 
     if (argerr) {
         crm_help('?', EX_USAGE);
     }
 
     if (do_init()) {
         int res = 0;
 
         res = do_work();
         if (res > 0) {
             /* wait for the reply by creating a mainloop and running it until
              * the callbacks are invoked...
              */
             mainloop = g_main_new(FALSE);
             crm_trace("Waiting for %d replies from the local CRM", expected_responses);
 
             message_timer_id = g_timeout_add(message_timeout_ms, admin_message_timeout, NULL);
 
             g_main_run(mainloop);
 
         } else if (res < 0) {
             crm_err("No message to send");
             operation_status = -1;
         }
     } else {
         crm_warn("Init failed, could not perform requested operations");
         operation_status = -2;
     }
 
     crm_trace("%s exiting normally", crm_system_name);
     return operation_status;
 }
 
 int
 do_work(void)
 {
     int ret = 1;
 
     /* construct the request */
     xmlNode *msg_data = NULL;
     gboolean all_is_good = TRUE;
 
     msg_options = create_xml_node(NULL, XML_TAG_OPTIONS);
     crm_xml_add(msg_options, XML_ATTR_VERBOSE, admin_verbose);
     crm_xml_add(msg_options, XML_ATTR_TIMEOUT, "0");
 
     if (DO_HEALTH == TRUE) {
         crm_trace("Querying the system");
 
         sys_to = CRM_SYSTEM_DC;
 
         if (dest_node != NULL) {
             sys_to = CRM_SYSTEM_CRMD;
             crmd_operation = CRM_OP_PING;
 
             if (BE_VERBOSE) {
                 expected_responses = 1;
             }
 
             crm_xml_add(msg_options, XML_ATTR_TIMEOUT, "0");
 
         } else {
             crm_info("Cluster-wide health not available yet");
             all_is_good = FALSE;
         }
 
     } else if (DO_ELECT_DC) {
         /* tell the local node to initiate an election */
 
         dest_node = NULL;
         sys_to = CRM_SYSTEM_CRMD;
         crmd_operation = CRM_OP_VOTE;
 
         crm_xml_add(msg_options, XML_ATTR_TIMEOUT, "0");
         ret = 0;                /* no return message */
 
     } else if (DO_WHOIS_DC) {
         dest_node = NULL;
         sys_to = CRM_SYSTEM_DC;
         crmd_operation = CRM_OP_PING;
         crm_xml_add(msg_options, XML_ATTR_TIMEOUT, "0");
 
 
     } else if (DO_NODE_LIST) {
 
         cib_t *the_cib = cib_new();
         xmlNode *output = NULL;
 
         int rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command);
 
         if (rc != pcmk_ok) {
             return -1;
         }
 
         output = get_cib_copy(the_cib);
         do_find_node_list(output);
 
         free_xml(output);
         the_cib->cmds->signoff(the_cib);
         exit(rc);
 
     } else if (DO_RESET) {
         /* tell dest_node to initiate the shutdown proceedure
          *
          * if dest_node is NULL, the request will be sent to the
          *   local node
          */
         sys_to = CRM_SYSTEM_CRMD;
         crm_xml_add(msg_options, XML_ATTR_TIMEOUT, "0");
 
         ret = 0;                /* no return message */
 
     } else if (DO_DEBUG == debug_inc) {
         /* tell dest_node to increase its debug level
          *
          * if dest_node is NULL, the request will be sent to the
          *   local node
          */
         sys_to = CRM_SYSTEM_CRMD;
         crmd_operation = CRM_OP_DEBUG_UP;
 
         ret = 0;                /* no return message */
 
     } else if (DO_DEBUG == debug_dec) {
         /* tell dest_node to increase its debug level
          *
          * if dest_node is NULL, the request will be sent to the
          *   local node
          */
         sys_to = CRM_SYSTEM_CRMD;
         crmd_operation = CRM_OP_DEBUG_DOWN;
 
         ret = 0;                /* no return message */
 
     } else {
         crm_err("Unknown options");
         all_is_good = FALSE;
     }
 
     if (all_is_good == FALSE) {
         crm_err("Creation of request failed.  No message to send");
         return -1;
     }
 
 /* send it */
     if (crmd_channel == NULL) {
         crm_err("The IPC connection is not valid, cannot send anything");
         return -1;
     }
 
     if (sys_to == NULL) {
         if (dest_node != NULL) {
             sys_to = CRM_SYSTEM_CRMD;
         } else {
             sys_to = CRM_SYSTEM_DC;
         }
     }
 
     {
         xmlNode *cmd = create_request(crmd_operation, msg_data, dest_node, sys_to,
                                       crm_system_name, admin_uuid);
 
         crm_ipc_send(crmd_channel, cmd, NULL, 0);
         free_xml(cmd);
     }
 
     return ret;
 }
 
 void
 crmadmin_ipc_connection_destroy(gpointer user_data)
 {
     crm_err("Connection to CRMd was terminated");
     if (mainloop) {
         g_main_quit(mainloop);
     } else {
         exit(1);
     }
 }
 
 struct ipc_client_callbacks crm_callbacks = 
 {
     .dispatch = admin_msg_callback,
     .destroy = crmadmin_ipc_connection_destroy
 };
 
 gboolean
 do_init(void)
 {
     mainloop_io_t *source = mainloop_add_ipc_client(CRM_SYSTEM_CRMD, 0, NULL, &crm_callbacks);
 
     admin_uuid = calloc(1, 11);
     if (admin_uuid != NULL) {
         snprintf(admin_uuid, 10, "%d", getpid());
         admin_uuid[10] = '\0';
     }
 
     crmd_channel = mainloop_get_ipc_client(source);
 
     if (DO_RESOURCE || DO_RESOURCE_LIST || DO_NODE_LIST) {
         return TRUE;
 
     } else if (crmd_channel != NULL) {
         xmlNode *xml = create_hello_message(admin_uuid, crm_system_name, "0", "1");
         crm_ipc_send(crmd_channel, xml, NULL, 0);
         return TRUE;
     }
     return FALSE;
 }
 
 static xmlNode *
 validate_crm_message(xmlNode * msg, const char *sys, const char *uuid, const char *msg_type)
 {
     const char *to = NULL;
     const char *type = NULL;
     const char *crm_msg_reference = NULL;
     xmlNode *action = NULL;
     const char *true_sys;
     char *local_sys = NULL;
 
     if (msg == NULL) {
         return NULL;
     }
 
     to = crm_element_value(msg, F_CRM_SYS_TO);
     type = crm_element_value(msg, F_CRM_MSG_TYPE);
 
     crm_msg_reference = crm_element_value(msg, XML_ATTR_REFERENCE);
     action = msg;
     true_sys = sys;
 
     if (uuid != NULL) {
         local_sys = generate_hash_key(sys, uuid);
         true_sys = local_sys;
     }
 
     if (to == NULL) {
         crm_info("No sub-system defined.");
         action = NULL;
     } else if (true_sys != NULL && strcasecmp(to, true_sys) != 0) {
         crm_trace("The message is not for this sub-system (%s != %s).", to, true_sys);
         action = NULL;
     }
 
     free(local_sys);
 
     if (type == NULL) {
         crm_info("No message type defined.");
         return NULL;
 
     } else if (msg_type != NULL && strcasecmp(msg_type, type) != 0) {
         crm_info("Expecting a (%s) message but received a (%s).", msg_type, type);
         action = NULL;
     }
 
     if (crm_msg_reference == NULL) {
         crm_info("No message crm_msg_reference defined.");
         action = NULL;
     }
 /*
  	if(action != NULL) 
 		crm_trace(
 		       "XML is valid and node with message type (%s) found.",
 		       type);
 	crm_trace("Returning node (%s)", crm_element_name(action));
 */
 
     return action;
 }
 
 int
 admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata)
 {
     static int received_responses = 0;
     const char *result = NULL;
     xmlNode *xml = string2xml(buffer);
 
     received_responses++;
     g_source_remove(message_timer_id);
 
     crm_log_xml_trace(xml, "ipc");
 
     if (xml == NULL) {
         crm_info("XML in IPC message was not valid... " "discarding.");
 
     } else if (validate_crm_message(xml, crm_system_name, admin_uuid, XML_ATTR_RESPONSE) == FALSE) {
         crm_trace("Message was not a CRM response. Discarding.");
 
     } else {
         result = crm_element_value(xml, XML_ATTR_RESULT);
         if (result == NULL || strcasecmp(result, "ok") == 0) {
             result = "pass";
 
         } else {
             result = "fail";
         }
 
         if (DO_HEALTH) {
             xmlNode *data = get_message_xml(xml, F_CRM_DATA);
             const char *state = crm_element_value(data, "crmd_state");
 
             printf("Status of %s@%s: %s (%s)\n",
                    crm_element_value(data, XML_PING_ATTR_SYSFROM),
                    crm_element_value(xml, F_CRM_HOST_FROM),
                    state, crm_element_value(data, XML_PING_ATTR_STATUS));
 
             if (BE_SILENT && state != NULL) {
                 fprintf(stderr, "%s\n", state);
             }
 
         } else if (DO_WHOIS_DC) {
             const char *dc = crm_element_value(xml, F_CRM_HOST_FROM);
 
             printf("Designated Controller is: %s\n", dc);
             if (BE_SILENT && dc != NULL) {
                 fprintf(stderr, "%s\n", dc);
             }
             exit(0);
         }
     }
     
     free_xml(xml);
 
     if (received_responses >= expected_responses) {
         crm_trace("Received expected number (%d) of messages from Heartbeat."
                     "  Exiting normally.", expected_responses);
         exit(0);
     }
 
     message_timer_id = g_timeout_add(message_timeout_ms, admin_message_timeout, NULL);
     return 0;
 }
 
 gboolean
 admin_message_timeout(gpointer data)
 {
     fprintf(stderr, "No messages received in %d seconds.. aborting\n",
             (int)message_timeout_ms / 1000);
     crm_err("No messages received in %d seconds", (int)message_timeout_ms / 1000);
     operation_status = -3;
     g_main_quit(mainloop);
     return FALSE;
 }
 
 gboolean
 is_node_online(xmlNode * node_state)
 {
     const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);
     const char *join_state = crm_element_value(node_state, XML_NODE_JOIN_STATE);
     const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
     const char *crm_state = crm_element_value(node_state, XML_NODE_IS_PEER);
-    const char *ha_state = crm_element_value(node_state, XML_CIB_ATTR_HASTATE);
     const char *ccm_state = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
 
     if (safe_str_neq(join_state, CRMD_JOINSTATE_DOWN)
-        && (ha_state == NULL || safe_str_eq(ha_state, "active"))
         && crm_is_true(ccm_state)
         && safe_str_eq(crm_state, "online")) {
         crm_trace("Node %s is online", uname);
         return TRUE;
     }
-    crm_trace("Node %s: ha=%s ccm=%s join=%s exp=%s crm=%s",
-                uname, crm_str(ha_state), crm_str(ccm_state),
+    crm_trace("Node %s: ccm=%s join=%s exp=%s crm=%s",
+                uname, crm_str(ccm_state),
                 crm_str(join_state), crm_str(exp_state), crm_str(crm_state));
     crm_trace("Node %s is offline", uname);
     return FALSE;
 }
 
 int
 do_find_node_list(xmlNode * xml_node)
 {
     int found = 0;
     xmlNode *node = NULL;
     xmlNode *nodes = get_object_root(XML_CIB_TAG_NODES, xml_node);
 
     for (node = __xml_first_child(nodes); node != NULL; node = __xml_next(node)) {
         if (crm_str_eq((const char *)node->name, XML_CIB_TAG_NODE, TRUE)) {
 
             if (BASH_EXPORT) {
                 printf("export %s=%s\n",
                        crm_element_value(node, XML_ATTR_UNAME),
                        crm_element_value(node, XML_ATTR_ID));
             } else {
                 printf("%s node: %s (%s)\n",
                        crm_element_value(node, XML_ATTR_TYPE),
                        crm_element_value(node, XML_ATTR_UNAME),
                        crm_element_value(node, XML_ATTR_ID));
             }
             found++;
         }
     }
 
     if (found == 0) {
         printf("NO nodes configured\n");
     }
 
     return found;
 }