diff --git a/crmd/crmd_utils.h b/crmd/crmd_utils.h
index 1acba01f38..a1aaad32da 100644
--- a/crmd/crmd_utils.h
+++ b/crmd/crmd_utils.h
@@ -1,134 +1,133 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 #ifndef CRMD_UTILS__H
 #  define CRMD_UTILS__H
 
 #  include <crm/crm.h>
 #  include <crm/common/xml.h>
 #  include <crm/cib/internal.h> /* For CIB_OP_MODIFY */
 #  include "crmd_alerts.h"
 
 #  define CLIENT_EXIT_WAIT 30
 #  define FAKE_TE_ID	"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
 
 
 #  define fsa_cib_delete(section, data, options, call_id, user_name)	\
 	if(fsa_cib_conn != NULL) {					\
 	    call_id = cib_internal_op(                                  \
 		fsa_cib_conn, CIB_OP_DELETE, NULL, section, data,	\
 		NULL, options, user_name);				\
 									\
 	} else {							\
 		crm_err("No CIB connection available");			\
 	}
 
 #  define fsa_cib_update(section, data, options, call_id, user_name)	\
 	if(fsa_cib_conn != NULL) {					\
 	    call_id = cib_internal_op(                                  \
 		fsa_cib_conn, CIB_OP_MODIFY, NULL, section, data,	\
 		NULL, options, user_name);				\
 									\
 	} else {							\
 		crm_err("No CIB connection available");			\
 	}
 
 #  define fsa_cib_anon_update(section, data, options)			\
 	if(fsa_cib_conn != NULL) {					\
 	    fsa_cib_conn->cmds->modify(					\
 		fsa_cib_conn, section, data, options);			\
 									\
 	} else {							\
 		crm_err("No CIB connection available");			\
 	}
 
 extern gboolean fsa_has_quorum;
 extern int last_peer_update;
 extern int last_resource_update;
 
 enum node_update_flags {
     node_update_none = 0x0000,
     node_update_quick = 0x0001,
     node_update_cluster = 0x0010,
     node_update_peer = 0x0020,
     node_update_join = 0x0040,
     node_update_expected = 0x0100,
     node_update_all = node_update_cluster|node_update_peer|node_update_join|node_update_expected,
 };
 
 gboolean crm_timer_stop(fsa_timer_t * timer);
 gboolean crm_timer_start(fsa_timer_t * timer);
 gboolean crm_timer_popped(gpointer data);
 gboolean is_timer_started(fsa_timer_t * timer);
 
 int crmd_exit(int rc);
 int crmd_fast_exit(int rc);
 gboolean stop_subsystem(struct crm_subsystem_s *centry, gboolean force_quit);
 gboolean start_subsystem(struct crm_subsystem_s *centry);
 
 void fsa_dump_actions(long long action, const char *text);
 void fsa_dump_inputs(int log_level, const char *text, long long input_register);
 
 gboolean update_dc(xmlNode * msg);
 void crm_update_peer_join(const char *source, crm_node_t * node, enum crm_join_phase phase);
 xmlNode *create_node_state_update(crm_node_t *node, int flags,
                                   xmlNode *parent, const char *source);
 void populate_cib_nodes(enum node_update_flags flags, const char *source);
 void crm_update_quorum(gboolean quorum, gboolean force_update);
 void erase_status_tag(const char *uname, const char *tag, int options);
-void init_transient_attrs(const char *uname, const char *start_state, int options);
 void update_attrd(const char *host, const char *name, const char *value, const char *user_name, gboolean is_remote_node);
 void update_attrd_remote_node_removed(const char *host, const char *user_name);
 void update_attrd_clear_failures(const char *host, const char *rsc,
                                  const char *op, const char *interval,
                                  gboolean is_remote_node);
 
 int crmd_join_phase_count(enum crm_join_phase phase);
 void crmd_join_phase_log(int level);
 
 const char *get_timer_desc(fsa_timer_t * timer);
 gboolean too_many_st_failures(void);
 void st_fail_count_reset(const char * target);
 void crmd_peer_down(crm_node_t *peer, bool full);
 
 /* Convenience macro for registering a CIB callback
  * (assumes that data can be freed with free())
  */
 #  define fsa_register_cib_callback(id, flag, data, fn) do {            \
     CRM_ASSERT(fsa_cib_conn);                                           \
     fsa_cib_conn->cmds->register_callback_full(                         \
             fsa_cib_conn, id, 10 * (1 + crm_active_peers()),            \
             flag, data, #fn, fn, free);                                 \
     } while(0)
 
 #  define start_transition(state) do {					\
 	switch(state) {							\
 	    case S_TRANSITION_ENGINE:					\
 		register_fsa_action(A_TE_CANCEL);			\
 		break;							\
 	    case S_POLICY_ENGINE:					\
 	    case S_IDLE:						\
 		register_fsa_input(C_FSA_INTERNAL, I_PE_CALC, NULL);	\
 		break;							\
 	    default:							\
 		crm_debug("NOT starting a new transition in state %s",	\
 			  fsa_state2string(fsa_state));			\
 		break;							\
 	}								\
     } while(0)
 
 #endif
diff --git a/crmd/join_client.c b/crmd/join_client.c
index ede9bd103d..319272d0fc 100644
--- a/crmd/join_client.c
+++ b/crmd/join_client.c
@@ -1,280 +1,302 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 #include <crm_internal.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crmd_fsa.h>
 #include <crmd_messages.h>
 
 int reannounce_count = 0;
 void join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);
 
 extern ha_msg_input_t *copy_ha_msg_input(ha_msg_input_t * orig);
 
 /*	A_CL_JOIN_QUERY		*/
 /* is there a DC out there? */
 void
 do_cl_join_query(long long action,
                  enum crmd_fsa_cause cause,
                  enum crmd_fsa_state cur_state,
                  enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
                                   CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
 
     sleep(1);                   /* give the CCM time to propogate to the DC */
     update_dc(NULL);            /* Unset any existing value so that the result is not discarded */
     crm_debug("Querying for a DC");
     send_cluster_message(NULL, crm_msg_crmd, req, FALSE);
     free_xml(req);
 }
 
 /*	 A_CL_JOIN_ANNOUNCE	*/
 
 /* this is kind of a workaround for the fact that we may not be around or
  * are otherwise unable to reply when the DC sends out A_DC_JOIN_OFFER_ALL
  */
 void
 do_cl_join_announce(long long action,
                     enum crmd_fsa_cause cause,
                     enum crmd_fsa_state cur_state,
                     enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     /* Once we hear from the DC, we can stop the timer
      *
      * This timer was started either on startup or when a node
      * left the CCM list
      */
 
     /* don't announce if we're in one of these states */
     if (cur_state != S_PENDING) {
         crm_warn("Not announcing cluster join because in state %s",
                  fsa_state2string(cur_state));
         return;
     }
 
     if (AM_I_OPERATIONAL) {
         /* send as a broadcast */
         xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
                                       CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
 
         crm_debug("Announcing availability");
         update_dc(NULL);
         send_cluster_message(NULL, crm_msg_crmd, req, FALSE);
         free_xml(req);
 
     } else {
         /* Delay announce until we have finished local startup */
         crm_warn("Delaying announce of cluster join until local startup is complete");
         return;
     }
 }
 
 static int query_call_id = 0;
 
 /*	 A_CL_JOIN_REQUEST	*/
 /* aka. accept the welcome offer */
 void
 do_cl_join_offer_respond(long long action,
                          enum crmd_fsa_cause cause,
                          enum crmd_fsa_state cur_state,
                          enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
     const char *welcome_from = crm_element_value(input->msg, F_CRM_HOST_FROM);
     const char *join_id = crm_element_value(input->msg, F_CRM_JOIN_ID);
 
 #if 0
     if (we are sick) {
         log error;
 
         /* save the request for later? */
         return;
     }
 #endif
 
     crm_trace("Accepting cluster join offer from node %s "CRM_XS" join-%s",
               welcome_from, crm_element_value(input->msg, F_CRM_JOIN_ID));
 
     /* we only ever want the last one */
     if (query_call_id > 0) {
         crm_trace("Cancelling previous join query: %d", query_call_id);
         remove_cib_op_callback(query_call_id, FALSE);
         query_call_id = 0;
     }
 
     if (update_dc(input->msg) == FALSE) {
         crm_warn("Discarding cluster join offer from node %s (expected %s)",
                  welcome_from, fsa_our_dc);
         return;
     }
 
     CRM_LOG_ASSERT(input != NULL);
     query_call_id =
         fsa_cib_conn->cmds->query(fsa_cib_conn, NULL, NULL, cib_scope_local | cib_no_children);
     fsa_register_cib_callback(query_call_id, FALSE, strdup(join_id), join_query_callback);
     crm_trace("Registered join query callback: %d", query_call_id);
 
     register_fsa_action(A_DC_TIMER_STOP);
 }
 
 void
 join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     char *join_id = user_data;
     xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);
 
     CRM_LOG_ASSERT(join_id != NULL);
 
     if (query_call_id != call_id) {
         crm_trace("Query %d superseded", call_id);
         goto done;
     }
 
     query_call_id = 0;
     if(rc != pcmk_ok || output == NULL) {
         crm_err("Could not retrieve version details for join-%s: %s (%d)",
                 join_id, pcmk_strerror(rc), rc);
         register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __FUNCTION__);
 
     } else if (fsa_our_dc == NULL) {
         crm_debug("Membership is in flux, not continuing join-%s", join_id);
 
     } else {
         xmlNode *reply = NULL;
 
         crm_debug("Respond to join offer join-%s from %s", join_id, fsa_our_dc);
         copy_in_properties(generation, output);
 
         reply = create_request(CRM_OP_JOIN_REQUEST, generation, fsa_our_dc,
                                CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
 
         crm_xml_add(reply, F_CRM_JOIN_ID, join_id);
         send_cluster_message(crm_get_peer(0, fsa_our_dc), crm_msg_crmd, reply, TRUE);
         free_xml(reply);
     }
 
   done:
     free_xml(generation);
 }
 
+static void
+set_join_state(const char * start_state)
+{
+    if (safe_str_eq(start_state, "standby")) {
+        crm_notice("Forcing node %s to join in %s state per configured environment",
+                   fsa_our_uname, start_state);
+        update_attr_delegate(fsa_cib_conn, cib_sync_call, XML_CIB_TAG_NODES, fsa_our_uuid,
+                             NULL, NULL, NULL, "standby", "on", TRUE, NULL, NULL);
+
+    } else if (safe_str_eq(start_state, "online")) {
+        crm_notice("Forcing node %s to join in %s state per configured environment",
+                   fsa_our_uname, start_state);
+        update_attr_delegate(fsa_cib_conn, cib_sync_call, XML_CIB_TAG_NODES, fsa_our_uuid,
+                             NULL, NULL, NULL, "standby", "off", TRUE, NULL, NULL);
+
+    } else if (safe_str_eq(start_state, "default")) {
+        crm_debug("Not forcing a starting state on node %s", fsa_our_uname);
+
+    } else {
+        crm_warn("Unrecognized start state '%s', using 'default' (%s)",
+                 start_state, fsa_our_uname);
+    }
+}
+
 /*	A_CL_JOIN_RESULT	*/
 /* aka. this is notification that we have (or have not) been accepted */
 void
 do_cl_join_finalize_respond(long long action,
                             enum crmd_fsa_cause cause,
                             enum crmd_fsa_state cur_state,
                             enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     xmlNode *tmp1 = NULL;
     gboolean was_nack = TRUE;
     static gboolean first_join = TRUE;
     ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
     const char *start_state = daemon_option("node_start_state");
 
     int join_id = -1;
     const char *op = crm_element_value(input->msg, F_CRM_TASK);
     const char *ack_nack = crm_element_value(input->msg, CRM_OP_JOIN_ACKNAK);
     const char *welcome_from = crm_element_value(input->msg, F_CRM_HOST_FROM);
 
     if (safe_str_neq(op, CRM_OP_JOIN_ACKNAK)) {
         crm_trace("Ignoring op=%s message", op);
         return;
     }
 
     /* calculate if it was an ack or a nack */
     if (crm_is_true(ack_nack)) {
         was_nack = FALSE;
     }
 
     crm_element_value_int(input->msg, F_CRM_JOIN_ID, &join_id);
 
     if (was_nack) {
         crm_err("Shutting down because cluster join with leader %s failed "
                 CRM_XS" join-%d NACK'd", welcome_from, join_id);
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
         return;
     }
 
     if (AM_I_DC == FALSE && safe_str_eq(welcome_from, fsa_our_uname)) {
         crm_warn("Discarding our own welcome - we're no longer the DC");
         return;
     }
 
     if (update_dc(input->msg) == FALSE) {
         crm_warn("Discarding %s from node %s (expected from %s)",
                  op, welcome_from, fsa_our_dc);
         return;
     }
 
     /* send our status section to the DC */
     tmp1 = do_lrm_query(TRUE, fsa_our_uname);
     if (tmp1 != NULL) {
         xmlNode *reply = create_request(CRM_OP_JOIN_CONFIRM, tmp1, fsa_our_dc,
                                         CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
 
         crm_xml_add_int(reply, F_CRM_JOIN_ID, join_id);
 
         crm_debug("Confirming join-%d: sending local operation history to %s",
                   join_id, fsa_our_dc);
 
         /*
          * If this is the node's first join since the crmd started on it, clear
          * any previous transient node attributes, to handle the case where
          * the node restarted so quickly that the cluster layer didn't notice.
          *
          * Do not remove the resources though, they'll be cleaned up in
          * do_dc_join_ack(). Removing them here creates a race condition if the
          * crmd is being recovered. Instead of a list of active resources from
          * the lrmd, we may end up with a blank status section. If we are _NOT_
          * lucky, we will probe for the "wrong" instance of anonymous clones and
          * end up with multiple active instances on the machine.
          */
         if (first_join && is_not_set(fsa_input_register, R_SHUTDOWN)) {
             first_join = FALSE;
+            erase_status_tag(fsa_our_uname, XML_TAG_TRANSIENT_NODEATTRS, 0);
+            update_attrd(fsa_our_uname, "terminate", NULL, NULL, FALSE);
+            update_attrd(fsa_our_uname, XML_CIB_ATTR_SHUTDOWN, "0", NULL, FALSE);
 
             if (start_state) {
-                init_transient_attrs(fsa_our_uname, start_state, 0);
-            } else {
-                erase_status_tag(fsa_our_uname, XML_TAG_TRANSIENT_NODEATTRS, 0);
+                set_join_state(start_state);
             }
-
-            update_attrd(fsa_our_uname, "terminate", NULL, NULL, FALSE);
-            update_attrd(fsa_our_uname, XML_CIB_ATTR_SHUTDOWN, "0", NULL, FALSE);
         }
 
         send_cluster_message(crm_get_peer(0, fsa_our_dc), crm_msg_crmd, reply, TRUE);
         free_xml(reply);
 
         if (AM_I_DC == FALSE) {
             register_fsa_input_adv(cause, I_NOT_DC, NULL, A_NOTHING, TRUE, __FUNCTION__);
             update_attrd(NULL, NULL, NULL, NULL, FALSE);
         }
 
         free_xml(tmp1);
 
     } else {
         crm_err("Could not confirm join-%d with %s: Local operation history failed",
                 join_id, fsa_our_dc);
         register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
     }
 }
diff --git a/crmd/te_actions.c b/crmd/te_actions.c
index c41d44d2ca..a8ad86f941 100644
--- a/crmd/te_actions.c
+++ b/crmd/te_actions.c
@@ -1,774 +1,768 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 
 #include <crm/common/xml.h>
 #include <tengine.h>
 
 #include <crmd_fsa.h>
 #include <crmd_lrm.h>
 #include <crmd_messages.h>
 #include <crm/cluster.h>
 #include <throttle.h>
 
 char *te_uuid = NULL;
 GHashTable *te_targets = NULL;
 void send_rsc_command(crm_action_t * action);
 static void te_update_job_count(crm_action_t * action, int offset);
 
 static void
 te_start_action_timer(crm_graph_t * graph, crm_action_t * action)
 {
     action->timer = calloc(1, sizeof(crm_action_timer_t));
     action->timer->timeout = action->timeout;
     action->timer->reason = timeout_action;
     action->timer->action = action;
     action->timer->source_id = g_timeout_add(action->timer->timeout + graph->network_delay,
                                              action_timer_callback, (void *)action->timer);
 
     CRM_ASSERT(action->timer->source_id != 0);
 }
 
 static gboolean
 te_pseudo_action(crm_graph_t * graph, crm_action_t * pseudo)
 {
     const char *task = crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK);
 
     /* send to peers as well? */
     if (safe_str_eq(task, CRM_OP_MAINTENANCE_NODES)) {
         GHashTableIter iter;
         crm_node_t *node = NULL;
 
         g_hash_table_iter_init(&iter, crm_peer_cache);
         while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
             xmlNode *cmd = NULL;
 
             if (safe_str_eq(fsa_our_uname, node->uname)) {
                 continue;
             }
 
             cmd = create_request(task, pseudo->xml, node->uname,
                                  CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
             send_cluster_message(node, crm_msg_crmd, cmd, FALSE);
             free_xml(cmd);
         }
 
         remote_ra_process_maintenance_nodes(pseudo->xml);
     } else {
         /* Check action for Pacemaker Remote node side effects */
         remote_ra_process_pseudo(pseudo->xml);
     }
 
     crm_debug("Pseudo-action %d (%s) fired and confirmed", pseudo->id,
               crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK_KEY));
     te_action_confirmed(pseudo);
     update_graph(graph, pseudo);
     trigger_graph();
     return TRUE;
 }
 
 void
 send_stonith_update(crm_action_t * action, const char *target, const char *uuid)
 {
     int rc = pcmk_ok;
     crm_node_t *peer = NULL;
 
     /* We (usually) rely on the membership layer to do node_update_cluster,
      * and the peer status callback to do node_update_peer, because the node
      * might have already rejoined before we get the stonith result here.
      */
     int flags = node_update_join | node_update_expected;
 
     /* zero out the node-status & remove all LRM status info */
     xmlNode *node_state = NULL;
 
-    const char *start_state = daemon_option("node_start_state");
-
     CRM_CHECK(target != NULL, return);
     CRM_CHECK(uuid != NULL, return);
 
     /* Make sure the membership and join caches are accurate */
     peer = crm_get_peer_full(0, target, CRM_GET_PEER_ANY);
 
     CRM_CHECK(peer != NULL, return);
 
     if (peer->state == NULL) {
         /* Usually, we rely on the membership layer to update the cluster state
          * in the CIB. However, if the node has never been seen, do it here, so
          * the node is not considered unclean.
          */
         flags |= node_update_cluster;
     }
 
     if (peer->uuid == NULL) {
         crm_info("Recording uuid '%s' for node '%s'", uuid, target);
         peer->uuid = strdup(uuid);
     }
 
     crmd_peer_down(peer, TRUE);
 
     /* Generate a node state update for the CIB */
     node_state = create_node_state_update(peer, flags, NULL, __FUNCTION__);
 
     /* we have to mark whether or not remote nodes have already been fenced */
     if (peer->flags & crm_remote_node) {
         time_t now = time(NULL);
         char *now_s = crm_itoa(now);
         crm_xml_add(node_state, XML_NODE_IS_FENCED, now_s);
         free(now_s);
     }
 
     /* Force our known ID */
     crm_xml_add(node_state, XML_ATTR_UUID, uuid);
 
     rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, node_state,
                                     cib_quorum_override | cib_scope_local | cib_can_create);
 
     /* Delay processing the trigger until the update completes */
     crm_debug("Sending fencing update %d for %s", rc, target);
     fsa_register_cib_callback(rc, FALSE, strdup(target), cib_fencing_updated);
 
     /* Make sure it sticks */
     /* fsa_cib_conn->cmds->bump_epoch(fsa_cib_conn, cib_quorum_override|cib_scope_local);    */
 
     erase_status_tag(peer->uname, XML_CIB_TAG_LRM, cib_scope_local);
-    if (start_state) {
-        init_transient_attrs(peer->uname, start_state, cib_scope_local);
-    } else {
-        erase_status_tag(peer->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
-    }
+    erase_status_tag(peer->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
 
     free_xml(node_state);
     return;
 }
 
 static gboolean
 te_fence_node(crm_graph_t * graph, crm_action_t * action)
 {
     int rc = 0;
     const char *id = NULL;
     const char *uuid = NULL;
     const char *target = NULL;
     const char *type = NULL;
     gboolean invalid_action = FALSE;
     enum stonith_call_options options = st_opt_none;
 
     id = ID(action->xml);
     target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
     type = crm_meta_value(action->params, "stonith_action");
 
     CRM_CHECK(id != NULL, invalid_action = TRUE);
     CRM_CHECK(uuid != NULL, invalid_action = TRUE);
     CRM_CHECK(type != NULL, invalid_action = TRUE);
     CRM_CHECK(target != NULL, invalid_action = TRUE);
 
     if (invalid_action) {
         crm_log_xml_warn(action->xml, "BadAction");
         return FALSE;
     }
 
     crm_notice("Requesting fencing (%s) of node %s "
                CRM_XS " action=%s timeout=%d",
                type, target, id, transition_graph->stonith_timeout);
 
     /* Passing NULL means block until we can connect... */
     te_connect_stonith(NULL);
 
     if (crmd_join_phase_count(crm_join_confirmed) == 1) {
         options |= st_opt_allow_suicide;
     }
 
     rc = stonith_api->cmds->fence(stonith_api, options, target, type,
                                   transition_graph->stonith_timeout / 1000, 0);
 
     stonith_api->cmds->register_callback(stonith_api, rc, transition_graph->stonith_timeout / 1000,
                                          st_opt_timeout_updates,
                                          generate_transition_key(transition_graph->id, action->id,
                                                                  0, te_uuid),
                                          "tengine_stonith_callback", tengine_stonith_callback);
 
     return TRUE;
 }
 
 static int
 get_target_rc(crm_action_t * action)
 {
     const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);
 
     if (target_rc_s != NULL) {
         return crm_parse_int(target_rc_s, "0");
     }
     return 0;
 }
 
 static gboolean
 te_crm_command(crm_graph_t * graph, crm_action_t * action)
 {
     char *counter = NULL;
     xmlNode *cmd = NULL;
     gboolean is_local = FALSE;
 
     const char *id = NULL;
     const char *task = NULL;
     const char *value = NULL;
     const char *on_node = NULL;
     const char *router_node = NULL;
 
     gboolean rc = TRUE;
     gboolean no_wait = FALSE;
 
     id = ID(action->xml);
     task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 
     if (!router_node) {
         router_node = on_node;
     }
 
     CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
               crm_err("Corrupted command (id=%s) %s: no node", crm_str(id), crm_str(task));
               return FALSE);
 
     crm_info("Executing crm-event (%s): %s on %s%s%s",
              crm_str(id), crm_str(task), on_node,
              is_local ? " (local)" : "", no_wait ? " - no waiting" : "");
 
     if (safe_str_eq(router_node, fsa_our_uname)) {
         is_local = TRUE;
     }
 
     value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
     if (crm_is_true(value)) {
         no_wait = TRUE;
     }
 
     if (is_local && safe_str_eq(task, CRM_OP_SHUTDOWN)) {
         /* defer until everything else completes */
         crm_info("crm-event (%s) is a local shutdown", crm_str(id));
         graph->completion_action = tg_shutdown;
         graph->abort_reason = "local shutdown";
         te_action_confirmed(action);
         update_graph(graph, action);
         trigger_graph();
         return TRUE;
 
     } else if (safe_str_eq(task, CRM_OP_SHUTDOWN)) {
         crm_node_t *peer = crm_get_peer(0, router_node);
         crm_update_peer_expected(__FUNCTION__, peer, CRMD_JOINSTATE_DOWN);
     }
 
     cmd = create_request(task, action->xml, router_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
 
     counter =
         generate_transition_key(transition_graph->id, action->id, get_target_rc(action), te_uuid);
     crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter);
 
     rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_crmd, cmd, TRUE);
     free(counter);
     free_xml(cmd);
 
     if (rc == FALSE) {
         crm_err("Action %d failed: send", action->id);
         return FALSE;
 
     } else if (no_wait) {
         te_action_confirmed(action);
         update_graph(graph, action);
         trigger_graph();
 
     } else {
         if (action->timeout <= 0) {
             crm_err("Action %d: %s on %s had an invalid timeout (%dms).  Using %dms instead",
                     action->id, task, on_node, action->timeout, graph->network_delay);
             action->timeout = graph->network_delay;
         }
         te_start_action_timer(graph, action);
     }
 
     return TRUE;
 }
 
 gboolean
 cib_action_update(crm_action_t * action, int status, int op_rc)
 {
     lrmd_event_data_t *op = NULL;
     xmlNode *state = NULL;
     xmlNode *rsc = NULL;
     xmlNode *xml_op = NULL;
     xmlNode *action_rsc = NULL;
 
     int rc = pcmk_ok;
 
     const char *rsc_id = NULL;
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
     const char *target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
 
     int call_options = cib_quorum_override | cib_scope_local;
     int target_rc = get_target_rc(action);
 
     if (status == PCMK_LRM_OP_PENDING) {
         crm_debug("%s %d: Recording pending operation %s on %s",
                   crm_element_name(action->xml), action->id, task_uuid, target);
     } else {
         crm_warn("%s %d: %s on %s timed out",
                  crm_element_name(action->xml), action->id, task_uuid, target);
     }
 
     action_rsc = find_xml_node(action->xml, XML_CIB_TAG_RESOURCE, TRUE);
     if (action_rsc == NULL) {
         return FALSE;
     }
 
     rsc_id = ID(action_rsc);
     CRM_CHECK(rsc_id != NULL, crm_log_xml_err(action->xml, "Bad:action");
               return FALSE);
 
 /*
   update the CIB
 
 <node_state id="hadev">
       <lrm>
         <lrm_resources>
           <lrm_resource id="rsc2" last_op="start" op_code="0" target="hadev"/>
 */
 
     state = create_xml_node(NULL, XML_CIB_TAG_STATE);
 
     crm_xml_add(state, XML_ATTR_UUID, target_uuid);
     crm_xml_add(state, XML_ATTR_UNAME, target);
 
     rsc = create_xml_node(state, XML_CIB_TAG_LRM);
     crm_xml_add(rsc, XML_ATTR_ID, target_uuid);
 
     rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCES);
     rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCE);
     crm_xml_add(rsc, XML_ATTR_ID, rsc_id);
 
 
     crm_copy_xml_element(action_rsc, rsc, XML_ATTR_TYPE);
     crm_copy_xml_element(action_rsc, rsc, XML_AGENT_ATTR_CLASS);
     crm_copy_xml_element(action_rsc, rsc, XML_AGENT_ATTR_PROVIDER);
 
     op = convert_graph_action(NULL, action, status, op_rc);
     op->call_id = -1;
     op->user_data = generate_transition_key(transition_graph->id, action->id, target_rc, te_uuid);
 
     xml_op = create_operation_update(rsc, op, CRM_FEATURE_SET, target_rc, target, __FUNCTION__, LOG_INFO);
     lrmd_free_event(op);
 
     crm_trace("Updating CIB with \"%s\" (%s): %s %s on %s",
               status < 0 ? "new action" : XML_ATTR_TIMEOUT,
               crm_element_name(action->xml), crm_str(task), rsc_id, target);
     crm_log_xml_trace(xml_op, "Op");
 
     rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, state, call_options);
 
     crm_trace("Updating CIB with %s action %d: %s on %s (call_id=%d)",
               services_lrm_status_str(status), action->id, task_uuid, target, rc);
 
     fsa_register_cib_callback(rc, FALSE, NULL, cib_action_updated);
     free_xml(state);
 
     action->sent_update = TRUE;
 
     if (rc < pcmk_ok) {
         return FALSE;
     }
 
     return TRUE;
 }
 
 static gboolean
 te_rsc_command(crm_graph_t * graph, crm_action_t * action)
 {
     /* never overwrite stop actions in the CIB with
      *   anything other than completed results
      *
      * Writing pending stops makes it look like the
      *   resource is running again
      */
     xmlNode *cmd = NULL;
     xmlNode *rsc_op = NULL;
 
     gboolean rc = TRUE;
     gboolean no_wait = FALSE;
     gboolean is_local = FALSE;
 
     char *counter = NULL;
     const char *task = NULL;
     const char *value = NULL;
     const char *on_node = NULL;
     const char *router_node = NULL;
     const char *task_uuid = NULL;
 
     CRM_ASSERT(action != NULL);
     CRM_ASSERT(action->xml != NULL);
 
     action->executed = FALSE;
     on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 
     CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
               crm_err("Corrupted command(id=%s) %s: no node", ID(action->xml), crm_str(task));
               return FALSE);
 
     rsc_op = action->xml;
     task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
     task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
     router_node = crm_element_value(rsc_op, XML_LRM_ATTR_ROUTER_NODE);
 
     if (!router_node) {
         router_node = on_node;
     }
 
     counter =
         generate_transition_key(transition_graph->id, action->id, get_target_rc(action), te_uuid);
     crm_xml_add(rsc_op, XML_ATTR_TRANSITION_KEY, counter);
 
     if (safe_str_eq(router_node, fsa_our_uname)) {
         is_local = TRUE;
     }
 
     value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
     if (crm_is_true(value)) {
         no_wait = TRUE;
     }
 
     crm_notice("Initiating %s operation %s%s on %s%s "CRM_XS" action %d",
                task, task_uuid, (is_local? " locally" : ""), on_node,
                (no_wait? " without waiting" : ""), action->id);
 
     cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, router_node,
                          CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);
 
     if (is_local) {
         /* shortcut local resource commands */
         ha_msg_input_t data = {
             .msg = cmd,
             .xml = rsc_op,
         };
 
         fsa_data_t msg = {
             .id = 0,
             .data = &data,
             .data_type = fsa_dt_ha_msg,
             .fsa_input = I_NULL,
             .fsa_cause = C_FSA_INTERNAL,
             .actions = A_LRM_INVOKE,
             .origin = __FUNCTION__,
         };
 
         do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, fsa_state, I_NULL, &msg);
 
     } else {
         rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_lrmd, cmd, TRUE);
     }
 
     free(counter);
     free_xml(cmd);
 
     action->executed = TRUE;
 
     if (rc == FALSE) {
         crm_err("Action %d failed: send", action->id);
         return FALSE;
 
     } else if (no_wait) {
         crm_info("Action %d confirmed - no wait", action->id);
         action->confirmed = TRUE; /* Just mark confirmed.
                                    * Don't bump the job count only to immediately decrement it
                                    */
         update_graph(transition_graph, action);
         trigger_graph();
 
     } else if (action->confirmed == TRUE) {
         crm_debug("Action %d: %s %s on %s(timeout %dms) was already confirmed.",
                   action->id, task, task_uuid, on_node, action->timeout);
     } else {
         if (action->timeout <= 0) {
             crm_err("Action %d: %s %s on %s had an invalid timeout (%dms).  Using %dms instead",
                     action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
             action->timeout = graph->network_delay;
         }
         te_update_job_count(action, 1);
         te_start_action_timer(graph, action);
     }
 
     return TRUE;
 }
 
 struct te_peer_s
 {
         char *name;
         int jobs;
         int migrate_jobs;
 };
 
 static void te_peer_free(gpointer p)
 {
     struct te_peer_s *peer = p;
 
     free(peer->name);
     free(peer);
 }
 
 void te_reset_job_counts(void)
 {
     GHashTableIter iter;
     struct te_peer_s *peer = NULL;
 
     if(te_targets == NULL) {
         te_targets = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, te_peer_free);
     }
 
     g_hash_table_iter_init(&iter, te_targets);
     while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & peer)) {
         peer->jobs = 0;
         peer->migrate_jobs = 0;
     }
 }
 
 static void
 te_update_job_count_on(const char *target, int offset, bool migrate)
 {
     struct te_peer_s *r = NULL;
 
     if(target == NULL || te_targets == NULL) {
         return;
     }
 
     r = g_hash_table_lookup(te_targets, target);
     if(r == NULL) {
         r = calloc(1, sizeof(struct te_peer_s));
         r->name = strdup(target);
         g_hash_table_insert(te_targets, r->name, r);
     }
 
     r->jobs += offset;
     if(migrate) {
         r->migrate_jobs += offset;
     }
     crm_trace("jobs[%s] = %d", target, r->jobs);
 }
 
 static void
 te_update_job_count(crm_action_t * action, int offset)
 {
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 
     if (action->type != action_type_rsc || target == NULL) {
         /* No limit on these */
         return;
     }
 
     /* if we have a router node, this means the action is performing
      * on a remote node. For now, we count all action occuring on a
      * remote node against the job list on the cluster node hosting
      * the connection resources */
     target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 
     if ((target == NULL) &&
         (safe_str_eq(task, CRMD_ACTION_MIGRATE) || safe_str_eq(task, CRMD_ACTION_MIGRATED))) {
 
         const char *t1 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
         const char *t2 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);
 
         te_update_job_count_on(t1, offset, TRUE);
         te_update_job_count_on(t2, offset, TRUE);
         return;
     } else if (target == NULL) {
         target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     }
 
     te_update_job_count_on(target, offset, FALSE);
 }
 
 static gboolean
 te_should_perform_action_on(crm_graph_t * graph, crm_action_t * action, const char *target)
 {
     int limit = 0;
     struct te_peer_s *r = NULL;
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     const char *id = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 
     if(target == NULL) {
         /* No limit on these */
         return TRUE;
 
     } else if(te_targets == NULL) {
         return FALSE;
     }
 
     r = g_hash_table_lookup(te_targets, target);
     limit = throttle_get_job_limit(target);
 
     if(r == NULL) {
         r = calloc(1, sizeof(struct te_peer_s));
         r->name = strdup(target);
         g_hash_table_insert(te_targets, r->name, r);
     }
 
     if(limit <= r->jobs) {
         crm_trace("Peer %s is over their job limit of %d (%d): deferring %s",
                   target, limit, r->jobs, id);
         return FALSE;
 
     } else if(graph->migration_limit > 0 && r->migrate_jobs >= graph->migration_limit) {
         if (safe_str_eq(task, CRMD_ACTION_MIGRATE) || safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
             crm_trace("Peer %s is over their migration job limit of %d (%d): deferring %s",
                       target, graph->migration_limit, r->migrate_jobs, id);
             return FALSE;
         }
     }
 
     crm_trace("Peer %s has not hit their limit yet. current jobs = %d limit= %d limit", target, r->jobs, limit);
 
     return TRUE;
 }
 
 static gboolean
 te_should_perform_action(crm_graph_t * graph, crm_action_t * action)
 {
     const char *target = NULL;
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 
     if (action->type != action_type_rsc) {
         /* No limit on these */
         return TRUE;
     }
 
     /* if we have a router node, this means the action is performing
      * on a remote node. For now, we count all action occuring on a
      * remote node against the job list on the cluster node hosting
      * the connection resources */
     target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 
     if ((target == NULL) &&
         (safe_str_eq(task, CRMD_ACTION_MIGRATE) || safe_str_eq(task, CRMD_ACTION_MIGRATED))) {
 
         target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
         if(te_should_perform_action_on(graph, action, target) == FALSE) {
             return FALSE;
         }
 
         target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);
 
     } else if (target == NULL) {
         target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     }
 
     return te_should_perform_action_on(graph, action, target);
 }
 
 void
 te_action_confirmed(crm_action_t * action)
 {
     const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 
     if (action->confirmed == FALSE && action->type == action_type_rsc && target != NULL) {
         te_update_job_count(action, -1);
     }
     action->confirmed = TRUE;
 }
 
 
 crm_graph_functions_t te_graph_fns = {
     te_pseudo_action,
     te_rsc_command,
     te_crm_command,
     te_fence_node,
     te_should_perform_action,
 };
 
 void
 notify_crmd(crm_graph_t * graph)
 {
     const char *type = "unknown";
     enum crmd_fsa_input event = I_NULL;
 
     crm_debug("Processing transition completion in state %s", fsa_state2string(fsa_state));
 
     CRM_CHECK(graph->complete, graph->complete = TRUE);
 
     switch (graph->completion_action) {
         case tg_stop:
             type = "stop";
             if (fsa_state == S_TRANSITION_ENGINE) {
                 event = I_TE_SUCCESS;
             }
             break;
         case tg_done:
             type = "done";
             if (fsa_state == S_TRANSITION_ENGINE) {
                 event = I_TE_SUCCESS;
             }
             break;
 
         case tg_restart:
             type = "restart";
             if (fsa_state == S_TRANSITION_ENGINE) {
                 if (too_many_st_failures() == FALSE) {
                     if (transition_timer->period_ms > 0) {
                         crm_timer_stop(transition_timer);
                         crm_timer_start(transition_timer);
                     } else {
                         event = I_PE_CALC;
                     }
                 } else {
                     event = I_TE_SUCCESS;
                 }
 
             } else if (fsa_state == S_POLICY_ENGINE) {
                 register_fsa_action(A_PE_INVOKE);
             }
             break;
 
         case tg_shutdown:
             type = "shutdown";
             if (is_set(fsa_input_register, R_SHUTDOWN)) {
                 event = I_STOP;
 
             } else {
                 crm_err("We didn't ask to be shut down, yet our PE is telling us to.");
                 event = I_TERMINATE;
             }
     }
 
     crm_debug("Transition %d status: %s - %s", graph->id, type, crm_str(graph->abort_reason));
 
     graph->abort_reason = NULL;
     graph->completion_action = tg_done;
     clear_bit(fsa_input_register, R_IN_TRANSITION);
 
     if (event != I_NULL) {
         register_fsa_input(C_FSA_INTERNAL, event, NULL);
 
     } else if (fsa_source) {
         mainloop_set_trigger(fsa_source);
     }
 }
diff --git a/crmd/utils.c b/crmd/utils.c
index 82b5448691..07c323d2fa 100644
--- a/crmd/utils.c
+++ b/crmd/utils.c
@@ -1,1069 +1,1019 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/cluster.h>
 
 #include <crmd_fsa.h>
 #include <crmd_utils.h>
 #include <crmd_messages.h>
 
 /*	A_DC_TIMER_STOP, A_DC_TIMER_START,
  *	A_FINALIZE_TIMER_STOP, A_FINALIZE_TIMER_START
  *	A_INTEGRATE_TIMER_STOP, A_INTEGRATE_TIMER_START
  */
 void
 do_timer_control(long long action,
                  enum crmd_fsa_cause cause,
                  enum crmd_fsa_state cur_state,
                  enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     gboolean timer_op_ok = TRUE;
 
     if (action & A_DC_TIMER_STOP) {
         timer_op_ok = crm_timer_stop(election_trigger);
 
     } else if (action & A_FINALIZE_TIMER_STOP) {
         timer_op_ok = crm_timer_stop(finalization_timer);
 
     } else if (action & A_INTEGRATE_TIMER_STOP) {
         timer_op_ok = crm_timer_stop(integration_timer);
 
 /* 	} else if(action & A_ELECTION_TIMEOUT_STOP) { */
 /* 		timer_op_ok = crm_timer_stop(election_timeout); */
     }
 
     /* don't start a timer that wasn't already running */
     if (action & A_DC_TIMER_START && timer_op_ok) {
         crm_timer_start(election_trigger);
         if (AM_I_DC) {
             /* there can be only one */
             register_fsa_input(cause, I_ELECTION, NULL);
         }
 
     } else if (action & A_FINALIZE_TIMER_START) {
         crm_timer_start(finalization_timer);
 
     } else if (action & A_INTEGRATE_TIMER_START) {
         crm_timer_start(integration_timer);
 
 /* 	} else if(action & A_ELECTION_TIMEOUT_START) { */
 /* 		crm_timer_start(election_timeout); */
     }
 }
 
 const char *
 get_timer_desc(fsa_timer_t * timer)
 {
     if (timer == election_trigger) {
         return "Election Trigger";
 
     } else if (timer == shutdown_escalation_timer) {
         return "Shutdown Escalation";
 
     } else if (timer == integration_timer) {
         return "Integration Timer";
 
     } else if (timer == finalization_timer) {
         return "Finalization Timer";
 
     } else if (timer == transition_timer) {
         return "New Transition Timer";
 
     } else if (timer == wait_timer) {
         return "Wait Timer";
 
     } else if (timer == recheck_timer) {
         return "PEngine Recheck Timer";
 
     }
     return "Unknown Timer";
 }
 
 gboolean
 crm_timer_popped(gpointer data)
 {
     fsa_timer_t *timer = (fsa_timer_t *) data;
 
     if (timer == wait_timer
         || timer == recheck_timer
         || timer == transition_timer || timer == finalization_timer || timer == election_trigger) {
         crm_info("%s (%s) just popped (%dms)",
                  get_timer_desc(timer), fsa_input2string(timer->fsa_input), timer->period_ms);
         timer->counter++;
 
     } else {
         crm_err("%s (%s) just popped in state %s! (%dms)",
                 get_timer_desc(timer), fsa_input2string(timer->fsa_input),
                 fsa_state2string(fsa_state), timer->period_ms);
     }
 
     if (timer == election_trigger && election_trigger->counter > 5) {
         crm_notice("We appear to be in an election loop, something may be wrong");
         crm_write_blackbox(0, NULL);
         election_trigger->counter = 0;
     }
 
     if (timer->repeat == FALSE) {
         crm_timer_stop(timer);  /* make it _not_ go off again */
     }
 
     if (timer->fsa_input == I_INTEGRATED) {
         crm_info("Welcomed: %d, Integrated: %d",
                  crmd_join_phase_count(crm_join_welcomed),
                  crmd_join_phase_count(crm_join_integrated));
         if (crmd_join_phase_count(crm_join_welcomed) == 0) {
             /* If we don't even have ourself, start again */
             register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, NULL, __FUNCTION__);
 
         } else {
             register_fsa_input_before(C_TIMER_POPPED, timer->fsa_input, NULL);
         }
 
     } else if (timer == recheck_timer && fsa_state != S_IDLE) {
         crm_debug("Discarding %s event in state: %s",
                   fsa_input2string(timer->fsa_input), fsa_state2string(fsa_state));
 
     } else if (timer == finalization_timer && fsa_state != S_FINALIZE_JOIN) {
         crm_debug("Discarding %s event in state: %s",
                   fsa_input2string(timer->fsa_input), fsa_state2string(fsa_state));
 
     } else if (timer->fsa_input != I_NULL) {
         register_fsa_input(C_TIMER_POPPED, timer->fsa_input, NULL);
     }
 
     crm_trace("Triggering FSA: %s", __FUNCTION__);
     mainloop_set_trigger(fsa_source);
 
     return TRUE;
 }
 
 gboolean
 is_timer_started(fsa_timer_t * timer)
 {
     if (timer->period_ms > 0) {
         if (transition_timer->source_id == 0) {
             return FALSE;
         } else {
             return TRUE;
         }
     }
     return FALSE;
 }
 
 gboolean
 crm_timer_start(fsa_timer_t * timer)
 {
     const char *timer_desc = get_timer_desc(timer);
 
     if (timer->source_id == 0 && timer->period_ms > 0) {
         timer->source_id = g_timeout_add(timer->period_ms, timer->callback, (void *)timer);
         CRM_ASSERT(timer->source_id != 0);
         crm_debug("Started %s (%s:%dms), src=%d",
                   timer_desc, fsa_input2string(timer->fsa_input),
                   timer->period_ms, timer->source_id);
 
     } else if (timer->period_ms < 0) {
         crm_err("Tried to start %s (%s:%dms) with a -ve period",
                 timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms);
 
     } else {
         crm_debug("%s (%s:%dms) already running: src=%d",
                   timer_desc, fsa_input2string(timer->fsa_input),
                   timer->period_ms, timer->source_id);
         return FALSE;
     }
     return TRUE;
 }
 
 gboolean
 crm_timer_stop(fsa_timer_t * timer)
 {
     const char *timer_desc = get_timer_desc(timer);
 
     if (timer == NULL) {
         crm_err("Attempted to stop NULL timer");
         return FALSE;
 
     } else if (timer->source_id != 0) {
         crm_trace("Stopping %s (%s:%dms), src=%d",
                   timer_desc, fsa_input2string(timer->fsa_input),
                   timer->period_ms, timer->source_id);
         g_source_remove(timer->source_id);
         timer->source_id = 0;
 
     } else {
         crm_trace("%s (%s:%dms) already stopped",
                   timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms);
         return FALSE;
     }
     return TRUE;
 }
 
 const char *
 fsa_input2string(enum crmd_fsa_input input)
 {
     const char *inputAsText = NULL;
 
     switch (input) {
         case I_NULL:
             inputAsText = "I_NULL";
             break;
         case I_CIB_OP:
             inputAsText = "I_CIB_OP (unused)";
             break;
         case I_CIB_UPDATE:
             inputAsText = "I_CIB_UPDATE";
             break;
         case I_DC_TIMEOUT:
             inputAsText = "I_DC_TIMEOUT";
             break;
         case I_ELECTION:
             inputAsText = "I_ELECTION";
             break;
         case I_PE_CALC:
             inputAsText = "I_PE_CALC";
             break;
         case I_RELEASE_DC:
             inputAsText = "I_RELEASE_DC";
             break;
         case I_ELECTION_DC:
             inputAsText = "I_ELECTION_DC";
             break;
         case I_ERROR:
             inputAsText = "I_ERROR";
             break;
         case I_FAIL:
             inputAsText = "I_FAIL";
             break;
         case I_INTEGRATED:
             inputAsText = "I_INTEGRATED";
             break;
         case I_FINALIZED:
             inputAsText = "I_FINALIZED";
             break;
         case I_NODE_JOIN:
             inputAsText = "I_NODE_JOIN";
             break;
         case I_JOIN_OFFER:
             inputAsText = "I_JOIN_OFFER";
             break;
         case I_JOIN_REQUEST:
             inputAsText = "I_JOIN_REQUEST";
             break;
         case I_JOIN_RESULT:
             inputAsText = "I_JOIN_RESULT";
             break;
         case I_NOT_DC:
             inputAsText = "I_NOT_DC";
             break;
         case I_RECOVERED:
             inputAsText = "I_RECOVERED";
             break;
         case I_RELEASE_FAIL:
             inputAsText = "I_RELEASE_FAIL";
             break;
         case I_RELEASE_SUCCESS:
             inputAsText = "I_RELEASE_SUCCESS";
             break;
         case I_RESTART:
             inputAsText = "I_RESTART";
             break;
         case I_PE_SUCCESS:
             inputAsText = "I_PE_SUCCESS";
             break;
         case I_ROUTER:
             inputAsText = "I_ROUTER";
             break;
         case I_SHUTDOWN:
             inputAsText = "I_SHUTDOWN";
             break;
         case I_STARTUP:
             inputAsText = "I_STARTUP";
             break;
         case I_TE_SUCCESS:
             inputAsText = "I_TE_SUCCESS";
             break;
         case I_STOP:
             inputAsText = "I_STOP";
             break;
         case I_DC_HEARTBEAT:
             inputAsText = "I_DC_HEARTBEAT";
             break;
         case I_WAIT_FOR_EVENT:
             inputAsText = "I_WAIT_FOR_EVENT";
             break;
         case I_LRM_EVENT:
             inputAsText = "I_LRM_EVENT";
             break;
         case I_PENDING:
             inputAsText = "I_PENDING";
             break;
         case I_HALT:
             inputAsText = "I_HALT";
             break;
         case I_TERMINATE:
             inputAsText = "I_TERMINATE";
             break;
         case I_ILLEGAL:
             inputAsText = "I_ILLEGAL";
             break;
     }
 
     if (inputAsText == NULL) {
         crm_err("Input %d is unknown", input);
         inputAsText = "<UNKNOWN_INPUT>";
     }
 
     return inputAsText;
 }
 
 const char *
 fsa_state2string(enum crmd_fsa_state state)
 {
     const char *stateAsText = NULL;
 
     switch (state) {
         case S_IDLE:
             stateAsText = "S_IDLE";
             break;
         case S_ELECTION:
             stateAsText = "S_ELECTION";
             break;
         case S_INTEGRATION:
             stateAsText = "S_INTEGRATION";
             break;
         case S_FINALIZE_JOIN:
             stateAsText = "S_FINALIZE_JOIN";
             break;
         case S_NOT_DC:
             stateAsText = "S_NOT_DC";
             break;
         case S_POLICY_ENGINE:
             stateAsText = "S_POLICY_ENGINE";
             break;
         case S_RECOVERY:
             stateAsText = "S_RECOVERY";
             break;
         case S_RELEASE_DC:
             stateAsText = "S_RELEASE_DC";
             break;
         case S_PENDING:
             stateAsText = "S_PENDING";
             break;
         case S_STOPPING:
             stateAsText = "S_STOPPING";
             break;
         case S_TERMINATE:
             stateAsText = "S_TERMINATE";
             break;
         case S_TRANSITION_ENGINE:
             stateAsText = "S_TRANSITION_ENGINE";
             break;
         case S_STARTING:
             stateAsText = "S_STARTING";
             break;
         case S_HALT:
             stateAsText = "S_HALT";
             break;
         case S_ILLEGAL:
             stateAsText = "S_ILLEGAL";
             break;
     }
 
     if (stateAsText == NULL) {
         crm_err("State %d is unknown", state);
         stateAsText = "<UNKNOWN_STATE>";
     }
 
     return stateAsText;
 }
 
 const char *
 fsa_cause2string(enum crmd_fsa_cause cause)
 {
     const char *causeAsText = NULL;
 
     switch (cause) {
         case C_UNKNOWN:
             causeAsText = "C_UNKNOWN";
             break;
         case C_STARTUP:
             causeAsText = "C_STARTUP";
             break;
         case C_IPC_MESSAGE:
             causeAsText = "C_IPC_MESSAGE";
             break;
         case C_HA_MESSAGE:
             causeAsText = "C_HA_MESSAGE";
             break;
         case C_CCM_CALLBACK:
             causeAsText = "C_CCM_CALLBACK";
             break;
         case C_TIMER_POPPED:
             causeAsText = "C_TIMER_POPPED";
             break;
         case C_SHUTDOWN:
             causeAsText = "C_SHUTDOWN";
             break;
         case C_HEARTBEAT_FAILED:
             causeAsText = "C_HEARTBEAT_FAILED";
             break;
         case C_SUBSYSTEM_CONNECT:
             causeAsText = "C_SUBSYSTEM_CONNECT";
             break;
         case C_LRM_OP_CALLBACK:
             causeAsText = "C_LRM_OP_CALLBACK";
             break;
         case C_LRM_MONITOR_CALLBACK:
             causeAsText = "C_LRM_MONITOR_CALLBACK";
             break;
         case C_CRMD_STATUS_CALLBACK:
             causeAsText = "C_CRMD_STATUS_CALLBACK";
             break;
         case C_HA_DISCONNECT:
             causeAsText = "C_HA_DISCONNECT";
             break;
         case C_FSA_INTERNAL:
             causeAsText = "C_FSA_INTERNAL";
             break;
         case C_ILLEGAL:
             causeAsText = "C_ILLEGAL";
             break;
     }
 
     if (causeAsText == NULL) {
         crm_err("Cause %d is unknown", cause);
         causeAsText = "<UNKNOWN_CAUSE>";
     }
 
     return causeAsText;
 }
 
 const char *
 fsa_action2string(long long action)
 {
     const char *actionAsText = NULL;
 
     switch (action) {
 
         case A_NOTHING:
             actionAsText = "A_NOTHING";
             break;
         case A_ELECTION_START:
             actionAsText = "A_ELECTION_START";
             break;
         case A_DC_JOIN_FINAL:
             actionAsText = "A_DC_JOIN_FINAL";
             break;
         case A_READCONFIG:
             actionAsText = "A_READCONFIG";
             break;
         case O_RELEASE:
             actionAsText = "O_RELEASE";
             break;
         case A_STARTUP:
             actionAsText = "A_STARTUP";
             break;
         case A_STARTED:
             actionAsText = "A_STARTED";
             break;
         case A_HA_CONNECT:
             actionAsText = "A_HA_CONNECT";
             break;
         case A_HA_DISCONNECT:
             actionAsText = "A_HA_DISCONNECT";
             break;
         case A_LRM_CONNECT:
             actionAsText = "A_LRM_CONNECT";
             break;
         case A_LRM_EVENT:
             actionAsText = "A_LRM_EVENT";
             break;
         case A_LRM_INVOKE:
             actionAsText = "A_LRM_INVOKE";
             break;
         case A_LRM_DISCONNECT:
             actionAsText = "A_LRM_DISCONNECT";
             break;
         case O_LRM_RECONNECT:
             actionAsText = "O_LRM_RECONNECT";
             break;
         case A_CL_JOIN_QUERY:
             actionAsText = "A_CL_JOIN_QUERY";
             break;
         case A_DC_TIMER_STOP:
             actionAsText = "A_DC_TIMER_STOP";
             break;
         case A_DC_TIMER_START:
             actionAsText = "A_DC_TIMER_START";
             break;
         case A_INTEGRATE_TIMER_START:
             actionAsText = "A_INTEGRATE_TIMER_START";
             break;
         case A_INTEGRATE_TIMER_STOP:
             actionAsText = "A_INTEGRATE_TIMER_STOP";
             break;
         case A_FINALIZE_TIMER_START:
             actionAsText = "A_FINALIZE_TIMER_START";
             break;
         case A_FINALIZE_TIMER_STOP:
             actionAsText = "A_FINALIZE_TIMER_STOP";
             break;
         case A_ELECTION_COUNT:
             actionAsText = "A_ELECTION_COUNT";
             break;
         case A_ELECTION_VOTE:
             actionAsText = "A_ELECTION_VOTE";
             break;
         case A_ELECTION_CHECK:
             actionAsText = "A_ELECTION_CHECK";
             break;
         case A_CL_JOIN_ANNOUNCE:
             actionAsText = "A_CL_JOIN_ANNOUNCE";
             break;
         case A_CL_JOIN_REQUEST:
             actionAsText = "A_CL_JOIN_REQUEST";
             break;
         case A_CL_JOIN_RESULT:
             actionAsText = "A_CL_JOIN_RESULT";
             break;
         case A_DC_JOIN_OFFER_ALL:
             actionAsText = "A_DC_JOIN_OFFER_ALL";
             break;
         case A_DC_JOIN_OFFER_ONE:
             actionAsText = "A_DC_JOIN_OFFER_ONE";
             break;
         case A_DC_JOIN_PROCESS_REQ:
             actionAsText = "A_DC_JOIN_PROCESS_REQ";
             break;
         case A_DC_JOIN_PROCESS_ACK:
             actionAsText = "A_DC_JOIN_PROCESS_ACK";
             break;
         case A_DC_JOIN_FINALIZE:
             actionAsText = "A_DC_JOIN_FINALIZE";
             break;
         case A_MSG_PROCESS:
             actionAsText = "A_MSG_PROCESS";
             break;
         case A_MSG_ROUTE:
             actionAsText = "A_MSG_ROUTE";
             break;
         case A_RECOVER:
             actionAsText = "A_RECOVER";
             break;
         case A_DC_RELEASE:
             actionAsText = "A_DC_RELEASE";
             break;
         case A_DC_RELEASED:
             actionAsText = "A_DC_RELEASED";
             break;
         case A_DC_TAKEOVER:
             actionAsText = "A_DC_TAKEOVER";
             break;
         case A_SHUTDOWN:
             actionAsText = "A_SHUTDOWN";
             break;
         case A_SHUTDOWN_REQ:
             actionAsText = "A_SHUTDOWN_REQ";
             break;
         case A_STOP:
             actionAsText = "A_STOP  ";
             break;
         case A_EXIT_0:
             actionAsText = "A_EXIT_0";
             break;
         case A_EXIT_1:
             actionAsText = "A_EXIT_1";
             break;
         case A_CCM_CONNECT:
             actionAsText = "A_CCM_CONNECT";
             break;
         case A_CCM_DISCONNECT:
             actionAsText = "A_CCM_DISCONNECT";
             break;
         case O_CIB_RESTART:
             actionAsText = "O_CIB_RESTART";
             break;
         case A_CIB_START:
             actionAsText = "A_CIB_START";
             break;
         case A_CIB_STOP:
             actionAsText = "A_CIB_STOP";
             break;
         case A_TE_INVOKE:
             actionAsText = "A_TE_INVOKE";
             break;
         case O_TE_RESTART:
             actionAsText = "O_TE_RESTART";
             break;
         case A_TE_START:
             actionAsText = "A_TE_START";
             break;
         case A_TE_STOP:
             actionAsText = "A_TE_STOP";
             break;
         case A_TE_HALT:
             actionAsText = "A_TE_HALT";
             break;
         case A_TE_CANCEL:
             actionAsText = "A_TE_CANCEL";
             break;
         case A_PE_INVOKE:
             actionAsText = "A_PE_INVOKE";
             break;
         case O_PE_RESTART:
             actionAsText = "O_PE_RESTART";
             break;
         case A_PE_START:
             actionAsText = "A_PE_START";
             break;
         case A_PE_STOP:
             actionAsText = "A_PE_STOP";
             break;
         case A_NODE_BLOCK:
             actionAsText = "A_NODE_BLOCK";
             break;
         case A_UPDATE_NODESTATUS:
             actionAsText = "A_UPDATE_NODESTATUS";
             break;
         case A_LOG:
             actionAsText = "A_LOG   ";
             break;
         case A_ERROR:
             actionAsText = "A_ERROR ";
             break;
         case A_WARN:
             actionAsText = "A_WARN  ";
             break;
             /* Composite actions */
         case A_DC_TIMER_START | A_CL_JOIN_QUERY:
             actionAsText = "A_DC_TIMER_START|A_CL_JOIN_QUERY";
             break;
     }
 
     if (actionAsText == NULL) {
         crm_err("Action %.16llx is unknown", action);
         actionAsText = "<UNKNOWN_ACTION>";
     }
 
     return actionAsText;
 }
 
 void
 fsa_dump_inputs(int log_level, const char *text, long long input_register)
 {
     if (input_register == A_NOTHING) {
         return;
     }
     if (text == NULL) {
         text = "Input register contents:";
     }
 
     if (is_set(input_register, R_THE_DC)) {
         crm_trace("%s %.16llx (R_THE_DC)", text, R_THE_DC);
     }
     if (is_set(input_register, R_STARTING)) {
         crm_trace("%s %.16llx (R_STARTING)", text, R_STARTING);
     }
     if (is_set(input_register, R_SHUTDOWN)) {
         crm_trace("%s %.16llx (R_SHUTDOWN)", text, R_SHUTDOWN);
     }
     if (is_set(input_register, R_STAYDOWN)) {
         crm_trace("%s %.16llx (R_STAYDOWN)", text, R_STAYDOWN);
     }
     if (is_set(input_register, R_JOIN_OK)) {
         crm_trace("%s %.16llx (R_JOIN_OK)", text, R_JOIN_OK);
     }
     if (is_set(input_register, R_READ_CONFIG)) {
         crm_trace("%s %.16llx (R_READ_CONFIG)", text, R_READ_CONFIG);
     }
     if (is_set(input_register, R_INVOKE_PE)) {
         crm_trace("%s %.16llx (R_INVOKE_PE)", text, R_INVOKE_PE);
     }
     if (is_set(input_register, R_CIB_CONNECTED)) {
         crm_trace("%s %.16llx (R_CIB_CONNECTED)", text, R_CIB_CONNECTED);
     }
     if (is_set(input_register, R_PE_CONNECTED)) {
         crm_trace("%s %.16llx (R_PE_CONNECTED)", text, R_PE_CONNECTED);
     }
     if (is_set(input_register, R_TE_CONNECTED)) {
         crm_trace("%s %.16llx (R_TE_CONNECTED)", text, R_TE_CONNECTED);
     }
     if (is_set(input_register, R_LRM_CONNECTED)) {
         crm_trace("%s %.16llx (R_LRM_CONNECTED)", text, R_LRM_CONNECTED);
     }
     if (is_set(input_register, R_CIB_REQUIRED)) {
         crm_trace("%s %.16llx (R_CIB_REQUIRED)", text, R_CIB_REQUIRED);
     }
     if (is_set(input_register, R_PE_REQUIRED)) {
         crm_trace("%s %.16llx (R_PE_REQUIRED)", text, R_PE_REQUIRED);
     }
     if (is_set(input_register, R_TE_REQUIRED)) {
         crm_trace("%s %.16llx (R_TE_REQUIRED)", text, R_TE_REQUIRED);
     }
     if (is_set(input_register, R_REQ_PEND)) {
         crm_trace("%s %.16llx (R_REQ_PEND)", text, R_REQ_PEND);
     }
     if (is_set(input_register, R_PE_PEND)) {
         crm_trace("%s %.16llx (R_PE_PEND)", text, R_PE_PEND);
     }
     if (is_set(input_register, R_TE_PEND)) {
         crm_trace("%s %.16llx (R_TE_PEND)", text, R_TE_PEND);
     }
     if (is_set(input_register, R_RESP_PEND)) {
         crm_trace("%s %.16llx (R_RESP_PEND)", text, R_RESP_PEND);
     }
     if (is_set(input_register, R_CIB_DONE)) {
         crm_trace("%s %.16llx (R_CIB_DONE)", text, R_CIB_DONE);
     }
     if (is_set(input_register, R_HAVE_CIB)) {
         crm_trace("%s %.16llx (R_HAVE_CIB)", text, R_HAVE_CIB);
     }
     if (is_set(input_register, R_CIB_ASKED)) {
         crm_trace("%s %.16llx (R_CIB_ASKED)", text, R_CIB_ASKED);
     }
     if (is_set(input_register, R_MEMBERSHIP)) {
         crm_trace("%s %.16llx (R_MEMBERSHIP)", text, R_MEMBERSHIP);
     }
     if (is_set(input_register, R_PEER_DATA)) {
         crm_trace("%s %.16llx (R_PEER_DATA)", text, R_PEER_DATA);
     }
     if (is_set(input_register, R_IN_RECOVERY)) {
         crm_trace("%s %.16llx (R_IN_RECOVERY)", text, R_IN_RECOVERY);
     }
 }
 
 void
 fsa_dump_actions(long long action, const char *text)
 {
     if (is_set(action, A_READCONFIG)) {
         crm_trace("Action %.16llx (A_READCONFIG) %s", A_READCONFIG, text);
     }
     if (is_set(action, A_STARTUP)) {
         crm_trace("Action %.16llx (A_STARTUP) %s", A_STARTUP, text);
     }
     if (is_set(action, A_STARTED)) {
         crm_trace("Action %.16llx (A_STARTED) %s", A_STARTED, text);
     }
     if (is_set(action, A_HA_CONNECT)) {
         crm_trace("Action %.16llx (A_CONNECT) %s", A_HA_CONNECT, text);
     }
     if (is_set(action, A_HA_DISCONNECT)) {
         crm_trace("Action %.16llx (A_DISCONNECT) %s", A_HA_DISCONNECT, text);
     }
     if (is_set(action, A_LRM_CONNECT)) {
         crm_trace("Action %.16llx (A_LRM_CONNECT) %s", A_LRM_CONNECT, text);
     }
     if (is_set(action, A_LRM_EVENT)) {
         crm_trace("Action %.16llx (A_LRM_EVENT) %s", A_LRM_EVENT, text);
     }
     if (is_set(action, A_LRM_INVOKE)) {
         crm_trace("Action %.16llx (A_LRM_INVOKE) %s", A_LRM_INVOKE, text);
     }
     if (is_set(action, A_LRM_DISCONNECT)) {
         crm_trace("Action %.16llx (A_LRM_DISCONNECT) %s", A_LRM_DISCONNECT, text);
     }
     if (is_set(action, A_DC_TIMER_STOP)) {
         crm_trace("Action %.16llx (A_DC_TIMER_STOP) %s", A_DC_TIMER_STOP, text);
     }
     if (is_set(action, A_DC_TIMER_START)) {
         crm_trace("Action %.16llx (A_DC_TIMER_START) %s", A_DC_TIMER_START, text);
     }
     if (is_set(action, A_INTEGRATE_TIMER_START)) {
         crm_trace("Action %.16llx (A_INTEGRATE_TIMER_START) %s", A_INTEGRATE_TIMER_START, text);
     }
     if (is_set(action, A_INTEGRATE_TIMER_STOP)) {
         crm_trace("Action %.16llx (A_INTEGRATE_TIMER_STOP) %s", A_INTEGRATE_TIMER_STOP, text);
     }
     if (is_set(action, A_FINALIZE_TIMER_START)) {
         crm_trace("Action %.16llx (A_FINALIZE_TIMER_START) %s", A_FINALIZE_TIMER_START, text);
     }
     if (is_set(action, A_FINALIZE_TIMER_STOP)) {
         crm_trace("Action %.16llx (A_FINALIZE_TIMER_STOP) %s", A_FINALIZE_TIMER_STOP, text);
     }
     if (is_set(action, A_ELECTION_COUNT)) {
         crm_trace("Action %.16llx (A_ELECTION_COUNT) %s", A_ELECTION_COUNT, text);
     }
     if (is_set(action, A_ELECTION_VOTE)) {
         crm_trace("Action %.16llx (A_ELECTION_VOTE) %s", A_ELECTION_VOTE, text);
     }
     if (is_set(action, A_ELECTION_CHECK)) {
         crm_trace("Action %.16llx (A_ELECTION_CHECK) %s", A_ELECTION_CHECK, text);
     }
     if (is_set(action, A_CL_JOIN_ANNOUNCE)) {
         crm_trace("Action %.16llx (A_CL_JOIN_ANNOUNCE) %s", A_CL_JOIN_ANNOUNCE, text);
     }
     if (is_set(action, A_CL_JOIN_REQUEST)) {
         crm_trace("Action %.16llx (A_CL_JOIN_REQUEST) %s", A_CL_JOIN_REQUEST, text);
     }
     if (is_set(action, A_CL_JOIN_RESULT)) {
         crm_trace("Action %.16llx (A_CL_JOIN_RESULT) %s", A_CL_JOIN_RESULT, text);
     }
     if (is_set(action, A_DC_JOIN_OFFER_ALL)) {
         crm_trace("Action %.16llx (A_DC_JOIN_OFFER_ALL) %s", A_DC_JOIN_OFFER_ALL, text);
     }
     if (is_set(action, A_DC_JOIN_OFFER_ONE)) {
         crm_trace("Action %.16llx (A_DC_JOIN_OFFER_ONE) %s", A_DC_JOIN_OFFER_ONE, text);
     }
     if (is_set(action, A_DC_JOIN_PROCESS_REQ)) {
         crm_trace("Action %.16llx (A_DC_JOIN_PROCESS_REQ) %s", A_DC_JOIN_PROCESS_REQ, text);
     }
     if (is_set(action, A_DC_JOIN_PROCESS_ACK)) {
         crm_trace("Action %.16llx (A_DC_JOIN_PROCESS_ACK) %s", A_DC_JOIN_PROCESS_ACK, text);
     }
     if (is_set(action, A_DC_JOIN_FINALIZE)) {
         crm_trace("Action %.16llx (A_DC_JOIN_FINALIZE) %s", A_DC_JOIN_FINALIZE, text);
     }
     if (is_set(action, A_MSG_PROCESS)) {
         crm_trace("Action %.16llx (A_MSG_PROCESS) %s", A_MSG_PROCESS, text);
     }
     if (is_set(action, A_MSG_ROUTE)) {
         crm_trace("Action %.16llx (A_MSG_ROUTE) %s", A_MSG_ROUTE, text);
     }
     if (is_set(action, A_RECOVER)) {
         crm_trace("Action %.16llx (A_RECOVER) %s", A_RECOVER, text);
     }
     if (is_set(action, A_DC_RELEASE)) {
         crm_trace("Action %.16llx (A_DC_RELEASE) %s", A_DC_RELEASE, text);
     }
     if (is_set(action, A_DC_RELEASED)) {
         crm_trace("Action %.16llx (A_DC_RELEASED) %s", A_DC_RELEASED, text);
     }
     if (is_set(action, A_DC_TAKEOVER)) {
         crm_trace("Action %.16llx (A_DC_TAKEOVER) %s", A_DC_TAKEOVER, text);
     }
     if (is_set(action, A_SHUTDOWN)) {
         crm_trace("Action %.16llx (A_SHUTDOWN) %s", A_SHUTDOWN, text);
     }
     if (is_set(action, A_SHUTDOWN_REQ)) {
         crm_trace("Action %.16llx (A_SHUTDOWN_REQ) %s", A_SHUTDOWN_REQ, text);
     }
     if (is_set(action, A_STOP)) {
         crm_trace("Action %.16llx (A_STOP  ) %s", A_STOP, text);
     }
     if (is_set(action, A_EXIT_0)) {
         crm_trace("Action %.16llx (A_EXIT_0) %s", A_EXIT_0, text);
     }
     if (is_set(action, A_EXIT_1)) {
         crm_trace("Action %.16llx (A_EXIT_1) %s", A_EXIT_1, text);
     }
     if (is_set(action, A_CCM_CONNECT)) {
         crm_trace("Action %.16llx (A_CCM_CONNECT) %s", A_CCM_CONNECT, text);
     }
     if (is_set(action, A_CCM_DISCONNECT)) {
         crm_trace("Action %.16llx (A_CCM_DISCONNECT) %s", A_CCM_DISCONNECT, text);
     }
     if (is_set(action, A_CIB_START)) {
         crm_trace("Action %.16llx (A_CIB_START) %s", A_CIB_START, text);
     }
     if (is_set(action, A_CIB_STOP)) {
         crm_trace("Action %.16llx (A_CIB_STOP) %s", A_CIB_STOP, text);
     }
     if (is_set(action, A_TE_INVOKE)) {
         crm_trace("Action %.16llx (A_TE_INVOKE) %s", A_TE_INVOKE, text);
     }
     if (is_set(action, A_TE_START)) {
         crm_trace("Action %.16llx (A_TE_START) %s", A_TE_START, text);
     }
     if (is_set(action, A_TE_STOP)) {
         crm_trace("Action %.16llx (A_TE_STOP) %s", A_TE_STOP, text);
     }
     if (is_set(action, A_TE_CANCEL)) {
         crm_trace("Action %.16llx (A_TE_CANCEL) %s", A_TE_CANCEL, text);
     }
     if (is_set(action, A_PE_INVOKE)) {
         crm_trace("Action %.16llx (A_PE_INVOKE) %s", A_PE_INVOKE, text);
     }
     if (is_set(action, A_PE_START)) {
         crm_trace("Action %.16llx (A_PE_START) %s", A_PE_START, text);
     }
     if (is_set(action, A_PE_STOP)) {
         crm_trace("Action %.16llx (A_PE_STOP) %s", A_PE_STOP, text);
     }
     if (is_set(action, A_NODE_BLOCK)) {
         crm_trace("Action %.16llx (A_NODE_BLOCK) %s", A_NODE_BLOCK, text);
     }
     if (is_set(action, A_UPDATE_NODESTATUS)) {
         crm_trace("Action %.16llx (A_UPDATE_NODESTATUS) %s", A_UPDATE_NODESTATUS, text);
     }
     if (is_set(action, A_LOG)) {
         crm_trace("Action %.16llx (A_LOG   ) %s", A_LOG, text);
     }
     if (is_set(action, A_ERROR)) {
         crm_trace("Action %.16llx (A_ERROR ) %s", A_ERROR, text);
     }
     if (is_set(action, A_WARN)) {
         crm_trace("Action %.16llx (A_WARN  ) %s", A_WARN, text);
     }
 }
 
 gboolean
 update_dc(xmlNode * msg)
 {
     char *last_dc = fsa_our_dc;
     const char *dc_version = NULL;
     const char *welcome_from = NULL;
 
     if (msg != NULL) {
         gboolean invalid = FALSE;
 
         dc_version = crm_element_value(msg, F_CRM_VERSION);
         welcome_from = crm_element_value(msg, F_CRM_HOST_FROM);
 
         CRM_CHECK(dc_version != NULL, return FALSE);
         CRM_CHECK(welcome_from != NULL, return FALSE);
 
         if (AM_I_DC && safe_str_neq(welcome_from, fsa_our_uname)) {
             invalid = TRUE;
 
         } else if (fsa_our_dc && safe_str_neq(welcome_from, fsa_our_dc)) {
             invalid = TRUE;
         }
 
         if (invalid) {
             CRM_CHECK(fsa_our_dc != NULL, crm_err("We have no DC"));
             if (AM_I_DC) {
                 crm_err("Not updating DC to %s (%s): we are also a DC", welcome_from, dc_version);
             } else {
                 crm_warn("New DC %s is not %s", welcome_from, fsa_our_dc);
             }
 
             register_fsa_action(A_CL_JOIN_QUERY | A_DC_TIMER_START);
             return FALSE;
         }
     }
 
     free(fsa_our_dc_version);
     fsa_our_dc_version = NULL;
 
     fsa_our_dc = NULL;          /* Free'd as last_dc */
 
     if (welcome_from != NULL) {
         fsa_our_dc = strdup(welcome_from);
     }
     if (dc_version != NULL) {
         fsa_our_dc_version = strdup(dc_version);
     }
 
     if (safe_str_eq(fsa_our_dc, last_dc)) {
         /* do nothing */
 
     } else if (fsa_our_dc != NULL) {
         crm_node_t *dc_node = crm_get_peer(0, fsa_our_dc);
 
         crm_info("Set DC to %s (%s)", crm_str(fsa_our_dc), crm_str(fsa_our_dc_version));
         crm_update_peer_expected(__FUNCTION__, dc_node, CRMD_JOINSTATE_MEMBER);
 
     } else if (last_dc != NULL) {
         crm_info("Unset DC. Was %s", crm_str(last_dc));
     }
 
     free(last_dc);
     return TRUE;
 }
 
 #define STATUS_PATH_MAX 512
 static void
 erase_xpath_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     char *xpath = user_data;
 
     do_crm_log_unlikely(rc == 0 ? LOG_DEBUG : LOG_NOTICE,
                         "Deletion of \"%s\": %s (rc=%d)", xpath, pcmk_strerror(rc), rc);
 }
 
 void
 erase_status_tag(const char *uname, const char *tag, int options)
 {
     int rc = pcmk_ok;
     char xpath[STATUS_PATH_MAX];
     int cib_opts = cib_quorum_override | cib_xpath | options;
 
     if (fsa_cib_conn && uname) {
         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", uname, tag);
         crm_info("Deleting xpath: %s", xpath);
         rc = fsa_cib_conn->cmds->delete(fsa_cib_conn, xpath, NULL, cib_opts);
         fsa_register_cib_callback(rc, FALSE, strdup(xpath), erase_xpath_callback);
     }
 }
 
-static void
-crm_set_join_state(const char *uname, const char *start_state)
-{
-    if (safe_str_eq(start_state, "standby")) {
-        crm_notice("Forcing node %s to join in %s state per configured environment", uname, start_state);
-        update_attrd(uname, XML_CIB_ATTR_STANDBY, "on", NULL, FALSE);
-    } else if (safe_str_eq(start_state, "online")) {
-        crm_notice("Forcing node %s to join in %s state per configured environment", uname, start_state);
-        update_attrd(uname, XML_CIB_ATTR_STANDBY, "off", NULL, FALSE);
-    } else if (safe_str_eq(start_state, "default")) {
-        crm_debug("Not forcing a starting state on node %s", uname);
-    } else {
-        crm_warn("Unrecognized start state '%s', using 'default' (%s)", start_state, uname);
-    }
-}
-
-static void
-init_attrs_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
-{
-    char **data = (char**)user_data;
-    char *uname = data[0];
-    char *start_state = data[1];
-    char *xpath = data[2];
-
-    do_crm_log_unlikely(rc == 0 ? LOG_DEBUG : LOG_NOTICE,
-                        "Erasing of \"%s\": %s (rc=%d)", xpath, pcmk_strerror(rc), rc);
-    crm_set_join_state(uname, start_state);
-    free(data[0]);
-    free(data[1]);
-    free(data[2]);
-}
-
-void
-init_transient_attrs(const char *uname, const char *start_state, int options)
-{
-    if (fsa_cib_conn && uname) {
-        int rc;
-        char *xpath = crm_strdup_printf("//node_state[@uname='%s']/%s", uname, XML_TAG_TRANSIENT_NODEATTRS);
-        int cib_opts = cib_quorum_override | cib_xpath | options;
-        const char **data = malloc(sizeof(char*)*3);
-        data[0] = strdup(uname);
-        data[1] = strdup(start_state);
-        data[2] = strdup(xpath);
-
-        crm_info("Erasing transient attributes for %s", uname);
-        rc = fsa_cib_conn->cmds->delete(fsa_cib_conn, xpath, NULL, cib_opts);
-        fsa_register_cib_callback(rc, FALSE, data, init_attrs_callback);
-    }
-}
-
 void crmd_peer_down(crm_node_t *peer, bool full) 
 {
     if(full && peer->state == NULL) {
         crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_LOST, 0);
         crm_update_peer_proc(__FUNCTION__, peer, crm_proc_none, NULL);
     }
     crm_update_peer_join(__FUNCTION__, peer, crm_join_none);
     crm_update_peer_expected(__FUNCTION__, peer, CRMD_JOINSTATE_DOWN);
 }