diff --git a/daemons/controld/controld_callbacks.c b/daemons/controld/controld_callbacks.c
index d07593e0ac..0cefc3b2b9 100644
--- a/daemons/controld/controld_callbacks.c
+++ b/daemons/controld/controld_callbacks.c
@@ -1,343 +1,343 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <string.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/cluster.h>
 #include <crm/cib.h>
 
 #include <pacemaker-controld.h>
 
 /* From join_dc... */
 extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
 
 void
 crmd_ha_msg_filter(xmlNode * msg)
 {
     if (AM_I_DC) {
         const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
 
         if (pcmk__str_eq(sys_from, CRM_SYSTEM_DC, pcmk__str_casei)) {
             const char *from = crm_element_value(msg, F_ORIG);
 
             if (!pcmk__str_eq(from, fsa_our_uname, pcmk__str_casei)) {
                 int level = LOG_INFO;
                 const char *op = crm_element_value(msg, F_CRM_TASK);
 
                 /* make sure the election happens NOW */
                 if (fsa_state != S_ELECTION) {
                     ha_msg_input_t new_input;
 
                     level = LOG_WARNING;
                     new_input.msg = msg;
                     register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, &new_input,
                                            __FUNCTION__);
                 }
 
                 do_crm_log(level, "Another DC detected: %s (op=%s)", from, op);
                 goto done;
             }
         }
 
     } else {
         const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
 
         if (pcmk__str_eq(sys_to, CRM_SYSTEM_DC, pcmk__str_casei)) {
             return;
         }
     }
 
     /* crm_log_xml_trace("HA[inbound]", msg); */
     route_message(C_HA_MESSAGE, msg);
 
   done:
-    trigger_fsa(fsa_source);
+    trigger_fsa();
 }
 
 /*!
  * \internal
  * \brief Check whether a node is online
  *
  * \param[in] node  Node to check
  *
  * \retval -1 if completely dead
  * \retval  0 if partially alive
  * \retval  1 if completely alive
  */
 static int
 node_alive(const crm_node_t *node)
 {
     if (is_set(node->flags, crm_remote_node)) {
         // Pacemaker Remote nodes can't be partially alive
         return pcmk__str_eq(node->state, CRM_NODE_MEMBER, pcmk__str_casei) ? 1: -1;
 
     } else if (crm_is_peer_active(node)) {
         // Completely up cluster node: both cluster member and peer
         return 1;
 
     } else if (is_not_set(node->processes, crm_get_cluster_proc())
                && !pcmk__str_eq(node->state, CRM_NODE_MEMBER, pcmk__str_casei)) {
         // Completely down cluster node: neither cluster member nor peer
         return -1;
     }
 
     // Partially up cluster node: only cluster member or only peer
     return 0;
 }
 
 #define state_text(state) ((state)? (const char *)(state) : "in unknown state")
 
 void
 peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
 {
     uint32_t old = 0;
     bool appeared = FALSE;
     bool is_remote = is_set(node->flags, crm_remote_node);
 
     /* The controller waits to receive some information from the membership
      * layer before declaring itself operational. If this is being called for a
      * cluster node, indicate that we have it.
      */
     if (!is_remote) {
         controld_set_fsa_input_flags(R_PEER_DATA);
     }
 
     if (node->uname == NULL) {
         return;
     }
 
     switch (type) {
         case crm_status_uname:
             /* If we've never seen the node, then it also won't be in the status section */
             crm_info("%s node %s is now %s",
                      (is_remote? "Remote" : "Cluster"),
                      node->uname, state_text(node->state));
             return;
 
         case crm_status_nstate:
             /* This callback should not be called unless the state actually
              * changed, but here's a failsafe just in case.
              */
             CRM_CHECK(!pcmk__str_eq(data, node->state, pcmk__str_casei),
                       return);
 
             crm_info("%s node %s is now %s (was %s)",
                      (is_remote? "Remote" : "Cluster"),
                      node->uname, state_text(node->state), state_text(data));
 
             if (pcmk__str_eq(CRM_NODE_MEMBER, node->state, pcmk__str_casei)) {
                 appeared = TRUE;
                 if (!is_remote) {
                     remove_stonith_cleanup(node->uname);
                 }
             } else {
                 controld_remove_voter(node->uname);
             }
 
             crmd_alert_node_event(node);
             break;
 
         case crm_status_processes:
             CRM_CHECK(data != NULL, return);
             old = *(const uint32_t *)data;
             appeared = is_set(node->processes, crm_get_cluster_proc());
 
             crm_info("Node %s is %s a peer " CRM_XS " DC=%s old=0x%07x new=0x%07x",
                      node->uname, (appeared? "now" : "no longer"),
                      (AM_I_DC? "true" : (fsa_our_dc? fsa_our_dc : "<none>")),
                      old, node->processes);
 
             if (is_not_set((node->processes ^ old), crm_get_cluster_proc())) {
                 /* Peer status did not change. This should not be possible,
                  * since we don't track process flags other than peer status.
                  */
                 crm_trace("Process flag 0x%7x did not change from 0x%7x to 0x%7x",
                           crm_get_cluster_proc(), old, node->processes);
                 return;
 
             }
 
             if (!appeared) {
                 controld_remove_voter(node->uname);
             }
 
             if (is_not_set(fsa_input_register, R_CIB_CONNECTED)) {
                 crm_trace("Ignoring peer status change because not connected to CIB");
                 return;
 
             } else if (fsa_state == S_STOPPING) {
                 crm_trace("Ignoring peer status change because stopping");
                 return;
             }
 
             if (pcmk__str_eq(node->uname, fsa_our_uname, pcmk__str_casei) && !appeared) {
                 /* Did we get evicted? */
                 crm_notice("Our peer connection failed");
                 register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ERROR, NULL);
 
             } else if (pcmk__str_eq(node->uname, fsa_our_dc, pcmk__str_casei) && crm_is_peer_active(node) == FALSE) {
                 /* Did the DC leave us? */
                 crm_notice("Our peer on the DC (%s) is dead", fsa_our_dc);
                 register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL);
 
                 /* @COMPAT DC < 1.1.13: If a DC shuts down normally, we don't
                  * want to fence it. Newer DCs will send their shutdown request
                  * to all peers, who will update the DC's expected state to
                  * down, thus avoiding fencing. We can safely erase the DC's
                  * transient attributes when it leaves in that case. However,
                  * the only way to avoid fencing older DCs is to leave the
                  * transient attributes intact until it rejoins.
                  */
                 if (compare_version(fsa_our_dc_version, "3.0.9") > 0) {
                     controld_delete_node_state(node->uname,
                                                controld_section_attrs,
                                                cib_scope_local);
                 }
 
             } else if (AM_I_DC || (fsa_our_dc == NULL)) {
                 /* This only needs to be done once, so normally the DC should do
                  * it. However if there is no DC, every node must do it, since
                  * there is no other way to ensure some one node does it.
                  */
                 if (appeared) {
                     te_trigger_stonith_history_sync(FALSE);
                 } else {
                     controld_delete_node_state(node->uname,
                                                controld_section_attrs,
                                                cib_scope_local);
                 }
             }
             break;
     }
 
     if (AM_I_DC) {
         xmlNode *update = NULL;
         int flags = node_update_peer;
         int alive = node_alive(node);
         crm_action_t *down = match_down_event(node->uuid);
 
         crm_trace("Alive=%d, appeared=%d, down=%d",
                   alive, appeared, (down? down->id : -1));
 
         if (appeared && (alive > 0) && !is_remote) {
             register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
         }
 
         if (down) {
             const char *task = crm_element_value(down->xml, XML_LRM_ATTR_TASK);
 
             if (pcmk__str_eq(task, CRM_OP_FENCE, pcmk__str_casei)) {
 
                 /* tengine_stonith_callback() confirms fence actions */
                 crm_trace("Updating CIB %s fencer reported fencing of %s complete",
                           (down->confirmed? "after" : "before"), node->uname);
 
             } else if (!appeared && pcmk__str_eq(task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
 
                 // Shutdown actions are immediately confirmed (i.e. no_wait)
                 if (!is_remote) {
                     flags |= node_update_join | node_update_expected;
                     crmd_peer_down(node, FALSE);
                     check_join_state(fsa_state, __FUNCTION__);
                 }
                 if (alive >= 0) {
                     crm_info("%s of peer %s is in progress " CRM_XS " action=%d",
                              task, node->uname, down->id);
                 } else {
                     crm_notice("%s of peer %s is complete " CRM_XS " action=%d",
                                task, node->uname, down->id);
                     update_graph(transition_graph, down);
                     trigger_graph();
                 }
 
             } else {
                 crm_trace("Node %s is %s, was expected to %s (op %d)",
                           node->uname,
                           ((alive > 0)? "alive" :
                            ((alive < 0)? "dead" : "partially alive")),
                           task, down->id);
             }
 
         } else if (appeared == FALSE) {
             crm_warn("Stonith/shutdown of node %s was not expected",
                      node->uname);
             if (!is_remote) {
                 crm_update_peer_join(__FUNCTION__, node, crm_join_none);
                 check_join_state(fsa_state, __FUNCTION__);
             }
             abort_transition(INFINITY, tg_restart, "Node failure", NULL);
             fail_incompletable_actions(transition_graph, node->uuid);
 
         } else {
             crm_trace("Node %s came up, was not expected to be down",
                       node->uname);
         }
 
         if (is_remote) {
             /* A pacemaker_remote node won't have its cluster status updated
              * in the CIB by membership-layer callbacks, so do it here.
              */
             flags |= node_update_cluster;
 
             /* Trigger resource placement on newly integrated nodes */
             if (appeared) {
                 abort_transition(INFINITY, tg_restart,
                                  "pacemaker_remote node integrated", NULL);
             }
         }
 
         /* Update the CIB node state */
         update = create_node_state_update(node, flags, NULL, __FUNCTION__);
         if (update == NULL) {
             crm_debug("Node state update not yet possible for %s", node->uname);
         } else {
             fsa_cib_anon_update(XML_CIB_TAG_STATUS, update);
         }
         free_xml(update);
     }
 
-    trigger_fsa(fsa_source);
+    trigger_fsa();
 }
 
 void
 crmd_cib_connection_destroy(gpointer user_data)
 {
     CRM_CHECK(user_data == fsa_cib_conn,;);
 
     crm_trace("Invoked");
-    trigger_fsa(fsa_source);
+    trigger_fsa();
     fsa_cib_conn->state = cib_disconnected;
 
     if (is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) {
         crm_info("Connection to the CIB manager terminated");
         return;
     }
 
     // @TODO This should trigger a reconnect, not a shutdown
     crm_crit("Lost connection to the CIB manager, shutting down");
     register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
     controld_clear_fsa_input_flags(R_CIB_CONNECTED);
 
     return;
 }
 
 gboolean
 crm_fsa_trigger(gpointer user_data)
 {
     crm_trace("Invoked (queue len: %d)", g_list_length(fsa_message_queue));
     s_crmd_fsa(C_FSA_INTERNAL);
     crm_trace("Exited  (queue len: %d)", g_list_length(fsa_message_queue));
     return TRUE;
 }
diff --git a/daemons/controld/controld_control.c b/daemons/controld/controld_control.c
index c9c142af45..3331801f32 100644
--- a/daemons/controld/controld_control.c
+++ b/daemons/controld/controld_control.c
@@ -1,818 +1,818 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/pengine/rules.h>
 #include <crm/cluster/internal.h>
 #include <crm/cluster/election.h>
 #include <crm/common/ipc_internal.h>
 
 #include <pacemaker-controld.h>
 
 qb_ipcs_service_t *ipcs = NULL;
 
 #if SUPPORT_COROSYNC
 extern gboolean crm_connect_corosync(crm_cluster_t * cluster);
 #endif
 
 void crm_shutdown(int nsig);
 gboolean crm_read_options(gpointer user_data);
 
 gboolean fsa_has_quorum = FALSE;
 crm_trigger_t *fsa_source = NULL;
 crm_trigger_t *config_read = NULL;
 bool no_quorum_suicide_escalation = FALSE;
 bool controld_shutdown_lock_enabled = false;
 
 /*	 A_HA_CONNECT	*/
 void
 do_ha_control(long long action,
               enum crmd_fsa_cause cause,
               enum crmd_fsa_state cur_state,
               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     gboolean registered = FALSE;
     static crm_cluster_t *cluster = NULL;
 
     if (cluster == NULL) {
         cluster = calloc(1, sizeof(crm_cluster_t));
     }
 
     if (action & A_HA_DISCONNECT) {
         crm_cluster_disconnect(cluster);
         crm_info("Disconnected from the cluster");
 
         controld_set_fsa_input_flags(R_HA_DISCONNECTED);
     }
 
     if (action & A_HA_CONNECT) {
         crm_set_status_callback(&peer_update_callback);
         crm_set_autoreap(FALSE);
 
         if (is_corosync_cluster()) {
 #if SUPPORT_COROSYNC
             registered = crm_connect_corosync(cluster);
 #endif
         }
 
         if (registered == TRUE) {
             controld_election_init(cluster->uname);
             fsa_our_uname = cluster->uname;
             fsa_our_uuid = cluster->uuid;
             if(cluster->uuid == NULL) {
                 crm_err("Could not obtain local uuid");
                 registered = FALSE;
             }
         }
 
         if (registered == FALSE) {
             controld_set_fsa_input_flags(R_HA_DISCONNECTED);
             register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
             return;
         }
 
         populate_cib_nodes(node_update_none, __FUNCTION__);
         controld_clear_fsa_input_flags(R_HA_DISCONNECTED);
         crm_info("Connected to the cluster");
     }
 
     if (action & ~(A_HA_CONNECT | A_HA_DISCONNECT)) {
         crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__);
     }
 }
 
 /*	 A_SHUTDOWN	*/
 void
 do_shutdown(long long action,
             enum crmd_fsa_cause cause,
             enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     /* just in case */
     controld_set_fsa_input_flags(R_SHUTDOWN);
     controld_disconnect_fencer(FALSE);
 }
 
 /*	 A_SHUTDOWN_REQ	*/
 void
 do_shutdown_req(long long action,
                 enum crmd_fsa_cause cause,
                 enum crmd_fsa_state cur_state,
                 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     xmlNode *msg = NULL;
 
     controld_set_fsa_input_flags(R_SHUTDOWN);
     //controld_set_fsa_input_flags(R_STAYDOWN);
     crm_info("Sending shutdown request to all peers (DC is %s)",
              (fsa_our_dc? fsa_our_dc : "not set"));
     msg = create_request(CRM_OP_SHUTDOWN_REQ, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
 
     if (send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) {
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
     free_xml(msg);
 }
 
 extern char *max_generation_from;
 extern xmlNode *max_generation_xml;
 extern GHashTable *resource_history;
 extern GHashTable *voted;
 
 void
 crmd_fast_exit(crm_exit_t exit_code)
 {
     if (is_set(fsa_input_register, R_STAYDOWN)) {
         crm_warn("Inhibiting respawn "CRM_XS" remapping exit code %d to %d",
                  exit_code, CRM_EX_FATAL);
         exit_code = CRM_EX_FATAL;
 
     } else if ((exit_code == CRM_EX_OK)
                && is_set(fsa_input_register, R_IN_RECOVERY)) {
         crm_err("Could not recover from internal error");
         exit_code = CRM_EX_ERROR;
     }
     crm_exit(exit_code);
 }
 
 crm_exit_t
 crmd_exit(crm_exit_t exit_code)
 {
     GListPtr gIter = NULL;
     GMainLoop *mloop = crmd_mainloop;
 
     static bool in_progress = FALSE;
 
     if (in_progress && (exit_code == CRM_EX_OK)) {
         crm_debug("Exit is already in progress");
         return exit_code;
 
     } else if(in_progress) {
         crm_notice("Error during shutdown process, exiting now with status %d (%s)",
                    exit_code, crm_exit_str(exit_code));
         crm_write_blackbox(SIGTRAP, NULL);
         crmd_fast_exit(exit_code);
     }
 
     in_progress = TRUE;
     crm_trace("Preparing to exit with status %d (%s)",
               exit_code, crm_exit_str(exit_code));
 
     /* Suppress secondary errors resulting from us disconnecting everything */
     controld_set_fsa_input_flags(R_HA_DISCONNECTED);
 
 /* Close all IPC servers and clients to ensure any and all shared memory files are cleaned up */
 
     if(ipcs) {
         crm_trace("Closing IPC server");
         mainloop_del_ipc_server(ipcs);
         ipcs = NULL;
     }
 
     controld_close_attrd_ipc();
     pe_subsystem_free();
     controld_disconnect_fencer(TRUE);
 
     if ((exit_code == CRM_EX_OK) && (crmd_mainloop == NULL)) {
         crm_debug("No mainloop detected");
         exit_code = CRM_EX_ERROR;
     }
 
     /* On an error, just get out.
      *
      * Otherwise, make the effort to have mainloop exit gracefully so
      * that it (mostly) cleans up after itself and valgrind has less
      * to report on - allowing real errors stand out
      */
     if (exit_code != CRM_EX_OK) {
         crm_notice("Forcing immediate exit with status %d (%s)",
                    exit_code, crm_exit_str(exit_code));
         crm_write_blackbox(SIGTRAP, NULL);
         crmd_fast_exit(exit_code);
     }
 
 /* Clean up as much memory as possible for valgrind */
 
     for (gIter = fsa_message_queue; gIter != NULL; gIter = gIter->next) {
         fsa_data_t *fsa_data = gIter->data;
 
         crm_info("Dropping %s: [ state=%s cause=%s origin=%s ]",
                  fsa_input2string(fsa_data->fsa_input),
                  fsa_state2string(fsa_state),
                  fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
         delete_fsa_input(fsa_data);
     }
 
     controld_clear_fsa_input_flags(R_MEMBERSHIP);
     g_list_free(fsa_message_queue); fsa_message_queue = NULL;
 
     metadata_cache_fini();
     controld_election_fini();
 
     /* Tear down the CIB manager connection, but don't free it yet -- it could
      * be used when we drain the mainloop later.
      */
     cib_free_callbacks(fsa_cib_conn);
     fsa_cib_conn->cmds->signoff(fsa_cib_conn);
 
     verify_stopped(fsa_state, LOG_WARNING);
     controld_clear_fsa_input_flags(R_LRM_CONNECTED);
     lrm_state_destroy_all();
 
     /* This basically will not work, since mainloop has a reference to it */
     mainloop_destroy_trigger(fsa_source); fsa_source = NULL;
 
     mainloop_destroy_trigger(config_read); config_read = NULL;
     mainloop_destroy_trigger(transition_trigger); transition_trigger = NULL;
 
     pcmk__client_cleanup();
     crm_peer_destroy();
 
     controld_free_fsa_timers();
     te_cleanup_stonith_history_sync(NULL, TRUE);
     controld_free_sched_timer();
 
     free(fsa_our_dc_version); fsa_our_dc_version = NULL;
     free(fsa_our_uname); fsa_our_uname = NULL;
     free(fsa_our_uuid); fsa_our_uuid = NULL;
     free(fsa_our_dc); fsa_our_dc = NULL;
 
     free(fsa_cluster_name); fsa_cluster_name = NULL;
 
     free(te_uuid); te_uuid = NULL;
     free(failed_stop_offset); failed_stop_offset = NULL;
     free(failed_start_offset); failed_start_offset = NULL;
 
     free(max_generation_from); max_generation_from = NULL;
     free_xml(max_generation_xml); max_generation_xml = NULL;
 
     mainloop_destroy_signal(SIGPIPE);
     mainloop_destroy_signal(SIGUSR1);
     mainloop_destroy_signal(SIGTERM);
     mainloop_destroy_signal(SIGTRAP);
     /* leave SIGCHLD engaged as we might still want to drain some service-actions */
 
     if (mloop) {
         GMainContext *ctx = g_main_loop_get_context(crmd_mainloop);
 
         /* Don't re-enter this block */
         crmd_mainloop = NULL;
 
         /* no signals on final draining anymore */
         mainloop_destroy_signal(SIGCHLD);
 
         crm_trace("Draining mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
 
         {
             int lpc = 0;
 
             while((g_main_context_pending(ctx) && lpc < 10)) {
                 lpc++;
                 crm_trace("Iteration %d", lpc);
                 g_main_context_dispatch(ctx);
             }
         }
 
         crm_trace("Closing mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
         g_main_loop_quit(mloop);
 
         /* Won't do anything yet, since we're inside it now */
         g_main_loop_unref(mloop);
     } else {
         mainloop_destroy_signal(SIGCHLD);
     }
 
     cib_delete(fsa_cib_conn);
     fsa_cib_conn = NULL;
 
     throttle_fini();
 
     /* Graceful */
     crm_trace("Done preparing for exit with status %d (%s)",
               exit_code, crm_exit_str(exit_code));
     return exit_code;
 }
 
 /*	 A_EXIT_0, A_EXIT_1	*/
 void
 do_exit(long long action,
         enum crmd_fsa_cause cause,
         enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     crm_exit_t exit_code = CRM_EX_OK;
     int log_level = LOG_INFO;
     const char *exit_type = "gracefully";
 
     if (action & A_EXIT_1) {
         log_level = LOG_ERR;
         exit_type = "forcefully";
         exit_code = CRM_EX_ERROR;
     }
 
     verify_stopped(cur_state, LOG_ERR);
     do_crm_log(log_level, "Performing %s - %s exiting the controller",
                fsa_action2string(action), exit_type);
 
     crm_info("[%s] stopped (%d)", crm_system_name, exit_code);
     crmd_exit(exit_code);
 }
 
 static void sigpipe_ignore(int nsig) { return; }
 
 /*	 A_STARTUP	*/
 void
 do_startup(long long action,
            enum crmd_fsa_cause cause,
            enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     crm_debug("Registering Signal Handlers");
     mainloop_add_signal(SIGTERM, crm_shutdown);
     mainloop_add_signal(SIGPIPE, sigpipe_ignore);
 
     fsa_source = mainloop_add_trigger(G_PRIORITY_HIGH, crm_fsa_trigger, NULL);
     config_read = mainloop_add_trigger(G_PRIORITY_HIGH, crm_read_options, NULL);
     transition_trigger = mainloop_add_trigger(G_PRIORITY_LOW, te_graph_trigger, NULL);
 
     crm_debug("Creating CIB manager and executor objects");
     fsa_cib_conn = cib_new();
 
     lrm_state_init_local();
     if (controld_init_fsa_timers() == FALSE) {
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
 }
 
 // \return libqb error code (0 on success, -errno on error)
 static int32_t
 accept_controller_client(qb_ipcs_connection_t *c, uid_t uid, gid_t gid)
 {
     crm_trace("Accepting new IPC client connection");
     if (pcmk__new_client(c, uid, gid) == NULL) {
         return -EIO;
     }
     return 0;
 }
 
 // \return libqb error code (0 on success, -errno on error)
 static int32_t
 dispatch_controller_ipc(qb_ipcs_connection_t * c, void *data, size_t size)
 {
     uint32_t id = 0;
     uint32_t flags = 0;
     pcmk__client_t *client = pcmk__find_client(c);
 
     xmlNode *msg = pcmk__client_data2xml(client, data, &id, &flags);
 
     pcmk__ipc_send_ack(client, id, flags, "ack");
     if (msg == NULL) {
         return 0;
     }
 
 #if ENABLE_ACL
     CRM_ASSERT(client->user != NULL);
     pcmk__update_acl_user(msg, F_CRM_USER, client->user);
 #endif
 
     crm_xml_add(msg, F_CRM_SYS_FROM, client->id);
     if (controld_authorize_ipc_message(msg, client, NULL)) {
         crm_trace("Processing IPC message from %s", pcmk__client_name(client));
         route_message(C_IPC_MESSAGE, msg);
     }
 
-    trigger_fsa(fsa_source);
+    trigger_fsa();
     free_xml(msg);
     return 0;
 }
 
 static int32_t
 crmd_ipc_closed(qb_ipcs_connection_t * c)
 {
     pcmk__client_t *client = pcmk__find_client(c);
 
     if (client) {
         crm_trace("Disconnecting %sregistered client %s (%p/%p)",
                   (client->userdata? "" : "un"), pcmk__client_name(client),
                   c, client);
         free(client->userdata);
         pcmk__free_client(client);
-        trigger_fsa(fsa_source);
+        trigger_fsa();
     }
     return 0;
 }
 
 static void
 crmd_ipc_destroy(qb_ipcs_connection_t * c)
 {
     crm_trace("Connection %p", c);
     crmd_ipc_closed(c);
 }
 
 /*	 A_STOP	*/
 void
 do_stop(long long action,
         enum crmd_fsa_cause cause,
         enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     crm_trace("Closing IPC server");
     mainloop_del_ipc_server(ipcs); ipcs = NULL;
     register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
 }
 
 /*	 A_STARTED	*/
 void
 do_started(long long action,
            enum crmd_fsa_cause cause,
            enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     static struct qb_ipcs_service_handlers crmd_callbacks = {
         .connection_accept = accept_controller_client,
         .connection_created = NULL,
         .msg_process = dispatch_controller_ipc,
         .connection_closed = crmd_ipc_closed,
         .connection_destroyed = crmd_ipc_destroy
     };
 
     if (cur_state != S_STARTING) {
         crm_err("Start cancelled... %s", fsa_state2string(cur_state));
         return;
 
     } else if (is_set(fsa_input_register, R_MEMBERSHIP) == FALSE) {
         crm_info("Delaying start, no membership data (%.16llx)", R_MEMBERSHIP);
 
         crmd_fsa_stall(TRUE);
         return;
 
     } else if (is_set(fsa_input_register, R_LRM_CONNECTED) == FALSE) {
         crm_info("Delaying start, not connected to executor (%.16llx)", R_LRM_CONNECTED);
 
         crmd_fsa_stall(TRUE);
         return;
 
     } else if (is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) {
         crm_info("Delaying start, CIB not connected (%.16llx)", R_CIB_CONNECTED);
 
         crmd_fsa_stall(TRUE);
         return;
 
     } else if (is_set(fsa_input_register, R_READ_CONFIG) == FALSE) {
         crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG);
 
         crmd_fsa_stall(TRUE);
         return;
 
     } else if (is_set(fsa_input_register, R_PEER_DATA) == FALSE) {
 
         crm_info("Delaying start, No peer data (%.16llx)", R_PEER_DATA);
         crmd_fsa_stall(TRUE);
         return;
     }
 
     crm_debug("Init server comms");
     ipcs = pcmk__serve_controld_ipc(&crmd_callbacks);
     if (ipcs == NULL) {
         crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     } else {
         crm_notice("Pacemaker controller successfully started and accepting connections");
     }
     controld_trigger_fencer_connect();
 
     controld_clear_fsa_input_flags(R_STARTING);
     register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL);
 }
 
 /*	 A_RECOVER	*/
 void
 do_recover(long long action,
            enum crmd_fsa_cause cause,
            enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     controld_set_fsa_input_flags(R_IN_RECOVERY);
     crm_warn("Fast-tracking shutdown in response to errors");
 
     register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
 }
 
 static pcmk__cluster_option_t crmd_opts[] = {
     /* name, old name, type, allowed values,
      * default value, validator,
      * short description,
      * long description
      */
     {
         "dc-version", NULL, "string", NULL, "none", NULL,
         "Pacemaker version on cluster node elected Designated Controller (DC)",
         "Includes a hash which identifies the exact changeset the code was "
             "built from. Used for diagnostic purposes."
     },
     {
         "cluster-infrastructure", NULL, "string", NULL, "corosync", NULL,
         "The messaging stack on which Pacemaker is currently running",
         "Used for informational and diagnostic purposes."
     },
     {
         "cluster-name", NULL, "string", NULL, NULL, NULL,
         "An arbitrary name for the cluster",
         "This optional value is mostly for users' convenience as desired "
             "in administration, but may also be used in Pacemaker "
             "configuration rules via the #cluster-name node attribute, and "
             "by higher-level tools and resource agents."
     },
     {
         XML_CONFIG_ATTR_DC_DEADTIME, NULL, "time",
         NULL, "20s", pcmk__valid_interval_spec,
         "How long to wait for a response from other nodes during start-up",
         "The optimal value will depend on the speed and load of your network "
             "and the type of switches used."
     },
     {
         XML_CONFIG_ATTR_RECHECK, NULL, "time",
         "Zero disables polling, while positive values are an interval in seconds"
             "(unless other units are specified, for example \"5min\")",
         "15min", pcmk__valid_interval_spec,
         "Polling interval to recheck cluster state and evaluate rules "
             "with date specifications",
         "Pacemaker is primarily event-driven, and looks ahead to know when to "
             "recheck cluster state for failure timeouts and most time-based "
             "rules. However, it will also recheck the cluster after this "
             "amount of inactivity, to evaluate rules with date specifications "
             "and serve as a fail-safe for certain types of scheduler bugs."
     },
     {
         "load-threshold", NULL, "percentage", NULL,
         "80%", pcmk__valid_utilization,
         "Maximum amount of system load that should be used by cluster nodes",
         "The cluster will slow down its recovery process when the amount of "
             "system resources used (currently CPU) approaches this limit",
     },
     {
         "node-action-limit", NULL, "integer", NULL,
         "0", pcmk__valid_number,
         "Maximum number of jobs that can be scheduled per node "
             "(defaults to 2x cores)"
     },
     { XML_CONFIG_ATTR_FENCE_REACTION, NULL, "string", NULL, "stop", NULL,
         "How a cluster node should react if notified of its own fencing",
         "A cluster node may receive notification of its own fencing if fencing "
         "is misconfigured, or if fabric fencing is in use that doesn't cut "
         "cluster communication. Allowed values are \"stop\" to attempt to "
         "immediately stop pacemaker and stay stopped, or \"panic\" to attempt "
         "to immediately reboot the local node, falling back to stop on failure."
     },
     {
         XML_CONFIG_ATTR_ELECTION_FAIL, NULL, "time", NULL,
         "2min", pcmk__valid_interval_spec,
         "*** Advanced Use Only ***",
         "Declare an election failed if it is not decided within this much "
             "time. If you need to adjust this value, it probably indicates "
             "the presence of a bug."
     },
     {
         XML_CONFIG_ATTR_FORCE_QUIT, NULL, "time", NULL,
         "20min", pcmk__valid_interval_spec,
         "*** Advanced Use Only ***",
         "Exit immediately if shutdown does not complete within this much "
             "time. If you need to adjust this value, it probably indicates "
             "the presence of a bug."
     },
     {
         "join-integration-timeout", "crmd-integration-timeout", "time", NULL,
         "3min", pcmk__valid_interval_spec,
         "*** Advanced Use Only ***",
         "If you need to adjust this value, it probably indicates "
             "the presence of a bug."
     },
     {
         "join-finalization-timeout", "crmd-finalization-timeout", "time", NULL,
         "30min", pcmk__valid_interval_spec,
         "*** Advanced Use Only ***",
         "If you need to adjust this value, it probably indicates "
             "the presence of a bug."
     },
     {
         "transition-delay", "crmd-transition-delay", "time", NULL,
         "0s", pcmk__valid_interval_spec,
         "*** Advanced Use Only *** Enabling this option will slow down "
             "cluster recovery under all conditions",
         "Delay cluster recovery for this much time to allow for additional "
             "events to occur. Useful if your configuration is sensitive to "
             "the order in which ping updates arrive."
     },
     {
         "stonith-watchdog-timeout", NULL, "time", NULL,
         NULL, pcmk__valid_sbd_timeout,
         "How long to wait before we can assume nodes are safely down "
             "when sbd is in use",
         NULL
     },
     {
         "stonith-max-attempts", NULL, "integer", NULL,
         "10", pcmk__valid_positive_number,
         "How many times fencing can fail before it will no longer be "
             "immediately re-attempted on a target"
     },
 
     // Already documented in libpe_status (other values must be kept identical)
     {
         "no-quorum-policy", NULL, "enum", "stop, freeze, ignore, demote, suicide",
         "stop", pcmk__valid_quorum, NULL, NULL
     },
     {
         XML_CONFIG_ATTR_SHUTDOWN_LOCK, NULL, "boolean", NULL,
         "false", pcmk__valid_boolean, NULL, NULL
     },
 };
 
 void
 crmd_metadata(void)
 {
     pcmk__print_option_metadata("pacemaker-controld", "1.0",
                                 "Pacemaker controller options",
                                 "Cluster options used by Pacemaker's "
                                     "controller (formerly called crmd)",
                                 crmd_opts, DIMOF(crmd_opts));
 }
 
 static void
 verify_crmd_options(GHashTable * options)
 {
     pcmk__validate_cluster_options(options, crmd_opts, DIMOF(crmd_opts));
 }
 
 static const char *
 crmd_pref(GHashTable * options, const char *name)
 {
     return pcmk__cluster_option(options, crmd_opts, DIMOF(crmd_opts), name);
 }
 
 static void
 config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     const char *value = NULL;
     GHashTable *config_hash = NULL;
     crm_time_t *now = crm_time_new(NULL);
     xmlNode *crmconfig = NULL;
     xmlNode *alerts = NULL;
 
     if (rc != pcmk_ok) {
         fsa_data_t *msg_data = NULL;
 
         crm_err("Local CIB query resulted in an error: %s", pcmk_strerror(rc));
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
 
         if (rc == -EACCES || rc == -pcmk_err_schema_validation) {
             crm_err("The cluster is mis-configured - shutting down and staying down");
             controld_set_fsa_input_flags(R_STAYDOWN);
         }
         goto bail;
     }
 
     crmconfig = output;
     if ((crmconfig) &&
         (crm_element_name(crmconfig)) &&
         (strcmp(crm_element_name(crmconfig), XML_CIB_TAG_CRMCONFIG) != 0)) {
         crmconfig = first_named_child(crmconfig, XML_CIB_TAG_CRMCONFIG);
     }
     if (!crmconfig) {
         fsa_data_t *msg_data = NULL;
 
         crm_err("Local CIB query for " XML_CIB_TAG_CRMCONFIG " section failed");
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
         goto bail;
     }
 
     crm_debug("Call %d : Parsing CIB options", call_id);
     config_hash = crm_str_table_new();
     pe_unpack_nvpairs(crmconfig, crmconfig, XML_CIB_TAG_PROPSET, NULL,
                       config_hash, CIB_OPTIONS_FIRST, FALSE, now, NULL);
 
     verify_crmd_options(config_hash);
 
     value = crmd_pref(config_hash, XML_CONFIG_ATTR_DC_DEADTIME);
     election_trigger->period_ms = crm_parse_interval_spec(value);
 
     value = crmd_pref(config_hash, "node-action-limit"); /* Also checks migration-limit */
     throttle_update_job_max(value);
 
     value = crmd_pref(config_hash, "load-threshold");
     if(value) {
         throttle_set_load_target(strtof(value, NULL) / 100.0);
     }
 
     value = crmd_pref(config_hash, "no-quorum-policy");
     if (pcmk__str_eq(value, "suicide", pcmk__str_casei) && pcmk_locate_sbd()) {
         no_quorum_suicide_escalation = TRUE;
     }
 
     set_fence_reaction(crmd_pref(config_hash, XML_CONFIG_ATTR_FENCE_REACTION));
 
     value = crmd_pref(config_hash,"stonith-max-attempts");
     update_stonith_max_attempts(value);
 
     value = crmd_pref(config_hash, XML_CONFIG_ATTR_FORCE_QUIT);
     shutdown_escalation_timer->period_ms = crm_parse_interval_spec(value);
     crm_debug("Shutdown escalation occurs if DC has not responded to request in %ums",
               shutdown_escalation_timer->period_ms);
 
     value = crmd_pref(config_hash, XML_CONFIG_ATTR_ELECTION_FAIL);
     controld_set_election_period(value);
 
     value = crmd_pref(config_hash, XML_CONFIG_ATTR_RECHECK);
     recheck_interval_ms = crm_parse_interval_spec(value);
     crm_debug("Re-run scheduler after %dms of inactivity", recheck_interval_ms);
 
     value = crmd_pref(config_hash, "transition-delay");
     transition_timer->period_ms = crm_parse_interval_spec(value);
 
     value = crmd_pref(config_hash, "join-integration-timeout");
     integration_timer->period_ms = crm_parse_interval_spec(value);
 
     value = crmd_pref(config_hash, "join-finalization-timeout");
     finalization_timer->period_ms = crm_parse_interval_spec(value);
 
     value = crmd_pref(config_hash, XML_CONFIG_ATTR_SHUTDOWN_LOCK);
     controld_shutdown_lock_enabled = crm_is_true(value);
 
     free(fsa_cluster_name);
     fsa_cluster_name = NULL;
 
     value = g_hash_table_lookup(config_hash, "cluster-name");
     if (value) {
         fsa_cluster_name = strdup(value);
     }
 
     alerts = first_named_child(output, XML_CIB_TAG_ALERTS);
     crmd_unpack_alerts(alerts);
 
     controld_set_fsa_input_flags(R_READ_CONFIG);
     crm_trace("Triggering FSA: %s", __FUNCTION__);
     mainloop_set_trigger(fsa_source);
 
     g_hash_table_destroy(config_hash);
   bail:
     crm_time_free(now);
 }
 
 gboolean
 crm_read_options(gpointer user_data)
 {
     int call_id =
         fsa_cib_conn->cmds->query(fsa_cib_conn,
             "//" XML_CIB_TAG_CRMCONFIG " | //" XML_CIB_TAG_ALERTS,
             NULL, cib_xpath | cib_scope_local);
 
     fsa_register_cib_callback(call_id, FALSE, NULL, config_query_callback);
     crm_trace("Querying the CIB... call %d", call_id);
     return TRUE;
 }
 
 /*	 A_READCONFIG	*/
 void
 do_read_config(long long action,
                enum crmd_fsa_cause cause,
                enum crmd_fsa_state cur_state,
                enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     throttle_init();
     mainloop_set_trigger(config_read);
 }
 
 void
 crm_shutdown(int nsig)
 {
     if ((crmd_mainloop == NULL) || !g_main_loop_is_running(crmd_mainloop)) {
         crmd_exit(CRM_EX_OK);
         return;
     }
 
     if (is_set(fsa_input_register, R_SHUTDOWN)) {
         crm_err("Escalating shutdown");
         register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL);
         return;
     }
 
     controld_set_fsa_input_flags(R_SHUTDOWN);
     register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
 
     if (shutdown_escalation_timer->period_ms == 0) {
         const char *value = crmd_pref(NULL, XML_CONFIG_ATTR_FORCE_QUIT);
 
         shutdown_escalation_timer->period_ms = crm_parse_interval_spec(value);
     }
 
     crm_notice("Initiating controller shutdown sequence " CRM_XS
                " limit=%ums", shutdown_escalation_timer->period_ms);
     controld_start_timer(shutdown_escalation_timer);
 }
diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c
index 05fe4016d3..b1a54c4139 100644
--- a/daemons/controld/controld_execd.c
+++ b/daemons/controld/controld_execd.c
@@ -1,2875 +1,2875 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <regex.h>
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 
 #include <crm/crm.h>
 #include <crm/lrmd.h>           // lrmd_event_data_t, lrmd_rsc_info_t, etc.
 #include <crm/services.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/pengine/rules.h>
 
 #include <pacemaker-internal.h>
 #include <pacemaker-controld.h>
 
 #define START_DELAY_THRESHOLD 5 * 60 * 1000
 #define MAX_LRM_REG_FAILS 30
 
 struct delete_event_s {
     int rc;
     const char *rsc;
     lrm_state_t *lrm_state;
 };
 
 static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id);
 static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list);
 static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
 
 static lrmd_event_data_t *construct_op(lrm_state_t * lrm_state, xmlNode * rsc_op,
                                        const char *rsc_id, const char *operation);
 static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc,
                           const char *operation, xmlNode *msg);
 
 static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state,
                                          int log_level);
 static int do_update_resource(const char *node_name, lrmd_rsc_info_t *rsc,
                               lrmd_event_data_t *op, time_t lock_time);
 
 static void
 lrm_connection_destroy(void)
 {
     if (is_set(fsa_input_register, R_LRM_CONNECTED)) {
         crm_crit("Connection to executor failed");
         register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
         controld_clear_fsa_input_flags(R_LRM_CONNECTED);
 
     } else {
         crm_info("Disconnected from executor");
     }
 
 }
 
 static char *
 make_stop_id(const char *rsc, int call_id)
 {
     return crm_strdup_printf("%s:%d", rsc, call_id);
 }
 
 static void
 copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
 {
     if (strstr(key, CRM_META "_") == NULL) {
         g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
     }
 }
 
 static void
 copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
 {
     if (strstr(key, CRM_META "_") != NULL) {
         g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
     }
 }
 
 /*!
  * \internal
  * \brief Remove a recurring operation from a resource's history
  *
  * \param[in,out] history  Resource history to modify
  * \param[in]     op       Operation to remove
  *
  * \return TRUE if the operation was found and removed, FALSE otherwise
  */
 static gboolean
 history_remove_recurring_op(rsc_history_t *history, const lrmd_event_data_t *op)
 {
     GList *iter;
 
     for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
         lrmd_event_data_t *existing = iter->data;
 
         if ((op->interval_ms == existing->interval_ms)
             && pcmk__str_eq(op->rsc_id, existing->rsc_id, pcmk__str_none)
             && pcmk__str_eq(op->op_type, existing->op_type, pcmk__str_casei)) {
 
             history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter);
             lrmd_free_event(existing);
             return TRUE;
         }
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Free all recurring operations in resource history
  *
  * \param[in,out] history  Resource history to modify
  */
 static void
 history_free_recurring_ops(rsc_history_t *history)
 {
     GList *iter;
 
     for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
         lrmd_free_event(iter->data);
     }
     g_list_free(history->recurring_op_list);
     history->recurring_op_list = NULL;
 }
 
 /*!
  * \internal
  * \brief Free resource history
  *
  * \param[in,out] history  Resource history to free
  */
 void
 history_free(gpointer data)
 {
     rsc_history_t *history = (rsc_history_t*)data;
 
     if (history->stop_params) {
         g_hash_table_destroy(history->stop_params);
     }
 
     /* Don't need to free history->rsc.id because it's set to history->id */
     free(history->rsc.type);
     free(history->rsc.standard);
     free(history->rsc.provider);
 
     lrmd_free_event(history->failed);
     lrmd_free_event(history->last);
     free(history->id);
     history_free_recurring_ops(history);
     free(history);
 }
 
 static void
 update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
 {
     int target_rc = 0;
     rsc_history_t *entry = NULL;
 
     if (op->rsc_deleted) {
         crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
         controld_delete_resource_history(op->rsc_id, lrm_state->node_name,
                                          NULL, crmd_cib_smart_opt());
         return;
     }
 
     if (pcmk__str_eq(op->op_type, RSC_NOTIFY, pcmk__str_casei)) {
         return;
     }
 
     crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);
 
     entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
     if (entry == NULL && rsc) {
         entry = calloc(1, sizeof(rsc_history_t));
         entry->id = strdup(op->rsc_id);
         g_hash_table_insert(lrm_state->resource_history, entry->id, entry);
 
         entry->rsc.id = entry->id;
         entry->rsc.type = strdup(rsc->type);
         entry->rsc.standard = strdup(rsc->standard);
         if (rsc->provider) {
             entry->rsc.provider = strdup(rsc->provider);
         } else {
             entry->rsc.provider = NULL;
         }
 
     } else if (entry == NULL) {
         crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
         return;
     }
 
     entry->last_callid = op->call_id;
     target_rc = rsc_op_expected_rc(op);
     if (op->op_status == PCMK_LRM_OP_CANCELLED) {
         if (op->interval_ms > 0) {
             crm_trace("Removing cancelled recurring op: " PCMK__OP_FMT,
                       op->rsc_id, op->op_type, op->interval_ms);
             history_remove_recurring_op(entry, op);
             return;
         } else {
             crm_trace("Skipping " PCMK__OP_FMT " rc=%d, status=%d",
                       op->rsc_id, op->op_type, op->interval_ms, op->rc,
                       op->op_status);
         }
 
     } else if (did_rsc_op_fail(op, target_rc)) {
         /* Store failed monitors here, otherwise the block below will cause them
          * to be forgotten when a stop happens.
          */
         if (entry->failed) {
             lrmd_free_event(entry->failed);
         }
         entry->failed = lrmd_copy_event(op);
 
     } else if (op->interval_ms == 0) {
         if (entry->last) {
             lrmd_free_event(entry->last);
         }
         entry->last = lrmd_copy_event(op);
 
         if (op->params && pcmk__strcase_any_of(op->op_type, CRMD_ACTION_START,
                                                "reload", CRMD_ACTION_STATUS, NULL)) {
             if (entry->stop_params) {
                 g_hash_table_destroy(entry->stop_params);
             }
             entry->stop_params = crm_str_table_new();
 
             g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
         }
     }
 
     if (op->interval_ms > 0) {
         /* Ensure there are no duplicates */
         history_remove_recurring_op(entry, op);
 
         crm_trace("Adding recurring op: " PCMK__OP_FMT,
                   op->rsc_id, op->op_type, op->interval_ms);
         entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));
 
     } else if (entry->recurring_op_list && !pcmk__str_eq(op->op_type, RSC_STATUS, pcmk__str_casei)) {
         crm_trace("Dropping %d recurring ops because of: " PCMK__OP_FMT,
                   g_list_length(entry->recurring_op_list), op->rsc_id,
                   op->op_type, op->interval_ms);
         history_free_recurring_ops(entry);
     }
 }
 
 /*!
  * \internal
  * \brief Send a direct OK ack for a resource task
  *
  * \param[in] lrm_state  LRM connection
  * \param[in] input      Input message being ack'ed
  * \param[in] rsc_id     ID of affected resource
  * \param[in] rsc        Affected resource (if available)
  * \param[in] task       Operation task being ack'ed
  * \param[in] ack_host   Name of host to send ack to
  * \param[in] ack_sys    IPC system name to ack
  */
 static void
 send_task_ok_ack(lrm_state_t *lrm_state, ha_msg_input_t *input,
                  const char *rsc_id, lrmd_rsc_info_t *rsc, const char *task,
                  const char *ack_host, const char *ack_sys)
 {
     lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc_id, task);
 
     op->rc = PCMK_OCF_OK;
     op->op_status = PCMK_LRM_OP_DONE;
     controld_ack_event_directly(ack_host, ack_sys, rsc, op, rsc_id);
     lrmd_free_event(op);
 }
 
 static inline const char *
 op_node_name(lrmd_event_data_t *op)
 {
     return op->remote_nodename? op->remote_nodename : fsa_our_uname;
 }
 
 void
 lrm_op_callback(lrmd_event_data_t * op)
 {
     CRM_CHECK(op != NULL, return);
     switch (op->type) {
         case lrmd_event_disconnect:
             if (op->remote_nodename == NULL) {
                 /* If this is the local executor IPC connection, set the right
                  * bits in the controller when the connection goes down.
                  */
                 lrm_connection_destroy();
             }
             break;
 
         case lrmd_event_exec_complete:
             {
                 lrm_state_t *lrm_state = lrm_state_find(op_node_name(op));
 
                 CRM_ASSERT(lrm_state != NULL);
                 process_lrm_event(lrm_state, op, NULL, NULL);
             }
             break;
 
         default:
             break;
     }
 }
 
 /*	 A_LRM_CONNECT	*/
 void
 do_lrm_control(long long action,
                enum crmd_fsa_cause cause,
                enum crmd_fsa_state cur_state,
                enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     /* This only pertains to local executor connections. Remote connections are
      * handled as resources within the scheduler. Connecting and disconnecting
      * from remote executor instances is handled differently.
      */
 
     lrm_state_t *lrm_state = NULL;
 
     if(fsa_our_uname == NULL) {
         return; /* Nothing to do */
     }
     lrm_state = lrm_state_find_or_create(fsa_our_uname);
     if (lrm_state == NULL) {
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
         return;
     }
 
     if (action & A_LRM_DISCONNECT) {
         if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) {
             if (action == A_LRM_DISCONNECT) {
                 crmd_fsa_stall(FALSE);
                 return;
             }
         }
 
         controld_clear_fsa_input_flags(R_LRM_CONNECTED);
         crm_info("Disconnecting from the executor");
         lrm_state_disconnect(lrm_state);
         lrm_state_reset_tables(lrm_state, FALSE);
         crm_notice("Disconnected from the executor");
     }
 
     if (action & A_LRM_CONNECT) {
         int ret = pcmk_ok;
 
         crm_debug("Connecting to the executor");
         ret = lrm_state_ipc_connect(lrm_state);
 
         if (ret != pcmk_ok) {
             if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
                 crm_warn("Failed to connect to the executor %d time%s (%d max)",
                          lrm_state->num_lrm_register_fails,
                          pcmk__plural_s(lrm_state->num_lrm_register_fails),
                          MAX_LRM_REG_FAILS);
 
                 controld_start_timer(wait_timer);
                 crmd_fsa_stall(FALSE);
                 return;
             }
         }
 
         if (ret != pcmk_ok) {
             crm_err("Failed to connect to the executor the max allowed %d time%s",
                     lrm_state->num_lrm_register_fails,
                     pcmk__plural_s(lrm_state->num_lrm_register_fails));
             register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
             return;
         }
 
         controld_set_fsa_input_flags(R_LRM_CONNECTED);
         crm_info("Connection to the executor established");
     }
 
     if (action & ~(A_LRM_CONNECT | A_LRM_DISCONNECT)) {
         crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__);
     }
 }
 
 static gboolean
 lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level)
 {
     int counter = 0;
     gboolean rc = TRUE;
     const char *when = "lrm disconnect";
 
     GHashTableIter gIter;
     const char *key = NULL;
     rsc_history_t *entry = NULL;
     active_op_t *pending = NULL;
 
     crm_debug("Checking for active resources before exit");
 
     if (cur_state == S_TERMINATE) {
         log_level = LOG_ERR;
         when = "shutdown";
 
     } else if (is_set(fsa_input_register, R_SHUTDOWN)) {
         when = "shutdown... waiting";
     }
 
     if (lrm_state->pending_ops && lrm_state_is_connected(lrm_state) == TRUE) {
         guint removed = g_hash_table_foreach_remove(
             lrm_state->pending_ops, stop_recurring_actions, lrm_state);
         guint nremaining = g_hash_table_size(lrm_state->pending_ops);
 
         if (removed || nremaining) {
             crm_notice("Stopped %u recurring operation%s at %s (%u remaining)",
                        removed, pcmk__plural_s(removed), when, nremaining);
         }
     }
 
     if (lrm_state->pending_ops) {
         g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
         while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) {
             /* Ignore recurring actions in the shutdown calculations */
             if (pending->interval_ms == 0) {
                 counter++;
             }
         }
     }
 
     if (counter > 0) {
         do_crm_log(log_level, "%d pending executor operation%s at %s",
                    counter, pcmk__plural_s(counter), when);
 
         if (cur_state == S_TERMINATE || !is_set(fsa_input_register, R_SENT_RSC_STOP)) {
             g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
             while (g_hash_table_iter_next(&gIter, (gpointer*)&key, (gpointer*)&pending)) {
                 do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key);
             }
 
         } else {
             rc = FALSE;
         }
         return rc;
     }
 
     if (lrm_state->resource_history == NULL) {
         return rc;
     }
 
     if (is_set(fsa_input_register, R_SHUTDOWN)) {
         /* At this point we're not waiting, we're just shutting down */
         when = "shutdown";
     }
 
     counter = 0;
     g_hash_table_iter_init(&gIter, lrm_state->resource_history);
     while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) {
         if (is_rsc_active(lrm_state, entry->id) == FALSE) {
             continue;
         }
 
         counter++;
         if (log_level == LOG_ERR) {
             crm_info("Found %s active at %s", entry->id, when);
         } else {
             crm_trace("Found %s active at %s", entry->id, when);
         }
         if (lrm_state->pending_ops) {
             GHashTableIter hIter;
 
             g_hash_table_iter_init(&hIter, lrm_state->pending_ops);
             while (g_hash_table_iter_next(&hIter, (gpointer*)&key, (gpointer*)&pending)) {
                 if (pcmk__str_eq(entry->id, pending->rsc_id, pcmk__str_none)) {
                     crm_notice("%sction %s (%s) incomplete at %s",
                                pending->interval_ms == 0 ? "A" : "Recurring a",
                                key, pending->op_key, when);
                 }
             }
         }
     }
 
     if (counter) {
         crm_err("%d resource%s active at %s",
                 counter, (counter == 1)? " was" : "s were", when);
     }
 
     return rc;
 }
 
 static char *
 build_parameter_list(const lrmd_event_data_t *op,
                      const struct ra_metadata_s *metadata,
                      xmlNode *result, enum ra_param_flags_e param_type,
                      bool invert_for_xml)
 {
     int len = 0;
     int max = 0;
     char *list = NULL;
     GList *iter = NULL;
 
     /* Newer resource agents support the "private" parameter attribute to
      * indicate sensitive parameters. For backward compatibility with older
      * agents, this list is used if the agent doesn't specify any as "private".
      */
     const char *secure_terms[] = {
         "password",
         "passwd",
         "user",
     };
 
     if (is_not_set(metadata->ra_flags, ra_uses_private)
         && (param_type == ra_param_private)) {
 
         max = DIMOF(secure_terms);
     }
 
     for (iter = metadata->ra_params; iter != NULL; iter = iter->next) {
         struct ra_param_s *param = (struct ra_param_s *) iter->data;
         bool accept = FALSE;
 
         if (is_set(param->rap_flags, param_type)) {
             accept = TRUE;
 
         } else if (max) {
             for (int lpc = 0; lpc < max; lpc++) {
                 if (pcmk__str_eq(secure_terms[lpc], param->rap_name, pcmk__str_casei)) {
                     accept = TRUE;
                     break;
                 }
             }
         }
 
         if (accept) {
             int start = len;
 
             crm_trace("Attr %s is %s", param->rap_name, ra_param_flag2text(param_type));
 
             len += strlen(param->rap_name) + 2; // include spaces around
             list = realloc_safe(list, len + 1); // include null terminator
 
             // spaces before and after make parsing simpler
             sprintf(list + start, " %s ", param->rap_name);
 
         } else {
             crm_trace("Rejecting %s for %s", param->rap_name, ra_param_flag2text(param_type));
         }
 
         if (result && (invert_for_xml? !accept : accept)) {
             const char *v = g_hash_table_lookup(op->params, param->rap_name);
 
             if (v != NULL) {
                 crm_trace("Adding attr %s=%s to the xml result", param->rap_name, v);
                 crm_xml_add(result, param->rap_name, v);
             }
         }
     }
 
     return list;
 }
 
 static void
 append_restart_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
                     xmlNode *update, const char *version)
 {
     char *list = NULL;
     char *digest = NULL;
     xmlNode *restart = NULL;
 
     CRM_LOG_ASSERT(op->params != NULL);
 
     if (op->interval_ms > 0) {
         /* monitors are not reloadable */
         return;
     }
 
     if (is_set(metadata->ra_flags, ra_supports_reload)) {
         restart = create_xml_node(NULL, XML_TAG_PARAMS);
         /* Add any parameters with unique="1" to the "op-force-restart" list.
          *
          * (Currently, we abuse "unique=0" to indicate reloadability. This is
          * nonstandard and should eventually be replaced once the OCF standard
          * is updated with something better.)
          */
         list = build_parameter_list(op, metadata, restart, ra_param_unique,
                                     FALSE);
 
     } else {
         /* Resource does not support reloads */
         return;
     }
 
     digest = calculate_operation_digest(restart, version);
     /* Add "op-force-restart" and "op-restart-digest" to indicate the resource supports reload,
      * no matter if it actually supports any parameters with unique="1"). */
     crm_xml_add(update, XML_LRM_ATTR_OP_RESTART, list? list: "");
     crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);
 
     crm_trace("%s: %s, %s", op->rsc_id, digest, list);
     crm_log_xml_trace(restart, "restart digest source");
 
     free_xml(restart);
     free(digest);
     free(list);
 }
 
 static void
 append_secure_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
                    xmlNode *update, const char *version)
 {
     char *list = NULL;
     char *digest = NULL;
     xmlNode *secure = NULL;
 
     CRM_LOG_ASSERT(op->params != NULL);
 
     /*
      * To keep XML_LRM_ATTR_OP_SECURE short, we want it to contain the
      * secure parameters but XML_LRM_ATTR_SECURE_DIGEST to be based on
      * the insecure ones
      */
     secure = create_xml_node(NULL, XML_TAG_PARAMS);
     list = build_parameter_list(op, metadata, secure, ra_param_private, TRUE);
 
     if (list != NULL) {
         digest = calculate_operation_digest(secure, version);
         crm_xml_add(update, XML_LRM_ATTR_OP_SECURE, list);
         crm_xml_add(update, XML_LRM_ATTR_SECURE_DIGEST, digest);
 
         crm_trace("%s: %s, %s", op->rsc_id, digest, list);
         crm_log_xml_trace(secure, "secure digest source");
     } else {
         crm_trace("%s: no secure parameters", op->rsc_id);
     }
 
     free_xml(secure);
     free(digest);
     free(list);
 }
 
 static gboolean
 build_operation_update(xmlNode * parent, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op,
                        const char *node_name, const char *src)
 {
     int target_rc = 0;
     xmlNode *xml_op = NULL;
     struct ra_metadata_s *metadata = NULL;
     const char *caller_version = NULL;
     lrm_state_t *lrm_state = NULL;
 
     if (op == NULL) {
         return FALSE;
     }
 
     target_rc = rsc_op_expected_rc(op);
 
     /* there is a small risk in formerly mixed clusters that it will
      * be sub-optimal.
      *
      * however with our upgrade policy, the update we send should
      * still be completely supported anyway
      */
     caller_version = g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION);
     CRM_LOG_ASSERT(caller_version != NULL);
 
     if(caller_version == NULL) {
         caller_version = CRM_FEATURE_SET;
     }
 
     crm_trace("Building %s operation update with originator version: %s", op->rsc_id, caller_version);
     xml_op = pcmk__create_history_xml(parent, op, caller_version, target_rc,
                                       fsa_our_uname, src, LOG_DEBUG);
     if (xml_op == NULL) {
         return TRUE;
     }
 
     if ((rsc == NULL) || (op->params == NULL)
         || !crm_op_needs_metadata(rsc->standard, op->op_type)) {
 
         crm_trace("No digests needed for %s action on %s (params=%p rsc=%p)",
                   op->op_type, op->rsc_id, op->params, rsc);
         return TRUE;
     }
 
     lrm_state = lrm_state_find(node_name);
     if (lrm_state == NULL) {
         crm_warn("Cannot calculate digests for operation " PCMK__OP_FMT
                  " because we have no connection to executor for %s",
                  op->rsc_id, op->op_type, op->interval_ms, node_name);
         return TRUE;
     }
 
     metadata = metadata_cache_get(lrm_state->metadata_cache, rsc);
     if (metadata == NULL) {
         /* For now, we always collect resource agent meta-data via a local,
          * synchronous, direct execution of the agent. This has multiple issues:
          * the executor should execute agents, not the controller; meta-data for
          * Pacemaker Remote nodes should be collected on those nodes, not
          * locally; and the meta-data call shouldn't eat into the timeout of the
          * real action being performed.
          *
          * These issues are planned to be addressed by having the scheduler
          * schedule a meta-data cache check at the beginning of each transition.
          * Once that is working, this block will only be a fallback in case the
          * initial collection fails.
          */
         char *metadata_str = NULL;
 
         int rc = lrm_state_get_metadata(lrm_state, rsc->standard,
                                         rsc->provider, rsc->type,
                                         &metadata_str, 0);
 
         if (rc != pcmk_ok) {
             crm_warn("Failed to get metadata for %s (%s:%s:%s)",
                      rsc->id, rsc->standard, rsc->provider, rsc->type);
             return TRUE;
         }
 
         metadata = metadata_cache_update(lrm_state->metadata_cache, rsc,
                                          metadata_str);
         free(metadata_str);
         if (metadata == NULL) {
             crm_warn("Failed to update metadata for %s (%s:%s:%s)",
                      rsc->id, rsc->standard, rsc->provider, rsc->type);
             return TRUE;
         }
     }
 
 #if ENABLE_VERSIONED_ATTRS
     crm_xml_add(xml_op, XML_ATTR_RA_VERSION, metadata->ra_version);
 #endif
 
     crm_trace("Including additional digests for %s::%s:%s", rsc->standard, rsc->provider, rsc->type);
     append_restart_list(op, metadata, xml_op, caller_version);
     append_secure_list(op, metadata, xml_op, caller_version);
 
     return TRUE;
 }
 
 static gboolean
 is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id)
 {
     rsc_history_t *entry = NULL;
 
     entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
     if (entry == NULL || entry->last == NULL) {
         return FALSE;
     }
 
     crm_trace("Processing %s: %s.%d=%d", rsc_id, entry->last->op_type,
               entry->last->interval_ms, entry->last->rc);
     if (entry->last->rc == PCMK_OCF_OK && pcmk__str_eq(entry->last->op_type, CRMD_ACTION_STOP, pcmk__str_casei)) {
         return FALSE;
 
     } else if (entry->last->rc == PCMK_OCF_OK
                && pcmk__str_eq(entry->last->op_type, CRMD_ACTION_MIGRATE, pcmk__str_casei)) {
         // A stricter check is too complex ... leave that to the scheduler
         return FALSE;
 
     } else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) {
         return FALSE;
 
     } else if ((entry->last->interval_ms == 0)
                && (entry->last->rc == PCMK_OCF_NOT_CONFIGURED)) {
         /* Badly configured resources can't be reliably stopped */
         return FALSE;
     }
 
     return TRUE;
 }
 
 static gboolean
 build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list)
 {
     GHashTableIter iter;
     rsc_history_t *entry = NULL;
 
     g_hash_table_iter_init(&iter, lrm_state->resource_history);
     while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
 
         GList *gIter = NULL;
         xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE);
 
         crm_xml_add(xml_rsc, XML_ATTR_ID, entry->id);
         crm_xml_add(xml_rsc, XML_ATTR_TYPE, entry->rsc.type);
         crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, entry->rsc.standard);
         crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, entry->rsc.provider);
 
         if (entry->last && entry->last->params) {
             const char *container = g_hash_table_lookup(entry->last->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
             if (container) {
                 crm_trace("Resource %s is a part of container resource %s", entry->id, container);
                 crm_xml_add(xml_rsc, XML_RSC_ATTR_CONTAINER, container);
             }
         }
         build_operation_update(xml_rsc, &(entry->rsc), entry->failed, lrm_state->node_name, __FUNCTION__);
         build_operation_update(xml_rsc, &(entry->rsc), entry->last, lrm_state->node_name, __FUNCTION__);
         for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
             build_operation_update(xml_rsc, &(entry->rsc), gIter->data, lrm_state->node_name, __FUNCTION__);
         }
     }
 
     return FALSE;
 }
 
 static xmlNode *
 do_lrm_query_internal(lrm_state_t *lrm_state, int update_flags)
 {
     xmlNode *xml_state = NULL;
     xmlNode *xml_data = NULL;
     xmlNode *rsc_list = NULL;
     crm_node_t *peer = NULL;
 
     peer = crm_get_peer_full(0, lrm_state->node_name, CRM_GET_PEER_ANY);
     CRM_CHECK(peer != NULL, return NULL);
 
     xml_state = create_node_state_update(peer, update_flags, NULL,
                                          __FUNCTION__);
     if (xml_state == NULL) {
         return NULL;
     }
 
     xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM);
     crm_xml_add(xml_data, XML_ATTR_ID, peer->uuid);
     rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES);
 
     /* Build a list of active (not always running) resources */
     build_active_RAs(lrm_state, rsc_list);
 
     crm_log_xml_trace(xml_state, "Current executor state");
 
     return xml_state;
 }
 
 xmlNode *
 controld_query_executor_state(const char *node_name)
 {
     lrm_state_t *lrm_state = lrm_state_find(node_name);
 
     if (!lrm_state) {
         crm_err("Could not find executor state for node %s", node_name);
         return NULL;
     }
     return do_lrm_query_internal(lrm_state,
                                  node_update_cluster|node_update_peer);
 }
 
 /*!
  * \internal
  * \brief Map standard Pacemaker return code to operation status and OCF code
  *
  * \param[out] event  Executor event whose status and return code should be set
  * \param[in]  rc     Standard Pacemaker return code
  */
 void
 controld_rc2event(lrmd_event_data_t *event, int rc)
 {
     switch (rc) {
         case pcmk_rc_ok:
             event->rc = PCMK_OCF_OK;
             event->op_status = PCMK_LRM_OP_DONE;
             break;
         case EACCES:
             event->rc = PCMK_OCF_INSUFFICIENT_PRIV;
             event->op_status = PCMK_LRM_OP_ERROR;
             break;
         default:
             event->rc = PCMK_OCF_UNKNOWN_ERROR;
             event->op_status = PCMK_LRM_OP_ERROR;
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Trigger a new transition after CIB status was deleted
  *
  * If a CIB status delete was not expected (as part of the transition graph),
  * trigger a new transition by updating the (arbitrary) "last-lrm-refresh"
  * cluster property.
  *
  * \param[in] from_sys  IPC name that requested the delete
  * \param[in] rsc_id    Resource whose status was deleted (for logging only)
  */
 void
 controld_trigger_delete_refresh(const char *from_sys, const char *rsc_id)
 {
     if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
         char *now_s = crm_strdup_printf("%lld", (long long) time(NULL));
 
         crm_debug("Triggering a refresh after %s cleaned %s", from_sys, rsc_id);
         update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG,
                              NULL, NULL, NULL, NULL, "last-lrm-refresh", now_s,
                              FALSE, NULL, NULL);
         free(now_s);
     }
 }
 
 static void
 notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc)
 {
     lrmd_event_data_t *op = NULL;
     const char *from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
     const char *from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
 
     crm_info("Notifying %s on %s that %s was%s deleted",
              from_sys, (from_host? from_host : "localhost"), rsc_id,
              ((rc == pcmk_ok)? "" : " not"));
     op = construct_op(lrm_state, input->xml, rsc_id, CRMD_ACTION_DELETE);
     controld_rc2event(op, pcmk_legacy2rc(rc));
     controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
     lrmd_free_event(op);
     controld_trigger_delete_refresh(from_sys, rsc_id);
 }
 
 static gboolean
 lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
 {
     struct delete_event_s *event = user_data;
     struct pending_deletion_op_s *op = value;
 
     if (pcmk__str_eq(event->rsc, op->rsc, pcmk__str_none)) {
         notify_deleted(event->lrm_state, op->input, event->rsc, event->rc);
         return TRUE;
     }
     return FALSE;
 }
 
 static gboolean
 lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
 {
     const char *rsc = user_data;
     active_op_t *pending = value;
 
     if (pcmk__str_eq(rsc, pending->rsc_id, pcmk__str_none)) {
         crm_info("Removing op %s:%d for deleted resource %s",
                  pending->op_key, pending->call_id, rsc);
         return TRUE;
     }
     return FALSE;
 }
 
 static void
 delete_rsc_entry(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id,
                  GHashTableIter * rsc_gIter, int rc, const char *user_name)
 {
     struct delete_event_s event;
 
     CRM_CHECK(rsc_id != NULL, return);
 
     if (rc == pcmk_ok) {
         char *rsc_id_copy = strdup(rsc_id);
 
         if (rsc_gIter) {
             g_hash_table_iter_remove(rsc_gIter);
         } else {
             g_hash_table_remove(lrm_state->resource_history, rsc_id_copy);
         }
         controld_delete_resource_history(rsc_id_copy, lrm_state->node_name,
                                          user_name, crmd_cib_smart_opt());
         g_hash_table_foreach_remove(lrm_state->pending_ops, lrm_remove_deleted_op, rsc_id_copy);
         free(rsc_id_copy);
     }
 
     if (input) {
         notify_deleted(lrm_state, input, rsc_id, rc);
     }
 
     event.rc = rc;
     event.rsc = rsc_id;
     event.lrm_state = lrm_state;
     g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event);
 }
 
 /*!
  * \internal
  * \brief Erase an LRM history entry from the CIB, given the operation data
  *
  * \param[in] lrm_state  LRM state of the desired node
  * \param[in] op         Operation whose history should be deleted
  */
 static void
 erase_lrm_history_by_op(lrm_state_t *lrm_state, lrmd_event_data_t *op)
 {
     xmlNode *xml_top = NULL;
 
     CRM_CHECK(op != NULL, return);
 
     xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
     crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
     crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);
 
     if (op->interval_ms > 0) {
         char *op_id = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
 
         /* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
         crm_xml_add(xml_top, XML_ATTR_ID, op_id);
         free(op_id);
     }
 
     crm_debug("Erasing resource operation history for " PCMK__OP_FMT " (call=%d)",
               op->rsc_id, op->op_type, op->interval_ms, op->call_id);
 
     fsa_cib_conn->cmds->remove(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top,
                                cib_quorum_override);
 
     crm_log_xml_trace(xml_top, "op:cancel");
     free_xml(xml_top);
 }
 
 /* Define xpath to find LRM resource history entry by node and resource */
 #define XPATH_HISTORY                                   \
     "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS              \
     "/" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"  \
     "/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES       \
     "/" XML_LRM_TAG_RESOURCE "[@" XML_ATTR_ID "='%s']"  \
     "/" XML_LRM_TAG_RSC_OP
 
 /* ... and also by operation key */
 #define XPATH_HISTORY_ID XPATH_HISTORY \
     "[@" XML_ATTR_ID "='%s']"
 
 /* ... and also by operation key and operation call ID */
 #define XPATH_HISTORY_CALL XPATH_HISTORY \
     "[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_CALLID "='%d']"
 
 /* ... and also by operation key and original operation key */
 #define XPATH_HISTORY_ORIG XPATH_HISTORY \
     "[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_TASK_KEY "='%s']"
 
 /*!
  * \internal
  * \brief Erase an LRM history entry from the CIB, given operation identifiers
  *
  * \param[in] lrm_state  LRM state of the node to clear history for
  * \param[in] rsc_id     Name of resource to clear history for
  * \param[in] key        Operation key of operation to clear history for
  * \param[in] orig_op    If specified, delete only if it has this original op
  * \param[in] call_id    If specified, delete entry only if it has this call ID
  */
 static void
 erase_lrm_history_by_id(lrm_state_t *lrm_state, const char *rsc_id,
                         const char *key, const char *orig_op, int call_id)
 {
     char *op_xpath = NULL;
 
     CRM_CHECK((rsc_id != NULL) && (key != NULL), return);
 
     if (call_id > 0) {
         op_xpath = crm_strdup_printf(XPATH_HISTORY_CALL,
                                      lrm_state->node_name, rsc_id, key,
                                      call_id);
 
     } else if (orig_op) {
         op_xpath = crm_strdup_printf(XPATH_HISTORY_ORIG,
                                      lrm_state->node_name, rsc_id, key,
                                      orig_op);
     } else {
         op_xpath = crm_strdup_printf(XPATH_HISTORY_ID,
                                      lrm_state->node_name, rsc_id, key);
     }
 
     crm_debug("Erasing resource operation history for %s on %s (call=%d)",
               key, rsc_id, call_id);
     fsa_cib_conn->cmds->remove(fsa_cib_conn, op_xpath, NULL,
                                cib_quorum_override | cib_xpath);
     free(op_xpath);
 }
 
 static inline gboolean
 last_failed_matches_op(rsc_history_t *entry, const char *op, guint interval_ms)
 {
     if (entry == NULL) {
         return FALSE;
     }
     if (op == NULL) {
         return TRUE;
     }
     return (pcmk__str_eq(op, entry->failed->op_type, pcmk__str_casei)
             && (interval_ms == entry->failed->interval_ms));
 }
 
 /*!
  * \internal
  * \brief Clear a resource's last failure
  *
  * Erase a resource's last failure on a particular node from both the
  * LRM resource history in the CIB, and the resource history remembered
  * for the LRM state.
  *
  * \param[in] rsc_id      Resource name
  * \param[in] node_name   Node name
  * \param[in] operation   If specified, only clear if matching this operation
  * \param[in] interval_ms If operation is specified, it has this interval
  */
 void
 lrm_clear_last_failure(const char *rsc_id, const char *node_name,
                        const char *operation, guint interval_ms)
 {
     char *op_key = NULL;
     char *orig_op_key = NULL;
     lrm_state_t *lrm_state = NULL;
 
     lrm_state = lrm_state_find(node_name);
     if (lrm_state == NULL) {
         return;
     }
 
     /* Erase from CIB */
     op_key = pcmk__op_key(rsc_id, "last_failure", 0);
     if (operation) {
         orig_op_key = pcmk__op_key(rsc_id, operation, interval_ms);
     }
     erase_lrm_history_by_id(lrm_state, rsc_id, op_key, orig_op_key, 0);
     free(op_key);
     free(orig_op_key);
 
     /* Remove from memory */
     if (lrm_state->resource_history) {
         rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history,
                                                    rsc_id);
 
         if (last_failed_matches_op(entry, operation, interval_ms)) {
             lrmd_free_event(entry->failed);
             entry->failed = NULL;
         }
     }
 }
 
 /* Returns: gboolean - cancellation is in progress */
 static gboolean
 cancel_op(lrm_state_t * lrm_state, const char *rsc_id, const char *key, int op, gboolean remove)
 {
     int rc = pcmk_ok;
     char *local_key = NULL;
     active_op_t *pending = NULL;
 
     CRM_CHECK(op != 0, return FALSE);
     CRM_CHECK(rsc_id != NULL, return FALSE);
     if (key == NULL) {
         local_key = make_stop_id(rsc_id, op);
         key = local_key;
     }
     pending = g_hash_table_lookup(lrm_state->pending_ops, key);
 
     if (pending) {
         if (remove && is_not_set(pending->flags, active_op_remove)) {
             set_bit(pending->flags, active_op_remove);
             crm_debug("Scheduling %s for removal", key);
         }
 
         if (is_set(pending->flags, active_op_cancelled)) {
             crm_debug("Operation %s already cancelled", key);
             free(local_key);
             return FALSE;
         }
         set_bit(pending->flags, active_op_cancelled);
 
     } else {
         crm_info("No pending op found for %s", key);
         free(local_key);
         return FALSE;
     }
 
     crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);
     rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type,
                           pending->interval_ms);
     if (rc == pcmk_ok) {
         crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
         free(local_key);
         return TRUE;
     }
 
     crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
     /* The caller needs to make sure the entry is
      * removed from the pending_ops list
      *
      * Usually by returning TRUE inside the worker function
      * supplied to g_hash_table_foreach_remove()
      *
      * Not removing the entry from pending_ops will block
      * the node from shutting down
      */
     free(local_key);
     return FALSE;
 }
 
 struct cancel_data {
     gboolean done;
     gboolean remove;
     const char *key;
     lrmd_rsc_info_t *rsc;
     lrm_state_t *lrm_state;
 };
 
 static gboolean
 cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
 {
     gboolean remove = FALSE;
     struct cancel_data *data = user_data;
     active_op_t *op = value;
 
     if (pcmk__str_eq(op->op_key, data->key, pcmk__str_none)) {
         data->done = TRUE;
         remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove);
     }
     return remove;
 }
 
 static gboolean
 cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
 {
     guint removed = 0;
     struct cancel_data data;
 
     CRM_CHECK(rsc != NULL, return FALSE);
     CRM_CHECK(key != NULL, return FALSE);
 
     data.key = key;
     data.rsc = rsc;
     data.done = FALSE;
     data.remove = remove;
     data.lrm_state = lrm_state;
 
     removed = g_hash_table_foreach_remove(lrm_state->pending_ops, cancel_action_by_key, &data);
     crm_trace("Removed %u op cache entries, new size: %u",
               removed, g_hash_table_size(lrm_state->pending_ops));
     return data.done;
 }
 
 /*!
  * \internal
  * \brief Retrieve resource information from LRM
  *
  * \param[in]  lrm_state LRM connection to use
  * \param[in]  rsc_xml   XML containing resource configuration
  * \param[in]  do_create If true, register resource with LRM if not already
  * \param[out] rsc_info  Where to store resource information obtained from LRM
  *
  * \retval pcmk_ok   Success (and rsc_info holds newly allocated result)
  * \retval -EINVAL   Required information is missing from arguments
  * \retval -ENOTCONN No active connection to LRM
  * \retval -ENODEV   Resource not found
  * \retval -errno    Error communicating with executor when registering resource
  *
  * \note Caller is responsible for freeing result on success.
  */
 static int
 get_lrm_resource(lrm_state_t *lrm_state, xmlNode *rsc_xml, gboolean do_create,
                  lrmd_rsc_info_t **rsc_info)
 {
     const char *id = ID(rsc_xml);
 
     CRM_CHECK(lrm_state && rsc_xml && rsc_info, return -EINVAL);
     CRM_CHECK(id, return -EINVAL);
 
     if (lrm_state_is_connected(lrm_state) == FALSE) {
         return -ENOTCONN;
     }
 
     crm_trace("Retrieving resource information for %s from the executor", id);
     *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
 
     // If resource isn't known by ID, try clone name, if provided
     if (!*rsc_info) {
         const char *long_id = crm_element_value(rsc_xml, XML_ATTR_ID_LONG);
 
         if (long_id) {
             *rsc_info = lrm_state_get_rsc_info(lrm_state, long_id, 0);
         }
     }
 
     if ((*rsc_info == NULL) && do_create) {
         const char *class = crm_element_value(rsc_xml, XML_AGENT_ATTR_CLASS);
         const char *provider = crm_element_value(rsc_xml, XML_AGENT_ATTR_PROVIDER);
         const char *type = crm_element_value(rsc_xml, XML_ATTR_TYPE);
         int rc;
 
         crm_trace("Registering resource %s with the executor", id);
         rc = lrm_state_register_rsc(lrm_state, id, class, provider, type,
                                     lrmd_opt_drop_recurring);
         if (rc != pcmk_ok) {
             fsa_data_t *msg_data = NULL;
 
             crm_err("Could not register resource %s with the executor on %s: %s "
                     CRM_XS " rc=%d",
                     id, lrm_state->node_name, pcmk_strerror(rc), rc);
 
             /* Register this as an internal error if this involves the local
              * executor. Otherwise, we're likely dealing with an unresponsive
              * remote node, which is not an FSA failure.
              */
             if (lrm_state_is_local(lrm_state) == TRUE) {
                 register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
             }
             return rc;
         }
 
         *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
     }
     return *rsc_info? pcmk_ok : -ENODEV;
 }
 
 static void
 delete_resource(lrm_state_t * lrm_state,
                 const char *id,
                 lrmd_rsc_info_t * rsc,
                 GHashTableIter * gIter,
                 const char *sys,
                 const char *user,
                 ha_msg_input_t * request,
                 gboolean unregister)
 {
     int rc = pcmk_ok;
 
     crm_info("Removing resource %s from executor for %s%s%s",
              id, sys, (user? " as " : ""), (user? user : ""));
 
     if (rsc && unregister) {
         rc = lrm_state_unregister_rsc(lrm_state, id, 0);
     }
 
     if (rc == pcmk_ok) {
         crm_trace("Resource %s deleted from executor", id);
     } else if (rc == -EINPROGRESS) {
         crm_info("Deletion of resource '%s' from executor is pending", id);
         if (request) {
             struct pending_deletion_op_s *op = NULL;
             char *ref = crm_element_value_copy(request->msg, XML_ATTR_REFERENCE);
 
             op = calloc(1, sizeof(struct pending_deletion_op_s));
             op->rsc = strdup(rsc->id);
             op->input = copy_ha_msg_input(request);
             g_hash_table_insert(lrm_state->deletion_ops, ref, op);
         }
         return;
     } else {
         crm_warn("Could not delete '%s' from executor for %s%s%s: %s "
                  CRM_XS " rc=%d", id, sys, (user? " as " : ""),
                  (user? user : ""), pcmk_strerror(rc), rc);
     }
 
     delete_rsc_entry(lrm_state, request, id, gIter, rc, user);
 }
 
 static int
 get_fake_call_id(lrm_state_t *lrm_state, const char *rsc_id)
 {
     int call_id = 999999999;
     rsc_history_t *entry = NULL;
 
     if(lrm_state) {
         entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
     }
 
     /* Make sure the call id is greater than the last successful operation,
      * otherwise the failure will not result in a possible recovery of the resource
      * as it could appear the failure occurred before the successful start */
     if (entry) {
         call_id = entry->last_callid + 1;
     }
 
     if (call_id < 0) {
         call_id = 1;
     }
     return call_id;
 }
 
 static void
 fake_op_status(lrm_state_t *lrm_state, lrmd_event_data_t *op, int op_status,
                enum ocf_exitcode op_exitcode)
 {
     op->call_id = get_fake_call_id(lrm_state, op->rsc_id);
     op->t_run = time(NULL);
     op->t_rcchange = op->t_run;
     op->op_status = op_status;
     op->rc = op_exitcode;
 }
 
 static void
 force_reprobe(lrm_state_t *lrm_state, const char *from_sys,
               const char *from_host, const char *user_name,
               gboolean is_remote_node)
 {
     GHashTableIter gIter;
     rsc_history_t *entry = NULL;
 
     crm_info("Clearing resource history on node %s", lrm_state->node_name);
     g_hash_table_iter_init(&gIter, lrm_state->resource_history);
     while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
         /* only unregister the resource during a reprobe if it is not a remote connection
          * resource. otherwise unregistering the connection will terminate remote-node
          * membership */
         gboolean unregister = TRUE;
 
         if (is_remote_lrmd_ra(NULL, NULL, entry->id)) {
             lrm_state_t *remote_lrm_state = lrm_state_find(entry->id);
             if (remote_lrm_state) {
                 /* when forcing a reprobe, make sure to clear remote node before
                  * clearing the remote node's connection resource */ 
                 force_reprobe(remote_lrm_state, from_sys, from_host, user_name, TRUE);
             }
             unregister = FALSE;
         }
 
         delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys,
                         user_name, NULL, unregister);
     }
 
     /* Now delete the copy in the CIB */
     controld_delete_node_state(lrm_state->node_name, controld_section_lrm,
                                cib_scope_local);
 
     /* Finally, _delete_ the value in pacemaker-attrd -- setting it to FALSE
      * would result in the scheduler sending us back here again
      */
     update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node);
 }
 
 /*!
  * \internal
  * \brief Fail a requested action without actually executing it
  *
  * For an action that can't be executed, process it similarly to an actual
  * execution result, with specified error status (except for notify actions,
  * which will always be treated as successful).
  *
  * \param[in] lrm_state  Executor connection that action is for
  * \param[in] action     Action XML from request
  * \param[in] rc         Desired return code to use
  * \param[in] op_status  Desired operation status to use
  */
 static void
 synthesize_lrmd_failure(lrm_state_t *lrm_state, xmlNode *action,
                         int op_status, enum ocf_exitcode rc)
 {
     lrmd_event_data_t *op = NULL;
     const char *operation = crm_element_value(action, XML_LRM_ATTR_TASK);
     const char *target_node = crm_element_value(action, XML_LRM_ATTR_TARGET);
     xmlNode *xml_rsc = find_xml_node(action, XML_CIB_TAG_RESOURCE, TRUE);
 
     if ((xml_rsc == NULL) || (ID(xml_rsc) == NULL)) {
         /* @TODO Should we do something else, like direct ack? */
         crm_info("Can't fake %s failure (%d) on %s without resource configuration",
                  crm_element_value(action, XML_LRM_ATTR_TASK_KEY), rc,
                  target_node);
         return;
 
     } else if(operation == NULL) {
         /* This probably came from crm_resource -C, nothing to do */
         crm_info("Can't fake %s failure (%d) on %s without operation",
                  ID(xml_rsc), rc, target_node);
         return;
     }
 
     op = construct_op(lrm_state, action, ID(xml_rsc), operation);
 
     if (pcmk__str_eq(operation, RSC_NOTIFY, pcmk__str_casei)) { // Notifications can't fail
         fake_op_status(lrm_state, op, PCMK_LRM_OP_DONE, PCMK_OCF_OK);
     } else {
         fake_op_status(lrm_state, op, op_status, rc);
     }
 
     crm_info("Faking " PCMK__OP_FMT " result (%d) on %s",
              op->rsc_id, op->op_type, op->interval_ms, op->rc, target_node);
 
     // Process the result as if it came from the LRM
     process_lrm_event(lrm_state, op, NULL, action);
     lrmd_free_event(op);
 }
 
 /*!
  * \internal
  * \brief Get target of an LRM operation
  *
  * \param[in] xml  LRM operation data XML
  *
  * \return LRM operation target node name (local node or Pacemaker Remote node)
  */
 static const char *
 lrm_op_target(xmlNode *xml)
 {
     const char *target = NULL;
 
     if (xml) {
         target = crm_element_value(xml, XML_LRM_ATTR_TARGET);
     }
     if (target == NULL) {
         target = fsa_our_uname;
     }
     return target;
 }
 
 static void
 fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name,
                   const char *from_host, const char *from_sys)
 {
     lrmd_event_data_t *op = NULL;
     lrmd_rsc_info_t *rsc = NULL;
     xmlNode *xml_rsc = find_xml_node(xml, XML_CIB_TAG_RESOURCE, TRUE);
 
     CRM_CHECK(xml_rsc != NULL, return);
 
     /* The executor simply executes operations and reports the results, without
      * any concept of success or failure, so to fail a resource, we must fake
      * what a failure looks like.
      *
      * To do this, we create a fake executor operation event for the resource,
      * and pass that event to the executor client callback so it will be
      * processed as if it came from the executor.
      */
     op = construct_op(lrm_state, xml, ID(xml_rsc), "asyncmon");
     fake_op_status(lrm_state, op, PCMK_LRM_OP_DONE, PCMK_OCF_UNKNOWN_ERROR);
 
     free((char*) op->user_data);
     op->user_data = NULL;
     op->interval_ms = 0;
 
 #if ENABLE_ACL
     if (user_name && !pcmk__is_privileged(user_name)) {
         crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc));
         controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc));
         lrmd_free_event(op);
         return;
     }
 #endif
 
     if (get_lrm_resource(lrm_state, xml_rsc, TRUE, &rsc) == pcmk_ok) {
         crm_info("Failing resource %s...", rsc->id);
         op->exit_reason = strdup("Simulated failure");
         process_lrm_event(lrm_state, op, NULL, xml);
         op->op_status = PCMK_LRM_OP_DONE;
         op->rc = PCMK_OCF_OK;
         lrmd_free_rsc_info(rsc);
 
     } else {
         crm_info("Cannot find/create resource in order to fail it...");
         crm_log_xml_warn(xml, "bad input");
     }
 
     controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc));
     lrmd_free_event(op);
 }
 
 static void
 handle_refresh_op(lrm_state_t *lrm_state, const char *user_name,
                   const char *from_host, const char *from_sys)
 {
     int rc = pcmk_ok;
     xmlNode *fragment = do_lrm_query_internal(lrm_state, node_update_all);
 
     fsa_cib_update(XML_CIB_TAG_STATUS, fragment, cib_quorum_override, rc, user_name);
     crm_info("Forced a local resource history refresh: call=%d", rc);
 
     if (!pcmk__str_eq(CRM_SYSTEM_CRMD, from_sys, pcmk__str_casei)) {
         xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, fragment, from_host,
                                         from_sys, CRM_SYSTEM_LRMD,
                                         fsa_our_uuid);
 
         crm_debug("ACK'ing refresh from %s (%s)", from_sys, from_host);
 
         if (relay_message(reply, TRUE) == FALSE) {
             crm_log_xml_err(reply, "Unable to route reply");
         }
         free_xml(reply);
     }
 
     free_xml(fragment);
 }
 
 static void
 handle_query_op(xmlNode *msg, lrm_state_t *lrm_state)
 {
     xmlNode *data = do_lrm_query_internal(lrm_state, node_update_all);
     xmlNode *reply = create_reply(msg, data);
 
     if (relay_message(reply, TRUE) == FALSE) {
         crm_err("Unable to route reply");
         crm_log_xml_err(reply, "reply");
     }
     free_xml(reply);
     free_xml(data);
 }
 
 static void
 handle_reprobe_op(lrm_state_t *lrm_state, const char *from_sys,
                   const char *from_host, const char *user_name,
                   gboolean is_remote_node)
 {
     crm_notice("Forcing the status of all resources to be redetected");
     force_reprobe(lrm_state, from_sys, from_host, user_name, is_remote_node);
 
     if (!pcmk__strcase_any_of(from_sys, CRM_SYSTEM_PENGINE, CRM_SYSTEM_TENGINE, NULL)) {
 
         xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, NULL, from_host,
                                         from_sys, CRM_SYSTEM_LRMD,
                                         fsa_our_uuid);
 
         crm_debug("ACK'ing re-probe from %s (%s)", from_sys, from_host);
 
         if (relay_message(reply, TRUE) == FALSE) {
             crm_log_xml_err(reply, "Unable to route reply");
         }
         free_xml(reply);
     }
 }
 
 static bool do_lrm_cancel(ha_msg_input_t *input, lrm_state_t *lrm_state,
               lrmd_rsc_info_t *rsc, const char *from_host, const char *from_sys)
 {
     char *op_key = NULL;
     char *meta_key = NULL;
     int call = 0;
     const char *call_id = NULL;
     const char *op_task = NULL;
     guint interval_ms = 0;
     gboolean in_progress = FALSE;
     xmlNode *params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE);
 
     CRM_CHECK(params != NULL, return FALSE);
 
     meta_key = crm_meta_name(XML_LRM_ATTR_TASK);
     op_task = crm_element_value(params, meta_key);
     free(meta_key);
     CRM_CHECK(op_task != NULL, return FALSE);
 
     meta_key = crm_meta_name(XML_LRM_ATTR_INTERVAL_MS);
     if (crm_element_value_ms(params, meta_key, &interval_ms) != pcmk_ok) {
         free(meta_key);
         return FALSE;
     }
     free(meta_key);
 
     op_key = pcmk__op_key(rsc->id, op_task, interval_ms);
 
     meta_key = crm_meta_name(XML_LRM_ATTR_CALLID);
     call_id = crm_element_value(params, meta_key);
     free(meta_key);
 
     crm_debug("Scheduler requested op %s (call=%s) be cancelled",
               op_key, (call_id? call_id : "NA"));
     call = crm_parse_int(call_id, "0");
     if (call == 0) {
         // Normal case when the scheduler cancels a recurring op
         in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE);
 
     } else {
         // Normal case when the scheduler cancels an orphan op
         in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE);
     }
 
     // Acknowledge cancellation operation if for a remote connection resource
     if (!in_progress || is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
         char *op_id = make_stop_id(rsc->id, call);
 
         if (is_remote_lrmd_ra(NULL, NULL, rsc->id) == FALSE) {
             crm_info("Nothing known about operation %d for %s", call, op_key);
         }
         erase_lrm_history_by_id(lrm_state, rsc->id, op_key, NULL, call);
         send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
                          from_host, from_sys);
 
         /* needed at least for cancellation of a remote operation */
         g_hash_table_remove(lrm_state->pending_ops, op_id);
         free(op_id);
 
     } else {
         /* No ack is needed since abcdaa8, but peers with older versions
          * in a rolling upgrade need one. We didn't bump the feature set
          * at that commit, so we can only compare against the previous
          * CRM version (3.0.8). If any peers have feature set 3.0.9 but
          * not abcdaa8, they will time out waiting for the ack (no
          * released versions of Pacemaker are affected).
          */
         const char *peer_version = crm_element_value(params, XML_ATTR_CRM_VERSION);
 
         if (compare_version(peer_version, "3.0.8") <= 0) {
             crm_info("Sending compatibility ack for %s cancellation to %s (CRM version %s)",
                      op_key, from_host, peer_version);
             send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
                              from_host, from_sys);
         }
     }
 
     free(op_key);
     return TRUE;
 }
 
 static void
 do_lrm_delete(ha_msg_input_t *input, lrm_state_t *lrm_state,
               lrmd_rsc_info_t *rsc, const char *from_sys, const char *from_host,
               bool crm_rsc_delete, const char *user_name)
 {
     gboolean unregister = TRUE;
 
 #if ENABLE_ACL
     int cib_rc = controld_delete_resource_history(rsc->id, lrm_state->node_name,
                                                   user_name,
                                                   cib_dryrun|cib_sync_call);
 
     if (cib_rc != pcmk_rc_ok) {
         lrmd_event_data_t *op = NULL;
 
         op = construct_op(lrm_state, input->xml, rsc->id, CRMD_ACTION_DELETE);
         op->op_status = PCMK_LRM_OP_ERROR;
 
         if (cib_rc == EACCES) {
             op->rc = PCMK_OCF_INSUFFICIENT_PRIV;
         } else {
             op->rc = PCMK_OCF_UNKNOWN_ERROR;
         }
         controld_ack_event_directly(from_host, from_sys, NULL, op, rsc->id);
         lrmd_free_event(op);
         return;
     }
 #endif
 
     if (crm_rsc_delete && is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
         unregister = FALSE;
     }
 
     delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys,
                     user_name, input, unregister);
 }
 
 /*	 A_LRM_INVOKE	*/
 void
 do_lrm_invoke(long long action,
               enum crmd_fsa_cause cause,
               enum crmd_fsa_state cur_state,
               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     lrm_state_t *lrm_state = NULL;
     const char *crm_op = NULL;
     const char *from_sys = NULL;
     const char *from_host = NULL;
     const char *operation = NULL;
     ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
     const char *user_name = NULL;
     const char *target_node = NULL;
     gboolean is_remote_node = FALSE;
     bool crm_rsc_delete = FALSE;
 
     target_node = lrm_op_target(input->xml);
     is_remote_node = !pcmk__str_eq(target_node, fsa_our_uname,
                                    pcmk__str_casei);
 
     lrm_state = lrm_state_find(target_node);
     if ((lrm_state == NULL) && is_remote_node) {
         crm_err("Failing action because local node has never had connection to remote node %s",
                 target_node);
         synthesize_lrmd_failure(NULL, input->xml, PCMK_LRM_OP_NOT_CONNECTED,
                                 PCMK_OCF_UNKNOWN_ERROR);
         return;
     }
     CRM_ASSERT(lrm_state != NULL);
 
 #if ENABLE_ACL
     user_name = pcmk__update_acl_user(input->msg, F_CRM_USER, NULL);
 #endif
 
     crm_op = crm_element_value(input->msg, F_CRM_TASK);
     from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
     if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
         from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
     }
 #if ENABLE_ACL
     crm_trace("Executor %s command from %s as user %s",
               crm_op, from_sys, user_name);
 #else
     crm_trace("Executor %s command from %s",
               crm_op, from_sys);
 #endif
 
     if (pcmk__str_eq(crm_op, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
         if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
             crm_rsc_delete = TRUE; // from crm_resource
         }
         operation = CRMD_ACTION_DELETE;
 
     } else if (pcmk__str_eq(crm_op, CRM_OP_LRM_FAIL, pcmk__str_casei)) {
         fail_lrm_resource(input->xml, lrm_state, user_name, from_host,
                           from_sys);
         return;
 
     } else if (input->xml != NULL) {
         operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK);
     }
 
     if (pcmk__str_eq(crm_op, CRM_OP_LRM_REFRESH, pcmk__str_casei)) {
         handle_refresh_op(lrm_state, user_name, from_host, from_sys);
 
     } else if (pcmk__str_eq(crm_op, CRM_OP_LRM_QUERY, pcmk__str_casei)) {
         handle_query_op(input->msg, lrm_state);
 
     } else if (pcmk__str_eq(operation, CRM_OP_PROBED, pcmk__str_casei)) {
         update_attrd(lrm_state->node_name, CRM_OP_PROBED, XML_BOOLEAN_TRUE,
                      user_name, is_remote_node);
 
     } else if (pcmk__strcase_any_of(CRM_OP_REPROBE, operation, crm_op, NULL)) {
         handle_reprobe_op(lrm_state, from_sys, from_host, user_name,
                           is_remote_node);
 
     } else if (operation != NULL) {
         lrmd_rsc_info_t *rsc = NULL;
         xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);
         gboolean create_rsc = !pcmk__str_eq(operation, CRMD_ACTION_DELETE,
                                             pcmk__str_casei);
         int rc;
 
         // We can't return anything meaningful without a resource ID
         CRM_CHECK(xml_rsc && ID(xml_rsc), return);
 
         rc = get_lrm_resource(lrm_state, xml_rsc, create_rsc, &rsc);
         if (rc == -ENOTCONN) {
             synthesize_lrmd_failure(lrm_state, input->xml,
                                     PCMK_LRM_OP_NOT_CONNECTED,
                                     PCMK_OCF_UNKNOWN_ERROR);
             return;
 
         } else if ((rc < 0) && !create_rsc) {
             /* Delete of malformed or nonexistent resource
              * (deleting something that does not exist is a success)
              */
             crm_notice("Not registering resource '%s' for a %s event "
                        CRM_XS " get-rc=%d (%s) transition-key=%s",
                        ID(xml_rsc), operation,
                        rc, pcmk_strerror(rc), ID(input->xml));
             delete_rsc_entry(lrm_state, input, ID(xml_rsc), NULL, pcmk_ok,
                              user_name);
             return;
 
         } else if (rc == -EINVAL) {
             // Resource operation on malformed resource
             crm_err("Invalid resource definition for %s", ID(xml_rsc));
             crm_log_xml_warn(input->msg, "invalid resource");
             synthesize_lrmd_failure(lrm_state, input->xml, PCMK_LRM_OP_ERROR,
                                     PCMK_OCF_NOT_CONFIGURED); // fatal error
             return;
 
         } else if (rc < 0) {
             // Error communicating with the executor
             crm_err("Could not register resource '%s' with executor: %s "
                     CRM_XS " rc=%d",
                     ID(xml_rsc), pcmk_strerror(rc), rc);
             crm_log_xml_warn(input->msg, "failed registration");
             synthesize_lrmd_failure(lrm_state, input->xml, PCMK_LRM_OP_ERROR,
                                     PCMK_OCF_INVALID_PARAM); // hard error
             return;
         }
 
         if (pcmk__str_eq(operation, CRMD_ACTION_CANCEL, pcmk__str_casei)) {
             if (!do_lrm_cancel(input, lrm_state, rsc, from_host, from_sys)) {
                 crm_log_xml_warn(input->xml, "Bad command");
             }
 
         } else if (pcmk__str_eq(operation, CRMD_ACTION_DELETE, pcmk__str_casei)) {
             do_lrm_delete(input, lrm_state, rsc, from_sys, from_host,
                           crm_rsc_delete, user_name);
 
         } else {
             do_lrm_rsc_op(lrm_state, rsc, operation, input->xml);
         }
 
         lrmd_free_rsc_info(rsc);
 
     } else {
         crm_err("Cannot perform operation %s of unknown type", crm_str(crm_op));
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
 }
 
 #if ENABLE_VERSIONED_ATTRS
 static void
 resolve_versioned_parameters(lrm_state_t *lrm_state, const char *rsc_id,
                              const xmlNode *rsc_op, GHashTable *params)
 {
     /* Resource info *should* already be cached, so we don't get
      * executor call */
     lrmd_rsc_info_t *rsc = lrm_state_get_rsc_info(lrm_state, rsc_id, 0);
     struct ra_metadata_s *metadata;
 
     metadata = metadata_cache_get(lrm_state->metadata_cache, rsc);
     if (metadata) {
         xmlNode *versioned_attrs = NULL;
         GHashTable *hash = NULL;
         char *key = NULL;
         char *value = NULL;
         GHashTableIter iter;
 
         versioned_attrs = first_named_child(rsc_op, XML_TAG_OP_VER_ATTRS);
         hash = pe_unpack_versioned_parameters(versioned_attrs, metadata->ra_version);
         g_hash_table_iter_init(&iter, hash);
         while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
             g_hash_table_iter_steal(&iter);
             g_hash_table_replace(params, key, value);
         }
         g_hash_table_destroy(hash);
 
         versioned_attrs = first_named_child(rsc_op, XML_TAG_OP_VER_META);
         hash = pe_unpack_versioned_parameters(versioned_attrs, metadata->ra_version);
         g_hash_table_iter_init(&iter, hash);
         while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
             g_hash_table_replace(params, crm_meta_name(key), strdup(value));
 
             if (pcmk__str_eq(key, XML_ATTR_TIMEOUT, pcmk__str_casei)) {
                 op->timeout = crm_parse_int(value, "0");
             } else if (pcmk__str_eq(key, XML_OP_ATTR_START_DELAY, pcmk__str_casei)) {
                 op->start_delay = crm_parse_int(value, "0");
             }
         }
         g_hash_table_destroy(hash);
 
         versioned_attrs = first_named_child(rsc_op, XML_TAG_RSC_VER_ATTRS);
         hash = pe_unpack_versioned_parameters(versioned_attrs, metadata->ra_version);
         g_hash_table_iter_init(&iter, hash);
         while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
             g_hash_table_iter_steal(&iter);
             g_hash_table_replace(params, key, value);
         }
         g_hash_table_destroy(hash);
     }
 
     lrmd_free_rsc_info(rsc);
 }
 #endif
 
 static lrmd_event_data_t *
 construct_op(lrm_state_t *lrm_state, xmlNode *rsc_op, const char *rsc_id,
              const char *operation)
 {
     lrmd_event_data_t *op = NULL;
     const char *op_delay = NULL;
     const char *op_timeout = NULL;
     GHashTable *params = NULL;
 
     xmlNode *primitive = NULL;
     const char *class = NULL;
 
     const char *transition = NULL;
 
     CRM_ASSERT(rsc_id && operation);
 
     op = lrmd_new_event(rsc_id, operation, 0);
     op->type = lrmd_event_exec_complete;
     op->op_status = PCMK_LRM_OP_PENDING;
     op->rc = -1;
     op->timeout = 0;
     op->start_delay = 0;
 
     if (rsc_op == NULL) {
         CRM_LOG_ASSERT(pcmk__str_eq(CRMD_ACTION_STOP, operation, pcmk__str_casei));
         op->user_data = NULL;
         /* the stop_all_resources() case
          * by definition there is no DC (or they'd be shutting
          *   us down).
          * So we should put our version here.
          */
         op->params = crm_str_table_new();
 
         g_hash_table_insert(op->params, strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));
 
         crm_trace("Constructed %s op for %s", operation, rsc_id);
         return op;
     }
 
     params = xml2list(rsc_op);
     g_hash_table_remove(params, CRM_META "_op_target_rc");
 
     op_delay = crm_meta_value(params, XML_OP_ATTR_START_DELAY);
     op->start_delay = crm_parse_int(op_delay, "0");
 
     op_timeout = crm_meta_value(params, XML_ATTR_TIMEOUT);
     op->timeout = crm_parse_int(op_timeout, "0");
 
     if (pcmk__guint_from_hash(params, CRM_META "_" XML_LRM_ATTR_INTERVAL_MS, 0,
                               &(op->interval_ms)) != pcmk_rc_ok) {
         op->interval_ms = 0;
     }
 
     /* Use pcmk_monitor_timeout instead of meta timeout for stonith
        recurring monitor, if set */
     primitive = find_xml_node(rsc_op, XML_CIB_TAG_RESOURCE, FALSE);
     class = crm_element_value(primitive, XML_AGENT_ATTR_CLASS);
 
     if (is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_fence_params)
             && pcmk__str_eq(operation, CRMD_ACTION_STATUS, pcmk__str_casei)
             && (op->interval_ms > 0)) {
 
         op_timeout = g_hash_table_lookup(params, "pcmk_monitor_timeout");
         if (op_timeout != NULL) {
             op->timeout = crm_get_msec(op_timeout);
         }
     }
 
 #if ENABLE_VERSIONED_ATTRS
     if (lrm_state && !is_remote_lrmd_ra(NULL, NULL, rsc_id)
         && !pcmk__strcase_any_of(op_type, CRMD_ACTION_METADATA, CRMD_ACTION_DELETE,
                                  NULL)) {
         resolve_versioned_parameters(lrm_state, rsc_id, rsc_op, params);
     }
 #endif
 
     if (!pcmk__str_eq(operation, RSC_STOP, pcmk__str_casei)) {
         op->params = params;
 
     } else {
         rsc_history_t *entry = NULL;
 
         if (lrm_state) {
             entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
         }
 
         /* If we do not have stop parameters cached, use
          * whatever we are given */
         if (!entry || !entry->stop_params) {
             op->params = params;
         } else {
             /* Copy the cached parameter list so that we stop the resource
              * with the old attributes, not the new ones */
             op->params = crm_str_table_new();
 
             g_hash_table_foreach(params, copy_meta_keys, op->params);
             g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
             g_hash_table_destroy(params);
             params = NULL;
         }
     }
 
     /* sanity */
     if (op->timeout <= 0) {
         op->timeout = op->interval_ms;
     }
     if (op->start_delay < 0) {
         op->start_delay = 0;
     }
 
     transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY);
     CRM_CHECK(transition != NULL, return op);
 
     op->user_data = strdup(transition);
 
     if (op->interval_ms != 0) {
         if (pcmk__strcase_any_of(operation, CRMD_ACTION_START, CRMD_ACTION_STOP, NULL)) {
             crm_err("Start and Stop actions cannot have an interval: %u",
                     op->interval_ms);
             op->interval_ms = 0;
         }
     }
 
     crm_trace("Constructed %s op for %s: interval=%u",
               operation, rsc_id, op->interval_ms);
 
     return op;
 }
 
 /*!
  * \internal
  * \brief Send a (synthesized) event result
  *
  * Reply with a synthesized event result directly, as opposed to going through
  * the executor.
  *
  * \param[in] to_host  Host to send result to
  * \param[in] to_sys   IPC name to send result to (NULL for transition engine)
  * \param[in] rsc      Type information about resource the result is for
  * \param[in] op       Event with result to send
  * \param[in] rsc_id   ID of resource the result is for
  */
 void
 controld_ack_event_directly(const char *to_host, const char *to_sys,
                             lrmd_rsc_info_t *rsc, lrmd_event_data_t *op,
                             const char *rsc_id)
 {
     xmlNode *reply = NULL;
     xmlNode *update, *iter;
     crm_node_t *peer = NULL;
 
     CRM_CHECK(op != NULL, return);
     if (op->rsc_id == NULL) {
         CRM_ASSERT(rsc_id != NULL);
         op->rsc_id = strdup(rsc_id);
     }
     if (to_sys == NULL) {
         to_sys = CRM_SYSTEM_TENGINE;
     }
 
     peer = crm_get_peer(0, fsa_our_uname);
     update = create_node_state_update(peer, node_update_none, NULL,
                                       __FUNCTION__);
 
     iter = create_xml_node(update, XML_CIB_TAG_LRM);
     crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
 
     crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);
 
     build_operation_update(iter, rsc, op, fsa_our_uname, __FUNCTION__);
     reply = create_request(CRM_OP_INVOKE_LRM, update, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);
 
     crm_log_xml_trace(update, "[direct ACK]");
 
     crm_debug("ACK'ing resource op " PCMK__OP_FMT " from %s: %s",
               op->rsc_id, op->op_type, op->interval_ms, op->user_data,
               crm_element_value(reply, XML_ATTR_REFERENCE));
 
     if (relay_message(reply, TRUE) == FALSE) {
         crm_log_xml_err(reply, "Unable to route reply");
     }
 
     free_xml(update);
     free_xml(reply);
 }
 
 gboolean
 verify_stopped(enum crmd_fsa_state cur_state, int log_level)
 {
     gboolean res = TRUE;
     GList *lrm_state_list = lrm_state_get_list();
     GList *state_entry;
 
     for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
         lrm_state_t *lrm_state = state_entry->data;
 
         if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) {
             /* keep iterating through all even when false is returned */
             res = FALSE;
         }
     }
 
     controld_set_fsa_input_flags(R_SENT_RSC_STOP);
     g_list_free(lrm_state_list); lrm_state_list = NULL;
     return res;
 }
 
 struct stop_recurring_action_s {
     lrmd_rsc_info_t *rsc;
     lrm_state_t *lrm_state;
 };
 
 static gboolean
 stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
 {
     gboolean remove = FALSE;
     struct stop_recurring_action_s *event = user_data;
     active_op_t *op = value;
 
     if ((op->interval_ms != 0)
         && pcmk__str_eq(op->rsc_id, event->rsc->id, pcmk__str_none)) {
 
         crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (char*)key);
         remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE);
     }
 
     return remove;
 }
 
 static gboolean
 stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
 {
     gboolean remove = FALSE;
     lrm_state_t *lrm_state = user_data;
     active_op_t *op = value;
 
     if (op->interval_ms != 0) {
         crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id,
                  (const char *) key);
         remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE);
     }
 
     return remove;
 }
 
 static void
 record_pending_op(const char *node_name, lrmd_rsc_info_t *rsc, lrmd_event_data_t *op)
 {
     const char *record_pending = NULL;
 
     CRM_CHECK(node_name != NULL, return);
     CRM_CHECK(rsc != NULL, return);
     CRM_CHECK(op != NULL, return);
 
     // Never record certain operation types as pending
     if ((op->op_type == NULL) || (op->params == NULL)
         || !controld_action_is_recordable(op->op_type)) {
         return;
     }
 
     // defaults to true
     record_pending = crm_meta_value(op->params, XML_OP_ATTR_PENDING);
     if (record_pending && !crm_is_true(record_pending)) {
         return;
     }
 
     op->call_id = -1;
     op->op_status = PCMK_LRM_OP_PENDING;
     op->rc = PCMK_OCF_UNKNOWN;
 
     op->t_run = time(NULL);
     op->t_rcchange = op->t_run;
 
     /* write a "pending" entry to the CIB, inhibit notification */
     crm_debug("Recording pending op " PCMK__OP_FMT " on %s in the CIB",
               op->rsc_id, op->op_type, op->interval_ms, node_name);
 
     do_update_resource(node_name, rsc, op, 0);
 }
 
 static void
 do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc,
               const char *operation, xmlNode *msg)
 {
     int call_id = 0;
     char *op_id = NULL;
     lrmd_event_data_t *op = NULL;
     lrmd_key_value_t *params = NULL;
     fsa_data_t *msg_data = NULL;
     const char *transition = NULL;
     gboolean stop_recurring = FALSE;
     bool send_nack = FALSE;
 
     CRM_CHECK(rsc != NULL, return);
     CRM_CHECK(operation != NULL, return);
 
     if (msg != NULL) {
         transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
         if (transition == NULL) {
             crm_log_xml_err(msg, "Missing transition number");
         }
     }
 
     op = construct_op(lrm_state, msg, rsc->id, operation);
     CRM_CHECK(op != NULL, return);
 
     if (is_remote_lrmd_ra(NULL, NULL, rsc->id)
         && (op->interval_ms == 0)
         && strcmp(operation, CRMD_ACTION_MIGRATE) == 0) {
 
         /* pcmk remote connections are a special use case.
          * We never ever want to stop monitoring a connection resource until
          * the entire migration has completed. If the connection is unexpectedly
          * severed, even during a migration, this is an event we must detect.*/
         stop_recurring = FALSE;
 
     } else if ((op->interval_ms == 0)
         && strcmp(operation, CRMD_ACTION_STATUS) != 0
         && strcmp(operation, CRMD_ACTION_NOTIFY) != 0) {
 
         /* stop any previous monitor operations before changing the resource state */
         stop_recurring = TRUE;
     }
 
     if (stop_recurring == TRUE) {
         guint removed = 0;
         struct stop_recurring_action_s data;
 
         data.rsc = rsc;
         data.lrm_state = lrm_state;
         removed = g_hash_table_foreach_remove(
             lrm_state->pending_ops, stop_recurring_action_by_rsc, &data);
 
         if (removed) {
             crm_debug("Stopped %u recurring operation%s in preparation for "
                       PCMK__OP_FMT, removed, pcmk__plural_s(removed),
                       rsc->id, operation, op->interval_ms);
         }
     }
 
     /* now do the op */
     crm_notice("Requesting local execution of %s operation for %s on %s "
                CRM_XS " transition_key=%s op_key=" PCMK__OP_FMT,
                crm_action_str(op->op_type, op->interval_ms), rsc->id, lrm_state->node_name,
                transition, rsc->id, operation, op->interval_ms);
 
     if (is_set(fsa_input_register, R_SHUTDOWN) && pcmk__str_eq(operation, RSC_START, pcmk__str_casei)) {
         register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
         send_nack = TRUE;
 
     } else if (fsa_state != S_NOT_DC
                && fsa_state != S_POLICY_ENGINE /* Recalculating */
                && fsa_state != S_TRANSITION_ENGINE
                && !pcmk__str_eq(operation, CRMD_ACTION_STOP, pcmk__str_casei)) {
         send_nack = TRUE;
     }
 
     if(send_nack) {
         crm_notice("Discarding attempt to perform action %s on %s in state %s (shutdown=%s)",
                    operation, rsc->id, fsa_state2string(fsa_state),
                    is_set(fsa_input_register, R_SHUTDOWN)?"true":"false");
 
         op->rc = PCMK_OCF_UNKNOWN_ERROR;
         op->op_status = PCMK_LRM_OP_INVALID;
         controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
         lrmd_free_event(op);
         free(op_id);
         return;
     }
 
     record_pending_op(lrm_state->node_name, rsc, op);
 
     op_id = pcmk__op_key(rsc->id, op->op_type, op->interval_ms);
 
     if (op->interval_ms > 0) {
         /* cancel it so we can then restart it without conflict */
         cancel_op_key(lrm_state, rsc, op_id, FALSE);
     }
 
     if (op->params) {
         char *key = NULL;
         char *value = NULL;
         GHashTableIter iter;
 
         g_hash_table_iter_init(&iter, op->params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             params = lrmd_key_value_add(params, key, value);
         }
     }
 
     call_id = lrm_state_exec(lrm_state, rsc->id, op->op_type, op->user_data,
                              op->interval_ms, op->timeout, op->start_delay,
                              params);
 
     if (call_id <= 0 && lrm_state_is_local(lrm_state)) {
         crm_err("Operation %s on %s failed: %d", operation, rsc->id, call_id);
         register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
 
     } else if (call_id <= 0) {
         crm_err("Operation %s on resource %s failed to execute on remote node %s: %d",
                 operation, rsc->id, lrm_state->node_name, call_id);
         fake_op_status(lrm_state, op, PCMK_LRM_OP_DONE, PCMK_OCF_UNKNOWN_ERROR);
         process_lrm_event(lrm_state, op, NULL, NULL);
 
     } else {
         /* record all operations so we can wait
          * for them to complete during shutdown
          */
         char *call_id_s = make_stop_id(rsc->id, call_id);
         active_op_t *pending = NULL;
 
         pending = calloc(1, sizeof(active_op_t));
         crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);
 
         pending->call_id = call_id;
         pending->interval_ms = op->interval_ms;
         pending->op_type = strdup(operation);
         pending->op_key = strdup(op_id);
         pending->rsc_id = strdup(rsc->id);
         pending->start_time = time(NULL);
         pending->user_data = op->user_data? strdup(op->user_data) : NULL;
         if (crm_element_value_epoch(msg, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
                                     &(pending->lock_time)) != pcmk_ok) {
             pending->lock_time = 0;
         }
         g_hash_table_replace(lrm_state->pending_ops, call_id_s, pending);
 
         if ((op->interval_ms > 0)
             && (op->start_delay > START_DELAY_THRESHOLD)) {
             int target_rc = 0;
 
             crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);
             decode_transition_key(op->user_data, NULL, NULL, NULL, &target_rc);
             op->rc = target_rc;
             op->op_status = PCMK_LRM_OP_DONE;
             controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
         }
 
         pending->params = op->params;
         op->params = NULL;
     }
 
     free(op_id);
     lrmd_free_event(op);
     return;
 }
 
 int last_resource_update = 0;
 
 static void
 cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     switch (rc) {
         case pcmk_ok:
         case -pcmk_err_diff_failed:
         case -pcmk_err_diff_resync:
             crm_trace("Resource update %d complete: rc=%d", call_id, rc);
             break;
         default:
             crm_warn("Resource update %d failed: (rc=%d) %s", call_id, rc, pcmk_strerror(rc));
     }
 
     if (call_id == last_resource_update) {
         last_resource_update = 0;
-        trigger_fsa(fsa_source);
+        trigger_fsa();
     }
 }
 
 /* Only successful stops, and probes that found the resource inactive, get locks
  * recorded in the history. This ensures the resource stays locked to the node
  * until it is active there again after the node comes back up.
  */
 static bool
 should_preserve_lock(lrmd_event_data_t *op)
 {
     if (!controld_shutdown_lock_enabled) {
         return false;
     }
     if (!strcmp(op->op_type, RSC_STOP) && (op->rc == PCMK_OCF_OK)) {
         return true;
     }
     if (!strcmp(op->op_type, RSC_STATUS) && (op->rc == PCMK_OCF_NOT_RUNNING)) {
         return true;
     }
     return false;
 }
 
 static int
 do_update_resource(const char *node_name, lrmd_rsc_info_t *rsc,
                    lrmd_event_data_t *op, time_t lock_time)
 {
 /*
   <status>
   <nodes_status id=uname>
   <lrm>
   <lrm_resources>
   <lrm_resource id=...>
   </...>
 */
     int rc = pcmk_ok;
     xmlNode *update, *iter = NULL;
     int call_opt = crmd_cib_smart_opt();
     const char *uuid = NULL;
 
     CRM_CHECK(op != NULL, return 0);
 
     iter = create_xml_node(iter, XML_CIB_TAG_STATUS);
     update = iter;
     iter = create_xml_node(iter, XML_CIB_TAG_STATE);
 
     if (pcmk__str_eq(node_name, fsa_our_uname, pcmk__str_casei)) {
         uuid = fsa_our_uuid;
 
     } else {
         /* remote nodes uuid and uname are equal */
         uuid = node_name;
         crm_xml_add(iter, XML_NODE_IS_REMOTE, "true");
     }
 
     CRM_LOG_ASSERT(uuid != NULL);
     if(uuid == NULL) {
         rc = -EINVAL;
         goto done;
     }
 
     crm_xml_add(iter, XML_ATTR_UUID,  uuid);
     crm_xml_add(iter, XML_ATTR_UNAME, node_name);
     crm_xml_add(iter, XML_ATTR_ORIGIN, __FUNCTION__);
 
     iter = create_xml_node(iter, XML_CIB_TAG_LRM);
     crm_xml_add(iter, XML_ATTR_ID, uuid);
 
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
     crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);
 
     build_operation_update(iter, rsc, op, node_name, __FUNCTION__);
 
     if (rsc) {
         const char *container = NULL;
 
         crm_xml_add(iter, XML_ATTR_TYPE, rsc->type);
         crm_xml_add(iter, XML_AGENT_ATTR_CLASS, rsc->standard);
         crm_xml_add(iter, XML_AGENT_ATTR_PROVIDER, rsc->provider);
         if (lock_time != 0) {
             /* Actions on a locked resource should either preserve the lock by
              * recording it with the action result, or clear it.
              */
             if (!should_preserve_lock(op)) {
                 lock_time = 0;
             }
             crm_xml_add_ll(iter, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
                            (long long) lock_time);
         }
 
         if (op->params) {
             container = g_hash_table_lookup(op->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
         }
         if (container) {
             crm_trace("Resource %s is a part of container resource %s", op->rsc_id, container);
             crm_xml_add(iter, XML_RSC_ATTR_CONTAINER, container);
         }
 
     } else {
         crm_warn("Resource %s no longer exists in the executor", op->rsc_id);
         controld_ack_event_directly(NULL, NULL, rsc, op, op->rsc_id);
         goto cleanup;
     }
 
     crm_log_xml_trace(update, __FUNCTION__);
 
     /* make it an asynchronous call and be done with it
      *
      * Best case:
      *   the resource state will be discovered during
      *   the next signup or election.
      *
      * Bad case:
      *   we are shutting down and there is no DC at the time,
      *   but then why were we shutting down then anyway?
      *   (probably because of an internal error)
      *
      * Worst case:
      *   we get shot for having resources "running" that really weren't
      *
      * the alternative however means blocking here for too long, which
      * isn't acceptable
      */
     fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, rc, NULL);
 
     if (rc > 0) {
         last_resource_update = rc;
     }
   done:
     /* the return code is a call number, not an error code */
     crm_trace("Sent resource state update message: %d for %s=%u on %s",
               rc, op->op_type, op->interval_ms, op->rsc_id);
     fsa_register_cib_callback(rc, FALSE, NULL, cib_rsc_callback);
 
   cleanup:
     free_xml(update);
     return rc;
 }
 
 void
 do_lrm_event(long long action,
              enum crmd_fsa_cause cause,
              enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
 {
     CRM_CHECK(FALSE, return);
 }
 
 static char *
 unescape_newlines(const char *string)
 {
     char *pch = NULL;
     char *ret = NULL;
     static const char *escaped_newline = "\\n";
 
     if (!string) {
         return NULL;
     }
 
     ret = strdup(string);
     pch = strstr(ret, escaped_newline);
     while (pch != NULL) {
         /* Replace newline escape pattern with actual newline (and a space so we
          * don't have to shuffle the rest of the buffer)
          */
         pch[0] = '\n';
         pch[1] = ' ';
         pch = strstr(pch, escaped_newline);
     }
 
     return ret;
 }
 
 static bool
 did_lrm_rsc_op_fail(lrm_state_t *lrm_state, const char * rsc_id,
                     const char * op_type, guint interval_ms)
 {
     rsc_history_t *entry = NULL;
 
     CRM_CHECK(lrm_state != NULL, return FALSE);
     CRM_CHECK(rsc_id != NULL, return FALSE);
     CRM_CHECK(op_type != NULL, return FALSE);
 
     entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
     if (entry == NULL || entry->failed == NULL) {
         return FALSE;
     }
 
     if (pcmk__str_eq(entry->failed->rsc_id, rsc_id, pcmk__str_none)
         && pcmk__str_eq(entry->failed->op_type, op_type, pcmk__str_casei)
         && entry->failed->interval_ms == interval_ms) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 void
 process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
                   active_op_t *pending, xmlNode *action_xml)
 {
     char *op_id = NULL;
     char *op_key = NULL;
 
     int update_id = 0;
     gboolean remove = FALSE;
     gboolean removed = FALSE;
     bool need_direct_ack = FALSE;
     lrmd_rsc_info_t *rsc = NULL;
     const char *node_name = NULL;
 
     CRM_CHECK(op != NULL, return);
     CRM_CHECK(op->rsc_id != NULL, return);
 
     // Remap new status codes for older DCs
     if (compare_version(fsa_our_dc_version, "3.2.0") < 0) {
         switch (op->op_status) {
             case PCMK_LRM_OP_NOT_CONNECTED:
                 op->op_status = PCMK_LRM_OP_ERROR;
                 op->rc = PCMK_OCF_CONNECTION_DIED;
                 break;
             case PCMK_LRM_OP_INVALID:
                 op->op_status = PCMK_LRM_OP_ERROR;
                 op->rc = CRM_DIRECT_NACK_RC;
                 break;
             default:
                 break;
         }
     }
 
     op_id = make_stop_id(op->rsc_id, op->call_id);
     op_key = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
 
     // Get resource info if available (from executor state or action XML)
     if (lrm_state) {
         rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
     }
     if ((rsc == NULL) && action_xml) {
         xmlNode *xml = find_xml_node(action_xml, XML_CIB_TAG_RESOURCE, TRUE);
 
         const char *standard = crm_element_value(xml, XML_AGENT_ATTR_CLASS);
         const char *provider = crm_element_value(xml, XML_AGENT_ATTR_PROVIDER);
         const char *type = crm_element_value(xml, XML_ATTR_TYPE);
 
         if (standard && type) {
             crm_info("%s agent information not cached, using %s%s%s:%s from action XML",
                      op->rsc_id, standard,
                      (provider? ":" : ""), (provider? provider : ""), type);
             rsc = lrmd_new_rsc_info(op->rsc_id, standard, provider, type);
         } else {
             crm_err("Can't process %s result because %s agent information not cached or in XML",
                     op_key, op->rsc_id);
         }
     }
 
     // Get node name if available (from executor state or action XML)
     if (lrm_state) {
         node_name = lrm_state->node_name;
     } else if (action_xml) {
         node_name = crm_element_value(action_xml, XML_LRM_ATTR_TARGET);
     }
 
     if(pending == NULL) {
         remove = TRUE;
         if (lrm_state) {
             pending = g_hash_table_lookup(lrm_state->pending_ops, op_id);
         }
     }
 
     if (op->op_status == PCMK_LRM_OP_ERROR) {
         switch(op->rc) {
             case PCMK_OCF_NOT_RUNNING:
             case PCMK_OCF_RUNNING_MASTER:
             case PCMK_OCF_DEGRADED:
             case PCMK_OCF_DEGRADED_MASTER:
                 // Leave it to the TE/scheduler to decide if this is an error
                 op->op_status = PCMK_LRM_OP_DONE;
                 break;
             default:
                 /* Nothing to do */
                 break;
         }
     }
 
     if (op->op_status != PCMK_LRM_OP_CANCELLED) {
         /* We might not record the result, so directly acknowledge it to the
          * originator instead, so it doesn't time out waiting for the result
          * (especially important if part of a transition).
          */
         need_direct_ack = TRUE;
 
         if (controld_action_is_recordable(op->op_type)) {
             if (node_name && rsc) {
                 // We should record the result, and happily, we can
                 update_id = do_update_resource(node_name, rsc, op,
                                                pending? pending->lock_time : 0);
                 need_direct_ack = FALSE;
 
             } else if (op->rsc_deleted) {
                 /* We shouldn't record the result (likely the resource was
                  * refreshed, cleaned, or removed while this operation was
                  * in flight).
                  */
                 crm_notice("Not recording %s result in CIB because "
                            "resource information was removed since it was initiated",
                            op_key);
             } else {
                 /* This shouldn't be possible; the executor didn't consider the
                  * resource deleted, but we couldn't find resource or node
                  * information.
                  */
                 crm_err("Unable to record %s result in CIB: %s", op_key,
                         (node_name? "No resource information" : "No node name"));
             }
         }
 
     } else if (op->interval_ms == 0) {
         /* A non-recurring operation was cancelled. Most likely, the
          * never-initiated action was removed from the executor's pending
          * operations list upon resource removal.
          */
         need_direct_ack = TRUE;
 
     } else if (pending == NULL) {
         /* This recurring operation was cancelled, but was not pending. No
          * transition actions are waiting on it, nothing needs to be done.
          */
 
     } else if (op->user_data == NULL) {
         /* This recurring operation was cancelled and pending, but we don't
          * have a transition key. This should never happen.
          */
         crm_err("Recurring operation %s was cancelled without transition information",
                 op_key);
 
     } else if (is_set(pending->flags, active_op_remove)) {
         /* This recurring operation was cancelled (by us) and pending, and we
          * have been waiting for it to finish.
          */
         if (lrm_state) {
             erase_lrm_history_by_op(lrm_state, op);
         }
 
         /* If the recurring operation had failed, the lrm_rsc_op is recorded as
          * "last_failure" which won't get erased from the cib given the logic on
          * purpose in erase_lrm_history_by_op(). So that the cancel action won't
          * have a chance to get confirmed by DC with process_op_deletion().
          * Cluster transition would get stuck waiting for the remaining action
          * timer to time out.
          *
          * Directly acknowledge the cancel operation in this case.
          */
         if (did_lrm_rsc_op_fail(lrm_state, pending->rsc_id,
                                 pending->op_type, pending->interval_ms)) {
             need_direct_ack = TRUE;
         }
 
     } else if (op->rsc_deleted) {
         /* This recurring operation was cancelled (but not by us, and the
          * executor does not have resource information, likely due to resource
          * cleanup, refresh, or removal) and pending.
          */
         crm_debug("Recurring op %s was cancelled due to resource deletion",
                   op_key);
         need_direct_ack = TRUE;
 
     } else {
         /* This recurring operation was cancelled (but not by us, likely by the
          * executor before stopping the resource) and pending. We don't need to
          * do anything special.
          */
     }
 
     if (need_direct_ack) {
         controld_ack_event_directly(NULL, NULL, NULL, op, op->rsc_id);
     }
 
     if(remove == FALSE) {
         /* The caller will do this afterwards, but keep the logging consistent */
         removed = TRUE;
 
     } else if (lrm_state && ((op->interval_ms == 0)
                              || (op->op_status == PCMK_LRM_OP_CANCELLED))) {
 
         gboolean found = g_hash_table_remove(lrm_state->pending_ops, op_id);
 
         if (op->interval_ms != 0) {
             removed = TRUE;
         } else if (found) {
             removed = TRUE;
             crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
                       op_key, op->call_id, op_id,
                       g_hash_table_size(lrm_state->pending_ops));
         }
     }
 
     if (node_name == NULL) {
         node_name = "unknown node"; // for logging
     }
 
     switch (op->op_status) {
         case PCMK_LRM_OP_CANCELLED:
             crm_info("Result of %s operation for %s on %s: %s "
                      CRM_XS " call=%d key=%s confirmed=%s",
                      crm_action_str(op->op_type, op->interval_ms),
                      op->rsc_id, node_name,
                      services_lrm_status_str(op->op_status),
                      op->call_id, op_key, (removed? "true" : "false"));
             break;
 
         case PCMK_LRM_OP_DONE:
             crm_notice("Result of %s operation for %s on %s: %s "
                        CRM_XS " rc=%d call=%d key=%s confirmed=%s cib-update=%d",
                        crm_action_str(op->op_type, op->interval_ms),
                        op->rsc_id, node_name,
                        services_ocf_exitcode_str(op->rc), op->rc,
                        op->call_id, op_key, (removed? "true" : "false"),
                        update_id);
             break;
 
         case PCMK_LRM_OP_TIMEOUT:
             crm_err("Result of %s operation for %s on %s: %s "
                     CRM_XS " call=%d key=%s timeout=%dms",
                     crm_action_str(op->op_type, op->interval_ms),
                     op->rsc_id, node_name,
                     services_lrm_status_str(op->op_status),
                     op->call_id, op_key, op->timeout);
             break;
 
         default:
             crm_err("Result of %s operation for %s on %s: %s "
                     CRM_XS " call=%d key=%s confirmed=%s status=%d cib-update=%d",
                     crm_action_str(op->op_type, op->interval_ms),
                     op->rsc_id, node_name,
                     services_lrm_status_str(op->op_status), op->call_id, op_key,
                     (removed? "true" : "false"), op->op_status, update_id);
     }
 
     if (op->output) {
         char *prefix =
             crm_strdup_printf("%s-" PCMK__OP_FMT ":%d", node_name,
                               op->rsc_id, op->op_type, op->interval_ms,
                               op->call_id);
 
         if (op->rc) {
             crm_log_output(LOG_NOTICE, prefix, op->output);
         } else {
             crm_log_output(LOG_DEBUG, prefix, op->output);
         }
         free(prefix);
     }
 
     if (lrm_state) {
         if (!pcmk__str_eq(op->op_type, RSC_METADATA, pcmk__str_casei)) {
             crmd_alert_resource_op(lrm_state->node_name, op);
         } else if (rsc && (op->rc == PCMK_OCF_OK)) {
             char *metadata = unescape_newlines(op->output);
 
             metadata_cache_update(lrm_state->metadata_cache, rsc, metadata);
             free(metadata);
         }
     }
 
     if (op->rsc_deleted) {
         crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
         if (lrm_state) {
             delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL);
         }
     }
 
     /* If a shutdown was escalated while operations were pending,
      * then the FSA will be stalled right now... allow it to continue
      */
     mainloop_set_trigger(fsa_source);
     if (lrm_state && rsc) {
         update_history_cache(lrm_state, rsc, op);
     }
 
     lrmd_free_rsc_info(rsc);
     free(op_key);
     free(op_id);
 }
diff --git a/daemons/controld/controld_execd_state.c b/daemons/controld/controld_execd_state.c
index 062c83fd32..61e2acd409 100644
--- a/daemons/controld/controld_execd_state.c
+++ b/daemons/controld/controld_execd_state.c
@@ -1,826 +1,826 @@
 /*
  * Copyright 2012-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/iso8601.h>
 #include <crm/pengine/rules.h>
 #include <crm/pengine/rules_internal.h>
 #include <crm/lrmd_internal.h>
 
 #include <pacemaker-internal.h>
 #include <pacemaker-controld.h>
 
 GHashTable *lrm_state_table = NULL;
 extern GHashTable *proxy_table;
 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
 
 static void
 free_rsc_info(gpointer value)
 {
     lrmd_rsc_info_t *rsc_info = value;
 
     lrmd_free_rsc_info(rsc_info);
 }
 
 static void
 free_deletion_op(gpointer value)
 {
     struct pending_deletion_op_s *op = value;
 
     free(op->rsc);
     delete_ha_msg_input(op->input);
     free(op);
 }
 
 static void
 free_recurring_op(gpointer value)
 {
     active_op_t *op = value;
 
     free(op->user_data);
     free(op->rsc_id);
     free(op->op_type);
     free(op->op_key);
     if (op->params) {
         g_hash_table_destroy(op->params);
     }
     free(op);
 }
 
 static gboolean
 fail_pending_op(gpointer key, gpointer value, gpointer user_data)
 {
     lrmd_event_data_t event = { 0, };
     lrm_state_t *lrm_state = user_data;
     active_op_t *op = value;
 
     crm_trace("Pre-emptively failing " PCMK__OP_FMT " on %s (call=%s, %s)",
               op->rsc_id, op->op_type, op->interval_ms,
               lrm_state->node_name, (char*)key, op->user_data);
 
     event.type = lrmd_event_exec_complete;
     event.rsc_id = op->rsc_id;
     event.op_type = op->op_type;
     event.user_data = op->user_data;
     event.timeout = 0;
     event.interval_ms = op->interval_ms;
     event.rc = PCMK_OCF_UNKNOWN_ERROR;
     event.op_status = PCMK_LRM_OP_NOT_CONNECTED;
     event.t_run = (unsigned int) op->start_time;
     event.t_rcchange = (unsigned int) op->start_time;
 
     event.call_id = op->call_id;
     event.remote_nodename = lrm_state->node_name;
     event.params = op->params;
 
     process_lrm_event(lrm_state, &event, op, NULL);
     return TRUE;
 }
 
 gboolean
 lrm_state_is_local(lrm_state_t *lrm_state)
 {
     if (lrm_state == NULL || fsa_our_uname == NULL) {
         return FALSE;
     }
 
     if (strcmp(lrm_state->node_name, fsa_our_uname) != 0) {
         return FALSE;
     }
 
     return TRUE;
 
 }
 
 lrm_state_t *
 lrm_state_create(const char *node_name)
 {
     lrm_state_t *state = NULL;
 
     if (!node_name) {
         crm_err("No node name given for lrm state object");
         return NULL;
     }
 
     state = calloc(1, sizeof(lrm_state_t));
     if (!state) {
         return NULL;
     }
 
     state->node_name = strdup(node_name);
 
     state->rsc_info_cache = g_hash_table_new_full(crm_str_hash,
                                                 g_str_equal, NULL, free_rsc_info);
 
     state->deletion_ops = g_hash_table_new_full(crm_str_hash, g_str_equal, free,
                                                 free_deletion_op);
 
     state->pending_ops = g_hash_table_new_full(crm_str_hash, g_str_equal, free,
                                                free_recurring_op);
 
     state->resource_history = g_hash_table_new_full(crm_str_hash,
                                                     g_str_equal, NULL, history_free);
 
     state->metadata_cache = metadata_cache_new();
 
     g_hash_table_insert(lrm_state_table, (char *)state->node_name, state);
     return state;
 
 }
 
 void
 lrm_state_destroy(const char *node_name)
 {
     g_hash_table_remove(lrm_state_table, node_name);
 }
 
 static gboolean
 remote_proxy_remove_by_node(gpointer key, gpointer value, gpointer user_data)
 {
     remote_proxy_t *proxy = value;
     const char *node_name = user_data;
 
     if (pcmk__str_eq(node_name, proxy->node_name, pcmk__str_casei)) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 static void
 internal_lrm_state_destroy(gpointer data)
 {
     lrm_state_t *lrm_state = data;
 
     if (!lrm_state) {
         return;
     }
 
     crm_trace("Destroying proxy table %s with %d members", lrm_state->node_name, g_hash_table_size(proxy_table));
     g_hash_table_foreach_remove(proxy_table, remote_proxy_remove_by_node, (char *) lrm_state->node_name);
     remote_ra_cleanup(lrm_state);
     lrmd_api_delete(lrm_state->conn);
 
     if (lrm_state->rsc_info_cache) {
         crm_trace("Destroying rsc info cache with %d members", g_hash_table_size(lrm_state->rsc_info_cache));
         g_hash_table_destroy(lrm_state->rsc_info_cache);
     }
     if (lrm_state->resource_history) {
         crm_trace("Destroying history op cache with %d members", g_hash_table_size(lrm_state->resource_history));
         g_hash_table_destroy(lrm_state->resource_history);
     }
     if (lrm_state->deletion_ops) {
         crm_trace("Destroying deletion op cache with %d members", g_hash_table_size(lrm_state->deletion_ops));
         g_hash_table_destroy(lrm_state->deletion_ops);
     }
     if (lrm_state->pending_ops) {
         crm_trace("Destroying pending op cache with %d members", g_hash_table_size(lrm_state->pending_ops));
         g_hash_table_destroy(lrm_state->pending_ops);
     }
     metadata_cache_free(lrm_state->metadata_cache);
 
     free((char *)lrm_state->node_name);
     free(lrm_state);
 }
 
 void
 lrm_state_reset_tables(lrm_state_t * lrm_state, gboolean reset_metadata)
 {
     if (lrm_state->resource_history) {
         crm_trace("Re-setting history op cache with %d members",
                   g_hash_table_size(lrm_state->resource_history));
         g_hash_table_remove_all(lrm_state->resource_history);
     }
     if (lrm_state->deletion_ops) {
         crm_trace("Re-setting deletion op cache with %d members",
                   g_hash_table_size(lrm_state->deletion_ops));
         g_hash_table_remove_all(lrm_state->deletion_ops);
     }
     if (lrm_state->pending_ops) {
         crm_trace("Re-setting pending op cache with %d members",
                   g_hash_table_size(lrm_state->pending_ops));
         g_hash_table_remove_all(lrm_state->pending_ops);
     }
     if (lrm_state->rsc_info_cache) {
         crm_trace("Re-setting rsc info cache with %d members",
                   g_hash_table_size(lrm_state->rsc_info_cache));
         g_hash_table_remove_all(lrm_state->rsc_info_cache);
     }
     if (reset_metadata) {
         metadata_cache_reset(lrm_state->metadata_cache);
     }
 }
 
 gboolean
 lrm_state_init_local(void)
 {
     if (lrm_state_table) {
         return TRUE;
     }
 
     lrm_state_table =
         g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, internal_lrm_state_destroy);
     if (!lrm_state_table) {
         return FALSE;
     }
 
     proxy_table =
         g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, remote_proxy_free);
     if (!proxy_table) {
         g_hash_table_destroy(lrm_state_table);
         lrm_state_table = NULL;
         return FALSE;
     }
 
     return TRUE;
 }
 
 void
 lrm_state_destroy_all(void)
 {
     if (lrm_state_table) {
         crm_trace("Destroying state table with %d members", g_hash_table_size(lrm_state_table));
         g_hash_table_destroy(lrm_state_table); lrm_state_table = NULL;
     }
     if(proxy_table) {
         crm_trace("Destroying proxy table with %d members", g_hash_table_size(proxy_table));
         g_hash_table_destroy(proxy_table); proxy_table = NULL;
     }
 }
 
 lrm_state_t *
 lrm_state_find(const char *node_name)
 {
     if (!node_name) {
         return NULL;
     }
     return g_hash_table_lookup(lrm_state_table, node_name);
 }
 
 lrm_state_t *
 lrm_state_find_or_create(const char *node_name)
 {
     lrm_state_t *lrm_state;
 
     lrm_state = g_hash_table_lookup(lrm_state_table, node_name);
     if (!lrm_state) {
         lrm_state = lrm_state_create(node_name);
     }
 
     return lrm_state;
 }
 
 GList *
 lrm_state_get_list(void)
 {
     return g_hash_table_get_values(lrm_state_table);
 }
 
 static remote_proxy_t *
 find_connected_proxy_by_node(const char * node_name)
 {
     GHashTableIter gIter;
     remote_proxy_t *proxy = NULL;
 
     CRM_CHECK(proxy_table != NULL, return NULL);
 
     g_hash_table_iter_init(&gIter, proxy_table);
 
     while (g_hash_table_iter_next(&gIter, NULL, (gpointer *) &proxy)) {
         if (proxy->source
             && pcmk__str_eq(node_name, proxy->node_name, pcmk__str_casei)) {
             return proxy;
         }
     }
 
     return NULL;
 }
 
 static void
 remote_proxy_disconnect_by_node(const char * node_name)
 {
     remote_proxy_t *proxy = NULL;
 
     CRM_CHECK(proxy_table != NULL, return);
 
     while ((proxy = find_connected_proxy_by_node(node_name)) != NULL) {
         /* mainloop_del_ipc_client() eventually calls remote_proxy_disconnected()
          * , which removes the entry from proxy_table.
          * Do not do this in a g_hash_table_iter_next() loop. */
         if (proxy->source) {
             mainloop_del_ipc_client(proxy->source);
         }
     }
 
     return;
 }
 
 void
 lrm_state_disconnect_only(lrm_state_t * lrm_state)
 {
     int removed = 0;
 
     if (!lrm_state->conn) {
         return;
     }
     crm_trace("Disconnecting %s", lrm_state->node_name);
 
     remote_proxy_disconnect_by_node(lrm_state->node_name);
 
     ((lrmd_t *) lrm_state->conn)->cmds->disconnect(lrm_state->conn);
 
     if (is_not_set(fsa_input_register, R_SHUTDOWN)) {
         removed = g_hash_table_foreach_remove(lrm_state->pending_ops, fail_pending_op, lrm_state);
         crm_trace("Synthesized %d operation failures for %s", removed, lrm_state->node_name);
     }
 }
 
 void
 lrm_state_disconnect(lrm_state_t * lrm_state)
 {
     if (!lrm_state->conn) {
         return;
     }
 
     lrm_state_disconnect_only(lrm_state);
 
     lrmd_api_delete(lrm_state->conn);
     lrm_state->conn = NULL;
 }
 
 int
 lrm_state_is_connected(lrm_state_t * lrm_state)
 {
     if (!lrm_state->conn) {
         return FALSE;
     }
     return ((lrmd_t *) lrm_state->conn)->cmds->is_connected(lrm_state->conn);
 }
 
 int
 lrm_state_poke_connection(lrm_state_t * lrm_state)
 {
 
     if (!lrm_state->conn) {
         return -1;
     }
     return ((lrmd_t *) lrm_state->conn)->cmds->poke_connection(lrm_state->conn);
 }
 
 int
 lrm_state_ipc_connect(lrm_state_t * lrm_state)
 {
     int ret;
 
     if (!lrm_state->conn) {
         lrm_state->conn = lrmd_api_new();
         ((lrmd_t *) lrm_state->conn)->cmds->set_callback(lrm_state->conn, lrm_op_callback);
     }
 
     ret = ((lrmd_t *) lrm_state->conn)->cmds->connect(lrm_state->conn, CRM_SYSTEM_CRMD, NULL);
 
     if (ret != pcmk_ok) {
         lrm_state->num_lrm_register_fails++;
     } else {
         lrm_state->num_lrm_register_fails = 0;
     }
 
     return ret;
 }
 
 static remote_proxy_t *
 crmd_remote_proxy_new(lrmd_t *lrmd, const char *node_name, const char *session_id, const char *channel)
 {
     struct ipc_client_callbacks proxy_callbacks = {
         .dispatch = remote_proxy_dispatch,
         .destroy = remote_proxy_disconnected
     };
     remote_proxy_t *proxy = remote_proxy_new(lrmd, &proxy_callbacks, node_name,
                                              session_id, channel);
     return proxy;
 }
 
 gboolean
 crmd_is_proxy_session(const char *session)
 {
     return g_hash_table_lookup(proxy_table, session) ? TRUE : FALSE;
 }
 
 void
 crmd_proxy_send(const char *session, xmlNode *msg)
 {
     remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);
     lrm_state_t *lrm_state = NULL;
 
     if (!proxy) {
         return;
     }
     crm_log_xml_trace(msg, "to-proxy");
     lrm_state = lrm_state_find(proxy->node_name);
     if (lrm_state) {
         crm_trace("Sending event to %.8s on %s", proxy->session_id, proxy->node_name);
         remote_proxy_relay_event(proxy, msg);
     }
 }
 
 static void
 crmd_proxy_dispatch(const char *session, xmlNode *msg)
 {
     crm_trace("Processing proxied IPC message from session %s", session);
     crm_log_xml_trace(msg, "controller[inbound]");
     crm_xml_add(msg, F_CRM_SYS_FROM, session);
     if (controld_authorize_ipc_message(msg, NULL, session)) {
         route_message(C_IPC_MESSAGE, msg);
     }
-    trigger_fsa(fsa_source);
+    trigger_fsa();
 }
 
 static void
 remote_config_check(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     if (rc != pcmk_ok) {
         crm_err("Query resulted in an error: %s", pcmk_strerror(rc));
 
         if (rc == -EACCES || rc == -pcmk_err_schema_validation) {
             crm_err("The cluster is mis-configured - shutting down and staying down");
         }
 
     } else {
         lrmd_t * lrmd = (lrmd_t *)user_data;
         crm_time_t *now = crm_time_new(NULL);
         GHashTable *config_hash = crm_str_table_new();
 
         crm_debug("Call %d : Parsing CIB options", call_id);
 
         pe_unpack_nvpairs(output, output, XML_CIB_TAG_PROPSET, NULL,
                           config_hash, CIB_OPTIONS_FIRST, FALSE, now, NULL);
 
         /* Now send it to the remote peer */
         remote_proxy_check(lrmd, config_hash);
 
         g_hash_table_destroy(config_hash);
         crm_time_free(now);
     }
 }
 
 static void
 crmd_remote_proxy_cb(lrmd_t *lrmd, void *userdata, xmlNode *msg)
 {
     lrm_state_t *lrm_state = userdata;
     const char *session = crm_element_value(msg, F_LRMD_IPC_SESSION);
     remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);
 
     const char *op = crm_element_value(msg, F_LRMD_IPC_OP);
     if (pcmk__str_eq(op, LRMD_IPC_OP_NEW, pcmk__str_casei)) {
         const char *channel = crm_element_value(msg, F_LRMD_IPC_IPC_SERVER);
 
         proxy = crmd_remote_proxy_new(lrmd, lrm_state->node_name, session, channel);
         if (!remote_ra_controlling_guest(lrm_state)) {
             if (proxy != NULL) {
                 /* Look up stonith-watchdog-timeout and send to the remote peer for validation */
                 int rc = fsa_cib_conn->cmds->query(fsa_cib_conn, XML_CIB_TAG_CRMCONFIG, NULL, cib_scope_local);
                 fsa_cib_conn->cmds->register_callback_full(fsa_cib_conn, rc, 10, FALSE, lrmd,
                                                         "remote_config_check", remote_config_check, NULL);
             }
         } else {
             crm_debug("Skipping remote_config_check for guest-nodes");
         }
 
     } else if (pcmk__str_eq(op, LRMD_IPC_OP_SHUTDOWN_REQ, pcmk__str_casei)) {
         char *now_s = NULL;
         time_t now = time(NULL);
 
         crm_notice("%s requested shutdown of its remote connection",
                    lrm_state->node_name);
 
         if (!remote_ra_is_in_maintenance(lrm_state)) {
             now_s = crm_itoa(now);
             update_attrd(lrm_state->node_name, XML_CIB_ATTR_SHUTDOWN, now_s, NULL, TRUE);
             free(now_s);
 
             remote_proxy_ack_shutdown(lrmd);
 
             crm_warn("Reconnection attempts to %s may result in failures that must be cleared",
                     lrm_state->node_name);
         } else {
             remote_proxy_nack_shutdown(lrmd);
 
             crm_notice("Remote resource for %s is not managed so no ordered shutdown happening",
                     lrm_state->node_name);
         }
         return;
 
     } else if (pcmk__str_eq(op, LRMD_IPC_OP_REQUEST, pcmk__str_casei) && proxy && proxy->is_local) {
         /* This is for the controller, which we are, so don't try
          * to send to ourselves over IPC -- do it directly.
          */
         int flags = 0;
         xmlNode *request = get_message_xml(msg, F_LRMD_IPC_MSG);
 
         CRM_CHECK(request != NULL, return);
 #if ENABLE_ACL
         CRM_CHECK(lrm_state->node_name, return);
         crm_xml_add(request, XML_ACL_TAG_ROLE, "pacemaker-remote");
         pcmk__update_acl_user(request, F_LRMD_IPC_USER, lrm_state->node_name);
 #endif
 
         /* Pacemaker Remote nodes don't know their own names (as known to the
          * cluster). When getting a node info request with no name or ID, add
          * the name, so we don't return info for ourselves instead of the
          * Pacemaker Remote node.
          */
         if (pcmk__str_eq(crm_element_value(request, F_CRM_TASK), CRM_OP_NODE_INFO, pcmk__str_casei)) {
             int node_id = 0;
 
             crm_element_value_int(request, XML_ATTR_ID, &node_id);
             if ((node_id <= 0)
                 && (crm_element_value(request, XML_ATTR_UNAME) == NULL)) {
                 crm_xml_add(request, XML_ATTR_UNAME, lrm_state->node_name);
             }
         }
 
         crmd_proxy_dispatch(session, request);
 
         crm_element_value_int(msg, F_LRMD_IPC_MSG_FLAGS, &flags);
         if (flags & crm_ipc_client_response) {
             int msg_id = 0;
             xmlNode *op_reply = create_xml_node(NULL, "ack");
 
             crm_xml_add(op_reply, "function", __FUNCTION__);
             crm_xml_add_int(op_reply, "line", __LINE__);
 
             crm_element_value_int(msg, F_LRMD_IPC_MSG_ID, &msg_id);
             remote_proxy_relay_response(proxy, op_reply, msg_id);
 
             free_xml(op_reply);
         }
 
     } else {
         remote_proxy_cb(lrmd, lrm_state->node_name, msg);
     }
 }
 
 
 int
 lrm_state_remote_connect_async(lrm_state_t * lrm_state, const char *server, int port,
                                int timeout_ms)
 {
     int ret;
 
     if (!lrm_state->conn) {
         lrm_state->conn = lrmd_remote_api_new(lrm_state->node_name, server, port);
         if (!lrm_state->conn) {
             return -1;
         }
         ((lrmd_t *) lrm_state->conn)->cmds->set_callback(lrm_state->conn, remote_lrm_op_callback);
         lrmd_internal_set_proxy_callback(lrm_state->conn, lrm_state, crmd_remote_proxy_cb);
     }
 
     crm_trace("initiating remote connection to %s at %d with timeout %d", server, port, timeout_ms);
     ret =
         ((lrmd_t *) lrm_state->conn)->cmds->connect_async(lrm_state->conn, lrm_state->node_name,
                                                           timeout_ms);
 
     if (ret != pcmk_ok) {
         lrm_state->num_lrm_register_fails++;
     } else {
         lrm_state->num_lrm_register_fails = 0;
     }
 
     return ret;
 }
 
 int
 lrm_state_get_metadata(lrm_state_t * lrm_state,
                        const char *class,
                        const char *provider,
                        const char *agent, char **output, enum lrmd_call_options options)
 {
     lrmd_key_value_t *params = NULL;
 
     if (!lrm_state->conn) {
         return -ENOTCONN;
     }
 
     /* Add the node name to the environment, as is done with normal resource
      * action calls. Meta-data calls shouldn't need it, but some agents are
      * written with an ocf_local_nodename call at the beginning regardless of
      * action. Without the environment variable, the agent would try to contact
      * the controller to get the node name -- but the controller would be
      * blocking on the synchronous meta-data call.
      *
      * At this point, we have to assume that agents are unlikely to make other
      * calls that require the controller, such as crm_node --quorum or
      * --cluster-id.
      *
      * @TODO Make meta-data calls asynchronous. (This will be part of a larger
      * project to make meta-data calls via the executor rather than directly.)
      */
     params = lrmd_key_value_add(params, CRM_META "_" XML_LRM_ATTR_TARGET,
                                 lrm_state->node_name);
 
     return ((lrmd_t *) lrm_state->conn)->cmds->get_metadata_params(lrm_state->conn,
             class, provider, agent, output, options, params);
 }
 
 int
 lrm_state_cancel(lrm_state_t *lrm_state, const char *rsc_id, const char *action,
                  guint interval_ms)
 {
     if (!lrm_state->conn) {
         return -ENOTCONN;
     }
 
     /* Figure out a way to make this async?
      * NOTICE: Currently it's synced and directly acknowledged in do_lrm_invoke(). */
     if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
         return remote_ra_cancel(lrm_state, rsc_id, action, interval_ms);
     }
     return ((lrmd_t *) lrm_state->conn)->cmds->cancel(lrm_state->conn, rsc_id,
                                                       action, interval_ms);
 }
 
 lrmd_rsc_info_t *
 lrm_state_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id, enum lrmd_call_options options)
 {
     lrmd_rsc_info_t *rsc = NULL;
 
     if (!lrm_state->conn) {
         return NULL;
     }
     if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
         return remote_ra_get_rsc_info(lrm_state, rsc_id);
     }
 
     rsc = g_hash_table_lookup(lrm_state->rsc_info_cache, rsc_id);
     if (rsc == NULL) {
         /* only contact the lrmd if we don't already have a cached rsc info */
         rsc = ((lrmd_t *) lrm_state->conn)->cmds->get_rsc_info(lrm_state->conn, rsc_id, options);
         if (rsc == NULL) {
 		    return NULL;
         }
         /* cache the result */
         g_hash_table_insert(lrm_state->rsc_info_cache, rsc->id, rsc);
     }
 
     return lrmd_copy_rsc_info(rsc);
 
 }
 
 int
 lrm_state_exec(lrm_state_t *lrm_state, const char *rsc_id, const char *action,
                const char *userdata, guint interval_ms,
                int timeout,     /* ms */
                int start_delay, /* ms */
                lrmd_key_value_t * params)
 {
 
     if (!lrm_state->conn) {
         lrmd_key_value_freeall(params);
         return -ENOTCONN;
     }
 
     if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
         return remote_ra_exec(lrm_state, rsc_id, action, userdata, interval_ms,
                               timeout, start_delay, params);
     }
 
     return ((lrmd_t *) lrm_state->conn)->cmds->exec(lrm_state->conn,
                                                     rsc_id,
                                                     action,
                                                     userdata,
                                                     interval_ms,
                                                     timeout,
                                                     start_delay,
                                                     lrmd_opt_notify_changes_only, params);
 }
 
 int
 lrm_state_register_rsc(lrm_state_t * lrm_state,
                        const char *rsc_id,
                        const char *class,
                        const char *provider, const char *agent, enum lrmd_call_options options)
 {
     lrmd_t *conn = (lrmd_t *) lrm_state->conn;
 
     if (conn == NULL) {
         return -ENOTCONN;
     }
 
     if (is_remote_lrmd_ra(agent, provider, NULL)) {
         return lrm_state_find_or_create(rsc_id)? pcmk_ok : -EINVAL;
     }
 
     /* @TODO Implement an asynchronous version of this (currently a blocking
      * call to the lrmd).
      */
     return conn->cmds->register_rsc(lrm_state->conn, rsc_id, class, provider,
                                     agent, options);
 }
 
 int
 lrm_state_unregister_rsc(lrm_state_t * lrm_state,
                          const char *rsc_id, enum lrmd_call_options options)
 {
     if (!lrm_state->conn) {
         return -ENOTCONN;
     }
 
     if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
         lrm_state_destroy(rsc_id);
         return pcmk_ok;
     }
 
     g_hash_table_remove(lrm_state->rsc_info_cache, rsc_id);
 
     /* @TODO Optimize this ... this function is a blocking round trip from
      * client to daemon. The controld_execd_state.c code path that uses this
      * function should always treat it as an async operation. The executor API
      * should make an async version available.
      */
     return ((lrmd_t *) lrm_state->conn)->cmds->unregister_rsc(lrm_state->conn, rsc_id, options);
 }
 
 /*
  * Functions for sending alerts via local executor connection
  */
 
 static GListPtr crmd_alert_list = NULL;
 
 void
 crmd_unpack_alerts(xmlNode *alerts)
 {
     pe_free_alert_list(crmd_alert_list);
     crmd_alert_list = pe_unpack_alerts(alerts);
 }
 
 void
 crmd_alert_node_event(crm_node_t *node)
 {
     lrm_state_t *lrm_state;
 
     if (crmd_alert_list == NULL) {
         return;
     }
 
     lrm_state = lrm_state_find(fsa_our_uname);
     if (lrm_state == NULL) {
         return;
     }
 
     lrmd_send_node_alert((lrmd_t *) lrm_state->conn, crmd_alert_list,
                          node->uname, node->id, node->state);
 }
 
 void
 crmd_alert_fencing_op(stonith_event_t * e)
 {
     char *desc;
     lrm_state_t *lrm_state;
 
     if (crmd_alert_list == NULL) {
         return;
     }
 
     lrm_state = lrm_state_find(fsa_our_uname);
     if (lrm_state == NULL) {
         return;
     }
 
     desc = crm_strdup_printf("Operation %s of %s by %s for %s@%s: %s (ref=%s)",
                              e->action, e->target,
                              (e->executioner? e->executioner : "<no-one>"),
                              e->client_origin, e->origin,
                              pcmk_strerror(e->result), e->id);
 
     lrmd_send_fencing_alert((lrmd_t *) lrm_state->conn, crmd_alert_list,
                             e->target, e->operation, desc, e->result);
     free(desc);
 }
 
 void
 crmd_alert_resource_op(const char *node, lrmd_event_data_t * op)
 {
     lrm_state_t *lrm_state;
 
     if (crmd_alert_list == NULL) {
         return;
     }
 
     lrm_state = lrm_state_find(fsa_our_uname);
     if (lrm_state == NULL) {
         return;
     }
 
     lrmd_send_resource_alert((lrmd_t *) lrm_state->conn, crmd_alert_list, node,
                              op);
 }
diff --git a/daemons/controld/controld_fsa.h b/daemons/controld/controld_fsa.h
index 9ee59f8abd..ce492cd2dd 100644
--- a/daemons/controld/controld_fsa.h
+++ b/daemons/controld/controld_fsa.h
@@ -1,700 +1,702 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef CRMD_FSA__H
 #  define CRMD_FSA__H
 
 #  include <crm/crm.h>
 #  include <crm/cib.h>
 #  include <crm/common/xml.h>
 #  include <crm/common/mainloop.h>
 #  include <crm/cluster.h>
 #  include <crm/cluster/election.h>
 #  include <crm/common/ipc_internal.h>
 
 /*! States the controller can be in */
 enum crmd_fsa_state {
     S_IDLE = 0,                 /* Nothing happening */
 
     S_ELECTION,                 /* Take part in the election algorithm as
                                  * described below
                                  */
     S_INTEGRATION,              /* integrate that status of new nodes (which is
                                  * all of them if we have just been elected DC)
                                  * to form a complete and up-to-date picture of
                                  * the CIB
                                  */
     S_FINALIZE_JOIN,            /* integrate that status of new nodes (which is
                                  * all of them if we have just been elected DC)
                                  * to form a complete and up-to-date picture of
                                  * the CIB
                                  */
     S_NOT_DC,                   /* we are in non-DC mode */
     S_POLICY_ENGINE,            /* Determine next stable state of the cluster */
     S_RECOVERY,                 /* Something bad happened, check everything is ok
                                  * before continuing and attempt to recover if
                                  * required
                                  */
     S_RELEASE_DC,               /* we were the DC, but now we arent anymore,
                                  * possibly by our own request, and we should
                                  * release all unnecessary sub-systems, finish
                                  * any pending actions, do general cleanup and
                                  * unset anything that makes us think we are
                                  * special :)
                                  */
     S_STARTING,                 /* we are just starting out */
     S_PENDING,                  /* we are not a full/active member yet */
     S_STOPPING,                 /* We are in the final stages of shutting down */
     S_TERMINATE,                /* We are going to shutdown, this is the equiv of
                                  * "Sending TERM signal to all processes" in Linux
                                  * and in worst case scenarios could be considered
                                  * a self STONITH
                                  */
     S_TRANSITION_ENGINE,        /* Attempt to make the calculated next stable
                                  * state of the cluster a reality
                                  */
 
     S_HALT,                     /* Freeze - don't do anything
                                  * Something bad happened that needs the admin to fix
                                  * Wait for I_ELECTION
                                  */
 
     /*  ----------- Last input found in table is above ---------- */
     S_ILLEGAL                   /* This is an illegal FSA state */
         /* (must be last) */
 };
 
 #  define MAXSTATE S_ILLEGAL
 
 /*
       Once we start and do some basic sanity checks, we go into the
       S_NOT_DC state and await instructions from the DC or input from
       the cluster layer which indicates the election algorithm needs to run.
 
       If the election algorithm is triggered, we enter the S_ELECTION state
       from where we can either go back to the S_NOT_DC state or progress
       to the S_INTEGRATION state (or S_RELEASE_DC if we used to be the DC
       but aren't anymore). See the libcrmcluster API documentation for more
       information about the election algorithm.
 
       Once the election is complete, if we are the DC, we enter the
       S_INTEGRATION state which is a DC-in-waiting style state.  We are
       the DC, but we shouldn't do anything yet because we may not have an
       up-to-date picture of the cluster.  There may of course be times
       when this fails, so we should go back to the S_RECOVERY stage and
       check everything is ok.  We may also end up here if a new node came
       online, since each node is authoritative about itself, and we would want
       to incorporate its information into the CIB.
 
       Once we have the latest CIB, we then enter the S_POLICY_ENGINE state
       where invoke the scheduler. It is possible that between
       invoking the scheduler and receiving an answer, that we receive
       more input. In this case, we would discard the orginal result and
       invoke it again.
 
       Once we are satisfied with the output from the scheduler, we
       enter S_TRANSITION_ENGINE and feed the scheduler's output to the
       Transition Engine who attempts to make the scheduler's
       calculation a reality. If the transition completes successfully,
       we enter S_IDLE, otherwise we go back to S_POLICY_ENGINE with the
       current unstable state and try again.
 
       Of course, we may be asked to shutdown at any time, however we must
       progress to S_NOT_DC before doing so.  Once we have handed over DC
       duties to another node, we can then shut down like everyone else,
       that is, by asking the DC for permission and waiting for it to take all
       our resources away.
 
       The case where we are the DC and the only node in the cluster is a
       special case and handled as an escalation which takes us to
       S_SHUTDOWN. Similarly, if any other point in the shutdown
       fails or stalls, this is escalated and we end up in S_TERMINATE.
 
       At any point, the controller can relay messages for its subsystems,
       but outbound messages (from subsystems) should probably be blocked
       until S_INTEGRATION (for the DC) or the join protocol has
       completed (for non-DC controllers).
 */
 
 /*======================================
  *
  *  Inputs/Events/Stimuli to be given to the finite state machine
  *
  *  Some of these a true events, and others are synthesised based on
  *  the "register" (see below) and the contents or source of messages.
  *
  *  The machine keeps processing until receiving I_NULL
  *
  *======================================*/
 enum crmd_fsa_input {
 /* 0 */
     I_NULL,                     /* Nothing happened */
 /* 1 */
 
     I_CIB_OP,                   /* An update to the CIB occurred */
     I_CIB_UPDATE,               /* An update to the CIB occurred */
     I_DC_TIMEOUT,               /* We have lost communication with the DC */
     I_ELECTION,                 /* Someone started an election */
     I_PE_CALC,                  /* The scheduler needs to be invoked */
     I_RELEASE_DC,               /* The election completed and we were not
                                  * elected, but we were the DC beforehand
                                  */
     I_ELECTION_DC,              /* The election completed and we were (re-)elected
                                  * DC
                                  */
     I_ERROR,                    /* Something bad happened (more serious than
                                  * I_FAIL) and may not have been due to the action
                                  * being performed.  For example, we may have lost
                                  * our connection to the CIB.
                                  */
 /* 9 */
     I_FAIL,                     /* The action failed to complete successfully */
     I_INTEGRATED,
     I_FINALIZED,
     I_NODE_JOIN,                /* A node has entered the cluster */
     I_NOT_DC,                   /* We are not and were not the DC before or after
                                  * the current operation or state
                                  */
     I_RECOVERED,                /* The recovery process completed successfully */
     I_RELEASE_FAIL,             /* We could not give up DC status for some reason
                                  */
     I_RELEASE_SUCCESS,          /* We are no longer the DC */
     I_RESTART,                  /* The current set of actions needs to be
                                  * restarted
                                  */
     I_TE_SUCCESS,               /* Some non-resource, non-cluster-layer action
                                  * is required of us, e.g. ping
                                  */
 /* 20 */
     I_ROUTER,                   /* Do our job as router and forward this to the
                                  * right place
                                  */
     I_SHUTDOWN,                 /* We are asking to shutdown */
     I_STOP,                     /* We have been told to shutdown */
     I_TERMINATE,                /* Actually exit */
     I_STARTUP,
     I_PE_SUCCESS,               /* The action completed successfully */
 
     I_JOIN_OFFER,               /* The DC is offering membership */
     I_JOIN_REQUEST,             /* The client is requesting membership */
     I_JOIN_RESULT,              /* If not the DC: The result of a join request
                                  * Else: A client is responding with its local state info
                                  */
 
     I_WAIT_FOR_EVENT,           /* we may be waiting for an async task to "happen"
                                  * and until it does, we can't do anything else
                                  */
 
     I_DC_HEARTBEAT,             /* The DC is telling us that it is alive and well */
 
     I_LRM_EVENT,
 
 /* 30 */
     I_PENDING,
     I_HALT,
 
     /*  ------------ Last input found in table is above ----------- */
     I_ILLEGAL                   /* This is an illegal value for an FSA input */
         /* (must be last) */
 };
 
 #  define MAXINPUT  I_ILLEGAL
 
 #  define I_MESSAGE I_ROUTER
 
 /*======================================
  *
  * actions
  *
  * Some of the actions below will always occur together for now, but this may
  * not always be the case, so they are split up so that they can easily be
  * called independently in the future, if necessary.
  *
  * For example, separating A_LRM_CONNECT from A_STARTUP might be useful
  * if we ever try to recover from a faulty or disconnected executor.
  *
  *======================================*/
 
          /* Don't do anything */
 #  define A_NOTHING                 0x0000000000000000ULL
 
 /* -- Startup actions -- */
         /* Hook to perform any actions (other than connecting to other daemons)
          * that might be needed as part of the startup.
          */
 #  define A_STARTUP                 0x0000000000000001ULL
         /* Hook to perform any actions that might be needed as part
          * after startup is successful.
          */
 #  define A_STARTED                 0x0000000000000002ULL
         /* Connect to cluster layer */
 #  define A_HA_CONNECT              0x0000000000000004ULL
 #  define A_HA_DISCONNECT           0x0000000000000008ULL
 
 #  define A_INTEGRATE_TIMER_START   0x0000000000000010ULL
 #  define A_INTEGRATE_TIMER_STOP    0x0000000000000020ULL
 #  define A_FINALIZE_TIMER_START    0x0000000000000040ULL
 #  define A_FINALIZE_TIMER_STOP     0x0000000000000080ULL
 
 /* -- Election actions -- */
 #  define A_DC_TIMER_START          0x0000000000000100ULL
 #  define A_DC_TIMER_STOP           0x0000000000000200ULL
 #  define A_ELECTION_COUNT          0x0000000000000400ULL
 #  define A_ELECTION_VOTE           0x0000000000000800ULL
 
 #  define A_ELECTION_START          0x0000000000001000ULL
 
 /* -- Message processing -- */
         /* Process the queue of requests */
 #  define A_MSG_PROCESS             0x0000000000002000ULL
         /* Send the message to the correct recipient */
 #  define A_MSG_ROUTE               0x0000000000004000ULL
 
         /* Send a welcome message to new node(s) */
 #  define A_DC_JOIN_OFFER_ONE       0x0000000000008000ULL
 
 /* -- Server Join protocol actions -- */
         /* Send a welcome message to all nodes */
 #  define A_DC_JOIN_OFFER_ALL       0x0000000000010000ULL
         /* Process the remote node's ack of our join message */
 #  define A_DC_JOIN_PROCESS_REQ     0x0000000000020000ULL
         /* Send out the results of the Join phase */
 #  define A_DC_JOIN_FINALIZE        0x0000000000040000ULL
         /* Send out the results of the Join phase */
 #  define A_DC_JOIN_PROCESS_ACK     0x0000000000080000ULL
 
 /* -- Client Join protocol actions -- */
 #  define A_CL_JOIN_QUERY           0x0000000000100000ULL
 #  define A_CL_JOIN_ANNOUNCE        0x0000000000200000ULL
         /* Request membership to the DC list */
 #  define A_CL_JOIN_REQUEST         0x0000000000400000ULL
         /* Did the DC accept or reject the request */
 #  define A_CL_JOIN_RESULT          0x0000000000800000ULL
 
 /* -- Recovery, DC start/stop -- */
         /* Something bad happened, try to recover */
 #  define A_RECOVER                 0x0000000001000000ULL
         /* Hook to perform any actions (apart from starting, the TE, scheduler,
          * and gathering the latest CIB) that might be necessary before
          * giving up the responsibilities of being the DC.
          */
 #  define A_DC_RELEASE              0x0000000002000000ULL
         /* */
 #  define A_DC_RELEASED             0x0000000004000000ULL
         /* Hook to perform any actions (apart from starting, the TE, scheduler,
          * and gathering the latest CIB) that might be necessary before
          * taking over the responsibilities of being the DC.
          */
 #  define A_DC_TAKEOVER             0x0000000008000000ULL
 
 /* -- Shutdown actions -- */
 #  define A_SHUTDOWN                0x0000000010000000ULL
 #  define A_STOP                    0x0000000020000000ULL
 #  define A_EXIT_0                  0x0000000040000000ULL
 #  define A_EXIT_1                  0x0000000080000000ULL
 
 #  define A_SHUTDOWN_REQ            0x0000000100000000ULL
 #  define A_ELECTION_CHECK          0x0000000200000000ULL
 #  define A_DC_JOIN_FINAL           0x0000000400000000ULL
 
 /* -- CIB actions -- */
 #  define A_CIB_START               0x0000020000000000ULL
 #  define A_CIB_STOP                0x0000040000000000ULL
 
 /* -- Transition Engine actions -- */
         /* Attempt to reach the newly calculated cluster state. This is
          * only called once per transition (except if it is asked to
          * stop the transition or start a new one).
          * Once given a cluster state to reach, the TE will determine
          * tasks that can be performed in parallel, execute them, wait
          * for replies and then determine the next set until the new
          * state is reached or no further tasks can be taken.
          */
 #  define A_TE_INVOKE               0x0000100000000000ULL
 #  define A_TE_START                0x0000200000000000ULL
 #  define A_TE_STOP                 0x0000400000000000ULL
 #  define A_TE_CANCEL               0x0000800000000000ULL
 #  define A_TE_HALT                 0x0001000000000000ULL
 
 /* -- Scheduler actions -- */
         /* Calculate the next state for the cluster.  This is only
          * invoked once per needed calculation.
          */
 #  define A_PE_INVOKE               0x0002000000000000ULL
 #  define A_PE_START                0x0004000000000000ULL
 #  define A_PE_STOP                 0x0008000000000000ULL
 /* -- Misc actions -- */
         /* Add a system generate "block" so that resources arent moved
          * to or are activly moved away from the affected node.  This
          * way we can return quickly even if busy with other things.
          */
 #  define A_NODE_BLOCK              0x0010000000000000ULL
         /* Update our information in the local CIB */
 #  define A_UPDATE_NODESTATUS       0x0020000000000000ULL
 #  define A_READCONFIG              0x0080000000000000ULL
 
 /* -- LRM Actions -- */
         /* Connect to pacemaker-execd */
 #  define A_LRM_CONNECT             0x0100000000000000ULL
         /* Disconnect from pacemaker-execd */
 #  define A_LRM_DISCONNECT          0x0200000000000000ULL
 #  define A_LRM_INVOKE              0x0400000000000000ULL
 #  define A_LRM_EVENT               0x0800000000000000ULL
 
 /* -- Logging actions -- */
 #  define A_LOG                     0x1000000000000000ULL
 #  define A_ERROR                   0x2000000000000000ULL
 #  define A_WARN                    0x4000000000000000ULL
 
 #  define O_EXIT                (A_SHUTDOWN|A_STOP|A_LRM_DISCONNECT|A_HA_DISCONNECT|A_EXIT_0|A_CIB_STOP)
 #  define O_RELEASE             (A_DC_TIMER_STOP|A_DC_RELEASE|A_PE_STOP|A_TE_STOP|A_DC_RELEASED)
 #  define O_PE_RESTART          (A_PE_START|A_PE_STOP)
 #  define O_TE_RESTART          (A_TE_START|A_TE_STOP)
 #  define O_CIB_RESTART         (A_CIB_START|A_CIB_STOP)
 #  define O_LRM_RECONNECT       (A_LRM_CONNECT|A_LRM_DISCONNECT)
 #  define O_DC_TIMER_RESTART    (A_DC_TIMER_STOP|A_DC_TIMER_START)
 /*======================================
  *
  * "register" contents
  *
  * Things we may want to remember regardless of which state we are in.
  *
  * These also count as inputs for synthesizing I_*
  *
  *======================================*/
 #  define R_THE_DC          0x00000001ULL
                                         /* Are we the DC? */
 #  define R_STARTING        0x00000002ULL
                                         /* Are we starting up? */
 #  define R_SHUTDOWN        0x00000004ULL
                                         /* Are we trying to shut down? */
 #  define R_STAYDOWN        0x00000008ULL
                                         /* Should we restart? */
 
 #  define R_JOIN_OK         0x00000010ULL   /* Have we completed the join process */
 #  define R_READ_CONFIG     0x00000040ULL
 #  define R_INVOKE_PE       0x00000080ULL   // Should the scheduler be invoked?
 
 #  define R_CIB_CONNECTED   0x00000100ULL
                                         /* Is the CIB connected? */
 #  define R_PE_CONNECTED    0x00000200ULL   // Is the scheduler connected?
 #  define R_TE_CONNECTED    0x00000400ULL
                                         /* Is the Transition Engine connected? */
 #  define R_LRM_CONNECTED   0x00000800ULL   // Is pacemaker-execd connected?
 
 #  define R_CIB_REQUIRED    0x00001000ULL
                                         /* Is the CIB required? */
 #  define R_PE_REQUIRED     0x00002000ULL   // Is the scheduler required?
 #  define R_TE_REQUIRED     0x00004000ULL
                                         /* Is the Transition Engine required? */
 #  define R_ST_REQUIRED     0x00008000ULL
                                         /* Is the Stonith daemon required? */
 
 #  define R_CIB_DONE        0x00010000ULL
                                         /* Have we calculated the CIB? */
 #  define R_HAVE_CIB        0x00020000ULL   /* Do we have an up-to-date CIB */
 #  define R_CIB_ASKED       0x00040000ULL   /* Have we asked for an up-to-date CIB */
 
 #  define R_MEMBERSHIP      0x00100000ULL   /* Have we got cluster layer data yet */
 #  define R_PEER_DATA       0x00200000ULL   /* Have we got T_CL_STATUS data yet */
 
 #  define R_HA_DISCONNECTED 0x00400000ULL      /* did we sign out of our own accord */
 
 #  define R_REQ_PEND        0x01000000ULL
                                         /* Are there Requests waiting for
                                            processing? */
 #  define R_PE_PEND         0x02000000ULL   // Are we awaiting reply from scheduler?
 #  define R_TE_PEND         0x04000000ULL
                                         /* Has the TE been invoked and we're
                                            awaiting completion? */
 #  define R_RESP_PEND       0x08000000ULL
                                         /* Do we have clients waiting on a
                                            response? if so perhaps we shouldn't
                                            stop yet */
 
 #  define R_IN_TRANSITION   0x10000000ULL
                                         /*  */
 #  define R_SENT_RSC_STOP   0x20000000ULL /* Have we sent a stop action to all
                                          * resources in preparation for
                                          * shutting down */
 
 #  define R_IN_RECOVERY     0x80000000ULL
 
 #define CRM_DIRECT_NACK_RC (99) // Deprecated (see PCMK_LRM_OP_INVALID)
 
 enum crmd_fsa_cause {
     C_UNKNOWN = 0,
     C_STARTUP,
     C_IPC_MESSAGE,
     C_HA_MESSAGE,
     C_CRMD_STATUS_CALLBACK,
     C_LRM_OP_CALLBACK,
     C_TIMER_POPPED,
     C_SHUTDOWN,
     C_FSA_INTERNAL,
 };
 
 enum fsa_data_type {
     fsa_dt_none,
     fsa_dt_ha_msg,
     fsa_dt_xml,
     fsa_dt_lrm,
 };
 
 typedef struct fsa_data_s fsa_data_t;
 struct fsa_data_s {
     int id;
     enum crmd_fsa_input fsa_input;
     enum crmd_fsa_cause fsa_cause;
     long long actions;
     const char *origin;
     void *data;
     enum fsa_data_type data_type;
 };
 
 /* Global FSA stuff */
 extern gboolean do_fsa_stall;
 extern enum crmd_fsa_state fsa_state;
 extern uint64_t fsa_input_register;
 extern long long fsa_actions;
 
 #define controld_set_fsa_input_flags(flags_to_set) do {                     \
         fsa_input_register = pcmk__set_flags_as(__FUNCTION__, __LINE__,     \
                                                 LOG_TRACE,                  \
                                                 "FSA input", "controller",  \
                                                 fsa_input_register,         \
                                                 (flags_to_set),             \
                                                 #flags_to_set);             \
     } while (0)
 
 #define controld_clear_fsa_input_flags(flags_to_clear) do {                 \
         fsa_input_register = pcmk__clear_flags_as(__FUNCTION__, __LINE__,   \
                                                   LOG_TRACE,                \
                                                   "FSA input", "controller",\
                                                   fsa_input_register,       \
                                                   (flags_to_clear),         \
                                                   #flags_to_clear);         \
     } while (0)
 
 extern cib_t *fsa_cib_conn;
 
 extern char *fsa_our_uname;
 extern char *fsa_our_uuid;
 extern char *fsa_pe_ref;        // Last invocation of the scheduler
 extern char *fsa_our_dc;
 extern char *fsa_our_dc_version;
 extern GListPtr fsa_message_queue;
 
 extern char *fsa_cluster_name;
 
 extern crm_trigger_t *fsa_source;
 extern crm_trigger_t *config_read;
 
 extern unsigned long long saved_ccm_membership_id;
 extern gboolean ever_had_quorum;
 
 // These should be moved elsewhere
 void do_update_cib_nodes(gboolean overwrite, const char *caller);
 int crmd_cib_smart_opt(void);
 xmlNode *controld_query_executor_state(const char *node_name);
 
 const char *fsa_input2string(enum crmd_fsa_input input);
 const char *fsa_state2string(enum crmd_fsa_state state);
 const char *fsa_cause2string(enum crmd_fsa_cause cause);
 const char *fsa_action2string(long long action);
 
 enum crmd_fsa_state s_crmd_fsa(enum crmd_fsa_cause cause);
 
 #  define AM_I_DC is_set(fsa_input_register, R_THE_DC)
 #  define AM_I_OPERATIONAL (is_set(fsa_input_register, R_STARTING) == FALSE)
-#  define trigger_fsa(source) do { \
-        crm_trace("Triggering FSA: %s", __FUNCTION__); \
-        mainloop_set_trigger(source); \
+#  define trigger_fsa() do {                    \
+        if (fsa_source != NULL) {               \
+            crm_trace("Triggering FSA");        \
+            mainloop_set_trigger(fsa_source);   \
+        }                                       \
     } while(0)
 
 /* A_READCONFIG */
 void do_read_config(long long action, enum crmd_fsa_cause cause,
                     enum crmd_fsa_state cur_state,
                     enum crmd_fsa_input current_input, fsa_data_t *msg_data);
 
 /* A_PE_INVOKE */
 void do_pe_invoke(long long action, enum crmd_fsa_cause cause,
                   enum crmd_fsa_state cur_state,
                   enum crmd_fsa_input current_input, fsa_data_t *msg_data);
 
 /* A_LOG */
 void do_log(long long action, enum crmd_fsa_cause cause,
             enum crmd_fsa_state cur_state,
             enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_STARTUP */
 void do_startup(long long action, enum crmd_fsa_cause cause,
                 enum crmd_fsa_state cur_state,
                 enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_CIB_START, STOP, RESTART */
 void do_cib_control(long long action, enum crmd_fsa_cause cause,
                     enum crmd_fsa_state cur_state,
                     enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_HA_CONNECT */
 void do_ha_control(long long action, enum crmd_fsa_cause cause,
                    enum crmd_fsa_state cur_state,
                    enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_LRM_CONNECT */
 void do_lrm_control(long long action, enum crmd_fsa_cause cause,
                     enum crmd_fsa_state cur_state,
                     enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_PE_START, STOP, RESTART */
 void do_pe_control(long long action, enum crmd_fsa_cause cause,
                    enum crmd_fsa_state cur_state,
                    enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_TE_START, STOP, RESTART */
 void do_te_control(long long action, enum crmd_fsa_cause cause,
                    enum crmd_fsa_state cur_state,
                    enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_STARTED */
 void do_started(long long action, enum crmd_fsa_cause cause,
                 enum crmd_fsa_state cur_state,
                 enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_MSG_ROUTE */
 void do_msg_route(long long action, enum crmd_fsa_cause cause,
                   enum crmd_fsa_state cur_state,
                   enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_RECOVER */
 void do_recover(long long action, enum crmd_fsa_cause cause,
                 enum crmd_fsa_state cur_state,
                 enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_ELECTION_VOTE */
 void do_election_vote(long long action, enum crmd_fsa_cause cause,
                       enum crmd_fsa_state cur_state,
                       enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_ELECTION_COUNT */
 void do_election_count_vote(long long action, enum crmd_fsa_cause cause,
                             enum crmd_fsa_state cur_state,
                             enum crmd_fsa_input cur_input,
                             fsa_data_t *msg_data);
 
 /* A_ELECTION_CHECK */
 void do_election_check(long long action, enum crmd_fsa_cause cause,
                        enum crmd_fsa_state cur_state,
                        enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_DC_TIMER_STOP */
 void do_timer_control(long long action, enum crmd_fsa_cause cause,
                       enum crmd_fsa_state cur_state,
                       enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_DC_TAKEOVER */
 void do_dc_takeover(long long action, enum crmd_fsa_cause cause,
                     enum crmd_fsa_state cur_state,
                     enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_DC_RELEASE */
 void do_dc_release(long long action, enum crmd_fsa_cause cause,
                    enum crmd_fsa_state cur_state,
                    enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_DC_JOIN_OFFER_ALL */
 void do_dc_join_offer_all(long long action, enum crmd_fsa_cause cause,
                           enum crmd_fsa_state cur_state,
                           enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_DC_JOIN_OFFER_ONE */
 void do_dc_join_offer_one(long long action, enum crmd_fsa_cause cause,
                           enum crmd_fsa_state cur_state,
                           enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_DC_JOIN_ACK */
 void do_dc_join_ack(long long action, enum crmd_fsa_cause cause,
                     enum crmd_fsa_state cur_state,
                     enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_DC_JOIN_REQ */
 void do_dc_join_filter_offer(long long action, enum crmd_fsa_cause cause,
                              enum crmd_fsa_state cur_state,
                              enum crmd_fsa_input cur_input,
                              fsa_data_t *msg_data);
 
 /* A_DC_JOIN_FINALIZE */
 void do_dc_join_finalize(long long action, enum crmd_fsa_cause cause,
                          enum crmd_fsa_state cur_state,
                          enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_CL_JOIN_QUERY */
 /* is there a DC out there? */
 void do_cl_join_query(long long action, enum crmd_fsa_cause cause,
                       enum crmd_fsa_state cur_state,
                       enum crmd_fsa_input current_input, fsa_data_t *msg_data);
 
 /* A_CL_JOIN_ANNOUNCE */
 void do_cl_join_announce(long long action, enum crmd_fsa_cause cause,
                          enum crmd_fsa_state cur_state,
                          enum crmd_fsa_input current_input, fsa_data_t *msg_data);
 
 /* A_CL_JOIN_REQUEST */
 void do_cl_join_offer_respond(long long action, enum crmd_fsa_cause cause,
                               enum crmd_fsa_state cur_state,
                               enum crmd_fsa_input current_input,
                               fsa_data_t *msg_data);
 
 /* A_CL_JOIN_RESULT */
 void do_cl_join_finalize_respond(long long action, enum crmd_fsa_cause cause,
                                  enum crmd_fsa_state cur_state,
                                  enum crmd_fsa_input current_input,
                                  fsa_data_t *msg_data);
 
 /* A_LRM_INVOKE */
 void do_lrm_invoke(long long action, enum crmd_fsa_cause cause,
                    enum crmd_fsa_state cur_state,
                    enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_LRM_EVENT */
 void do_lrm_event(long long action, enum crmd_fsa_cause cause,
                   enum crmd_fsa_state cur_state,
                   enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_TE_INVOKE, A_TE_CANCEL */
 void do_te_invoke(long long action, enum crmd_fsa_cause cause,
                   enum crmd_fsa_state cur_state,
                   enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_SHUTDOWN_REQ */
 void do_shutdown_req(long long action, enum crmd_fsa_cause cause,
                      enum crmd_fsa_state cur_state,
                      enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_SHUTDOWN */
 void do_shutdown(long long action, enum crmd_fsa_cause cause,
                  enum crmd_fsa_state cur_state,
                  enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_STOP */
 void do_stop(long long action, enum crmd_fsa_cause cause,
              enum crmd_fsa_state cur_state,
              enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_EXIT_0, A_EXIT_1 */
 void do_exit(long long action, enum crmd_fsa_cause cause,
              enum crmd_fsa_state cur_state,
              enum crmd_fsa_input cur_input, fsa_data_t *msg_data);
 
 /* A_DC_JOIN_FINAL */
 void do_dc_join_final(long long action, enum crmd_fsa_cause cause,
                       enum crmd_fsa_state cur_state,
                       enum crmd_fsa_input current_input, fsa_data_t *msg_data);
 #endif
diff --git a/daemons/controld/controld_join_client.c b/daemons/controld/controld_join_client.c
index beebe8e791..b3b5682db8 100644
--- a/daemons/controld/controld_join_client.c
+++ b/daemons/controld/controld_join_client.c
@@ -1,310 +1,311 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <pacemaker-controld.h>
 
 int reannounce_count = 0;
 void join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);
 
 extern ha_msg_input_t *copy_ha_msg_input(ha_msg_input_t * orig);
 
 /*!
  * \internal
  * \brief Remember if DC is shutting down as we join
  *
  * If we're joining while the current DC is shutting down, update its expected
  * state, so we don't fence it if we become the new DC. (We weren't a peer
  * when it broadcast its shutdown request.)
  *
  * \param[in] msg  A join message from the DC
  */
 static void
 update_dc_expected(xmlNode *msg)
 {
     if (fsa_our_dc && crm_is_true(crm_element_value(msg, F_CRM_DC_LEAVING))) {
         crm_node_t *dc_node = crm_get_peer(0, fsa_our_dc);
 
         crm_update_peer_expected(__FUNCTION__, dc_node, CRMD_JOINSTATE_DOWN);
     }
 }
 
 /*	A_CL_JOIN_QUERY		*/
 /* is there a DC out there? */
 void
 do_cl_join_query(long long action,
                  enum crmd_fsa_cause cause,
                  enum crmd_fsa_state cur_state,
                  enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
                                   CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
 
     sleep(1);                   // Give the cluster layer time to propagate to the DC
     update_dc(NULL);            /* Unset any existing value so that the result is not discarded */
     crm_debug("Querying for a DC");
     send_cluster_message(NULL, crm_msg_crmd, req, FALSE);
     free_xml(req);
 }
 
 /*	 A_CL_JOIN_ANNOUNCE	*/
 
 /* this is kind of a workaround for the fact that we may not be around or
  * are otherwise unable to reply when the DC sends out A_DC_JOIN_OFFER_ALL
  */
 void
 do_cl_join_announce(long long action,
                     enum crmd_fsa_cause cause,
                     enum crmd_fsa_state cur_state,
                     enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     /* don't announce if we're in one of these states */
     if (cur_state != S_PENDING) {
         crm_warn("Not announcing cluster join because in state %s",
                  fsa_state2string(cur_state));
         return;
     }
 
     if (AM_I_OPERATIONAL) {
         /* send as a broadcast */
         xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
                                       CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
 
         crm_debug("Announcing availability");
         update_dc(NULL);
         send_cluster_message(NULL, crm_msg_crmd, req, FALSE);
         free_xml(req);
 
     } else {
         /* Delay announce until we have finished local startup */
         crm_warn("Delaying announce of cluster join until local startup is complete");
         return;
     }
 }
 
 static int query_call_id = 0;
 
 /*	 A_CL_JOIN_REQUEST	*/
 /* aka. accept the welcome offer */
 void
 do_cl_join_offer_respond(long long action,
                          enum crmd_fsa_cause cause,
                          enum crmd_fsa_state cur_state,
                          enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
     const char *welcome_from;
     const char *join_id;
 
     CRM_CHECK(input != NULL, return);
 
 #if 0
     if (we are sick) {
         log error;
 
         /* save the request for later? */
         return;
     }
 #endif
 
     welcome_from = crm_element_value(input->msg, F_CRM_HOST_FROM);
     join_id = crm_element_value(input->msg, F_CRM_JOIN_ID);
     crm_trace("Accepting cluster join offer from node %s "CRM_XS" join-%s",
               welcome_from, crm_element_value(input->msg, F_CRM_JOIN_ID));
 
     /* we only ever want the last one */
     if (query_call_id > 0) {
         crm_trace("Cancelling previous join query: %d", query_call_id);
         remove_cib_op_callback(query_call_id, FALSE);
         query_call_id = 0;
     }
 
     if (update_dc(input->msg) == FALSE) {
         crm_warn("Discarding cluster join offer from node %s (expected %s)",
                  welcome_from, fsa_our_dc);
         return;
     }
 
     update_dc_expected(input->msg);
 
     query_call_id =
         fsa_cib_conn->cmds->query(fsa_cib_conn, NULL, NULL, cib_scope_local | cib_no_children);
     fsa_register_cib_callback(query_call_id, FALSE, strdup(join_id), join_query_callback);
     crm_trace("Registered join query callback: %d", query_call_id);
 
     register_fsa_action(A_DC_TIMER_STOP);
+    trigger_fsa();
 }
 
 void
 join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     char *join_id = user_data;
     xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);
 
     CRM_LOG_ASSERT(join_id != NULL);
 
     if (query_call_id != call_id) {
         crm_trace("Query %d superseded", call_id);
         goto done;
     }
 
     query_call_id = 0;
     if(rc != pcmk_ok || output == NULL) {
         crm_err("Could not retrieve version details for join-%s: %s (%d)",
                 join_id, pcmk_strerror(rc), rc);
         register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __FUNCTION__);
 
     } else if (fsa_our_dc == NULL) {
         crm_debug("Membership is in flux, not continuing join-%s", join_id);
 
     } else {
         xmlNode *reply = NULL;
 
         crm_debug("Respond to join offer join-%s from %s", join_id, fsa_our_dc);
         copy_in_properties(generation, output);
 
         reply = create_request(CRM_OP_JOIN_REQUEST, generation, fsa_our_dc,
                                CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
 
         crm_xml_add(reply, F_CRM_JOIN_ID, join_id);
         crm_xml_add(reply, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
         send_cluster_message(crm_get_peer(0, fsa_our_dc), crm_msg_crmd, reply, TRUE);
         free_xml(reply);
     }
 
   done:
     free_xml(generation);
 }
 
 static void
 set_join_state(const char * start_state)
 {
     if (pcmk__str_eq(start_state, "standby", pcmk__str_casei)) {
         crm_notice("Forcing node %s to join in %s state per configured environment",
                    fsa_our_uname, start_state);
         update_attr_delegate(fsa_cib_conn, cib_sync_call, XML_CIB_TAG_NODES, fsa_our_uuid,
                              NULL, NULL, NULL, "standby", "on", TRUE, NULL, NULL);
 
     } else if (pcmk__str_eq(start_state, "online", pcmk__str_casei)) {
         crm_notice("Forcing node %s to join in %s state per configured environment",
                    fsa_our_uname, start_state);
         update_attr_delegate(fsa_cib_conn, cib_sync_call, XML_CIB_TAG_NODES, fsa_our_uuid,
                              NULL, NULL, NULL, "standby", "off", TRUE, NULL, NULL);
 
     } else if (pcmk__str_eq(start_state, "default", pcmk__str_casei)) {
         crm_debug("Not forcing a starting state on node %s", fsa_our_uname);
 
     } else {
         crm_warn("Unrecognized start state '%s', using 'default' (%s)",
                  start_state, fsa_our_uname);
     }
 }
 
 /*	A_CL_JOIN_RESULT	*/
 /* aka. this is notification that we have (or have not) been accepted */
 void
 do_cl_join_finalize_respond(long long action,
                             enum crmd_fsa_cause cause,
                             enum crmd_fsa_state cur_state,
                             enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     xmlNode *tmp1 = NULL;
     gboolean was_nack = TRUE;
     static gboolean first_join = TRUE;
     ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
     const char *start_state = pcmk__env_option("node_start_state");
 
     int join_id = -1;
     const char *op = crm_element_value(input->msg, F_CRM_TASK);
     const char *ack_nack = crm_element_value(input->msg, CRM_OP_JOIN_ACKNAK);
     const char *welcome_from = crm_element_value(input->msg, F_CRM_HOST_FROM);
 
     if (!pcmk__str_eq(op, CRM_OP_JOIN_ACKNAK, pcmk__str_casei)) {
         crm_trace("Ignoring op=%s message", op);
         return;
     }
 
     /* calculate if it was an ack or a nack */
     if (crm_is_true(ack_nack)) {
         was_nack = FALSE;
     }
 
     crm_element_value_int(input->msg, F_CRM_JOIN_ID, &join_id);
 
     if (was_nack) {
         crm_err("Shutting down because cluster join with leader %s failed "
                 CRM_XS" join-%d NACK'd", welcome_from, join_id);
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
         return;
     }
 
     if (AM_I_DC == FALSE && pcmk__str_eq(welcome_from, fsa_our_uname, pcmk__str_casei)) {
         crm_warn("Discarding our own welcome - we're no longer the DC");
         return;
     }
 
     if (update_dc(input->msg) == FALSE) {
         crm_warn("Discarding %s from node %s (expected from %s)",
                  op, welcome_from, fsa_our_dc);
         return;
     }
 
     update_dc_expected(input->msg);
 
     /* send our status section to the DC */
     tmp1 = controld_query_executor_state(fsa_our_uname);
     if (tmp1 != NULL) {
         xmlNode *reply = create_request(CRM_OP_JOIN_CONFIRM, tmp1, fsa_our_dc,
                                         CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
 
         crm_xml_add_int(reply, F_CRM_JOIN_ID, join_id);
 
         crm_debug("Confirming join-%d: sending local operation history to %s",
                   join_id, fsa_our_dc);
 
         /*
          * If this is the node's first join since the controller started on it,
          * set its initial state (standby or member) according to the user's
          * preference.
          *
          * We do not clear the LRM history here. Even if the DC failed to do it
          * when we last left, removing them here creates a race condition if the
          * controller is being recovered. Instead of a list of active resources
          * from the executor, we may end up with a blank status section. If we
          * are _NOT_ lucky, we will probe for the "wrong" instance of anonymous
          * clones and end up with multiple active instances on the machine.
          */
         if (first_join && is_not_set(fsa_input_register, R_SHUTDOWN)) {
             first_join = FALSE;
             if (start_state) {
                 set_join_state(start_state);
             }
         }
 
         send_cluster_message(crm_get_peer(0, fsa_our_dc), crm_msg_crmd, reply, TRUE);
         free_xml(reply);
 
         if (AM_I_DC == FALSE) {
             register_fsa_input_adv(cause, I_NOT_DC, NULL, A_NOTHING, TRUE, __FUNCTION__);
         }
 
         free_xml(tmp1);
 
     } else {
         crm_err("Could not confirm join-%d with %s: Local operation history failed",
                 join_id, fsa_our_dc);
         register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
     }
 }
diff --git a/daemons/controld/controld_membership.c b/daemons/controld/controld_membership.c
index 29e5f6d8d1..41d222a213 100644
--- a/daemons/controld/controld_membership.c
+++ b/daemons/controld/controld_membership.c
@@ -1,436 +1,437 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 /* put these first so that uuid_t is defined without conflicts */
 #include <crm_internal.h>
 
 #include <string.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/cluster/internal.h>
 
 #include <pacemaker-controld.h>
 
 gboolean membership_flux_hack = FALSE;
 void post_cache_update(int instance);
 
 int last_peer_update = 0;
 guint highest_born_on = -1;
 
 extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
 
 static void
 reap_dead_nodes(gpointer key, gpointer value, gpointer user_data)
 {
     crm_node_t *node = value;
 
     if (crm_is_peer_active(node) == FALSE) {
         crm_update_peer_join(__FUNCTION__, node, crm_join_none);
 
         if(node && node->uname) {
             if (pcmk__str_eq(fsa_our_uname, node->uname, pcmk__str_casei)) {
                 crm_err("We're not part of the cluster anymore");
                 register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
 
             } else if (AM_I_DC == FALSE && pcmk__str_eq(node->uname, fsa_our_dc, pcmk__str_casei)) {
                 crm_warn("Our DC node (%s) left the cluster", node->uname);
                 register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
             }
         }
 
         if (fsa_state == S_INTEGRATION || fsa_state == S_FINALIZE_JOIN) {
             check_join_state(fsa_state, __FUNCTION__);
         }
         if(node && node->uuid) {
             fail_incompletable_actions(transition_graph, node->uuid);
         }
     }
 }
 
 gboolean ever_had_quorum = FALSE;
 
 void
 post_cache_update(int instance)
 {
     xmlNode *no_op = NULL;
 
     crm_peer_seq = instance;
     crm_debug("Updated cache after membership event %d.", instance);
 
     g_hash_table_foreach(crm_peer_cache, reap_dead_nodes, NULL);
     controld_set_fsa_input_flags(R_MEMBERSHIP);
 
     if (AM_I_DC) {
         populate_cib_nodes(node_update_quick | node_update_cluster | node_update_peer |
                            node_update_expected, __FUNCTION__);
     }
 
     /*
      * If we lost nodes, we should re-check the election status
      * Safe to call outside of an election
      */
     register_fsa_action(A_ELECTION_CHECK);
+    trigger_fsa();
 
     /* Membership changed, remind everyone we're here.
      * This will aid detection of duplicate DCs
      */
     no_op = create_request(CRM_OP_NOOP, NULL, NULL, CRM_SYSTEM_CRMD,
                            AM_I_DC ? CRM_SYSTEM_DC : CRM_SYSTEM_CRMD, NULL);
     send_cluster_message(NULL, crm_msg_crmd, no_op, FALSE);
     free_xml(no_op);
 }
 
 static void
 crmd_node_update_complete(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     fsa_data_t *msg_data = NULL;
 
     last_peer_update = 0;
 
     if (rc == pcmk_ok) {
         crm_trace("Node update %d complete", call_id);
 
     } else if(call_id < pcmk_ok) {
         crm_err("Node update failed: %s (%d)", pcmk_strerror(call_id), call_id);
         crm_log_xml_debug(msg, "failed");
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
 
     } else {
         crm_err("Node update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
         crm_log_xml_debug(msg, "failed");
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
 }
 
 /*!
  * \internal
  * \brief Create an XML node state tag with updates
  *
  * \param[in,out] node    Node whose state will be used for update
  * \param[in]     flags   Bitmask of node_update_flags indicating what to update
  * \param[in,out] parent  XML node to contain update (or NULL)
  * \param[in]     source  Who requested the update (only used for logging)
  *
  * \return Pointer to created node state tag
  */
 xmlNode *
 create_node_state_update(crm_node_t *node, int flags, xmlNode *parent,
                          const char *source)
 {
     const char *value = NULL;
     xmlNode *node_state;
 
     if (!node->state) {
         crm_info("Node update for %s cancelled: no state, not seen yet", node->uname);
        return NULL;
     }
 
     node_state = create_xml_node(parent, XML_CIB_TAG_STATE);
 
     if (is_set(node->flags, crm_remote_node)) {
         crm_xml_add(node_state, XML_NODE_IS_REMOTE, XML_BOOLEAN_TRUE);
     }
 
     set_uuid(node_state, XML_ATTR_UUID, node);
 
     if (crm_element_value(node_state, XML_ATTR_UUID) == NULL) {
         crm_info("Node update for %s cancelled: no id", node->uname);
         free_xml(node_state);
         return NULL;
     }
 
     crm_xml_add(node_state, XML_ATTR_UNAME, node->uname);
 
     if ((flags & node_update_cluster) && node->state) {
         crm_xml_add_boolean(node_state, XML_NODE_IN_CLUSTER,
                             pcmk__str_eq(node->state, CRM_NODE_MEMBER, pcmk__str_casei));
     }
 
     if (!is_set(node->flags, crm_remote_node)) {
         if (flags & node_update_peer) {
             value = OFFLINESTATUS;
             if (is_set(node->processes, crm_get_cluster_proc())) {
                 value = ONLINESTATUS;
             }
             crm_xml_add(node_state, XML_NODE_IS_PEER, value);
         }
 
         if (flags & node_update_join) {
             if (node->join <= crm_join_none) {
                 value = CRMD_JOINSTATE_DOWN;
             } else {
                 value = CRMD_JOINSTATE_MEMBER;
             }
             crm_xml_add(node_state, XML_NODE_JOIN_STATE, value);
         }
 
         if (flags & node_update_expected) {
             crm_xml_add(node_state, XML_NODE_EXPECTED, node->expected);
         }
     }
 
     crm_xml_add(node_state, XML_ATTR_ORIGIN, source);
 
     return node_state;
 }
 
 static void
 remove_conflicting_node_callback(xmlNode * msg, int call_id, int rc,
                                  xmlNode * output, void *user_data)
 {
     char *node_uuid = user_data;
 
     do_crm_log_unlikely(rc == 0 ? LOG_DEBUG : LOG_NOTICE,
                         "Deletion of the unknown conflicting node \"%s\": %s (rc=%d)",
                         node_uuid, pcmk_strerror(rc), rc);
 }
 
 static void
 search_conflicting_node_callback(xmlNode * msg, int call_id, int rc,
                                  xmlNode * output, void *user_data)
 {
     char *new_node_uuid = user_data;
     xmlNode *node_xml = NULL;
 
     if (rc != pcmk_ok) {
         if (rc != -ENXIO) {
             crm_notice("Searching conflicting nodes for %s failed: %s (%d)",
                        new_node_uuid, pcmk_strerror(rc), rc);
         }
         return;
 
     } else if (output == NULL) {
         return;
     }
 
     if (pcmk__str_eq(crm_element_name(output), XML_CIB_TAG_NODE, pcmk__str_casei)) {
         node_xml = output;
 
     } else {
         node_xml = __xml_first_child(output);
     }
 
     for (; node_xml != NULL; node_xml = __xml_next(node_xml)) {
         const char *node_uuid = NULL;
         const char *node_uname = NULL;
         GHashTableIter iter;
         crm_node_t *node = NULL;
         gboolean known = FALSE;
 
         if (!pcmk__str_eq(crm_element_name(node_xml), XML_CIB_TAG_NODE, pcmk__str_casei)) {
             continue;
         }
 
         node_uuid = crm_element_value(node_xml, XML_ATTR_ID);
         node_uname = crm_element_value(node_xml, XML_ATTR_UNAME);
 
         if (node_uuid == NULL || node_uname == NULL) {
             continue;
         }
 
         g_hash_table_iter_init(&iter, crm_peer_cache);
         while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
             if (node->uuid
                 && pcmk__str_eq(node->uuid, node_uuid, pcmk__str_casei)
                 && node->uname
                 && pcmk__str_eq(node->uname, node_uname, pcmk__str_casei)) {
 
                 known = TRUE;
                 break;
             }
         }
 
         if (known == FALSE) {
             int delete_call_id = 0;
             xmlNode *node_state_xml = NULL;
 
             crm_notice("Deleting unknown node %s/%s which has conflicting uname with %s",
                        node_uuid, node_uname, new_node_uuid);
 
             delete_call_id = fsa_cib_conn->cmds->remove(fsa_cib_conn, XML_CIB_TAG_NODES, node_xml,
                                                         cib_scope_local | cib_quorum_override);
             fsa_register_cib_callback(delete_call_id, FALSE, strdup(node_uuid),
                                       remove_conflicting_node_callback);
 
             node_state_xml = create_xml_node(NULL, XML_CIB_TAG_STATE);
             crm_xml_add(node_state_xml, XML_ATTR_ID, node_uuid);
             crm_xml_add(node_state_xml, XML_ATTR_UNAME, node_uname);
 
             delete_call_id = fsa_cib_conn->cmds->remove(fsa_cib_conn, XML_CIB_TAG_STATUS, node_state_xml,
                                                         cib_scope_local | cib_quorum_override);
             fsa_register_cib_callback(delete_call_id, FALSE, strdup(node_uuid),
                                       remove_conflicting_node_callback);
             free_xml(node_state_xml);
         }
     }
 }
 
 static void
 node_list_update_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     fsa_data_t *msg_data = NULL;
 
     if(call_id < pcmk_ok) {
         crm_err("Node list update failed: %s (%d)", pcmk_strerror(call_id), call_id);
         crm_log_xml_debug(msg, "update:failed");
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
 
     } else if(rc < pcmk_ok) {
         crm_err("Node update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
         crm_log_xml_debug(msg, "update:failed");
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
 }
 
 #define NODE_PATH_MAX 512
 
 void
 populate_cib_nodes(enum node_update_flags flags, const char *source)
 {
     int call_id = 0;
     gboolean from_hashtable = TRUE;
     int call_options = cib_scope_local | cib_quorum_override;
     xmlNode *node_list = create_xml_node(NULL, XML_CIB_TAG_NODES);
 
 #if SUPPORT_COROSYNC
     if (is_not_set(flags, node_update_quick) && is_corosync_cluster()) {
         from_hashtable = corosync_initialize_nodelist(NULL, FALSE, node_list);
     }
 #endif
 
     if (from_hashtable) {
         GHashTableIter iter;
         crm_node_t *node = NULL;
 
         g_hash_table_iter_init(&iter, crm_peer_cache);
         while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
             xmlNode *new_node = NULL;
 
             crm_trace("Creating node entry for %s/%s", node->uname, node->uuid);
             if(node->uuid && node->uname) {
                 char xpath[NODE_PATH_MAX];
 
                 /* We need both to be valid */
                 new_node = create_xml_node(node_list, XML_CIB_TAG_NODE);
                 crm_xml_add(new_node, XML_ATTR_ID, node->uuid);
                 crm_xml_add(new_node, XML_ATTR_UNAME, node->uname);
 
                 /* Search and remove unknown nodes with the conflicting uname from CIB */
                 snprintf(xpath, NODE_PATH_MAX,
                          "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_NODES
                          "/" XML_CIB_TAG_NODE "[@uname='%s'][@id!='%s']",
                          node->uname, node->uuid);
 
                 call_id = fsa_cib_conn->cmds->query(fsa_cib_conn, xpath, NULL,
                                                     cib_scope_local | cib_xpath);
                 fsa_register_cib_callback(call_id, FALSE, strdup(node->uuid),
                                           search_conflicting_node_callback);
             }
         }
     }
 
     crm_trace("Populating <nodes> section from %s", from_hashtable ? "hashtable" : "cluster");
 
     fsa_cib_update(XML_CIB_TAG_NODES, node_list, call_options, call_id, NULL);
     fsa_register_cib_callback(call_id, FALSE, NULL, node_list_update_callback);
 
     free_xml(node_list);
 
     if (call_id >= pcmk_ok && crm_peer_cache != NULL && AM_I_DC) {
         /*
          * There is no need to update the local CIB with our values if
          * we've not seen valid membership data
          */
         GHashTableIter iter;
         crm_node_t *node = NULL;
 
         node_list = create_xml_node(NULL, XML_CIB_TAG_STATUS);
 
         g_hash_table_iter_init(&iter, crm_peer_cache);
         while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
             create_node_state_update(node, flags, node_list, source);
         }
 
         if (crm_remote_peer_cache) {
             g_hash_table_iter_init(&iter, crm_remote_peer_cache);
             while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
                 create_node_state_update(node, flags, node_list, source);
             }
         }
 
         fsa_cib_update(XML_CIB_TAG_STATUS, node_list, call_options, call_id, NULL);
         fsa_register_cib_callback(call_id, FALSE, NULL, crmd_node_update_complete);
         last_peer_update = call_id;
 
         free_xml(node_list);
     }
 }
 
 static void
 cib_quorum_update_complete(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     fsa_data_t *msg_data = NULL;
 
     if (rc == pcmk_ok) {
         crm_trace("Quorum update %d complete", call_id);
 
     } else {
         crm_err("Quorum update %d failed: %s (%d)", call_id, pcmk_strerror(rc), rc);
         crm_log_xml_debug(msg, "failed");
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
 }
 
 void
 crm_update_quorum(gboolean quorum, gboolean force_update)
 {
     ever_had_quorum |= quorum;
 
     if(ever_had_quorum && quorum == FALSE && no_quorum_suicide_escalation) {
         pcmk_panic(__FUNCTION__);
     }
 
     if (AM_I_DC && (force_update || fsa_has_quorum != quorum)) {
         int call_id = 0;
         xmlNode *update = NULL;
         int call_options = cib_scope_local | cib_quorum_override;
 
         update = create_xml_node(NULL, XML_TAG_CIB);
         crm_xml_add_int(update, XML_ATTR_HAVE_QUORUM, quorum);
         crm_xml_add(update, XML_ATTR_DC_UUID, fsa_our_uuid);
 
         fsa_cib_update(XML_TAG_CIB, update, call_options, call_id, NULL);
         crm_debug("Updating quorum status to %s (call=%d)", quorum ? "true" : "false", call_id);
         fsa_register_cib_callback(call_id, FALSE, NULL, cib_quorum_update_complete);
         free_xml(update);
 
         /* Quorum changes usually cause a new transition via other activity:
          * quorum gained via a node joining will abort via the node join,
          * and quorum lost via a node leaving will usually abort via resource
          * activity and/or fencing.
          *
          * However, it is possible that nothing else causes a transition (e.g.
          * someone forces quorum via corosync-cmaptcl, or quorum is lost due to
          * a node in standby shutting down cleanly), so here ensure a new
          * transition is triggered.
          */
         if (quorum) {
             /* If quorum was gained, abort after a short delay, in case multiple
              * nodes are joining around the same time, so the one that brings us
              * to quorum doesn't cause all the remaining ones to be fenced.
              */
             abort_after_delay(INFINITY, tg_restart, "Quorum gained", 5000);
         } else {
             abort_transition(INFINITY, tg_restart, "Quorum lost", NULL);
         }
     }
     fsa_has_quorum = quorum;
 }
diff --git a/daemons/controld/controld_messages.h b/daemons/controld/controld_messages.h
index 4018deb0c4..f3372fb8ef 100644
--- a/daemons/controld/controld_messages.h
+++ b/daemons/controld/controld_messages.h
@@ -1,91 +1,88 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #ifndef XML_CRM_MESSAGES__H
 #  define XML_CRM_MESSAGES__H
 
 #  include <crm/crm.h>
 #  include <crm/common/ipc_internal.h>
 #  include <crm/common/xml.h>
 #  include <crm/cluster/internal.h>
 #  include <controld_fsa.h>
 
 typedef struct ha_msg_input_s {
     xmlNode *msg;
     xmlNode *xml;
 
 } ha_msg_input_t;
 
 extern ha_msg_input_t *new_ha_msg_input(xmlNode * orig);
 extern void delete_ha_msg_input(ha_msg_input_t * orig);
 
 extern void *fsa_typed_data_adv(fsa_data_t * fsa_data, enum fsa_data_type a_type,
                                 const char *caller);
 
 #  define fsa_typed_data(x) fsa_typed_data_adv(msg_data, x, __FUNCTION__)
 
 extern void register_fsa_error_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
                                    fsa_data_t * cur_data, void *new_data, const char *raised_from);
 
 #  define register_fsa_error(cause, input, new_data) register_fsa_error_adv(cause, input, msg_data, new_data, __FUNCTION__)
 
 extern int register_fsa_input_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
                                   void *data, long long with_actions,
                                   gboolean prepend, const char *raised_from);
 
 extern void fsa_dump_queue(int log_level);
 extern void route_message(enum crmd_fsa_cause cause, xmlNode * input);
 
 #  define crmd_fsa_stall(suppress) do {                                 \
     if(suppress == FALSE && msg_data != NULL) {                         \
         register_fsa_input_adv(                                         \
             ((fsa_data_t*)msg_data)->fsa_cause, I_WAIT_FOR_EVENT,       \
             ((fsa_data_t*)msg_data)->data, action, TRUE, __FUNCTION__); \
     } else {                                                            \
         register_fsa_input_adv(                                         \
             C_FSA_INTERNAL, I_WAIT_FOR_EVENT,                           \
             NULL, action, TRUE, __FUNCTION__);                          \
     }                                                                   \
     } while(0)
 
 #  define register_fsa_input(cause, input, data) register_fsa_input_adv(cause, input, data, A_NOTHING, FALSE, __FUNCTION__)
 
 #  define register_fsa_action(action) {					\
 		fsa_actions |= action;					\
-		if(fsa_source) {					\
-			mainloop_set_trigger(fsa_source);			\
-		}							\
 		crm_debug("%s added action %s to the FSA",		\
 			  __FUNCTION__, fsa_action2string(action));	\
 	}
 
 #  define register_fsa_input_before(cause, input, data) register_fsa_input_adv(cause, input, data, A_NOTHING, TRUE, __FUNCTION__)
 
 #  define register_fsa_input_later(cause, input, data) register_fsa_input_adv(cause, input, data, A_NOTHING, FALSE, __FUNCTION__)
 
 void delete_fsa_input(fsa_data_t * fsa_data);
 
 fsa_data_t *get_message(void);
 
 extern gboolean relay_message(xmlNode * relay_message, gboolean originated_locally);
 
 gboolean crmd_is_proxy_session(const char *session);
 void crmd_proxy_send(const char *session, xmlNode *msg);
 
 bool controld_authorize_ipc_message(xmlNode *client_msg,
                                     pcmk__client_t *curr_client,
                                     const char *proxy_session);
 
 extern gboolean send_request(xmlNode * msg, char **msg_reference);
 
 extern ha_msg_input_t *copy_ha_msg_input(ha_msg_input_t * orig);
 
 void send_remote_state_message(const char *node_name, gboolean node_up);
 
 #endif
diff --git a/daemons/controld/controld_schedulerd.c b/daemons/controld/controld_schedulerd.c
index 3dfc7e4268..f3400a554a 100644
--- a/daemons/controld/controld_schedulerd.c
+++ b/daemons/controld/controld_schedulerd.c
@@ -1,468 +1,470 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <unistd.h>  /* pid_t, sleep, ssize_t */
 
 #include <crm/cib.h>
 #include <crm/cluster.h>
 #include <crm/common/xml.h>
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 
 #include <pacemaker-controld.h>
 
 static mainloop_io_t *pe_subsystem = NULL;
 
 /*!
  * \internal
  * \brief Close any scheduler connection and free associated memory
  */
 void
 pe_subsystem_free(void)
 {
     controld_clear_fsa_input_flags(R_PE_REQUIRED);
     if (pe_subsystem) {
         controld_expect_sched_reply(NULL);
         mainloop_del_ipc_client(pe_subsystem);
         pe_subsystem = NULL;
         controld_clear_fsa_input_flags(R_PE_CONNECTED);
     }
 }
 
 /*!
  * \internal
  * \brief Save CIB query result to file, raising FSA error
  *
  * \param[in] msg        Ignored
  * \param[in] call_id    Call ID of CIB query
  * \param[in] rc         Return code of CIB query
  * \param[in] output     Result of CIB query
  * \param[in] user_data  Unique identifier for filename (will be freed)
  *
  * \note This is intended to be called after a scheduler connection fails.
  */
 static void
 save_cib_contents(xmlNode *msg, int call_id, int rc, xmlNode *output,
                   void *user_data)
 {
     char *id = user_data;
 
     register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __FUNCTION__);
     CRM_CHECK(id != NULL, return);
 
     if (rc == pcmk_ok) {
         char *filename = crm_strdup_printf(PE_STATE_DIR "/pe-core-%s.bz2", id);
 
         if (write_xml_file(output, filename, TRUE) < 0) {
             crm_err("Could not save Cluster Information Base to %s after scheduler crash",
                     filename);
         } else {
             crm_notice("Saved Cluster Information Base to %s after scheduler crash",
                        filename);
         }
         free(filename);
     }
 }
 
 /*!
  * \internal
  * \brief Respond to scheduler connection failure
  *
  * \param[in] user_data  Ignored
  */
 static void
 pe_ipc_destroy(gpointer user_data)
 {
     // If we aren't connected to the scheduler, we can't expect a reply
     controld_expect_sched_reply(NULL);
 
     if (is_set(fsa_input_register, R_PE_REQUIRED)) {
         int rc = pcmk_ok;
         char *uuid_str = crm_generate_uuid();
 
         crm_crit("Connection to the scheduler failed "
                  CRM_XS " uuid=%s", uuid_str);
 
         /*
          * The scheduler died...
          *
          * Save the current CIB so that we have a chance of
          * figuring out what killed it.
          *
          * Delay raising the I_ERROR until the query below completes or
          * 5s is up, whichever comes first.
          *
          */
         rc = fsa_cib_conn->cmds->query(fsa_cib_conn, NULL, NULL, cib_scope_local);
         fsa_register_cib_callback(rc, FALSE, uuid_str, save_cib_contents);
 
     } else {
         crm_info("Connection to the scheduler released");
     }
 
     controld_clear_fsa_input_flags(R_PE_CONNECTED);
     pe_subsystem = NULL;
     mainloop_set_trigger(fsa_source);
     return;
 }
 
 /*!
  * \internal
  * \brief Handle message from scheduler connection
  *
  * \param[in] buffer    XML message (will be freed)
  * \param[in] length    Ignored
  * \param[in] userdata  Ignored
  *
  * \return 0
  */
 static int
 pe_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
 {
     xmlNode *msg = string2xml(buffer);
 
     if (msg) {
         route_message(C_IPC_MESSAGE, msg);
     }
     free_xml(msg);
     return 0;
 }
 
 /*!
  * \internal
  * \brief Make new connection to scheduler
  *
  * \return TRUE on success, FALSE otherwise
  */
 static bool
 pe_subsystem_new(void)
 {
     struct ipc_client_callbacks pe_callbacks = {
         .dispatch = pe_ipc_dispatch,
         .destroy = pe_ipc_destroy
     };
 
     controld_set_fsa_input_flags(R_PE_REQUIRED);
     pe_subsystem = mainloop_add_ipc_client(CRM_SYSTEM_PENGINE,
                                            G_PRIORITY_DEFAULT,
                                            5 * 1024 * 1024 /* 5MB */,
                                            NULL, &pe_callbacks);
     if (pe_subsystem == NULL) {
         return FALSE;
     }
     controld_set_fsa_input_flags(R_PE_CONNECTED);
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Send an XML message to the scheduler
  *
  * \param[in] cmd  XML message to send
  *
  * \return pcmk_ok on success, -errno otherwise
  */
 static int
 pe_subsystem_send(xmlNode *cmd)
 {
     if (pe_subsystem) {
         int sent = crm_ipc_send(mainloop_get_ipc_client(pe_subsystem), cmd,
                                 0, 0, NULL);
 
         if (sent == 0) {
             sent = -ENODATA;
         } else if (sent > 0) {
             sent = pcmk_ok;
         }
         return sent;
     }
     return -ENOTCONN;
 }
 
 static void do_pe_invoke_callback(xmlNode *msg, int call_id, int rc,
                                   xmlNode *output, void *user_data);
 
 /*	 A_PE_START, A_PE_STOP, O_PE_RESTART	*/
 void
 do_pe_control(long long action,
               enum crmd_fsa_cause cause,
               enum crmd_fsa_state cur_state,
               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     if (action & A_PE_STOP) {
         pe_subsystem_free();
     }
     if ((action & A_PE_START)
         && (is_not_set(fsa_input_register, R_PE_CONNECTED))) {
 
         if (cur_state == S_STOPPING) {
             crm_info("Ignoring request to connect to scheduler while shutting down");
 
         } else if (!pe_subsystem_new()) {
             crm_warn("Could not connect to scheduler");
             register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
         }
     }
 }
 
 int fsa_pe_query = 0;
 char *fsa_pe_ref = NULL;
 static mainloop_timer_t *controld_sched_timer = NULL;
 
 // @TODO Make this a configurable cluster option if there's demand for it
 #define SCHED_TIMEOUT_MS (120000)
 
 /*!
  * \internal
  * \brief Handle a timeout waiting for scheduler reply
  *
  * \param[in] user_data  Ignored
  *
  * \return FALSE (indicating that timer should not be restarted)
  */
 static gboolean
 controld_sched_timeout(gpointer user_data)
 {
     if (AM_I_DC) {
         /* If this node is the DC but can't communicate with the scheduler, just
          * exit (and likely get fenced) so this node doesn't interfere with any
          * further DC elections.
          *
          * @TODO We could try something less drastic first, like disconnecting
          * and reconnecting to the scheduler, but something is likely going
          * seriously wrong, so perhaps it's better to just fail as quickly as
          * possible.
          */
         crmd_exit(CRM_EX_FATAL);
     }
     return FALSE;
 }
 
 void
 controld_stop_sched_timer(void)
 {
     if (controld_sched_timer && fsa_pe_ref) {
         crm_trace("Stopping timer for scheduler reply %s", fsa_pe_ref);
     }
     mainloop_timer_stop(controld_sched_timer);
 }
 
 /*!
  * \internal
  * \brief Set the scheduler request currently being waited on
  *
  * \param[in] msg  Request to expect reply to (or NULL for none)
  */
 void
 controld_expect_sched_reply(xmlNode *msg)
 {
     char *ref = NULL;
 
     if (msg) {
         ref = crm_element_value_copy(msg, XML_ATTR_REFERENCE);
         CRM_ASSERT(ref != NULL);
 
         if (controld_sched_timer == NULL) {
             controld_sched_timer = mainloop_timer_add("scheduler_reply_timer",
                                                       SCHED_TIMEOUT_MS, FALSE,
                                                       controld_sched_timeout,
                                                       NULL);
         }
         mainloop_timer_start(controld_sched_timer);
     } else {
         controld_stop_sched_timer();
     }
     free(fsa_pe_ref);
     fsa_pe_ref = ref;
 }
 
 /*!
  * \internal
  * \brief Free the scheduler reply timer
  */
 void
 controld_free_sched_timer(void)
 {
     if (controld_sched_timer != NULL) {
         mainloop_timer_del(controld_sched_timer);
         controld_sched_timer = NULL;
     }
 }
 
 /*	 A_PE_INVOKE	*/
 void
 do_pe_invoke(long long action,
              enum crmd_fsa_cause cause,
              enum crmd_fsa_state cur_state,
              enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     if (AM_I_DC == FALSE) {
         crm_err("Not invoking scheduler because not DC: %s",
                 fsa_action2string(action));
         return;
     }
 
     if (is_set(fsa_input_register, R_PE_CONNECTED) == FALSE) {
         if (is_set(fsa_input_register, R_SHUTDOWN)) {
             crm_err("Cannot shut down gracefully without the scheduler");
             register_fsa_input_before(C_FSA_INTERNAL, I_TERMINATE, NULL);
 
         } else {
             crm_info("Waiting for the scheduler to connect");
             crmd_fsa_stall(FALSE);
             register_fsa_action(A_PE_START);
+            trigger_fsa();
         }
         return;
     }
 
     if (cur_state != S_POLICY_ENGINE) {
         crm_notice("Not invoking scheduler because in state %s",
                    fsa_state2string(cur_state));
         return;
     }
     if (is_set(fsa_input_register, R_HAVE_CIB) == FALSE) {
         crm_err("Attempted to invoke scheduler without consistent Cluster Information Base!");
 
         /* start the join from scratch */
         register_fsa_input_before(C_FSA_INTERNAL, I_ELECTION, NULL);
         return;
     }
 
     fsa_pe_query = fsa_cib_conn->cmds->query(fsa_cib_conn, NULL, NULL, cib_scope_local);
 
     crm_debug("Query %d: Requesting the current CIB: %s", fsa_pe_query,
               fsa_state2string(fsa_state));
 
     controld_expect_sched_reply(NULL);
     fsa_register_cib_callback(fsa_pe_query, FALSE, NULL, do_pe_invoke_callback);
 }
 
 static void
 force_local_option(xmlNode *xml, const char *attr_name, const char *attr_value)
 {
     int max = 0;
     int lpc = 0;
     char *xpath_string = NULL;
     xmlXPathObjectPtr xpathObj = NULL;
 
     xpath_string = crm_strdup_printf("%.128s//%s//nvpair[@name='%.128s']",
                                      get_object_path(XML_CIB_TAG_CRMCONFIG),
                                      XML_CIB_TAG_PROPSET, attr_name);
     xpathObj = xpath_search(xml, xpath_string);
     max = numXpathResults(xpathObj);
     free(xpath_string);
 
     for (lpc = 0; lpc < max; lpc++) {
         xmlNode *match = getXpathResult(xpathObj, lpc);
         crm_trace("Forcing %s/%s = %s", ID(match), attr_name, attr_value);
         crm_xml_add(match, XML_NVPAIR_ATTR_VALUE, attr_value);
     }
 
     if(max == 0) {
         xmlNode *configuration = NULL;
         xmlNode *crm_config = NULL;
         xmlNode *cluster_property_set = NULL;
 
         crm_trace("Creating %s-%s for %s=%s",
                   CIB_OPTIONS_FIRST, attr_name, attr_name, attr_value);
 
         configuration = find_entity(xml, XML_CIB_TAG_CONFIGURATION, NULL);
         if (configuration == NULL) {
             configuration = create_xml_node(xml, XML_CIB_TAG_CONFIGURATION);
         }
 
         crm_config = find_entity(configuration, XML_CIB_TAG_CRMCONFIG, NULL);
         if (crm_config == NULL) {
             crm_config = create_xml_node(configuration, XML_CIB_TAG_CRMCONFIG);
         }
 
         cluster_property_set = find_entity(crm_config, XML_CIB_TAG_PROPSET, NULL);
         if (cluster_property_set == NULL) {
             cluster_property_set = create_xml_node(crm_config, XML_CIB_TAG_PROPSET);
             crm_xml_add(cluster_property_set, XML_ATTR_ID, CIB_OPTIONS_FIRST);
         }
 
         xml = create_xml_node(cluster_property_set, XML_CIB_TAG_NVPAIR);
 
         crm_xml_set_id(xml, "%s-%s", CIB_OPTIONS_FIRST, attr_name);
         crm_xml_add(xml, XML_NVPAIR_ATTR_NAME, attr_name);
         crm_xml_add(xml, XML_NVPAIR_ATTR_VALUE, attr_value);
     }
     freeXpathObject(xpathObj);
 }
 
 static void
 do_pe_invoke_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     xmlNode *cmd = NULL;
     pid_t watchdog = pcmk_locate_sbd();
 
     if (rc != pcmk_ok) {
         crm_err("Could not retrieve the Cluster Information Base: %s "
                 CRM_XS " rc=%d call=%d", pcmk_strerror(rc), rc, call_id);
         register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __FUNCTION__);
         return;
 
     } else if (call_id != fsa_pe_query) {
         crm_trace("Skipping superseded CIB query: %d (current=%d)", call_id, fsa_pe_query);
         return;
 
     } else if (AM_I_DC == FALSE || is_set(fsa_input_register, R_PE_CONNECTED) == FALSE) {
         crm_debug("No need to invoke the scheduler anymore");
         return;
 
     } else if (fsa_state != S_POLICY_ENGINE) {
         crm_debug("Discarding scheduler request in state: %s",
                   fsa_state2string(fsa_state));
         return;
 
     /* this callback counts as 1 */
     } else if (num_cib_op_callbacks() > 1) {
         crm_debug("Re-asking for the CIB: %d other peer updates still pending",
                   (num_cib_op_callbacks() - 1));
         sleep(1);
         register_fsa_action(A_PE_INVOKE);
+        trigger_fsa();
         return;
 
     } else if (fsa_state != S_POLICY_ENGINE) {
         crm_err("Invoking scheduler in state: %s", fsa_state2string(fsa_state));
         return;
     }
 
     CRM_LOG_ASSERT(output != NULL);
 
     /* Refresh the remote node cache and the known node cache when the
      * scheduler is invoked */
     crm_peer_caches_refresh(output);
 
     crm_xml_add(output, XML_ATTR_DC_UUID, fsa_our_uuid);
     crm_xml_add_int(output, XML_ATTR_HAVE_QUORUM, fsa_has_quorum);
 
     force_local_option(output, XML_ATTR_HAVE_WATCHDOG, watchdog?"true":"false");
 
     if (ever_had_quorum && crm_have_quorum == FALSE) {
         crm_xml_add_int(output, XML_ATTR_QUORUM_PANIC, 1);
     }
 
     cmd = create_request(CRM_OP_PECALC, output, NULL, CRM_SYSTEM_PENGINE, CRM_SYSTEM_DC, NULL);
 
     rc = pe_subsystem_send(cmd);
     if (rc < 0) {
         crm_err("Could not contact the scheduler: %s " CRM_XS " rc=%d",
                 pcmk_strerror(rc), rc);
         register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __FUNCTION__);
     } else {
         controld_expect_sched_reply(cmd);
         crm_debug("Invoking the scheduler: query=%d, ref=%s, seq=%llu, quorate=%d",
                   fsa_pe_query, fsa_pe_ref, crm_peer_seq, fsa_has_quorum);
     }
     free_xml(cmd);
 }
diff --git a/daemons/controld/controld_te_actions.c b/daemons/controld/controld_te_actions.c
index 64e30bb90d..55903c2808 100644
--- a/daemons/controld/controld_te_actions.c
+++ b/daemons/controld/controld_te_actions.c
@@ -1,642 +1,643 @@
 /*
  * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/lrmd.h>               // lrmd_event_data_t, lrmd_free_event()
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/cluster.h>
 
 #include <pacemaker-internal.h>
 #include <pacemaker-controld.h>
 
 char *te_uuid = NULL;
 GHashTable *te_targets = NULL;
 void send_rsc_command(crm_action_t * action);
 static void te_update_job_count(crm_action_t * action, int offset);
 
 static void
 te_start_action_timer(crm_graph_t * graph, crm_action_t * action)
 {
     action->timer = calloc(1, sizeof(crm_action_timer_t));
     action->timer->timeout = action->timeout;
     action->timer->action = action;
     action->timer->source_id = g_timeout_add(action->timer->timeout + graph->network_delay,
                                              action_timer_callback, (void *)action->timer);
 
     CRM_ASSERT(action->timer->source_id != 0);
 }
 
 static gboolean
 te_pseudo_action(crm_graph_t * graph, crm_action_t * pseudo)
 {
     const char *task = crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK);
 
     /* send to peers as well? */
     if (pcmk__str_eq(task, CRM_OP_MAINTENANCE_NODES, pcmk__str_casei)) {
         GHashTableIter iter;
         crm_node_t *node = NULL;
 
         g_hash_table_iter_init(&iter, crm_peer_cache);
         while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
             xmlNode *cmd = NULL;
 
             if (pcmk__str_eq(fsa_our_uname, node->uname, pcmk__str_casei)) {
                 continue;
             }
 
             cmd = create_request(task, pseudo->xml, node->uname,
                                  CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
             send_cluster_message(node, crm_msg_crmd, cmd, FALSE);
             free_xml(cmd);
         }
 
         remote_ra_process_maintenance_nodes(pseudo->xml);
     } else {
         /* Check action for Pacemaker Remote node side effects */
         remote_ra_process_pseudo(pseudo->xml);
     }
 
     crm_debug("Pseudo-action %d (%s) fired and confirmed", pseudo->id,
               crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK_KEY));
     te_action_confirmed(pseudo, graph);
     return TRUE;
 }
 
 static int
 get_target_rc(crm_action_t * action)
 {
     const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);
 
     if (target_rc_s != NULL) {
         return crm_parse_int(target_rc_s, "0");
     }
     return 0;
 }
 
 static gboolean
 te_crm_command(crm_graph_t * graph, crm_action_t * action)
 {
     char *counter = NULL;
     xmlNode *cmd = NULL;
     gboolean is_local = FALSE;
 
     const char *id = NULL;
     const char *task = NULL;
     const char *value = NULL;
     const char *on_node = NULL;
     const char *router_node = NULL;
 
     gboolean rc = TRUE;
     gboolean no_wait = FALSE;
 
     id = ID(action->xml);
     task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 
     if (!router_node) {
         router_node = on_node;
         if (pcmk__str_eq(task, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
             const char *mode = crm_element_value(action->xml, PCMK__XA_MODE);
 
             if (pcmk__str_eq(mode, XML_TAG_CIB, pcmk__str_casei)) {
                 router_node = fsa_our_uname;
             }
         }
     }
 
     CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
               crm_err("Corrupted command (id=%s) %s: no node", crm_str(id), crm_str(task));
               return FALSE);
 
     if (pcmk__str_eq(router_node, fsa_our_uname, pcmk__str_casei)) {
         is_local = TRUE;
     }
 
     value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
     if (crm_is_true(value)) {
         no_wait = TRUE;
     }
 
     crm_info("Executing crm-event (%s)%s%s: %s on %s",
              crm_str(id), (is_local? " locally" : ""),
              (no_wait? " without waiting" : ""), crm_str(task), on_node);
 
     if (is_local && pcmk__str_eq(task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
         /* defer until everything else completes */
         crm_info("crm-event (%s) is a local shutdown", crm_str(id));
         graph->completion_action = tg_shutdown;
         graph->abort_reason = "local shutdown";
         te_action_confirmed(action, graph);
         return TRUE;
 
     } else if (pcmk__str_eq(task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
         crm_node_t *peer = crm_get_peer(0, router_node);
         crm_update_peer_expected(__FUNCTION__, peer, CRMD_JOINSTATE_DOWN);
     }
 
     cmd = create_request(task, action->xml, router_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
 
     counter = pcmk__transition_key(transition_graph->id, action->id,
                                    get_target_rc(action), te_uuid);
     crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter);
 
     rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_crmd, cmd, TRUE);
     free(counter);
     free_xml(cmd);
 
     if (rc == FALSE) {
         crm_err("Action %d failed: send", action->id);
         return FALSE;
 
     } else if (no_wait) {
         te_action_confirmed(action, graph);
 
     } else {
         if (action->timeout <= 0) {
             crm_err("Action %d: %s on %s had an invalid timeout (%dms).  Using %ums instead",
                     action->id, task, on_node, action->timeout, graph->network_delay);
             action->timeout = (int) graph->network_delay;
         }
         te_start_action_timer(graph, action);
     }
 
     return TRUE;
 }
 
 void
 controld_record_action_timeout(crm_action_t *action)
 {
     lrmd_event_data_t *op = NULL;
     xmlNode *state = NULL;
     xmlNode *rsc = NULL;
     xmlNode *xml_op = NULL;
     xmlNode *action_rsc = NULL;
 
     int rc = pcmk_ok;
 
     const char *rsc_id = NULL;
     const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
     const char *target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
 
     int call_options = cib_quorum_override | cib_scope_local;
     int target_rc = get_target_rc(action);
 
     crm_warn("%s %d: %s on %s timed out",
              crm_element_name(action->xml), action->id, task_uuid, target);
 
     action_rsc = find_xml_node(action->xml, XML_CIB_TAG_RESOURCE, TRUE);
     if (action_rsc == NULL) {
         return;
     }
 
     rsc_id = ID(action_rsc);
     CRM_CHECK(rsc_id != NULL,
               crm_log_xml_err(action->xml, "Bad:action"); return);
 
 /*
   update the CIB
 
 <node_state id="hadev">
       <lrm>
         <lrm_resources>
           <lrm_resource id="rsc2" last_op="start" op_code="0" target="hadev"/>
 */
 
     state = create_xml_node(NULL, XML_CIB_TAG_STATE);
 
     crm_xml_add(state, XML_ATTR_UUID, target_uuid);
     crm_xml_add(state, XML_ATTR_UNAME, target);
 
     rsc = create_xml_node(state, XML_CIB_TAG_LRM);
     crm_xml_add(rsc, XML_ATTR_ID, target_uuid);
 
     rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCES);
     rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCE);
     crm_xml_add(rsc, XML_ATTR_ID, rsc_id);
 
 
     crm_copy_xml_element(action_rsc, rsc, XML_ATTR_TYPE);
     crm_copy_xml_element(action_rsc, rsc, XML_AGENT_ATTR_CLASS);
     crm_copy_xml_element(action_rsc, rsc, XML_AGENT_ATTR_PROVIDER);
 
     /* If the executor gets a timeout while waiting for the action to complete,
      * that will be reported via the usual callback. This timeout means that we
      * didn't hear from the executor or the controller that relayed the action
      * to the executor.
      *
      * @TODO Using PCMK_OCF_UNKNOWN_ERROR instead of PCMK_OCF_TIMEOUT is one way
      * to distinguish those situations, but perhaps PCMK_OCF_TIMEOUT would be
      * preferable anyway.
      */
     op = convert_graph_action(NULL, action, PCMK_LRM_OP_TIMEOUT,
                               PCMK_OCF_UNKNOWN_ERROR);
     op->call_id = -1;
     op->user_data = pcmk__transition_key(transition_graph->id, action->id,
                                          target_rc, te_uuid);
 
     xml_op = pcmk__create_history_xml(rsc, op, CRM_FEATURE_SET, target_rc,
                                       target, __FUNCTION__, LOG_INFO);
     lrmd_free_event(op);
 
     crm_log_xml_trace(xml_op, "Action timeout");
 
     rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, state, call_options);
     fsa_register_cib_callback(rc, FALSE, NULL, cib_action_updated);
     free_xml(state);
 
     crm_trace("Sent CIB update (call ID %d) for timeout of action %d (%s on %s)",
               rc, action->id, task_uuid, target);
     action->sent_update = TRUE;
 }
 
 static gboolean
 te_rsc_command(crm_graph_t * graph, crm_action_t * action)
 {
     /* never overwrite stop actions in the CIB with
      *   anything other than completed results
      *
      * Writing pending stops makes it look like the
      *   resource is running again
      */
     xmlNode *cmd = NULL;
     xmlNode *rsc_op = NULL;
 
     gboolean rc = TRUE;
     gboolean no_wait = FALSE;
     gboolean is_local = FALSE;
 
     char *counter = NULL;
     const char *task = NULL;
     const char *value = NULL;
     const char *on_node = NULL;
     const char *router_node = NULL;
     const char *task_uuid = NULL;
 
     CRM_ASSERT(action != NULL);
     CRM_ASSERT(action->xml != NULL);
 
     action->executed = FALSE;
     on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 
     CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
               crm_err("Corrupted command(id=%s) %s: no node", ID(action->xml), crm_str(task));
               return FALSE);
 
     rsc_op = action->xml;
     task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
     task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
     router_node = crm_element_value(rsc_op, XML_LRM_ATTR_ROUTER_NODE);
 
     if (!router_node) {
         router_node = on_node;
     }
 
     counter = pcmk__transition_key(transition_graph->id, action->id,
                                    get_target_rc(action), te_uuid);
     crm_xml_add(rsc_op, XML_ATTR_TRANSITION_KEY, counter);
 
     if (pcmk__str_eq(router_node, fsa_our_uname, pcmk__str_casei)) {
         is_local = TRUE;
     }
 
     value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
     if (crm_is_true(value)) {
         no_wait = TRUE;
     }
 
     crm_notice("Initiating %s operation %s%s on %s%s "CRM_XS" action %d",
                task, task_uuid, (is_local? " locally" : ""), on_node,
                (no_wait? " without waiting" : ""), action->id);
 
     cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, router_node,
                          CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);
 
     if (is_local) {
         /* shortcut local resource commands */
         ha_msg_input_t data = {
             .msg = cmd,
             .xml = rsc_op,
         };
 
         fsa_data_t msg = {
             .id = 0,
             .data = &data,
             .data_type = fsa_dt_ha_msg,
             .fsa_input = I_NULL,
             .fsa_cause = C_FSA_INTERNAL,
             .actions = A_LRM_INVOKE,
             .origin = __FUNCTION__,
         };
 
         do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, fsa_state, I_NULL, &msg);
 
     } else {
         rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_lrmd, cmd, TRUE);
     }
 
     free(counter);
     free_xml(cmd);
 
     action->executed = TRUE;
 
     if (rc == FALSE) {
         crm_err("Action %d failed: send", action->id);
         return FALSE;
 
     } else if (no_wait) {
         crm_info("Action %d confirmed - no wait", action->id);
         action->confirmed = TRUE; /* Just mark confirmed.
                                    * Don't bump the job count only to immediately decrement it
                                    */
         update_graph(transition_graph, action);
         trigger_graph();
 
     } else if (action->confirmed == TRUE) {
         crm_debug("Action %d: %s %s on %s(timeout %dms) was already confirmed.",
                   action->id, task, task_uuid, on_node, action->timeout);
     } else {
         if (action->timeout <= 0) {
             crm_err("Action %d: %s %s on %s had an invalid timeout (%dms).  Using %ums instead",
                     action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
             action->timeout = (int) graph->network_delay;
         }
         te_update_job_count(action, 1);
         te_start_action_timer(graph, action);
     }
 
     return TRUE;
 }
 
 struct te_peer_s
 {
         char *name;
         int jobs;
         int migrate_jobs;
 };
 
 static void te_peer_free(gpointer p)
 {
     struct te_peer_s *peer = p;
 
     free(peer->name);
     free(peer);
 }
 
 void te_reset_job_counts(void)
 {
     GHashTableIter iter;
     struct te_peer_s *peer = NULL;
 
     if(te_targets == NULL) {
         te_targets = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, te_peer_free);
     }
 
     g_hash_table_iter_init(&iter, te_targets);
     while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & peer)) {
         peer->jobs = 0;
         peer->migrate_jobs = 0;
     }
 }
 
 static void
 te_update_job_count_on(const char *target, int offset, bool migrate)
 {
     struct te_peer_s *r = NULL;
 
     if(target == NULL || te_targets == NULL) {
         return;
     }
 
     r = g_hash_table_lookup(te_targets, target);
     if(r == NULL) {
         r = calloc(1, sizeof(struct te_peer_s));
         r->name = strdup(target);
         g_hash_table_insert(te_targets, r->name, r);
     }
 
     r->jobs += offset;
     if(migrate) {
         r->migrate_jobs += offset;
     }
     crm_trace("jobs[%s] = %d", target, r->jobs);
 }
 
 static void
 te_update_job_count(crm_action_t * action, int offset)
 {
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 
     if (action->type != action_type_rsc || target == NULL) {
         /* No limit on these */
         return;
     }
 
     /* if we have a router node, this means the action is performing
      * on a remote node. For now, we count all actions occurring on a
      * remote node against the job list on the cluster node hosting
      * the connection resources */
     target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 
     if ((target == NULL) && pcmk__strcase_any_of(task, CRMD_ACTION_MIGRATE,
                                                  CRMD_ACTION_MIGRATED, NULL)) {
 
         const char *t1 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
         const char *t2 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);
 
         te_update_job_count_on(t1, offset, TRUE);
         te_update_job_count_on(t2, offset, TRUE);
         return;
     } else if (target == NULL) {
         target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     }
 
     te_update_job_count_on(target, offset, FALSE);
 }
 
 static gboolean
 te_should_perform_action_on(crm_graph_t * graph, crm_action_t * action, const char *target)
 {
     int limit = 0;
     struct te_peer_s *r = NULL;
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     const char *id = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 
     if(target == NULL) {
         /* No limit on these */
         return TRUE;
 
     } else if(te_targets == NULL) {
         return FALSE;
     }
 
     r = g_hash_table_lookup(te_targets, target);
     limit = throttle_get_job_limit(target);
 
     if(r == NULL) {
         r = calloc(1, sizeof(struct te_peer_s));
         r->name = strdup(target);
         g_hash_table_insert(te_targets, r->name, r);
     }
 
     if(limit <= r->jobs) {
         crm_trace("Peer %s is over their job limit of %d (%d): deferring %s",
                   target, limit, r->jobs, id);
         return FALSE;
 
     } else if(graph->migration_limit > 0 && r->migrate_jobs >= graph->migration_limit) {
         if (pcmk__strcase_any_of(task, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
             crm_trace("Peer %s is over their migration job limit of %d (%d): deferring %s",
                       target, graph->migration_limit, r->migrate_jobs, id);
             return FALSE;
         }
     }
 
     crm_trace("Peer %s has not hit their limit yet. current jobs = %d limit= %d limit", target, r->jobs, limit);
 
     return TRUE;
 }
 
 static gboolean
 te_should_perform_action(crm_graph_t * graph, crm_action_t * action)
 {
     const char *target = NULL;
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 
     if (action->type != action_type_rsc) {
         /* No limit on these */
         return TRUE;
     }
 
     /* if we have a router node, this means the action is performing
      * on a remote node. For now, we count all actions occurring on a
      * remote node against the job list on the cluster node hosting
      * the connection resources */
     target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 
     if ((target == NULL) && pcmk__strcase_any_of(task, CRMD_ACTION_MIGRATE,
                                                  CRMD_ACTION_MIGRATED, NULL)) {
         target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
         if(te_should_perform_action_on(graph, action, target) == FALSE) {
             return FALSE;
         }
 
         target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);
 
     } else if (target == NULL) {
         target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     }
 
     return te_should_perform_action_on(graph, action, target);
 }
 
 /*!
  * \brief Confirm a graph action (and optionally update graph)
  *
  * \param[in] action  Action to confirm
  * \param[in] graph   Update and trigger this graph (if non-NULL)
  */
 void
 te_action_confirmed(crm_action_t *action, crm_graph_t *graph)
 {
     if (action->confirmed == FALSE) {
         if ((action->type == action_type_rsc)
             && (crm_element_value(action->xml, XML_LRM_ATTR_TARGET) != NULL)) {
             te_update_job_count(action, -1);
         }
         action->confirmed = TRUE;
     }
     if (graph) {
         update_graph(graph, action);
         trigger_graph();
     }
 }
 
 
 crm_graph_functions_t te_graph_fns = {
     te_pseudo_action,
     te_rsc_command,
     te_crm_command,
     te_fence_node,
     te_should_perform_action,
 };
 
 void
 notify_crmd(crm_graph_t * graph)
 {
     const char *type = "unknown";
     enum crmd_fsa_input event = I_NULL;
 
     crm_debug("Processing transition completion in state %s", fsa_state2string(fsa_state));
 
     if (graph->complete == FALSE) {
         CRM_CHECK(graph->complete,);
         graph->complete = TRUE;
     }
 
     switch (graph->completion_action) {
         case tg_stop:
             type = "stop";
             if (fsa_state == S_TRANSITION_ENGINE) {
                 event = I_TE_SUCCESS;
             }
             break;
         case tg_done:
             type = "done";
             if (fsa_state == S_TRANSITION_ENGINE) {
                 event = I_TE_SUCCESS;
             }
             break;
 
         case tg_restart:
             type = "restart";
             if (fsa_state == S_TRANSITION_ENGINE) {
                 if (transition_timer->period_ms > 0) {
                     controld_stop_timer(transition_timer);
                     controld_start_timer(transition_timer);
                 } else {
                     event = I_PE_CALC;
                 }
 
             } else if (fsa_state == S_POLICY_ENGINE) {
                 register_fsa_action(A_PE_INVOKE);
+                trigger_fsa();
             }
             break;
 
         case tg_shutdown:
             type = "shutdown";
             if (is_set(fsa_input_register, R_SHUTDOWN)) {
                 event = I_STOP;
 
             } else {
                 crm_err("We didn't ask to be shut down, yet the scheduler is telling us to");
                 event = I_TERMINATE;
             }
     }
 
     crm_debug("Transition %d status: %s - %s", graph->id, type, crm_str(graph->abort_reason));
 
     graph->abort_reason = NULL;
     graph->completion_action = tg_done;
     controld_clear_fsa_input_flags(R_IN_TRANSITION);
 
     if (event != I_NULL) {
         register_fsa_input(C_FSA_INTERNAL, event, NULL);
 
     } else if (fsa_source) {
         mainloop_set_trigger(fsa_source);
     }
 }
diff --git a/daemons/controld/controld_utils.c b/daemons/controld/controld_utils.c
index f4c5927573..b8ad55fac1 100644
--- a/daemons/controld/controld_utils.c
+++ b/daemons/controld/controld_utils.c
@@ -1,873 +1,874 @@
 /*
- * Copyright 2004-2019 the Pacemaker project contributors
+ * Copyright 2004-2020 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdlib.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <pacemaker-controld.h>
 
 const char *
 fsa_input2string(enum crmd_fsa_input input)
 {
     const char *inputAsText = NULL;
 
     switch (input) {
         case I_NULL:
             inputAsText = "I_NULL";
             break;
         case I_CIB_OP:
             inputAsText = "I_CIB_OP (unused)";
             break;
         case I_CIB_UPDATE:
             inputAsText = "I_CIB_UPDATE";
             break;
         case I_DC_TIMEOUT:
             inputAsText = "I_DC_TIMEOUT";
             break;
         case I_ELECTION:
             inputAsText = "I_ELECTION";
             break;
         case I_PE_CALC:
             inputAsText = "I_PE_CALC";
             break;
         case I_RELEASE_DC:
             inputAsText = "I_RELEASE_DC";
             break;
         case I_ELECTION_DC:
             inputAsText = "I_ELECTION_DC";
             break;
         case I_ERROR:
             inputAsText = "I_ERROR";
             break;
         case I_FAIL:
             inputAsText = "I_FAIL";
             break;
         case I_INTEGRATED:
             inputAsText = "I_INTEGRATED";
             break;
         case I_FINALIZED:
             inputAsText = "I_FINALIZED";
             break;
         case I_NODE_JOIN:
             inputAsText = "I_NODE_JOIN";
             break;
         case I_JOIN_OFFER:
             inputAsText = "I_JOIN_OFFER";
             break;
         case I_JOIN_REQUEST:
             inputAsText = "I_JOIN_REQUEST";
             break;
         case I_JOIN_RESULT:
             inputAsText = "I_JOIN_RESULT";
             break;
         case I_NOT_DC:
             inputAsText = "I_NOT_DC";
             break;
         case I_RECOVERED:
             inputAsText = "I_RECOVERED";
             break;
         case I_RELEASE_FAIL:
             inputAsText = "I_RELEASE_FAIL";
             break;
         case I_RELEASE_SUCCESS:
             inputAsText = "I_RELEASE_SUCCESS";
             break;
         case I_RESTART:
             inputAsText = "I_RESTART";
             break;
         case I_PE_SUCCESS:
             inputAsText = "I_PE_SUCCESS";
             break;
         case I_ROUTER:
             inputAsText = "I_ROUTER";
             break;
         case I_SHUTDOWN:
             inputAsText = "I_SHUTDOWN";
             break;
         case I_STARTUP:
             inputAsText = "I_STARTUP";
             break;
         case I_TE_SUCCESS:
             inputAsText = "I_TE_SUCCESS";
             break;
         case I_STOP:
             inputAsText = "I_STOP";
             break;
         case I_DC_HEARTBEAT:
             inputAsText = "I_DC_HEARTBEAT";
             break;
         case I_WAIT_FOR_EVENT:
             inputAsText = "I_WAIT_FOR_EVENT";
             break;
         case I_LRM_EVENT:
             inputAsText = "I_LRM_EVENT";
             break;
         case I_PENDING:
             inputAsText = "I_PENDING";
             break;
         case I_HALT:
             inputAsText = "I_HALT";
             break;
         case I_TERMINATE:
             inputAsText = "I_TERMINATE";
             break;
         case I_ILLEGAL:
             inputAsText = "I_ILLEGAL";
             break;
     }
 
     if (inputAsText == NULL) {
         crm_err("Input %d is unknown", input);
         inputAsText = "<UNKNOWN_INPUT>";
     }
 
     return inputAsText;
 }
 
 const char *
 fsa_state2string(enum crmd_fsa_state state)
 {
     const char *stateAsText = NULL;
 
     switch (state) {
         case S_IDLE:
             stateAsText = "S_IDLE";
             break;
         case S_ELECTION:
             stateAsText = "S_ELECTION";
             break;
         case S_INTEGRATION:
             stateAsText = "S_INTEGRATION";
             break;
         case S_FINALIZE_JOIN:
             stateAsText = "S_FINALIZE_JOIN";
             break;
         case S_NOT_DC:
             stateAsText = "S_NOT_DC";
             break;
         case S_POLICY_ENGINE:
             stateAsText = "S_POLICY_ENGINE";
             break;
         case S_RECOVERY:
             stateAsText = "S_RECOVERY";
             break;
         case S_RELEASE_DC:
             stateAsText = "S_RELEASE_DC";
             break;
         case S_PENDING:
             stateAsText = "S_PENDING";
             break;
         case S_STOPPING:
             stateAsText = "S_STOPPING";
             break;
         case S_TERMINATE:
             stateAsText = "S_TERMINATE";
             break;
         case S_TRANSITION_ENGINE:
             stateAsText = "S_TRANSITION_ENGINE";
             break;
         case S_STARTING:
             stateAsText = "S_STARTING";
             break;
         case S_HALT:
             stateAsText = "S_HALT";
             break;
         case S_ILLEGAL:
             stateAsText = "S_ILLEGAL";
             break;
     }
 
     if (stateAsText == NULL) {
         crm_err("State %d is unknown", state);
         stateAsText = "<UNKNOWN_STATE>";
     }
 
     return stateAsText;
 }
 
 const char *
 fsa_cause2string(enum crmd_fsa_cause cause)
 {
     const char *causeAsText = NULL;
 
     switch (cause) {
         case C_UNKNOWN:
             causeAsText = "C_UNKNOWN";
             break;
         case C_STARTUP:
             causeAsText = "C_STARTUP";
             break;
         case C_IPC_MESSAGE:
             causeAsText = "C_IPC_MESSAGE";
             break;
         case C_HA_MESSAGE:
             causeAsText = "C_HA_MESSAGE";
             break;
         case C_TIMER_POPPED:
             causeAsText = "C_TIMER_POPPED";
             break;
         case C_SHUTDOWN:
             causeAsText = "C_SHUTDOWN";
             break;
         case C_LRM_OP_CALLBACK:
             causeAsText = "C_LRM_OP_CALLBACK";
             break;
         case C_CRMD_STATUS_CALLBACK:
             causeAsText = "C_CRMD_STATUS_CALLBACK";
             break;
         case C_FSA_INTERNAL:
             causeAsText = "C_FSA_INTERNAL";
             break;
     }
 
     if (causeAsText == NULL) {
         crm_err("Cause %d is unknown", cause);
         causeAsText = "<UNKNOWN_CAUSE>";
     }
 
     return causeAsText;
 }
 
 const char *
 fsa_action2string(long long action)
 {
     const char *actionAsText = NULL;
 
     switch (action) {
 
         case A_NOTHING:
             actionAsText = "A_NOTHING";
             break;
         case A_ELECTION_START:
             actionAsText = "A_ELECTION_START";
             break;
         case A_DC_JOIN_FINAL:
             actionAsText = "A_DC_JOIN_FINAL";
             break;
         case A_READCONFIG:
             actionAsText = "A_READCONFIG";
             break;
         case O_RELEASE:
             actionAsText = "O_RELEASE";
             break;
         case A_STARTUP:
             actionAsText = "A_STARTUP";
             break;
         case A_STARTED:
             actionAsText = "A_STARTED";
             break;
         case A_HA_CONNECT:
             actionAsText = "A_HA_CONNECT";
             break;
         case A_HA_DISCONNECT:
             actionAsText = "A_HA_DISCONNECT";
             break;
         case A_LRM_CONNECT:
             actionAsText = "A_LRM_CONNECT";
             break;
         case A_LRM_EVENT:
             actionAsText = "A_LRM_EVENT";
             break;
         case A_LRM_INVOKE:
             actionAsText = "A_LRM_INVOKE";
             break;
         case A_LRM_DISCONNECT:
             actionAsText = "A_LRM_DISCONNECT";
             break;
         case O_LRM_RECONNECT:
             actionAsText = "O_LRM_RECONNECT";
             break;
         case A_CL_JOIN_QUERY:
             actionAsText = "A_CL_JOIN_QUERY";
             break;
         case A_DC_TIMER_STOP:
             actionAsText = "A_DC_TIMER_STOP";
             break;
         case A_DC_TIMER_START:
             actionAsText = "A_DC_TIMER_START";
             break;
         case A_INTEGRATE_TIMER_START:
             actionAsText = "A_INTEGRATE_TIMER_START";
             break;
         case A_INTEGRATE_TIMER_STOP:
             actionAsText = "A_INTEGRATE_TIMER_STOP";
             break;
         case A_FINALIZE_TIMER_START:
             actionAsText = "A_FINALIZE_TIMER_START";
             break;
         case A_FINALIZE_TIMER_STOP:
             actionAsText = "A_FINALIZE_TIMER_STOP";
             break;
         case A_ELECTION_COUNT:
             actionAsText = "A_ELECTION_COUNT";
             break;
         case A_ELECTION_VOTE:
             actionAsText = "A_ELECTION_VOTE";
             break;
         case A_ELECTION_CHECK:
             actionAsText = "A_ELECTION_CHECK";
             break;
         case A_CL_JOIN_ANNOUNCE:
             actionAsText = "A_CL_JOIN_ANNOUNCE";
             break;
         case A_CL_JOIN_REQUEST:
             actionAsText = "A_CL_JOIN_REQUEST";
             break;
         case A_CL_JOIN_RESULT:
             actionAsText = "A_CL_JOIN_RESULT";
             break;
         case A_DC_JOIN_OFFER_ALL:
             actionAsText = "A_DC_JOIN_OFFER_ALL";
             break;
         case A_DC_JOIN_OFFER_ONE:
             actionAsText = "A_DC_JOIN_OFFER_ONE";
             break;
         case A_DC_JOIN_PROCESS_REQ:
             actionAsText = "A_DC_JOIN_PROCESS_REQ";
             break;
         case A_DC_JOIN_PROCESS_ACK:
             actionAsText = "A_DC_JOIN_PROCESS_ACK";
             break;
         case A_DC_JOIN_FINALIZE:
             actionAsText = "A_DC_JOIN_FINALIZE";
             break;
         case A_MSG_PROCESS:
             actionAsText = "A_MSG_PROCESS";
             break;
         case A_MSG_ROUTE:
             actionAsText = "A_MSG_ROUTE";
             break;
         case A_RECOVER:
             actionAsText = "A_RECOVER";
             break;
         case A_DC_RELEASE:
             actionAsText = "A_DC_RELEASE";
             break;
         case A_DC_RELEASED:
             actionAsText = "A_DC_RELEASED";
             break;
         case A_DC_TAKEOVER:
             actionAsText = "A_DC_TAKEOVER";
             break;
         case A_SHUTDOWN:
             actionAsText = "A_SHUTDOWN";
             break;
         case A_SHUTDOWN_REQ:
             actionAsText = "A_SHUTDOWN_REQ";
             break;
         case A_STOP:
             actionAsText = "A_STOP  ";
             break;
         case A_EXIT_0:
             actionAsText = "A_EXIT_0";
             break;
         case A_EXIT_1:
             actionAsText = "A_EXIT_1";
             break;
         case O_CIB_RESTART:
             actionAsText = "O_CIB_RESTART";
             break;
         case A_CIB_START:
             actionAsText = "A_CIB_START";
             break;
         case A_CIB_STOP:
             actionAsText = "A_CIB_STOP";
             break;
         case A_TE_INVOKE:
             actionAsText = "A_TE_INVOKE";
             break;
         case O_TE_RESTART:
             actionAsText = "O_TE_RESTART";
             break;
         case A_TE_START:
             actionAsText = "A_TE_START";
             break;
         case A_TE_STOP:
             actionAsText = "A_TE_STOP";
             break;
         case A_TE_HALT:
             actionAsText = "A_TE_HALT";
             break;
         case A_TE_CANCEL:
             actionAsText = "A_TE_CANCEL";
             break;
         case A_PE_INVOKE:
             actionAsText = "A_PE_INVOKE";
             break;
         case O_PE_RESTART:
             actionAsText = "O_PE_RESTART";
             break;
         case A_PE_START:
             actionAsText = "A_PE_START";
             break;
         case A_PE_STOP:
             actionAsText = "A_PE_STOP";
             break;
         case A_NODE_BLOCK:
             actionAsText = "A_NODE_BLOCK";
             break;
         case A_UPDATE_NODESTATUS:
             actionAsText = "A_UPDATE_NODESTATUS";
             break;
         case A_LOG:
             actionAsText = "A_LOG   ";
             break;
         case A_ERROR:
             actionAsText = "A_ERROR ";
             break;
         case A_WARN:
             actionAsText = "A_WARN  ";
             break;
             /* Composite actions */
         case A_DC_TIMER_START | A_CL_JOIN_QUERY:
             actionAsText = "A_DC_TIMER_START|A_CL_JOIN_QUERY";
             break;
     }
 
     if (actionAsText == NULL) {
         crm_err("Action %.16llx is unknown", action);
         actionAsText = "<UNKNOWN_ACTION>";
     }
 
     return actionAsText;
 }
 
 void
 fsa_dump_inputs(int log_level, const char *text, long long input_register)
 {
     if (input_register == A_NOTHING) {
         return;
     }
     if (text == NULL) {
         text = "Input register contents:";
     }
 
     if (is_set(input_register, R_THE_DC)) {
         crm_trace("%s %.16llx (R_THE_DC)", text, R_THE_DC);
     }
     if (is_set(input_register, R_STARTING)) {
         crm_trace("%s %.16llx (R_STARTING)", text, R_STARTING);
     }
     if (is_set(input_register, R_SHUTDOWN)) {
         crm_trace("%s %.16llx (R_SHUTDOWN)", text, R_SHUTDOWN);
     }
     if (is_set(input_register, R_STAYDOWN)) {
         crm_trace("%s %.16llx (R_STAYDOWN)", text, R_STAYDOWN);
     }
     if (is_set(input_register, R_JOIN_OK)) {
         crm_trace("%s %.16llx (R_JOIN_OK)", text, R_JOIN_OK);
     }
     if (is_set(input_register, R_READ_CONFIG)) {
         crm_trace("%s %.16llx (R_READ_CONFIG)", text, R_READ_CONFIG);
     }
     if (is_set(input_register, R_INVOKE_PE)) {
         crm_trace("%s %.16llx (R_INVOKE_PE)", text, R_INVOKE_PE);
     }
     if (is_set(input_register, R_CIB_CONNECTED)) {
         crm_trace("%s %.16llx (R_CIB_CONNECTED)", text, R_CIB_CONNECTED);
     }
     if (is_set(input_register, R_PE_CONNECTED)) {
         crm_trace("%s %.16llx (R_PE_CONNECTED)", text, R_PE_CONNECTED);
     }
     if (is_set(input_register, R_TE_CONNECTED)) {
         crm_trace("%s %.16llx (R_TE_CONNECTED)", text, R_TE_CONNECTED);
     }
     if (is_set(input_register, R_LRM_CONNECTED)) {
         crm_trace("%s %.16llx (R_LRM_CONNECTED)", text, R_LRM_CONNECTED);
     }
     if (is_set(input_register, R_CIB_REQUIRED)) {
         crm_trace("%s %.16llx (R_CIB_REQUIRED)", text, R_CIB_REQUIRED);
     }
     if (is_set(input_register, R_PE_REQUIRED)) {
         crm_trace("%s %.16llx (R_PE_REQUIRED)", text, R_PE_REQUIRED);
     }
     if (is_set(input_register, R_TE_REQUIRED)) {
         crm_trace("%s %.16llx (R_TE_REQUIRED)", text, R_TE_REQUIRED);
     }
     if (is_set(input_register, R_REQ_PEND)) {
         crm_trace("%s %.16llx (R_REQ_PEND)", text, R_REQ_PEND);
     }
     if (is_set(input_register, R_PE_PEND)) {
         crm_trace("%s %.16llx (R_PE_PEND)", text, R_PE_PEND);
     }
     if (is_set(input_register, R_TE_PEND)) {
         crm_trace("%s %.16llx (R_TE_PEND)", text, R_TE_PEND);
     }
     if (is_set(input_register, R_RESP_PEND)) {
         crm_trace("%s %.16llx (R_RESP_PEND)", text, R_RESP_PEND);
     }
     if (is_set(input_register, R_CIB_DONE)) {
         crm_trace("%s %.16llx (R_CIB_DONE)", text, R_CIB_DONE);
     }
     if (is_set(input_register, R_HAVE_CIB)) {
         crm_trace("%s %.16llx (R_HAVE_CIB)", text, R_HAVE_CIB);
     }
     if (is_set(input_register, R_CIB_ASKED)) {
         crm_trace("%s %.16llx (R_CIB_ASKED)", text, R_CIB_ASKED);
     }
     if (is_set(input_register, R_MEMBERSHIP)) {
         crm_trace("%s %.16llx (R_MEMBERSHIP)", text, R_MEMBERSHIP);
     }
     if (is_set(input_register, R_PEER_DATA)) {
         crm_trace("%s %.16llx (R_PEER_DATA)", text, R_PEER_DATA);
     }
     if (is_set(input_register, R_IN_RECOVERY)) {
         crm_trace("%s %.16llx (R_IN_RECOVERY)", text, R_IN_RECOVERY);
     }
 }
 
 void
 fsa_dump_actions(long long action, const char *text)
 {
     if (is_set(action, A_READCONFIG)) {
         crm_trace("Action %.16llx (A_READCONFIG) %s", A_READCONFIG, text);
     }
     if (is_set(action, A_STARTUP)) {
         crm_trace("Action %.16llx (A_STARTUP) %s", A_STARTUP, text);
     }
     if (is_set(action, A_STARTED)) {
         crm_trace("Action %.16llx (A_STARTED) %s", A_STARTED, text);
     }
     if (is_set(action, A_HA_CONNECT)) {
         crm_trace("Action %.16llx (A_CONNECT) %s", A_HA_CONNECT, text);
     }
     if (is_set(action, A_HA_DISCONNECT)) {
         crm_trace("Action %.16llx (A_DISCONNECT) %s", A_HA_DISCONNECT, text);
     }
     if (is_set(action, A_LRM_CONNECT)) {
         crm_trace("Action %.16llx (A_LRM_CONNECT) %s", A_LRM_CONNECT, text);
     }
     if (is_set(action, A_LRM_EVENT)) {
         crm_trace("Action %.16llx (A_LRM_EVENT) %s", A_LRM_EVENT, text);
     }
     if (is_set(action, A_LRM_INVOKE)) {
         crm_trace("Action %.16llx (A_LRM_INVOKE) %s", A_LRM_INVOKE, text);
     }
     if (is_set(action, A_LRM_DISCONNECT)) {
         crm_trace("Action %.16llx (A_LRM_DISCONNECT) %s", A_LRM_DISCONNECT, text);
     }
     if (is_set(action, A_DC_TIMER_STOP)) {
         crm_trace("Action %.16llx (A_DC_TIMER_STOP) %s", A_DC_TIMER_STOP, text);
     }
     if (is_set(action, A_DC_TIMER_START)) {
         crm_trace("Action %.16llx (A_DC_TIMER_START) %s", A_DC_TIMER_START, text);
     }
     if (is_set(action, A_INTEGRATE_TIMER_START)) {
         crm_trace("Action %.16llx (A_INTEGRATE_TIMER_START) %s", A_INTEGRATE_TIMER_START, text);
     }
     if (is_set(action, A_INTEGRATE_TIMER_STOP)) {
         crm_trace("Action %.16llx (A_INTEGRATE_TIMER_STOP) %s", A_INTEGRATE_TIMER_STOP, text);
     }
     if (is_set(action, A_FINALIZE_TIMER_START)) {
         crm_trace("Action %.16llx (A_FINALIZE_TIMER_START) %s", A_FINALIZE_TIMER_START, text);
     }
     if (is_set(action, A_FINALIZE_TIMER_STOP)) {
         crm_trace("Action %.16llx (A_FINALIZE_TIMER_STOP) %s", A_FINALIZE_TIMER_STOP, text);
     }
     if (is_set(action, A_ELECTION_COUNT)) {
         crm_trace("Action %.16llx (A_ELECTION_COUNT) %s", A_ELECTION_COUNT, text);
     }
     if (is_set(action, A_ELECTION_VOTE)) {
         crm_trace("Action %.16llx (A_ELECTION_VOTE) %s", A_ELECTION_VOTE, text);
     }
     if (is_set(action, A_ELECTION_CHECK)) {
         crm_trace("Action %.16llx (A_ELECTION_CHECK) %s", A_ELECTION_CHECK, text);
     }
     if (is_set(action, A_CL_JOIN_ANNOUNCE)) {
         crm_trace("Action %.16llx (A_CL_JOIN_ANNOUNCE) %s", A_CL_JOIN_ANNOUNCE, text);
     }
     if (is_set(action, A_CL_JOIN_REQUEST)) {
         crm_trace("Action %.16llx (A_CL_JOIN_REQUEST) %s", A_CL_JOIN_REQUEST, text);
     }
     if (is_set(action, A_CL_JOIN_RESULT)) {
         crm_trace("Action %.16llx (A_CL_JOIN_RESULT) %s", A_CL_JOIN_RESULT, text);
     }
     if (is_set(action, A_DC_JOIN_OFFER_ALL)) {
         crm_trace("Action %.16llx (A_DC_JOIN_OFFER_ALL) %s", A_DC_JOIN_OFFER_ALL, text);
     }
     if (is_set(action, A_DC_JOIN_OFFER_ONE)) {
         crm_trace("Action %.16llx (A_DC_JOIN_OFFER_ONE) %s", A_DC_JOIN_OFFER_ONE, text);
     }
     if (is_set(action, A_DC_JOIN_PROCESS_REQ)) {
         crm_trace("Action %.16llx (A_DC_JOIN_PROCESS_REQ) %s", A_DC_JOIN_PROCESS_REQ, text);
     }
     if (is_set(action, A_DC_JOIN_PROCESS_ACK)) {
         crm_trace("Action %.16llx (A_DC_JOIN_PROCESS_ACK) %s", A_DC_JOIN_PROCESS_ACK, text);
     }
     if (is_set(action, A_DC_JOIN_FINALIZE)) {
         crm_trace("Action %.16llx (A_DC_JOIN_FINALIZE) %s", A_DC_JOIN_FINALIZE, text);
     }
     if (is_set(action, A_MSG_PROCESS)) {
         crm_trace("Action %.16llx (A_MSG_PROCESS) %s", A_MSG_PROCESS, text);
     }
     if (is_set(action, A_MSG_ROUTE)) {
         crm_trace("Action %.16llx (A_MSG_ROUTE) %s", A_MSG_ROUTE, text);
     }
     if (is_set(action, A_RECOVER)) {
         crm_trace("Action %.16llx (A_RECOVER) %s", A_RECOVER, text);
     }
     if (is_set(action, A_DC_RELEASE)) {
         crm_trace("Action %.16llx (A_DC_RELEASE) %s", A_DC_RELEASE, text);
     }
     if (is_set(action, A_DC_RELEASED)) {
         crm_trace("Action %.16llx (A_DC_RELEASED) %s", A_DC_RELEASED, text);
     }
     if (is_set(action, A_DC_TAKEOVER)) {
         crm_trace("Action %.16llx (A_DC_TAKEOVER) %s", A_DC_TAKEOVER, text);
     }
     if (is_set(action, A_SHUTDOWN)) {
         crm_trace("Action %.16llx (A_SHUTDOWN) %s", A_SHUTDOWN, text);
     }
     if (is_set(action, A_SHUTDOWN_REQ)) {
         crm_trace("Action %.16llx (A_SHUTDOWN_REQ) %s", A_SHUTDOWN_REQ, text);
     }
     if (is_set(action, A_STOP)) {
         crm_trace("Action %.16llx (A_STOP  ) %s", A_STOP, text);
     }
     if (is_set(action, A_EXIT_0)) {
         crm_trace("Action %.16llx (A_EXIT_0) %s", A_EXIT_0, text);
     }
     if (is_set(action, A_EXIT_1)) {
         crm_trace("Action %.16llx (A_EXIT_1) %s", A_EXIT_1, text);
     }
     if (is_set(action, A_CIB_START)) {
         crm_trace("Action %.16llx (A_CIB_START) %s", A_CIB_START, text);
     }
     if (is_set(action, A_CIB_STOP)) {
         crm_trace("Action %.16llx (A_CIB_STOP) %s", A_CIB_STOP, text);
     }
     if (is_set(action, A_TE_INVOKE)) {
         crm_trace("Action %.16llx (A_TE_INVOKE) %s", A_TE_INVOKE, text);
     }
     if (is_set(action, A_TE_START)) {
         crm_trace("Action %.16llx (A_TE_START) %s", A_TE_START, text);
     }
     if (is_set(action, A_TE_STOP)) {
         crm_trace("Action %.16llx (A_TE_STOP) %s", A_TE_STOP, text);
     }
     if (is_set(action, A_TE_CANCEL)) {
         crm_trace("Action %.16llx (A_TE_CANCEL) %s", A_TE_CANCEL, text);
     }
     if (is_set(action, A_PE_INVOKE)) {
         crm_trace("Action %.16llx (A_PE_INVOKE) %s", A_PE_INVOKE, text);
     }
     if (is_set(action, A_PE_START)) {
         crm_trace("Action %.16llx (A_PE_START) %s", A_PE_START, text);
     }
     if (is_set(action, A_PE_STOP)) {
         crm_trace("Action %.16llx (A_PE_STOP) %s", A_PE_STOP, text);
     }
     if (is_set(action, A_NODE_BLOCK)) {
         crm_trace("Action %.16llx (A_NODE_BLOCK) %s", A_NODE_BLOCK, text);
     }
     if (is_set(action, A_UPDATE_NODESTATUS)) {
         crm_trace("Action %.16llx (A_UPDATE_NODESTATUS) %s", A_UPDATE_NODESTATUS, text);
     }
     if (is_set(action, A_LOG)) {
         crm_trace("Action %.16llx (A_LOG   ) %s", A_LOG, text);
     }
     if (is_set(action, A_ERROR)) {
         crm_trace("Action %.16llx (A_ERROR ) %s", A_ERROR, text);
     }
     if (is_set(action, A_WARN)) {
         crm_trace("Action %.16llx (A_WARN  ) %s", A_WARN, text);
     }
 }
 
 gboolean
 update_dc(xmlNode * msg)
 {
     char *last_dc = fsa_our_dc;
     const char *dc_version = NULL;
     const char *welcome_from = NULL;
 
     if (msg != NULL) {
         gboolean invalid = FALSE;
 
         dc_version = crm_element_value(msg, F_CRM_VERSION);
         welcome_from = crm_element_value(msg, F_CRM_HOST_FROM);
 
         CRM_CHECK(dc_version != NULL, return FALSE);
         CRM_CHECK(welcome_from != NULL, return FALSE);
 
         if (AM_I_DC && !pcmk__str_eq(welcome_from, fsa_our_uname, pcmk__str_casei)) {
             invalid = TRUE;
 
         } else if (fsa_our_dc && !pcmk__str_eq(welcome_from, fsa_our_dc, pcmk__str_casei)) {
             invalid = TRUE;
         }
 
         if (invalid) {
             CRM_CHECK(fsa_our_dc != NULL, crm_err("We have no DC"));
             if (AM_I_DC) {
                 crm_err("Not updating DC to %s (%s): we are also a DC", welcome_from, dc_version);
             } else {
                 crm_warn("New DC %s is not %s", welcome_from, fsa_our_dc);
             }
 
             register_fsa_action(A_CL_JOIN_QUERY | A_DC_TIMER_START);
+            trigger_fsa();
             return FALSE;
         }
     }
 
     free(fsa_our_dc_version);
     fsa_our_dc_version = NULL;
 
     fsa_our_dc = NULL;          /* Free'd as last_dc */
 
     if (welcome_from != NULL) {
         fsa_our_dc = strdup(welcome_from);
     }
     if (dc_version != NULL) {
         fsa_our_dc_version = strdup(dc_version);
     }
 
     if (pcmk__str_eq(fsa_our_dc, last_dc, pcmk__str_casei)) {
         /* do nothing */
 
     } else if (fsa_our_dc != NULL) {
         crm_node_t *dc_node = crm_get_peer(0, fsa_our_dc);
 
         crm_info("Set DC to %s (%s)", crm_str(fsa_our_dc), crm_str(fsa_our_dc_version));
         crm_update_peer_expected(__FUNCTION__, dc_node, CRMD_JOINSTATE_MEMBER);
 
     } else if (last_dc != NULL) {
         crm_info("Unset DC. Was %s", crm_str(last_dc));
     }
 
     free(last_dc);
     return TRUE;
 }
 
 void crmd_peer_down(crm_node_t *peer, bool full) 
 {
     if(full && peer->state == NULL) {
         crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_LOST, 0);
         crm_update_peer_proc(__FUNCTION__, peer, crm_proc_none, NULL);
     }
     crm_update_peer_join(__FUNCTION__, peer, crm_join_none);
     crm_update_peer_expected(__FUNCTION__, peer, CRMD_JOINSTATE_DOWN);
 }
 
 #define MIN_CIB_OP_TIMEOUT (30)
 
 unsigned int
 cib_op_timeout()
 {
     static int env_timeout = -1;
     unsigned int calculated_timeout = 0;
 
     if (env_timeout == -1) {
         const char *env = getenv("PCMK_cib_timeout");
 
         if (env) {
             env_timeout = crm_parse_int(env, "0");
         }
         env_timeout = QB_MAX(env_timeout, MIN_CIB_OP_TIMEOUT);
         crm_trace("Minimum CIB op timeout: %us (environment: %s)",
                   env_timeout, (env? env : "none"));
     }
 
     calculated_timeout = 1 + crm_active_peers();
     if (crm_remote_peer_cache) {
         calculated_timeout += g_hash_table_size(crm_remote_peer_cache);
     }
     calculated_timeout *= 10;
 
     calculated_timeout = QB_MAX(calculated_timeout, env_timeout);
     crm_trace("Calculated timeout: %us", calculated_timeout);
 
     if (fsa_cib_conn) {
         fsa_cib_conn->call_timeout = calculated_timeout;
     }
     return calculated_timeout;
 }
 
 /*!
  * \internal
  * \brief Check feature set compatibility of DC and joining node
  *
  * Return true if a joining node's CRM feature set is compatible with the
  * current DC's. The feature sets are compatible if they have the same major
  * version number, and the DC's minor version number is the same or older than
  * the joining node's. The minor-minor version is intended solely to allow
  * resource agents to detect feature support, and so is ignored.
  *
  * \param[in] dc_version    DC's feature set
  * \param[in] join_version  Joining node's version
  */
 bool
 feature_set_compatible(const char *dc_version, const char *join_version)
 {
     char *dc_minor = NULL;
     char *join_minor = NULL;
     long dc_v = 0;
     long join_v = 0;
 
     // Get DC's major version
     errno = 0;
     dc_v = strtol(dc_version, &dc_minor, 10);
     if (errno) {
         return FALSE;
     }
 
     // Get joining node's major version
     errno = 0;
     join_v = strtol(join_version, &join_minor, 10);
     if (errno) {
         return FALSE;
     }
 
     // Major version component must be identical
     if (dc_v != join_v) {
         return FALSE;
     }
 
     // Get DC's minor version
     if (*dc_minor == '.') {
         ++dc_minor;
     }
     errno = 0;
     dc_v = strtol(dc_minor, NULL, 10);
     if (errno) {
         return FALSE;
     }
 
     // Get joining node's minor version
     if (*join_minor == '.') {
         ++join_minor;
     }
     errno = 0;
     join_v = strtol(join_minor, NULL, 10);
     if (errno) {
         return FALSE;
     }
 
     // DC's minor version must be the same or older
     return dc_v <= join_v;
 }
 
 const char *
 get_node_id(xmlNode *lrm_rsc_op)
 {
     xmlNode *node = lrm_rsc_op;
 
     while (node != NULL && !pcmk__str_eq(XML_CIB_TAG_STATE, TYPE(node), pcmk__str_casei)) {
         node = node->parent;
     }
 
     CRM_CHECK(node != NULL, return NULL);
     return ID(node);
 }