diff --git a/daemons/controld/controld_execd.c b/daemons/controld/controld_execd.c
index b4ba16a534..4fc2b8861a 100644
--- a/daemons/controld/controld_execd.c
+++ b/daemons/controld/controld_execd.c
@@ -1,2893 +1,2900 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <regex.h>
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 
 #include <crm/crm.h>
 #include <crm/lrmd.h>           // lrmd_event_data_t, lrmd_rsc_info_t, etc.
 #include <crm/services.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/pengine/rules.h>
 #include <crm/lrmd_internal.h>
 
 #include <pacemaker-internal.h>
 #include <pacemaker-controld.h>
 
 #define START_DELAY_THRESHOLD 5 * 60 * 1000
 #define MAX_LRM_REG_FAILS 30
 
 struct delete_event_s {
     int rc;
     const char *rsc;
     lrm_state_t *lrm_state;
 };
 
 static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id);
 static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list);
 static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
 
 static lrmd_event_data_t *construct_op(lrm_state_t * lrm_state, xmlNode * rsc_op,
                                        const char *rsc_id, const char *operation);
 static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc,
                           const char *operation, xmlNode *msg);
 
 static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state,
                                          int log_level);
 static int do_update_resource(const char *node_name, lrmd_rsc_info_t *rsc,
                               lrmd_event_data_t *op, time_t lock_time);
 
 static void
 lrm_connection_destroy(void)
 {
     if (pcmk_is_set(fsa_input_register, R_LRM_CONNECTED)) {
         crm_crit("Connection to executor failed");
         register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
         controld_clear_fsa_input_flags(R_LRM_CONNECTED);
 
     } else {
         crm_info("Disconnected from executor");
     }
 
 }
 
 static char *
 make_stop_id(const char *rsc, int call_id)
 {
     return crm_strdup_printf("%s:%d", rsc, call_id);
 }
 
 static void
 copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
 {
     if (strstr(key, CRM_META "_") == NULL) {
         g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
     }
 }
 
 static void
 copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
 {
     if (strstr(key, CRM_META "_") != NULL) {
         g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
     }
 }
 
 /*!
  * \internal
  * \brief Remove a recurring operation from a resource's history
  *
  * \param[in,out] history  Resource history to modify
  * \param[in]     op       Operation to remove
  *
  * \return TRUE if the operation was found and removed, FALSE otherwise
  */
 static gboolean
 history_remove_recurring_op(rsc_history_t *history, const lrmd_event_data_t *op)
 {
     GList *iter;
 
     for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
         lrmd_event_data_t *existing = iter->data;
 
         if ((op->interval_ms == existing->interval_ms)
             && pcmk__str_eq(op->rsc_id, existing->rsc_id, pcmk__str_none)
             && pcmk__str_eq(op->op_type, existing->op_type, pcmk__str_casei)) {
 
             history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter);
             lrmd_free_event(existing);
             return TRUE;
         }
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Free all recurring operations in resource history
  *
  * \param[in,out] history  Resource history to modify
  */
 static void
 history_free_recurring_ops(rsc_history_t *history)
 {
     GList *iter;
 
     for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
         lrmd_free_event(iter->data);
     }
     g_list_free(history->recurring_op_list);
     history->recurring_op_list = NULL;
 }
 
 /*!
  * \internal
  * \brief Free resource history
  *
  * \param[in,out] history  Resource history to free
  */
 void
 history_free(gpointer data)
 {
     rsc_history_t *history = (rsc_history_t*)data;
 
     if (history->stop_params) {
         g_hash_table_destroy(history->stop_params);
     }
 
     /* Don't need to free history->rsc.id because it's set to history->id */
     free(history->rsc.type);
     free(history->rsc.standard);
     free(history->rsc.provider);
 
     lrmd_free_event(history->failed);
     lrmd_free_event(history->last);
     free(history->id);
     history_free_recurring_ops(history);
     free(history);
 }
 
 static void
 update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
 {
     int target_rc = 0;
     rsc_history_t *entry = NULL;
 
     if (op->rsc_deleted) {
         crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
         controld_delete_resource_history(op->rsc_id, lrm_state->node_name,
                                          NULL, crmd_cib_smart_opt());
         return;
     }
 
     if (pcmk__str_eq(op->op_type, RSC_NOTIFY, pcmk__str_casei)) {
         return;
     }
 
     crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);
 
     entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
     if (entry == NULL && rsc) {
         entry = calloc(1, sizeof(rsc_history_t));
         entry->id = strdup(op->rsc_id);
         g_hash_table_insert(lrm_state->resource_history, entry->id, entry);
 
         entry->rsc.id = entry->id;
         entry->rsc.type = strdup(rsc->type);
         entry->rsc.standard = strdup(rsc->standard);
         if (rsc->provider) {
             entry->rsc.provider = strdup(rsc->provider);
         } else {
             entry->rsc.provider = NULL;
         }
 
     } else if (entry == NULL) {
         crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
         return;
     }
 
     entry->last_callid = op->call_id;
     target_rc = rsc_op_expected_rc(op);
     if (op->op_status == PCMK_EXEC_CANCELLED) {
         if (op->interval_ms > 0) {
             crm_trace("Removing cancelled recurring op: " PCMK__OP_FMT,
                       op->rsc_id, op->op_type, op->interval_ms);
             history_remove_recurring_op(entry, op);
             return;
         } else {
             crm_trace("Skipping " PCMK__OP_FMT " rc=%d, status=%d",
                       op->rsc_id, op->op_type, op->interval_ms, op->rc,
                       op->op_status);
         }
 
     } else if (did_rsc_op_fail(op, target_rc)) {
         /* Store failed monitors here, otherwise the block below will cause them
          * to be forgotten when a stop happens.
          */
         if (entry->failed) {
             lrmd_free_event(entry->failed);
         }
         entry->failed = lrmd_copy_event(op);
 
     } else if (op->interval_ms == 0) {
         if (entry->last) {
             lrmd_free_event(entry->last);
         }
         entry->last = lrmd_copy_event(op);
 
         if (op->params && pcmk__strcase_any_of(op->op_type, CRMD_ACTION_START,
                                                CRMD_ACTION_RELOAD,
                                                CRMD_ACTION_RELOAD_AGENT,
                                                CRMD_ACTION_STATUS, NULL)) {
             if (entry->stop_params) {
                 g_hash_table_destroy(entry->stop_params);
             }
             entry->stop_params = pcmk__strkey_table(free, free);
 
             g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
         }
     }
 
     if (op->interval_ms > 0) {
         /* Ensure there are no duplicates */
         history_remove_recurring_op(entry, op);
 
         crm_trace("Adding recurring op: " PCMK__OP_FMT,
                   op->rsc_id, op->op_type, op->interval_ms);
         entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));
 
     } else if (entry->recurring_op_list && !pcmk__str_eq(op->op_type, RSC_STATUS, pcmk__str_casei)) {
         crm_trace("Dropping %d recurring ops because of: " PCMK__OP_FMT,
                   g_list_length(entry->recurring_op_list), op->rsc_id,
                   op->op_type, op->interval_ms);
         history_free_recurring_ops(entry);
     }
 }
 
 /*!
  * \internal
  * \brief Send a direct OK ack for a resource task
  *
  * \param[in] lrm_state  LRM connection
  * \param[in] input      Input message being ack'ed
  * \param[in] rsc_id     ID of affected resource
  * \param[in] rsc        Affected resource (if available)
  * \param[in] task       Operation task being ack'ed
  * \param[in] ack_host   Name of host to send ack to
  * \param[in] ack_sys    IPC system name to ack
  */
 static void
 send_task_ok_ack(lrm_state_t *lrm_state, ha_msg_input_t *input,
                  const char *rsc_id, lrmd_rsc_info_t *rsc, const char *task,
                  const char *ack_host, const char *ack_sys)
 {
     lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc_id, task);
 
     lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
     controld_ack_event_directly(ack_host, ack_sys, rsc, op, rsc_id);
     lrmd_free_event(op);
 }
 
 static inline const char *
 op_node_name(lrmd_event_data_t *op)
 {
     return op->remote_nodename? op->remote_nodename : fsa_our_uname;
 }
 
 void
 lrm_op_callback(lrmd_event_data_t * op)
 {
     CRM_CHECK(op != NULL, return);
     switch (op->type) {
         case lrmd_event_disconnect:
             if (op->remote_nodename == NULL) {
                 /* If this is the local executor IPC connection, set the right
                  * bits in the controller when the connection goes down.
                  */
                 lrm_connection_destroy();
             }
             break;
 
         case lrmd_event_exec_complete:
             {
                 lrm_state_t *lrm_state = lrm_state_find(op_node_name(op));
 
                 CRM_ASSERT(lrm_state != NULL);
                 process_lrm_event(lrm_state, op, NULL, NULL);
             }
             break;
 
         default:
             break;
     }
 }
 
 static void
 try_local_executor_connect(long long action, fsa_data_t *msg_data,
                            lrm_state_t *lrm_state)
 {
     int rc = pcmk_rc_ok;
 
     crm_debug("Connecting to the local executor");
 
     // If we can connect, great
     rc = controld_connect_local_executor(lrm_state);
     if (rc == pcmk_rc_ok) {
         controld_set_fsa_input_flags(R_LRM_CONNECTED);
         crm_info("Connection to the local executor established");
         return;
     }
 
     // Otherwise, if we can try again, set a timer to do so
     if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
         crm_warn("Failed to connect to the local executor %d time%s "
                  "(%d max): %s", lrm_state->num_lrm_register_fails,
                  pcmk__plural_s(lrm_state->num_lrm_register_fails),
                  MAX_LRM_REG_FAILS, pcmk_rc_str(rc));
         controld_start_timer(wait_timer);
         crmd_fsa_stall(FALSE);
         return;
     }
 
     // Otherwise give up
     crm_err("Failed to connect to the executor the max allowed "
             "%d time%s: %s", lrm_state->num_lrm_register_fails,
             pcmk__plural_s(lrm_state->num_lrm_register_fails),
             pcmk_rc_str(rc));
     register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
 }
 
 /*	 A_LRM_CONNECT	*/
 void
 do_lrm_control(long long action,
                enum crmd_fsa_cause cause,
                enum crmd_fsa_state cur_state,
                enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     /* This only pertains to local executor connections. Remote connections are
      * handled as resources within the scheduler. Connecting and disconnecting
      * from remote executor instances is handled differently.
      */
 
     lrm_state_t *lrm_state = NULL;
 
     if(fsa_our_uname == NULL) {
         return; /* Nothing to do */
     }
     lrm_state = lrm_state_find_or_create(fsa_our_uname);
     if (lrm_state == NULL) {
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
         return;
     }
 
     if (action & A_LRM_DISCONNECT) {
         if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) {
             if (action == A_LRM_DISCONNECT) {
                 crmd_fsa_stall(FALSE);
                 return;
             }
         }
 
         controld_clear_fsa_input_flags(R_LRM_CONNECTED);
         crm_info("Disconnecting from the executor");
         lrm_state_disconnect(lrm_state);
         lrm_state_reset_tables(lrm_state, FALSE);
         crm_notice("Disconnected from the executor");
     }
 
     if (action & A_LRM_CONNECT) {
         try_local_executor_connect(action, msg_data, lrm_state);
     }
 
     if (action & ~(A_LRM_CONNECT | A_LRM_DISCONNECT)) {
         crm_err("Unexpected action %s in %s", fsa_action2string(action),
                 __func__);
     }
 }
 
 static gboolean
 lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level)
 {
     int counter = 0;
     gboolean rc = TRUE;
     const char *when = "lrm disconnect";
 
     GHashTableIter gIter;
     const char *key = NULL;
     rsc_history_t *entry = NULL;
     active_op_t *pending = NULL;
 
     crm_debug("Checking for active resources before exit");
 
     if (cur_state == S_TERMINATE) {
         log_level = LOG_ERR;
         when = "shutdown";
 
     } else if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
         when = "shutdown... waiting";
     }
 
     if (lrm_state->pending_ops && lrm_state_is_connected(lrm_state) == TRUE) {
         guint removed = g_hash_table_foreach_remove(
             lrm_state->pending_ops, stop_recurring_actions, lrm_state);
         guint nremaining = g_hash_table_size(lrm_state->pending_ops);
 
         if (removed || nremaining) {
             crm_notice("Stopped %u recurring operation%s at %s (%u remaining)",
                        removed, pcmk__plural_s(removed), when, nremaining);
         }
     }
 
     if (lrm_state->pending_ops) {
         g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
         while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) {
             /* Ignore recurring actions in the shutdown calculations */
             if (pending->interval_ms == 0) {
                 counter++;
             }
         }
     }
 
     if (counter > 0) {
         do_crm_log(log_level, "%d pending executor operation%s at %s",
                    counter, pcmk__plural_s(counter), when);
 
         if ((cur_state == S_TERMINATE)
             || !pcmk_is_set(fsa_input_register, R_SENT_RSC_STOP)) {
             g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
             while (g_hash_table_iter_next(&gIter, (gpointer*)&key, (gpointer*)&pending)) {
                 do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key);
             }
 
         } else {
             rc = FALSE;
         }
         return rc;
     }
 
     if (lrm_state->resource_history == NULL) {
         return rc;
     }
 
     if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
         /* At this point we're not waiting, we're just shutting down */
         when = "shutdown";
     }
 
     counter = 0;
     g_hash_table_iter_init(&gIter, lrm_state->resource_history);
     while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) {
         if (is_rsc_active(lrm_state, entry->id) == FALSE) {
             continue;
         }
 
         counter++;
         if (log_level == LOG_ERR) {
             crm_info("Found %s active at %s", entry->id, when);
         } else {
             crm_trace("Found %s active at %s", entry->id, when);
         }
         if (lrm_state->pending_ops) {
             GHashTableIter hIter;
 
             g_hash_table_iter_init(&hIter, lrm_state->pending_ops);
             while (g_hash_table_iter_next(&hIter, (gpointer*)&key, (gpointer*)&pending)) {
                 if (pcmk__str_eq(entry->id, pending->rsc_id, pcmk__str_none)) {
                     crm_notice("%sction %s (%s) incomplete at %s",
                                pending->interval_ms == 0 ? "A" : "Recurring a",
                                key, pending->op_key, when);
                 }
             }
         }
     }
 
     if (counter) {
         crm_err("%d resource%s active at %s",
                 counter, (counter == 1)? " was" : "s were", when);
     }
 
     return rc;
 }
 
 /*!
  * \internal
  * \brief Build XML and string of parameters meeting some criteria, for digest
  *
  * \param[in]  op              Executor event with parameter table to use
  * \param[in]  metadata        Parsed meta-data for executed resource agent
  * \param[in]  param_type      Flag used for selection criteria
  * \param[out] result          Will be set to newly created XML with selected
  *                             parameters as attributes
  *
  * \return Newly allocated space-separated string of parameter names
  * \note Selection criteria varies by param_type: for the restart digest, we
  *       want parameters that are *not* marked reloadable (OCF 1.1) or that
  *       *are* marked unique (pre-1.1), for both string and XML results; for the
  *       secure digest, we want parameters that *are* marked private for the
  *       string, but parameters that are *not* marked private for the XML.
  * \note It is the caller's responsibility to free the string return value with
  *       free() and the XML result with free_xml().
  */
 static char *
 build_parameter_list(const lrmd_event_data_t *op,
                      const struct ra_metadata_s *metadata,
                      enum ra_param_flags_e param_type, xmlNode **result)
 {
     char *list = NULL;
     size_t len = 0;
 
     *result = create_xml_node(NULL, XML_TAG_PARAMS);
 
     for (GList *iter = metadata->ra_params; iter != NULL; iter = iter->next) {
         struct ra_param_s *param = (struct ra_param_s *) iter->data;
 
         bool accept_for_list = false;
         bool accept_for_xml = false;
 
         switch (param_type) {
             case ra_param_reloadable:
                 accept_for_list = !pcmk_is_set(param->rap_flags, param_type);
                 accept_for_xml = accept_for_list;
                 break;
 
             case ra_param_unique:
                 accept_for_list = pcmk_is_set(param->rap_flags, param_type);
                 accept_for_xml = accept_for_list;
                 break;
 
             case ra_param_private:
                 accept_for_list = pcmk_is_set(param->rap_flags, param_type);
                 accept_for_xml = !accept_for_list;
                 break;
         }
 
         if (accept_for_list) {
             crm_trace("Attr %s is %s", param->rap_name, ra_param_flag2text(param_type));
 
             if (list == NULL) {
                 // We will later search for " WORD ", so start list with a space
                 pcmk__add_word(&list, &len, " ");
             }
             pcmk__add_word(&list, &len, param->rap_name);
 
         } else {
             crm_trace("Rejecting %s for %s", param->rap_name, ra_param_flag2text(param_type));
         }
 
         if (accept_for_xml) {
             const char *v = g_hash_table_lookup(op->params, param->rap_name);
 
             if (v != NULL) {
                 crm_trace("Adding attr %s=%s to the xml result", param->rap_name, v);
                 crm_xml_add(*result, param->rap_name, v);
             }
         }
     }
 
     if (list != NULL) {
         // We will later search for " WORD ", so end list with a space
         pcmk__add_word(&list, &len, " ");
     }
     return list;
 }
 
 static void
 append_restart_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
                     xmlNode *update, const char *version)
 {
     char *list = NULL;
     char *digest = NULL;
     xmlNode *restart = NULL;
 
     CRM_LOG_ASSERT(op->params != NULL);
 
     if (op->interval_ms > 0) {
         /* monitors are not reloadable */
         return;
     }
 
     if (pcmk_is_set(metadata->ra_flags, ra_supports_reload_agent)) {
         // Add parameters not marked reloadable to the "op-force-restart" list
         list = build_parameter_list(op, metadata, ra_param_reloadable,
                                     &restart);
 
     } else if (pcmk_is_set(metadata->ra_flags, ra_supports_legacy_reload)) {
         /* @COMPAT pre-OCF-1.1 resource agents
          *
          * Before OCF 1.1, Pacemaker abused "unique=0" to indicate
          * reloadability. Add any parameters with unique="1" to the
          * "op-force-restart" list.
          */
         list = build_parameter_list(op, metadata, ra_param_unique, &restart);
 
     } else {
         // Resource does not support agent reloads
         return;
     }
 
     digest = calculate_operation_digest(restart, version);
     /* Add "op-force-restart" and "op-restart-digest" to indicate the resource supports reload,
      * no matter if it actually supports any parameters with unique="1"). */
     crm_xml_add(update, XML_LRM_ATTR_OP_RESTART, list? list: "");
     crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);
 
     crm_trace("%s: %s, %s", op->rsc_id, digest, list);
     crm_log_xml_trace(restart, "restart digest source");
 
     free_xml(restart);
     free(digest);
     free(list);
 }
 
 static void
 append_secure_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
                    xmlNode *update, const char *version)
 {
     char *list = NULL;
     char *digest = NULL;
     xmlNode *secure = NULL;
 
     CRM_LOG_ASSERT(op->params != NULL);
 
     /*
      * To keep XML_LRM_ATTR_OP_SECURE short, we want it to contain the
      * secure parameters but XML_LRM_ATTR_SECURE_DIGEST to be based on
      * the insecure ones
      */
     list = build_parameter_list(op, metadata, ra_param_private, &secure);
 
     if (list != NULL) {
         digest = calculate_operation_digest(secure, version);
         crm_xml_add(update, XML_LRM_ATTR_OP_SECURE, list);
         crm_xml_add(update, XML_LRM_ATTR_SECURE_DIGEST, digest);
 
         crm_trace("%s: %s, %s", op->rsc_id, digest, list);
         crm_log_xml_trace(secure, "secure digest source");
     } else {
         crm_trace("%s: no secure parameters", op->rsc_id);
     }
 
     free_xml(secure);
     free(digest);
     free(list);
 }
 
 static gboolean
 build_operation_update(xmlNode * parent, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op,
                        const char *node_name, const char *src)
 {
     int target_rc = 0;
     xmlNode *xml_op = NULL;
     struct ra_metadata_s *metadata = NULL;
     const char *caller_version = NULL;
     lrm_state_t *lrm_state = NULL;
     uint32_t metadata_source = controld_metadata_from_agent;
 
     if (op == NULL) {
         return FALSE;
     }
 
     target_rc = rsc_op_expected_rc(op);
 
     /* there is a small risk in formerly mixed clusters that it will
      * be sub-optimal.
      *
      * however with our upgrade policy, the update we send should
      * still be completely supported anyway
      */
     caller_version = g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION);
     CRM_LOG_ASSERT(caller_version != NULL);
 
     if(caller_version == NULL) {
         caller_version = CRM_FEATURE_SET;
     }
 
     crm_trace("Building %s operation update with originator version: %s", op->rsc_id, caller_version);
     xml_op = pcmk__create_history_xml(parent, op, caller_version, target_rc,
                                       fsa_our_uname, src, LOG_DEBUG);
     if (xml_op == NULL) {
         return TRUE;
     }
 
     if ((rsc == NULL) || (op->params == NULL)
         || !crm_op_needs_metadata(rsc->standard, op->op_type)) {
 
         crm_trace("No digests needed for %s action on %s (params=%p rsc=%p)",
                   op->op_type, op->rsc_id, op->params, rsc);
         return TRUE;
     }
 
     lrm_state = lrm_state_find(node_name);
     if (lrm_state == NULL) {
         crm_warn("Cannot calculate digests for operation " PCMK__OP_FMT
                  " because we have no connection to executor for %s",
                  op->rsc_id, op->op_type, op->interval_ms, node_name);
         return TRUE;
     }
 
     /* Getting meta-data from cache is OK unless this is a successful start
      * action -- always refresh from the agent for those, in case the
      * resource agent was updated.
      *
      * @TODO Only refresh the meta-data after starts if the agent actually
      * changed (using something like inotify, or a hash or modification time of
      * the agent executable).
      */
     if ((op->op_status != PCMK_EXEC_DONE) || (op->rc != target_rc)
         || !pcmk__str_eq(op->op_type, CRMD_ACTION_START, pcmk__str_none)) {
         metadata_source |= controld_metadata_from_cache;
     }
     metadata = controld_get_rsc_metadata(lrm_state, rsc, metadata_source);
     if (metadata == NULL) {
         return TRUE;
     }
 
 #if ENABLE_VERSIONED_ATTRS
     crm_xml_add(xml_op, XML_ATTR_RA_VERSION, metadata->ra_version);
 #endif
 
     crm_trace("Including additional digests for %s:%s:%s",
               rsc->standard, rsc->provider, rsc->type);
     append_restart_list(op, metadata, xml_op, caller_version);
     append_secure_list(op, metadata, xml_op, caller_version);
 
     return TRUE;
 }
 
 static gboolean
 is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id)
 {
     rsc_history_t *entry = NULL;
 
     entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
     if (entry == NULL || entry->last == NULL) {
         return FALSE;
     }
 
     crm_trace("Processing %s: %s.%d=%d", rsc_id, entry->last->op_type,
               entry->last->interval_ms, entry->last->rc);
     if (entry->last->rc == PCMK_OCF_OK && pcmk__str_eq(entry->last->op_type, CRMD_ACTION_STOP, pcmk__str_casei)) {
         return FALSE;
 
     } else if (entry->last->rc == PCMK_OCF_OK
                && pcmk__str_eq(entry->last->op_type, CRMD_ACTION_MIGRATE, pcmk__str_casei)) {
         // A stricter check is too complex ... leave that to the scheduler
         return FALSE;
 
     } else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) {
         return FALSE;
 
     } else if ((entry->last->interval_ms == 0)
                && (entry->last->rc == PCMK_OCF_NOT_CONFIGURED)) {
         /* Badly configured resources can't be reliably stopped */
         return FALSE;
     }
 
     return TRUE;
 }
 
 static gboolean
 build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list)
 {
     GHashTableIter iter;
     rsc_history_t *entry = NULL;
 
     g_hash_table_iter_init(&iter, lrm_state->resource_history);
     while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
 
         GList *gIter = NULL;
         xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE);
 
         crm_xml_add(xml_rsc, XML_ATTR_ID, entry->id);
         crm_xml_add(xml_rsc, XML_ATTR_TYPE, entry->rsc.type);
         crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, entry->rsc.standard);
         crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, entry->rsc.provider);
 
         if (entry->last && entry->last->params) {
             const char *container = g_hash_table_lookup(entry->last->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
             if (container) {
                 crm_trace("Resource %s is a part of container resource %s", entry->id, container);
                 crm_xml_add(xml_rsc, XML_RSC_ATTR_CONTAINER, container);
             }
         }
         build_operation_update(xml_rsc, &(entry->rsc), entry->failed, lrm_state->node_name,
                                __func__);
         build_operation_update(xml_rsc, &(entry->rsc), entry->last, lrm_state->node_name,
                                __func__);
         for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
             build_operation_update(xml_rsc, &(entry->rsc), gIter->data, lrm_state->node_name,
                                    __func__);
         }
     }
 
     return FALSE;
 }
 
 static xmlNode *
 do_lrm_query_internal(lrm_state_t *lrm_state, int update_flags)
 {
     xmlNode *xml_state = NULL;
     xmlNode *xml_data = NULL;
     xmlNode *rsc_list = NULL;
     crm_node_t *peer = NULL;
 
     peer = crm_get_peer_full(0, lrm_state->node_name, CRM_GET_PEER_ANY);
     CRM_CHECK(peer != NULL, return NULL);
 
     xml_state = create_node_state_update(peer, update_flags, NULL,
                                          __func__);
     if (xml_state == NULL) {
         return NULL;
     }
 
     xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM);
     crm_xml_add(xml_data, XML_ATTR_ID, peer->uuid);
     rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES);
 
     /* Build a list of active (not always running) resources */
     build_active_RAs(lrm_state, rsc_list);
 
     crm_log_xml_trace(xml_state, "Current executor state");
 
     return xml_state;
 }
 
 xmlNode *
 controld_query_executor_state(const char *node_name)
 {
     lrm_state_t *lrm_state = lrm_state_find(node_name);
 
     if (!lrm_state) {
         crm_err("Could not find executor state for node %s", node_name);
         return NULL;
     }
     return do_lrm_query_internal(lrm_state,
                                  node_update_cluster|node_update_peer);
 }
 
 /*!
  * \internal
  * \brief Map standard Pacemaker return code to operation status and OCF code
  *
  * \param[out] event  Executor event whose status and return code should be set
  * \param[in]  rc     Standard Pacemaker return code
  */
 void
 controld_rc2event(lrmd_event_data_t *event, int rc)
 {
+    /* This is called for cleanup requests from controller peers/clients, not
+     * for resource actions, so no exit reason is needed.
+     */
     switch (rc) {
         case pcmk_rc_ok:
             lrmd__set_result(event, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
             break;
         case EACCES:
             lrmd__set_result(event, PCMK_OCF_INSUFFICIENT_PRIV,
                              PCMK_EXEC_ERROR, NULL);
             break;
         default:
             lrmd__set_result(event, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                              NULL);
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Trigger a new transition after CIB status was deleted
  *
  * If a CIB status delete was not expected (as part of the transition graph),
  * trigger a new transition by updating the (arbitrary) "last-lrm-refresh"
  * cluster property.
  *
  * \param[in] from_sys  IPC name that requested the delete
  * \param[in] rsc_id    Resource whose status was deleted (for logging only)
  */
 void
 controld_trigger_delete_refresh(const char *from_sys, const char *rsc_id)
 {
     if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
         char *now_s = crm_strdup_printf("%lld", (long long) time(NULL));
 
         crm_debug("Triggering a refresh after %s cleaned %s", from_sys, rsc_id);
         update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG,
                              NULL, NULL, NULL, NULL, "last-lrm-refresh", now_s,
                              FALSE, NULL, NULL);
         free(now_s);
     }
 }
 
 static void
 notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc)
 {
     lrmd_event_data_t *op = NULL;
     const char *from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
     const char *from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
 
     crm_info("Notifying %s on %s that %s was%s deleted",
              from_sys, (from_host? from_host : "localhost"), rsc_id,
              ((rc == pcmk_ok)? "" : " not"));
     op = construct_op(lrm_state, input->xml, rsc_id, CRMD_ACTION_DELETE);
     controld_rc2event(op, pcmk_legacy2rc(rc));
     controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
     lrmd_free_event(op);
     controld_trigger_delete_refresh(from_sys, rsc_id);
 }
 
 static gboolean
 lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
 {
     struct delete_event_s *event = user_data;
     struct pending_deletion_op_s *op = value;
 
     if (pcmk__str_eq(event->rsc, op->rsc, pcmk__str_none)) {
         notify_deleted(event->lrm_state, op->input, event->rsc, event->rc);
         return TRUE;
     }
     return FALSE;
 }
 
 static gboolean
 lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
 {
     const char *rsc = user_data;
     active_op_t *pending = value;
 
     if (pcmk__str_eq(rsc, pending->rsc_id, pcmk__str_none)) {
         crm_info("Removing op %s:%d for deleted resource %s",
                  pending->op_key, pending->call_id, rsc);
         return TRUE;
     }
     return FALSE;
 }
 
 static void
 delete_rsc_entry(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id,
                  GHashTableIter * rsc_gIter, int rc, const char *user_name)
 {
     struct delete_event_s event;
 
     CRM_CHECK(rsc_id != NULL, return);
 
     if (rc == pcmk_ok) {
         char *rsc_id_copy = strdup(rsc_id);
 
         if (rsc_gIter) {
             g_hash_table_iter_remove(rsc_gIter);
         } else {
             g_hash_table_remove(lrm_state->resource_history, rsc_id_copy);
         }
         controld_delete_resource_history(rsc_id_copy, lrm_state->node_name,
                                          user_name, crmd_cib_smart_opt());
         g_hash_table_foreach_remove(lrm_state->pending_ops, lrm_remove_deleted_op, rsc_id_copy);
         free(rsc_id_copy);
     }
 
     if (input) {
         notify_deleted(lrm_state, input, rsc_id, rc);
     }
 
     event.rc = rc;
     event.rsc = rsc_id;
     event.lrm_state = lrm_state;
     g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event);
 }
 
 /*!
  * \internal
  * \brief Erase an LRM history entry from the CIB, given the operation data
  *
  * \param[in] lrm_state  LRM state of the desired node
  * \param[in] op         Operation whose history should be deleted
  */
 static void
 erase_lrm_history_by_op(lrm_state_t *lrm_state, lrmd_event_data_t *op)
 {
     xmlNode *xml_top = NULL;
 
     CRM_CHECK(op != NULL, return);
 
     xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
     crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
     crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);
 
     if (op->interval_ms > 0) {
         char *op_id = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
 
         /* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
         crm_xml_add(xml_top, XML_ATTR_ID, op_id);
         free(op_id);
     }
 
     crm_debug("Erasing resource operation history for " PCMK__OP_FMT " (call=%d)",
               op->rsc_id, op->op_type, op->interval_ms, op->call_id);
 
     fsa_cib_conn->cmds->remove(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top,
                                cib_quorum_override);
 
     crm_log_xml_trace(xml_top, "op:cancel");
     free_xml(xml_top);
 }
 
 /* Define xpath to find LRM resource history entry by node and resource */
 #define XPATH_HISTORY                                   \
     "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS              \
     "/" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"  \
     "/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES       \
     "/" XML_LRM_TAG_RESOURCE "[@" XML_ATTR_ID "='%s']"  \
     "/" XML_LRM_TAG_RSC_OP
 
 /* ... and also by operation key */
 #define XPATH_HISTORY_ID XPATH_HISTORY \
     "[@" XML_ATTR_ID "='%s']"
 
 /* ... and also by operation key and operation call ID */
 #define XPATH_HISTORY_CALL XPATH_HISTORY \
     "[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_CALLID "='%d']"
 
 /* ... and also by operation key and original operation key */
 #define XPATH_HISTORY_ORIG XPATH_HISTORY \
     "[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_TASK_KEY "='%s']"
 
 /*!
  * \internal
  * \brief Erase an LRM history entry from the CIB, given operation identifiers
  *
  * \param[in] lrm_state  LRM state of the node to clear history for
  * \param[in] rsc_id     Name of resource to clear history for
  * \param[in] key        Operation key of operation to clear history for
  * \param[in] orig_op    If specified, delete only if it has this original op
  * \param[in] call_id    If specified, delete entry only if it has this call ID
  */
 static void
 erase_lrm_history_by_id(lrm_state_t *lrm_state, const char *rsc_id,
                         const char *key, const char *orig_op, int call_id)
 {
     char *op_xpath = NULL;
 
     CRM_CHECK((rsc_id != NULL) && (key != NULL), return);
 
     if (call_id > 0) {
         op_xpath = crm_strdup_printf(XPATH_HISTORY_CALL,
                                      lrm_state->node_name, rsc_id, key,
                                      call_id);
 
     } else if (orig_op) {
         op_xpath = crm_strdup_printf(XPATH_HISTORY_ORIG,
                                      lrm_state->node_name, rsc_id, key,
                                      orig_op);
     } else {
         op_xpath = crm_strdup_printf(XPATH_HISTORY_ID,
                                      lrm_state->node_name, rsc_id, key);
     }
 
     crm_debug("Erasing resource operation history for %s on %s (call=%d)",
               key, rsc_id, call_id);
     fsa_cib_conn->cmds->remove(fsa_cib_conn, op_xpath, NULL,
                                cib_quorum_override | cib_xpath);
     free(op_xpath);
 }
 
 static inline gboolean
 last_failed_matches_op(rsc_history_t *entry, const char *op, guint interval_ms)
 {
     if (entry == NULL) {
         return FALSE;
     }
     if (op == NULL) {
         return TRUE;
     }
     return (pcmk__str_eq(op, entry->failed->op_type, pcmk__str_casei)
             && (interval_ms == entry->failed->interval_ms));
 }
 
 /*!
  * \internal
  * \brief Clear a resource's last failure
  *
  * Erase a resource's last failure on a particular node from both the
  * LRM resource history in the CIB, and the resource history remembered
  * for the LRM state.
  *
  * \param[in] rsc_id      Resource name
  * \param[in] node_name   Node name
  * \param[in] operation   If specified, only clear if matching this operation
  * \param[in] interval_ms If operation is specified, it has this interval
  */
 void
 lrm_clear_last_failure(const char *rsc_id, const char *node_name,
                        const char *operation, guint interval_ms)
 {
     char *op_key = NULL;
     char *orig_op_key = NULL;
     lrm_state_t *lrm_state = NULL;
 
     lrm_state = lrm_state_find(node_name);
     if (lrm_state == NULL) {
         return;
     }
 
     /* Erase from CIB */
     op_key = pcmk__op_key(rsc_id, "last_failure", 0);
     if (operation) {
         orig_op_key = pcmk__op_key(rsc_id, operation, interval_ms);
     }
     erase_lrm_history_by_id(lrm_state, rsc_id, op_key, orig_op_key, 0);
     free(op_key);
     free(orig_op_key);
 
     /* Remove from memory */
     if (lrm_state->resource_history) {
         rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history,
                                                    rsc_id);
 
         if (last_failed_matches_op(entry, operation, interval_ms)) {
             lrmd_free_event(entry->failed);
             entry->failed = NULL;
         }
     }
 }
 
 /* Returns: gboolean - cancellation is in progress */
 static gboolean
 cancel_op(lrm_state_t * lrm_state, const char *rsc_id, const char *key, int op, gboolean remove)
 {
     int rc = pcmk_ok;
     char *local_key = NULL;
     active_op_t *pending = NULL;
 
     CRM_CHECK(op != 0, return FALSE);
     CRM_CHECK(rsc_id != NULL, return FALSE);
     if (key == NULL) {
         local_key = make_stop_id(rsc_id, op);
         key = local_key;
     }
     pending = g_hash_table_lookup(lrm_state->pending_ops, key);
 
     if (pending) {
         if (remove && !pcmk_is_set(pending->flags, active_op_remove)) {
             controld_set_active_op_flags(pending, active_op_remove);
             crm_debug("Scheduling %s for removal", key);
         }
 
         if (pcmk_is_set(pending->flags, active_op_cancelled)) {
             crm_debug("Operation %s already cancelled", key);
             free(local_key);
             return FALSE;
         }
         controld_set_active_op_flags(pending, active_op_cancelled);
 
     } else {
         crm_info("No pending op found for %s", key);
         free(local_key);
         return FALSE;
     }
 
     crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);
     rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type,
                           pending->interval_ms);
     if (rc == pcmk_ok) {
         crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
         free(local_key);
         return TRUE;
     }
 
     crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
     /* The caller needs to make sure the entry is
      * removed from the pending_ops list
      *
      * Usually by returning TRUE inside the worker function
      * supplied to g_hash_table_foreach_remove()
      *
      * Not removing the entry from pending_ops will block
      * the node from shutting down
      */
     free(local_key);
     return FALSE;
 }
 
 struct cancel_data {
     gboolean done;
     gboolean remove;
     const char *key;
     lrmd_rsc_info_t *rsc;
     lrm_state_t *lrm_state;
 };
 
 static gboolean
 cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
 {
     gboolean remove = FALSE;
     struct cancel_data *data = user_data;
     active_op_t *op = value;
 
     if (pcmk__str_eq(op->op_key, data->key, pcmk__str_none)) {
         data->done = TRUE;
         remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove);
     }
     return remove;
 }
 
 static gboolean
 cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
 {
     guint removed = 0;
     struct cancel_data data;
 
     CRM_CHECK(rsc != NULL, return FALSE);
     CRM_CHECK(key != NULL, return FALSE);
 
     data.key = key;
     data.rsc = rsc;
     data.done = FALSE;
     data.remove = remove;
     data.lrm_state = lrm_state;
 
     removed = g_hash_table_foreach_remove(lrm_state->pending_ops, cancel_action_by_key, &data);
     crm_trace("Removed %u op cache entries, new size: %u",
               removed, g_hash_table_size(lrm_state->pending_ops));
     return data.done;
 }
 
 /*!
  * \internal
  * \brief Retrieve resource information from LRM
  *
  * \param[in]  lrm_state LRM connection to use
  * \param[in]  rsc_xml   XML containing resource configuration
  * \param[in]  do_create If true, register resource with LRM if not already
  * \param[out] rsc_info  Where to store resource information obtained from LRM
  *
  * \retval pcmk_ok   Success (and rsc_info holds newly allocated result)
  * \retval -EINVAL   Required information is missing from arguments
  * \retval -ENOTCONN No active connection to LRM
  * \retval -ENODEV   Resource not found
  * \retval -errno    Error communicating with executor when registering resource
  *
  * \note Caller is responsible for freeing result on success.
  */
 static int
 get_lrm_resource(lrm_state_t *lrm_state, xmlNode *rsc_xml, gboolean do_create,
                  lrmd_rsc_info_t **rsc_info)
 {
     const char *id = ID(rsc_xml);
 
     CRM_CHECK(lrm_state && rsc_xml && rsc_info, return -EINVAL);
     CRM_CHECK(id, return -EINVAL);
 
     if (lrm_state_is_connected(lrm_state) == FALSE) {
         return -ENOTCONN;
     }
 
     crm_trace("Retrieving resource information for %s from the executor", id);
     *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
 
     // If resource isn't known by ID, try clone name, if provided
     if (!*rsc_info) {
         const char *long_id = crm_element_value(rsc_xml, XML_ATTR_ID_LONG);
 
         if (long_id) {
             *rsc_info = lrm_state_get_rsc_info(lrm_state, long_id, 0);
         }
     }
 
     if ((*rsc_info == NULL) && do_create) {
         const char *class = crm_element_value(rsc_xml, XML_AGENT_ATTR_CLASS);
         const char *provider = crm_element_value(rsc_xml, XML_AGENT_ATTR_PROVIDER);
         const char *type = crm_element_value(rsc_xml, XML_ATTR_TYPE);
         int rc;
 
         crm_trace("Registering resource %s with the executor", id);
         rc = lrm_state_register_rsc(lrm_state, id, class, provider, type,
                                     lrmd_opt_drop_recurring);
         if (rc != pcmk_ok) {
             fsa_data_t *msg_data = NULL;
 
             crm_err("Could not register resource %s with the executor on %s: %s "
                     CRM_XS " rc=%d",
                     id, lrm_state->node_name, pcmk_strerror(rc), rc);
 
             /* Register this as an internal error if this involves the local
              * executor. Otherwise, we're likely dealing with an unresponsive
              * remote node, which is not an FSA failure.
              */
             if (lrm_state_is_local(lrm_state) == TRUE) {
                 register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
             }
             return rc;
         }
 
         *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
     }
     return *rsc_info? pcmk_ok : -ENODEV;
 }
 
 static void
 delete_resource(lrm_state_t * lrm_state,
                 const char *id,
                 lrmd_rsc_info_t * rsc,
                 GHashTableIter * gIter,
                 const char *sys,
                 const char *user,
                 ha_msg_input_t * request,
                 gboolean unregister)
 {
     int rc = pcmk_ok;
 
     crm_info("Removing resource %s from executor for %s%s%s",
              id, sys, (user? " as " : ""), (user? user : ""));
 
     if (rsc && unregister) {
         rc = lrm_state_unregister_rsc(lrm_state, id, 0);
     }
 
     if (rc == pcmk_ok) {
         crm_trace("Resource %s deleted from executor", id);
     } else if (rc == -EINPROGRESS) {
         crm_info("Deletion of resource '%s' from executor is pending", id);
         if (request) {
             struct pending_deletion_op_s *op = NULL;
             char *ref = crm_element_value_copy(request->msg, XML_ATTR_REFERENCE);
 
             op = calloc(1, sizeof(struct pending_deletion_op_s));
             op->rsc = strdup(rsc->id);
             op->input = copy_ha_msg_input(request);
             g_hash_table_insert(lrm_state->deletion_ops, ref, op);
         }
         return;
     } else {
         crm_warn("Could not delete '%s' from executor for %s%s%s: %s "
                  CRM_XS " rc=%d", id, sys, (user? " as " : ""),
                  (user? user : ""), pcmk_strerror(rc), rc);
     }
 
     delete_rsc_entry(lrm_state, request, id, gIter, rc, user);
 }
 
 static int
 get_fake_call_id(lrm_state_t *lrm_state, const char *rsc_id)
 {
     int call_id = 999999999;
     rsc_history_t *entry = NULL;
 
     if(lrm_state) {
         entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
     }
 
     /* Make sure the call id is greater than the last successful operation,
      * otherwise the failure will not result in a possible recovery of the resource
      * as it could appear the failure occurred before the successful start */
     if (entry) {
         call_id = entry->last_callid + 1;
     }
 
     if (call_id < 0) {
         call_id = 1;
     }
     return call_id;
 }
 
 static void
 fake_op_status(lrm_state_t *lrm_state, lrmd_event_data_t *op, int op_status,
                enum ocf_exitcode op_exitcode)
 {
     op->call_id = get_fake_call_id(lrm_state, op->rsc_id);
     op->t_run = time(NULL);
     op->t_rcchange = op->t_run;
     lrmd__set_result(op, op_exitcode, op_status, NULL);
 }
 
 static void
 force_reprobe(lrm_state_t *lrm_state, const char *from_sys,
               const char *from_host, const char *user_name,
               gboolean is_remote_node)
 {
     GHashTableIter gIter;
     rsc_history_t *entry = NULL;
 
     crm_info("Clearing resource history on node %s", lrm_state->node_name);
     g_hash_table_iter_init(&gIter, lrm_state->resource_history);
     while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
         /* only unregister the resource during a reprobe if it is not a remote connection
          * resource. otherwise unregistering the connection will terminate remote-node
          * membership */
         gboolean unregister = TRUE;
 
         if (is_remote_lrmd_ra(NULL, NULL, entry->id)) {
             lrm_state_t *remote_lrm_state = lrm_state_find(entry->id);
             if (remote_lrm_state) {
                 /* when forcing a reprobe, make sure to clear remote node before
                  * clearing the remote node's connection resource */ 
                 force_reprobe(remote_lrm_state, from_sys, from_host, user_name, TRUE);
             }
             unregister = FALSE;
         }
 
         delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys,
                         user_name, NULL, unregister);
     }
 
     /* Now delete the copy in the CIB */
     controld_delete_node_state(lrm_state->node_name, controld_section_lrm,
                                cib_scope_local);
 
     /* Finally, _delete_ the value in pacemaker-attrd -- setting it to FALSE
      * would result in the scheduler sending us back here again
      */
     update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node);
 }
 
 /*!
  * \internal
  * \brief Fail a requested action without actually executing it
  *
  * For an action that can't be executed, process it similarly to an actual
  * execution result, with specified error status (except for notify actions,
  * which will always be treated as successful).
  *
  * \param[in] lrm_state  Executor connection that action is for
  * \param[in] action     Action XML from request
  * \param[in] rc         Desired return code to use
  * \param[in] op_status  Desired operation status to use
  */
 static void
 synthesize_lrmd_failure(lrm_state_t *lrm_state, xmlNode *action,
                         int op_status, enum ocf_exitcode rc)
 {
     lrmd_event_data_t *op = NULL;
     const char *operation = crm_element_value(action, XML_LRM_ATTR_TASK);
     const char *target_node = crm_element_value(action, XML_LRM_ATTR_TARGET);
     xmlNode *xml_rsc = find_xml_node(action, XML_CIB_TAG_RESOURCE, TRUE);
 
     if ((xml_rsc == NULL) || (ID(xml_rsc) == NULL)) {
         /* @TODO Should we do something else, like direct ack? */
         crm_info("Can't fake %s failure (%d) on %s without resource configuration",
                  crm_element_value(action, XML_LRM_ATTR_TASK_KEY), rc,
                  target_node);
         return;
 
     } else if(operation == NULL) {
         /* This probably came from crm_resource -C, nothing to do */
         crm_info("Can't fake %s failure (%d) on %s without operation",
                  ID(xml_rsc), rc, target_node);
         return;
     }
 
     op = construct_op(lrm_state, action, ID(xml_rsc), operation);
 
     if (pcmk__str_eq(operation, RSC_NOTIFY, pcmk__str_casei)) { // Notifications can't fail
         fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_OK);
     } else {
         fake_op_status(lrm_state, op, op_status, rc);
     }
 
     crm_info("Faking " PCMK__OP_FMT " result (%d) on %s",
              op->rsc_id, op->op_type, op->interval_ms, op->rc, target_node);
 
     // Process the result as if it came from the LRM
     process_lrm_event(lrm_state, op, NULL, action);
     lrmd_free_event(op);
 }
 
 /*!
  * \internal
  * \brief Get target of an LRM operation
  *
  * \param[in] xml  LRM operation data XML
  *
  * \return LRM operation target node name (local node or Pacemaker Remote node)
  */
 static const char *
 lrm_op_target(xmlNode *xml)
 {
     const char *target = NULL;
 
     if (xml) {
         target = crm_element_value(xml, XML_LRM_ATTR_TARGET);
     }
     if (target == NULL) {
         target = fsa_our_uname;
     }
     return target;
 }
 
 static void
 fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name,
                   const char *from_host, const char *from_sys)
 {
     lrmd_event_data_t *op = NULL;
     lrmd_rsc_info_t *rsc = NULL;
     xmlNode *xml_rsc = find_xml_node(xml, XML_CIB_TAG_RESOURCE, TRUE);
 
     CRM_CHECK(xml_rsc != NULL, return);
 
     /* The executor simply executes operations and reports the results, without
      * any concept of success or failure, so to fail a resource, we must fake
      * what a failure looks like.
      *
      * To do this, we create a fake executor operation event for the resource,
      * and pass that event to the executor client callback so it will be
      * processed as if it came from the executor.
      */
     op = construct_op(lrm_state, xml, ID(xml_rsc), "asyncmon");
     fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_UNKNOWN_ERROR);
 
     free((char*) op->user_data);
     op->user_data = NULL;
     op->interval_ms = 0;
 
     if (user_name && !pcmk__is_privileged(user_name)) {
         crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc));
         controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc));
         lrmd_free_event(op);
         return;
     }
 
     if (get_lrm_resource(lrm_state, xml_rsc, TRUE, &rsc) == pcmk_ok) {
         crm_info("Failing resource %s...", rsc->id);
         lrmd__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_DONE,
                          "Simulated failure");
         process_lrm_event(lrm_state, op, NULL, xml);
         op->rc = PCMK_OCF_OK; // The request to fail the resource succeeded
         lrmd_free_rsc_info(rsc);
 
     } else {
         crm_info("Cannot find/create resource in order to fail it...");
         crm_log_xml_warn(xml, "bad input");
     }
 
     controld_ack_event_directly(from_host, from_sys, NULL, op, ID(xml_rsc));
     lrmd_free_event(op);
 }
 
 static void
 handle_refresh_op(lrm_state_t *lrm_state, const char *user_name,
                   const char *from_host, const char *from_sys)
 {
     int rc = pcmk_ok;
     xmlNode *fragment = do_lrm_query_internal(lrm_state, node_update_all);
 
     fsa_cib_update(XML_CIB_TAG_STATUS, fragment, cib_quorum_override, rc, user_name);
     crm_info("Forced a local resource history refresh: call=%d", rc);
 
     if (!pcmk__str_eq(CRM_SYSTEM_CRMD, from_sys, pcmk__str_casei)) {
         xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, fragment, from_host,
                                         from_sys, CRM_SYSTEM_LRMD,
                                         fsa_our_uuid);
 
         crm_debug("ACK'ing refresh from %s (%s)", from_sys, from_host);
 
         if (relay_message(reply, TRUE) == FALSE) {
             crm_log_xml_err(reply, "Unable to route reply");
         }
         free_xml(reply);
     }
 
     free_xml(fragment);
 }
 
 static void
 handle_query_op(xmlNode *msg, lrm_state_t *lrm_state)
 {
     xmlNode *data = do_lrm_query_internal(lrm_state, node_update_all);
     xmlNode *reply = create_reply(msg, data);
 
     if (relay_message(reply, TRUE) == FALSE) {
         crm_err("Unable to route reply");
         crm_log_xml_err(reply, "reply");
     }
     free_xml(reply);
     free_xml(data);
 }
 
 static void
 handle_reprobe_op(lrm_state_t *lrm_state, const char *from_sys,
                   const char *from_host, const char *user_name,
                   gboolean is_remote_node)
 {
     crm_notice("Forcing the status of all resources to be redetected");
     force_reprobe(lrm_state, from_sys, from_host, user_name, is_remote_node);
 
     if (!pcmk__strcase_any_of(from_sys, CRM_SYSTEM_PENGINE, CRM_SYSTEM_TENGINE, NULL)) {
 
         xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, NULL, from_host,
                                         from_sys, CRM_SYSTEM_LRMD,
                                         fsa_our_uuid);
 
         crm_debug("ACK'ing re-probe from %s (%s)", from_sys, from_host);
 
         if (relay_message(reply, TRUE) == FALSE) {
             crm_log_xml_err(reply, "Unable to route reply");
         }
         free_xml(reply);
     }
 }
 
 static bool do_lrm_cancel(ha_msg_input_t *input, lrm_state_t *lrm_state,
               lrmd_rsc_info_t *rsc, const char *from_host, const char *from_sys)
 {
     char *op_key = NULL;
     char *meta_key = NULL;
     int call = 0;
     const char *call_id = NULL;
     const char *op_task = NULL;
     guint interval_ms = 0;
     gboolean in_progress = FALSE;
     xmlNode *params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE);
 
     CRM_CHECK(params != NULL, return FALSE);
 
     meta_key = crm_meta_name(XML_LRM_ATTR_TASK);
     op_task = crm_element_value(params, meta_key);
     free(meta_key);
     CRM_CHECK(op_task != NULL, return FALSE);
 
     meta_key = crm_meta_name(XML_LRM_ATTR_INTERVAL_MS);
     if (crm_element_value_ms(params, meta_key, &interval_ms) != pcmk_ok) {
         free(meta_key);
         return FALSE;
     }
     free(meta_key);
 
     op_key = pcmk__op_key(rsc->id, op_task, interval_ms);
 
     meta_key = crm_meta_name(XML_LRM_ATTR_CALLID);
     call_id = crm_element_value(params, meta_key);
     free(meta_key);
 
     crm_debug("Scheduler requested op %s (call=%s) be cancelled",
               op_key, (call_id? call_id : "NA"));
     pcmk__scan_min_int(call_id, &call, 0);
     if (call == 0) {
         // Normal case when the scheduler cancels a recurring op
         in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE);
 
     } else {
         // Normal case when the scheduler cancels an orphan op
         in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE);
     }
 
     // Acknowledge cancellation operation if for a remote connection resource
     if (!in_progress || is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
         char *op_id = make_stop_id(rsc->id, call);
 
         if (is_remote_lrmd_ra(NULL, NULL, rsc->id) == FALSE) {
             crm_info("Nothing known about operation %d for %s", call, op_key);
         }
         erase_lrm_history_by_id(lrm_state, rsc->id, op_key, NULL, call);
         send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
                          from_host, from_sys);
 
         /* needed at least for cancellation of a remote operation */
         g_hash_table_remove(lrm_state->pending_ops, op_id);
         free(op_id);
 
     } else {
         /* No ack is needed since abcdaa8, but peers with older versions
          * in a rolling upgrade need one. We didn't bump the feature set
          * at that commit, so we can only compare against the previous
          * CRM version (3.0.8). If any peers have feature set 3.0.9 but
          * not abcdaa8, they will time out waiting for the ack (no
          * released versions of Pacemaker are affected).
          */
         const char *peer_version = crm_element_value(params, XML_ATTR_CRM_VERSION);
 
         if (compare_version(peer_version, "3.0.8") <= 0) {
             crm_info("Sending compatibility ack for %s cancellation to %s (CRM version %s)",
                      op_key, from_host, peer_version);
             send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
                              from_host, from_sys);
         }
     }
 
     free(op_key);
     return TRUE;
 }
 
 static void
 do_lrm_delete(ha_msg_input_t *input, lrm_state_t *lrm_state,
               lrmd_rsc_info_t *rsc, const char *from_sys, const char *from_host,
               bool crm_rsc_delete, const char *user_name)
 {
     gboolean unregister = TRUE;
     int cib_rc = controld_delete_resource_history(rsc->id, lrm_state->node_name,
                                                   user_name,
                                                   cib_dryrun|cib_sync_call);
 
     if (cib_rc != pcmk_rc_ok) {
         lrmd_event_data_t *op = NULL;
 
         op = construct_op(lrm_state, input->xml, rsc->id, CRMD_ACTION_DELETE);
 
+        /* These are resource clean-ups, not actions, so no exit reason is
+         * needed.
+         */
         if (cib_rc == EACCES) {
             lrmd__set_result(op, PCMK_OCF_INSUFFICIENT_PRIV, PCMK_EXEC_ERROR,
                              NULL);
         } else {
             lrmd__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
                              NULL);
         }
         controld_ack_event_directly(from_host, from_sys, NULL, op, rsc->id);
         lrmd_free_event(op);
         return;
     }
 
     if (crm_rsc_delete && is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
         unregister = FALSE;
     }
 
     delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys,
                     user_name, input, unregister);
 }
 
 /*	 A_LRM_INVOKE	*/
 void
 do_lrm_invoke(long long action,
               enum crmd_fsa_cause cause,
               enum crmd_fsa_state cur_state,
               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     lrm_state_t *lrm_state = NULL;
     const char *crm_op = NULL;
     const char *from_sys = NULL;
     const char *from_host = NULL;
     const char *operation = NULL;
     ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
     const char *user_name = NULL;
     const char *target_node = NULL;
     gboolean is_remote_node = FALSE;
     bool crm_rsc_delete = FALSE;
 
     target_node = lrm_op_target(input->xml);
     is_remote_node = !pcmk__str_eq(target_node, fsa_our_uname,
                                    pcmk__str_casei);
 
     lrm_state = lrm_state_find(target_node);
     if ((lrm_state == NULL) && is_remote_node) {
         crm_err("Failing action because local node has never had connection to remote node %s",
                 target_node);
         synthesize_lrmd_failure(NULL, input->xml, PCMK_EXEC_NOT_CONNECTED,
                                 PCMK_OCF_UNKNOWN_ERROR);
         return;
     }
     CRM_ASSERT(lrm_state != NULL);
 
     user_name = pcmk__update_acl_user(input->msg, F_CRM_USER, NULL);
     crm_op = crm_element_value(input->msg, F_CRM_TASK);
     from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
     if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
         from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
     }
     crm_trace("Executor %s command from %s as user %s",
               crm_op, from_sys, user_name);
 
     if (pcmk__str_eq(crm_op, CRM_OP_LRM_DELETE, pcmk__str_casei)) {
         if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
             crm_rsc_delete = TRUE; // from crm_resource
         }
         operation = CRMD_ACTION_DELETE;
 
     } else if (pcmk__str_eq(crm_op, CRM_OP_LRM_FAIL, pcmk__str_casei)) {
         fail_lrm_resource(input->xml, lrm_state, user_name, from_host,
                           from_sys);
         return;
 
     } else if (input->xml != NULL) {
         operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK);
     }
 
     if (pcmk__str_eq(crm_op, CRM_OP_LRM_REFRESH, pcmk__str_casei)) {
         handle_refresh_op(lrm_state, user_name, from_host, from_sys);
 
     } else if (pcmk__str_eq(crm_op, CRM_OP_LRM_QUERY, pcmk__str_casei)) {
         handle_query_op(input->msg, lrm_state);
 
     } else if (pcmk__str_eq(operation, CRM_OP_PROBED, pcmk__str_casei)) {
         update_attrd(lrm_state->node_name, CRM_OP_PROBED, XML_BOOLEAN_TRUE,
                      user_name, is_remote_node);
 
     } else if (pcmk__str_eq(crm_op, CRM_OP_REPROBE, pcmk__str_casei)
                || pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_casei)) {
         handle_reprobe_op(lrm_state, from_sys, from_host, user_name,
                           is_remote_node);
 
     } else if (operation != NULL) {
         lrmd_rsc_info_t *rsc = NULL;
         xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);
         gboolean create_rsc = !pcmk__str_eq(operation, CRMD_ACTION_DELETE,
                                             pcmk__str_casei);
         int rc;
 
         // We can't return anything meaningful without a resource ID
         CRM_CHECK(xml_rsc && ID(xml_rsc), return);
 
         rc = get_lrm_resource(lrm_state, xml_rsc, create_rsc, &rsc);
         if (rc == -ENOTCONN) {
             synthesize_lrmd_failure(lrm_state, input->xml,
                                     PCMK_EXEC_NOT_CONNECTED,
                                     PCMK_OCF_UNKNOWN_ERROR);
             return;
 
         } else if ((rc < 0) && !create_rsc) {
             /* Delete of malformed or nonexistent resource
              * (deleting something that does not exist is a success)
              */
             crm_notice("Not registering resource '%s' for a %s event "
                        CRM_XS " get-rc=%d (%s) transition-key=%s",
                        ID(xml_rsc), operation,
                        rc, pcmk_strerror(rc), ID(input->xml));
             delete_rsc_entry(lrm_state, input, ID(xml_rsc), NULL, pcmk_ok,
                              user_name);
             return;
 
         } else if (rc == -EINVAL) {
             // Resource operation on malformed resource
             crm_err("Invalid resource definition for %s", ID(xml_rsc));
             crm_log_xml_warn(input->msg, "invalid resource");
             synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
                                     PCMK_OCF_NOT_CONFIGURED); // fatal error
             return;
 
         } else if (rc < 0) {
             // Error communicating with the executor
             crm_err("Could not register resource '%s' with executor: %s "
                     CRM_XS " rc=%d",
                     ID(xml_rsc), pcmk_strerror(rc), rc);
             crm_log_xml_warn(input->msg, "failed registration");
             synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
                                     PCMK_OCF_INVALID_PARAM); // hard error
             return;
         }
 
         if (pcmk__str_eq(operation, CRMD_ACTION_CANCEL, pcmk__str_casei)) {
             if (!do_lrm_cancel(input, lrm_state, rsc, from_host, from_sys)) {
                 crm_log_xml_warn(input->xml, "Bad command");
             }
 
         } else if (pcmk__str_eq(operation, CRMD_ACTION_DELETE, pcmk__str_casei)) {
             do_lrm_delete(input, lrm_state, rsc, from_sys, from_host,
                           crm_rsc_delete, user_name);
 
         } else if (pcmk__str_any_of(operation, CRMD_ACTION_RELOAD,
                                     CRMD_ACTION_RELOAD_AGENT, NULL)) {
             /* Pre-2.1.0 DCs will schedule reload actions only, and 2.1.0+ DCs
              * will schedule reload-agent actions only. In either case, we need
              * to map that to whatever the resource agent actually supports.
              * Default to the OCF 1.1 name.
              */
             struct ra_metadata_s *md = NULL;
             const char *reload_name = CRMD_ACTION_RELOAD_AGENT;
 
             md = controld_get_rsc_metadata(lrm_state, rsc,
                                            controld_metadata_from_cache);
             if ((md != NULL)
                 && pcmk_is_set(md->ra_flags, ra_supports_legacy_reload)) {
                 reload_name = CRMD_ACTION_RELOAD;
             }
             do_lrm_rsc_op(lrm_state, rsc, reload_name, input->xml);
 
         } else {
             do_lrm_rsc_op(lrm_state, rsc, operation, input->xml);
         }
 
         lrmd_free_rsc_info(rsc);
 
     } else {
         crm_err("Cannot perform operation %s of unknown type", crm_str(crm_op));
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
 }
 
 #if ENABLE_VERSIONED_ATTRS
 static void
 resolve_versioned_parameters(lrm_state_t *lrm_state, const char *rsc_id,
                              const xmlNode *rsc_op, GHashTable *params)
 {
     /* Resource info *should* already be cached, so we don't get
      * executor call */
     lrmd_rsc_info_t *rsc = lrm_state_get_rsc_info(lrm_state, rsc_id, 0);
     struct ra_metadata_s *metadata;
 
     metadata = controld_get_rsc_metadata(lrm_state, rsc,
                                          controld_metadata_from_cache);
     if (metadata) {
         xmlNode *versioned_attrs = NULL;
         GHashTable *hash = NULL;
         char *key = NULL;
         char *value = NULL;
         GHashTableIter iter;
 
         versioned_attrs = first_named_child(rsc_op, XML_TAG_OP_VER_ATTRS);
         hash = pe_unpack_versioned_parameters(versioned_attrs, metadata->ra_version);
         g_hash_table_iter_init(&iter, hash);
         while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
             g_hash_table_iter_steal(&iter);
             g_hash_table_replace(params, key, value);
         }
         g_hash_table_destroy(hash);
 
         versioned_attrs = first_named_child(rsc_op, XML_TAG_OP_VER_META);
         hash = pe_unpack_versioned_parameters(versioned_attrs, metadata->ra_version);
         g_hash_table_iter_init(&iter, hash);
         while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
             g_hash_table_replace(params, crm_meta_name(key), strdup(value));
 
             if (pcmk__str_eq(key, XML_ATTR_TIMEOUT, pcmk__str_casei)) {
                 pcmk__scan_min_int(value, &op->timeout, 0);
             } else if (pcmk__str_eq(key, XML_OP_ATTR_START_DELAY, pcmk__str_casei)) {
                 pcmk__scan_min_int(value, &op->start_delay, 0);
             }
         }
         g_hash_table_destroy(hash);
 
         versioned_attrs = first_named_child(rsc_op, XML_TAG_RSC_VER_ATTRS);
         hash = pe_unpack_versioned_parameters(versioned_attrs, metadata->ra_version);
         g_hash_table_iter_init(&iter, hash);
         while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
             g_hash_table_iter_steal(&iter);
             g_hash_table_replace(params, key, value);
         }
         g_hash_table_destroy(hash);
     }
 
     lrmd_free_rsc_info(rsc);
 }
 #endif
 
 static lrmd_event_data_t *
 construct_op(lrm_state_t *lrm_state, xmlNode *rsc_op, const char *rsc_id,
              const char *operation)
 {
     lrmd_event_data_t *op = NULL;
     const char *op_delay = NULL;
     const char *op_timeout = NULL;
     GHashTable *params = NULL;
 
     xmlNode *primitive = NULL;
     const char *class = NULL;
 
     const char *transition = NULL;
 
     CRM_ASSERT(rsc_id && operation);
 
     op = lrmd_new_event(rsc_id, operation, 0);
     op->type = lrmd_event_exec_complete;
     op->timeout = 0;
     op->start_delay = 0;
     lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
 
     if (rsc_op == NULL) {
         CRM_LOG_ASSERT(pcmk__str_eq(CRMD_ACTION_STOP, operation, pcmk__str_casei));
         op->user_data = NULL;
         /* the stop_all_resources() case
          * by definition there is no DC (or they'd be shutting
          *   us down).
          * So we should put our version here.
          */
         op->params = pcmk__strkey_table(free, free);
 
         g_hash_table_insert(op->params, strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));
 
         crm_trace("Constructed %s op for %s", operation, rsc_id);
         return op;
     }
 
     params = xml2list(rsc_op);
     g_hash_table_remove(params, CRM_META "_op_target_rc");
 
     op_delay = crm_meta_value(params, XML_OP_ATTR_START_DELAY);
     pcmk__scan_min_int(op_delay, &op->start_delay, 0);
 
     op_timeout = crm_meta_value(params, XML_ATTR_TIMEOUT);
     pcmk__scan_min_int(op_timeout, &op->timeout, 0);
 
     if (pcmk__guint_from_hash(params, CRM_META "_" XML_LRM_ATTR_INTERVAL_MS, 0,
                               &(op->interval_ms)) != pcmk_rc_ok) {
         op->interval_ms = 0;
     }
 
     /* Use pcmk_monitor_timeout instead of meta timeout for stonith
        recurring monitor, if set */
     primitive = find_xml_node(rsc_op, XML_CIB_TAG_RESOURCE, FALSE);
     class = crm_element_value(primitive, XML_AGENT_ATTR_CLASS);
 
     if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_fence_params)
             && pcmk__str_eq(operation, CRMD_ACTION_STATUS, pcmk__str_casei)
             && (op->interval_ms > 0)) {
 
         op_timeout = g_hash_table_lookup(params, "pcmk_monitor_timeout");
         if (op_timeout != NULL) {
             op->timeout = crm_get_msec(op_timeout);
         }
     }
 
 #if ENABLE_VERSIONED_ATTRS
     if (lrm_state && !is_remote_lrmd_ra(NULL, NULL, rsc_id)
         && !pcmk__strcase_any_of(op_type, CRMD_ACTION_METADATA, CRMD_ACTION_DELETE,
                                  NULL)) {
         resolve_versioned_parameters(lrm_state, rsc_id, rsc_op, params);
     }
 #endif
 
     if (!pcmk__str_eq(operation, RSC_STOP, pcmk__str_casei)) {
         op->params = params;
 
     } else {
         rsc_history_t *entry = NULL;
 
         if (lrm_state) {
             entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
         }
 
         /* If we do not have stop parameters cached, use
          * whatever we are given */
         if (!entry || !entry->stop_params) {
             op->params = params;
         } else {
             /* Copy the cached parameter list so that we stop the resource
              * with the old attributes, not the new ones */
             op->params = pcmk__strkey_table(free, free);
 
             g_hash_table_foreach(params, copy_meta_keys, op->params);
             g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
             g_hash_table_destroy(params);
             params = NULL;
         }
     }
 
     /* sanity */
     if (op->timeout <= 0) {
         op->timeout = op->interval_ms;
     }
     if (op->start_delay < 0) {
         op->start_delay = 0;
     }
 
     transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY);
     CRM_CHECK(transition != NULL, return op);
 
     op->user_data = strdup(transition);
 
     if (op->interval_ms != 0) {
         if (pcmk__strcase_any_of(operation, CRMD_ACTION_START, CRMD_ACTION_STOP, NULL)) {
             crm_err("Start and Stop actions cannot have an interval: %u",
                     op->interval_ms);
             op->interval_ms = 0;
         }
     }
 
     crm_trace("Constructed %s op for %s: interval=%u",
               operation, rsc_id, op->interval_ms);
 
     return op;
 }
 
 /*!
  * \internal
  * \brief Send a (synthesized) event result
  *
  * Reply with a synthesized event result directly, as opposed to going through
  * the executor.
  *
  * \param[in] to_host  Host to send result to
  * \param[in] to_sys   IPC name to send result to (NULL for transition engine)
  * \param[in] rsc      Type information about resource the result is for
  * \param[in] op       Event with result to send
  * \param[in] rsc_id   ID of resource the result is for
  */
 void
 controld_ack_event_directly(const char *to_host, const char *to_sys,
                             lrmd_rsc_info_t *rsc, lrmd_event_data_t *op,
                             const char *rsc_id)
 {
     xmlNode *reply = NULL;
     xmlNode *update, *iter;
     crm_node_t *peer = NULL;
 
     CRM_CHECK(op != NULL, return);
     if (op->rsc_id == NULL) {
         CRM_ASSERT(rsc_id != NULL);
         op->rsc_id = strdup(rsc_id);
     }
     if (to_sys == NULL) {
         to_sys = CRM_SYSTEM_TENGINE;
     }
 
     peer = crm_get_peer(0, fsa_our_uname);
     update = create_node_state_update(peer, node_update_none, NULL,
                                       __func__);
 
     iter = create_xml_node(update, XML_CIB_TAG_LRM);
     crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
 
     crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);
 
     build_operation_update(iter, rsc, op, fsa_our_uname, __func__);
     reply = create_request(CRM_OP_INVOKE_LRM, update, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);
 
     crm_log_xml_trace(update, "[direct ACK]");
 
     crm_debug("ACK'ing resource op " PCMK__OP_FMT " from %s: %s",
               op->rsc_id, op->op_type, op->interval_ms, op->user_data,
               crm_element_value(reply, XML_ATTR_REFERENCE));
 
     if (relay_message(reply, TRUE) == FALSE) {
         crm_log_xml_err(reply, "Unable to route reply");
     }
 
     free_xml(update);
     free_xml(reply);
 }
 
 gboolean
 verify_stopped(enum crmd_fsa_state cur_state, int log_level)
 {
     gboolean res = TRUE;
     GList *lrm_state_list = lrm_state_get_list();
     GList *state_entry;
 
     for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
         lrm_state_t *lrm_state = state_entry->data;
 
         if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) {
             /* keep iterating through all even when false is returned */
             res = FALSE;
         }
     }
 
     controld_set_fsa_input_flags(R_SENT_RSC_STOP);
     g_list_free(lrm_state_list); lrm_state_list = NULL;
     return res;
 }
 
 struct stop_recurring_action_s {
     lrmd_rsc_info_t *rsc;
     lrm_state_t *lrm_state;
 };
 
 static gboolean
 stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
 {
     gboolean remove = FALSE;
     struct stop_recurring_action_s *event = user_data;
     active_op_t *op = value;
 
     if ((op->interval_ms != 0)
         && pcmk__str_eq(op->rsc_id, event->rsc->id, pcmk__str_none)) {
 
         crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (char*)key);
         remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE);
     }
 
     return remove;
 }
 
 static gboolean
 stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
 {
     gboolean remove = FALSE;
     lrm_state_t *lrm_state = user_data;
     active_op_t *op = value;
 
     if (op->interval_ms != 0) {
         crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id,
                  (const char *) key);
         remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE);
     }
 
     return remove;
 }
 
 static void
 record_pending_op(const char *node_name, lrmd_rsc_info_t *rsc, lrmd_event_data_t *op)
 {
     const char *record_pending = NULL;
 
     CRM_CHECK(node_name != NULL, return);
     CRM_CHECK(rsc != NULL, return);
     CRM_CHECK(op != NULL, return);
 
     // Never record certain operation types as pending
     if ((op->op_type == NULL) || (op->params == NULL)
         || !controld_action_is_recordable(op->op_type)) {
         return;
     }
 
     // defaults to true
     record_pending = crm_meta_value(op->params, XML_OP_ATTR_PENDING);
     if (record_pending && !crm_is_true(record_pending)) {
         return;
     }
 
     op->call_id = -1;
     lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
 
     op->t_run = time(NULL);
     op->t_rcchange = op->t_run;
 
     /* write a "pending" entry to the CIB, inhibit notification */
     crm_debug("Recording pending op " PCMK__OP_FMT " on %s in the CIB",
               op->rsc_id, op->op_type, op->interval_ms, node_name);
 
     do_update_resource(node_name, rsc, op, 0);
 }
 
 static void
 do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc,
               const char *operation, xmlNode *msg)
 {
     int rc;
     int call_id = 0;
     char *op_id = NULL;
     lrmd_event_data_t *op = NULL;
     lrmd_key_value_t *params = NULL;
     fsa_data_t *msg_data = NULL;
     const char *transition = NULL;
     gboolean stop_recurring = FALSE;
-    bool send_nack = FALSE;
+    const char *nack_reason = NULL;
 
     CRM_CHECK(rsc != NULL, return);
     CRM_CHECK(operation != NULL, return);
 
     if (msg != NULL) {
         transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
         if (transition == NULL) {
             crm_log_xml_err(msg, "Missing transition number");
         }
     }
 
     op = construct_op(lrm_state, msg, rsc->id, operation);
     CRM_CHECK(op != NULL, return);
 
     if (is_remote_lrmd_ra(NULL, NULL, rsc->id)
         && (op->interval_ms == 0)
         && strcmp(operation, CRMD_ACTION_MIGRATE) == 0) {
 
         /* pcmk remote connections are a special use case.
          * We never ever want to stop monitoring a connection resource until
          * the entire migration has completed. If the connection is unexpectedly
          * severed, even during a migration, this is an event we must detect.*/
         stop_recurring = FALSE;
 
     } else if ((op->interval_ms == 0)
         && strcmp(operation, CRMD_ACTION_STATUS) != 0
         && strcmp(operation, CRMD_ACTION_NOTIFY) != 0) {
 
         /* stop any previous monitor operations before changing the resource state */
         stop_recurring = TRUE;
     }
 
     if (stop_recurring == TRUE) {
         guint removed = 0;
         struct stop_recurring_action_s data;
 
         data.rsc = rsc;
         data.lrm_state = lrm_state;
         removed = g_hash_table_foreach_remove(
             lrm_state->pending_ops, stop_recurring_action_by_rsc, &data);
 
         if (removed) {
             crm_debug("Stopped %u recurring operation%s in preparation for "
                       PCMK__OP_FMT, removed, pcmk__plural_s(removed),
                       rsc->id, operation, op->interval_ms);
         }
     }
 
     /* now do the op */
     crm_notice("Requesting local execution of %s operation for %s on %s "
                CRM_XS " transition_key=%s op_key=" PCMK__OP_FMT,
                crm_action_str(op->op_type, op->interval_ms), rsc->id, lrm_state->node_name,
                transition, rsc->id, operation, op->interval_ms);
 
     if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)
         && pcmk__str_eq(operation, RSC_START, pcmk__str_casei)) {
 
         register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
-        send_nack = TRUE;
+        nack_reason = "Not attempting start due to shutdown in progress";
 
     } else if (fsa_state != S_NOT_DC
                && fsa_state != S_POLICY_ENGINE /* Recalculating */
                && fsa_state != S_TRANSITION_ENGINE
                && !pcmk__str_eq(operation, CRMD_ACTION_STOP, pcmk__str_casei)) {
-        send_nack = TRUE;
+        nack_reason = "Controller cannot attempt actions at this time";
     }
 
-    if(send_nack) {
+    if (nack_reason != NULL) {
         crm_notice("Discarding attempt to perform action %s on %s in state %s (shutdown=%s)",
                    operation, rsc->id, fsa_state2string(fsa_state),
                    pcmk__btoa(pcmk_is_set(fsa_input_register, R_SHUTDOWN)));
 
-        lrmd__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_INVALID, NULL);
+        lrmd__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_INVALID,
+                         nack_reason);
         controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
         lrmd_free_event(op);
         free(op_id);
         return;
     }
 
     record_pending_op(lrm_state->node_name, rsc, op);
 
     op_id = pcmk__op_key(rsc->id, op->op_type, op->interval_ms);
 
     if (op->interval_ms > 0) {
         /* cancel it so we can then restart it without conflict */
         cancel_op_key(lrm_state, rsc, op_id, FALSE);
     }
 
     if (op->params) {
         char *key = NULL;
         char *value = NULL;
         GHashTableIter iter;
 
         g_hash_table_iter_init(&iter, op->params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             params = lrmd_key_value_add(params, key, value);
         }
     }
 
     rc = controld_execute_resource_agent(lrm_state, rsc->id, op->op_type,
                                          op->user_data, op->interval_ms,
                                          op->timeout, op->start_delay, params,
                                          &call_id);
     if (rc == pcmk_rc_ok) {
         /* record all operations so we can wait
          * for them to complete during shutdown
          */
         char *call_id_s = make_stop_id(rsc->id, call_id);
         active_op_t *pending = NULL;
 
         pending = calloc(1, sizeof(active_op_t));
         crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);
 
         pending->call_id = call_id;
         pending->interval_ms = op->interval_ms;
         pending->op_type = strdup(operation);
         pending->op_key = strdup(op_id);
         pending->rsc_id = strdup(rsc->id);
         pending->start_time = time(NULL);
         pending->user_data = op->user_data? strdup(op->user_data) : NULL;
         if (crm_element_value_epoch(msg, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
                                     &(pending->lock_time)) != pcmk_ok) {
             pending->lock_time = 0;
         }
         g_hash_table_replace(lrm_state->pending_ops, call_id_s, pending);
 
         if ((op->interval_ms > 0)
             && (op->start_delay > START_DELAY_THRESHOLD)) {
             int target_rc = PCMK_OCF_OK;
 
             crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);
             decode_transition_key(op->user_data, NULL, NULL, NULL, &target_rc);
             lrmd__set_result(op, target_rc, PCMK_EXEC_DONE, NULL);
             controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
         }
 
         pending->params = op->params;
         op->params = NULL;
 
     } else if (lrm_state_is_local(lrm_state)) {
         crm_err("Could not initiate %s action for resource %s locally: %s "
                 CRM_XS " rc=%d", operation, rsc->id, pcmk_rc_str(rc), rc);
         fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
                        PCMK_OCF_UNKNOWN_ERROR);
         process_lrm_event(lrm_state, op, NULL, NULL);
         register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
 
     } else {
         crm_err("Could not initiate %s action for resource %s remotely on %s: "
                 "%s " CRM_XS " rc=%d",
                 operation, rsc->id, lrm_state->node_name, pcmk_rc_str(rc), rc);
         fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
                        PCMK_OCF_UNKNOWN_ERROR);
         process_lrm_event(lrm_state, op, NULL, NULL);
     }
 
     free(op_id);
     lrmd_free_event(op);
 }
 
 int last_resource_update = 0;
 
 static void
 cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     switch (rc) {
         case pcmk_ok:
         case -pcmk_err_diff_failed:
         case -pcmk_err_diff_resync:
             crm_trace("Resource update %d complete: rc=%d", call_id, rc);
             break;
         default:
             crm_warn("Resource update %d failed: (rc=%d) %s", call_id, rc, pcmk_strerror(rc));
     }
 
     if (call_id == last_resource_update) {
         last_resource_update = 0;
         trigger_fsa();
     }
 }
 
 /* Only successful stops, and probes that found the resource inactive, get locks
  * recorded in the history. This ensures the resource stays locked to the node
  * until it is active there again after the node comes back up.
  */
 static bool
 should_preserve_lock(lrmd_event_data_t *op)
 {
     if (!controld_shutdown_lock_enabled) {
         return false;
     }
     if (!strcmp(op->op_type, RSC_STOP) && (op->rc == PCMK_OCF_OK)) {
         return true;
     }
     if (!strcmp(op->op_type, RSC_STATUS) && (op->rc == PCMK_OCF_NOT_RUNNING)) {
         return true;
     }
     return false;
 }
 
 static int
 do_update_resource(const char *node_name, lrmd_rsc_info_t *rsc,
                    lrmd_event_data_t *op, time_t lock_time)
 {
 /*
   <status>
   <nodes_status id=uname>
   <lrm>
   <lrm_resources>
   <lrm_resource id=...>
   </...>
 */
     int rc = pcmk_ok;
     xmlNode *update, *iter = NULL;
     int call_opt = crmd_cib_smart_opt();
     const char *uuid = NULL;
 
     CRM_CHECK(op != NULL, return 0);
 
     iter = create_xml_node(iter, XML_CIB_TAG_STATUS);
     update = iter;
     iter = create_xml_node(iter, XML_CIB_TAG_STATE);
 
     if (pcmk__str_eq(node_name, fsa_our_uname, pcmk__str_casei)) {
         uuid = fsa_our_uuid;
 
     } else {
         /* remote nodes uuid and uname are equal */
         uuid = node_name;
         crm_xml_add(iter, XML_NODE_IS_REMOTE, "true");
     }
 
     CRM_LOG_ASSERT(uuid != NULL);
     if(uuid == NULL) {
         rc = -EINVAL;
         goto done;
     }
 
     crm_xml_add(iter, XML_ATTR_UUID,  uuid);
     crm_xml_add(iter, XML_ATTR_UNAME, node_name);
     crm_xml_add(iter, XML_ATTR_ORIGIN, __func__);
 
     iter = create_xml_node(iter, XML_CIB_TAG_LRM);
     crm_xml_add(iter, XML_ATTR_ID, uuid);
 
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
     crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);
 
     build_operation_update(iter, rsc, op, node_name, __func__);
 
     if (rsc) {
         const char *container = NULL;
 
         crm_xml_add(iter, XML_ATTR_TYPE, rsc->type);
         crm_xml_add(iter, XML_AGENT_ATTR_CLASS, rsc->standard);
         crm_xml_add(iter, XML_AGENT_ATTR_PROVIDER, rsc->provider);
         if (lock_time != 0) {
             /* Actions on a locked resource should either preserve the lock by
              * recording it with the action result, or clear it.
              */
             if (!should_preserve_lock(op)) {
                 lock_time = 0;
             }
             crm_xml_add_ll(iter, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
                            (long long) lock_time);
         }
 
         if (op->params) {
             container = g_hash_table_lookup(op->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
         }
         if (container) {
             crm_trace("Resource %s is a part of container resource %s", op->rsc_id, container);
             crm_xml_add(iter, XML_RSC_ATTR_CONTAINER, container);
         }
 
     } else {
         crm_warn("Resource %s no longer exists in the executor", op->rsc_id);
         controld_ack_event_directly(NULL, NULL, rsc, op, op->rsc_id);
         goto cleanup;
     }
 
     crm_log_xml_trace(update, __func__);
 
     /* make it an asynchronous call and be done with it
      *
      * Best case:
      *   the resource state will be discovered during
      *   the next signup or election.
      *
      * Bad case:
      *   we are shutting down and there is no DC at the time,
      *   but then why were we shutting down then anyway?
      *   (probably because of an internal error)
      *
      * Worst case:
      *   we get shot for having resources "running" that really weren't
      *
      * the alternative however means blocking here for too long, which
      * isn't acceptable
      */
     fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, rc, NULL);
 
     if (rc > 0) {
         last_resource_update = rc;
     }
   done:
     /* the return code is a call number, not an error code */
     crm_trace("Sent resource state update message: %d for %s=%u on %s",
               rc, op->op_type, op->interval_ms, op->rsc_id);
     fsa_register_cib_callback(rc, FALSE, NULL, cib_rsc_callback);
 
   cleanup:
     free_xml(update);
     return rc;
 }
 
 void
 do_lrm_event(long long action,
              enum crmd_fsa_cause cause,
              enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
 {
     CRM_CHECK(FALSE, return);
 }
 
 static char *
 unescape_newlines(const char *string)
 {
     char *pch = NULL;
     char *ret = NULL;
     static const char *escaped_newline = "\\n";
 
     if (!string) {
         return NULL;
     }
 
     ret = strdup(string);
     pch = strstr(ret, escaped_newline);
     while (pch != NULL) {
         /* Replace newline escape pattern with actual newline (and a space so we
          * don't have to shuffle the rest of the buffer)
          */
         pch[0] = '\n';
         pch[1] = ' ';
         pch = strstr(pch, escaped_newline);
     }
 
     return ret;
 }
 
 static bool
 did_lrm_rsc_op_fail(lrm_state_t *lrm_state, const char * rsc_id,
                     const char * op_type, guint interval_ms)
 {
     rsc_history_t *entry = NULL;
 
     CRM_CHECK(lrm_state != NULL, return FALSE);
     CRM_CHECK(rsc_id != NULL, return FALSE);
     CRM_CHECK(op_type != NULL, return FALSE);
 
     entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
     if (entry == NULL || entry->failed == NULL) {
         return FALSE;
     }
 
     if (pcmk__str_eq(entry->failed->rsc_id, rsc_id, pcmk__str_none)
         && pcmk__str_eq(entry->failed->op_type, op_type, pcmk__str_casei)
         && entry->failed->interval_ms == interval_ms) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 void
 process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
                   active_op_t *pending, xmlNode *action_xml)
 {
     char *op_id = NULL;
     char *op_key = NULL;
 
     int update_id = 0;
     gboolean remove = FALSE;
     gboolean removed = FALSE;
     bool need_direct_ack = FALSE;
     lrmd_rsc_info_t *rsc = NULL;
     const char *node_name = NULL;
 
     CRM_CHECK(op != NULL, return);
     CRM_CHECK(op->rsc_id != NULL, return);
 
     // Remap new status codes for older DCs
     if (compare_version(fsa_our_dc_version, "3.2.0") < 0) {
         switch (op->op_status) {
             case PCMK_EXEC_NOT_CONNECTED:
                 lrmd__set_result(op, PCMK_OCF_CONNECTION_DIED,
                                  PCMK_EXEC_ERROR, op->exit_reason);
                 break;
             case PCMK_EXEC_INVALID:
                 lrmd__set_result(op, CRM_DIRECT_NACK_RC, PCMK_EXEC_ERROR,
                                  op->exit_reason);
                 break;
             default:
                 break;
         }
     }
 
     op_id = make_stop_id(op->rsc_id, op->call_id);
     op_key = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
 
     // Get resource info if available (from executor state or action XML)
     if (lrm_state) {
         rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
     }
     if ((rsc == NULL) && action_xml) {
         xmlNode *xml = find_xml_node(action_xml, XML_CIB_TAG_RESOURCE, TRUE);
 
         const char *standard = crm_element_value(xml, XML_AGENT_ATTR_CLASS);
         const char *provider = crm_element_value(xml, XML_AGENT_ATTR_PROVIDER);
         const char *type = crm_element_value(xml, XML_ATTR_TYPE);
 
         if (standard && type) {
             crm_info("%s agent information not cached, using %s%s%s:%s from action XML",
                      op->rsc_id, standard,
                      (provider? ":" : ""), (provider? provider : ""), type);
             rsc = lrmd_new_rsc_info(op->rsc_id, standard, provider, type);
         } else {
             crm_err("Can't process %s result because %s agent information not cached or in XML",
                     op_key, op->rsc_id);
         }
     }
 
     // Get node name if available (from executor state or action XML)
     if (lrm_state) {
         node_name = lrm_state->node_name;
     } else if (action_xml) {
         node_name = crm_element_value(action_xml, XML_LRM_ATTR_TARGET);
     }
 
     if(pending == NULL) {
         remove = TRUE;
         if (lrm_state) {
             pending = g_hash_table_lookup(lrm_state->pending_ops, op_id);
         }
     }
 
     if (op->op_status == PCMK_EXEC_ERROR) {
         switch(op->rc) {
             case PCMK_OCF_NOT_RUNNING:
             case PCMK_OCF_RUNNING_PROMOTED:
             case PCMK_OCF_DEGRADED:
             case PCMK_OCF_DEGRADED_PROMOTED:
                 // Leave it to the TE/scheduler to decide if this is an error
                 op->op_status = PCMK_EXEC_DONE;
                 break;
             default:
                 /* Nothing to do */
                 break;
         }
     }
 
     if (op->op_status != PCMK_EXEC_CANCELLED) {
         /* We might not record the result, so directly acknowledge it to the
          * originator instead, so it doesn't time out waiting for the result
          * (especially important if part of a transition).
          */
         need_direct_ack = TRUE;
 
         if (controld_action_is_recordable(op->op_type)) {
             if (node_name && rsc) {
                 // We should record the result, and happily, we can
                 update_id = do_update_resource(node_name, rsc, op,
                                                pending? pending->lock_time : 0);
                 need_direct_ack = FALSE;
 
             } else if (op->rsc_deleted) {
                 /* We shouldn't record the result (likely the resource was
                  * refreshed, cleaned, or removed while this operation was
                  * in flight).
                  */
                 crm_notice("Not recording %s result in CIB because "
                            "resource information was removed since it was initiated",
                            op_key);
             } else {
                 /* This shouldn't be possible; the executor didn't consider the
                  * resource deleted, but we couldn't find resource or node
                  * information.
                  */
                 crm_err("Unable to record %s result in CIB: %s", op_key,
                         (node_name? "No resource information" : "No node name"));
             }
         }
 
     } else if (op->interval_ms == 0) {
         /* A non-recurring operation was cancelled. Most likely, the
          * never-initiated action was removed from the executor's pending
          * operations list upon resource removal.
          */
         need_direct_ack = TRUE;
 
     } else if (pending == NULL) {
         /* This recurring operation was cancelled, but was not pending. No
          * transition actions are waiting on it, nothing needs to be done.
          */
 
     } else if (op->user_data == NULL) {
         /* This recurring operation was cancelled and pending, but we don't
          * have a transition key. This should never happen.
          */
         crm_err("Recurring operation %s was cancelled without transition information",
                 op_key);
 
     } else if (pcmk_is_set(pending->flags, active_op_remove)) {
         /* This recurring operation was cancelled (by us) and pending, and we
          * have been waiting for it to finish.
          */
         if (lrm_state) {
             erase_lrm_history_by_op(lrm_state, op);
         }
 
         /* If the recurring operation had failed, the lrm_rsc_op is recorded as
          * "last_failure" which won't get erased from the cib given the logic on
          * purpose in erase_lrm_history_by_op(). So that the cancel action won't
          * have a chance to get confirmed by DC with process_op_deletion().
          * Cluster transition would get stuck waiting for the remaining action
          * timer to time out.
          *
          * Directly acknowledge the cancel operation in this case.
          */
         if (did_lrm_rsc_op_fail(lrm_state, pending->rsc_id,
                                 pending->op_type, pending->interval_ms)) {
             need_direct_ack = TRUE;
         }
 
     } else if (op->rsc_deleted) {
         /* This recurring operation was cancelled (but not by us, and the
          * executor does not have resource information, likely due to resource
          * cleanup, refresh, or removal) and pending.
          */
         crm_debug("Recurring op %s was cancelled due to resource deletion",
                   op_key);
         need_direct_ack = TRUE;
 
     } else {
         /* This recurring operation was cancelled (but not by us, likely by the
          * executor before stopping the resource) and pending. We don't need to
          * do anything special.
          */
     }
 
     if (need_direct_ack) {
         controld_ack_event_directly(NULL, NULL, NULL, op, op->rsc_id);
     }
 
     if(remove == FALSE) {
         /* The caller will do this afterwards, but keep the logging consistent */
         removed = TRUE;
 
     } else if (lrm_state && ((op->interval_ms == 0)
                              || (op->op_status == PCMK_EXEC_CANCELLED))) {
 
         gboolean found = g_hash_table_remove(lrm_state->pending_ops, op_id);
 
         if (op->interval_ms != 0) {
             removed = TRUE;
         } else if (found) {
             removed = TRUE;
             crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
                       op_key, op->call_id, op_id,
                       g_hash_table_size(lrm_state->pending_ops));
         }
     }
 
     if (node_name == NULL) {
         node_name = "unknown node"; // for logging
     }
 
     switch (op->op_status) {
         case PCMK_EXEC_CANCELLED:
             crm_info("Result of %s operation for %s on %s: %s "
                      CRM_XS " call=%d key=%s confirmed=%s",
                      crm_action_str(op->op_type, op->interval_ms),
                      op->rsc_id, node_name, pcmk_exec_status_str(op->op_status),
                      op->call_id, op_key, pcmk__btoa(removed));
             break;
 
         case PCMK_EXEC_DONE:
             crm_notice("Result of %s operation for %s on %s: %s "
                        CRM_XS " rc=%d call=%d key=%s confirmed=%s cib-update=%d",
                        crm_action_str(op->op_type, op->interval_ms),
                        op->rsc_id, node_name,
                        services_ocf_exitcode_str(op->rc), op->rc,
                        op->call_id, op_key, pcmk__btoa(removed), update_id);
             break;
 
         case PCMK_EXEC_TIMEOUT:
             crm_err("Result of %s operation for %s on %s: %s "
                     CRM_XS " call=%d key=%s timeout=%dms",
                     crm_action_str(op->op_type, op->interval_ms),
                     op->rsc_id, node_name, pcmk_exec_status_str(op->op_status),
                     op->call_id, op_key, op->timeout);
             break;
 
         default:
             crm_err("Result of %s operation for %s on %s: %s "
                     CRM_XS " call=%d key=%s confirmed=%s status=%d cib-update=%d",
                     crm_action_str(op->op_type, op->interval_ms),
                     op->rsc_id, node_name, pcmk_exec_status_str(op->op_status),
                     op->call_id, op_key, pcmk__btoa(removed), op->op_status,
                     update_id);
     }
 
     if (op->output) {
         char *prefix =
             crm_strdup_printf("%s-" PCMK__OP_FMT ":%d", node_name,
                               op->rsc_id, op->op_type, op->interval_ms,
                               op->call_id);
 
         if (op->rc) {
             crm_log_output(LOG_NOTICE, prefix, op->output);
         } else {
             crm_log_output(LOG_DEBUG, prefix, op->output);
         }
         free(prefix);
     }
 
     if (lrm_state) {
         if (!pcmk__str_eq(op->op_type, RSC_METADATA, pcmk__str_casei)) {
             crmd_alert_resource_op(lrm_state->node_name, op);
         } else if (rsc && (op->rc == PCMK_OCF_OK)) {
             char *metadata = unescape_newlines(op->output);
 
             metadata_cache_update(lrm_state->metadata_cache, rsc, metadata);
             free(metadata);
         }
     }
 
     if (op->rsc_deleted) {
         crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
         if (lrm_state) {
             delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL);
         }
     }
 
     /* If a shutdown was escalated while operations were pending,
      * then the FSA will be stalled right now... allow it to continue
      */
     mainloop_set_trigger(fsa_source);
     if (lrm_state && rsc) {
         update_history_cache(lrm_state, rsc, op);
     }
 
     lrmd_free_rsc_info(rsc);
     free(op_key);
     free(op_id);
 }
diff --git a/daemons/controld/controld_execd_state.c b/daemons/controld/controld_execd_state.c
index 5435402717..a0129bd514 100644
--- a/daemons/controld/controld_execd_state.c
+++ b/daemons/controld/controld_execd_state.c
@@ -1,853 +1,853 @@
 /*
  * Copyright 2012-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <errno.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/iso8601.h>
 #include <crm/pengine/rules.h>
 #include <crm/pengine/rules_internal.h>
 #include <crm/lrmd_internal.h>
 
 #include <pacemaker-internal.h>
 #include <pacemaker-controld.h>
 
 GHashTable *lrm_state_table = NULL;
 extern GHashTable *proxy_table;
 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
 
 static void
 free_rsc_info(gpointer value)
 {
     lrmd_rsc_info_t *rsc_info = value;
 
     lrmd_free_rsc_info(rsc_info);
 }
 
 static void
 free_deletion_op(gpointer value)
 {
     struct pending_deletion_op_s *op = value;
 
     free(op->rsc);
     delete_ha_msg_input(op->input);
     free(op);
 }
 
 static void
 free_recurring_op(gpointer value)
 {
     active_op_t *op = value;
 
     free(op->user_data);
     free(op->rsc_id);
     free(op->op_type);
     free(op->op_key);
     if (op->params) {
         g_hash_table_destroy(op->params);
     }
     free(op);
 }
 
 static gboolean
 fail_pending_op(gpointer key, gpointer value, gpointer user_data)
 {
     lrmd_event_data_t event = { 0, };
     lrm_state_t *lrm_state = user_data;
     active_op_t *op = value;
 
     crm_trace("Pre-emptively failing " PCMK__OP_FMT " on %s (call=%s, %s)",
               op->rsc_id, op->op_type, op->interval_ms,
               lrm_state->node_name, (char*)key, op->user_data);
 
     event.type = lrmd_event_exec_complete;
     event.rsc_id = op->rsc_id;
     event.op_type = op->op_type;
     event.user_data = op->user_data;
     event.timeout = 0;
     event.interval_ms = op->interval_ms;
     lrmd__set_result(&event, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_NOT_CONNECTED,
-                     NULL);
+                     "Action was pending when executor connection was dropped");
     event.t_run = (unsigned int) op->start_time;
     event.t_rcchange = (unsigned int) op->start_time;
 
     event.call_id = op->call_id;
     event.remote_nodename = lrm_state->node_name;
     event.params = op->params;
 
     process_lrm_event(lrm_state, &event, op, NULL);
     return TRUE;
 }
 
 gboolean
 lrm_state_is_local(lrm_state_t *lrm_state)
 {
     if (lrm_state == NULL || fsa_our_uname == NULL) {
         return FALSE;
     }
 
     if (strcmp(lrm_state->node_name, fsa_our_uname) != 0) {
         return FALSE;
     }
 
     return TRUE;
 
 }
 
 lrm_state_t *
 lrm_state_create(const char *node_name)
 {
     lrm_state_t *state = NULL;
 
     if (!node_name) {
         crm_err("No node name given for lrm state object");
         return NULL;
     }
 
     state = calloc(1, sizeof(lrm_state_t));
     if (!state) {
         return NULL;
     }
 
     state->node_name = strdup(node_name);
     state->rsc_info_cache = pcmk__strkey_table(NULL, free_rsc_info);
     state->deletion_ops = pcmk__strkey_table(free, free_deletion_op);
     state->pending_ops = pcmk__strkey_table(free, free_recurring_op);
     state->resource_history = pcmk__strkey_table(NULL, history_free);
     state->metadata_cache = metadata_cache_new();
 
     g_hash_table_insert(lrm_state_table, (char *)state->node_name, state);
     return state;
 }
 
 void
 lrm_state_destroy(const char *node_name)
 {
     g_hash_table_remove(lrm_state_table, node_name);
 }
 
 static gboolean
 remote_proxy_remove_by_node(gpointer key, gpointer value, gpointer user_data)
 {
     remote_proxy_t *proxy = value;
     const char *node_name = user_data;
 
     if (pcmk__str_eq(node_name, proxy->node_name, pcmk__str_casei)) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 static void
 internal_lrm_state_destroy(gpointer data)
 {
     lrm_state_t *lrm_state = data;
 
     if (!lrm_state) {
         return;
     }
 
     crm_trace("Destroying proxy table %s with %d members", lrm_state->node_name, g_hash_table_size(proxy_table));
     g_hash_table_foreach_remove(proxy_table, remote_proxy_remove_by_node, (char *) lrm_state->node_name);
     remote_ra_cleanup(lrm_state);
     lrmd_api_delete(lrm_state->conn);
 
     if (lrm_state->rsc_info_cache) {
         crm_trace("Destroying rsc info cache with %d members", g_hash_table_size(lrm_state->rsc_info_cache));
         g_hash_table_destroy(lrm_state->rsc_info_cache);
     }
     if (lrm_state->resource_history) {
         crm_trace("Destroying history op cache with %d members", g_hash_table_size(lrm_state->resource_history));
         g_hash_table_destroy(lrm_state->resource_history);
     }
     if (lrm_state->deletion_ops) {
         crm_trace("Destroying deletion op cache with %d members", g_hash_table_size(lrm_state->deletion_ops));
         g_hash_table_destroy(lrm_state->deletion_ops);
     }
     if (lrm_state->pending_ops) {
         crm_trace("Destroying pending op cache with %d members", g_hash_table_size(lrm_state->pending_ops));
         g_hash_table_destroy(lrm_state->pending_ops);
     }
     metadata_cache_free(lrm_state->metadata_cache);
 
     free((char *)lrm_state->node_name);
     free(lrm_state);
 }
 
 void
 lrm_state_reset_tables(lrm_state_t * lrm_state, gboolean reset_metadata)
 {
     if (lrm_state->resource_history) {
         crm_trace("Re-setting history op cache with %d members",
                   g_hash_table_size(lrm_state->resource_history));
         g_hash_table_remove_all(lrm_state->resource_history);
     }
     if (lrm_state->deletion_ops) {
         crm_trace("Re-setting deletion op cache with %d members",
                   g_hash_table_size(lrm_state->deletion_ops));
         g_hash_table_remove_all(lrm_state->deletion_ops);
     }
     if (lrm_state->pending_ops) {
         crm_trace("Re-setting pending op cache with %d members",
                   g_hash_table_size(lrm_state->pending_ops));
         g_hash_table_remove_all(lrm_state->pending_ops);
     }
     if (lrm_state->rsc_info_cache) {
         crm_trace("Re-setting rsc info cache with %d members",
                   g_hash_table_size(lrm_state->rsc_info_cache));
         g_hash_table_remove_all(lrm_state->rsc_info_cache);
     }
     if (reset_metadata) {
         metadata_cache_reset(lrm_state->metadata_cache);
     }
 }
 
 gboolean
 lrm_state_init_local(void)
 {
     if (lrm_state_table) {
         return TRUE;
     }
 
     lrm_state_table = pcmk__strikey_table(NULL, internal_lrm_state_destroy);
     if (!lrm_state_table) {
         return FALSE;
     }
 
     proxy_table = pcmk__strikey_table(NULL, remote_proxy_free);
     if (!proxy_table) {
         g_hash_table_destroy(lrm_state_table);
         lrm_state_table = NULL;
         return FALSE;
     }
 
     return TRUE;
 }
 
 void
 lrm_state_destroy_all(void)
 {
     if (lrm_state_table) {
         crm_trace("Destroying state table with %d members", g_hash_table_size(lrm_state_table));
         g_hash_table_destroy(lrm_state_table); lrm_state_table = NULL;
     }
     if(proxy_table) {
         crm_trace("Destroying proxy table with %d members", g_hash_table_size(proxy_table));
         g_hash_table_destroy(proxy_table); proxy_table = NULL;
     }
 }
 
 lrm_state_t *
 lrm_state_find(const char *node_name)
 {
     if (!node_name) {
         return NULL;
     }
     return g_hash_table_lookup(lrm_state_table, node_name);
 }
 
 lrm_state_t *
 lrm_state_find_or_create(const char *node_name)
 {
     lrm_state_t *lrm_state;
 
     lrm_state = g_hash_table_lookup(lrm_state_table, node_name);
     if (!lrm_state) {
         lrm_state = lrm_state_create(node_name);
     }
 
     return lrm_state;
 }
 
 GList *
 lrm_state_get_list(void)
 {
     return g_hash_table_get_values(lrm_state_table);
 }
 
 static remote_proxy_t *
 find_connected_proxy_by_node(const char * node_name)
 {
     GHashTableIter gIter;
     remote_proxy_t *proxy = NULL;
 
     CRM_CHECK(proxy_table != NULL, return NULL);
 
     g_hash_table_iter_init(&gIter, proxy_table);
 
     while (g_hash_table_iter_next(&gIter, NULL, (gpointer *) &proxy)) {
         if (proxy->source
             && pcmk__str_eq(node_name, proxy->node_name, pcmk__str_casei)) {
             return proxy;
         }
     }
 
     return NULL;
 }
 
 static void
 remote_proxy_disconnect_by_node(const char * node_name)
 {
     remote_proxy_t *proxy = NULL;
 
     CRM_CHECK(proxy_table != NULL, return);
 
     while ((proxy = find_connected_proxy_by_node(node_name)) != NULL) {
         /* mainloop_del_ipc_client() eventually calls remote_proxy_disconnected()
          * , which removes the entry from proxy_table.
          * Do not do this in a g_hash_table_iter_next() loop. */
         if (proxy->source) {
             mainloop_del_ipc_client(proxy->source);
         }
     }
 
     return;
 }
 
 void
 lrm_state_disconnect_only(lrm_state_t * lrm_state)
 {
     int removed = 0;
 
     if (!lrm_state->conn) {
         return;
     }
     crm_trace("Disconnecting %s", lrm_state->node_name);
 
     remote_proxy_disconnect_by_node(lrm_state->node_name);
 
     ((lrmd_t *) lrm_state->conn)->cmds->disconnect(lrm_state->conn);
 
     if (!pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
         removed = g_hash_table_foreach_remove(lrm_state->pending_ops, fail_pending_op, lrm_state);
         crm_trace("Synthesized %d operation failures for %s", removed, lrm_state->node_name);
     }
 }
 
 void
 lrm_state_disconnect(lrm_state_t * lrm_state)
 {
     if (!lrm_state->conn) {
         return;
     }
 
     lrm_state_disconnect_only(lrm_state);
 
     lrmd_api_delete(lrm_state->conn);
     lrm_state->conn = NULL;
 }
 
 int
 lrm_state_is_connected(lrm_state_t * lrm_state)
 {
     if (!lrm_state->conn) {
         return FALSE;
     }
     return ((lrmd_t *) lrm_state->conn)->cmds->is_connected(lrm_state->conn);
 }
 
 int
 lrm_state_poke_connection(lrm_state_t * lrm_state)
 {
 
     if (!lrm_state->conn) {
         return -ENOTCONN;
     }
     return ((lrmd_t *) lrm_state->conn)->cmds->poke_connection(lrm_state->conn);
 }
 
 // \return Standard Pacemaker return code
 int
 controld_connect_local_executor(lrm_state_t *lrm_state)
 {
     int rc = pcmk_rc_ok;
 
     if (lrm_state->conn == NULL) {
         lrmd_t *api = NULL;
 
         rc = lrmd__new(&api, NULL, NULL, 0);
         if (rc != pcmk_rc_ok) {
             return rc;
         }
         api->cmds->set_callback(api, lrm_op_callback);
         lrm_state->conn = api;
     }
 
     rc = ((lrmd_t *) lrm_state->conn)->cmds->connect(lrm_state->conn,
                                                      CRM_SYSTEM_CRMD, NULL);
     rc = pcmk_legacy2rc(rc);
 
     if (rc == pcmk_rc_ok) {
         lrm_state->num_lrm_register_fails = 0;
     } else {
         lrm_state->num_lrm_register_fails++;
     }
     return rc;
 }
 
 static remote_proxy_t *
 crmd_remote_proxy_new(lrmd_t *lrmd, const char *node_name, const char *session_id, const char *channel)
 {
     struct ipc_client_callbacks proxy_callbacks = {
         .dispatch = remote_proxy_dispatch,
         .destroy = remote_proxy_disconnected
     };
     remote_proxy_t *proxy = remote_proxy_new(lrmd, &proxy_callbacks, node_name,
                                              session_id, channel);
     return proxy;
 }
 
 gboolean
 crmd_is_proxy_session(const char *session)
 {
     return g_hash_table_lookup(proxy_table, session) ? TRUE : FALSE;
 }
 
 void
 crmd_proxy_send(const char *session, xmlNode *msg)
 {
     remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);
     lrm_state_t *lrm_state = NULL;
 
     if (!proxy) {
         return;
     }
     crm_log_xml_trace(msg, "to-proxy");
     lrm_state = lrm_state_find(proxy->node_name);
     if (lrm_state) {
         crm_trace("Sending event to %.8s on %s", proxy->session_id, proxy->node_name);
         remote_proxy_relay_event(proxy, msg);
     }
 }
 
 static void
 crmd_proxy_dispatch(const char *session, xmlNode *msg)
 {
     crm_trace("Processing proxied IPC message from session %s", session);
     crm_log_xml_trace(msg, "controller[inbound]");
     crm_xml_add(msg, F_CRM_SYS_FROM, session);
     if (controld_authorize_ipc_message(msg, NULL, session)) {
         route_message(C_IPC_MESSAGE, msg);
     }
     trigger_fsa();
 }
 
 static void
 remote_config_check(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     if (rc != pcmk_ok) {
         crm_err("Query resulted in an error: %s", pcmk_strerror(rc));
 
         if (rc == -EACCES || rc == -pcmk_err_schema_validation) {
             crm_err("The cluster is mis-configured - shutting down and staying down");
         }
 
     } else {
         lrmd_t * lrmd = (lrmd_t *)user_data;
         crm_time_t *now = crm_time_new(NULL);
         GHashTable *config_hash = pcmk__strkey_table(free, free);
 
         crm_debug("Call %d : Parsing CIB options", call_id);
 
         pe_unpack_nvpairs(output, output, XML_CIB_TAG_PROPSET, NULL,
                           config_hash, CIB_OPTIONS_FIRST, FALSE, now, NULL);
 
         /* Now send it to the remote peer */
         lrmd__validate_remote_settings(lrmd, config_hash);
 
         g_hash_table_destroy(config_hash);
         crm_time_free(now);
     }
 }
 
 static void
 crmd_remote_proxy_cb(lrmd_t *lrmd, void *userdata, xmlNode *msg)
 {
     lrm_state_t *lrm_state = userdata;
     const char *session = crm_element_value(msg, F_LRMD_IPC_SESSION);
     remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);
 
     const char *op = crm_element_value(msg, F_LRMD_IPC_OP);
     if (pcmk__str_eq(op, LRMD_IPC_OP_NEW, pcmk__str_casei)) {
         const char *channel = crm_element_value(msg, F_LRMD_IPC_IPC_SERVER);
 
         proxy = crmd_remote_proxy_new(lrmd, lrm_state->node_name, session, channel);
         if (!remote_ra_controlling_guest(lrm_state)) {
             if (proxy != NULL) {
                 /* Look up stonith-watchdog-timeout and send to the remote peer for validation */
                 int rc = fsa_cib_conn->cmds->query(fsa_cib_conn, XML_CIB_TAG_CRMCONFIG, NULL, cib_scope_local);
                 fsa_cib_conn->cmds->register_callback_full(fsa_cib_conn, rc, 10, FALSE, lrmd,
                                                         "remote_config_check", remote_config_check, NULL);
             }
         } else {
             crm_debug("Skipping remote_config_check for guest-nodes");
         }
 
     } else if (pcmk__str_eq(op, LRMD_IPC_OP_SHUTDOWN_REQ, pcmk__str_casei)) {
         char *now_s = NULL;
 
         crm_notice("%s requested shutdown of its remote connection",
                    lrm_state->node_name);
 
         if (!remote_ra_is_in_maintenance(lrm_state)) {
             now_s = pcmk__ttoa(time(NULL));
             update_attrd(lrm_state->node_name, XML_CIB_ATTR_SHUTDOWN, now_s, NULL, TRUE);
             free(now_s);
 
             remote_proxy_ack_shutdown(lrmd);
 
             crm_warn("Reconnection attempts to %s may result in failures that must be cleared",
                     lrm_state->node_name);
         } else {
             remote_proxy_nack_shutdown(lrmd);
 
             crm_notice("Remote resource for %s is not managed so no ordered shutdown happening",
                     lrm_state->node_name);
         }
         return;
 
     } else if (pcmk__str_eq(op, LRMD_IPC_OP_REQUEST, pcmk__str_casei) && proxy && proxy->is_local) {
         /* This is for the controller, which we are, so don't try
          * to send to ourselves over IPC -- do it directly.
          */
         int flags = 0;
         xmlNode *request = get_message_xml(msg, F_LRMD_IPC_MSG);
 
         CRM_CHECK(request != NULL, return);
         CRM_CHECK(lrm_state->node_name, return);
         crm_xml_add(request, XML_ACL_TAG_ROLE, "pacemaker-remote");
         pcmk__update_acl_user(request, F_LRMD_IPC_USER, lrm_state->node_name);
 
         /* Pacemaker Remote nodes don't know their own names (as known to the
          * cluster). When getting a node info request with no name or ID, add
          * the name, so we don't return info for ourselves instead of the
          * Pacemaker Remote node.
          */
         if (pcmk__str_eq(crm_element_value(request, F_CRM_TASK), CRM_OP_NODE_INFO, pcmk__str_casei)) {
             int node_id = 0;
 
             crm_element_value_int(request, XML_ATTR_ID, &node_id);
             if ((node_id <= 0)
                 && (crm_element_value(request, XML_ATTR_UNAME) == NULL)) {
                 crm_xml_add(request, XML_ATTR_UNAME, lrm_state->node_name);
             }
         }
 
         crmd_proxy_dispatch(session, request);
 
         crm_element_value_int(msg, F_LRMD_IPC_MSG_FLAGS, &flags);
         if (flags & crm_ipc_client_response) {
             int msg_id = 0;
             xmlNode *op_reply = create_xml_node(NULL, "ack");
 
             crm_xml_add(op_reply, "function", __func__);
             crm_xml_add_int(op_reply, "line", __LINE__);
 
             crm_element_value_int(msg, F_LRMD_IPC_MSG_ID, &msg_id);
             remote_proxy_relay_response(proxy, op_reply, msg_id);
 
             free_xml(op_reply);
         }
 
     } else {
         remote_proxy_cb(lrmd, lrm_state->node_name, msg);
     }
 }
 
 
 // \return Standard Pacemaker return code
 int
 controld_connect_remote_executor(lrm_state_t *lrm_state, const char *server,
                                  int port, int timeout_ms)
 {
     int rc = pcmk_rc_ok;
 
     if (lrm_state->conn == NULL) {
         lrmd_t *api = NULL;
 
         rc = lrmd__new(&api, lrm_state->node_name, server, port);
         if (rc != pcmk_rc_ok) {
             crm_warn("Pacemaker Remote connection to %s:%s failed: %s "
                      CRM_XS " rc=%d", server, port, pcmk_rc_str(rc), rc);
 
             return rc;
         }
         lrm_state->conn = api;
         api->cmds->set_callback(api, remote_lrm_op_callback);
         lrmd_internal_set_proxy_callback(api, lrm_state, crmd_remote_proxy_cb);
     }
 
     crm_trace("Initiating remote connection to %s:%d with timeout %dms",
               server, port, timeout_ms);
     rc = ((lrmd_t *) lrm_state->conn)->cmds->connect_async(lrm_state->conn,
                                                            lrm_state->node_name,
                                                            timeout_ms);
     if (rc == pcmk_ok) {
         lrm_state->num_lrm_register_fails = 0;
     } else {
         lrm_state->num_lrm_register_fails++; // Ignored for remote connections
     }
     return pcmk_legacy2rc(rc);
 }
 
 int
 lrm_state_get_metadata(lrm_state_t * lrm_state,
                        const char *class,
                        const char *provider,
                        const char *agent, char **output, enum lrmd_call_options options)
 {
     lrmd_key_value_t *params = NULL;
 
     if (!lrm_state->conn) {
         return -ENOTCONN;
     }
 
     /* Add the node name to the environment, as is done with normal resource
      * action calls. Meta-data calls shouldn't need it, but some agents are
      * written with an ocf_local_nodename call at the beginning regardless of
      * action. Without the environment variable, the agent would try to contact
      * the controller to get the node name -- but the controller would be
      * blocking on the synchronous meta-data call.
      *
      * At this point, we have to assume that agents are unlikely to make other
      * calls that require the controller, such as crm_node --quorum or
      * --cluster-id.
      *
      * @TODO Make meta-data calls asynchronous. (This will be part of a larger
      * project to make meta-data calls via the executor rather than directly.)
      */
     params = lrmd_key_value_add(params, CRM_META "_" XML_LRM_ATTR_TARGET,
                                 lrm_state->node_name);
 
     return ((lrmd_t *) lrm_state->conn)->cmds->get_metadata_params(lrm_state->conn,
             class, provider, agent, output, options, params);
 }
 
 int
 lrm_state_cancel(lrm_state_t *lrm_state, const char *rsc_id, const char *action,
                  guint interval_ms)
 {
     if (!lrm_state->conn) {
         return -ENOTCONN;
     }
 
     /* Figure out a way to make this async?
      * NOTICE: Currently it's synced and directly acknowledged in do_lrm_invoke(). */
     if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
         return remote_ra_cancel(lrm_state, rsc_id, action, interval_ms);
     }
     return ((lrmd_t *) lrm_state->conn)->cmds->cancel(lrm_state->conn, rsc_id,
                                                       action, interval_ms);
 }
 
 lrmd_rsc_info_t *
 lrm_state_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id, enum lrmd_call_options options)
 {
     lrmd_rsc_info_t *rsc = NULL;
 
     if (!lrm_state->conn) {
         return NULL;
     }
     if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
         return remote_ra_get_rsc_info(lrm_state, rsc_id);
     }
 
     rsc = g_hash_table_lookup(lrm_state->rsc_info_cache, rsc_id);
     if (rsc == NULL) {
         /* only contact the lrmd if we don't already have a cached rsc info */
         rsc = ((lrmd_t *) lrm_state->conn)->cmds->get_rsc_info(lrm_state->conn, rsc_id, options);
         if (rsc == NULL) {
 		    return NULL;
         }
         /* cache the result */
         g_hash_table_insert(lrm_state->rsc_info_cache, rsc->id, rsc);
     }
 
     return lrmd_copy_rsc_info(rsc);
 
 }
 
 /*!
  * \internal
  * \brief Initiate a resource agent action
  *
  * \param[in]  lrm_state       Executor state object
  * \param[in]  rsc_id          ID of resource for action
  * \param[in]  action          Action to execute
  * \param[in]  userdata        String to copy and pass to execution callback
  * \param[in]  interval_ms     Action interval (in milliseconds)
  * \param[in]  timeout_ms      Action timeout (in milliseconds)
  * \param[in]  start_delay_ms  Delay (in milliseconds) before initiating action
  * \param[in]  params          Resource parameters
  * \param[out] call_id         Where to store call ID on success
  *
  * \return Standard Pacemaker return code
  * \note This takes ownership of \p params, which should not be used or freed
  *       after calling this function.
  */
 int
 controld_execute_resource_agent(lrm_state_t *lrm_state, const char *rsc_id,
                                 const char *action, const char *userdata,
                                 guint interval_ms, int timeout_ms,
                                 int start_delay_ms, lrmd_key_value_t *params,
                                 int *call_id)
 {
     int rc = pcmk_rc_ok;
 
     if (lrm_state->conn == NULL) {
         lrmd_key_value_freeall(params);
         rc = ENOTCONN;
 
     } else if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
         rc = controld_execute_remote_agent(lrm_state, rsc_id, action,
                                            userdata, interval_ms, timeout_ms,
                                            start_delay_ms, params, call_id);
 
     } else {
         rc = ((lrmd_t *) lrm_state->conn)->cmds->exec(lrm_state->conn, rsc_id,
                                                       action, userdata,
                                                       interval_ms, timeout_ms,
                                                       start_delay_ms,
                                                       lrmd_opt_notify_changes_only,
                                                       params);
         if (rc < 0) {
             rc = pcmk_legacy2rc(rc);
         } else {
             *call_id = rc;
             rc = pcmk_rc_ok;
         }
     }
     return rc;
 }
 
 int
 lrm_state_register_rsc(lrm_state_t * lrm_state,
                        const char *rsc_id,
                        const char *class,
                        const char *provider, const char *agent, enum lrmd_call_options options)
 {
     lrmd_t *conn = (lrmd_t *) lrm_state->conn;
 
     if (conn == NULL) {
         return -ENOTCONN;
     }
 
     if (is_remote_lrmd_ra(agent, provider, NULL)) {
         return lrm_state_find_or_create(rsc_id)? pcmk_ok : -EINVAL;
     }
 
     /* @TODO Implement an asynchronous version of this (currently a blocking
      * call to the lrmd).
      */
     return conn->cmds->register_rsc(lrm_state->conn, rsc_id, class, provider,
                                     agent, options);
 }
 
 int
 lrm_state_unregister_rsc(lrm_state_t * lrm_state,
                          const char *rsc_id, enum lrmd_call_options options)
 {
     if (!lrm_state->conn) {
         return -ENOTCONN;
     }
 
     if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
         lrm_state_destroy(rsc_id);
         return pcmk_ok;
     }
 
     g_hash_table_remove(lrm_state->rsc_info_cache, rsc_id);
 
     /* @TODO Optimize this ... this function is a blocking round trip from
      * client to daemon. The controld_execd_state.c code path that uses this
      * function should always treat it as an async operation. The executor API
      * should make an async version available.
      */
     return ((lrmd_t *) lrm_state->conn)->cmds->unregister_rsc(lrm_state->conn, rsc_id, options);
 }
 
 /*
  * Functions for sending alerts via local executor connection
  */
 
 static GList *crmd_alert_list = NULL;
 
 void
 crmd_unpack_alerts(xmlNode *alerts)
 {
     pe_free_alert_list(crmd_alert_list);
     crmd_alert_list = pe_unpack_alerts(alerts);
 }
 
 void
 crmd_alert_node_event(crm_node_t *node)
 {
     lrm_state_t *lrm_state;
 
     if (crmd_alert_list == NULL) {
         return;
     }
 
     lrm_state = lrm_state_find(fsa_our_uname);
     if (lrm_state == NULL) {
         return;
     }
 
     lrmd_send_node_alert((lrmd_t *) lrm_state->conn, crmd_alert_list,
                          node->uname, node->id, node->state);
 }
 
 void
 crmd_alert_fencing_op(stonith_event_t * e)
 {
     char *desc;
     lrm_state_t *lrm_state;
 
     if (crmd_alert_list == NULL) {
         return;
     }
 
     lrm_state = lrm_state_find(fsa_our_uname);
     if (lrm_state == NULL) {
         return;
     }
 
     desc = crm_strdup_printf("Operation %s of %s by %s for %s@%s: %s (ref=%s)",
                              e->action, e->target,
                              (e->executioner? e->executioner : "<no-one>"),
                              e->client_origin, e->origin,
                              pcmk_strerror(e->result), e->id);
 
     lrmd_send_fencing_alert((lrmd_t *) lrm_state->conn, crmd_alert_list,
                             e->target, e->operation, desc, e->result);
     free(desc);
 }
 
 void
 crmd_alert_resource_op(const char *node, lrmd_event_data_t * op)
 {
     lrm_state_t *lrm_state;
 
     if (crmd_alert_list == NULL) {
         return;
     }
 
     lrm_state = lrm_state_find(fsa_our_uname);
     if (lrm_state == NULL) {
         return;
     }
 
     lrmd_send_resource_alert((lrmd_t *) lrm_state->conn, crmd_alert_list, node,
                              op);
 }
diff --git a/daemons/controld/controld_remote_ra.c b/daemons/controld/controld_remote_ra.c
index e36df711c4..ffddd2f3cf 100644
--- a/daemons/controld/controld_remote_ra.c
+++ b/daemons/controld/controld_remote_ra.c
@@ -1,1337 +1,1340 @@
 /*
  * Copyright 2013-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml_internal.h>
 #include <crm/lrmd.h>
 #include <crm/lrmd_internal.h>
 #include <crm/services.h>
 
 #include <pacemaker-controld.h>
 
 #define REMOTE_LRMD_RA "remote"
 
 /* The max start timeout before cmd retry */
 #define MAX_START_TIMEOUT_MS 10000
 
 typedef struct remote_ra_cmd_s {
     /*! the local node the cmd is issued from */
     char *owner;
     /*! the remote node the cmd is executed on */
     char *rsc_id;
     /*! the action to execute */
     char *action;
     /*! some string the client wants us to give it back */
     char *userdata;
     /*! start delay in ms */
     int start_delay;
     /*! timer id used for start delay. */
     int delay_id;
     /*! timeout in ms for cmd */
     int timeout;
     int remaining_timeout;
     /*! recurring interval in ms */
     guint interval_ms;
     /*! interval timer id */
     int interval_id;
     int reported_success;
     int monitor_timeout_id;
     int takeover_timeout_id;
     /*! action parameters */
     lrmd_key_value_t *params;
     pcmk__action_result_t result;
     int call_id;
     time_t start_time;
     gboolean cancel;
 } remote_ra_cmd_t;
 
 enum remote_migration_status {
     expect_takeover = 1,
     takeover_complete,
 };
 
 typedef struct remote_ra_data_s {
     crm_trigger_t *work;
     remote_ra_cmd_t *cur_cmd;
     GList *cmds;
     GList *recurring_cmds;
 
     enum remote_migration_status migrate_status;
 
     gboolean active;
 
     /* Maintenance mode is difficult to determine from the controller's context,
      * so we have it signalled back with the transition from the scheduler.
      */
     gboolean is_maintenance;
 
     /* Similar for whether we are controlling a guest node or remote node.
      * Fortunately there is a meta-attribute in the transition already and
      * as the situation doesn't change over time we can use the
      * resource start for noting down the information for later use when
      * the attributes aren't at hand.
      */
     gboolean controlling_guest;
 } remote_ra_data_t;
 
 static int handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeout_ms);
 static void handle_remote_ra_stop(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd);
 static GList *fail_all_monitor_cmds(GList * list);
 
 static void
 free_cmd(gpointer user_data)
 {
     remote_ra_cmd_t *cmd = user_data;
 
     if (!cmd) {
         return;
     }
     if (cmd->delay_id) {
         g_source_remove(cmd->delay_id);
     }
     if (cmd->interval_id) {
         g_source_remove(cmd->interval_id);
     }
     if (cmd->monitor_timeout_id) {
         g_source_remove(cmd->monitor_timeout_id);
     }
     if (cmd->takeover_timeout_id) {
         g_source_remove(cmd->takeover_timeout_id);
     }
     free(cmd->owner);
     free(cmd->rsc_id);
     free(cmd->action);
     free(cmd->userdata);
     pcmk__reset_result(&(cmd->result));
     lrmd_key_value_freeall(cmd->params);
     free(cmd);
 }
 
 static int
 generate_callid(void)
 {
     static int remote_ra_callid = 0;
 
     remote_ra_callid++;
     if (remote_ra_callid <= 0) {
         remote_ra_callid = 1;
     }
 
     return remote_ra_callid;
 }
 
 static gboolean
 recurring_helper(gpointer data)
 {
     remote_ra_cmd_t *cmd = data;
     lrm_state_t *connection_rsc = NULL;
 
     cmd->interval_id = 0;
     connection_rsc = lrm_state_find(cmd->rsc_id);
     if (connection_rsc && connection_rsc->remote_ra_data) {
         remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;
 
         ra_data->recurring_cmds = g_list_remove(ra_data->recurring_cmds, cmd);
 
         ra_data->cmds = g_list_append(ra_data->cmds, cmd);
         mainloop_set_trigger(ra_data->work);
     }
     return FALSE;
 }
 
 static gboolean
 start_delay_helper(gpointer data)
 {
     remote_ra_cmd_t *cmd = data;
     lrm_state_t *connection_rsc = NULL;
 
     cmd->delay_id = 0;
     connection_rsc = lrm_state_find(cmd->rsc_id);
     if (connection_rsc && connection_rsc->remote_ra_data) {
         remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;
 
         mainloop_set_trigger(ra_data->work);
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Handle cluster communication related to pacemaker_remote node joining
  *
  * \param[in] node_name  Name of newly integrated pacemaker_remote node
  */
 static void
 remote_node_up(const char *node_name)
 {
     int call_opt, call_id = 0;
     xmlNode *update, *state;
     crm_node_t *node;
     enum controld_section_e section = controld_section_all;
 
     CRM_CHECK(node_name != NULL, return);
     crm_info("Announcing Pacemaker Remote node %s", node_name);
 
     /* Clear node's entire state (resource history and transient attributes)
      * other than shutdown locks. The transient attributes should and normally
      * will be cleared when the node leaves, but since remote node state has a
      * number of corner cases, clear them here as well, to be sure.
      */
     call_opt = crmd_cib_smart_opt();
     if (controld_shutdown_lock_enabled) {
         section = controld_section_all_unlocked;
     }
     /* Purge node from attrd's memory */
     update_attrd_remote_node_removed(node_name, NULL);
 
     controld_delete_node_state(node_name, section, call_opt);
 
     /* Clear node's probed attribute */
     update_attrd(node_name, CRM_OP_PROBED, NULL, NULL, TRUE);
 
     /* Ensure node is in the remote peer cache with member status */
     node = crm_remote_peer_get(node_name);
     CRM_CHECK(node != NULL, return);
     pcmk__update_peer_state(__func__, node, CRM_NODE_MEMBER, 0);
 
     /* pacemaker_remote nodes don't participate in the membership layer,
      * so cluster nodes don't automatically get notified when they come and go.
      * We send a cluster message to the DC, and update the CIB node state entry,
      * so the DC will get it sooner (via message) or later (via CIB refresh),
      * and any other interested parties can query the CIB.
      */
     send_remote_state_message(node_name, TRUE);
 
     update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
     state = create_node_state_update(node, node_update_cluster, update,
                                      __func__);
 
     /* Clear the XML_NODE_IS_FENCED flag in the node state. If the node ever
      * needs to be fenced, this flag will allow various actions to determine
      * whether the fencing has happened yet.
      */
     crm_xml_add(state, XML_NODE_IS_FENCED, "0");
 
     /* TODO: If the remote connection drops, and this (async) CIB update either
      * failed or has not yet completed, later actions could mistakenly think the
      * node has already been fenced (if the XML_NODE_IS_FENCED attribute was
      * previously set, because it won't have been cleared). This could prevent
      * actual fencing or allow recurring monitor failures to be cleared too
      * soon. Ideally, we wouldn't rely on the CIB for the fenced status.
      */
     fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
     if (call_id < 0) {
         crm_perror(LOG_WARNING, "%s CIB node state setup", node_name);
     }
     free_xml(update);
 }
 
 enum down_opts {
     DOWN_KEEP_LRM,
     DOWN_ERASE_LRM
 };
 
 /*!
  * \internal
  * \brief Handle cluster communication related to pacemaker_remote node leaving
  *
  * \param[in] node_name  Name of lost node
  * \param[in] opts       Whether to keep or erase LRM history
  */
 static void
 remote_node_down(const char *node_name, const enum down_opts opts)
 {
     xmlNode *update;
     int call_id = 0;
     int call_opt = crmd_cib_smart_opt();
     crm_node_t *node;
 
     /* Purge node from attrd's memory */
     update_attrd_remote_node_removed(node_name, NULL);
 
     /* Normally, only node attributes should be erased, and the resource history
      * should be kept until the node comes back up. However, after a successful
      * fence, we want to clear the history as well, so we don't think resources
      * are still running on the node.
      */
     if (opts == DOWN_ERASE_LRM) {
         controld_delete_node_state(node_name, controld_section_all, call_opt);
     } else {
         controld_delete_node_state(node_name, controld_section_attrs, call_opt);
     }
 
     /* Ensure node is in the remote peer cache with lost state */
     node = crm_remote_peer_get(node_name);
     CRM_CHECK(node != NULL, return);
     pcmk__update_peer_state(__func__, node, CRM_NODE_LOST, 0);
 
     /* Notify DC */
     send_remote_state_message(node_name, FALSE);
 
     /* Update CIB node state */
     update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
     create_node_state_update(node, node_update_cluster, update, __func__);
     fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
     if (call_id < 0) {
         crm_perror(LOG_ERR, "%s CIB node state update", node_name);
     }
     free_xml(update);
 }
 
 /*!
  * \internal
  * \brief Handle effects of a remote RA command on node state
  *
  * \param[in] cmd  Completed remote RA command
  */
 static void
 check_remote_node_state(remote_ra_cmd_t *cmd)
 {
     /* Only successful actions can change node state */
     if (cmd->result.exit_status != PCMK_OCF_OK) {
         return;
     }
 
     if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)) {
         remote_node_up(cmd->rsc_id);
 
     } else if (pcmk__str_eq(cmd->action, "migrate_from", pcmk__str_casei)) {
         /* After a successful migration, we don't need to do remote_node_up()
          * because the DC already knows the node is up, and we don't want to
          * clear LRM history etc. We do need to add the remote node to this
          * host's remote peer cache, because (unless it happens to be DC)
          * it hasn't been tracking the remote node, and other code relies on
          * the cache to distinguish remote nodes from unseen cluster nodes.
          */
         crm_node_t *node = crm_remote_peer_get(cmd->rsc_id);
 
         CRM_CHECK(node != NULL, return);
         pcmk__update_peer_state(__func__, node, CRM_NODE_MEMBER, 0);
 
     } else if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
         lrm_state_t *lrm_state = lrm_state_find(cmd->rsc_id);
         remote_ra_data_t *ra_data = lrm_state? lrm_state->remote_ra_data : NULL;
 
         if (ra_data) {
             if (ra_data->migrate_status != takeover_complete) {
                 /* Stop means down if we didn't successfully migrate elsewhere */
                 remote_node_down(cmd->rsc_id, DOWN_KEEP_LRM);
             } else if (AM_I_DC == FALSE) {
                 /* Only the connection host and DC track node state,
                  * so if the connection migrated elsewhere and we aren't DC,
                  * un-cache the node, so we don't have stale info
                  */
                 crm_remote_peer_cache_remove(cmd->rsc_id);
             }
         }
     }
 
     /* We don't do anything for successful monitors, which is correct for
      * routine recurring monitors, and for monitors on nodes where the
      * connection isn't supposed to be (the cluster will stop the connection in
      * that case). However, if the initial probe finds the connection already
      * active on the node where we want it, we probably should do
      * remote_node_up(). Unfortunately, we can't distinguish that case here.
      * Given that connections have to be initiated by the cluster, the chance of
      * that should be close to zero.
      */
 }
 
 static void
 report_remote_ra_result(remote_ra_cmd_t * cmd)
 {
     lrmd_event_data_t op = { 0, };
 
     check_remote_node_state(cmd);
 
     op.type = lrmd_event_exec_complete;
     op.rsc_id = cmd->rsc_id;
     op.op_type = cmd->action;
     op.user_data = cmd->userdata;
     op.timeout = cmd->timeout;
     op.interval_ms = cmd->interval_ms;
     op.t_run = (unsigned int) cmd->start_time;
     op.t_rcchange = (unsigned int) cmd->start_time;
 
     lrmd__set_result(&op, cmd->result.exit_status, cmd->result.exec_status,
                      cmd->result.exit_reason);
 
     if (cmd->reported_success && (cmd->result.exit_status != PCMK_OCF_OK)) {
         op.t_rcchange = (unsigned int) time(NULL);
         /* This edge case will likely never ever occur, but if it does the
          * result is that a failure will not be processed correctly. This is only
          * remotely possible because we are able to detect a connection resource's tcp
          * connection has failed at any moment after start has completed. The actual
          * recurring operation is just a connectivity ping.
          *
          * basically, we are not guaranteed that the first successful monitor op and
          * a subsequent failed monitor op will not occur in the same timestamp. We have to
          * make it look like the operations occurred at separate times though. */
         if (op.t_rcchange == op.t_run) {
             op.t_rcchange++;
         }
     }
 
     if (cmd->params) {
         lrmd_key_value_t *tmp;
 
         op.params = pcmk__strkey_table(free, free);
         for (tmp = cmd->params; tmp; tmp = tmp->next) {
             g_hash_table_insert(op.params, strdup(tmp->key), strdup(tmp->value));
         }
 
     }
     op.call_id = cmd->call_id;
     op.remote_nodename = cmd->owner;
 
     lrm_op_callback(&op);
 
     if (op.params) {
         g_hash_table_destroy(op.params);
     }
     lrmd__reset_result(&op);
 }
 
 static void
 update_remaining_timeout(remote_ra_cmd_t * cmd)
 {
     cmd->remaining_timeout = ((cmd->timeout / 1000) - (time(NULL) - cmd->start_time)) * 1000;
 }
 
 static gboolean
 retry_start_cmd_cb(gpointer data)
 {
     lrm_state_t *lrm_state = data;
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
     remote_ra_cmd_t *cmd = NULL;
     int rc = pcmk_rc_error;
 
     if (!ra_data || !ra_data->cur_cmd) {
         return FALSE;
     }
     cmd = ra_data->cur_cmd;
     if (!pcmk__strcase_any_of(cmd->action, "start", "migrate_from", NULL)) {
         return FALSE;
     }
     update_remaining_timeout(cmd);
 
     if (cmd->remaining_timeout > 0) {
         rc = handle_remote_ra_start(lrm_state, cmd, cmd->remaining_timeout);
     }
 
     if (rc != pcmk_rc_ok) {
         report_remote_ra_result(cmd);
 
         if (ra_data->cmds) {
             mainloop_set_trigger(ra_data->work);
         }
         ra_data->cur_cmd = NULL;
         free_cmd(cmd);
     } else {
         /* wait for connection event */
     }
 
     return FALSE;
 }
 
 
 static gboolean
 connection_takeover_timeout_cb(gpointer data)
 {
     lrm_state_t *lrm_state = NULL;
     remote_ra_cmd_t *cmd = data;
 
     crm_info("takeover event timed out for node %s", cmd->rsc_id);
     cmd->takeover_timeout_id = 0;
 
     lrm_state = lrm_state_find(cmd->rsc_id);
 
     handle_remote_ra_stop(lrm_state, cmd);
     free_cmd(cmd);
 
     return FALSE;
 }
 
 static gboolean
 monitor_timeout_cb(gpointer data)
 {
     lrm_state_t *lrm_state = NULL;
     remote_ra_cmd_t *cmd = data;
 
     lrm_state = lrm_state_find(cmd->rsc_id);
 
     crm_info("Timed out waiting for remote poke response from %s%s",
              cmd->rsc_id, (lrm_state? "" : " (no LRM state)"));
     cmd->monitor_timeout_id = 0;
     pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_TIMEOUT,
-                     NULL);
+                     "Remote executor did not respond");
 
     if (lrm_state && lrm_state->remote_ra_data) {
         remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
         if (ra_data->cur_cmd == cmd) {
             ra_data->cur_cmd = NULL;
         }
         if (ra_data->cmds) {
             mainloop_set_trigger(ra_data->work);
         }
     }
 
     report_remote_ra_result(cmd);
     free_cmd(cmd);
 
     if(lrm_state) {
         lrm_state_disconnect(lrm_state);
     }
     return FALSE;
 }
 
 static void
 synthesize_lrmd_success(lrm_state_t *lrm_state, const char *rsc_id, const char *op_type)
 {
     lrmd_event_data_t op = { 0, };
 
     if (lrm_state == NULL) {
         /* if lrm_state not given assume local */
         lrm_state = lrm_state_find(fsa_our_uname);
     }
     CRM_ASSERT(lrm_state != NULL);
 
     op.type = lrmd_event_exec_complete;
     op.rsc_id = rsc_id;
     op.op_type = op_type;
     op.t_run = (unsigned int) time(NULL);
     op.t_rcchange = op.t_run;
     op.call_id = generate_callid();
     lrmd__set_result(&op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
     process_lrm_event(lrm_state, &op, NULL, NULL);
 }
 
 void
 remote_lrm_op_callback(lrmd_event_data_t * op)
 {
     gboolean cmd_handled = FALSE;
     lrm_state_t *lrm_state = NULL;
     remote_ra_data_t *ra_data = NULL;
     remote_ra_cmd_t *cmd = NULL;
 
     crm_debug("Processing '%s%s%s' event on remote connection to %s: %s "
               "(%d) status=%s (%d)",
               (op->op_type? op->op_type : ""), (op->op_type? " " : ""),
               lrmd_event_type2str(op->type), op->remote_nodename,
               services_ocf_exitcode_str(op->rc), op->rc,
               pcmk_exec_status_str(op->op_status), op->op_status);
 
     lrm_state = lrm_state_find(op->remote_nodename);
     if (!lrm_state || !lrm_state->remote_ra_data) {
         crm_debug("No state information found for remote connection event");
         return;
     }
     ra_data = lrm_state->remote_ra_data;
 
     if (op->type == lrmd_event_new_client) {
         // Another client has connected to the remote daemon
 
         if (ra_data->migrate_status == expect_takeover) {
             // Great, we knew this was coming
             ra_data->migrate_status = takeover_complete;
 
         } else {
             crm_err("Disconnecting from Pacemaker Remote node %s due to "
                     "unexpected client takeover", op->remote_nodename);
             /* In this case, lrmd_tls_connection_destroy() will be called under the control of mainloop. */
             /* Do not free lrm_state->conn yet. */
             /* It'll be freed in the following stop action. */
             lrm_state_disconnect_only(lrm_state);
         }
         return;
     }
 
     /* filter all EXEC events up */
     if (op->type == lrmd_event_exec_complete) {
         if (ra_data->migrate_status == takeover_complete) {
             crm_debug("ignoring event, this connection is taken over by another node");
         } else {
             lrm_op_callback(op);
         }
         return;
     }
 
     if ((op->type == lrmd_event_disconnect) && (ra_data->cur_cmd == NULL)) {
 
         if (ra_data->active == FALSE) {
             crm_debug("Disconnection from Pacemaker Remote node %s complete",
                       lrm_state->node_name);
 
         } else if (!remote_ra_is_in_maintenance(lrm_state)) {
             crm_err("Lost connection to Pacemaker Remote node %s",
                     lrm_state->node_name);
             ra_data->recurring_cmds = fail_all_monitor_cmds(ra_data->recurring_cmds);
             ra_data->cmds = fail_all_monitor_cmds(ra_data->cmds);
 
         } else {
             crm_notice("Unmanaged Pacemaker Remote node %s disconnected",
                        lrm_state->node_name);
             /* Do roughly what a 'stop' on the remote-resource would do */
             handle_remote_ra_stop(lrm_state, NULL);
             remote_node_down(lrm_state->node_name, DOWN_KEEP_LRM);
             /* now fake the reply of a successful 'stop' */
             synthesize_lrmd_success(NULL, lrm_state->node_name, "stop");
         }
         return;
     }
 
     if (!ra_data->cur_cmd) {
         crm_debug("no event to match");
         return;
     }
 
     cmd = ra_data->cur_cmd;
 
     /* Start actions and migrate from actions complete after connection
      * comes back to us. */
     if (op->type == lrmd_event_connect && pcmk__strcase_any_of(cmd->action, "start",
                                                                "migrate_from", NULL)) {
         if (op->connection_rc < 0) {
             update_remaining_timeout(cmd);
 
             if (op->connection_rc == -ENOKEY) {
                 // Hard error, don't retry
                 pcmk__set_result(&(cmd->result), PCMK_OCF_INVALID_PARAM,
                                  PCMK_EXEC_ERROR,
                                  "Authentication key not readable");
 
             } else if (cmd->remaining_timeout > 3000) {
                 crm_trace("rescheduling start, remaining timeout %d", cmd->remaining_timeout);
                 g_timeout_add(1000, retry_start_cmd_cb, lrm_state);
                 return;
 
             } else {
                 crm_trace("can't reschedule start, remaining timeout too small %d",
                           cmd->remaining_timeout);
                 pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
-                                 PCMK_EXEC_TIMEOUT, NULL);
+                                 PCMK_EXEC_TIMEOUT,
+                                 pcmk_strerror(op->connection_rc));
             }
 
         } else {
             lrm_state_reset_tables(lrm_state, TRUE);
             pcmk__set_result(&(cmd->result), PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
             ra_data->active = TRUE;
         }
 
         crm_debug("Remote connection event matched %s action", cmd->action);
         report_remote_ra_result(cmd);
         cmd_handled = TRUE;
 
     } else if (op->type == lrmd_event_poke && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
 
         if (cmd->monitor_timeout_id) {
             g_source_remove(cmd->monitor_timeout_id);
             cmd->monitor_timeout_id = 0;
         }
 
         /* Only report success the first time, after that only worry about failures.
          * For this function, if we get the poke pack, it is always a success. Pokes
          * only fail if the send fails, or the response times out. */
         if (!cmd->reported_success) {
             pcmk__set_result(&(cmd->result), PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
             report_remote_ra_result(cmd);
             cmd->reported_success = 1;
         }
 
         crm_debug("Remote poke event matched %s action", cmd->action);
 
         /* success, keep rescheduling if interval is present. */
         if (cmd->interval_ms && (cmd->cancel == FALSE)) {
             ra_data->recurring_cmds = g_list_append(ra_data->recurring_cmds, cmd);
             cmd->interval_id = g_timeout_add(cmd->interval_ms,
                                              recurring_helper, cmd);
             cmd = NULL;         /* prevent free */
         }
         cmd_handled = TRUE;
 
     } else if (op->type == lrmd_event_disconnect && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
         if (ra_data->active == TRUE && (cmd->cancel == FALSE)) {
             pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
-                             PCMK_EXEC_ERROR, NULL);
+                             PCMK_EXEC_ERROR,
+                             "Remote connection unexpectedly dropped "
+                             "during monitor");
             report_remote_ra_result(cmd);
             crm_err("Remote connection to %s unexpectedly dropped during monitor",
                     lrm_state->node_name);
         }
         cmd_handled = TRUE;
 
     } else if (op->type == lrmd_event_new_client && pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
 
         handle_remote_ra_stop(lrm_state, cmd);
         cmd_handled = TRUE;
 
     } else {
         crm_debug("Event did not match %s action", ra_data->cur_cmd->action);
     }
 
     if (cmd_handled) {
         ra_data->cur_cmd = NULL;
         if (ra_data->cmds) {
             mainloop_set_trigger(ra_data->work);
         }
         free_cmd(cmd);
     }
 }
 
 static void
 handle_remote_ra_stop(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd)
 {
     remote_ra_data_t *ra_data = NULL;
 
     CRM_ASSERT(lrm_state);
     ra_data = lrm_state->remote_ra_data;
 
     if (ra_data->migrate_status != takeover_complete) {
         /* delete pending ops when ever the remote connection is intentionally stopped */
         g_hash_table_remove_all(lrm_state->pending_ops);
     } else {
         /* we no longer hold the history if this connection has been migrated,
          * however, we keep metadata cache for future use */
         lrm_state_reset_tables(lrm_state, FALSE);
     }
 
     ra_data->active = FALSE;
     lrm_state_disconnect(lrm_state);
 
     if (ra_data->cmds) {
         g_list_free_full(ra_data->cmds, free_cmd);
     }
     if (ra_data->recurring_cmds) {
         g_list_free_full(ra_data->recurring_cmds, free_cmd);
     }
     ra_data->cmds = NULL;
     ra_data->recurring_cmds = NULL;
     ra_data->cur_cmd = NULL;
 
     if (cmd) {
         pcmk__set_result(&(cmd->result), PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
         report_remote_ra_result(cmd);
     }
 }
 
 // \return Standard Pacemaker return code
 static int
 handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeout_ms)
 {
     const char *server = NULL;
     lrmd_key_value_t *tmp = NULL;
     int port = 0;
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
     int timeout_used = timeout_ms > MAX_START_TIMEOUT_MS ? MAX_START_TIMEOUT_MS : timeout_ms;
     int rc = pcmk_rc_ok;
 
     for (tmp = cmd->params; tmp; tmp = tmp->next) {
         if (pcmk__strcase_any_of(tmp->key, XML_RSC_ATTR_REMOTE_RA_ADDR,
                                  XML_RSC_ATTR_REMOTE_RA_SERVER, NULL)) {
             server = tmp->value;
         } else if (pcmk__str_eq(tmp->key, XML_RSC_ATTR_REMOTE_RA_PORT, pcmk__str_casei)) {
             port = atoi(tmp->value);
         } else if (pcmk__str_eq(tmp->key, CRM_META "_" XML_RSC_ATTR_CONTAINER, pcmk__str_casei)) {
             ra_data->controlling_guest = TRUE;
         }
     }
 
     rc = controld_connect_remote_executor(lrm_state, server, port,
                                           timeout_used);
     if (rc != pcmk_rc_ok) {
         pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
                          PCMK_EXEC_ERROR, pcmk_rc_str(rc));
     }
     return rc;
 }
 
 static gboolean
 handle_remote_ra_exec(gpointer user_data)
 {
     int rc = 0;
     lrm_state_t *lrm_state = user_data;
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
     remote_ra_cmd_t *cmd;
     GList *first = NULL;
 
     if (ra_data->cur_cmd) {
         /* still waiting on previous cmd */
         return TRUE;
     }
 
     while (ra_data->cmds) {
         first = ra_data->cmds;
         cmd = first->data;
         if (cmd->delay_id) {
             /* still waiting for start delay timer to trip */
             return TRUE;
         }
 
         ra_data->cmds = g_list_remove_link(ra_data->cmds, first);
         g_list_free_1(first);
 
         if (!strcmp(cmd->action, "start") || !strcmp(cmd->action, "migrate_from")) {
             ra_data->migrate_status = 0;
             if (handle_remote_ra_start(lrm_state, cmd,
                                        cmd->timeout) == pcmk_rc_ok) {
                 /* take care of this later when we get async connection result */
                 crm_debug("Initiated async remote connection, %s action will complete after connect event",
                           cmd->action);
                 ra_data->cur_cmd = cmd;
                 return TRUE;
             }
             report_remote_ra_result(cmd);
 
         } else if (!strcmp(cmd->action, "monitor")) {
 
             if (lrm_state_is_connected(lrm_state) == TRUE) {
                 rc = lrm_state_poke_connection(lrm_state);
                 if (rc < 0) {
                     pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
-                                     PCMK_EXEC_ERROR, NULL);
+                                     PCMK_EXEC_ERROR, pcmk_strerror(rc));
                 }
             } else {
                 rc = -1;
                 pcmk__set_result(&(cmd->result), PCMK_OCF_NOT_RUNNING,
-                                 PCMK_EXEC_DONE, NULL);
+                                 PCMK_EXEC_DONE, "Remote connection inactive");
             }
 
             if (rc == 0) {
                 crm_debug("Poked Pacemaker Remote at node %s, waiting for async response",
                           cmd->rsc_id);
                 ra_data->cur_cmd = cmd;
                 cmd->monitor_timeout_id = g_timeout_add(cmd->timeout, monitor_timeout_cb, cmd);
                 return TRUE;
             }
             report_remote_ra_result(cmd);
 
         } else if (!strcmp(cmd->action, "stop")) {
 
             if (ra_data->migrate_status == expect_takeover) {
                 /* briefly wait on stop for the takeover event to occur. If the
                  * takeover event does not occur during the wait period, that's fine.
                  * It just means that the remote-node's lrm_status section is going to get
                  * cleared which will require all the resources running in the remote-node
                  * to be explicitly re-detected via probe actions.  If the takeover does occur
                  * successfully, then we can leave the status section intact. */
                 cmd->takeover_timeout_id = g_timeout_add((cmd->timeout/2), connection_takeover_timeout_cb, cmd);
                 ra_data->cur_cmd = cmd;
                 return TRUE;
             }
 
             handle_remote_ra_stop(lrm_state, cmd);
 
         } else if (!strcmp(cmd->action, "migrate_to")) {
             ra_data->migrate_status = expect_takeover;
             pcmk__set_result(&(cmd->result), PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
             report_remote_ra_result(cmd);
         } else if (pcmk__str_any_of(cmd->action, CRMD_ACTION_RELOAD,
                                     CRMD_ACTION_RELOAD_AGENT, NULL))  {
             /* Currently the only reloadable parameter is reconnect_interval,
              * which is only used by the scheduler via the CIB, so reloads are a
              * no-op.
              *
              * @COMPAT DC <2.1.0: We only need to check for "reload" in case
              * we're in a rolling upgrade with a DC scheduling "reload" instead
              * of "reload-agent". An OCF 1.1 "reload" would be a no-op anyway,
              * so this would work for that purpose as well.
              */
             pcmk__set_result(&(cmd->result), PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
             report_remote_ra_result(cmd);
         }
 
         free_cmd(cmd);
     }
 
     return TRUE;
 }
 
 static void
 remote_ra_data_init(lrm_state_t * lrm_state)
 {
     remote_ra_data_t *ra_data = NULL;
 
     if (lrm_state->remote_ra_data) {
         return;
     }
 
     ra_data = calloc(1, sizeof(remote_ra_data_t));
     ra_data->work = mainloop_add_trigger(G_PRIORITY_HIGH, handle_remote_ra_exec, lrm_state);
     lrm_state->remote_ra_data = ra_data;
 }
 
 void
 remote_ra_cleanup(lrm_state_t * lrm_state)
 {
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
     if (!ra_data) {
         return;
     }
 
     if (ra_data->cmds) {
         g_list_free_full(ra_data->cmds, free_cmd);
     }
 
     if (ra_data->recurring_cmds) {
         g_list_free_full(ra_data->recurring_cmds, free_cmd);
     }
     mainloop_destroy_trigger(ra_data->work);
     free(ra_data);
     lrm_state->remote_ra_data = NULL;
 }
 
 gboolean
 is_remote_lrmd_ra(const char *agent, const char *provider, const char *id)
 {
     if (agent && provider && !strcmp(agent, REMOTE_LRMD_RA) && !strcmp(provider, "pacemaker")) {
         return TRUE;
     }
     if (id && lrm_state_find(id) && !pcmk__str_eq(id, fsa_our_uname, pcmk__str_casei)) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 lrmd_rsc_info_t *
 remote_ra_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id)
 {
     lrmd_rsc_info_t *info = NULL;
 
     if ((lrm_state_find(rsc_id))) {
         info = calloc(1, sizeof(lrmd_rsc_info_t));
 
         info->id = strdup(rsc_id);
         info->type = strdup(REMOTE_LRMD_RA);
         info->standard = strdup(PCMK_RESOURCE_CLASS_OCF);
         info->provider = strdup("pacemaker");
     }
 
     return info;
 }
 
 static gboolean
 is_remote_ra_supported_action(const char *action)
 {
     return pcmk__str_any_of(action,
                             CRMD_ACTION_START,
                             CRMD_ACTION_STOP,
                             CRMD_ACTION_STATUS,
                             CRMD_ACTION_MIGRATE,
                             CRMD_ACTION_MIGRATED,
                             CRMD_ACTION_RELOAD_AGENT,
                             CRMD_ACTION_RELOAD,
                             NULL);
 }
 
 static GList *
 fail_all_monitor_cmds(GList * list)
 {
     GList *rm_list = NULL;
     remote_ra_cmd_t *cmd = NULL;
     GList *gIter = NULL;
 
     for (gIter = list; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
         if ((cmd->interval_ms > 0) && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
             rm_list = g_list_append(rm_list, cmd);
         }
     }
 
     for (gIter = rm_list; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
 
         pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
-                         PCMK_EXEC_ERROR, NULL);
+                         PCMK_EXEC_ERROR, "Lost connection to remote executor");
         crm_trace("Pre-emptively failing %s %s (interval=%u, %s)",
                   cmd->action, cmd->rsc_id, cmd->interval_ms, cmd->userdata);
         report_remote_ra_result(cmd);
 
         list = g_list_remove(list, cmd);
         free_cmd(cmd);
     }
 
     /* frees only the list data, not the cmds */
     g_list_free(rm_list);
     return list;
 }
 
 static GList *
 remove_cmd(GList * list, const char *action, guint interval_ms)
 {
     remote_ra_cmd_t *cmd = NULL;
     GList *gIter = NULL;
 
     for (gIter = list; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
         if ((cmd->interval_ms == interval_ms)
             && pcmk__str_eq(cmd->action, action, pcmk__str_casei)) {
             break;
         }
         cmd = NULL;
     }
     if (cmd) {
         list = g_list_remove(list, cmd);
         free_cmd(cmd);
     }
     return list;
 }
 
 int
 remote_ra_cancel(lrm_state_t *lrm_state, const char *rsc_id,
                  const char *action, guint interval_ms)
 {
     lrm_state_t *connection_rsc = NULL;
     remote_ra_data_t *ra_data = NULL;
 
     connection_rsc = lrm_state_find(rsc_id);
     if (!connection_rsc || !connection_rsc->remote_ra_data) {
         return -EINVAL;
     }
 
     ra_data = connection_rsc->remote_ra_data;
     ra_data->cmds = remove_cmd(ra_data->cmds, action, interval_ms);
     ra_data->recurring_cmds = remove_cmd(ra_data->recurring_cmds, action,
                                          interval_ms);
     if (ra_data->cur_cmd &&
         (ra_data->cur_cmd->interval_ms == interval_ms) &&
         (pcmk__str_eq(ra_data->cur_cmd->action, action, pcmk__str_casei))) {
 
         ra_data->cur_cmd->cancel = TRUE;
     }
 
     return 0;
 }
 
 static remote_ra_cmd_t *
 handle_dup_monitor(remote_ra_data_t *ra_data, guint interval_ms,
                    const char *userdata)
 {
     GList *gIter = NULL;
     remote_ra_cmd_t *cmd = NULL;
 
     /* there are 3 places a potential duplicate monitor operation
      * could exist.
      * 1. recurring_cmds list. where the op is waiting for its next interval
      * 2. cmds list, where the op is queued to get executed immediately
      * 3. cur_cmd, which means the monitor op is in flight right now.
      */
     if (interval_ms == 0) {
         return NULL;
     }
 
     if (ra_data->cur_cmd &&
         ra_data->cur_cmd->cancel == FALSE &&
         (ra_data->cur_cmd->interval_ms == interval_ms) &&
         pcmk__str_eq(ra_data->cur_cmd->action, "monitor", pcmk__str_casei)) {
 
         cmd = ra_data->cur_cmd;
         goto handle_dup;
     }
 
     for (gIter = ra_data->recurring_cmds; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
         if ((cmd->interval_ms == interval_ms)
             && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
             goto handle_dup;
         }
     }
 
     for (gIter = ra_data->cmds; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
         if ((cmd->interval_ms == interval_ms)
             && pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
             goto handle_dup;
         }
     }
 
     return NULL;
 
 handle_dup:
 
     crm_trace("merging duplicate monitor cmd " PCMK__OP_FMT,
               cmd->rsc_id, "monitor", interval_ms);
 
     /* update the userdata */
     if (userdata) {
        free(cmd->userdata);
        cmd->userdata = strdup(userdata);
     }
 
     /* if we've already reported success, generate a new call id */
     if (cmd->reported_success) {
         cmd->start_time = time(NULL);
         cmd->call_id = generate_callid();
         cmd->reported_success = 0;
     }
 
     /* if we have an interval_id set, that means we are in the process of
      * waiting for this cmd's next interval. instead of waiting, cancel
      * the timer and execute the action immediately */
     if (cmd->interval_id) {
         g_source_remove(cmd->interval_id);
         cmd->interval_id = 0;
         recurring_helper(cmd);
     }
 
     return cmd;
 }
 
 /*!
  * \internal
  * \brief Execute an action using the (internal) ocf:pacemaker:remote agent
  *
  * \param[in]  lrm_state       Executor state object for remote connection
  * \param[in]  rsc_id          Connection resource ID
  * \param[in]  action          Action to execute
  * \param[in]  userdata        String to copy and pass to execution callback
  * \param[in]  interval_ms     Action interval (in milliseconds)
  * \param[in]  timeout_ms      Action timeout (in milliseconds)
  * \param[in]  start_delay_ms  Delay (in milliseconds) before initiating action
  * \param[in]  params          Connection resource parameters
  * \param[out] call_id         Where to store call ID on success
  *
  * \return Standard Pacemaker return code
  * \note This takes ownership of \p params, which should not be used or freed
  *       after calling this function.
  */
 int
 controld_execute_remote_agent(lrm_state_t *lrm_state, const char *rsc_id,
                               const char *action, const char *userdata,
                               guint interval_ms, int timeout_ms,
                               int start_delay_ms, lrmd_key_value_t *params,
                               int *call_id)
 {
     lrm_state_t *connection_rsc = NULL;
     remote_ra_cmd_t *cmd = NULL;
     remote_ra_data_t *ra_data = NULL;
 
     *call_id = 0;
 
     CRM_CHECK((lrm_state != NULL) && (rsc_id != NULL) && (action != NULL)
               && (userdata != NULL) && (call_id != NULL),
               lrmd_key_value_freeall(params); return EINVAL);
 
     if (!is_remote_ra_supported_action(action)) {
         lrmd_key_value_freeall(params);
         return EOPNOTSUPP;
     }
 
     connection_rsc = lrm_state_find(rsc_id);
     if (connection_rsc == NULL) {
         lrmd_key_value_freeall(params);
         return ENOTCONN;
     }
 
     remote_ra_data_init(connection_rsc);
     ra_data = connection_rsc->remote_ra_data;
 
     cmd = handle_dup_monitor(ra_data, interval_ms, userdata);
     if (cmd) {
         *call_id = cmd->call_id;
         lrmd_key_value_freeall(params);
         return pcmk_rc_ok;
     }
 
     cmd = calloc(1, sizeof(remote_ra_cmd_t));
     if (cmd == NULL) {
         lrmd_key_value_freeall(params);
         return ENOMEM;
     }
 
     cmd->owner = strdup(lrm_state->node_name);
     cmd->rsc_id = strdup(rsc_id);
     cmd->action = strdup(action);
     cmd->userdata = strdup(userdata);
     if ((cmd->owner == NULL) || (cmd->rsc_id == NULL) || (cmd->action == NULL)
         || (cmd->userdata == NULL)) {
         free_cmd(cmd);
         lrmd_key_value_freeall(params);
         return ENOMEM;
     }
 
     cmd->interval_ms = interval_ms;
     cmd->timeout = timeout_ms;
     cmd->start_delay = start_delay_ms;
     cmd->params = params;
     cmd->start_time = time(NULL);
 
     cmd->call_id = generate_callid();
 
     if (cmd->start_delay) {
         cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
     }
 
     ra_data->cmds = g_list_append(ra_data->cmds, cmd);
     mainloop_set_trigger(ra_data->work);
 
     *call_id = cmd->call_id;
     return pcmk_rc_ok;
 }
 
 /*!
  * \internal
  * \brief Immediately fail all monitors of a remote node, if proxied here
  *
  * \param[in] node_name  Name of pacemaker_remote node
  */
 void
 remote_ra_fail(const char *node_name)
 {
     lrm_state_t *lrm_state = lrm_state_find(node_name);
 
     if (lrm_state && lrm_state_is_connected(lrm_state)) {
         remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
         crm_info("Failing monitors on Pacemaker Remote node %s", node_name);
         ra_data->recurring_cmds = fail_all_monitor_cmds(ra_data->recurring_cmds);
         ra_data->cmds = fail_all_monitor_cmds(ra_data->cmds);
     }
 }
 
 /* A guest node fencing implied by host fencing looks like:
  *
  *  <pseudo_event id="103" operation="stonith" operation_key="stonith-lxc1-off"
  *                on_node="lxc1" on_node_uuid="lxc1">
  *     <attributes CRM_meta_on_node="lxc1" CRM_meta_on_node_uuid="lxc1"
  *                 CRM_meta_stonith_action="off" crm_feature_set="3.0.12"/>
  *     <downed>
  *       <node id="lxc1"/>
  *     </downed>
  *  </pseudo_event>
  */
 #define XPATH_PSEUDO_FENCE "//" XML_GRAPH_TAG_PSEUDO_EVENT \
     "[@" XML_LRM_ATTR_TASK "='stonith']/" XML_GRAPH_TAG_DOWNED \
     "/" XML_CIB_TAG_NODE
 
 /*!
  * \internal
  * \brief Check a pseudo-action for Pacemaker Remote node side effects
  *
  * \param[in] xml  XML of pseudo-action to check
  */
 void
 remote_ra_process_pseudo(xmlNode *xml)
 {
     xmlXPathObjectPtr search = xpath_search(xml, XPATH_PSEUDO_FENCE);
 
     if (numXpathResults(search) == 1) {
         xmlNode *result = getXpathResult(search, 0);
 
         /* Normally, we handle the necessary side effects of a guest node stop
          * action when reporting the remote agent's result. However, if the stop
          * is implied due to fencing, it will be a fencing pseudo-event, and
          * there won't be a result to report. Handle that case here.
          *
          * This will result in a duplicate call to remote_node_down() if the
          * guest stop was real instead of implied, but that shouldn't hurt.
          *
          * There is still one corner case that isn't handled: if a guest node
          * isn't running any resources when its host is fenced, it will appear
          * to be cleanly stopped, so there will be no pseudo-fence, and our
          * peer cache state will be incorrect unless and until the guest is
          * recovered.
          */
         if (result) {
             const char *remote = ID(result);
 
             if (remote) {
                 remote_node_down(remote, DOWN_ERASE_LRM);
             }
         }
     }
     freeXpathObject(search);
 }
 
 static void
 remote_ra_maintenance(lrm_state_t * lrm_state, gboolean maintenance)
 {
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
     xmlNode *update, *state;
     int call_opt, call_id = 0;
     crm_node_t *node;
 
     call_opt = crmd_cib_smart_opt();
     node = crm_remote_peer_get(lrm_state->node_name);
     CRM_CHECK(node != NULL, return);
     update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
     state = create_node_state_update(node, node_update_none, update,
                                      __func__);
     crm_xml_add(state, XML_NODE_IS_MAINTENANCE, maintenance?"1":"0");
     fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
     if (call_id < 0) {
         crm_perror(LOG_WARNING, "%s CIB node state update failed", lrm_state->node_name);
     } else {
         /* TODO: still not 100% sure that async update will succeed ... */
         ra_data->is_maintenance = maintenance;
     }
     free_xml(update);
 }
 
 #define XPATH_PSEUDO_MAINTENANCE "//" XML_GRAPH_TAG_PSEUDO_EVENT \
     "[@" XML_LRM_ATTR_TASK "='" CRM_OP_MAINTENANCE_NODES "']/" \
     XML_GRAPH_TAG_MAINTENANCE
 
 /*!
  * \internal
  * \brief Check a pseudo-action holding updates for maintenance state
  *
  * \param[in] xml  XML of pseudo-action to check
  */
 
 void
 remote_ra_process_maintenance_nodes(xmlNode *xml)
 {
     xmlXPathObjectPtr search = xpath_search(xml, XPATH_PSEUDO_MAINTENANCE);
 
     if (numXpathResults(search) == 1) {
         xmlNode *node;
         int cnt = 0, cnt_remote = 0;
 
         for (node =
                 first_named_child(getXpathResult(search, 0), XML_CIB_TAG_NODE);
             node != NULL; node = pcmk__xml_next(node)) {
             lrm_state_t *lrm_state = lrm_state_find(ID(node));
 
             cnt++;
             if (lrm_state && lrm_state->remote_ra_data &&
                 ((remote_ra_data_t *) lrm_state->remote_ra_data)->active) {
                 int is_maint;
 
                 cnt_remote++;
                 pcmk__scan_min_int(crm_element_value(node, XML_NODE_IS_MAINTENANCE),
                                    &is_maint, 0);
                 remote_ra_maintenance(lrm_state, is_maint);
             }
         }
         crm_trace("Action holds %d nodes (%d remotes found) "
                     "adjusting maintenance-mode", cnt, cnt_remote);
     }
     freeXpathObject(search);
 }
 
 gboolean
 remote_ra_is_in_maintenance(lrm_state_t * lrm_state)
 {
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
     return ra_data->is_maintenance;
 }
 
 gboolean
 remote_ra_controlling_guest(lrm_state_t * lrm_state)
 {
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
     return ra_data->controlling_guest;
 }
diff --git a/daemons/execd/execd_commands.c b/daemons/execd/execd_commands.c
index 5ca4735704..3390518759 100644
--- a/daemons/execd/execd_commands.c
+++ b/daemons/execd/execd_commands.c
@@ -1,2000 +1,2003 @@
 /*
  * Copyright 2012-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <glib.h>
 
 // Check whether we have a high-resolution monotonic clock
 #undef PCMK__TIME_USE_CGT
 #if HAVE_DECL_CLOCK_MONOTONIC && defined(CLOCK_MONOTONIC)
 #  define PCMK__TIME_USE_CGT
 #  include <time.h>  /* clock_gettime */
 #endif
 
 #include <unistd.h>
 
 #include <crm/crm.h>
 #include <crm/services.h>
 #include <crm/services_internal.h>
 #include <crm/common/mainloop.h>
 #include <crm/common/ipc.h>
 #include <crm/common/ipc_internal.h>
 #include <crm/msg_xml.h>
 
 #include "pacemaker-execd.h"
 
 #define EXIT_REASON_MAX_LEN 128
 
 GHashTable *rsc_list = NULL;
 
 typedef struct lrmd_cmd_s {
     int timeout;
     guint interval_ms;
     int start_delay;
     int timeout_orig;
 
     int call_id;
 
     int call_opts;
     /* Timer ids, must be removed on cmd destruction. */
     int delay_id;
     int stonith_recurring_id;
 
     int rsc_deleted;
 
     int service_flags;
 
     char *client_id;
     char *origin;
     char *rsc_id;
     char *action;
     char *real_action;
     char *userdata_str;
 
     pcmk__action_result_t result;
 
     /* We can track operation queue time and run time, to be saved with the CIB
      * resource history (and displayed in cluster status). We need
      * high-resolution monotonic time for this purpose, so we use
      * clock_gettime(CLOCK_MONOTONIC, ...) (if available, otherwise this feature
      * is disabled).
      *
      * However, we also need epoch timestamps for recording the time the command
      * last ran and the time its return value last changed, for use in time
      * displays (as opposed to interval calculations). We keep time_t values for
      * this purpose.
      *
      * The last run time is used for both purposes, so we keep redundant
      * monotonic and epoch values for this. Technically the two could represent
      * different times, but since time_t has only second resolution and the
      * values are used for distinct purposes, that is not significant.
      */
 #ifdef PCMK__TIME_USE_CGT
     /* Recurring and systemd operations may involve more than one executor
      * command per operation, so they need info about the original and the most
      * recent.
      */
     struct timespec t_first_run;    // When op first ran
     struct timespec t_run;          // When op most recently ran
     struct timespec t_first_queue;  // When op was first queued
     struct timespec t_queue;        // When op was most recently queued
 #endif
     time_t epoch_last_run;          // Epoch timestamp of when op last ran
     time_t epoch_rcchange;          // Epoch timestamp of when rc last changed
 
     bool first_notify_sent;
     int last_notify_rc;
     int last_notify_op_status;
     int last_pid;
 
     GHashTable *params;
 } lrmd_cmd_t;
 
 static void cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc);
 static gboolean lrmd_rsc_dispatch(gpointer user_data);
 static void cancel_all_recurring(lrmd_rsc_t * rsc, const char *client_id);
 
 #ifdef PCMK__TIME_USE_CGT
 
 /*!
  * \internal
  * \brief Check whether a struct timespec has been set
  *
  * \param[in] timespec  Time to check
  *
  * \return true if timespec has been set (i.e. is nonzero), false otherwise
  */
 static inline bool
 time_is_set(struct timespec *timespec)
 {
     return (timespec != NULL) &&
            ((timespec->tv_sec != 0) || (timespec->tv_nsec != 0));
 }
 
 /*
  * \internal
  * \brief Set a timespec (and its original if unset) to the current time
  *
  * \param[out] t_current  Where to store current time
  * \param[out] t_orig     Where to copy t_current if unset
  */
 static void
 get_current_time(struct timespec *t_current, struct timespec *t_orig)
 {
     clock_gettime(CLOCK_MONOTONIC, t_current);
     if ((t_orig != NULL) && !time_is_set(t_orig)) {
         *t_orig = *t_current;
     }
 }
 
 /*!
  * \internal
  * \brief Return difference between two times in milliseconds
  *
  * \param[in] now  More recent time (or NULL to use current time)
  * \param[in] old  Earlier time
  *
  * \return milliseconds difference (or 0 if old is NULL or unset)
  *
  * \note Can overflow on 32bit machines when the differences is around
  *       24 days or more.
  */
 static int
 time_diff_ms(struct timespec *now, struct timespec *old)
 {
     int diff_ms = 0;
 
     if (time_is_set(old)) {
         struct timespec local_now = { 0, };
 
         if (now == NULL) {
             clock_gettime(CLOCK_MONOTONIC, &local_now);
             now = &local_now;
         }
         diff_ms = (now->tv_sec - old->tv_sec) * 1000
                   + (now->tv_nsec - old->tv_nsec) / 1000000;
     }
     return diff_ms;
 }
 
 /*!
  * \internal
  * \brief Reset a command's operation times to their original values.
  *
  * Reset a command's run and queued timestamps to the timestamps of the original
  * command, so we report the entire time since then and not just the time since
  * the most recent command (for recurring and systemd operations).
  *
  * \param[in] cmd  Executor command object to reset
  *
  * \note It's not obvious what the queued time should be for a systemd
  *       start/stop operation, which might go like this:
  *         initial command queued 5ms, runs 3s
  *         monitor command queued 10ms, runs 10s
  *         monitor command queued 10ms, runs 10s
  *       Is the queued time for that operation 5ms, 10ms or 25ms? The current
  *       implementation will report 5ms. If it's 25ms, then we need to
  *       subtract 20ms from the total exec time so as not to count it twice.
  *       We can implement that later if it matters to anyone ...
  */
 static void
 cmd_original_times(lrmd_cmd_t * cmd)
 {
     cmd->t_run = cmd->t_first_run;
     cmd->t_queue = cmd->t_first_queue;
 }
 #endif
 
 static inline bool
 action_matches(lrmd_cmd_t *cmd, const char *action, guint interval_ms)
 {
     return (cmd->interval_ms == interval_ms)
            && pcmk__str_eq(cmd->action, action, pcmk__str_casei);
 }
 
 static void
 log_finished(lrmd_cmd_t * cmd, int exec_time, int queue_time)
 {
     char pid_str[32] = { 0, };
     int log_level = LOG_INFO;
 
     if (cmd->last_pid) {
         snprintf(pid_str, 32, "%d", cmd->last_pid);
     }
 
     if (pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
         log_level = LOG_DEBUG;
     }
 #ifdef PCMK__TIME_USE_CGT
     do_crm_log(log_level, "%s %s (call %d%s%s) exited with status %d"
                " (execution time %dms, queue time %dms)",
                cmd->rsc_id, cmd->action, cmd->call_id,
                (cmd->last_pid? ", PID " : ""), pid_str,
                cmd->result.exit_status, exec_time, queue_time);
 #else
     do_crm_log(log_level, "%s %s (call %d%s%s) exited with status %d",
                cmd->rsc_id, cmd->action, cmd->call_id,
                (cmd->last_pid? ", PID " : ""), pid_str,
                cmd->result.exit_status);
 #endif
 }
 
 static void
 log_execute(lrmd_cmd_t * cmd)
 {
     int log_level = LOG_INFO;
 
     if (pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
         log_level = LOG_DEBUG;
     }
 
     do_crm_log(log_level, "executing - rsc:%s action:%s call_id:%d",
                cmd->rsc_id, cmd->action, cmd->call_id);
 }
 
 static const char *
 normalize_action_name(lrmd_rsc_t * rsc, const char *action)
 {
     if (pcmk__str_eq(action, "monitor", pcmk__str_casei) &&
         pcmk_is_set(pcmk_get_ra_caps(rsc->class), pcmk_ra_cap_status)) {
         return "status";
     }
     return action;
 }
 
 static lrmd_rsc_t *
 build_rsc_from_xml(xmlNode * msg)
 {
     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, msg, LOG_ERR);
     lrmd_rsc_t *rsc = NULL;
 
     rsc = calloc(1, sizeof(lrmd_rsc_t));
 
     crm_element_value_int(msg, F_LRMD_CALLOPTS, &rsc->call_opts);
 
     rsc->rsc_id = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ID);
     rsc->class = crm_element_value_copy(rsc_xml, F_LRMD_CLASS);
     rsc->provider = crm_element_value_copy(rsc_xml, F_LRMD_PROVIDER);
     rsc->type = crm_element_value_copy(rsc_xml, F_LRMD_TYPE);
     rsc->work = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_rsc_dispatch, rsc);
     rsc->st_probe_rc = -ENODEV; // if stonith, initialize to "not running"
     return rsc;
 }
 
 static lrmd_cmd_t *
 create_lrmd_cmd(xmlNode *msg, pcmk__client_t *client)
 {
     int call_options = 0;
     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, msg, LOG_ERR);
     lrmd_cmd_t *cmd = NULL;
 
     cmd = calloc(1, sizeof(lrmd_cmd_t));
 
     crm_element_value_int(msg, F_LRMD_CALLOPTS, &call_options);
     cmd->call_opts = call_options;
     cmd->client_id = strdup(client->id);
 
     crm_element_value_int(msg, F_LRMD_CALLID, &cmd->call_id);
     crm_element_value_ms(rsc_xml, F_LRMD_RSC_INTERVAL, &cmd->interval_ms);
     crm_element_value_int(rsc_xml, F_LRMD_TIMEOUT, &cmd->timeout);
     crm_element_value_int(rsc_xml, F_LRMD_RSC_START_DELAY, &cmd->start_delay);
     cmd->timeout_orig = cmd->timeout;
 
     cmd->origin = crm_element_value_copy(rsc_xml, F_LRMD_ORIGIN);
     cmd->action = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ACTION);
     cmd->userdata_str = crm_element_value_copy(rsc_xml, F_LRMD_RSC_USERDATA_STR);
     cmd->rsc_id = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ID);
 
     cmd->params = xml2list(rsc_xml);
 
     if (pcmk__str_eq(g_hash_table_lookup(cmd->params, "CRM_meta_on_fail"), "block", pcmk__str_casei)) {
         crm_debug("Setting flag to leave pid group on timeout and "
                   "only kill action pid for " PCMK__OP_FMT,
                   cmd->rsc_id, cmd->action, cmd->interval_ms);
         cmd->service_flags = pcmk__set_flags_as(__func__, __LINE__,
                                                 LOG_TRACE, "Action",
                                                 cmd->action, 0,
                                                 SVC_ACTION_LEAVE_GROUP,
                                                 "SVC_ACTION_LEAVE_GROUP");
     }
     return cmd;
 }
 
 static void
 stop_recurring_timer(lrmd_cmd_t *cmd)
 {
     if (cmd) {
         if (cmd->stonith_recurring_id) {
             g_source_remove(cmd->stonith_recurring_id);
         }
         cmd->stonith_recurring_id = 0;
     }
 }
 
 static void
 free_lrmd_cmd(lrmd_cmd_t * cmd)
 {
     stop_recurring_timer(cmd);
     if (cmd->delay_id) {
         g_source_remove(cmd->delay_id);
     }
     if (cmd->params) {
         g_hash_table_destroy(cmd->params);
     }
     pcmk__reset_result(&(cmd->result));
     free(cmd->origin);
     free(cmd->action);
     free(cmd->real_action);
     free(cmd->userdata_str);
     free(cmd->rsc_id);
     free(cmd->client_id);
     free(cmd);
 }
 
 static gboolean
 stonith_recurring_op_helper(gpointer data)
 {
     lrmd_cmd_t *cmd = data;
     lrmd_rsc_t *rsc;
 
     cmd->stonith_recurring_id = 0;
 
     if (!cmd->rsc_id) {
         return FALSE;
     }
 
     rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id);
 
     CRM_ASSERT(rsc != NULL);
     /* take it out of recurring_ops list, and put it in the pending ops
      * to be executed */
     rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd);
     rsc->pending_ops = g_list_append(rsc->pending_ops, cmd);
 #ifdef PCMK__TIME_USE_CGT
     get_current_time(&(cmd->t_queue), &(cmd->t_first_queue));
 #endif
     mainloop_set_trigger(rsc->work);
 
     return FALSE;
 }
 
 static inline void
 start_recurring_timer(lrmd_cmd_t *cmd)
 {
     if (cmd && (cmd->interval_ms > 0)) {
         cmd->stonith_recurring_id = g_timeout_add(cmd->interval_ms,
                                                   stonith_recurring_op_helper,
                                                   cmd);
     }
 }
 
 static gboolean
 start_delay_helper(gpointer data)
 {
     lrmd_cmd_t *cmd = data;
     lrmd_rsc_t *rsc = NULL;
 
     cmd->delay_id = 0;
     rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL;
 
     if (rsc) {
         mainloop_set_trigger(rsc->work);
     }
 
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Check whether a list already contains the equivalent of a given action
  */
 static lrmd_cmd_t *
 find_duplicate_action(GList *action_list, lrmd_cmd_t *cmd)
 {
     for (GList *item = action_list; item != NULL; item = item->next) {
         lrmd_cmd_t *dup = item->data;
 
         if (action_matches(cmd, dup->action, dup->interval_ms)) {
             return dup;
         }
     }
     return NULL;
 }
 
 static bool
 merge_recurring_duplicate(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
 {
     lrmd_cmd_t * dup = NULL;
     bool dup_pending = true;
 
     if (cmd->interval_ms == 0) {
         return false;
     }
 
     // Search for a duplicate of this action (in-flight or not)
     dup = find_duplicate_action(rsc->pending_ops, cmd);
     if (dup == NULL) {
         dup_pending = false;
         dup = find_duplicate_action(rsc->recurring_ops, cmd);
         if (dup == NULL) {
             return false;
         }
     }
 
     /* Do not merge fencing monitors marked for cancellation, so we can reply to
      * the cancellation separately.
      */
     if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH,
                      pcmk__str_casei)
         && (dup->result.exec_status == PCMK_EXEC_CANCELLED)) {
         return false;
     }
 
     /* This should not occur. If it does, we need to investigate how something
      * like this is possible in the controller.
      */
     crm_warn("Duplicate recurring op entry detected (" PCMK__OP_FMT
              "), merging with previous op entry",
              rsc->rsc_id, normalize_action_name(rsc, dup->action),
              dup->interval_ms);
 
     // Merge new action's call ID and user data into existing action
     dup->first_notify_sent = false;
     free(dup->userdata_str);
     dup->userdata_str = cmd->userdata_str;
     cmd->userdata_str = NULL;
     dup->call_id = cmd->call_id;
     free_lrmd_cmd(cmd);
     cmd = NULL;
 
     /* If dup is not pending, that means it has already executed at least once
      * and is waiting in the interval. In that case, stop waiting and initiate
      * a new instance now.
      */
     if (!dup_pending) {
         if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH,
                          pcmk__str_casei)) {
             stop_recurring_timer(dup);
             stonith_recurring_op_helper(dup);
         } else {
             services_action_kick(rsc->rsc_id,
                                  normalize_action_name(rsc, dup->action),
                                  dup->interval_ms);
         }
     }
     return true;
 }
 
 static void
 schedule_lrmd_cmd(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
 {
     CRM_CHECK(cmd != NULL, return);
     CRM_CHECK(rsc != NULL, return);
 
     crm_trace("Scheduling %s on %s", cmd->action, rsc->rsc_id);
 
     if (merge_recurring_duplicate(rsc, cmd)) {
         // Equivalent of cmd has already been scheduled
         return;
     }
 
     /* The controller expects the executor to automatically cancel
      * recurring operations before a resource stops.
      */
     if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
         cancel_all_recurring(rsc, NULL);
     }
 
     rsc->pending_ops = g_list_append(rsc->pending_ops, cmd);
 #ifdef PCMK__TIME_USE_CGT
     get_current_time(&(cmd->t_queue), &(cmd->t_first_queue));
 #endif
     mainloop_set_trigger(rsc->work);
 
     if (cmd->start_delay) {
         cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
     }
 }
 
 static xmlNode *
 create_lrmd_reply(const char *origin, int rc, int call_id)
 {
     xmlNode *reply = create_xml_node(NULL, T_LRMD_REPLY);
 
     crm_xml_add(reply, F_LRMD_ORIGIN, origin);
     crm_xml_add_int(reply, F_LRMD_RC, rc);
     crm_xml_add_int(reply, F_LRMD_CALLID, call_id);
     return reply;
 }
 
 static void
 send_client_notify(gpointer key, gpointer value, gpointer user_data)
 {
     xmlNode *update_msg = user_data;
     pcmk__client_t *client = value;
     int rc;
     int log_level = LOG_WARNING;
     const char *msg = NULL;
 
     CRM_CHECK(client != NULL, return);
     if (client->name == NULL) {
         crm_trace("Skipping notification to client without name");
         return;
     }
     if (pcmk_is_set(client->flags, pcmk__client_to_proxy)) {
         /* We only want to notify clients of the executor IPC API. If we are
          * running as Pacemaker Remote, we may have clients proxied to other
          * IPC services in the cluster, so skip those.
          */
         crm_trace("Skipping executor API notification to client %s",
                   pcmk__client_name(client));
         return;
     }
 
     rc = lrmd_server_send_notify(client, update_msg);
     if (rc == pcmk_rc_ok) {
         return;
     }
 
     switch (rc) {
         case ENOTCONN:
         case EPIPE: // Client exited without waiting for notification
             log_level = LOG_INFO;
             msg = "Disconnected";
             break;
 
         default:
             msg = pcmk_rc_str(rc);
             break;
     }
     do_crm_log(log_level, "Could not notify client %s: %s " CRM_XS " rc=%d",
                pcmk__client_name(client), msg, rc);
 }
 
 static void
 send_cmd_complete_notify(lrmd_cmd_t * cmd)
 {
     xmlNode *notify = NULL;
 
 #ifdef PCMK__TIME_USE_CGT
     int exec_time = time_diff_ms(NULL, &(cmd->t_run));
     int queue_time = time_diff_ms(&cmd->t_run, &(cmd->t_queue));
 
     log_finished(cmd, exec_time, queue_time);
 #else
     log_finished(cmd, 0, 0);
 #endif
 
     /* if the first notify result for a cmd has already been sent earlier, and the
      * the option to only send notifies on result changes is set. Check to see
      * if the last result is the same as the new one. If so, suppress this update */
     if (cmd->first_notify_sent && (cmd->call_opts & lrmd_opt_notify_changes_only)) {
         if ((cmd->last_notify_rc == cmd->result.exit_status) &&
             (cmd->last_notify_op_status == cmd->result.exec_status)) {
 
             /* only send changes */
             return;
         }
 
     }
 
     cmd->first_notify_sent = true;
     cmd->last_notify_rc = cmd->result.exit_status;
     cmd->last_notify_op_status = cmd->result.exec_status;
 
     notify = create_xml_node(NULL, T_LRMD_NOTIFY);
 
     crm_xml_add(notify, F_LRMD_ORIGIN, __func__);
     crm_xml_add_int(notify, F_LRMD_TIMEOUT, cmd->timeout);
     crm_xml_add_ms(notify, F_LRMD_RSC_INTERVAL, cmd->interval_ms);
     crm_xml_add_int(notify, F_LRMD_RSC_START_DELAY, cmd->start_delay);
     crm_xml_add_int(notify, F_LRMD_EXEC_RC, cmd->result.exit_status);
     crm_xml_add_int(notify, F_LRMD_OP_STATUS, cmd->result.exec_status);
     crm_xml_add_int(notify, F_LRMD_CALLID, cmd->call_id);
     crm_xml_add_int(notify, F_LRMD_RSC_DELETED, cmd->rsc_deleted);
 
     crm_xml_add_ll(notify, F_LRMD_RSC_RUN_TIME,
                    (long long) cmd->epoch_last_run);
     crm_xml_add_ll(notify, F_LRMD_RSC_RCCHANGE_TIME,
                    (long long) cmd->epoch_rcchange);
 #ifdef PCMK__TIME_USE_CGT
     crm_xml_add_int(notify, F_LRMD_RSC_EXEC_TIME, exec_time);
     crm_xml_add_int(notify, F_LRMD_RSC_QUEUE_TIME, queue_time);
 #endif
 
     crm_xml_add(notify, F_LRMD_OPERATION, LRMD_OP_RSC_EXEC);
     crm_xml_add(notify, F_LRMD_RSC_ID, cmd->rsc_id);
     if(cmd->real_action) {
         crm_xml_add(notify, F_LRMD_RSC_ACTION, cmd->real_action);
     } else {
         crm_xml_add(notify, F_LRMD_RSC_ACTION, cmd->action);
     }
     crm_xml_add(notify, F_LRMD_RSC_USERDATA_STR, cmd->userdata_str);
     crm_xml_add(notify, F_LRMD_RSC_EXIT_REASON, cmd->result.exit_reason);
 
     if (cmd->result.action_stderr != NULL) {
         crm_xml_add(notify, F_LRMD_RSC_OUTPUT, cmd->result.action_stderr);
 
     } else if (cmd->result.action_stdout != NULL) {
         crm_xml_add(notify, F_LRMD_RSC_OUTPUT, cmd->result.action_stdout);
     }
 
     if (cmd->params) {
         char *key = NULL;
         char *value = NULL;
         GHashTableIter iter;
 
         xmlNode *args = create_xml_node(notify, XML_TAG_ATTRS);
 
         g_hash_table_iter_init(&iter, cmd->params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             hash2smartfield((gpointer) key, (gpointer) value, args);
         }
     }
     if (cmd->client_id && (cmd->call_opts & lrmd_opt_notify_orig_only)) {
         pcmk__client_t *client = pcmk__find_client_by_id(cmd->client_id);
 
         if (client) {
             send_client_notify(client->id, client, notify);
         }
     } else {
         pcmk__foreach_ipc_client(send_client_notify, notify);
     }
 
     free_xml(notify);
 }
 
 static void
 send_generic_notify(int rc, xmlNode * request)
 {
     if (pcmk__ipc_client_count() != 0) {
         int call_id = 0;
         xmlNode *notify = NULL;
         xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
         const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
         const char *op = crm_element_value(request, F_LRMD_OPERATION);
 
         crm_element_value_int(request, F_LRMD_CALLID, &call_id);
 
         notify = create_xml_node(NULL, T_LRMD_NOTIFY);
         crm_xml_add(notify, F_LRMD_ORIGIN, __func__);
         crm_xml_add_int(notify, F_LRMD_RC, rc);
         crm_xml_add_int(notify, F_LRMD_CALLID, call_id);
         crm_xml_add(notify, F_LRMD_OPERATION, op);
         crm_xml_add(notify, F_LRMD_RSC_ID, rsc_id);
 
         pcmk__foreach_ipc_client(send_client_notify, notify);
 
         free_xml(notify);
     }
 }
 
 static void
 cmd_reset(lrmd_cmd_t * cmd)
 {
     cmd->last_pid = 0;
 #ifdef PCMK__TIME_USE_CGT
     memset(&cmd->t_run, 0, sizeof(cmd->t_run));
     memset(&cmd->t_queue, 0, sizeof(cmd->t_queue));
 #endif
     cmd->epoch_last_run = 0;
 
     pcmk__reset_result(&(cmd->result));
     cmd->result.exec_status = PCMK_EXEC_DONE;
 }
 
 static void
 cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc)
 {
     crm_trace("Resource operation rsc:%s action:%s completed (%p %p)", cmd->rsc_id, cmd->action,
               rsc ? rsc->active : NULL, cmd);
 
     if (rsc && (rsc->active == cmd)) {
         rsc->active = NULL;
         mainloop_set_trigger(rsc->work);
     }
 
     if (!rsc) {
         cmd->rsc_deleted = 1;
     }
 
     /* reset original timeout so client notification has correct information */
     cmd->timeout = cmd->timeout_orig;
 
     send_cmd_complete_notify(cmd);
 
     if ((cmd->interval_ms != 0)
         && (cmd->result.exec_status == PCMK_EXEC_CANCELLED)) {
 
         if (rsc) {
             rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd);
             rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd);
         }
         free_lrmd_cmd(cmd);
     } else if (cmd->interval_ms == 0) {
         if (rsc) {
             rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd);
         }
         free_lrmd_cmd(cmd);
     } else {
         /* Clear all the values pertaining just to the last iteration of a recurring op. */
         cmd_reset(cmd);
     }
 }
 
 static int
 ocf2uniform_rc(int rc)
 {
     switch (rc) {
         case PCMK_OCF_DEGRADED:
         case PCMK_OCF_DEGRADED_PROMOTED:
             break;
         default:
             if (rc < 0 || rc > PCMK_OCF_FAILED_PROMOTED) {
                 return PCMK_OCF_UNKNOWN_ERROR;
             }
     }
 
     return rc;
 }
 
 static int
 stonith2uniform_rc(const char *action, int rc)
 {
     switch (rc) {
         case pcmk_ok:
             rc = PCMK_OCF_OK;
             break;
 
         case -ENODEV:
             /* This should be possible only for probes in practice, but
              * interpret for all actions to be safe.
              */
             if (pcmk__str_eq(action, "monitor", pcmk__str_casei)) {
                 rc = PCMK_OCF_NOT_RUNNING;
             } else if (pcmk__str_eq(action, "stop", pcmk__str_casei)) {
                 rc = PCMK_OCF_OK;
             } else {
                 rc = PCMK_OCF_NOT_INSTALLED;
             }
             break;
 
         case -EOPNOTSUPP:
             rc = PCMK_OCF_UNIMPLEMENT_FEATURE;
             break;
 
         case -ETIME:
         case -ETIMEDOUT:
             rc = PCMK_OCF_TIMEOUT;
             break;
 
         default:
             rc = PCMK_OCF_UNKNOWN_ERROR;
             break;
     }
     return rc;
 }
 
 #if SUPPORT_NAGIOS
 static int
 nagios2uniform_rc(const char *action, int rc)
 {
     if (rc < 0) {
         return PCMK_OCF_UNKNOWN_ERROR;
     }
 
     switch (rc) {
         case NAGIOS_STATE_OK:
             return PCMK_OCF_OK;
         case NAGIOS_INSUFFICIENT_PRIV:
             return PCMK_OCF_INSUFFICIENT_PRIV;
         case NAGIOS_NOT_INSTALLED:
             return PCMK_OCF_NOT_INSTALLED;
         case NAGIOS_STATE_WARNING:
         case NAGIOS_STATE_CRITICAL:
         case NAGIOS_STATE_UNKNOWN:
         case NAGIOS_STATE_DEPENDENT:
         default:
             return PCMK_OCF_UNKNOWN_ERROR;
     }
 
     return PCMK_OCF_UNKNOWN_ERROR;
 }
 #endif
 
 static int
 get_uniform_rc(const char *standard, const char *action, int rc)
 {
     if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_OCF, pcmk__str_casei)) {
         return ocf2uniform_rc(rc);
     } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
         return stonith2uniform_rc(action, rc);
     } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_SYSTEMD, pcmk__str_casei)) {
         return rc;
     } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_UPSTART, pcmk__str_casei)) {
         return rc;
 #if SUPPORT_NAGIOS
     } else if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
         return nagios2uniform_rc(action, rc);
 #endif
     } else {
         return services_get_ocf_exitcode(action, rc);
     }
 }
 
 static int
 action_get_uniform_rc(svc_action_t * action)
 {
     lrmd_cmd_t *cmd = action->cb_data;
     return get_uniform_rc(action->standard, cmd->action, action->rc);
 }
 
 struct notify_new_client_data {
     xmlNode *notify;
     pcmk__client_t *new_client;
 };
 
 static void
 notify_one_client(gpointer key, gpointer value, gpointer user_data)
 {
     pcmk__client_t *client = value;
     struct notify_new_client_data *data = user_data;
 
     if (!pcmk__str_eq(client->id, data->new_client->id, pcmk__str_casei)) {
         send_client_notify(key, (gpointer) client, (gpointer) data->notify);
     }
 }
 
 void
 notify_of_new_client(pcmk__client_t *new_client)
 {
     struct notify_new_client_data data;
 
     data.new_client = new_client;
     data.notify = create_xml_node(NULL, T_LRMD_NOTIFY);
     crm_xml_add(data.notify, F_LRMD_ORIGIN, __func__);
     crm_xml_add(data.notify, F_LRMD_OPERATION, LRMD_OP_NEW_CLIENT);
     pcmk__foreach_ipc_client(notify_one_client, &data);
     free_xml(data.notify);
 }
 
 static char *
 parse_exit_reason(const char *output)
 {
     const char *cur = NULL;
     const char *last = NULL;
     static int cookie_len = 0;
     char *eol = NULL;
     size_t reason_len = EXIT_REASON_MAX_LEN;
 
     if (output == NULL) {
         return NULL;
     }
 
     if (!cookie_len) {
         cookie_len = strlen(PCMK_OCF_REASON_PREFIX);
     }
 
     cur = strstr(output, PCMK_OCF_REASON_PREFIX);
     for (; cur != NULL; cur = strstr(cur, PCMK_OCF_REASON_PREFIX)) {
         /* skip over the cookie delimiter string */
         cur += cookie_len;
         last = cur;
     }
     if (last == NULL) {
         return NULL;
     }
 
     // Truncate everything after a new line, and limit reason string size
     eol = strchr(last, '\n');
     if (eol) {
         reason_len = QB_MIN(reason_len, eol - last);
     }
     return strndup(last, reason_len);
 }
 
 void
 client_disconnect_cleanup(const char *client_id)
 {
     GHashTableIter iter;
     lrmd_rsc_t *rsc = NULL;
     char *key = NULL;
 
     g_hash_table_iter_init(&iter, rsc_list);
     while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & rsc)) {
         if (rsc->call_opts & lrmd_opt_drop_recurring) {
             /* This client is disconnecting, drop any recurring operations
              * it may have initiated on the resource */
             cancel_all_recurring(rsc, client_id);
         }
     }
 }
 
 static void
 action_complete(svc_action_t * action)
 {
     lrmd_rsc_t *rsc;
     lrmd_cmd_t *cmd = action->cb_data;
 
 #ifdef PCMK__TIME_USE_CGT
     const char *rclass = NULL;
     bool goagain = false;
 #endif
 
     if (!cmd) {
         crm_err("Completed executor action (%s) does not match any known operations",
                 action->id);
         return;
     }
 
 #ifdef PCMK__TIME_USE_CGT
     if (cmd->result.exit_status != action->rc) {
         cmd->epoch_rcchange = time(NULL);
     }
 #endif
 
     cmd->last_pid = action->pid;
     pcmk__set_result(&(cmd->result), action_get_uniform_rc(action),
                      action->status, NULL);
     rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL;
 
 #ifdef PCMK__TIME_USE_CGT
     if (rsc && pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
         rclass = resources_find_service_class(rsc->type);
     } else if(rsc) {
         rclass = rsc->class;
     }
 
     if (pcmk__str_eq(rclass, PCMK_RESOURCE_CLASS_SYSTEMD, pcmk__str_casei)) {
         if ((cmd->result.exit_status == PCMK_OCF_OK)
             && pcmk__strcase_any_of(cmd->action, "start", "stop", NULL)) {
             /* systemd returns from start and stop actions after the action
              * begins, not after it completes. We have to jump through a few
              * hoops so that we don't report 'complete' to the rest of pacemaker
              * until it's actually done.
              */
             goagain = true;
             cmd->real_action = cmd->action;
             cmd->action = strdup("monitor");
 
         } else if (cmd->real_action != NULL) {
             // This is follow-up monitor to check whether start/stop completed
             if ((cmd->result.exec_status == PCMK_EXEC_DONE)
                 && (cmd->result.exit_status == PCMK_OCF_PENDING)) {
                 goagain = true;
 
             } else if ((cmd->result.exit_status == PCMK_OCF_OK)
                        && pcmk__str_eq(cmd->real_action, "stop", pcmk__str_casei)) {
                 goagain = true;
 
             } else {
                 int time_sum = time_diff_ms(NULL, &(cmd->t_first_run));
                 int timeout_left = cmd->timeout_orig - time_sum;
 
                 crm_debug("%s systemd %s is now complete (elapsed=%dms, "
                           "remaining=%dms): %s (%d)",
                           cmd->rsc_id, cmd->real_action, time_sum, timeout_left,
                           services_ocf_exitcode_str(cmd->result.exit_status),
                           cmd->result.exit_status);
                 cmd_original_times(cmd);
 
                 // Monitors may return "not running", but start/stop shouldn't
                 if ((cmd->result.exec_status == PCMK_EXEC_DONE)
                     && (cmd->result.exit_status == PCMK_OCF_NOT_RUNNING)) {
 
                     if (pcmk__str_eq(cmd->real_action, "start", pcmk__str_casei)) {
                         cmd->result.exit_status = PCMK_OCF_UNKNOWN_ERROR;
                     } else if (pcmk__str_eq(cmd->real_action, "stop", pcmk__str_casei)) {
                         cmd->result.exit_status = PCMK_OCF_OK;
                     }
                 }
             }
         }
     }
 #endif
 
 #if SUPPORT_NAGIOS
     if (rsc && pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)) {
         if (action_matches(cmd, "monitor", 0)
             && (cmd->result.exit_status == PCMK_OCF_OK)) {
             /* Successfully executed --version for the nagios plugin */
             cmd->result.exit_status = PCMK_OCF_NOT_RUNNING;
 
         } else if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)
                    && (cmd->result.exit_status != PCMK_OCF_OK)) {
 #ifdef PCMK__TIME_USE_CGT
             goagain = true;
 #endif
         }
     }
 #endif
 
 #ifdef PCMK__TIME_USE_CGT
     if (goagain) {
         int time_sum = time_diff_ms(NULL, &(cmd->t_first_run));
         int timeout_left = cmd->timeout_orig - time_sum;
         int delay = cmd->timeout_orig / 10;
 
         if(delay >= timeout_left && timeout_left > 20) {
             delay = timeout_left/2;
         }
 
         delay = QB_MIN(2000, delay);
         if (delay < timeout_left) {
             cmd->start_delay = delay;
             cmd->timeout = timeout_left;
 
             if (cmd->result.exit_status == PCMK_OCF_OK) {
                 crm_debug("%s %s may still be in progress: re-scheduling (elapsed=%dms, remaining=%dms, start_delay=%dms)",
                           cmd->rsc_id, cmd->real_action, time_sum, timeout_left, delay);
 
             } else if (cmd->result.exit_status == PCMK_OCF_PENDING) {
                 crm_info("%s %s is still in progress: re-scheduling (elapsed=%dms, remaining=%dms, start_delay=%dms)",
                          cmd->rsc_id, cmd->action, time_sum, timeout_left, delay);
 
             } else {
                 crm_notice("%s %s failed '%s' (%d): re-scheduling (elapsed=%dms, remaining=%dms, start_delay=%dms)",
                            cmd->rsc_id, cmd->action,
                            services_ocf_exitcode_str(cmd->result.exit_status),
                            cmd->result.exit_status, time_sum, timeout_left,
                            delay);
             }
 
             cmd_reset(cmd);
             if(rsc) {
                 rsc->active = NULL;
             }
             schedule_lrmd_cmd(rsc, cmd);
 
             /* Don't finalize cmd, we're not done with it yet */
             return;
 
         } else {
             crm_notice("Giving up on %s %s (rc=%d): timeout (elapsed=%dms, remaining=%dms)",
                        cmd->rsc_id,
                        (cmd->real_action? cmd->real_action : cmd->action),
                        cmd->result.exit_status, time_sum, timeout_left);
             pcmk__set_result(&(cmd->result), PCMK_OCF_TIMEOUT,
-                             PCMK_EXEC_TIMEOUT, NULL);
+                             PCMK_EXEC_TIMEOUT,
+                             "Investigate reason for timeout, and adjust "
+                             "configured operation timeout if necessary");
             cmd_original_times(cmd);
         }
     }
 #endif
 
     pcmk__set_result_output(&(cmd->result),
                             action->stdout_data, action->stderr_data);
     if (action->stderr_data) {
         cmd->result.exit_reason = parse_exit_reason(action->stderr_data);
     }
 
     cmd_finalize(cmd, rsc);
 }
 
 /*!
  * \internal
  * \brief Determine operation status of a stonith operation
  *
  * Non-stonith resource operations get their operation status directly from the
  * service library, but the fencer does not have an equivalent, so we must infer
  * an operation status from the fencer API's return code.
  *
  * \param[in] action       Name of action performed on stonith resource
  * \param[in] interval_ms  Action interval
  * \param[in] rc           Action result from fencer
  *
  * \return Operation status corresponding to fencer API return code
  */
 static int
 stonith_rc2status(const char *action, guint interval_ms, int rc)
 {
     int status = PCMK_EXEC_DONE;
 
     switch (rc) {
         case pcmk_ok:
             break;
 
         case -EOPNOTSUPP:
         case -EPROTONOSUPPORT:
             status = PCMK_EXEC_NOT_SUPPORTED;
             break;
 
         case -ETIME:
         case -ETIMEDOUT:
             status = PCMK_EXEC_TIMEOUT;
             break;
 
         case -ENOTCONN:
         case -ECOMM:
             // Couldn't talk to fencer
             status = PCMK_EXEC_ERROR;
             break;
 
         case -ENODEV:
             // The device is not registered with the fencer
             status = PCMK_EXEC_ERROR;
             break;
 
         default:
             break;
     }
     return status;
 }
 
 static void
 stonith_action_complete(lrmd_cmd_t * cmd, int rc)
 {
     // This can be NULL if resource was removed before command completed
     lrmd_rsc_t *rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id);
 
     cmd->result.exit_status = stonith2uniform_rc(cmd->action, rc);
 
     /* This function may be called with status already set to cancelled, if a
      * pending action was aborted. Otherwise, we need to determine status from
      * the fencer return code.
      */
     if (cmd->result.exec_status != PCMK_EXEC_CANCELLED) {
         cmd->result.exec_status = stonith_rc2status(cmd->action,
                                                     cmd->interval_ms, rc);
 
         // Certain successful actions change the known state of the resource
         if ((rsc != NULL) && (cmd->result.exit_status == PCMK_OCF_OK)) {
             if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)) {
                 rsc->st_probe_rc = pcmk_ok; // maps to PCMK_OCF_OK
             } else if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
                 rsc->st_probe_rc = -ENODEV; // maps to PCMK_OCF_NOT_RUNNING
             }
         }
     }
 
     /* The recurring timer should not be running at this point in any case, but
      * as a failsafe, stop it if it is.
      */
     stop_recurring_timer(cmd);
 
     /* Reschedule this command if appropriate. If a recurring command is *not*
      * rescheduled, its status must be PCMK_EXEC_CANCELLED, otherwise it will
      * not be removed from recurring_ops by cmd_finalize().
      */
     if (rsc && (cmd->interval_ms > 0)
         && (cmd->result.exec_status != PCMK_EXEC_CANCELLED)) {
         start_recurring_timer(cmd);
     }
 
     cmd_finalize(cmd, rsc);
 }
 
 static void
 lrmd_stonith_callback(stonith_t * stonith, stonith_callback_data_t * data)
 {
     stonith_action_complete(data->userdata, data->rc);
 }
 
 void
 stonith_connection_failed(void)
 {
     GHashTableIter iter;
     GList *cmd_list = NULL;
     GList *cmd_iter = NULL;
     lrmd_rsc_t *rsc = NULL;
     char *key = NULL;
 
     g_hash_table_iter_init(&iter, rsc_list);
     while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & rsc)) {
         if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
             /* If we registered this fence device, we don't know whether the
              * fencer still has the registration or not. Cause future probes to
              * return PCMK_OCF_UNKNOWN_ERROR until the resource is stopped or
              * started successfully. This is especially important if the
              * controller also went away (possibly due to a cluster layer
              * restart) and won't receive our client notification of any
              * monitors finalized below.
              */
             if (rsc->st_probe_rc == pcmk_ok) {
                 rsc->st_probe_rc = pcmk_err_generic;
             }
 
             if (rsc->active) {
                 cmd_list = g_list_append(cmd_list, rsc->active);
             }
             if (rsc->recurring_ops) {
                 cmd_list = g_list_concat(cmd_list, rsc->recurring_ops);
             }
             if (rsc->pending_ops) {
                 cmd_list = g_list_concat(cmd_list, rsc->pending_ops);
             }
             rsc->pending_ops = rsc->recurring_ops = NULL;
         }
     }
 
     if (!cmd_list) {
         return;
     }
 
     crm_err("Connection to fencer failed, finalizing %d pending operations",
             g_list_length(cmd_list));
     for (cmd_iter = cmd_list; cmd_iter; cmd_iter = cmd_iter->next) {
         stonith_action_complete(cmd_iter->data, -ENOTCONN);
     }
     g_list_free(cmd_list);
 }
 
 /*!
  * \internal
  * \brief Execute a stonith resource "start" action
  *
  * Start a stonith resource by registering it with the fencer.
  * (Stonith agents don't have a start command.)
  *
  * \param[in] stonith_api  Connection to fencer
  * \param[in] rsc          Stonith resource to start
  * \param[in] cmd          Start command to execute
  *
  * \return pcmk_ok on success, -errno otherwise
  */
 static int
 execd_stonith_start(stonith_t *stonith_api, lrmd_rsc_t *rsc, lrmd_cmd_t *cmd)
 {
     char *key = NULL;
     char *value = NULL;
     stonith_key_value_t *device_params = NULL;
     int rc = pcmk_ok;
 
     // Convert command parameters to stonith API key/values
     if (cmd->params) {
         GHashTableIter iter;
 
         g_hash_table_iter_init(&iter, cmd->params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             device_params = stonith_key_value_add(device_params, key, value);
         }
     }
 
     /* The fencer will automatically register devices via CIB notifications
      * when the CIB changes, but to avoid a possible race condition between
      * the fencer receiving the notification and the executor requesting that
      * resource, the executor registers the device as well. The fencer knows how
      * to handle duplicate registrations.
      */
     rc = stonith_api->cmds->register_device(stonith_api, st_opt_sync_call,
                                             cmd->rsc_id, rsc->provider,
                                             rsc->type, device_params);
 
     stonith_key_value_freeall(device_params, 1, 1);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Execute a stonith resource "stop" action
  *
  * Stop a stonith resource by unregistering it with the fencer.
  * (Stonith agents don't have a stop command.)
  *
  * \param[in] stonith_api  Connection to fencer
  * \param[in] rsc          Stonith resource to stop
  *
  * \return pcmk_ok on success, -errno otherwise
  */
 static inline int
 execd_stonith_stop(stonith_t *stonith_api, const lrmd_rsc_t *rsc)
 {
     /* @TODO Failure would indicate a problem communicating with fencer;
      * perhaps we should try reconnecting and retrying a few times?
      */
     return stonith_api->cmds->remove_device(stonith_api, st_opt_sync_call,
                                             rsc->rsc_id);
 }
 
 /*!
  * \internal
  * \brief Initiate a stonith resource agent recurring "monitor" action
  *
  * \param[in] stonith_api  Connection to fencer
  * \param[in] rsc          Stonith resource to monitor
  * \param[in] cmd          Monitor command being executed
  *
  * \return pcmk_ok if monitor was successfully initiated, -errno otherwise
  */
 static inline int
 execd_stonith_monitor(stonith_t *stonith_api, lrmd_rsc_t *rsc, lrmd_cmd_t *cmd)
 {
     int rc = stonith_api->cmds->monitor(stonith_api, 0, cmd->rsc_id,
                                         cmd->timeout / 1000);
 
     rc = stonith_api->cmds->register_callback(stonith_api, rc, 0, 0, cmd,
                                               "lrmd_stonith_callback",
                                               lrmd_stonith_callback);
     if (rc == TRUE) {
         rsc->active = cmd;
         rc = pcmk_ok;
     } else {
         rc = -pcmk_err_generic;
     }
     return rc;
 }
 
 static void
 lrmd_rsc_execute_stonith(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
 {
     int rc = 0;
     bool do_monitor = FALSE;
 
     stonith_t *stonith_api = get_stonith_connection();
 
     if (!stonith_api) {
         rc = -ENOTCONN;
 
     } else if (pcmk__str_eq(cmd->action, "start", pcmk__str_casei)) {
         rc = execd_stonith_start(stonith_api, rsc, cmd);
         if (rc == 0) {
             do_monitor = TRUE;
         }
 
     } else if (pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
         rc = execd_stonith_stop(stonith_api, rsc);
 
     } else if (pcmk__str_eq(cmd->action, "monitor", pcmk__str_casei)) {
         if (cmd->interval_ms > 0) {
             do_monitor = TRUE;
         } else {
             rc = rsc->st_probe_rc;
         }
     }
 
     if (do_monitor) {
         rc = execd_stonith_monitor(stonith_api, rsc, cmd);
         if (rc == pcmk_ok) {
             // Don't clean up yet, we will find out result of the monitor later
             return;
         }
     }
 
     stonith_action_complete(cmd, rc);
 }
 
 static int
 lrmd_rsc_execute_service_lib(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
 {
     svc_action_t *action = NULL;
     GHashTable *params_copy = NULL;
 
     CRM_ASSERT(rsc);
     CRM_ASSERT(cmd);
 
     crm_trace("Creating action, resource:%s action:%s class:%s provider:%s agent:%s",
               rsc->rsc_id, cmd->action, rsc->class, rsc->provider, rsc->type);
 
 #if SUPPORT_NAGIOS
     /* Recurring operations are cancelled anyway for a stop operation */
     if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_NAGIOS, pcmk__str_casei)
         && pcmk__str_eq(cmd->action, "stop", pcmk__str_casei)) {
 
         cmd->result.exit_status = PCMK_OCF_OK;
         goto exec_done;
     }
 #endif
 
     params_copy = pcmk__str_table_dup(cmd->params);
 
     action = services__create_resource_action(rsc->rsc_id, rsc->class, rsc->provider,
                                      rsc->type,
                                      normalize_action_name(rsc, cmd->action),
                                      cmd->interval_ms, cmd->timeout,
                                      params_copy, cmd->service_flags);
 
     if (!action) {
+        // Invalid arguments (which would be a bug) or out-of-memory
         crm_err("Failed to create action, action:%s on resource %s", cmd->action, rsc->rsc_id);
         pcmk__set_result(&(cmd->result), PCMK_OCF_UNKNOWN_ERROR,
-                         PCMK_EXEC_ERROR, NULL);
+                         PCMK_EXEC_ERROR, "Internal Pacemaker error");
         goto exec_done;
     }
 
     if (action->rc != PCMK_OCF_OK) {
         pcmk__set_result(&(cmd->result), action->rc, action->status, NULL);
         services_action_free(action);
         goto exec_done;
     }
 
     action->cb_data = cmd;
 
     /* 'cmd' may not be valid after this point if
      * services_action_async() returned TRUE
      *
      * Upstart and systemd both synchronously determine monitor/status
      * results and call action_complete (which may free 'cmd') if necessary.
      */
     if (services_action_async(action, action_complete)) {
         return TRUE;
     }
 
     if (action->status == PCMK_EXEC_DONE) {
         action->status = PCMK_EXEC_ERROR;
     }
     pcmk__set_result(&(cmd->result), action->rc, action->status, NULL);
     services_action_free(action);
     action = NULL;
 
   exec_done:
     cmd_finalize(cmd, rsc);
     return TRUE;
 }
 
 static gboolean
 lrmd_rsc_execute(lrmd_rsc_t * rsc)
 {
     lrmd_cmd_t *cmd = NULL;
 
     CRM_CHECK(rsc != NULL, return FALSE);
 
     if (rsc->active) {
         crm_trace("%s is still active", rsc->rsc_id);
         return TRUE;
     }
 
     if (rsc->pending_ops) {
         GList *first = rsc->pending_ops;
 
         cmd = first->data;
         if (cmd->delay_id) {
             crm_trace
                 ("Command %s %s was asked to run too early, waiting for start_delay timeout of %dms",
                  cmd->rsc_id, cmd->action, cmd->start_delay);
             return TRUE;
         }
         rsc->pending_ops = g_list_remove_link(rsc->pending_ops, first);
         g_list_free_1(first);
 
 #ifdef PCMK__TIME_USE_CGT
         get_current_time(&(cmd->t_run), &(cmd->t_first_run));
 #endif
         cmd->epoch_last_run = time(NULL);
     }
 
     if (!cmd) {
         crm_trace("Nothing further to do for %s", rsc->rsc_id);
         return TRUE;
     }
 
     rsc->active = cmd;          /* only one op at a time for a rsc */
     if (cmd->interval_ms) {
         rsc->recurring_ops = g_list_append(rsc->recurring_ops, cmd);
     }
 
     log_execute(cmd);
 
     if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
         lrmd_rsc_execute_stonith(rsc, cmd);
     } else {
         lrmd_rsc_execute_service_lib(rsc, cmd);
     }
 
     return TRUE;
 }
 
 static gboolean
 lrmd_rsc_dispatch(gpointer user_data)
 {
     return lrmd_rsc_execute(user_data);
 }
 
 void
 free_rsc(gpointer data)
 {
     GList *gIter = NULL;
     lrmd_rsc_t *rsc = data;
     int is_stonith = pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH,
                                   pcmk__str_casei);
 
     gIter = rsc->pending_ops;
     while (gIter != NULL) {
         GList *next = gIter->next;
         lrmd_cmd_t *cmd = gIter->data;
 
         /* command was never executed */
         cmd->result.exec_status = PCMK_EXEC_CANCELLED;
         cmd_finalize(cmd, NULL);
 
         gIter = next;
     }
     /* frees list, but not list elements. */
     g_list_free(rsc->pending_ops);
 
     gIter = rsc->recurring_ops;
     while (gIter != NULL) {
         GList *next = gIter->next;
         lrmd_cmd_t *cmd = gIter->data;
 
         if (is_stonith) {
             cmd->result.exec_status = PCMK_EXEC_CANCELLED;
             /* If a stonith command is in-flight, just mark it as cancelled;
              * it is not safe to finalize/free the cmd until the stonith api
              * says it has either completed or timed out.
              */
             if (rsc->active != cmd) {
                 cmd_finalize(cmd, NULL);
             }
         } else {
             /* This command is already handed off to service library,
              * let service library cancel it and tell us via the callback
              * when it is cancelled. The rsc can be safely destroyed
              * even if we are waiting for the cancel result */
             services_action_cancel(rsc->rsc_id,
                                    normalize_action_name(rsc, cmd->action),
                                    cmd->interval_ms);
         }
 
         gIter = next;
     }
     /* frees list, but not list elements. */
     g_list_free(rsc->recurring_ops);
 
     free(rsc->rsc_id);
     free(rsc->class);
     free(rsc->provider);
     free(rsc->type);
     mainloop_destroy_trigger(rsc->work);
 
     free(rsc);
 }
 
 static int
 process_lrmd_signon(pcmk__client_t *client, xmlNode *request, int call_id,
                     xmlNode **reply)
 {
     int rc = pcmk_ok;
     const char *is_ipc_provider = crm_element_value(request, F_LRMD_IS_IPC_PROVIDER);
     const char *protocol_version = crm_element_value(request, F_LRMD_PROTOCOL_VERSION);
 
     if (compare_version(protocol_version, LRMD_MIN_PROTOCOL_VERSION) < 0) {
         crm_err("Cluster API version must be greater than or equal to %s, not %s",
                 LRMD_MIN_PROTOCOL_VERSION, protocol_version);
         rc = -EPROTO;
     }
 
     if (crm_is_true(is_ipc_provider)) {
 #ifdef PCMK__COMPILE_REMOTE
         if ((client->remote != NULL) && client->remote->tls_handshake_complete) {
             // This is a remote connection from a cluster node's controller
             ipc_proxy_add_provider(client);
         } else {
             rc = -EACCES;
         }
 #else
         rc = -EPROTONOSUPPORT;
 #endif
     }
 
     *reply = create_lrmd_reply(__func__, rc, call_id);
     crm_xml_add(*reply, F_LRMD_OPERATION, CRM_OP_REGISTER);
     crm_xml_add(*reply, F_LRMD_CLIENTID, client->id);
     crm_xml_add(*reply, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
 
     return rc;
 }
 
 static int
 process_lrmd_rsc_register(pcmk__client_t *client, uint32_t id, xmlNode *request)
 {
     int rc = pcmk_ok;
     lrmd_rsc_t *rsc = build_rsc_from_xml(request);
     lrmd_rsc_t *dup = g_hash_table_lookup(rsc_list, rsc->rsc_id);
 
     if (dup &&
         pcmk__str_eq(rsc->class, dup->class, pcmk__str_casei) &&
         pcmk__str_eq(rsc->provider, dup->provider, pcmk__str_casei) && pcmk__str_eq(rsc->type, dup->type, pcmk__str_casei)) {
 
         crm_notice("Ignoring duplicate registration of '%s'", rsc->rsc_id);
         free_rsc(rsc);
         return rc;
     }
 
     g_hash_table_replace(rsc_list, rsc->rsc_id, rsc);
     crm_info("Cached agent information for '%s'", rsc->rsc_id);
     return rc;
 }
 
 static xmlNode *
 process_lrmd_get_rsc_info(xmlNode *request, int call_id)
 {
     int rc = pcmk_ok;
     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
     const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
     xmlNode *reply = NULL;
     lrmd_rsc_t *rsc = NULL;
 
     if (rsc_id == NULL) {
         rc = -ENODEV;
     } else {
         rsc = g_hash_table_lookup(rsc_list, rsc_id);
         if (rsc == NULL) {
             crm_info("Agent information for '%s' not in cache", rsc_id);
             rc = -ENODEV;
         }
     }
 
     reply = create_lrmd_reply(__func__, rc, call_id);
     if (rsc) {
         crm_xml_add(reply, F_LRMD_RSC_ID, rsc->rsc_id);
         crm_xml_add(reply, F_LRMD_CLASS, rsc->class);
         crm_xml_add(reply, F_LRMD_PROVIDER, rsc->provider);
         crm_xml_add(reply, F_LRMD_TYPE, rsc->type);
     }
     return reply;
 }
 
 static int
 process_lrmd_rsc_unregister(pcmk__client_t *client, uint32_t id,
                             xmlNode *request)
 {
     int rc = pcmk_ok;
     lrmd_rsc_t *rsc = NULL;
     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
     const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
 
     if (!rsc_id) {
         return -ENODEV;
     }
 
     rsc = g_hash_table_lookup(rsc_list, rsc_id);
     if (rsc == NULL) {
         crm_info("Ignoring unregistration of resource '%s', which is not registered",
                  rsc_id);
         return pcmk_ok;
     }
 
     if (rsc->active) {
         /* let the caller know there are still active ops on this rsc to watch for */
         crm_trace("Operation (0x%p) still in progress for unregistered resource %s",
                   rsc->active, rsc_id);
         rc = -EINPROGRESS;
     }
 
     g_hash_table_remove(rsc_list, rsc_id);
 
     return rc;
 }
 
 static int
 process_lrmd_rsc_exec(pcmk__client_t *client, uint32_t id, xmlNode *request)
 {
     lrmd_rsc_t *rsc = NULL;
     lrmd_cmd_t *cmd = NULL;
     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
     const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
     int call_id;
 
     if (!rsc_id) {
         return -EINVAL;
     }
     if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) {
         crm_info("Resource '%s' not found (%d active resources)",
                  rsc_id, g_hash_table_size(rsc_list));
         return -ENODEV;
     }
 
     cmd = create_lrmd_cmd(request, client);
     call_id = cmd->call_id;
 
     /* Don't reference cmd after handing it off to be scheduled.
      * The cmd could get merged and freed. */
     schedule_lrmd_cmd(rsc, cmd);
 
     return call_id;
 }
 
 static int
 cancel_op(const char *rsc_id, const char *action, guint interval_ms)
 {
     GList *gIter = NULL;
     lrmd_rsc_t *rsc = g_hash_table_lookup(rsc_list, rsc_id);
 
     /* How to cancel an action.
      * 1. Check pending ops list, if it hasn't been handed off
      *    to the service library or stonith recurring list remove
      *    it there and that will stop it.
      * 2. If it isn't in the pending ops list, then it's either a
      *    recurring op in the stonith recurring list, or the service
      *    library's recurring list.  Stop it there
      * 3. If not found in any lists, then this operation has either
      *    been executed already and is not a recurring operation, or
      *    never existed.
      */
     if (!rsc) {
         return -ENODEV;
     }
 
     for (gIter = rsc->pending_ops; gIter != NULL; gIter = gIter->next) {
         lrmd_cmd_t *cmd = gIter->data;
 
         if (action_matches(cmd, action, interval_ms)) {
             cmd->result.exec_status = PCMK_EXEC_CANCELLED;
             cmd_finalize(cmd, rsc);
             return pcmk_ok;
         }
     }
 
     if (pcmk__str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
         /* The service library does not handle stonith operations.
          * We have to handle recurring stonith operations ourselves. */
         for (gIter = rsc->recurring_ops; gIter != NULL; gIter = gIter->next) {
             lrmd_cmd_t *cmd = gIter->data;
 
             if (action_matches(cmd, action, interval_ms)) {
                 cmd->result.exec_status = PCMK_EXEC_CANCELLED;
                 if (rsc->active != cmd) {
                     cmd_finalize(cmd, rsc);
                 }
                 return pcmk_ok;
             }
         }
     } else if (services_action_cancel(rsc_id,
                                       normalize_action_name(rsc, action),
                                       interval_ms) == TRUE) {
         /* The service library will tell the action_complete callback function
          * this action was cancelled, which will destroy the cmd and remove
          * it from the recurring_op list. Do not do that in this function
          * if the service library says it cancelled it. */
         return pcmk_ok;
     }
 
     return -EOPNOTSUPP;
 }
 
 static void
 cancel_all_recurring(lrmd_rsc_t * rsc, const char *client_id)
 {
     GList *cmd_list = NULL;
     GList *cmd_iter = NULL;
 
     /* Notice a copy of each list is created when concat is called.
      * This prevents odd behavior from occurring when the cmd_list
      * is iterated through later on.  It is possible the cancel_op
      * function may end up modifying the recurring_ops and pending_ops
      * lists.  If we did not copy those lists, our cmd_list iteration
      * could get messed up.*/
     if (rsc->recurring_ops) {
         cmd_list = g_list_concat(cmd_list, g_list_copy(rsc->recurring_ops));
     }
     if (rsc->pending_ops) {
         cmd_list = g_list_concat(cmd_list, g_list_copy(rsc->pending_ops));
     }
     if (!cmd_list) {
         return;
     }
 
     for (cmd_iter = cmd_list; cmd_iter; cmd_iter = cmd_iter->next) {
         lrmd_cmd_t *cmd = cmd_iter->data;
 
         if (cmd->interval_ms == 0) {
             continue;
         }
 
         if (client_id && !pcmk__str_eq(cmd->client_id, client_id, pcmk__str_casei)) {
             continue;
         }
 
         cancel_op(rsc->rsc_id, cmd->action, cmd->interval_ms);
     }
     /* frees only the copied list data, not the cmds */
     g_list_free(cmd_list);
 }
 
 static int
 process_lrmd_rsc_cancel(pcmk__client_t *client, uint32_t id, xmlNode *request)
 {
     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
     const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
     const char *action = crm_element_value(rsc_xml, F_LRMD_RSC_ACTION);
     guint interval_ms = 0;
 
     crm_element_value_ms(rsc_xml, F_LRMD_RSC_INTERVAL, &interval_ms);
 
     if (!rsc_id || !action) {
         return -EINVAL;
     }
 
     return cancel_op(rsc_id, action, interval_ms);
 }
 
 static void
 add_recurring_op_xml(xmlNode *reply, lrmd_rsc_t *rsc)
 {
     xmlNode *rsc_xml = create_xml_node(reply, F_LRMD_RSC);
 
     crm_xml_add(rsc_xml, F_LRMD_RSC_ID, rsc->rsc_id);
     for (GList *item = rsc->recurring_ops; item != NULL; item = item->next) {
         lrmd_cmd_t *cmd = item->data;
         xmlNode *op_xml = create_xml_node(rsc_xml, T_LRMD_RSC_OP);
 
         crm_xml_add(op_xml, F_LRMD_RSC_ACTION,
                     (cmd->real_action? cmd->real_action : cmd->action));
         crm_xml_add_ms(op_xml, F_LRMD_RSC_INTERVAL, cmd->interval_ms);
         crm_xml_add_int(op_xml, F_LRMD_TIMEOUT, cmd->timeout_orig);
     }
 }
 
 static xmlNode *
 process_lrmd_get_recurring(xmlNode *request, int call_id)
 {
     int rc = pcmk_ok;
     const char *rsc_id = NULL;
     lrmd_rsc_t *rsc = NULL;
     xmlNode *reply = NULL;
     xmlNode *rsc_xml = NULL;
 
     // Resource ID is optional
     rsc_xml = first_named_child(request, F_LRMD_CALLDATA);
     if (rsc_xml) {
         rsc_xml = first_named_child(rsc_xml, F_LRMD_RSC);
     }
     if (rsc_xml) {
         rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
     }
 
     // If resource ID is specified, resource must exist
     if (rsc_id != NULL) {
         rsc = g_hash_table_lookup(rsc_list, rsc_id);
         if (rsc == NULL) {
             crm_info("Resource '%s' not found (%d active resources)",
                      rsc_id, g_hash_table_size(rsc_list));
             rc = -ENODEV;
         }
     }
 
     reply = create_lrmd_reply(__func__, rc, call_id);
 
     // If resource ID is not specified, check all resources
     if (rsc_id == NULL) {
         GHashTableIter iter;
         char *key = NULL;
 
         g_hash_table_iter_init(&iter, rsc_list);
         while (g_hash_table_iter_next(&iter, (gpointer *) &key,
                                       (gpointer *) &rsc)) {
             add_recurring_op_xml(reply, rsc);
         }
     } else if (rsc) {
         add_recurring_op_xml(reply, rsc);
     }
     return reply;
 }
 
 void
 process_lrmd_message(pcmk__client_t *client, uint32_t id, xmlNode *request)
 {
     int rc = pcmk_ok;
     int call_id = 0;
     const char *op = crm_element_value(request, F_LRMD_OPERATION);
     int do_reply = 0;
     int do_notify = 0;
     xmlNode *reply = NULL;
 
     /* Certain IPC commands may be done only by privileged users (i.e. root or
      * hacluster), because they would otherwise provide a means of bypassing
      * ACLs.
      */
     bool allowed = pcmk_is_set(client->flags, pcmk__client_privileged);
 
     crm_trace("Processing %s operation from %s", op, client->id);
     crm_element_value_int(request, F_LRMD_CALLID, &call_id);
 
     if (pcmk__str_eq(op, CRM_OP_IPC_FWD, pcmk__str_none)) {
 #ifdef PCMK__COMPILE_REMOTE
         if (allowed) {
             ipc_proxy_forward_client(client, request);
         } else {
             rc = -EACCES;
         }
 #else
         rc = -EPROTONOSUPPORT;
 #endif
         do_reply = 1;
     } else if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
         rc = process_lrmd_signon(client, request, call_id, &reply);
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_RSC_REG, pcmk__str_none)) {
         if (allowed) {
             rc = process_lrmd_rsc_register(client, id, request);
             do_notify = 1;
         } else {
             rc = -EACCES;
         }
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_RSC_INFO, pcmk__str_none)) {
         if (allowed) {
             reply = process_lrmd_get_rsc_info(request, call_id);
         } else {
             rc = -EACCES;
         }
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
         if (allowed) {
             rc = process_lrmd_rsc_unregister(client, id, request);
             /* don't notify anyone about failed un-registers */
             if (rc == pcmk_ok || rc == -EINPROGRESS) {
                 do_notify = 1;
             }
         } else {
             rc = -EACCES;
         }
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
         if (allowed) {
             rc = process_lrmd_rsc_exec(client, id, request);
         } else {
             rc = -EACCES;
         }
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_RSC_CANCEL, pcmk__str_none)) {
         if (allowed) {
             rc = process_lrmd_rsc_cancel(client, id, request);
         } else {
             rc = -EACCES;
         }
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_POKE, pcmk__str_none)) {
         do_notify = 1;
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_CHECK, pcmk__str_none)) {
         if (allowed) {
             xmlNode *data = get_message_xml(request, F_LRMD_CALLDATA);
 
             CRM_LOG_ASSERT(data != NULL);
             pcmk__valid_sbd_timeout(crm_element_value(data, F_LRMD_WATCHDOG));
         } else {
             rc = -EACCES;
         }
     } else if (pcmk__str_eq(op, LRMD_OP_ALERT_EXEC, pcmk__str_none)) {
         if (allowed) {
             rc = process_lrmd_alert_exec(client, id, request);
         } else {
             rc = -EACCES;
         }
         do_reply = 1;
     } else if (pcmk__str_eq(op, LRMD_OP_GET_RECURRING, pcmk__str_none)) {
         if (allowed) {
             reply = process_lrmd_get_recurring(request, call_id);
         } else {
             rc = -EACCES;
         }
         do_reply = 1;
     } else {
         rc = -EOPNOTSUPP;
         do_reply = 1;
         crm_err("Unknown IPC request '%s' from client %s",
                 op, pcmk__client_name(client));
     }
 
     if (rc == -EACCES) {
         crm_warn("Rejecting IPC request '%s' from unprivileged client %s",
                  op, pcmk__client_name(client));
     }
 
     crm_debug("Processed %s operation from %s: rc=%d, reply=%d, notify=%d",
               op, client->id, rc, do_reply, do_notify);
 
     if (do_reply) {
         int send_rc = pcmk_rc_ok;
 
         if (reply == NULL) {
             reply = create_lrmd_reply(__func__, rc, call_id);
         }
         send_rc = lrmd_server_send_reply(client, id, reply);
         free_xml(reply);
         if (send_rc != pcmk_rc_ok) {
             crm_warn("Reply to client %s failed: %s " CRM_XS " rc=%d",
                      pcmk__client_name(client), pcmk_rc_str(send_rc), send_rc);
         }
     }
 
     if (do_notify) {
         send_generic_notify(rc, request);
     }
 }
diff --git a/lib/pacemaker/pcmk_sched_transition.c b/lib/pacemaker/pcmk_sched_transition.c
index 5b1882e3dd..1862c9cd3c 100644
--- a/lib/pacemaker/pcmk_sched_transition.c
+++ b/lib/pacemaker/pcmk_sched_transition.c
@@ -1,850 +1,850 @@
 /*
  * Copyright 2009-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <unistd.h>
 #include <stdlib.h>
 
 #include <sys/stat.h>
 #include <sys/param.h>
 #include <sys/types.h>
 #include <dirent.h>
 
 #include <crm/crm.h>
 #include <crm/lrmd.h>           // lrmd_event_data_t, lrmd_free_event()
 #include <crm/cib.h>
 #include <crm/common/util.h>
 #include <crm/common/iso8601.h>
 #include <crm/common/xml_internal.h>
 #include <crm/lrmd_internal.h>
 #include <crm/pengine/status.h>
 #include <pacemaker-internal.h>
 
 static pcmk__output_t *out = NULL;
 static cib_t *fake_cib = NULL;
 static GList *fake_resource_list = NULL;
 static GList *fake_op_fail_list = NULL;
 gboolean bringing_nodes_online = FALSE;
 
 #define STATUS_PATH_MAX 512
 
 #define NEW_NODE_TEMPLATE "//"XML_CIB_TAG_NODE"[@uname='%s']"
 #define NODE_TEMPLATE "//"XML_CIB_TAG_STATE"[@uname='%s']"
 #define RSC_TEMPLATE "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']"
 
 
 static void
 inject_transient_attr(xmlNode * cib_node, const char *name, const char *value)
 {
     xmlNode *attrs = NULL;
     xmlNode *instance_attrs = NULL;
     const char *node_uuid = ID(cib_node);
 
     out->message(out, "inject-attr", name, value, cib_node);
 
     attrs = first_named_child(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
     if (attrs == NULL) {
         attrs = create_xml_node(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
         crm_xml_add(attrs, XML_ATTR_ID, node_uuid);
     }
 
     instance_attrs = first_named_child(attrs, XML_TAG_ATTR_SETS);
     if (instance_attrs == NULL) {
         instance_attrs = create_xml_node(attrs, XML_TAG_ATTR_SETS);
         crm_xml_add(instance_attrs, XML_ATTR_ID, node_uuid);
     }
 
     crm_create_nvpair_xml(instance_attrs, NULL, name, value);
 }
 
 static void
 update_failcounts(xmlNode * cib_node, const char *resource, const char *task,
                   guint interval_ms, int rc)
 {
     if (rc == 0) {
         return;
 
     } else if ((rc == 7) && (interval_ms == 0)) {
         return;
 
     } else {
         char *name = NULL;
         char *now = pcmk__ttoa(time(NULL));
 
         name = pcmk__failcount_name(resource, task, interval_ms);
         inject_transient_attr(cib_node, name, "value++");
         free(name);
 
         name = pcmk__lastfailure_name(resource, task, interval_ms);
         inject_transient_attr(cib_node, name, now);
         free(name);
         free(now);
     }
 }
 
 static void
 create_node_entry(cib_t * cib_conn, const char *node)
 {
     int rc = pcmk_ok;
     char *xpath = crm_strdup_printf(NEW_NODE_TEMPLATE, node);
 
     rc = cib_conn->cmds->query(cib_conn, xpath, NULL, cib_xpath | cib_sync_call | cib_scope_local);
 
     if (rc == -ENXIO) {
         xmlNode *cib_object = create_xml_node(NULL, XML_CIB_TAG_NODE);
 
         crm_xml_add(cib_object, XML_ATTR_ID, node); // Use node name as ID
         crm_xml_add(cib_object, XML_ATTR_UNAME, node);
         cib_conn->cmds->create(cib_conn, XML_CIB_TAG_NODES, cib_object,
                                cib_sync_call | cib_scope_local);
         /* Not bothering with subsequent query to see if it exists,
            we'll bomb out later in the call to query_node_uuid()... */
 
         free_xml(cib_object);
     }
 
     free(xpath);
 }
 
 static lrmd_event_data_t *
 create_op(xmlNode *cib_resource, const char *task, guint interval_ms,
           int outcome)
 {
     lrmd_event_data_t *op = NULL;
     xmlNode *xop = NULL;
 
     op = lrmd_new_event(ID(cib_resource), task, interval_ms);
-    lrmd__set_result(op, outcome, PCMK_EXEC_DONE, NULL);
+    lrmd__set_result(op, outcome, PCMK_EXEC_DONE, "Simulated action result");
     op->params = NULL;          /* TODO: Fill me in */
     op->t_run = (unsigned int) time(NULL);
     op->t_rcchange = op->t_run;
 
     op->call_id = 0;
     for (xop = pcmk__xe_first_child(cib_resource); xop != NULL;
          xop = pcmk__xe_next(xop)) {
 
         int tmp = 0;
 
         crm_element_value_int(xop, XML_LRM_ATTR_CALLID, &tmp);
         if (tmp > op->call_id) {
             op->call_id = tmp;
         }
     }
     op->call_id++;
 
     return op;
 }
 
 static xmlNode *
 inject_op(xmlNode * cib_resource, lrmd_event_data_t * op, int target_rc)
 {
     return pcmk__create_history_xml(cib_resource, op, CRM_FEATURE_SET,
                                     target_rc, NULL, crm_system_name,
                                     LOG_TRACE);
 }
 
 static xmlNode *
 inject_node_state(cib_t * cib_conn, const char *node, const char *uuid)
 {
     int rc = pcmk_ok;
     xmlNode *cib_object = NULL;
     char *xpath = crm_strdup_printf(NODE_TEMPLATE, node);
 
     if (bringing_nodes_online) {
         create_node_entry(cib_conn, node);
     }
 
     rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
                                cib_xpath | cib_sync_call | cib_scope_local);
 
     if (cib_object && ID(cib_object) == NULL) {
         crm_err("Detected multiple node_state entries for xpath=%s, bailing", xpath);
         crm_log_xml_warn(cib_object, "Duplicates");
         free(xpath);
         crm_exit(CRM_EX_SOFTWARE);
         return NULL; // not reached, but makes static analysis happy
     }
 
     if (rc == -ENXIO) {
         char *found_uuid = NULL;
 
         if (uuid == NULL) {
             query_node_uuid(cib_conn, node, &found_uuid, NULL);
         } else {
             found_uuid = strdup(uuid);
         }
 
         cib_object = create_xml_node(NULL, XML_CIB_TAG_STATE);
         crm_xml_add(cib_object, XML_ATTR_UUID, found_uuid);
         crm_xml_add(cib_object, XML_ATTR_UNAME, node);
         cib_conn->cmds->create(cib_conn, XML_CIB_TAG_STATUS, cib_object,
                                cib_sync_call | cib_scope_local);
         free_xml(cib_object);
         free(found_uuid);
 
         rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
                                    cib_xpath | cib_sync_call | cib_scope_local);
         crm_trace("injecting node state for %s. rc is %d", node, rc);
     }
 
     free(xpath);
     CRM_ASSERT(rc == pcmk_ok);
     return cib_object;
 }
 
 static xmlNode *
 modify_node(cib_t * cib_conn, char *node, gboolean up)
 {
     xmlNode *cib_node = inject_node_state(cib_conn, node, NULL);
 
     if (up) {
         crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_YES);
         crm_xml_add(cib_node, XML_NODE_IS_PEER, ONLINESTATUS);
         crm_xml_add(cib_node, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_MEMBER);
         crm_xml_add(cib_node, XML_NODE_EXPECTED, CRMD_JOINSTATE_MEMBER);
 
     } else {
         crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO);
         crm_xml_add(cib_node, XML_NODE_IS_PEER, OFFLINESTATUS);
         crm_xml_add(cib_node, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_DOWN);
         crm_xml_add(cib_node, XML_NODE_EXPECTED, CRMD_JOINSTATE_DOWN);
     }
 
     crm_xml_add(cib_node, XML_ATTR_ORIGIN, crm_system_name);
     return cib_node;
 }
 
 static xmlNode *
 find_resource_xml(xmlNode * cib_node, const char *resource)
 {
     xmlNode *match = NULL;
     const char *node = crm_element_value(cib_node, XML_ATTR_UNAME);
     char *xpath = crm_strdup_printf(RSC_TEMPLATE, node, resource);
 
     match = get_xpath_object(xpath, cib_node, LOG_TRACE);
     free(xpath);
     return match;
 }
 
 
 static xmlNode *
 inject_resource(xmlNode * cib_node, const char *resource, const char *lrm_name,
                 const char *rclass, const char *rtype, const char *rprovider)
 {
     xmlNode *lrm = NULL;
     xmlNode *container = NULL;
     xmlNode *cib_resource = NULL;
     char *xpath = NULL;
 
     cib_resource = find_resource_xml(cib_node, resource);
     if (cib_resource != NULL) {
         /* If an existing LRM history entry uses the resource name,
          * continue using it, even if lrm_name is different.
          */
         return cib_resource;
     }
 
     // Check for history entry under preferred name
     if (strcmp(resource, lrm_name)) {
         cib_resource = find_resource_xml(cib_node, lrm_name);
         if (cib_resource != NULL) {
             return cib_resource;
         }
     }
 
     /* One day, add query for class, provider, type */
 
     if (rclass == NULL || rtype == NULL) {
         out->err(out, "Resource %s not found in the status section of %s."
                  "  Please supply the class and type to continue", resource, ID(cib_node));
         return NULL;
 
     } else if (!pcmk__strcase_any_of(rclass, PCMK_RESOURCE_CLASS_OCF, PCMK_RESOURCE_CLASS_STONITH,
                                      PCMK_RESOURCE_CLASS_SERVICE, PCMK_RESOURCE_CLASS_UPSTART,
                                      PCMK_RESOURCE_CLASS_SYSTEMD, PCMK_RESOURCE_CLASS_LSB, NULL)) {
         out->err(out, "Invalid class for %s: %s", resource, rclass);
         return NULL;
 
     } else if (pcmk_is_set(pcmk_get_ra_caps(rclass), pcmk_ra_cap_provider)
                 && (rprovider == NULL)) {
         out->err(out, "Please specify the provider for resource %s", resource);
         return NULL;
     }
 
     xpath = (char *)xmlGetNodePath(cib_node);
     crm_info("Injecting new resource %s into %s '%s'", lrm_name, xpath, ID(cib_node));
     free(xpath);
 
     lrm = first_named_child(cib_node, XML_CIB_TAG_LRM);
     if (lrm == NULL) {
         const char *node_uuid = ID(cib_node);
 
         lrm = create_xml_node(cib_node, XML_CIB_TAG_LRM);
         crm_xml_add(lrm, XML_ATTR_ID, node_uuid);
     }
 
     container = first_named_child(lrm, XML_LRM_TAG_RESOURCES);
     if (container == NULL) {
         container = create_xml_node(lrm, XML_LRM_TAG_RESOURCES);
     }
 
     cib_resource = create_xml_node(container, XML_LRM_TAG_RESOURCE);
 
     // If we're creating a new entry, use the preferred name
     crm_xml_add(cib_resource, XML_ATTR_ID, lrm_name);
 
     crm_xml_add(cib_resource, XML_AGENT_ATTR_CLASS, rclass);
     crm_xml_add(cib_resource, XML_AGENT_ATTR_PROVIDER, rprovider);
     crm_xml_add(cib_resource, XML_ATTR_TYPE, rtype);
 
     return cib_resource;
 }
 
 #define XPATH_MAX 1024
 
 static int
 find_ticket_state(cib_t * the_cib, const char *ticket_id, xmlNode ** ticket_state_xml)
 {
     int offset = 0;
     int rc = pcmk_ok;
     xmlNode *xml_search = NULL;
 
     char *xpath_string = NULL;
 
     CRM_ASSERT(ticket_state_xml != NULL);
     *ticket_state_xml = NULL;
 
     xpath_string = calloc(1, XPATH_MAX);
     offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "%s", "/cib/status/tickets");
 
     if (ticket_id) {
         offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "/%s[@id=\"%s\"]",
                            XML_CIB_TAG_TICKET_STATE, ticket_id);
     }
     CRM_LOG_ASSERT(offset > 0);
     rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
                               cib_sync_call | cib_scope_local | cib_xpath);
 
     if (rc != pcmk_ok) {
         goto bail;
     }
 
     crm_log_xml_debug(xml_search, "Match");
     if (xml_has_children(xml_search)) {
         if (ticket_id) {
             out->err(out, "Multiple ticket_states match ticket_id=%s", ticket_id);
         }
         *ticket_state_xml = xml_search;
     } else {
         *ticket_state_xml = xml_search;
     }
 
   bail:
     free(xpath_string);
     return rc;
 }
 
 static int
 set_ticket_state_attr(const char *ticket_id, const char *attr_name,
                       const char *attr_value, cib_t * cib, int cib_options)
 {
     int rc = pcmk_ok;
     xmlNode *xml_top = NULL;
     xmlNode *ticket_state_xml = NULL;
 
     rc = find_ticket_state(cib, ticket_id, &ticket_state_xml);
     if (rc == pcmk_ok) {
         crm_debug("Found a match state for ticket: id=%s", ticket_id);
         xml_top = ticket_state_xml;
 
     } else if (rc != -ENXIO) {
         return rc;
 
     } else {
         xmlNode *xml_obj = NULL;
 
         xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
         xml_obj = create_xml_node(xml_top, XML_CIB_TAG_TICKETS);
         ticket_state_xml = create_xml_node(xml_obj, XML_CIB_TAG_TICKET_STATE);
         crm_xml_add(ticket_state_xml, XML_ATTR_ID, ticket_id);
     }
 
     crm_xml_add(ticket_state_xml, attr_name, attr_value);
 
     crm_log_xml_debug(xml_top, "Update");
 
     rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, xml_top, cib_options);
 
     free_xml(xml_top);
 
     return rc;
 }
 
 void
 modify_configuration(pe_working_set_t * data_set, cib_t *cib, pcmk_injections_t *injections)
 {
     int rc = pcmk_ok;
     GList *gIter = NULL;
 
     xmlNode *cib_op = NULL;
     xmlNode *cib_node = NULL;
     xmlNode *cib_resource = NULL;
 
     lrmd_event_data_t *op = NULL;
 
     out = data_set->priv;
 
     out->message(out, "inject-modify-config", injections->quorum, injections->watchdog);
 
     if (injections->quorum) {
         xmlNode *top = create_xml_node(NULL, XML_TAG_CIB);
 
         /* crm_xml_add(top, XML_ATTR_DC_UUID, dc_uuid);      */
         crm_xml_add(top, XML_ATTR_HAVE_QUORUM, injections->quorum);
 
         rc = cib->cmds->modify(cib, NULL, top, cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     if (injections->watchdog) {
         rc = update_attr_delegate(cib, cib_sync_call | cib_scope_local,
                              XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
                              XML_ATTR_HAVE_WATCHDOG, injections->watchdog, FALSE, NULL, NULL);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = injections->node_up; gIter != NULL; gIter = gIter->next) {
         char *node = (char *)gIter->data;
 
         out->message(out, "inject-modify-node", "Online", node);
 
         cib_node = modify_node(cib, node, TRUE);
         CRM_ASSERT(cib_node != NULL);
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                                       cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
         free_xml(cib_node);
     }
 
     for (gIter = injections->node_down; gIter != NULL; gIter = gIter->next) {
         char xpath[STATUS_PATH_MAX];
         char *node = (char *)gIter->data;
 
         out->message(out, "inject-modify-node", "Offline", node);
 
         cib_node = modify_node(cib, node, FALSE);
         CRM_ASSERT(cib_node != NULL);
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                                       cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
         free_xml(cib_node);
 
         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", node, XML_CIB_TAG_LRM);
         cib->cmds->remove(cib, xpath, NULL,
                                       cib_xpath | cib_sync_call | cib_scope_local);
 
         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", node,
                  XML_TAG_TRANSIENT_NODEATTRS);
         cib->cmds->remove(cib, xpath, NULL,
                                       cib_xpath | cib_sync_call | cib_scope_local);
 
     }
 
     for (gIter = injections->node_fail; gIter != NULL; gIter = gIter->next) {
         char *node = (char *)gIter->data;
 
         out->message(out, "inject-modify-node", "Failing", node);
 
         cib_node = modify_node(cib, node, TRUE);
         crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO);
         CRM_ASSERT(cib_node != NULL);
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                                       cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
         free_xml(cib_node);
     }
 
     for (gIter = injections->ticket_grant; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         out->message(out, "inject-modify-ticket", "Granting", ticket_id);
 
         rc = set_ticket_state_attr(ticket_id, "granted", "true",
                                    cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = injections->ticket_revoke; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         out->message(out, "inject-modify-ticket", "Revoking", ticket_id);
 
         rc = set_ticket_state_attr(ticket_id, "granted", "false",
                                    cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = injections->ticket_standby; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         out->message(out, "inject-modify-ticket", "Standby", ticket_id);
 
         rc = set_ticket_state_attr(ticket_id, "standby", "true",
                                    cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = injections->ticket_activate; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         out->message(out, "inject-modify-ticket", "Activating", ticket_id);
 
         rc = set_ticket_state_attr(ticket_id, "standby", "false",
                                    cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = injections->op_inject; gIter != NULL; gIter = gIter->next) {
         char *spec = (char *)gIter->data;
 
         int rc = 0;
         int outcome = PCMK_OCF_OK;
         guint interval_ms = 0;
 
         char *key = NULL;
         char *node = NULL;
         char *task = NULL;
         char *resource = NULL;
 
         const char *rtype = NULL;
         const char *rclass = NULL;
         const char *rprovider = NULL;
 
         pe_resource_t *rsc = NULL;
 
         out->message(out, "inject-spec", spec);
 
         key = calloc(1, strlen(spec) + 1);
         node = calloc(1, strlen(spec) + 1);
         rc = sscanf(spec, "%[^@]@%[^=]=%d", key, node, &outcome);
         if (rc != 3) {
             out->err(out, "Invalid operation spec: %s.  Only found %d fields", spec, rc);
             free(key);
             free(node);
             continue;
         }
 
         parse_op_key(key, &resource, &task, &interval_ms);
 
         rsc = pe_find_resource(data_set->resources, resource);
         if (rsc == NULL) {
             out->err(out, "Invalid resource name: %s", resource);
         } else {
             rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
             rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
             rprovider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
 
             cib_node = inject_node_state(cib, node, NULL);
             CRM_ASSERT(cib_node != NULL);
 
             update_failcounts(cib_node, resource, task, interval_ms, outcome);
 
             cib_resource = inject_resource(cib_node, resource, resource,
                                            rclass, rtype, rprovider);
             CRM_ASSERT(cib_resource != NULL);
 
             op = create_op(cib_resource, task, interval_ms, outcome);
             CRM_ASSERT(op != NULL);
 
             cib_op = inject_op(cib_resource, op, 0);
             CRM_ASSERT(cib_op != NULL);
             lrmd_free_event(op);
 
             rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                                           cib_sync_call | cib_scope_local);
             CRM_ASSERT(rc == pcmk_ok);
         }
         free(task);
         free(node);
         free(key);
     }
 
     if (!out->is_quiet(out)) {
         out->end_list(out);
     }
 }
 
 static gboolean
 exec_pseudo_action(crm_graph_t * graph, crm_action_t * action)
 {
     const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 
     action->confirmed = TRUE;
     out->message(out, "inject-pseudo-action", node, task);
 
     pcmk__update_graph(graph, action);
     return TRUE;
 }
 
 static gboolean
 exec_rsc_action(crm_graph_t * graph, crm_action_t * action)
 {
     int rc = 0;
     GList *gIter = NULL;
     lrmd_event_data_t *op = NULL;
     int target_outcome = PCMK_OCF_OK;
 
     const char *rtype = NULL;
     const char *rclass = NULL;
     const char *resource = NULL;
     const char *rprovider = NULL;
     const char *lrm_name = NULL;
     const char *operation = crm_element_value(action->xml, "operation");
     const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);
 
     xmlNode *cib_node = NULL;
     xmlNode *cib_resource = NULL;
     xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
 
     char *node = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
     char *uuid = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET_UUID);
     const char *router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 
     if (pcmk__strcase_any_of(operation, CRM_OP_PROBED, CRM_OP_REPROBE, NULL)) {
         crm_info("Skipping %s op for %s", operation, node);
         goto done;
     }
 
     if (action_rsc == NULL) {
         crm_log_xml_err(action->xml, "Bad");
         free(node); free(uuid);
         return FALSE;
     }
 
     /* Look for the preferred name
      * If not found, try the expected 'local' name
      * If not found use the preferred name anyway
      */
     resource = crm_element_value(action_rsc, XML_ATTR_ID);
     CRM_ASSERT(resource != NULL); // makes static analysis happy
     lrm_name = resource; // Preferred name when writing history
     if (pe_find_resource(fake_resource_list, resource) == NULL) {
         const char *longname = crm_element_value(action_rsc, XML_ATTR_ID_LONG);
 
         if (longname && pe_find_resource(fake_resource_list, longname)) {
             resource = longname;
         }
     }
 
     if (pcmk__strcase_any_of(operation, "delete", RSC_METADATA, NULL)) {
         out->message(out, "inject-rsc-action", resource, operation, node, (guint) 0);
         goto done;
     }
 
     rclass = crm_element_value(action_rsc, XML_AGENT_ATTR_CLASS);
     rtype = crm_element_value(action_rsc, XML_ATTR_TYPE);
     rprovider = crm_element_value(action_rsc, XML_AGENT_ATTR_PROVIDER);
 
     pcmk__scan_min_int(target_rc_s, &target_outcome, 0);
 
     CRM_ASSERT(fake_cib->cmds->query(fake_cib, NULL, NULL, cib_sync_call | cib_scope_local) ==
                pcmk_ok);
 
     cib_node = inject_node_state(fake_cib, node, (router_node? node : uuid));
     CRM_ASSERT(cib_node != NULL);
 
     cib_resource = inject_resource(cib_node, resource, lrm_name,
                                    rclass, rtype, rprovider);
     if (cib_resource == NULL) {
         crm_err("invalid resource in transition");
         free(node); free(uuid);
         free_xml(cib_node);
         return FALSE;
     }
 
     op = pcmk__event_from_graph_action(cib_resource, action, PCMK_EXEC_DONE,
                                        target_outcome);
 
     out->message(out, "inject-rsc-action", resource, op->op_type, node, op->interval_ms);
 
     for (gIter = fake_op_fail_list; gIter != NULL; gIter = gIter->next) {
         char *spec = (char *)gIter->data;
         char *key = NULL;
         const char *match_name = NULL;
 
         // Allow user to specify anonymous clone with or without instance number
         key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource, op->op_type,
                                 op->interval_ms, node);
         if (strncasecmp(key, spec, strlen(key)) == 0) {
             match_name = resource;
         }
         free(key);
 
         if ((match_name == NULL) && strcmp(resource, lrm_name)) {
             key = crm_strdup_printf(PCMK__OP_FMT "@%s=", lrm_name, op->op_type,
                                     op->interval_ms, node);
             if (strncasecmp(key, spec, strlen(key)) == 0) {
                 match_name = lrm_name;
             }
             free(key);
         }
 
         if (match_name != NULL) {
 
             rc = sscanf(spec, "%*[^=]=%d", (int *) &op->rc);
             // ${match_name}_${task}_${interval_in_ms}@${node}=${rc}
 
             if (rc != 1) {
                 out->err(out,
                          "Invalid failed operation spec: %s. Result code must be integer",
                          spec);
                 continue;
             }
             action->failed = TRUE;
             graph->abort_priority = INFINITY;
             out->info(out, "Pretending action %d failed with rc=%d", action->id, op->rc);
             update_failcounts(cib_node, match_name, op->op_type,
                               op->interval_ms, op->rc);
             break;
         }
     }
 
     inject_op(cib_resource, op, target_outcome);
     lrmd_free_event(op);
 
     rc = fake_cib->cmds->modify(fake_cib, XML_CIB_TAG_STATUS, cib_node,
                                   cib_sync_call | cib_scope_local);
     CRM_ASSERT(rc == pcmk_ok);
 
   done:
     free(node); free(uuid);
     free_xml(cib_node);
     action->confirmed = TRUE;
     pcmk__update_graph(graph, action);
     return TRUE;
 }
 
 static gboolean
 exec_crmd_action(crm_graph_t * graph, crm_action_t * action)
 {
     const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     xmlNode *rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
 
     action->confirmed = TRUE;
     out->message(out, "inject-cluster-action", node, task, rsc);
     pcmk__update_graph(graph, action);
     return TRUE;
 }
 
 static gboolean
 exec_stonith_action(crm_graph_t * graph, crm_action_t * action)
 {
     const char *op = crm_meta_value(action->params, "stonith_action");
     char *target = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
 
     out->message(out, "inject-fencing-action", target, op);
 
     if(!pcmk__str_eq(op, "on", pcmk__str_casei)) {
         int rc = 0;
         char xpath[STATUS_PATH_MAX];
         xmlNode *cib_node = modify_node(fake_cib, target, FALSE);
 
         crm_xml_add(cib_node, XML_ATTR_ORIGIN, __func__);
         CRM_ASSERT(cib_node != NULL);
 
         rc = fake_cib->cmds->replace(fake_cib, XML_CIB_TAG_STATUS, cib_node,
                                    cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
 
         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target, XML_CIB_TAG_LRM);
         fake_cib->cmds->remove(fake_cib, xpath, NULL,
                                       cib_xpath | cib_sync_call | cib_scope_local);
 
         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target,
                  XML_TAG_TRANSIENT_NODEATTRS);
         fake_cib->cmds->remove(fake_cib, xpath, NULL,
                                       cib_xpath | cib_sync_call | cib_scope_local);
 
         free_xml(cib_node);
     }
 
     action->confirmed = TRUE;
     pcmk__update_graph(graph, action);
     free(target);
     return TRUE;
 }
 
 enum transition_status
 run_simulation(pe_working_set_t * data_set, cib_t *cib, GList *op_fail_list)
 {
     crm_graph_t *transition = NULL;
     enum transition_status graph_rc;
 
     crm_graph_functions_t exec_fns = {
         exec_pseudo_action,
         exec_rsc_action,
         exec_crmd_action,
         exec_stonith_action,
     };
 
     out = data_set->priv;
 
     fake_cib = cib;
     fake_op_fail_list = op_fail_list;
 
     if (!out->is_quiet(out)) {
         out->begin_list(out, NULL, NULL, "Executing Cluster Transition");
     }
 
     pcmk__set_graph_functions(&exec_fns);
     transition = pcmk__unpack_graph(data_set->graph, crm_system_name);
     pcmk__log_graph(LOG_DEBUG, transition);
 
     fake_resource_list = data_set->resources;
     do {
         graph_rc = pcmk__execute_graph(transition);
 
     } while (graph_rc == transition_active);
     fake_resource_list = NULL;
 
     if (graph_rc != transition_complete) {
         out->err(out, "Transition failed: %s",
                  pcmk__graph_status2text(graph_rc));
         pcmk__log_graph(LOG_ERR, transition);
     }
     pcmk__free_graph(transition);
     if (graph_rc != transition_complete) {
         out->err(out, "An invalid transition was produced");
     }
 
     if (!out->is_quiet(out)) {
         xmlNode *cib_object = NULL;
         int rc = fake_cib->cmds->query(fake_cib, NULL, &cib_object, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
         pe_reset_working_set(data_set);
         data_set->input = cib_object;
 
         out->end_list(out);
     }
 
     return graph_rc;
 }