diff --git a/crmd/lrm.c b/crmd/lrm.c
index 2d4da98a6a..5b412d5d11 100644
--- a/crmd/lrm.c
+++ b/crmd/lrm.c
@@ -1,2670 +1,2685 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 
 #include <crm/crm.h>
 #include <crm/services.h>
 
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crmd.h>
 #include <crmd_fsa.h>
 #include <crmd_messages.h>
 #include <crmd_callbacks.h>
 #include <crmd_lrm.h>
 #include <regex.h>
 #include <crm/pengine/rules.h>
 
 #define START_DELAY_THRESHOLD 5 * 60 * 1000
 #define MAX_LRM_REG_FAILS 30
 
 #define s_if_plural(i) (((i) == 1)? "" : "s")
 
 struct delete_event_s {
     int rc;
     const char *rsc;
     lrm_state_t *lrm_state;
 };
 
 static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id);
 static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list);
 static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
 static int delete_rsc_status(lrm_state_t * lrm_state, const char *rsc_id, int call_options,
                              const char *user_name);
 
 static lrmd_event_data_t *construct_op(lrm_state_t * lrm_state, xmlNode * rsc_op,
                                        const char *rsc_id, const char *operation);
 static void do_lrm_rsc_op(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *operation,
                           xmlNode * msg, xmlNode * request);
 
 void send_direct_ack(const char *to_host, const char *to_sys,
                      lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id);
 
 static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state,
                                          int log_level);
 static int do_update_resource(const char *node_name, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op);
 
 static void
 lrm_connection_destroy(void)
 {
     if (is_set(fsa_input_register, R_LRM_CONNECTED)) {
         crm_crit("LRM Connection failed");
         register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
         clear_bit(fsa_input_register, R_LRM_CONNECTED);
 
     } else {
         crm_info("LRM Connection disconnected");
     }
 
 }
 
 static char *
 make_stop_id(const char *rsc, int call_id)
 {
     return crm_strdup_printf("%s:%d", rsc, call_id);
 }
 
 static void
 copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
 {
     if (strstr(key, CRM_META "_") == NULL) {
         g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
     }
 }
 
 static void
 copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
 {
     if (strstr(key, CRM_META "_") != NULL) {
         g_hash_table_replace(user_data, strdup((const char *)key), strdup((const char *)value));
     }
 }
 
 /*!
  * \internal
  * \brief Remove a recurring operation from a resource's history
  *
  * \param[in,out] history  Resource history to modify
  * \param[in]     op       Operation to remove
  *
  * \return TRUE if the operation was found and removed, FALSE otherwise
  */
 static gboolean
 history_remove_recurring_op(rsc_history_t *history, const lrmd_event_data_t *op)
 {
     GList *iter;
 
     for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
         lrmd_event_data_t *existing = iter->data;
 
-        if ((op->interval == existing->interval)
+        if ((op->interval_ms == existing->interval_ms)
             && crm_str_eq(op->rsc_id, existing->rsc_id, TRUE)
             && safe_str_eq(op->op_type, existing->op_type)) {
 
             history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter);
             lrmd_free_event(existing);
             return TRUE;
         }
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Free all recurring operations in resource history
  *
  * \param[in,out] history  Resource history to modify
  */
 static void
 history_free_recurring_ops(rsc_history_t *history)
 {
     GList *iter;
 
     for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
         lrmd_free_event(iter->data);
     }
     g_list_free(history->recurring_op_list);
     history->recurring_op_list = NULL;
 }
 
 /*!
  * \internal
  * \brief Free resource history
  *
  * \param[in,out] history  Resource history to free
  */
 void
 history_free(gpointer data)
 {
     rsc_history_t *history = (rsc_history_t*)data;
 
     if (history->stop_params) {
         g_hash_table_destroy(history->stop_params);
     }
 
     /* Don't need to free history->rsc.id because it's set to history->id */
     free(history->rsc.type);
     free(history->rsc.standard);
     free(history->rsc.provider);
 
     lrmd_free_event(history->failed);
     lrmd_free_event(history->last);
     free(history->id);
     history_free_recurring_ops(history);
     free(history);
 }
 
 static void
 update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
 {
     int target_rc = 0;
     rsc_history_t *entry = NULL;
 
     if (op->rsc_deleted) {
         crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
         delete_rsc_status(lrm_state, op->rsc_id, cib_quorum_override, NULL);
         return;
     }
 
     if (safe_str_eq(op->op_type, RSC_NOTIFY)) {
         return;
     }
 
     crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);
 
     entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
     if (entry == NULL && rsc) {
         entry = calloc(1, sizeof(rsc_history_t));
         entry->id = strdup(op->rsc_id);
         g_hash_table_insert(lrm_state->resource_history, entry->id, entry);
 
         entry->rsc.id = entry->id;
         entry->rsc.type = strdup(rsc->type);
         entry->rsc.standard = strdup(rsc->standard);
         if (rsc->provider) {
             entry->rsc.provider = strdup(rsc->provider);
         } else {
             entry->rsc.provider = NULL;
         }
 
     } else if (entry == NULL) {
         crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
         return;
     }
 
     entry->last_callid = op->call_id;
     target_rc = rsc_op_expected_rc(op);
     if (op->op_status == PCMK_LRM_OP_CANCELLED) {
-        if (op->interval > 0) {
-            crm_trace("Removing cancelled recurring op: %s_%s_%d", op->rsc_id, op->op_type,
-                      op->interval);
+        if (op->interval_ms > 0) {
+            crm_trace("Removing cancelled recurring op: " CRM_OP_FMT,
+                      op->rsc_id, op->op_type, op->interval_ms);
             history_remove_recurring_op(entry, op);
             return;
         } else {
-            crm_trace("Skipping %s_%s_%d rc=%d, status=%d", op->rsc_id, op->op_type, op->interval,
-                      op->rc, op->op_status);
+            crm_trace("Skipping " CRM_OP_FMT " rc=%d, status=%d",
+                      op->rsc_id, op->op_type, op->interval_ms, op->rc,
+                      op->op_status);
         }
 
     } else if (did_rsc_op_fail(op, target_rc)) {
         /* Store failed monitors here, otherwise the block below will cause them
          * to be forgotten when a stop happens.
          */
         if (entry->failed) {
             lrmd_free_event(entry->failed);
         }
         entry->failed = lrmd_copy_event(op);
 
-    } else if (op->interval == 0) {
+    } else if (op->interval_ms == 0) {
         if (entry->last) {
             lrmd_free_event(entry->last);
         }
         entry->last = lrmd_copy_event(op);
 
         if (op->params &&
             (safe_str_eq(CRMD_ACTION_START, op->op_type) ||
              safe_str_eq("reload", op->op_type) ||
              safe_str_eq(CRMD_ACTION_STATUS, op->op_type))) {
 
             if (entry->stop_params) {
                 g_hash_table_destroy(entry->stop_params);
             }
             entry->stop_params = crm_str_table_new();
 
             g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
         }
     }
 
-    if (op->interval > 0) {
+    if (op->interval_ms > 0) {
         /* Ensure there are no duplicates */
         history_remove_recurring_op(entry, op);
 
-        crm_trace("Adding recurring op: %s_%s_%d", op->rsc_id, op->op_type, op->interval);
+        crm_trace("Adding recurring op: " CRM_OP_FMT,
+                  op->rsc_id, op->op_type, op->interval_ms);
         entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));
 
     } else if (entry->recurring_op_list && safe_str_eq(op->op_type, RSC_STATUS) == FALSE) {
-        crm_trace("Dropping %d recurring ops because of: %s_%s_%d",
+        crm_trace("Dropping %d recurring ops because of: " CRM_OP_FMT,
                   g_list_length(entry->recurring_op_list), op->rsc_id,
-                  op->op_type, op->interval);
+                  op->op_type, op->interval_ms);
         history_free_recurring_ops(entry);
     }
 }
 
 /*!
  * \internal
  * \brief Send a direct OK ack for a resource task
  *
  * \param[in] lrm_state  LRM connection
  * \param[in] input      Input message being ack'ed
  * \param[in] rsc_id     ID of affected resource
  * \param[in] rsc        Affected resource (if available)
  * \param[in] task       Operation task being ack'ed
  * \param[in] ack_host   Name of host to send ack to
  * \param[in] ack_sys    IPC system name to ack
  */
 static void
 send_task_ok_ack(lrm_state_t *lrm_state, ha_msg_input_t *input,
                  const char *rsc_id, lrmd_rsc_info_t *rsc, const char *task,
                  const char *ack_host, const char *ack_sys)
 {
     lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc_id, task);
 
     op->rc = PCMK_OCF_OK;
     op->op_status = PCMK_LRM_OP_DONE;
     send_direct_ack(ack_host, ack_sys, rsc, op, rsc_id);
     lrmd_free_event(op);
 }
 
 void
 lrm_op_callback(lrmd_event_data_t * op)
 {
     const char *nodename = NULL;
     lrm_state_t *lrm_state = NULL;
 
     CRM_CHECK(op != NULL, return);
 
     /* determine the node name for this connection. */
     nodename = op->remote_nodename ? op->remote_nodename : fsa_our_uname;
 
     if (op->type == lrmd_event_disconnect && (safe_str_eq(nodename, fsa_our_uname))) {
         /* if this is the local lrmd ipc connection, set the right bits in the
          * crmd when the connection goes down */
         lrm_connection_destroy();
         return;
     } else if (op->type != lrmd_event_exec_complete) {
         /* we only need to process execution results */
         return;
     }
 
     lrm_state = lrm_state_find(nodename);
     CRM_ASSERT(lrm_state != NULL);
 
     process_lrm_event(lrm_state, op, NULL);
 }
 
 /*	 A_LRM_CONNECT	*/
 void
 do_lrm_control(long long action,
                enum crmd_fsa_cause cause,
                enum crmd_fsa_state cur_state,
                enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     /* This only pertains to local lrmd connections.  Remote connections are handled as
      * resources within the pengine.  Connecting and disconnecting from remote lrmd instances
      * handled differently than the local. */
 
     lrm_state_t *lrm_state = NULL;
 
     if(fsa_our_uname == NULL) {
         return; /* Nothing to do */
     }
     lrm_state = lrm_state_find_or_create(fsa_our_uname);
     if (lrm_state == NULL) {
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
         return;
     }
 
     if (action & A_LRM_DISCONNECT) {
         if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) {
             if (action == A_LRM_DISCONNECT) {
                 crmd_fsa_stall(FALSE);
                 return;
             }
         }
 
         clear_bit(fsa_input_register, R_LRM_CONNECTED);
         crm_info("Disconnecting from the LRM");
         lrm_state_disconnect(lrm_state);
         lrm_state_reset_tables(lrm_state, FALSE);
         crm_notice("Disconnected from the LRM");
     }
 
     if (action & A_LRM_CONNECT) {
         int ret = pcmk_ok;
 
         crm_debug("Connecting to the LRM");
         ret = lrm_state_ipc_connect(lrm_state);
 
         if (ret != pcmk_ok) {
             if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
                 crm_warn("Failed to connect to the LRM %d time%s (%d max)",
                          lrm_state->num_lrm_register_fails,
                          s_if_plural(lrm_state->num_lrm_register_fails),
                          MAX_LRM_REG_FAILS);
 
                 crm_timer_start(wait_timer);
                 crmd_fsa_stall(FALSE);
                 return;
             }
         }
 
         if (ret != pcmk_ok) {
             crm_err("Failed to connect to the LRM the max allowed %d time%s",
                     lrm_state->num_lrm_register_fails,
                     s_if_plural(lrm_state->num_lrm_register_fails));
             register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
             return;
         }
 
         set_bit(fsa_input_register, R_LRM_CONNECTED);
         crm_info("LRM connection established");
     }
 
     if (action & ~(A_LRM_CONNECT | A_LRM_DISCONNECT)) {
         crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__);
     }
 }
 
 static gboolean
 lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level)
 {
     int counter = 0;
     gboolean rc = TRUE;
     const char *when = "lrm disconnect";
 
     GHashTableIter gIter;
     const char *key = NULL;
     rsc_history_t *entry = NULL;
     struct recurring_op_s *pending = NULL;
 
     crm_debug("Checking for active resources before exit");
 
     if (cur_state == S_TERMINATE) {
         log_level = LOG_ERR;
         when = "shutdown";
 
     } else if (is_set(fsa_input_register, R_SHUTDOWN)) {
         when = "shutdown... waiting";
     }
 
     if (lrm_state->pending_ops && lrm_state_is_connected(lrm_state) == TRUE) {
         guint removed = g_hash_table_foreach_remove(
             lrm_state->pending_ops, stop_recurring_actions, lrm_state);
         guint nremaining = g_hash_table_size(lrm_state->pending_ops);
 
         if (removed || nremaining) {
             crm_notice("Stopped %u recurring operation%s at %s (%u remaining)",
                        removed, s_if_plural(removed), when, nremaining);
         }
     }
 
     if (lrm_state->pending_ops) {
         g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
         while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) {
             /* Ignore recurring actions in the shutdown calculations */
             if (pending->interval_ms == 0) {
                 counter++;
             }
         }
     }
 
     if (counter > 0) {
         do_crm_log(log_level, "%d pending LRM operation%s at %s",
                    counter, s_if_plural(counter), when);
 
         if (cur_state == S_TERMINATE || !is_set(fsa_input_register, R_SENT_RSC_STOP)) {
             g_hash_table_iter_init(&gIter, lrm_state->pending_ops);
             while (g_hash_table_iter_next(&gIter, (gpointer*)&key, (gpointer*)&pending)) {
                 do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key);
             }
 
         } else {
             rc = FALSE;
         }
         return rc;
     }
 
     if (lrm_state->resource_history == NULL) {
         return rc;
     }
 
     if (is_set(fsa_input_register, R_SHUTDOWN)) {
         /* At this point we're not waiting, we're just shutting down */
         when = "shutdown";
     }
 
     counter = 0;
     g_hash_table_iter_init(&gIter, lrm_state->resource_history);
     while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) {
         if (is_rsc_active(lrm_state, entry->id) == FALSE) {
             continue;
         }
 
         counter++;
         if (log_level == LOG_ERR) {
             crm_info("Found %s active at %s", entry->id, when);
         } else {
             crm_trace("Found %s active at %s", entry->id, when);
         }
         if (lrm_state->pending_ops) {
             GHashTableIter hIter;
 
             g_hash_table_iter_init(&hIter, lrm_state->pending_ops);
             while (g_hash_table_iter_next(&hIter, (gpointer*)&key, (gpointer*)&pending)) {
                 if (crm_str_eq(entry->id, pending->rsc_id, TRUE)) {
                     crm_notice("%sction %s (%s) incomplete at %s",
                                pending->interval_ms == 0 ? "A" : "Recurring a",
                                key, pending->op_key, when);
                 }
             }
         }
     }
 
     if (counter) {
         crm_err("%d resource%s active at %s",
                 counter, (counter == 1)? " was" : "s were", when);
     }
 
     return rc;
 }
 
 static char *
 build_parameter_list(const lrmd_event_data_t *op,
                      const struct ra_metadata_s *metadata,
                      xmlNode *result, enum ra_param_flags_e param_type,
                      bool invert_for_xml)
 {
     int len = 0;
     int max = 0;
     char *list = NULL;
     GList *iter = NULL;
 
     /* Newer resource agents support the "private" parameter attribute to
      * indicate sensitive parameters. For backward compatibility with older
      * agents, this list is used if the agent doesn't specify any as "private".
      */
     const char *secure_terms[] = {
         "password",
         "passwd",
         "user",
     };
 
     if (is_not_set(metadata->ra_flags, ra_uses_private)
         && (param_type == ra_param_private)) {
 
         max = DIMOF(secure_terms);
     }
 
     for (iter = metadata->ra_params; iter != NULL; iter = iter->next) {
         struct ra_param_s *param = (struct ra_param_s *) iter->data;
         bool accept = FALSE;
 
         if (is_set(param->rap_flags, param_type)) {
             accept = TRUE;
 
         } else if (max) {
             for (int lpc = 0; lpc < max; lpc++) {
                 if (safe_str_eq(secure_terms[lpc], param->rap_name)) {
                     accept = TRUE;
                     break;
                 }
             }
         }
 
         if (accept) {
             int start = len;
 
             crm_trace("Attr %s is %s", param->rap_name, ra_param_flag2text(param_type));
 
             len += strlen(param->rap_name) + 2; // include spaces around
             list = realloc_safe(list, len + 1); // include null terminator
 
             // spaces before and after make parsing simpler
             sprintf(list + start, " %s ", param->rap_name);
 
         } else {
             crm_trace("Rejecting %s for %s", param->rap_name, ra_param_flag2text(param_type));
         }
 
         if (result && (invert_for_xml? !accept : accept)) {
             const char *v = g_hash_table_lookup(op->params, param->rap_name);
 
             if (v != NULL) {
                 crm_trace("Adding attr %s=%s to the xml result", param->rap_name, v);
                 crm_xml_add(result, param->rap_name, v);
             }
         }
     }
 
     return list;
 }
 
 static void
 append_restart_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
                     xmlNode *update, const char *version)
 {
     char *list = NULL;
     char *digest = NULL;
     xmlNode *restart = NULL;
 
     CRM_LOG_ASSERT(op->params != NULL);
 
-    if (op->interval > 0) {
+    if (op->interval_ms > 0) {
         /* monitors are not reloadable */
         return;
     }
 
     if (is_set(metadata->ra_flags, ra_supports_reload)) {
         restart = create_xml_node(NULL, XML_TAG_PARAMS);
         /* Add any parameters with unique="1" to the "op-force-restart" list.
          *
          * (Currently, we abuse "unique=0" to indicate reloadability. This is
          * nonstandard and should eventually be replaced once the OCF standard
          * is updated with something better.)
          */
         list = build_parameter_list(op, metadata, restart, ra_param_unique,
                                     FALSE);
 
     } else {
         /* Resource does not support reloads */
         return;
     }
 
     digest = calculate_operation_digest(restart, version);
     /* Add "op-force-restart" and "op-restart-digest" to indicate the resource supports reload,
      * no matter if it actually supports any parameters with unique="1"). */
     crm_xml_add(update, XML_LRM_ATTR_OP_RESTART, list? list: "");
     crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);
 
     crm_trace("%s: %s, %s", op->rsc_id, digest, list);
     crm_log_xml_trace(restart, "restart digest source");
 
     free_xml(restart);
     free(digest);
     free(list);
 }
 
 static void
 append_secure_list(lrmd_event_data_t *op, struct ra_metadata_s *metadata,
                    xmlNode *update, const char *version)
 {
     char *list = NULL;
     char *digest = NULL;
     xmlNode *secure = NULL;
 
     CRM_LOG_ASSERT(op->params != NULL);
 
     /*
      * To keep XML_LRM_ATTR_OP_SECURE short, we want it to contain the
      * secure parameters but XML_LRM_ATTR_SECURE_DIGEST to be based on
      * the insecure ones
      */
     secure = create_xml_node(NULL, XML_TAG_PARAMS);
     list = build_parameter_list(op, metadata, secure, ra_param_private, TRUE);
 
     if (list != NULL) {
         digest = calculate_operation_digest(secure, version);
         crm_xml_add(update, XML_LRM_ATTR_OP_SECURE, list);
         crm_xml_add(update, XML_LRM_ATTR_SECURE_DIGEST, digest);
 
         crm_trace("%s: %s, %s", op->rsc_id, digest, list);
         crm_log_xml_trace(secure, "secure digest source");
     } else {
         crm_trace("%s: no secure parameters", op->rsc_id);
     }
 
     free_xml(secure);
     free(digest);
     free(list);
 }
 
 static gboolean
 build_operation_update(xmlNode * parent, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op,
                        const char *node_name, const char *src)
 {
     int target_rc = 0;
     xmlNode *xml_op = NULL;
     struct ra_metadata_s *metadata = NULL;
     const char *caller_version = NULL;
     lrm_state_t *lrm_state = NULL;
 
     if (op == NULL) {
         return FALSE;
     }
 
     target_rc = rsc_op_expected_rc(op);
 
     /* there is a small risk in formerly mixed clusters that it will
      * be sub-optimal.
      *
      * however with our upgrade policy, the update we send should
      * still be completely supported anyway
      */
     caller_version = g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION);
     CRM_LOG_ASSERT(caller_version != NULL);
 
     if(caller_version == NULL) {
         caller_version = CRM_FEATURE_SET;
     }
 
     crm_trace("Building %s operation update with originator version: %s", op->rsc_id, caller_version);
     xml_op = create_operation_update(parent, op, caller_version, target_rc, fsa_our_uname, src, LOG_DEBUG);
     if (xml_op == NULL) {
         return TRUE;
     }
 
     if ((rsc == NULL) || (op == NULL) || (op->params == NULL)
         || !crm_op_needs_metadata(rsc->standard, op->op_type)) {
 
         crm_trace("No digests needed for %s action on %s (params=%p rsc=%p)",
                   op->op_type, op->rsc_id, op->params, rsc);
         return TRUE;
     }
 
     lrm_state = lrm_state_find(node_name);
     if (lrm_state == NULL) {
-        crm_warn("Cannot calculate digests for operation %s_%s_%d because we have no LRM connection to %s",
-                 op->rsc_id, op->op_type, op->interval, node_name);
+        crm_warn("Cannot calculate digests for operation " CRM_OP_FMT
+                 " because we have no LRM connection to %s",
+                 op->rsc_id, op->op_type, op->interval_ms, node_name);
         return TRUE;
     }
 
     metadata = metadata_cache_get(lrm_state->metadata_cache, rsc);
     if (metadata == NULL) {
         /* For now, we always collect resource agent meta-data via a local,
          * synchronous, direct execution of the agent. This has multiple issues:
          * the lrmd should execute agents, not the crmd; meta-data for
          * Pacemaker Remote nodes should be collected on those nodes, not
          * locally; and the meta-data call shouldn't eat into the timeout of the
          * real action being performed.
          *
          * These issues are planned to be addressed by having the PE schedule
          * a meta-data cache check at the beginning of each transition. Once
          * that is working, this block will only be a fallback in case the
          * initial collection fails.
          */
         char *metadata_str = NULL;
 
         int rc = lrm_state_get_metadata(lrm_state, rsc->standard,
                                         rsc->provider, rsc->type,
                                         &metadata_str, 0);
 
         if (rc != pcmk_ok) {
             crm_warn("Failed to get metadata for %s (%s:%s:%s)",
                      rsc->id, rsc->standard, rsc->provider, rsc->type);
             return TRUE;
         }
 
         metadata = metadata_cache_update(lrm_state->metadata_cache, rsc,
                                          metadata_str);
         free(metadata_str);
         if (metadata == NULL) {
             crm_warn("Failed to update metadata for %s (%s:%s:%s)",
                      rsc->id, rsc->standard, rsc->provider, rsc->type);
             return TRUE;
         }
     }
 
 #if ENABLE_VERSIONED_ATTRS
     crm_xml_add(xml_op, XML_ATTR_RA_VERSION, metadata->ra_version);
 #endif
 
     crm_trace("Including additional digests for %s::%s:%s", rsc->standard, rsc->provider, rsc->type);
     append_restart_list(op, metadata, xml_op, caller_version);
     append_secure_list(op, metadata, xml_op, caller_version);
 
     return TRUE;
 }
 
 static gboolean
 is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id)
 {
     rsc_history_t *entry = NULL;
 
     entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
     if (entry == NULL || entry->last == NULL) {
         return FALSE;
     }
 
-    crm_trace("Processing %s: %s.%d=%d",
-              rsc_id, entry->last->op_type, entry->last->interval, entry->last->rc);
+    crm_trace("Processing %s: %s.%d=%d", rsc_id, entry->last->op_type,
+              entry->last->interval_ms, entry->last->rc);
     if (entry->last->rc == PCMK_OCF_OK && safe_str_eq(entry->last->op_type, CRMD_ACTION_STOP)) {
         return FALSE;
 
     } else if (entry->last->rc == PCMK_OCF_OK
                && safe_str_eq(entry->last->op_type, CRMD_ACTION_MIGRATE)) {
         /* a stricter check is too complex...
          * leave that to the PE
          */
         return FALSE;
 
     } else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) {
         return FALSE;
 
-    } else if (entry->last->interval == 0 && entry->last->rc == PCMK_OCF_NOT_CONFIGURED) {
+    } else if ((entry->last->interval_ms == 0)
+               && (entry->last->rc == PCMK_OCF_NOT_CONFIGURED)) {
         /* Badly configured resources can't be reliably stopped */
         return FALSE;
     }
 
     return TRUE;
 }
 
 static gboolean
 build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list)
 {
     GHashTableIter iter;
     rsc_history_t *entry = NULL;
 
     g_hash_table_iter_init(&iter, lrm_state->resource_history);
     while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
 
         GList *gIter = NULL;
         xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE);
 
         crm_xml_add(xml_rsc, XML_ATTR_ID, entry->id);
         crm_xml_add(xml_rsc, XML_ATTR_TYPE, entry->rsc.type);
         crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, entry->rsc.standard);
         crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, entry->rsc.provider);
 
         if (entry->last && entry->last->params) {
             const char *container = g_hash_table_lookup(entry->last->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
             if (container) {
                 crm_trace("Resource %s is a part of container resource %s", entry->id, container);
                 crm_xml_add(xml_rsc, XML_RSC_ATTR_CONTAINER, container);
             }
         }
         build_operation_update(xml_rsc, &(entry->rsc), entry->failed, lrm_state->node_name, __FUNCTION__);
         build_operation_update(xml_rsc, &(entry->rsc), entry->last, lrm_state->node_name, __FUNCTION__);
         for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
             build_operation_update(xml_rsc, &(entry->rsc), gIter->data, lrm_state->node_name, __FUNCTION__);
         }
     }
 
     return FALSE;
 }
 
 static xmlNode *
 do_lrm_query_internal(lrm_state_t *lrm_state, int update_flags)
 {
     xmlNode *xml_state = NULL;
     xmlNode *xml_data = NULL;
     xmlNode *rsc_list = NULL;
     crm_node_t *peer = NULL;
 
     peer = crm_get_peer_full(0, lrm_state->node_name, CRM_GET_PEER_ANY);
     CRM_CHECK(peer != NULL, return NULL);
 
     xml_state = create_node_state_update(peer, update_flags, NULL,
                                          __FUNCTION__);
     if (xml_state == NULL) {
         return NULL;
     }
 
     xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM);
     crm_xml_add(xml_data, XML_ATTR_ID, peer->uuid);
     rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES);
 
     /* Build a list of active (not always running) resources */
     build_active_RAs(lrm_state, rsc_list);
 
     crm_log_xml_trace(xml_state, "Current state of the LRM");
 
     return xml_state;
 }
 
 xmlNode *
 do_lrm_query(gboolean is_replace, const char *node_name)
 {
     lrm_state_t *lrm_state = lrm_state_find(node_name);
 
     if (!lrm_state) {
         crm_err("Could not query lrm state for lrmd node %s", node_name);
         return NULL;
     }
     return do_lrm_query_internal(lrm_state,
                                  node_update_cluster|node_update_peer);
 }
 
 static void
 notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc)
 {
     lrmd_event_data_t *op = NULL;
     const char *from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
     const char *from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
 
     crm_info("Notifying %s on %s that %s was%s deleted",
              from_sys, (from_host? from_host : "localhost"), rsc_id,
              ((rc == pcmk_ok)? "" : " not"));
 
     op = construct_op(lrm_state, input->xml, rsc_id, CRMD_ACTION_DELETE);
 
     if (rc == pcmk_ok) {
         op->op_status = PCMK_LRM_OP_DONE;
         op->rc = PCMK_OCF_OK;
     } else {
         op->op_status = PCMK_LRM_OP_ERROR;
         op->rc = PCMK_OCF_UNKNOWN_ERROR;
     }
 
     send_direct_ack(from_host, from_sys, NULL, op, rsc_id);
     lrmd_free_event(op);
 
     if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
         /* this isn't expected - trigger a new transition */
         time_t now = time(NULL);
         char *now_s = crm_itoa(now);
 
         crm_debug("Triggering a refresh after %s deleted %s from the LRM", from_sys, rsc_id);
 
         update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
                              "last-lrm-refresh", now_s, FALSE, NULL, NULL);
 
         free(now_s);
     }
 }
 
 static gboolean
 lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
 {
     struct delete_event_s *event = user_data;
     struct pending_deletion_op_s *op = value;
 
     if (crm_str_eq(event->rsc, op->rsc, TRUE)) {
         notify_deleted(event->lrm_state, op->input, event->rsc, event->rc);
         return TRUE;
     }
     return FALSE;
 }
 
 static gboolean
 lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
 {
     const char *rsc = user_data;
     struct recurring_op_s *pending = value;
 
     if (crm_str_eq(rsc, pending->rsc_id, TRUE)) {
         crm_info("Removing op %s:%d for deleted resource %s",
                  pending->op_key, pending->call_id, rsc);
         return TRUE;
     }
     return FALSE;
 }
 
 /*
  * Remove the rsc from the CIB
  *
  * Avoids refreshing the entire LRM section of this host
  */
 #define rsc_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']"
 
 static int
 delete_rsc_status(lrm_state_t * lrm_state, const char *rsc_id, int call_options,
                   const char *user_name)
 {
     char *rsc_xpath = NULL;
     int rc = pcmk_ok;
 
     CRM_CHECK(rsc_id != NULL, return -ENXIO);
 
     rsc_xpath = crm_strdup_printf(rsc_template, lrm_state->node_name, rsc_id);
 
     rc = cib_internal_op(fsa_cib_conn, CIB_OP_DELETE, NULL, rsc_xpath,
                          NULL, NULL, call_options | cib_xpath, user_name);
 
     free(rsc_xpath);
     return rc;
 }
 
 static void
 delete_rsc_entry(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id,
                  GHashTableIter * rsc_gIter, int rc, const char *user_name)
 {
     struct delete_event_s event;
 
     CRM_CHECK(rsc_id != NULL, return);
 
     if (rc == pcmk_ok) {
         char *rsc_id_copy = strdup(rsc_id);
 
         if (rsc_gIter)
             g_hash_table_iter_remove(rsc_gIter);
         else
             g_hash_table_remove(lrm_state->resource_history, rsc_id_copy);
         crm_debug("sync: Sending delete op for %s", rsc_id_copy);
         delete_rsc_status(lrm_state, rsc_id_copy, cib_quorum_override, user_name);
 
         g_hash_table_foreach_remove(lrm_state->pending_ops, lrm_remove_deleted_op, rsc_id_copy);
         free(rsc_id_copy);
     }
 
     if (input) {
         notify_deleted(lrm_state, input, rsc_id, rc);
     }
 
     event.rc = rc;
     event.rsc = rsc_id;
     event.lrm_state = lrm_state;
     g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event);
 }
 
 /*!
  * \internal
  * \brief Erase an LRM history entry from the CIB, given the operation data
  *
  * \param[in] lrm_state  LRM state of the desired node
  * \param[in] op         Operation whose history should be deleted
  */
 static void
 erase_lrm_history_by_op(lrm_state_t *lrm_state, lrmd_event_data_t *op)
 {
     xmlNode *xml_top = NULL;
 
     CRM_CHECK(op != NULL, return);
 
     xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
     crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
     crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);
 
-    if (op->interval > 0) {
-        char *op_id = generate_op_key(op->rsc_id, op->op_type, op->interval);
+    if (op->interval_ms > 0) {
+        char *op_id = generate_op_key(op->rsc_id, op->op_type, op->interval_ms);
 
         /* Avoid deleting last_failure too (if it was a result of this recurring op failing) */
         crm_xml_add(xml_top, XML_ATTR_ID, op_id);
         free(op_id);
     }
 
-    crm_debug("Erasing LRM resource history for %s_%s_%d (call=%d)",
-              op->rsc_id, op->op_type, op->interval, op->call_id);
+    crm_debug("Erasing LRM resource history for " CRM_OP_FMT " (call=%d)",
+              op->rsc_id, op->op_type, op->interval_ms, op->call_id);
 
     fsa_cib_conn->cmds->remove(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top,
                                cib_quorum_override);
 
     crm_log_xml_trace(xml_top, "op:cancel");
     free_xml(xml_top);
 }
 
 /* Define xpath to find LRM resource history entry by node and resource */
 #define XPATH_HISTORY                                   \
     "/" XML_TAG_CIB "/" XML_CIB_TAG_STATUS              \
     "/" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"  \
     "/" XML_CIB_TAG_LRM "/" XML_LRM_TAG_RESOURCES       \
     "/" XML_LRM_TAG_RESOURCE "[@" XML_ATTR_ID "='%s']"  \
     "/" XML_LRM_TAG_RSC_OP
 
 /* ... and also by operation key */
 #define XPATH_HISTORY_ID XPATH_HISTORY \
     "[@" XML_ATTR_ID "='%s']"
 
 /* ... and also by operation key and operation call ID */
 #define XPATH_HISTORY_CALL XPATH_HISTORY \
     "[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_CALLID "='%d']"
 
 /* ... and also by operation key and original operation key */
 #define XPATH_HISTORY_ORIG XPATH_HISTORY \
     "[@" XML_ATTR_ID "='%s' and @" XML_LRM_ATTR_TASK_KEY "='%s']"
 
 /*!
  * \internal
  * \brief Erase an LRM history entry from the CIB, given operation identifiers
  *
  * \param[in] lrm_state  LRM state of the node to clear history for
  * \param[in] rsc_id     Name of resource to clear history for
  * \param[in] key        Operation key of operation to clear history for
  * \param[in] orig_op    If specified, delete only if it has this original op
  * \param[in] call_id    If specified, delete entry only if it has this call ID
  */
 static void
 erase_lrm_history_by_id(lrm_state_t *lrm_state, const char *rsc_id,
                         const char *key, const char *orig_op, int call_id)
 {
     char *op_xpath = NULL;
 
     CRM_CHECK((rsc_id != NULL) && (key != NULL), return);
 
     if (call_id > 0) {
         op_xpath = crm_strdup_printf(XPATH_HISTORY_CALL,
                                      lrm_state->node_name, rsc_id, key,
                                      call_id);
 
     } else if (orig_op) {
         op_xpath = crm_strdup_printf(XPATH_HISTORY_ORIG,
                                      lrm_state->node_name, rsc_id, key,
                                      orig_op);
     } else {
         op_xpath = crm_strdup_printf(XPATH_HISTORY_ID,
                                      lrm_state->node_name, rsc_id, key);
     }
 
     crm_debug("Erasing LRM resource history for %s on %s (call=%d)",
               key, rsc_id, call_id);
     fsa_cib_conn->cmds->remove(fsa_cib_conn, op_xpath, NULL,
                                cib_quorum_override | cib_xpath);
     free(op_xpath);
 }
 
 static inline gboolean
 last_failed_matches_op(rsc_history_t *entry, const char *op, guint interval_ms)
 {
     if (entry == NULL) {
         return FALSE;
     }
     if (op == NULL) {
         return TRUE;
     }
     return (safe_str_eq(op, entry->failed->op_type)
-            && (interval_ms == entry->failed->interval));
+            && (interval_ms == entry->failed->interval_ms));
 }
 
 /*!
  * \internal
  * \brief Clear a resource's last failure
  *
  * Erase a resource's last failure on a particular node from both the
  * LRM resource history in the CIB, and the resource history remembered
  * for the LRM state.
  *
  * \param[in] rsc_id      Resource name
  * \param[in] node_name   Node name
  * \param[in] operation   If specified, only clear if matching this operation
  * \param[in] interval_ms If operation is specified, it has this interval
  */
 void
 lrm_clear_last_failure(const char *rsc_id, const char *node_name,
                        const char *operation, guint interval_ms)
 {
     char *op_key = NULL;
     char *orig_op_key = NULL;
     lrm_state_t *lrm_state = NULL;
 
     lrm_state = lrm_state_find(node_name);
     if (lrm_state == NULL) {
         return;
     }
 
     /* Erase from CIB */
     op_key = generate_op_key(rsc_id, "last_failure", 0);
     if (operation) {
         orig_op_key = generate_op_key(rsc_id, operation, interval_ms);
     }
     erase_lrm_history_by_id(lrm_state, rsc_id, op_key, orig_op_key, 0);
     free(op_key);
     free(orig_op_key);
 
     /* Remove from memory */
     if (lrm_state->resource_history) {
         rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history,
                                                    rsc_id);
 
         if (last_failed_matches_op(entry, operation, interval_ms)) {
             lrmd_free_event(entry->failed);
             entry->failed = NULL;
         }
     }
 }
 
 /* Returns: gboolean - cancellation is in progress */
 static gboolean
 cancel_op(lrm_state_t * lrm_state, const char *rsc_id, const char *key, int op, gboolean remove)
 {
     int rc = pcmk_ok;
     char *local_key = NULL;
     struct recurring_op_s *pending = NULL;
 
     CRM_CHECK(op != 0, return FALSE);
     CRM_CHECK(rsc_id != NULL, return FALSE);
     if (key == NULL) {
         local_key = make_stop_id(rsc_id, op);
         key = local_key;
     }
     pending = g_hash_table_lookup(lrm_state->pending_ops, key);
 
     if (pending) {
         if (remove && pending->remove == FALSE) {
             pending->remove = TRUE;
             crm_debug("Scheduling %s for removal", key);
         }
 
         if (pending->cancelled) {
             crm_debug("Operation %s already cancelled", key);
             free(local_key);
             return FALSE;
         }
 
         pending->cancelled = TRUE;
 
     } else {
         crm_info("No pending op found for %s", key);
         free(local_key);
         return FALSE;
     }
 
     crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);
     rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type,
                           pending->interval_ms);
     if (rc == pcmk_ok) {
         crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
         free(local_key);
         return TRUE;
     }
 
     crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
     /* The caller needs to make sure the entry is
      * removed from the pending_ops list
      *
      * Usually by returning TRUE inside the worker function
      * supplied to g_hash_table_foreach_remove()
      *
      * Not removing the entry from pending_ops will block
      * the node from shutting down
      */
     free(local_key);
     return FALSE;
 }
 
 struct cancel_data {
     gboolean done;
     gboolean remove;
     const char *key;
     lrmd_rsc_info_t *rsc;
     lrm_state_t *lrm_state;
 };
 
 static gboolean
 cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
 {
     gboolean remove = FALSE;
     struct cancel_data *data = user_data;
     struct recurring_op_s *op = (struct recurring_op_s *)value;
 
     if (crm_str_eq(op->op_key, data->key, TRUE)) {
         data->done = TRUE;
         remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove);
     }
     return remove;
 }
 
 static gboolean
 cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
 {
     guint removed = 0;
     struct cancel_data data;
 
     CRM_CHECK(rsc != NULL, return FALSE);
     CRM_CHECK(key != NULL, return FALSE);
 
     data.key = key;
     data.rsc = rsc;
     data.done = FALSE;
     data.remove = remove;
     data.lrm_state = lrm_state;
 
     removed = g_hash_table_foreach_remove(lrm_state->pending_ops, cancel_action_by_key, &data);
     crm_trace("Removed %u op cache entries, new size: %u",
               removed, g_hash_table_size(lrm_state->pending_ops));
     return data.done;
 }
 
 /*!
  * \internal
  * \brief Retrieve resource information from LRM
  *
  * \param[in]  lrm_state LRM connection to use
  * \param[in]  rsc_xml   XML containing resource configuration
  * \param[in]  do_create If true, register resource with LRM if not already
  * \param[out] rsc_info  Where to store resource information obtained from LRM
  *
  * \retval pcmk_ok   Success (and rsc_info holds newly allocated result)
  * \retval -EINVAL   Required information is missing from arguments
  * \retval -ENOTCONN No active connection to LRM
  * \retval -ENODEV   Resource not found
  * \retval -errno    Error communicating with lrmd when registering resource
  *
  * \note Caller is responsible for freeing result on success.
  */
 static int
 get_lrm_resource(lrm_state_t *lrm_state, xmlNode *rsc_xml, gboolean do_create,
                  lrmd_rsc_info_t **rsc_info)
 {
     const char *id = ID(rsc_xml);
 
     CRM_CHECK(lrm_state && rsc_xml && rsc_info, return -EINVAL);
     CRM_CHECK(id, return -EINVAL);
 
     if (lrm_state_is_connected(lrm_state) == FALSE) {
         return -ENOTCONN;
     }
 
     crm_trace("Retrieving resource information for %s from the LRM", id);
     *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
 
     // If resource isn't known by ID, try clone name, if provided
     if (!*rsc_info) {
         const char *long_id = crm_element_value(rsc_xml, XML_ATTR_ID_LONG);
 
         if (long_id) {
             *rsc_info = lrm_state_get_rsc_info(lrm_state, long_id, 0);
         }
     }
 
     if ((*rsc_info == NULL) && do_create) {
         const char *class = crm_element_value(rsc_xml, XML_AGENT_ATTR_CLASS);
         const char *provider = crm_element_value(rsc_xml, XML_AGENT_ATTR_PROVIDER);
         const char *type = crm_element_value(rsc_xml, XML_ATTR_TYPE);
         int rc;
 
         crm_trace("Registering resource %s with LRM", id);
         rc = lrm_state_register_rsc(lrm_state, id, class, provider, type,
                                     lrmd_opt_drop_recurring);
         if (rc != pcmk_ok) {
             fsa_data_t *msg_data = NULL;
 
             crm_err("Could not register resource %s with LRM on %s: %s "
                     CRM_XS " rc=%d",
                     id, lrm_state->node_name, pcmk_strerror(rc), rc);
 
             /* Register this as an internal error if this involves the local
              * lrmd. Otherwise, we're likely dealing with an unresponsive remote
              * node, which is not an FSA failure.
              */
             if (lrm_state_is_local(lrm_state) == TRUE) {
                 register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
             }
             return rc;
         }
 
         *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
     }
     return *rsc_info? pcmk_ok : -ENODEV;
 }
 
 static void
 delete_resource(lrm_state_t * lrm_state,
                 const char *id,
                 lrmd_rsc_info_t * rsc,
                 GHashTableIter * gIter,
                 const char *sys,
                 const char *host,
                 const char *user,
                 ha_msg_input_t * request,
                 gboolean unregister)
 {
     int rc = pcmk_ok;
 
     crm_info("Removing resource %s for %s (%s) on %s", id, sys, user ? user : "internal", host);
 
     if (rsc && unregister) {
         rc = lrm_state_unregister_rsc(lrm_state, id, 0);
     }
 
     if (rc == pcmk_ok) {
         crm_trace("Resource '%s' deleted", id);
     } else if (rc == -EINPROGRESS) {
         crm_info("Deletion of resource '%s' pending", id);
         if (request) {
             struct pending_deletion_op_s *op = NULL;
             char *ref = crm_element_value_copy(request->msg, XML_ATTR_REFERENCE);
 
             op = calloc(1, sizeof(struct pending_deletion_op_s));
             op->rsc = strdup(rsc->id);
             op->input = copy_ha_msg_input(request);
             g_hash_table_insert(lrm_state->deletion_ops, ref, op);
         }
         return;
     } else {
         crm_warn("Deletion of resource '%s' for %s (%s) on %s failed: %d",
                  id, sys, user ? user : "internal", host, rc);
     }
 
     delete_rsc_entry(lrm_state, request, id, gIter, rc, user);
 }
 
 static int
 get_fake_call_id(lrm_state_t *lrm_state, const char *rsc_id)
 {
     int call_id = 999999999;
     rsc_history_t *entry = NULL;
 
     if(lrm_state) {
         entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
     }
 
     /* Make sure the call id is greater than the last successful operation,
      * otherwise the failure will not result in a possible recovery of the resource
      * as it could appear the failure occurred before the successful start */
     if (entry) {
         call_id = entry->last_callid + 1;
     }
 
     if (call_id < 0) {
         call_id = 1;
     }
     return call_id;
 }
 
 static void
 fake_op_status(lrm_state_t *lrm_state, lrmd_event_data_t *op, int op_status,
                enum ocf_exitcode op_exitcode)
 {
     op->call_id = get_fake_call_id(lrm_state, op->rsc_id);
     op->t_run = time(NULL);
     op->t_rcchange = op->t_run;
     op->op_status = op_status;
     op->rc = op_exitcode;
 }
 
 static void
 force_reprobe(lrm_state_t *lrm_state, const char *from_sys,
               const char *from_host, const char *user_name,
               gboolean is_remote_node)
 {
     GHashTableIter gIter;
     rsc_history_t *entry = NULL;
 
     crm_info("Clearing resource history on node %s", lrm_state->node_name);
     g_hash_table_iter_init(&gIter, lrm_state->resource_history);
     while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
         /* only unregister the resource during a reprobe if it is not a remote connection
          * resource. otherwise unregistering the connection will terminate remote-node
          * membership */
         gboolean unregister = TRUE;
 
         if (is_remote_lrmd_ra(NULL, NULL, entry->id)) {
             lrm_state_t *remote_lrm_state = lrm_state_find(entry->id);
             if (remote_lrm_state) {
                 /* when forcing a reprobe, make sure to clear remote node before
                  * clearing the remote node's connection resource */ 
                 force_reprobe(remote_lrm_state, from_sys, from_host, user_name, TRUE);
             }
             unregister = FALSE;
         }
 
         delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys, from_host,
                         user_name, NULL, unregister);
     }
 
     /* Now delete the copy in the CIB */
     erase_status_tag(lrm_state->node_name, XML_CIB_TAG_LRM, cib_scope_local);
 
     /* And finally, _delete_ the value in attrd
      * Setting it to FALSE results in the PE sending us back here again
      */
     update_attrd(lrm_state->node_name, CRM_OP_PROBED, NULL, user_name, is_remote_node);
 }
 
 static void
 synthesize_lrmd_failure(lrm_state_t *lrm_state, xmlNode *action, int rc) 
 {
     lrmd_event_data_t *op = NULL;
     lrmd_rsc_info_t *rsc_info = NULL;
     const char *operation = crm_element_value(action, XML_LRM_ATTR_TASK);
     const char *target_node = crm_element_value(action, XML_LRM_ATTR_TARGET);
     xmlNode *xml_rsc = find_xml_node(action, XML_CIB_TAG_RESOURCE, TRUE);
 
     if ((xml_rsc == NULL) || (ID(xml_rsc) == NULL)) {
         /* @TODO Should we do something else, like direct ack? */
         crm_info("Can't fake %s failure (%d) on %s without resource configuration",
                  crm_element_value(action, XML_LRM_ATTR_TASK_KEY), rc,
                  target_node);
         return;
 
     } else if(operation == NULL) {
         /* This probably came from crm_resource -C, nothing to do */
         crm_info("Can't fake %s failure (%d) on %s without operation",
                  ID(xml_rsc), rc, target_node);
         return;
     }
 
     op = construct_op(lrm_state, action, ID(xml_rsc), operation);
 
     if (safe_str_eq(operation, RSC_NOTIFY)) { // Notifications can't fail
         fake_op_status(lrm_state, op, PCMK_LRM_OP_DONE, PCMK_OCF_OK);
     } else {
         fake_op_status(lrm_state, op, PCMK_LRM_OP_ERROR, rc);
     }
 
     crm_info("Faking " CRM_OP_FMT " result (%d) on %s",
-             op->rsc_id, op->op_type, op->interval, op->rc, target_node);
+             op->rsc_id, op->op_type, op->interval_ms, op->rc, target_node);
 
     /* Process the result as if it came from the LRM, if possible
      * (i.e. resource info can be obtained from the lrm_state).
      */
     if (lrm_state) {
         rsc_info = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
     }
     if (rsc_info) {
         process_lrm_event(lrm_state, op, NULL);
 
     } else {
         /* If we can't process the result normally, at least write it to the CIB
          * if possible, so the PE can act on it.
          */
         char *standard = crm_element_value_copy(xml_rsc, XML_AGENT_ATTR_CLASS);
         char *provider = crm_element_value_copy(xml_rsc, XML_AGENT_ATTR_PROVIDER);
         char *type = crm_element_value_copy(xml_rsc, XML_ATTR_TYPE);
 
         if (standard && type) {
             rsc_info = lrmd_new_rsc_info(op->rsc_id, standard, provider, type);
             do_update_resource(target_node, rsc_info, op);
             lrmd_free_rsc_info(rsc_info);
         } else {
             // @TODO Should we direct ack?
             crm_info("Can't fake %s failure (%d) on %s without resource standard and type",
                      crm_element_value(action, XML_LRM_ATTR_TASK_KEY), rc,
                      target_node);
         }
     }
     lrmd_free_event(op);
 }
 
 /*!
  * \internal
  * \brief Get target of an LRM operation
  *
  * \param[in] xml  LRM operation data XML
  *
  * \return LRM operation target node name (local node or Pacemaker Remote node)
  */
 static const char *
 lrm_op_target(xmlNode *xml)
 {
     const char *target = NULL;
 
     if (xml) {
         target = crm_element_value(xml, XML_LRM_ATTR_TARGET);
     }
     if (target == NULL) {
         target = fsa_our_uname;
     }
     return target;
 }
 
 static void
 fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name,
                   const char *from_host, const char *from_sys)
 {
     lrmd_event_data_t *op = NULL;
     lrmd_rsc_info_t *rsc = NULL;
     xmlNode *xml_rsc = find_xml_node(xml, XML_CIB_TAG_RESOURCE, TRUE);
 
     CRM_CHECK(xml_rsc != NULL, return);
 
     /* The lrmd simply executes operations and reports the results, without any
      * concept of success or failure, so to fail a resource, we must fake what a
      * failure looks like.
      *
      * To do this, we create a fake lrmd operation event for the resource, and
      * pass that event to the lrmd client callback so it will be processed as if
      * it came from the lrmd.
      */
     op = construct_op(lrm_state, xml, ID(xml_rsc), "asyncmon");
     fake_op_status(lrm_state, op, PCMK_LRM_OP_DONE, PCMK_OCF_UNKNOWN_ERROR);
 
     free((char*) op->user_data);
     op->user_data = NULL;
-    op->interval = 0;
+    op->interval_ms = 0;
 
 #if ENABLE_ACL
     if (user_name && is_privileged(user_name) == FALSE) {
         crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc));
         send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
         lrmd_free_event(op);
         return;
     }
 #endif
 
     if (get_lrm_resource(lrm_state, xml_rsc, TRUE, &rsc) == pcmk_ok) {
         crm_info("Failing resource %s...", rsc->id);
         process_lrm_event(lrm_state, op, NULL);
         op->op_status = PCMK_LRM_OP_DONE;
         op->rc = PCMK_OCF_OK;
         lrmd_free_rsc_info(rsc);
 
     } else {
         crm_info("Cannot find/create resource in order to fail it...");
         crm_log_xml_warn(xml, "bad input");
     }
 
     send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc));
     lrmd_free_event(op);
 }
 
 static void
 handle_refresh_op(lrm_state_t *lrm_state, const char *user_name,
                   const char *from_host, const char *from_sys)
 {
     int rc = pcmk_ok;
     xmlNode *fragment = do_lrm_query_internal(lrm_state, node_update_all);
 
     fsa_cib_update(XML_CIB_TAG_STATUS, fragment, cib_quorum_override, rc, user_name);
     crm_info("Forced a local LRM refresh: call=%d", rc);
 
     if (safe_str_neq(CRM_SYSTEM_CRMD, from_sys)) {
         xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, fragment, from_host,
                                         from_sys, CRM_SYSTEM_LRMD,
                                         fsa_our_uuid);
 
         crm_debug("ACK'ing refresh from %s (%s)", from_sys, from_host);
 
         if (relay_message(reply, TRUE) == FALSE) {
             crm_log_xml_err(reply, "Unable to route reply");
         }
         free_xml(reply);
     }
 
     free_xml(fragment);
 }
 
 static void
 handle_query_op(xmlNode *msg, lrm_state_t *lrm_state)
 {
     xmlNode *data = do_lrm_query_internal(lrm_state, node_update_all);
     xmlNode *reply = create_reply(msg, data);
 
     if (relay_message(reply, TRUE) == FALSE) {
         crm_err("Unable to route reply");
         crm_log_xml_err(reply, "reply");
     }
     free_xml(reply);
     free_xml(data);
 }
 
 static void
 handle_reprobe_op(lrm_state_t *lrm_state, const char *from_sys,
                   const char *from_host, const char *user_name,
                   gboolean is_remote_node)
 {
     crm_notice("Forcing the status of all resources to be redetected");
     force_reprobe(lrm_state, from_sys, from_host, user_name, is_remote_node);
 
     if (safe_str_neq(CRM_SYSTEM_PENGINE, from_sys)
         && safe_str_neq(CRM_SYSTEM_TENGINE, from_sys)) {
 
         xmlNode *reply = create_request(CRM_OP_INVOKE_LRM, NULL, from_host,
                                         from_sys, CRM_SYSTEM_LRMD,
                                         fsa_our_uuid);
 
         crm_debug("ACK'ing re-probe from %s (%s)", from_sys, from_host);
 
         if (relay_message(reply, TRUE) == FALSE) {
             crm_log_xml_err(reply, "Unable to route reply");
         }
         free_xml(reply);
     }
 }
 
 static bool do_lrm_cancel(ha_msg_input_t *input, lrm_state_t *lrm_state,
               lrmd_rsc_info_t *rsc, const char *from_host, const char *from_sys)
 {
     char *op_key = NULL;
     char *meta_key = NULL;
     int call = 0;
     const char *call_id = NULL;
     const char *op_task = NULL;
     const char *interval_ms_s = NULL;
     gboolean in_progress = FALSE;
     xmlNode *params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE);
 
     CRM_CHECK(params != NULL, return FALSE);
 
     meta_key = crm_meta_name(XML_LRM_ATTR_INTERVAL_MS);
     interval_ms_s = crm_element_value(params, meta_key);
     free(meta_key);
     CRM_CHECK(interval_ms_s != NULL, return FALSE);
 
     meta_key = crm_meta_name(XML_LRM_ATTR_TASK);
     op_task = crm_element_value(params, meta_key);
     free(meta_key);
     CRM_CHECK(op_task != NULL, return FALSE);
 
     meta_key = crm_meta_name(XML_LRM_ATTR_CALLID);
     call_id = crm_element_value(params, meta_key);
     free(meta_key);
 
     op_key = generate_op_key(rsc->id, op_task, crm_parse_ms(interval_ms_s));
 
     crm_debug("PE requested op %s (call=%s) be cancelled",
               op_key, (call_id? call_id : "NA"));
     call = crm_parse_int(call_id, "0");
     if (call == 0) {
         /* the normal case when the PE cancels a recurring op */
         in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE);
 
     } else {
         /* the normal case when the PE cancels an orphan op */
         in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE);
     }
 
     // Acknowledge cancellation operation if for a remote connection resource
     if (!in_progress || is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
         char *op_id = make_stop_id(rsc->id, call);
 
         if (is_remote_lrmd_ra(NULL, NULL, rsc->id) == FALSE) {
             crm_info("Nothing known about operation %d for %s", call, op_key);
         }
         erase_lrm_history_by_id(lrm_state, rsc->id, op_key, NULL, call);
         send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
                          from_host, from_sys);
 
         /* needed at least for cancellation of a remote operation */
         g_hash_table_remove(lrm_state->pending_ops, op_id);
         free(op_id);
 
     } else {
         /* No ack is needed since abcdaa8, but peers with older versions
          * in a rolling upgrade need one. We didn't bump the feature set
          * at that commit, so we can only compare against the previous
          * CRM version (3.0.8). If any peers have feature set 3.0.9 but
          * not abcdaa8, they will time out waiting for the ack (no
          * released versions of Pacemaker are affected).
          */
         const char *peer_version = crm_element_value(params, XML_ATTR_CRM_VERSION);
 
         if (compare_version(peer_version, "3.0.8") <= 0) {
             crm_info("Sending compatibility ack for %s cancellation to %s (CRM version %s)",
                      op_key, from_host, peer_version);
             send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
                              from_host, from_sys);
         }
     }
 
     free(op_key);
     return TRUE;
 }
 
 static void
 do_lrm_delete(ha_msg_input_t *input, lrm_state_t *lrm_state,
               lrmd_rsc_info_t *rsc, const char *from_sys, const char *from_host,
               bool crm_rsc_delete, const char *user_name)
 {
     gboolean unregister = TRUE;
 
 #if ENABLE_ACL
     int cib_rc = delete_rsc_status(lrm_state, rsc->id,
                                    cib_dryrun|cib_sync_call, user_name);
 
     if (cib_rc != pcmk_ok) {
         lrmd_event_data_t *op = NULL;
 
         crm_err("Could not delete resource status of %s for %s (user %s) on %s: %s"
                 CRM_XS " rc=%d",
                 rsc->id, from_sys, (user_name? user_name : "unknown"),
                 from_host, pcmk_strerror(cib_rc), cib_rc);
 
         op = construct_op(lrm_state, input->xml, rsc->id, CRMD_ACTION_DELETE);
         op->op_status = PCMK_LRM_OP_ERROR;
 
         if (cib_rc == -EACCES) {
             op->rc = PCMK_OCF_INSUFFICIENT_PRIV;
         } else {
             op->rc = PCMK_OCF_UNKNOWN_ERROR;
         }
         send_direct_ack(from_host, from_sys, NULL, op, rsc->id);
         lrmd_free_event(op);
         lrmd_free_rsc_info(rsc);
         return;
     }
 #endif
 
     if (crm_rsc_delete && is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
         unregister = FALSE;
     }
 
     delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys, from_host,
                     user_name, input, unregister);
 }
 
 /*	 A_LRM_INVOKE	*/
 void
 do_lrm_invoke(long long action,
               enum crmd_fsa_cause cause,
               enum crmd_fsa_state cur_state,
               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     lrm_state_t *lrm_state = NULL;
     const char *crm_op = NULL;
     const char *from_sys = NULL;
     const char *from_host = NULL;
     const char *operation = NULL;
     ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
     const char *user_name = NULL;
     const char *target_node = NULL;
     gboolean is_remote_node = FALSE;
     bool crm_rsc_delete = FALSE;
 
     target_node = lrm_op_target(input->xml);
     is_remote_node = safe_str_neq(target_node, fsa_our_uname);
 
     lrm_state = lrm_state_find(target_node);
     if ((lrm_state == NULL) && is_remote_node) {
         crm_err("Failing action because local node has never had connection to remote node %s",
                 target_node);
         synthesize_lrmd_failure(NULL, input->xml, PCMK_OCF_CONNECTION_DIED);
         return;
     }
     CRM_ASSERT(lrm_state != NULL);
 
 #if ENABLE_ACL
     user_name = crm_acl_get_set_user(input->msg, F_CRM_USER, NULL);
     crm_trace("LRM command from user '%s'", user_name);
 #endif
 
     crm_op = crm_element_value(input->msg, F_CRM_TASK);
     from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM);
     if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
         from_host = crm_element_value(input->msg, F_CRM_HOST_FROM);
     }
     crm_trace("LRM %s command from %s", crm_op, from_sys);
 
     if (safe_str_eq(crm_op, CRM_OP_LRM_DELETE)) {
         crm_rsc_delete = TRUE; // Only crm_resource uses this op
         operation = CRMD_ACTION_DELETE;
 
     } else if (safe_str_eq(crm_op, CRM_OP_LRM_FAIL)) {
         fail_lrm_resource(input->xml, lrm_state, user_name, from_host,
                           from_sys);
         return;
 
     } else if (input->xml != NULL) {
         operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK);
     }
 
     if (safe_str_eq(crm_op, CRM_OP_LRM_REFRESH)) {
         handle_refresh_op(lrm_state, user_name, from_host, from_sys);
 
     } else if (safe_str_eq(crm_op, CRM_OP_LRM_QUERY)) {
         handle_query_op(input->msg, lrm_state);
 
     } else if (safe_str_eq(operation, CRM_OP_PROBED)) {
         update_attrd(lrm_state->node_name, CRM_OP_PROBED, XML_BOOLEAN_TRUE,
                      user_name, is_remote_node);
 
     } else if (safe_str_eq(operation, CRM_OP_REPROBE)
                || safe_str_eq(crm_op, CRM_OP_REPROBE)) {
         handle_reprobe_op(lrm_state, from_sys, from_host, user_name,
                           is_remote_node);
 
     } else if (operation != NULL) {
         lrmd_rsc_info_t *rsc = NULL;
         xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE);
         gboolean create_rsc = safe_str_neq(operation, CRMD_ACTION_DELETE);
         int rc;
 
         // We can't return anything meaningful without a resource ID
         CRM_CHECK(xml_rsc && ID(xml_rsc), return);
 
         rc = get_lrm_resource(lrm_state, xml_rsc, create_rsc, &rsc);
         if (rc == -ENOTCONN) {
             synthesize_lrmd_failure(lrm_state, input->xml,
                                     PCMK_OCF_CONNECTION_DIED);
             return;
 
         } else if (!create_rsc) {
             /* Delete of malformed or nonexistent resource
              * (deleting something that does not exist is a success)
              */
             crm_notice("Not registering resource '%s' for a %s event "
                        CRM_XS " get-rc=%d (%s) transition-key=%s",
                        ID(xml_rsc), operation,
                        rc, pcmk_strerror(rc), ID(input->xml));
             delete_rsc_entry(lrm_state, input, ID(xml_rsc), NULL, pcmk_ok,
                              user_name);
             send_task_ok_ack(lrm_state, input, ID(xml_rsc), NULL, operation,
                              from_host, from_sys);
             return;
 
         } else if (rc == -EINVAL) {
             // Resource operation on malformed resource
             crm_err("Invalid resource definition for %s", ID(xml_rsc));
             crm_log_xml_warn(input->msg, "invalid resource");
             synthesize_lrmd_failure(lrm_state, input->xml,
                                     PCMK_OCF_NOT_CONFIGURED); // fatal error
             return;
 
         } else if (rc < 0) {
             // Error communicating with lrmd
             crm_err("Could not register resource '%s' with lrmd: %s " CRM_XS " rc=%d",
                     ID(xml_rsc), pcmk_strerror(rc), rc);
             crm_log_xml_warn(input->msg, "failed registration");
             synthesize_lrmd_failure(lrm_state, input->xml,
                                     PCMK_OCF_INVALID_PARAM); // hard error
             return;
         }
 
         if (safe_str_eq(operation, CRMD_ACTION_CANCEL)) {
             if (!do_lrm_cancel(input, lrm_state, rsc, from_host, from_sys)) {
                 crm_log_xml_warn(input->xml, "Bad command");
             }
 
         } else if (safe_str_eq(operation, CRMD_ACTION_DELETE)) {
             do_lrm_delete(input, lrm_state, rsc, from_sys, from_host,
                           crm_rsc_delete, user_name);
 
         } else {
             do_lrm_rsc_op(lrm_state, rsc, operation, input->xml, input->msg);
         }
 
         lrmd_free_rsc_info(rsc);
 
     } else {
         crm_err("Cannot perform operation %s of unknown type", crm_str(crm_op));
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
 }
 
 static lrmd_event_data_t *
 construct_op(lrm_state_t * lrm_state, xmlNode * rsc_op, const char *rsc_id, const char *operation)
 {
     lrmd_event_data_t *op = NULL;
     const char *op_delay = NULL;
     const char *op_timeout = NULL;
     const char *interval_ms_s = NULL;
     GHashTable *params = NULL;
 
     const char *transition = NULL;
 
     CRM_ASSERT(rsc_id && operation);
 
     op = calloc(1, sizeof(lrmd_event_data_t));
     CRM_ASSERT(op != NULL);
 
     op->type = lrmd_event_exec_complete;
     op->op_type = strdup(operation);
     op->op_status = PCMK_LRM_OP_PENDING;
     op->rc = -1;
     op->rsc_id = strdup(rsc_id);
-    op->interval = 0;
+    op->interval_ms = 0;
     op->timeout = 0;
     op->start_delay = 0;
 
     if (rsc_op == NULL) {
         CRM_LOG_ASSERT(safe_str_eq(CRMD_ACTION_STOP, operation));
         op->user_data = NULL;
         /* the stop_all_resources() case
          * by definition there is no DC (or they'd be shutting
          *   us down).
          * So we should put our version here.
          */
         op->params = crm_str_table_new();
 
         g_hash_table_insert(op->params, strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));
 
         crm_trace("Constructed %s op for %s", operation, rsc_id);
         return op;
     }
 
     params = xml2list(rsc_op);
     g_hash_table_remove(params, CRM_META "_op_target_rc");
 
     op_delay = crm_meta_value(params, XML_OP_ATTR_START_DELAY);
     op_timeout = crm_meta_value(params, XML_ATTR_TIMEOUT);
     interval_ms_s = crm_meta_value(params, XML_LRM_ATTR_INTERVAL_MS);
 
-    op->interval = crm_parse_int(interval_ms_s, "0");
+    op->interval_ms = crm_parse_ms(interval_ms_s);
     op->timeout = crm_parse_int(op_timeout, "0");
     op->start_delay = crm_parse_int(op_delay, "0");
 
 #if ENABLE_VERSIONED_ATTRS
     // Resolve any versioned parameters
     if (lrm_state && safe_str_neq(op->op_type, RSC_METADATA)
         && safe_str_neq(op->op_type, CRMD_ACTION_DELETE)
         && !is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
 
         // Resource info *should* already be cached, so we don't get lrmd call
         lrmd_rsc_info_t *rsc = lrm_state_get_rsc_info(lrm_state, rsc_id, 0);
         struct ra_metadata_s *metadata;
 
         metadata = metadata_cache_get(lrm_state->metadata_cache, rsc);
         if (metadata) {
             xmlNode *versioned_attrs = NULL;
             GHashTable *hash = NULL;
             char *key = NULL;
             char *value = NULL;
             GHashTableIter iter;
 
             versioned_attrs = first_named_child(rsc_op, XML_TAG_OP_VER_ATTRS);
             hash = pe_unpack_versioned_parameters(versioned_attrs, metadata->ra_version);
             g_hash_table_iter_init(&iter, hash);
             while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
                 g_hash_table_iter_steal(&iter);
                 g_hash_table_replace(params, key, value);
             }
             g_hash_table_destroy(hash);
 
             versioned_attrs = first_named_child(rsc_op, XML_TAG_OP_VER_META);
             hash = pe_unpack_versioned_parameters(versioned_attrs, metadata->ra_version);
             g_hash_table_iter_init(&iter, hash);
             while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
                 g_hash_table_replace(params, crm_meta_name(key), strdup(value));
 
                 if (safe_str_eq(key, XML_ATTR_TIMEOUT)) {
                     op->timeout = crm_parse_int(value, "0");
                 } else if (safe_str_eq(key, XML_OP_ATTR_START_DELAY)) {
                     op->start_delay = crm_parse_int(value, "0");
                 }
             }
             g_hash_table_destroy(hash);
 
             versioned_attrs = first_named_child(rsc_op, XML_TAG_RSC_VER_ATTRS);
             hash = pe_unpack_versioned_parameters(versioned_attrs, metadata->ra_version);
             g_hash_table_iter_init(&iter, hash);
             while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
                 g_hash_table_iter_steal(&iter);
                 g_hash_table_replace(params, key, value);
             }
             g_hash_table_destroy(hash);
         }
 
         lrmd_free_rsc_info(rsc);
     }
 #endif
 
     if (safe_str_neq(operation, RSC_STOP)) {
         op->params = params;
 
     } else {
         rsc_history_t *entry = NULL;
 
         if (lrm_state) {
             entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
         }
 
         /* If we do not have stop parameters cached, use
          * whatever we are given */
         if (!entry || !entry->stop_params) {
             op->params = params;
         } else {
             /* Copy the cached parameter list so that we stop the resource
              * with the old attributes, not the new ones */
             op->params = crm_str_table_new();
 
             g_hash_table_foreach(params, copy_meta_keys, op->params);
             g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
             g_hash_table_destroy(params);
             params = NULL;
         }
     }
 
     /* sanity */
-    if (op->interval < 0) {
-        op->interval = 0;
+    if (op->interval_ms < 0) {
+        op->interval_ms = 0;
     }
     if (op->timeout <= 0) {
-        op->timeout = op->interval;
+        op->timeout = op->interval_ms;
     }
     if (op->start_delay < 0) {
         op->start_delay = 0;
     }
 
     transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY);
     CRM_CHECK(transition != NULL, return op);
 
     op->user_data = strdup(transition);
 
-    if (op->interval != 0) {
+    if (op->interval_ms != 0) {
         if (safe_str_eq(operation, CRMD_ACTION_START)
             || safe_str_eq(operation, CRMD_ACTION_STOP)) {
-            crm_err("Start and Stop actions cannot have an interval: %d", op->interval);
-            op->interval = 0;
+            crm_err("Start and Stop actions cannot have an interval: %u",
+                    op->interval_ms);
+            op->interval_ms = 0;
         }
     }
 
-    crm_trace("Constructed %s op for %s: interval=%d", operation, rsc_id, op->interval);
+    crm_trace("Constructed %s op for %s: interval=%u",
+              operation, rsc_id, op->interval_ms);
 
     return op;
 }
 
 void
 send_direct_ack(const char *to_host, const char *to_sys,
                 lrmd_rsc_info_t * rsc, lrmd_event_data_t * op, const char *rsc_id)
 {
     xmlNode *reply = NULL;
     xmlNode *update, *iter;
     crm_node_t *peer = NULL;
 
     CRM_CHECK(op != NULL, return);
     if (op->rsc_id == NULL) {
         CRM_ASSERT(rsc_id != NULL);
         op->rsc_id = strdup(rsc_id);
     }
     if (to_sys == NULL) {
         to_sys = CRM_SYSTEM_TENGINE;
     }
 
     peer = crm_get_peer(0, fsa_our_uname);
     update = create_node_state_update(peer, node_update_none, NULL,
                                       __FUNCTION__);
 
     iter = create_xml_node(update, XML_CIB_TAG_LRM);
     crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
 
     crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);
 
     build_operation_update(iter, rsc, op, fsa_our_uname, __FUNCTION__);
     reply = create_request(CRM_OP_INVOKE_LRM, update, to_host, to_sys, CRM_SYSTEM_LRMD, NULL);
 
     crm_log_xml_trace(update, "ACK Update");
 
-    crm_debug("ACK'ing resource op %s_%s_%d from %s: %s",
-              op->rsc_id, op->op_type, op->interval, op->user_data,
+    crm_debug("ACK'ing resource op " CRM_OP_FMT " from %s: %s",
+              op->rsc_id, op->op_type, op->interval_ms, op->user_data,
               crm_element_value(reply, XML_ATTR_REFERENCE));
 
     if (relay_message(reply, TRUE) == FALSE) {
         crm_log_xml_err(reply, "Unable to route reply");
     }
 
     free_xml(update);
     free_xml(reply);
 }
 
 gboolean
 verify_stopped(enum crmd_fsa_state cur_state, int log_level)
 {
     gboolean res = TRUE;
     GList *lrm_state_list = lrm_state_get_list();
     GList *state_entry;
 
     for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
         lrm_state_t *lrm_state = state_entry->data;
 
         if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) {
             /* keep iterating through all even when false is returned */
             res = FALSE;
         }
     }
 
     set_bit(fsa_input_register, R_SENT_RSC_STOP);
     g_list_free(lrm_state_list); lrm_state_list = NULL;
     return res;
 }
 
 struct stop_recurring_action_s {
     lrmd_rsc_info_t *rsc;
     lrm_state_t *lrm_state;
 };
 
 static gboolean
 stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
 {
     gboolean remove = FALSE;
     struct stop_recurring_action_s *event = user_data;
     struct recurring_op_s *op = (struct recurring_op_s *)value;
 
     if ((op->interval_ms != 0)
         && crm_str_eq(op->rsc_id, event->rsc->id, TRUE)) {
 
         crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (char*)key);
         remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE);
     }
 
     return remove;
 }
 
 static gboolean
 stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
 {
     gboolean remove = FALSE;
     lrm_state_t *lrm_state = user_data;
     struct recurring_op_s *op = (struct recurring_op_s *)value;
 
     if (op->interval_ms != 0) {
         crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, key);
         remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE);
     }
 
     return remove;
 }
 
 static void
 record_pending_op(const char *node_name, lrmd_rsc_info_t *rsc, lrmd_event_data_t *op)
 {
     const char *record_pending = NULL;
 
     CRM_CHECK(node_name != NULL, return);
     CRM_CHECK(rsc != NULL, return);
     CRM_CHECK(op != NULL, return);
 
     if ((op->op_type == NULL) || (op->params == NULL)
         || safe_str_eq(op->op_type, CRMD_ACTION_CANCEL)
         || safe_str_eq(op->op_type, CRMD_ACTION_DELETE)) {
         return;
     }
 
     // defaults to true
     record_pending = crm_meta_value(op->params, XML_OP_ATTR_PENDING);
     if (record_pending && !crm_is_true(record_pending)) {
         return;
     }
 
     op->call_id = -1;
     op->op_status = PCMK_LRM_OP_PENDING;
     op->rc = PCMK_OCF_UNKNOWN;
 
     op->t_run = time(NULL);
     op->t_rcchange = op->t_run;
 
     /* write a "pending" entry to the CIB, inhibit notification */
-    crm_debug("Recording pending op %s_%s_%d on %s in the CIB",
-              op->rsc_id, op->op_type, op->interval, node_name);
+    crm_debug("Recording pending op " CRM_OP_FMT " on %s in the CIB",
+              op->rsc_id, op->op_type, op->interval_ms, node_name);
 
     do_update_resource(node_name, rsc, op);
 }
 
 static void
 do_lrm_rsc_op(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *operation, xmlNode * msg,
               xmlNode * request)
 {
     int call_id = 0;
     char *op_id = NULL;
     lrmd_event_data_t *op = NULL;
     lrmd_key_value_t *params = NULL;
     fsa_data_t *msg_data = NULL;
     const char *transition = NULL;
     gboolean stop_recurring = FALSE;
     bool send_nack = FALSE;
 
     CRM_CHECK(rsc != NULL, return);
     CRM_CHECK(operation != NULL, return);
 
     if (msg != NULL) {
         transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
         if (transition == NULL) {
             crm_log_xml_err(msg, "Missing transition number");
         }
     }
 
     op = construct_op(lrm_state, msg, rsc->id, operation);
     CRM_CHECK(op != NULL, return);
 
     if (is_remote_lrmd_ra(NULL, NULL, rsc->id)
-        && op->interval == 0
+        && (op->interval_ms == 0)
         && strcmp(operation, CRMD_ACTION_MIGRATE) == 0) {
 
         /* pcmk remote connections are a special use case.
          * We never ever want to stop monitoring a connection resource until
          * the entire migration has completed. If the connection is unexpectedly
          * severed, even during a migration, this is an event we must detect.*/
         stop_recurring = FALSE;
 
-    } else if (op->interval == 0
+    } else if ((op->interval_ms == 0)
         && strcmp(operation, CRMD_ACTION_STATUS) != 0
         && strcmp(operation, CRMD_ACTION_NOTIFY) != 0) {
 
         /* stop any previous monitor operations before changing the resource state */
         stop_recurring = TRUE;
     }
 
     if (stop_recurring == TRUE) {
         guint removed = 0;
         struct stop_recurring_action_s data;
 
         data.rsc = rsc;
         data.lrm_state = lrm_state;
         removed = g_hash_table_foreach_remove(
             lrm_state->pending_ops, stop_recurring_action_by_rsc, &data);
 
         if (removed) {
-            crm_debug("Stopped %u recurring operation%s in preparation for %s_%s_%d",
-                      removed, s_if_plural(removed), rsc->id, operation, op->interval);
+            crm_debug("Stopped %u recurring operation%s in preparation for " CRM_OP_FMT,
+                      removed, s_if_plural(removed),
+                      rsc->id, operation, op->interval_ms);
         }
     }
 
     /* now do the op */
-    crm_info("Performing key=%s op=%s_%s_%d", transition, rsc->id, operation, op->interval);
+    crm_info("Performing key=%s op=" CRM_OP_FMT,
+             transition, rsc->id, operation, op->interval_ms);
 
     if (is_set(fsa_input_register, R_SHUTDOWN) && safe_str_eq(operation, RSC_START)) {
         register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
         send_nack = TRUE;
 
     } else if (fsa_state != S_NOT_DC
                && fsa_state != S_POLICY_ENGINE /* Recalculating */
                && fsa_state != S_TRANSITION_ENGINE
                && safe_str_neq(operation, CRMD_ACTION_STOP)) {
         send_nack = TRUE;
     }
 
     if(send_nack) {
         crm_notice("Discarding attempt to perform action %s on %s in state %s (shutdown=%s)",
                    operation, rsc->id, fsa_state2string(fsa_state),
                    is_set(fsa_input_register, R_SHUTDOWN)?"true":"false");
 
         op->rc = CRM_DIRECT_NACK_RC;
         op->op_status = PCMK_LRM_OP_ERROR;
         send_direct_ack(NULL, NULL, rsc, op, rsc->id);
         lrmd_free_event(op);
         free(op_id);
         return;
     }
 
     record_pending_op(lrm_state->node_name, rsc, op);
 
-    op_id = generate_op_key(rsc->id, op->op_type, op->interval);
+    op_id = generate_op_key(rsc->id, op->op_type, op->interval_ms);
 
-    if (op->interval > 0) {
+    if (op->interval_ms > 0) {
         /* cancel it so we can then restart it without conflict */
         cancel_op_key(lrm_state, rsc, op_id, FALSE);
     }
 
     if (op->params) {
         char *key = NULL;
         char *value = NULL;
         GHashTableIter iter;
 
         g_hash_table_iter_init(&iter, op->params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             params = lrmd_key_value_add(params, key, value);
         }
     }
 
-    call_id = lrm_state_exec(lrm_state,
-                             rsc->id,
-                             op->op_type,
-                             op->user_data, op->interval, op->timeout, op->start_delay, params);
+    call_id = lrm_state_exec(lrm_state, rsc->id, op->op_type, op->user_data,
+                             op->interval_ms, op->timeout, op->start_delay,
+                             params);
 
     if (call_id <= 0 && lrm_state_is_local(lrm_state)) {
         crm_err("Operation %s on %s failed: %d", operation, rsc->id, call_id);
         register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
 
     } else if (call_id <= 0) {
         crm_err("Operation %s on resource %s failed to execute on remote node %s: %d",
                 operation, rsc->id, lrm_state->node_name, call_id);
         fake_op_status(lrm_state, op, PCMK_LRM_OP_DONE, PCMK_OCF_UNKNOWN_ERROR);
         process_lrm_event(lrm_state, op, NULL);
 
     } else {
         /* record all operations so we can wait
          * for them to complete during shutdown
          */
         char *call_id_s = make_stop_id(rsc->id, call_id);
         struct recurring_op_s *pending = NULL;
 
         pending = calloc(1, sizeof(struct recurring_op_s));
         crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);
 
         pending->call_id = call_id;
-        pending->interval_ms = op->interval;
+        pending->interval_ms = op->interval_ms;
         pending->op_type = strdup(operation);
         pending->op_key = strdup(op_id);
         pending->rsc_id = strdup(rsc->id);
         pending->start_time = time(NULL);
         pending->user_data = strdup(op->user_data);
         g_hash_table_replace(lrm_state->pending_ops, call_id_s, pending);
 
-        if (op->interval > 0 && op->start_delay > START_DELAY_THRESHOLD) {
+        if ((op->interval_ms > 0)
+            && (op->start_delay > START_DELAY_THRESHOLD)) {
+
             char *uuid = NULL;
             int dummy = 0, target_rc = 0;
 
             crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);
 
             decode_transition_key(op->user_data, &uuid, &dummy, &dummy, &target_rc);
             free(uuid);
 
             op->rc = target_rc;
             op->op_status = PCMK_LRM_OP_DONE;
             send_direct_ack(NULL, NULL, rsc, op, rsc->id);
         }
 
         pending->params = op->params;
         op->params = NULL;
     }
 
     free(op_id);
     lrmd_free_event(op);
     return;
 }
 
 int last_resource_update = 0;
 
 static void
 cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     switch (rc) {
         case pcmk_ok:
         case -pcmk_err_diff_failed:
         case -pcmk_err_diff_resync:
             crm_trace("Resource update %d complete: rc=%d", call_id, rc);
             break;
         default:
             crm_warn("Resource update %d failed: (rc=%d) %s", call_id, rc, pcmk_strerror(rc));
     }
 
     if (call_id == last_resource_update) {
         last_resource_update = 0;
         trigger_fsa(fsa_source);
     }
 }
 
 static int
 do_update_resource(const char *node_name, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
 {
 /*
   <status>
   <nodes_status id=uname>
   <lrm>
   <lrm_resources>
   <lrm_resource id=...>
   </...>
 */
     int rc = pcmk_ok;
     xmlNode *update, *iter = NULL;
     int call_opt = crmd_cib_smart_opt();
     const char *uuid = NULL;
 
     CRM_CHECK(op != NULL, return 0);
 
     iter = create_xml_node(iter, XML_CIB_TAG_STATUS);
     update = iter;
     iter = create_xml_node(iter, XML_CIB_TAG_STATE);
 
     if (safe_str_eq(node_name, fsa_our_uname)) {
         uuid = fsa_our_uuid;
 
     } else {
         /* remote nodes uuid and uname are equal */
         uuid = node_name;
         crm_xml_add(iter, XML_NODE_IS_REMOTE, "true");
     }
 
     CRM_LOG_ASSERT(uuid != NULL);
     if(uuid == NULL) {
         rc = -EINVAL;
         goto done;
     }
 
     crm_xml_add(iter, XML_ATTR_UUID,  uuid);
     crm_xml_add(iter, XML_ATTR_UNAME, node_name);
     crm_xml_add(iter, XML_ATTR_ORIGIN, __FUNCTION__);
 
     iter = create_xml_node(iter, XML_CIB_TAG_LRM);
     crm_xml_add(iter, XML_ATTR_ID, uuid);
 
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
     iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE);
     crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);
 
     build_operation_update(iter, rsc, op, node_name, __FUNCTION__);
 
     if (rsc) {
         const char *container = NULL;
 
         crm_xml_add(iter, XML_ATTR_TYPE, rsc->type);
         crm_xml_add(iter, XML_AGENT_ATTR_CLASS, rsc->standard);
         crm_xml_add(iter, XML_AGENT_ATTR_PROVIDER, rsc->provider);
 
         if (op->params) {
             container = g_hash_table_lookup(op->params, CRM_META"_"XML_RSC_ATTR_CONTAINER);
         }
         if (container) {
             crm_trace("Resource %s is a part of container resource %s", op->rsc_id, container);
             crm_xml_add(iter, XML_RSC_ATTR_CONTAINER, container);
         }
 
     } else {
         crm_warn("Resource %s no longer exists in the lrmd", op->rsc_id);
         send_direct_ack(NULL, NULL, rsc, op, op->rsc_id);
         goto cleanup;
     }
 
     crm_log_xml_trace(update, __FUNCTION__);
 
     /* make it an asynchronous call and be done with it
      *
      * Best case:
      *   the resource state will be discovered during
      *   the next signup or election.
      *
      * Bad case:
      *   we are shutting down and there is no DC at the time,
      *   but then why were we shutting down then anyway?
      *   (probably because of an internal error)
      *
      * Worst case:
      *   we get shot for having resources "running" when the really weren't
      *
      * the alternative however means blocking here for too long, which
      * isn't acceptable
      */
     fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, rc, NULL);
 
     if (rc > 0) {
         last_resource_update = rc;
     }
   done:
     /* the return code is a call number, not an error code */
-    crm_trace("Sent resource state update message: %d for %s=%d on %s", rc,
-              op->op_type, op->interval, op->rsc_id);
+    crm_trace("Sent resource state update message: %d for %s=%u on %s",
+              rc, op->op_type, op->interval_ms, op->rsc_id);
     fsa_register_cib_callback(rc, FALSE, NULL, cib_rsc_callback);
 
   cleanup:
     free_xml(update);
     return rc;
 }
 
 void
 do_lrm_event(long long action,
              enum crmd_fsa_cause cause,
              enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
 {
     CRM_CHECK(FALSE, return);
 }
 
 static char *
 unescape_newlines(const char *string)
 {
     char *pch = NULL;
     char *ret = NULL;
     static const char *escaped_newline = "\\n";
 
     if (!string) {
         return NULL;
     }
 
     ret = strdup(string);
     pch = strstr(ret, escaped_newline);
     while (pch != NULL) {
         /* 2 chars for 2 chars, null-termination irrelevant */
         memcpy(pch, "\n ", 2 * sizeof(char));
         pch = strstr(pch, escaped_newline);
     }
 
     return ret;
 }
 
 gboolean
 process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurring_op_s *pending)
 {
     char *op_id = NULL;
     char *op_key = NULL;
 
     int update_id = 0;
     gboolean remove = FALSE;
     gboolean removed = FALSE;
     lrmd_rsc_info_t *rsc = NULL;
 
     CRM_CHECK(op != NULL, return FALSE);
     CRM_CHECK(op->rsc_id != NULL, return FALSE);
 
     op_id = make_stop_id(op->rsc_id, op->call_id);
-    op_key = generate_op_key(op->rsc_id, op->op_type, op->interval);
+    op_key = generate_op_key(op->rsc_id, op->op_type, op->interval_ms);
     rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
     if(pending == NULL) {
         remove = TRUE;
         pending = g_hash_table_lookup(lrm_state->pending_ops, op_id);
     }
 
     if (op->op_status == PCMK_LRM_OP_ERROR) {
         switch(op->rc) {
             case PCMK_OCF_NOT_RUNNING:
             case PCMK_OCF_RUNNING_MASTER:
             case PCMK_OCF_DEGRADED:
             case PCMK_OCF_DEGRADED_MASTER:
                 /* Leave it up to the TE/PE to decide if this is an error */
                 op->op_status = PCMK_LRM_OP_DONE;
                 break;
             default:
                 /* Nothing to do */
                 break;
         }
     }
 
     if (op->op_status != PCMK_LRM_OP_CANCELLED) {
         if (safe_str_eq(op->op_type, RSC_NOTIFY) || safe_str_eq(op->op_type, RSC_METADATA)) {
             /* Keep notify and meta-data ops out of the CIB */
             send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);
         } else {
             update_id = do_update_resource(lrm_state->node_name, rsc, op);
         }
-    } else if (op->interval == 0) {
+    } else if (op->interval_ms == 0) {
         /* This will occur when "crm resource cleanup" is called while actions are in-flight */
         crm_err("Op %s (call=%d): Cancelled", op_key, op->call_id);
         send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);
 
     } else if (pending == NULL) {
         /* We don't need to do anything for cancelled ops
          * that are not in our pending op list. There are no
          * transition actions waiting on these operations. */
 
     } else if (op->user_data == NULL) {
         /* At this point we have a pending entry, but no transition
          * key present in the user_data field. report this */
         crm_err("Op %s (call=%d): No user data", op_key, op->call_id);
 
     } else if (pending->remove) {
         /* The tengine canceled this op, we have been waiting for the cancel to finish. */
         erase_lrm_history_by_op(lrm_state, op);
 
     } else if (pending && op->rsc_deleted) {
         /* The tengine initiated this op, but it was cancelled outside of the
          * tengine's control during a resource cleanup/re-probe request. The tengine
          * must be alerted that this operation completed, otherwise the tengine
          * will continue waiting for this update to occur until it is timed out.
          * We don't want this update going to the cib though, so use a direct ack. */
         crm_trace("Op %s (call=%d): cancelled due to rsc deletion", op_key, op->call_id);
         send_direct_ack(NULL, NULL, NULL, op, op->rsc_id);
 
     } else {
         /* Before a stop is called, no need to direct ack */
         crm_trace("Op %s (call=%d): no delete event required", op_key, op->call_id);
     }
 
     if(remove == FALSE) {
         /* The caller will do this afterwards, but keep the logging consistent */
         removed = TRUE;
 
-    } else if ((op->interval == 0) && g_hash_table_remove(lrm_state->pending_ops, op_id)) {
+    } else if ((op->interval_ms == 0)
+               && g_hash_table_remove(lrm_state->pending_ops, op_id)) {
+
         removed = TRUE;
         crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
                   op_key, op->call_id, op_id, g_hash_table_size(lrm_state->pending_ops));
 
-    } else if(op->interval != 0 && op->op_status == PCMK_LRM_OP_CANCELLED) {
+    } else if ((op->interval_ms != 0)
+               && (op->op_status == PCMK_LRM_OP_CANCELLED)) {
+
         removed = TRUE;
         g_hash_table_remove(lrm_state->pending_ops, op_id);
     }
 
     switch (op->op_status) {
         case PCMK_LRM_OP_CANCELLED:
             crm_info("Result of %s operation for %s on %s: %s "
                      CRM_XS " call=%d key=%s confirmed=%s",
-                     crm_action_str(op->op_type, op->interval),
+                     crm_action_str(op->op_type, op->interval_ms),
                      op->rsc_id, lrm_state->node_name,
                      services_lrm_status_str(op->op_status),
                      op->call_id, op_key, (removed? "true" : "false"));
             break;
 
         case PCMK_LRM_OP_DONE:
-            do_crm_log(op->interval?LOG_INFO:LOG_NOTICE,
+            do_crm_log((op->interval_ms? LOG_INFO : LOG_NOTICE),
                        "Result of %s operation for %s on %s: %d (%s) "
                        CRM_XS " call=%d key=%s confirmed=%s cib-update=%d",
-                       crm_action_str(op->op_type, op->interval),
+                       crm_action_str(op->op_type, op->interval_ms),
                        op->rsc_id, lrm_state->node_name,
                        op->rc, services_ocf_exitcode_str(op->rc),
                        op->call_id, op_key, (removed? "true" : "false"),
                        update_id);
             break;
 
         case PCMK_LRM_OP_TIMEOUT:
             crm_err("Result of %s operation for %s on %s: %s "
                     CRM_XS " call=%d key=%s timeout=%dms",
-                    crm_action_str(op->op_type, op->interval),
+                    crm_action_str(op->op_type, op->interval_ms),
                     op->rsc_id, lrm_state->node_name,
                     services_lrm_status_str(op->op_status),
                     op->call_id, op_key, op->timeout);
             break;
 
         default:
             crm_err("Result of %s operation for %s on %s: %s "
                     CRM_XS " call=%d key=%s confirmed=%s status=%d cib-update=%d",
-                    crm_action_str(op->op_type, op->interval),
+                    crm_action_str(op->op_type, op->interval_ms),
                     op->rsc_id, lrm_state->node_name,
                     services_lrm_status_str(op->op_status), op->call_id, op_key,
                     (removed? "true" : "false"), op->op_status, update_id);
     }
 
     if (op->output) {
         char *prefix =
-            crm_strdup_printf("%s-%s_%s_%d:%d", lrm_state->node_name, op->rsc_id, op->op_type, op->interval, op->call_id);
+            crm_strdup_printf("%s-" CRM_OP_FMT ":%d", lrm_state->node_name,
+                              op->rsc_id, op->op_type, op->interval_ms,
+                              op->call_id);
 
         if (op->rc) {
             crm_log_output(LOG_NOTICE, prefix, op->output);
         } else {
             crm_log_output(LOG_DEBUG, prefix, op->output);
         }
         free(prefix);
     }
 
     if (safe_str_neq(op->op_type, RSC_METADATA)) {
         crmd_alert_resource_op(lrm_state->node_name, op);
     } else if (op->rc == PCMK_OCF_OK) {
         char *metadata = unescape_newlines(op->output);
 
         metadata_cache_update(lrm_state->metadata_cache, rsc, metadata);
         free(metadata);
     }
 
     if (op->rsc_deleted) {
         crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
         delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL);
     }
 
     /* If a shutdown was escalated while operations were pending,
      * then the FSA will be stalled right now... allow it to continue
      */
     mainloop_set_trigger(fsa_source);
     update_history_cache(lrm_state, rsc, op);
 
     lrmd_free_rsc_info(rsc);
     free(op_key);
     free(op_id);
 
     return TRUE;
 }
diff --git a/crmd/lrm_state.c b/crmd/lrm_state.c
index e0b3e2426b..829d6a9dd0 100644
--- a/crmd/lrm_state.c
+++ b/crmd/lrm_state.c
@@ -1,801 +1,801 @@
 /* 
  * Copyright (C) 2012 David Vossel <davidvossel@gmail.com>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/iso8601.h>
 
 #include <crmd.h>
 #include <crmd_fsa.h>
 #include <crmd_messages.h>
 #include <crmd_callbacks.h>
 #include <crmd_lrm.h>
 #include <crmd_alerts.h>
 #include <crm/pengine/rules.h>
 #include <crm/pengine/rules_internal.h>
 #include <crm/transition.h>
 #include <crm/lrmd_alerts_internal.h>
 
 GHashTable *lrm_state_table = NULL;
 extern GHashTable *proxy_table;
 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
 
 static void
 free_rsc_info(gpointer value)
 {
     lrmd_rsc_info_t *rsc_info = value;
 
     lrmd_free_rsc_info(rsc_info);
 }
 
 static void
 free_deletion_op(gpointer value)
 {
     struct pending_deletion_op_s *op = value;
 
     free(op->rsc);
     delete_ha_msg_input(op->input);
     free(op);
 }
 
 static void
 free_recurring_op(gpointer value)
 {
     struct recurring_op_s *op = (struct recurring_op_s *)value;
 
     free(op->user_data);
     free(op->rsc_id);
     free(op->op_type);
     free(op->op_key);
     if (op->params) {
         g_hash_table_destroy(op->params);
     }
     free(op);
 }
 
 static gboolean
 fail_pending_op(gpointer key, gpointer value, gpointer user_data)
 {
     lrmd_event_data_t event = { 0, };
     lrm_state_t *lrm_state = user_data;
     struct recurring_op_s *op = (struct recurring_op_s *)value;
 
     crm_trace("Pre-emptively failing " CRM_OP_FMT " on %s (call=%s, %s)",
               op->rsc_id, op->op_type, op->interval_ms,
               lrm_state->node_name, (char*)key, op->user_data);
 
     event.type = lrmd_event_exec_complete;
     event.rsc_id = op->rsc_id;
     event.op_type = op->op_type;
     event.user_data = op->user_data;
     event.timeout = 0;
-    event.interval = op->interval_ms;
+    event.interval_ms = op->interval_ms;
     event.rc = PCMK_OCF_CONNECTION_DIED;
     event.op_status = PCMK_LRM_OP_ERROR;
     event.t_run = op->start_time;
     event.t_rcchange = op->start_time;
 
     event.call_id = op->call_id;
     event.remote_nodename = lrm_state->node_name;
     event.params = op->params;
 
     process_lrm_event(lrm_state, &event, op);
     return TRUE;
 }
 
 gboolean
 lrm_state_is_local(lrm_state_t *lrm_state)
 {
     if (lrm_state == NULL || fsa_our_uname == NULL) {
         return FALSE;
     }
 
     if (strcmp(lrm_state->node_name, fsa_our_uname) != 0) {
         return FALSE;
     }
 
     return TRUE;
 
 }
 
 lrm_state_t *
 lrm_state_create(const char *node_name)
 {
     lrm_state_t *state = NULL;
 
     if (!node_name) {
         crm_err("No node name given for lrm state object");
         return NULL;
     }
 
     state = calloc(1, sizeof(lrm_state_t));
     if (!state) {
         return NULL;
     }
 
     state->node_name = strdup(node_name);
 
     state->rsc_info_cache = g_hash_table_new_full(crm_str_hash,
                                                 g_str_equal, NULL, free_rsc_info);
 
     state->deletion_ops = g_hash_table_new_full(crm_str_hash, g_str_equal, free,
                                                 free_deletion_op);
 
     state->pending_ops = g_hash_table_new_full(crm_str_hash, g_str_equal, free,
                                                free_recurring_op);
 
     state->resource_history = g_hash_table_new_full(crm_str_hash,
                                                     g_str_equal, NULL, history_free);
 
     state->metadata_cache = metadata_cache_new();
 
     g_hash_table_insert(lrm_state_table, (char *)state->node_name, state);
     return state;
 
 }
 
 void
 lrm_state_destroy(const char *node_name)
 {
     g_hash_table_remove(lrm_state_table, node_name);
 }
 
 static gboolean
 remote_proxy_remove_by_node(gpointer key, gpointer value, gpointer user_data)
 {
     remote_proxy_t *proxy = value;
     const char *node_name = user_data;
 
     if (safe_str_eq(node_name, proxy->node_name)) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 static void
 internal_lrm_state_destroy(gpointer data)
 {
     lrm_state_t *lrm_state = data;
 
     if (!lrm_state) {
         return;
     }
 
     crm_trace("Destroying proxy table %s with %d members", lrm_state->node_name, g_hash_table_size(proxy_table));
     g_hash_table_foreach_remove(proxy_table, remote_proxy_remove_by_node, (char *) lrm_state->node_name);
     remote_ra_cleanup(lrm_state);
     lrmd_api_delete(lrm_state->conn);
 
     if (lrm_state->rsc_info_cache) {
         crm_trace("Destroying rsc info cache with %d members", g_hash_table_size(lrm_state->rsc_info_cache));
         g_hash_table_destroy(lrm_state->rsc_info_cache);
     }
     if (lrm_state->resource_history) {
         crm_trace("Destroying history op cache with %d members", g_hash_table_size(lrm_state->resource_history));
         g_hash_table_destroy(lrm_state->resource_history);
     }
     if (lrm_state->deletion_ops) {
         crm_trace("Destroying deletion op cache with %d members", g_hash_table_size(lrm_state->deletion_ops));
         g_hash_table_destroy(lrm_state->deletion_ops);
     }
     if (lrm_state->pending_ops) {
         crm_trace("Destroying pending op cache with %d members", g_hash_table_size(lrm_state->pending_ops));
         g_hash_table_destroy(lrm_state->pending_ops);
     }
     metadata_cache_free(lrm_state->metadata_cache);
 
     free((char *)lrm_state->node_name);
     free(lrm_state);
 }
 
 void
 lrm_state_reset_tables(lrm_state_t * lrm_state, gboolean reset_metadata)
 {
     if (lrm_state->resource_history) {
         crm_trace("Re-setting history op cache with %d members",
                   g_hash_table_size(lrm_state->resource_history));
         g_hash_table_remove_all(lrm_state->resource_history);
     }
     if (lrm_state->deletion_ops) {
         crm_trace("Re-setting deletion op cache with %d members",
                   g_hash_table_size(lrm_state->deletion_ops));
         g_hash_table_remove_all(lrm_state->deletion_ops);
     }
     if (lrm_state->pending_ops) {
         crm_trace("Re-setting pending op cache with %d members",
                   g_hash_table_size(lrm_state->pending_ops));
         g_hash_table_remove_all(lrm_state->pending_ops);
     }
     if (lrm_state->rsc_info_cache) {
         crm_trace("Re-setting rsc info cache with %d members",
                   g_hash_table_size(lrm_state->rsc_info_cache));
         g_hash_table_remove_all(lrm_state->rsc_info_cache);
     }
     if (reset_metadata) {
         metadata_cache_reset(lrm_state->metadata_cache);
     }
 }
 
 gboolean
 lrm_state_init_local(void)
 {
     if (lrm_state_table) {
         return TRUE;
     }
 
     lrm_state_table =
         g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, internal_lrm_state_destroy);
     if (!lrm_state_table) {
         return FALSE;
     }
 
     proxy_table =
         g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, remote_proxy_free);
     if (!proxy_table) {
         g_hash_table_destroy(lrm_state_table);
         lrm_state_table = NULL;
         return FALSE;
     }
 
     return TRUE;
 }
 
 void
 lrm_state_destroy_all(void)
 {
     if (lrm_state_table) {
         crm_trace("Destroying state table with %d members", g_hash_table_size(lrm_state_table));
         g_hash_table_destroy(lrm_state_table); lrm_state_table = NULL;
     }
     if(proxy_table) {
         crm_trace("Destroying proxy table with %d members", g_hash_table_size(proxy_table));
         g_hash_table_destroy(proxy_table); proxy_table = NULL;
     }
 }
 
 lrm_state_t *
 lrm_state_find(const char *node_name)
 {
     if (!node_name) {
         return NULL;
     }
     return g_hash_table_lookup(lrm_state_table, node_name);
 }
 
 lrm_state_t *
 lrm_state_find_or_create(const char *node_name)
 {
     lrm_state_t *lrm_state;
 
     lrm_state = g_hash_table_lookup(lrm_state_table, node_name);
     if (!lrm_state) {
         lrm_state = lrm_state_create(node_name);
     }
 
     return lrm_state;
 }
 
 GList *
 lrm_state_get_list(void)
 {
     return g_hash_table_get_values(lrm_state_table);
 }
 
 static remote_proxy_t *
 find_connected_proxy_by_node(const char * node_name)
 {
     GHashTableIter gIter;
     remote_proxy_t *proxy = NULL;
 
     CRM_CHECK(proxy_table != NULL, return NULL);
 
     g_hash_table_iter_init(&gIter, proxy_table);
 
     while (g_hash_table_iter_next(&gIter, NULL, (gpointer *) &proxy)) {
         if (proxy->source
             && safe_str_eq(node_name, proxy->node_name)) {
             return proxy;
         }
     }
 
     return NULL;
 }
 
 static void
 remote_proxy_disconnect_by_node(const char * node_name)
 {
     remote_proxy_t *proxy = NULL;
 
     CRM_CHECK(proxy_table != NULL, return);
 
     while ((proxy = find_connected_proxy_by_node(node_name)) != NULL) {
         /* mainloop_del_ipc_client() eventually calls remote_proxy_disconnected()
          * , which removes the entry from proxy_table.
          * Do not do this in a g_hash_table_iter_next() loop. */
         if (proxy->source) {
             mainloop_del_ipc_client(proxy->source);
         }
     }
 
     return;
 }
 
 void
 lrm_state_disconnect_only(lrm_state_t * lrm_state)
 {
     int removed = 0;
 
     if (!lrm_state->conn) {
         return;
     }
     crm_trace("Disconnecting %s", lrm_state->node_name);
 
     remote_proxy_disconnect_by_node(lrm_state->node_name);
 
     ((lrmd_t *) lrm_state->conn)->cmds->disconnect(lrm_state->conn);
 
     if (is_not_set(fsa_input_register, R_SHUTDOWN)) {
         removed = g_hash_table_foreach_remove(lrm_state->pending_ops, fail_pending_op, lrm_state);
         crm_trace("Synthesized %d operation failures for %s", removed, lrm_state->node_name);
     }
 }
 
 void
 lrm_state_disconnect(lrm_state_t * lrm_state)
 {
     if (!lrm_state->conn) {
         return;
     }
 
     lrm_state_disconnect_only(lrm_state);
 
     lrmd_api_delete(lrm_state->conn);
     lrm_state->conn = NULL;
 }
 
 int
 lrm_state_is_connected(lrm_state_t * lrm_state)
 {
     if (!lrm_state->conn) {
         return FALSE;
     }
     return ((lrmd_t *) lrm_state->conn)->cmds->is_connected(lrm_state->conn);
 }
 
 int
 lrm_state_poke_connection(lrm_state_t * lrm_state)
 {
 
     if (!lrm_state->conn) {
         return -1;
     }
     return ((lrmd_t *) lrm_state->conn)->cmds->poke_connection(lrm_state->conn);
 }
 
 int
 lrm_state_ipc_connect(lrm_state_t * lrm_state)
 {
     int ret;
 
     if (!lrm_state->conn) {
         lrm_state->conn = lrmd_api_new();
         ((lrmd_t *) lrm_state->conn)->cmds->set_callback(lrm_state->conn, lrm_op_callback);
     }
 
     ret = ((lrmd_t *) lrm_state->conn)->cmds->connect(lrm_state->conn, CRM_SYSTEM_CRMD, NULL);
 
     if (ret != pcmk_ok) {
         lrm_state->num_lrm_register_fails++;
     } else {
         lrm_state->num_lrm_register_fails = 0;
     }
 
     return ret;
 }
 
 static remote_proxy_t *
 crmd_remote_proxy_new(lrmd_t *lrmd, const char *node_name, const char *session_id, const char *channel)
 {
     static struct ipc_client_callbacks proxy_callbacks = {
         .dispatch = remote_proxy_dispatch,
         .destroy = remote_proxy_disconnected
     };
     remote_proxy_t *proxy = remote_proxy_new(lrmd, &proxy_callbacks, node_name,
                                              session_id, channel);
     return proxy;
 }
 
 gboolean
 crmd_is_proxy_session(const char *session)
 {
     return g_hash_table_lookup(proxy_table, session) ? TRUE : FALSE;
 }
 
 void
 crmd_proxy_send(const char *session, xmlNode *msg)
 {
     remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);
     lrm_state_t *lrm_state = NULL;
 
     if (!proxy) {
         return;
     }
     crm_log_xml_trace(msg, "to-proxy");
     lrm_state = lrm_state_find(proxy->node_name);
     if (lrm_state) {
         crm_trace("Sending event to %.8s on %s", proxy->session_id, proxy->node_name);
         remote_proxy_relay_event(proxy, msg);
     }
 }
 
 static void
 crmd_proxy_dispatch(const char *session, xmlNode *msg)
 {
 
     crm_log_xml_trace(msg, "CRMd-PROXY[inbound]");
 
     crm_xml_add(msg, F_CRM_SYS_FROM, session);
     if (crmd_authorize_message(msg, NULL, session)) {
         route_message(C_IPC_MESSAGE, msg);
     }
 
     trigger_fsa(fsa_source);
 }
 
 static void
 remote_config_check(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     if (rc != pcmk_ok) {
         crm_err("Query resulted in an error: %s", pcmk_strerror(rc));
 
         if (rc == -EACCES || rc == -pcmk_err_schema_validation) {
             crm_err("The cluster is mis-configured - shutting down and staying down");
         }
 
     } else {
         lrmd_t * lrmd = (lrmd_t *)user_data;
         crm_time_t *now = crm_time_new(NULL);
         GHashTable *config_hash = crm_str_table_new();
 
         crm_debug("Call %d : Parsing CIB options", call_id);
 
         unpack_instance_attributes(
             output, output, XML_CIB_TAG_PROPSET, NULL, config_hash, CIB_OPTIONS_FIRST, FALSE, now);
 
         /* Now send it to the remote peer */
         remote_proxy_check(lrmd, config_hash);
 
         g_hash_table_destroy(config_hash);
         crm_time_free(now);
     }
 }
 
 static void
 crmd_remote_proxy_cb(lrmd_t *lrmd, void *userdata, xmlNode *msg)
 {
     lrm_state_t *lrm_state = userdata;
     const char *session = crm_element_value(msg, F_LRMD_IPC_SESSION);
     remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session);
 
     const char *op = crm_element_value(msg, F_LRMD_IPC_OP);
     if (safe_str_eq(op, LRMD_IPC_OP_NEW)) {
         const char *channel = crm_element_value(msg, F_LRMD_IPC_IPC_SERVER);
 
         proxy = crmd_remote_proxy_new(lrmd, lrm_state->node_name, session, channel);
         if (proxy != NULL) {
             /* Look up stonith-watchdog-timeout and send to the remote peer for validation */
             int rc = fsa_cib_conn->cmds->query(fsa_cib_conn, XML_CIB_TAG_CRMCONFIG, NULL, cib_scope_local);
             fsa_cib_conn->cmds->register_callback_full(fsa_cib_conn, rc, 10, FALSE, lrmd,
                                                        "remote_config_check", remote_config_check, NULL);
         }
 
     } else if (safe_str_eq(op, LRMD_IPC_OP_SHUTDOWN_REQ)) {
         char *now_s = NULL;
         time_t now = time(NULL);
 
         crm_notice("%s requested shutdown of its remote connection",
                    lrm_state->node_name);
 
         if (!remote_ra_is_in_maintenance(lrm_state)) {
             now_s = crm_itoa(now);
             update_attrd(lrm_state->node_name, XML_CIB_ATTR_SHUTDOWN, now_s, NULL, TRUE);
             free(now_s);
 
             remote_proxy_ack_shutdown(lrmd);
 
             crm_warn("Reconnection attempts to %s may result in failures that must be cleared",
                     lrm_state->node_name);
         } else {
             remote_proxy_nack_shutdown(lrmd);
 
             crm_notice("Remote resource for %s is not managed so no ordered shutdown happening",
                     lrm_state->node_name);
         }
         return;
 
     } else if (safe_str_eq(op, LRMD_IPC_OP_REQUEST) && proxy && proxy->is_local) {
         /* this is for the crmd, which we are, so don't try
          * and connect/send to ourselves over ipc. instead
          * do it directly.
          */
         int flags = 0;
         xmlNode *request = get_message_xml(msg, F_LRMD_IPC_MSG);
 
         CRM_CHECK(request != NULL, return);
 #if ENABLE_ACL
         CRM_CHECK(lrm_state->node_name, return);
         crm_xml_add(request, XML_ACL_TAG_ROLE, "pacemaker-remote");
         crm_acl_get_set_user(request, F_LRMD_IPC_USER, lrm_state->node_name);
 #endif
         crmd_proxy_dispatch(session, request);
 
         crm_element_value_int(msg, F_LRMD_IPC_MSG_FLAGS, &flags);
         if (flags & crm_ipc_client_response) {
             int msg_id = 0;
             xmlNode *op_reply = create_xml_node(NULL, "ack");
 
             crm_xml_add(op_reply, "function", __FUNCTION__);
             crm_xml_add_int(op_reply, "line", __LINE__);
 
             crm_element_value_int(msg, F_LRMD_IPC_MSG_ID, &msg_id);
             remote_proxy_relay_response(proxy, op_reply, msg_id);
 
             free_xml(op_reply);
         }
 
     } else {
         remote_proxy_cb(lrmd, lrm_state->node_name, msg);
     }
 }
 
 
 int
 lrm_state_remote_connect_async(lrm_state_t * lrm_state, const char *server, int port,
                                int timeout_ms)
 {
     int ret;
 
     if (!lrm_state->conn) {
         lrm_state->conn = lrmd_remote_api_new(lrm_state->node_name, server, port);
         if (!lrm_state->conn) {
             return -1;
         }
         ((lrmd_t *) lrm_state->conn)->cmds->set_callback(lrm_state->conn, remote_lrm_op_callback);
         lrmd_internal_set_proxy_callback(lrm_state->conn, lrm_state, crmd_remote_proxy_cb);
     }
 
     crm_trace("initiating remote connection to %s at %d with timeout %d", server, port, timeout_ms);
     ret =
         ((lrmd_t *) lrm_state->conn)->cmds->connect_async(lrm_state->conn, lrm_state->node_name,
                                                           timeout_ms);
 
     if (ret != pcmk_ok) {
         lrm_state->num_lrm_register_fails++;
     } else {
         lrm_state->num_lrm_register_fails = 0;
     }
 
     return ret;
 }
 
 int
 lrm_state_get_metadata(lrm_state_t * lrm_state,
                        const char *class,
                        const char *provider,
                        const char *agent, char **output, enum lrmd_call_options options)
 {
     if (!lrm_state->conn) {
         return -ENOTCONN;
     }
     return ((lrmd_t *) lrm_state->conn)->cmds->get_metadata(lrm_state->conn, class, provider, agent,
                                                             output, options);
 }
 
 int
 lrm_state_cancel(lrm_state_t *lrm_state, const char *rsc_id, const char *action,
                  guint interval_ms)
 {
     if (!lrm_state->conn) {
         return -ENOTCONN;
     }
 
     /* Figure out a way to make this async?
      * NOTICE: Currently it's synced and directly acknowledged in do_lrm_invoke(). */
     if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
         return remote_ra_cancel(lrm_state, rsc_id, action, interval_ms);
     }
     return ((lrmd_t *) lrm_state->conn)->cmds->cancel(lrm_state->conn, rsc_id,
                                                       action, interval_ms);
 }
 
 lrmd_rsc_info_t *
 lrm_state_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id, enum lrmd_call_options options)
 {
     lrmd_rsc_info_t *rsc = NULL;
 
     if (!lrm_state->conn) {
         return NULL;
     }
     if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
         return remote_ra_get_rsc_info(lrm_state, rsc_id);
     }
 
     rsc = g_hash_table_lookup(lrm_state->rsc_info_cache, rsc_id);
     if (rsc == NULL) {
         /* only contact the lrmd if we don't already have a cached rsc info */
         rsc = ((lrmd_t *) lrm_state->conn)->cmds->get_rsc_info(lrm_state->conn, rsc_id, options);
         if (rsc == NULL) {
 		    return NULL;
         }
         /* cache the result */
         g_hash_table_insert(lrm_state->rsc_info_cache, rsc->id, rsc);
     }
 
     return lrmd_copy_rsc_info(rsc);
 
 }
 
 int
 lrm_state_exec(lrm_state_t *lrm_state, const char *rsc_id, const char *action,
                const char *userdata, guint interval_ms,
                int timeout,     /* ms */
                int start_delay, /* ms */
                lrmd_key_value_t * params)
 {
 
     if (!lrm_state->conn) {
         lrmd_key_value_freeall(params);
         return -ENOTCONN;
     }
 
     if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
         return remote_ra_exec(lrm_state, rsc_id, action, userdata, interval_ms,
                               timeout, start_delay, params);
     }
 
     return ((lrmd_t *) lrm_state->conn)->cmds->exec(lrm_state->conn,
                                                     rsc_id,
                                                     action,
                                                     userdata,
                                                     interval_ms,
                                                     timeout,
                                                     start_delay,
                                                     lrmd_opt_notify_changes_only, params);
 }
 
 int
 lrm_state_register_rsc(lrm_state_t * lrm_state,
                        const char *rsc_id,
                        const char *class,
                        const char *provider, const char *agent, enum lrmd_call_options options)
 {
     lrmd_t *conn = (lrmd_t *) lrm_state->conn;
 
     if (conn == NULL) {
         return -ENOTCONN;
     }
 
     if (is_remote_lrmd_ra(agent, provider, NULL)) {
         return lrm_state_find_or_create(rsc_id)? pcmk_ok : -EINVAL;
     }
 
     /* @TODO Implement an asynchronous version of this (currently a blocking
      * call to the lrmd).
      */
     return conn->cmds->register_rsc(lrm_state->conn, rsc_id, class, provider,
                                     agent, options);
 }
 
 int
 lrm_state_unregister_rsc(lrm_state_t * lrm_state,
                          const char *rsc_id, enum lrmd_call_options options)
 {
     if (!lrm_state->conn) {
         return -ENOTCONN;
     }
 
     /* optimize this... this function is a synced round trip from client to daemon.
      * The crmd/lrm.c code path that uses this function should always treat it as an
      * async operation. The lrmd client api needs to make an async version unreg available. */
     if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) {
         lrm_state_destroy(rsc_id);
         return pcmk_ok;
     }
 
     g_hash_table_remove(lrm_state->rsc_info_cache, rsc_id);
 
     return ((lrmd_t *) lrm_state->conn)->cmds->unregister_rsc(lrm_state->conn, rsc_id, options);
 }
 
 /*
  * functions for sending alerts via local LRMD connection
  */
 
 static GListPtr crmd_alert_list = NULL;
 
 void
 crmd_unpack_alerts(xmlNode *alerts)
 {
     pe_free_alert_list(crmd_alert_list);
     crmd_alert_list = pe_unpack_alerts(alerts);
 }
 
 void
 crmd_alert_node_event(crm_node_t *node)
 {
     lrm_state_t *lrm_state;
 
     if (crmd_alert_list == NULL) {
         return;
     }
 
     lrm_state = lrm_state_find(fsa_our_uname);
     if (lrm_state == NULL) {
         return;
     }
 
     lrmd_send_node_alert((lrmd_t *) lrm_state->conn, crmd_alert_list,
                          node->uname, node->id, node->state);
 }
 
 void
 crmd_alert_fencing_op(stonith_event_t * e)
 {
     char *desc;
     lrm_state_t *lrm_state;
 
     if (crmd_alert_list == NULL) {
         return;
     }
 
     lrm_state = lrm_state_find(fsa_our_uname);
     if (lrm_state == NULL) {
         return;
     }
 
     desc = crm_strdup_printf("Operation %s of %s by %s for %s@%s: %s (ref=%s)",
                              e->action, e->target,
                              (e->executioner? e->executioner : "<no-one>"),
                              e->client_origin, e->origin,
                              pcmk_strerror(e->result), e->id);
 
     lrmd_send_fencing_alert((lrmd_t *) lrm_state->conn, crmd_alert_list,
                             e->target, e->operation, desc, e->result);
     free(desc);
 }
 
 void
 crmd_alert_resource_op(const char *node, lrmd_event_data_t * op)
 {
     lrm_state_t *lrm_state;
 
     if (crmd_alert_list == NULL) {
         return;
     }
 
     lrm_state = lrm_state_find(fsa_our_uname);
     if (lrm_state == NULL) {
         return;
     }
 
     lrmd_send_resource_alert((lrmd_t *) lrm_state->conn, crmd_alert_list, node,
                              op);
 }
diff --git a/crmd/remote_lrmd_ra.c b/crmd/remote_lrmd_ra.c
index d9911f88ce..57b0fdf721 100644
--- a/crmd/remote_lrmd_ra.c
+++ b/crmd/remote_lrmd_ra.c
@@ -1,1277 +1,1277 @@
 /* 
  * Copyright (C) 2013 David Vossel <davidvossel@gmail.com>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 
 #include <crmd.h>
 #include <crmd_fsa.h>
 #include <crmd_messages.h>
 #include <crmd_callbacks.h>
 #include <crmd_lrm.h>
 #include <crm/lrmd.h>
 #include <crm/services.h>
 
 #define REMOTE_LRMD_RA "remote"
 
 /* The max start timeout before cmd retry */
 #define MAX_START_TIMEOUT_MS 10000
 
 typedef struct remote_ra_cmd_s {
     /*! the local node the cmd is issued from */
     char *owner;
     /*! the remote node the cmd is executed on */
     char *rsc_id;
     /*! the action to execute */
     char *action;
     /*! some string the client wants us to give it back */
     char *userdata;
     char *exit_reason;          // descriptive text on error
     /*! start delay in ms */
     int start_delay;
     /*! timer id used for start delay. */
     int delay_id;
     /*! timeout in ms for cmd */
     int timeout;
     int remaining_timeout;
     /*! recurring interval in ms */
     guint interval_ms;
     /*! interval timer id */
     int interval_id;
     int reported_success;
     int monitor_timeout_id;
     int takeover_timeout_id;
     /*! action parameters */
     lrmd_key_value_t *params;
     /*! executed rc */
     int rc;
     int op_status;
     int call_id;
     time_t start_time;
     gboolean cancel;
 } remote_ra_cmd_t;
 
 enum remote_migration_status {
     expect_takeover = 1,
     takeover_complete,
 };
 
 typedef struct remote_ra_data_s {
     crm_trigger_t *work;
     remote_ra_cmd_t *cur_cmd;
     GList *cmds;
     GList *recurring_cmds;
 
     enum remote_migration_status migrate_status;
 
     gboolean active;
     gboolean is_maintenance; /* kind of complex to determine from crmd-context
                               * so we have it signalled back with the
                               * transition from pengine
                               */
 } remote_ra_data_t;
 
 static int handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeout_ms);
 static void handle_remote_ra_stop(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd);
 static GList *fail_all_monitor_cmds(GList * list);
 
 static void
 free_cmd(gpointer user_data)
 {
     remote_ra_cmd_t *cmd = user_data;
 
     if (!cmd) {
         return;
     }
     if (cmd->delay_id) {
         g_source_remove(cmd->delay_id);
     }
     if (cmd->interval_id) {
         g_source_remove(cmd->interval_id);
     }
     if (cmd->monitor_timeout_id) {
         g_source_remove(cmd->monitor_timeout_id);
     }
     if (cmd->takeover_timeout_id) {
         g_source_remove(cmd->takeover_timeout_id);
     }
     free(cmd->owner);
     free(cmd->rsc_id);
     free(cmd->action);
     free(cmd->userdata);
     free(cmd->exit_reason);
     lrmd_key_value_freeall(cmd->params);
     free(cmd);
 }
 
 static int
 generate_callid(void)
 {
     static int remote_ra_callid = 0;
 
     remote_ra_callid++;
     if (remote_ra_callid <= 0) {
         remote_ra_callid = 1;
     }
 
     return remote_ra_callid;
 }
 
 static gboolean
 recurring_helper(gpointer data)
 {
     remote_ra_cmd_t *cmd = data;
     lrm_state_t *connection_rsc = NULL;
 
     cmd->interval_id = 0;
     connection_rsc = lrm_state_find(cmd->rsc_id);
     if (connection_rsc && connection_rsc->remote_ra_data) {
         remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;
 
         ra_data->recurring_cmds = g_list_remove(ra_data->recurring_cmds, cmd);
 
         ra_data->cmds = g_list_append(ra_data->cmds, cmd);
         mainloop_set_trigger(ra_data->work);
     }
     return FALSE;
 }
 
 static gboolean
 start_delay_helper(gpointer data)
 {
     remote_ra_cmd_t *cmd = data;
     lrm_state_t *connection_rsc = NULL;
 
     cmd->delay_id = 0;
     connection_rsc = lrm_state_find(cmd->rsc_id);
     if (connection_rsc && connection_rsc->remote_ra_data) {
         remote_ra_data_t *ra_data = connection_rsc->remote_ra_data;
 
         mainloop_set_trigger(ra_data->work);
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Handle cluster communication related to pacemaker_remote node joining
  *
  * \param[in] node_name  Name of newly integrated pacemaker_remote node
  */
 static void
 remote_node_up(const char *node_name)
 {
     int call_opt, call_id = 0;
     xmlNode *update, *state;
     crm_node_t *node;
 
     CRM_CHECK(node_name != NULL, return);
     crm_info("Announcing pacemaker_remote node %s", node_name);
 
     /* Clear node's operation history. The node's transient attributes should
      * and normally will be cleared when the node leaves, but since remote node
      * state has a number of corner cases, clear them here as well, to be sure.
      */
     call_opt = crmd_cib_smart_opt();
     erase_status_tag(node_name, XML_CIB_TAG_LRM, call_opt);
     erase_status_tag(node_name, XML_TAG_TRANSIENT_NODEATTRS, call_opt);
 
     /* Clear node's probed attribute */
     update_attrd(node_name, CRM_OP_PROBED, NULL, NULL, TRUE);
 
     /* Ensure node is in the remote peer cache with member status */
     node = crm_remote_peer_get(node_name);
     CRM_CHECK(node != NULL, return);
     crm_update_peer_state(__FUNCTION__, node, CRM_NODE_MEMBER, 0);
 
     /* pacemaker_remote nodes don't participate in the membership layer,
      * so cluster nodes don't automatically get notified when they come and go.
      * We send a cluster message to the DC, and update the CIB node state entry,
      * so the DC will get it sooner (via message) or later (via CIB refresh),
      * and any other interested parties can query the CIB.
      */
     send_remote_state_message(node_name, TRUE);
 
     update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
     state = create_node_state_update(node, node_update_cluster, update,
                                      __FUNCTION__);
 
     /* Clear the XML_NODE_IS_FENCED flag in the node state. If the node ever
      * needs to be fenced, this flag will allow various actions to determine
      * whether the fencing has happened yet.
      */
     crm_xml_add(state, XML_NODE_IS_FENCED, "0");
 
     /* TODO: If the remote connection drops, and this (async) CIB update either
      * failed or has not yet completed, later actions could mistakenly think the
      * node has already been fenced (if the XML_NODE_IS_FENCED attribute was
      * previously set, because it won't have been cleared). This could prevent
      * actual fencing or allow recurring monitor failures to be cleared too
      * soon. Ideally, we wouldn't rely on the CIB for the fenced status.
      */
     fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
     if (call_id < 0) {
         crm_perror(LOG_WARNING, "%s CIB node state setup", node_name);
     }
     free_xml(update);
 }
 
 enum down_opts {
     DOWN_KEEP_LRM,
     DOWN_ERASE_LRM
 };
 
 /*!
  * \internal
  * \brief Handle cluster communication related to pacemaker_remote node leaving
  *
  * \param[in] node_name  Name of lost node
  * \param[in] opts       Whether to keep or erase LRM history
  */
 static void
 remote_node_down(const char *node_name, const enum down_opts opts)
 {
     xmlNode *update;
     int call_id = 0;
     int call_opt = crmd_cib_smart_opt();
     crm_node_t *node;
 
     /* Purge node from attrd's memory */
     update_attrd_remote_node_removed(node_name, NULL);
 
     /* Purge node's transient attributes */
     erase_status_tag(node_name, XML_TAG_TRANSIENT_NODEATTRS, call_opt);
 
     /* Normally, the LRM operation history should be kept until the node comes
      * back up. However, after a successful fence, we want to clear it, so we
      * don't think resources are still running on the node.
      */
     if (opts == DOWN_ERASE_LRM) {
         erase_status_tag(node_name, XML_CIB_TAG_LRM, call_opt);
     }
 
     /* Ensure node is in the remote peer cache with lost state */
     node = crm_remote_peer_get(node_name);
     CRM_CHECK(node != NULL, return);
     crm_update_peer_state(__FUNCTION__, node, CRM_NODE_LOST, 0);
 
     /* Notify DC */
     send_remote_state_message(node_name, FALSE);
 
     /* Update CIB node state */
     update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
     create_node_state_update(node, node_update_cluster, update, __FUNCTION__);
     fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
     if (call_id < 0) {
         crm_perror(LOG_ERR, "%s CIB node state update", node_name);
     }
     free_xml(update);
 }
 
 /*!
  * \internal
  * \brief Handle effects of a remote RA command on node state
  *
  * \param[in] cmd  Completed remote RA command
  */
 static void
 check_remote_node_state(remote_ra_cmd_t *cmd)
 {
     /* Only successful actions can change node state */
     if (cmd->rc != PCMK_OCF_OK) {
         return;
     }
 
     if (safe_str_eq(cmd->action, "start")) {
         remote_node_up(cmd->rsc_id);
 
     } else if (safe_str_eq(cmd->action, "migrate_from")) {
         /* After a successful migration, we don't need to do remote_node_up()
          * because the DC already knows the node is up, and we don't want to
          * clear LRM history etc. We do need to add the remote node to this
          * host's remote peer cache, because (unless it happens to be DC)
          * it hasn't been tracking the remote node, and other code relies on
          * the cache to distinguish remote nodes from unseen cluster nodes.
          */
         crm_node_t *node = crm_remote_peer_get(cmd->rsc_id);
 
         CRM_CHECK(node != NULL, return);
         crm_update_peer_state(__FUNCTION__, node, CRM_NODE_MEMBER, 0);
 
     } else if (safe_str_eq(cmd->action, "stop")) {
         lrm_state_t *lrm_state = lrm_state_find(cmd->rsc_id);
         remote_ra_data_t *ra_data = lrm_state? lrm_state->remote_ra_data : NULL;
 
         if (ra_data) {
             if (ra_data->migrate_status != takeover_complete) {
                 /* Stop means down if we didn't successfully migrate elsewhere */
                 remote_node_down(cmd->rsc_id, DOWN_KEEP_LRM);
             } else if (AM_I_DC == FALSE) {
                 /* Only the connection host and DC track node state,
                  * so if the connection migrated elsewhere and we aren't DC,
                  * un-cache the node, so we don't have stale info
                  */
                 crm_remote_peer_cache_remove(cmd->rsc_id);
             }
         }
     }
 
     /* We don't do anything for successful monitors, which is correct for
      * routine recurring monitors, and for monitors on nodes where the
      * connection isn't supposed to be (the cluster will stop the connection in
      * that case). However, if the initial probe finds the connection already
      * active on the node where we want it, we probably should do
      * remote_node_up(). Unfortunately, we can't distinguish that case here.
      * Given that connections have to be initiated by the cluster, the chance of
      * that should be close to zero.
      */
 }
 
 static void
 report_remote_ra_result(remote_ra_cmd_t * cmd)
 {
     lrmd_event_data_t op = { 0, };
 
     check_remote_node_state(cmd);
 
     op.type = lrmd_event_exec_complete;
     op.rsc_id = cmd->rsc_id;
     op.op_type = cmd->action;
     op.user_data = cmd->userdata;
     op.exit_reason = cmd->exit_reason;
     op.timeout = cmd->timeout;
-    op.interval = cmd->interval_ms;
+    op.interval_ms = cmd->interval_ms;
     op.rc = cmd->rc;
     op.op_status = cmd->op_status;
     op.t_run = cmd->start_time;
     op.t_rcchange = cmd->start_time;
     if (cmd->reported_success && cmd->rc != PCMK_OCF_OK) {
         op.t_rcchange = time(NULL);
         /* This edge case will likely never ever occur, but if it does the
          * result is that a failure will not be processed correctly. This is only
          * remotely possible because we are able to detect a connection resource's tcp
          * connection has failed at any moment after start has completed. The actual
          * recurring operation is just a connectivity ping.
          *
          * basically, we are not guaranteed that the first successful monitor op and
          * a subsequent failed monitor op will not occur in the same timestamp. We have to
          * make it look like the operations occurred at separate times though. */
         if (op.t_rcchange == op.t_run) {
             op.t_rcchange++;
         }
     }
 
     if (cmd->params) {
         lrmd_key_value_t *tmp;
 
         op.params = crm_str_table_new();
         for (tmp = cmd->params; tmp; tmp = tmp->next) {
             g_hash_table_insert(op.params, strdup(tmp->key), strdup(tmp->value));
         }
 
     }
     op.call_id = cmd->call_id;
     op.remote_nodename = cmd->owner;
 
     lrm_op_callback(&op);
 
     if (op.params) {
         g_hash_table_destroy(op.params);
     }
 }
 
 static void
 update_remaining_timeout(remote_ra_cmd_t * cmd)
 {
     cmd->remaining_timeout = ((cmd->timeout / 1000) - (time(NULL) - cmd->start_time)) * 1000;
 }
 
 static gboolean
 retry_start_cmd_cb(gpointer data)
 {
     lrm_state_t *lrm_state = data;
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
     remote_ra_cmd_t *cmd = NULL;
     int rc = -1;
 
     if (!ra_data || !ra_data->cur_cmd) {
         return FALSE;
     }
     cmd = ra_data->cur_cmd;
     if (safe_str_neq(cmd->action, "start") && safe_str_neq(cmd->action, "migrate_from")) {
         return FALSE;
     }
     update_remaining_timeout(cmd);
 
     if (cmd->remaining_timeout > 0) {
         rc = handle_remote_ra_start(lrm_state, cmd, cmd->remaining_timeout);
     }
 
     if (rc != 0) {
         cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
         cmd->op_status = PCMK_LRM_OP_ERROR;
         report_remote_ra_result(cmd);
 
         if (ra_data->cmds) {
             mainloop_set_trigger(ra_data->work);
         }
         ra_data->cur_cmd = NULL;
         free_cmd(cmd);
     } else {
         /* wait for connection event */
     }
 
     return FALSE;
 }
 
 
 static gboolean
 connection_takeover_timeout_cb(gpointer data)
 {
     lrm_state_t *lrm_state = NULL;
     remote_ra_cmd_t *cmd = data;
 
     crm_info("takeover event timed out for node %s", cmd->rsc_id);
     cmd->takeover_timeout_id = 0;
 
     lrm_state = lrm_state_find(cmd->rsc_id);
 
     handle_remote_ra_stop(lrm_state, cmd);
     free_cmd(cmd);
 
     return FALSE;
 }
 
 static gboolean
 monitor_timeout_cb(gpointer data)
 {
     lrm_state_t *lrm_state = NULL;
     remote_ra_cmd_t *cmd = data;
 
     lrm_state = lrm_state_find(cmd->rsc_id);
 
     crm_info("Timed out waiting for remote poke response from %s%s",
              cmd->rsc_id, (lrm_state? "" : " (no LRM state)"));
     cmd->monitor_timeout_id = 0;
     cmd->op_status = PCMK_LRM_OP_TIMEOUT;
     cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
 
     if (lrm_state && lrm_state->remote_ra_data) {
         remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
         if (ra_data->cur_cmd == cmd) {
             ra_data->cur_cmd = NULL;
         }
         if (ra_data->cmds) {
             mainloop_set_trigger(ra_data->work);
         }
     }
 
     report_remote_ra_result(cmd);
     free_cmd(cmd);
 
     if(lrm_state) {
         lrm_state_disconnect(lrm_state);
     }
     return FALSE;
 }
 
 static void
 synthesize_lrmd_success(lrm_state_t *lrm_state, const char *rsc_id, const char *op_type)
 {
     lrmd_event_data_t op = { 0, };
 
     if (lrm_state == NULL) {
         /* if lrm_state not given assume local */
         lrm_state = lrm_state_find(fsa_our_uname);
     }
     CRM_ASSERT(lrm_state != NULL);
 
     op.type = lrmd_event_exec_complete;
     op.rsc_id = rsc_id;
     op.op_type = op_type;
     op.rc = PCMK_OCF_OK;
     op.op_status = PCMK_LRM_OP_DONE;
     op.t_run = time(NULL);
     op.t_rcchange = op.t_run;
     op.call_id = generate_callid();
     process_lrm_event(lrm_state, &op, NULL);
 }
 
 void
 remote_lrm_op_callback(lrmd_event_data_t * op)
 {
     gboolean cmd_handled = FALSE;
     lrm_state_t *lrm_state = NULL;
     remote_ra_data_t *ra_data = NULL;
     remote_ra_cmd_t *cmd = NULL;
 
     crm_debug("remote connection event - event_type:%s node:%s action:%s rc:%s op_status:%s",
               lrmd_event_type2str(op->type),
               op->remote_nodename,
               op->op_type ? op->op_type : "none",
               services_ocf_exitcode_str(op->rc), services_lrm_status_str(op->op_status));
 
     lrm_state = lrm_state_find(op->remote_nodename);
     if (!lrm_state || !lrm_state->remote_ra_data) {
         crm_debug("lrm_state info not found for remote lrmd connection event");
         return;
     }
     ra_data = lrm_state->remote_ra_data;
 
     /* Another client has connected to the remote daemon,
      * determine if this is expected. */
     if (op->type == lrmd_event_new_client) {
         /* great, we new this was coming */
         if (ra_data->migrate_status == expect_takeover) {
             ra_data->migrate_status = takeover_complete;
         } else {
             crm_err("Unexpected pacemaker_remote client takeover for %s. Disconnecting", op->remote_nodename);
             /* In this case, lrmd_tls_connection_destroy() will be called under the control of mainloop. */
             /* Do not free lrm_state->conn yet. */
             /* It'll be freed in the following stop action. */
             lrm_state_disconnect_only(lrm_state);
         }
         return;
     }
 
     /* filter all EXEC events up */
     if (op->type == lrmd_event_exec_complete) {
         if (ra_data->migrate_status == takeover_complete) {
             crm_debug("ignoring event, this connection is taken over by another node");
         } else {
             lrm_op_callback(op);
         }
         return;
     }
 
     if ((op->type == lrmd_event_disconnect) &&
         (ra_data->cur_cmd == NULL) &&
         (ra_data->active == TRUE)) {
 
         if (!remote_ra_is_in_maintenance(lrm_state)) {
             crm_err("Lost connection to Pacemaker Remote node %s",
                     lrm_state->node_name);
             ra_data->recurring_cmds = fail_all_monitor_cmds(ra_data->recurring_cmds);
             ra_data->cmds = fail_all_monitor_cmds(ra_data->cmds);
         } else {
             crm_notice("Unmanaged Pacemaker Remote node %s disconnected",
                        lrm_state->node_name);
             /* Do roughly what a 'stop' on the remote-resource would do */
             handle_remote_ra_stop(lrm_state, NULL);
             remote_node_down(lrm_state->node_name, DOWN_KEEP_LRM);
             /* now fake the reply of a successful 'stop' */
             synthesize_lrmd_success(NULL, lrm_state->node_name, "stop");
         }
         return;
     }
 
     if (!ra_data->cur_cmd) {
         crm_debug("no event to match");
         return;
     }
 
     cmd = ra_data->cur_cmd;
 
     /* Start actions and migrate from actions complete after connection
      * comes back to us. */
     if (op->type == lrmd_event_connect && (safe_str_eq(cmd->action, "start") ||
                                            safe_str_eq(cmd->action, "migrate_from"))) {
 
         if (op->connection_rc < 0) {
             update_remaining_timeout(cmd);
 
             if (op->connection_rc == -ENOKEY) {
                 // Hard error, don't retry
                 cmd->op_status = PCMK_LRM_OP_ERROR;
                 cmd->rc = PCMK_OCF_INVALID_PARAM;
                 cmd->exit_reason = strdup("Authentication key not readable");
 
             } else if (cmd->remaining_timeout > 3000) {
                 crm_trace("rescheduling start, remaining timeout %d", cmd->remaining_timeout);
                 g_timeout_add(1000, retry_start_cmd_cb, lrm_state);
                 return;
 
             } else {
                 crm_trace("can't reschedule start, remaining timeout too small %d",
                           cmd->remaining_timeout);
                 cmd->op_status = PCMK_LRM_OP_TIMEOUT;
                 cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
             }
 
         } else {
             lrm_state_reset_tables(lrm_state, TRUE);
             cmd->rc = PCMK_OCF_OK;
             cmd->op_status = PCMK_LRM_OP_DONE;
             ra_data->active = TRUE;
         }
 
         crm_debug("remote lrmd connect event matched %s action. ", cmd->action);
         report_remote_ra_result(cmd);
         cmd_handled = TRUE;
 
     } else if (op->type == lrmd_event_poke && safe_str_eq(cmd->action, "monitor")) {
 
         if (cmd->monitor_timeout_id) {
             g_source_remove(cmd->monitor_timeout_id);
             cmd->monitor_timeout_id = 0;
         }
 
         /* Only report success the first time, after that only worry about failures.
          * For this function, if we get the poke pack, it is always a success. Pokes
          * only fail if the send fails, or the response times out. */
         if (!cmd->reported_success) {
             cmd->rc = PCMK_OCF_OK;
             cmd->op_status = PCMK_LRM_OP_DONE;
             report_remote_ra_result(cmd);
             cmd->reported_success = 1;
         }
 
         crm_debug("remote lrmd poke event matched %s action. ", cmd->action);
 
         /* success, keep rescheduling if interval is present. */
         if (cmd->interval_ms && (cmd->cancel == FALSE)) {
             ra_data->recurring_cmds = g_list_append(ra_data->recurring_cmds, cmd);
             cmd->interval_id = g_timeout_add(cmd->interval_ms,
                                              recurring_helper, cmd);
             cmd = NULL;         /* prevent free */
         }
         cmd_handled = TRUE;
 
     } else if (op->type == lrmd_event_disconnect && safe_str_eq(cmd->action, "monitor")) {
         if (ra_data->active == TRUE && (cmd->cancel == FALSE)) {
             cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
             cmd->op_status = PCMK_LRM_OP_ERROR;
             report_remote_ra_result(cmd);
             crm_err("remote-node %s unexpectedly disconneced during monitor operation", lrm_state->node_name);
         }
         cmd_handled = TRUE;
 
     } else if (op->type == lrmd_event_new_client && safe_str_eq(cmd->action, "stop")) {
 
         handle_remote_ra_stop(lrm_state, cmd);
         cmd_handled = TRUE;
 
     } else {
         crm_debug("Event did not match %s action", ra_data->cur_cmd->action);
     }
 
     if (cmd_handled) {
         ra_data->cur_cmd = NULL;
         if (ra_data->cmds) {
             mainloop_set_trigger(ra_data->work);
         }
         free_cmd(cmd);
     }
 }
 
 static void
 handle_remote_ra_stop(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd)
 {
     remote_ra_data_t *ra_data = NULL;
 
     CRM_ASSERT(lrm_state);
     ra_data = lrm_state->remote_ra_data;
 
     if (ra_data->migrate_status != takeover_complete) {
         /* delete pending ops when ever the remote connection is intentionally stopped */
         g_hash_table_remove_all(lrm_state->pending_ops);
     } else {
         /* we no longer hold the history if this connection has been migrated,
          * however, we keep metadata cache for future use */
         lrm_state_reset_tables(lrm_state, FALSE);
     }
 
     ra_data->active = FALSE;
     lrm_state_disconnect(lrm_state);
 
     if (ra_data->cmds) {
         g_list_free_full(ra_data->cmds, free_cmd);
     }
     if (ra_data->recurring_cmds) {
         g_list_free_full(ra_data->recurring_cmds, free_cmd);
     }
     ra_data->cmds = NULL;
     ra_data->recurring_cmds = NULL;
     ra_data->cur_cmd = NULL;
 
     if (cmd) {
         cmd->rc = PCMK_OCF_OK;
         cmd->op_status = PCMK_LRM_OP_DONE;
 
         report_remote_ra_result(cmd);
     }
 }
 
 static int
 handle_remote_ra_start(lrm_state_t * lrm_state, remote_ra_cmd_t * cmd, int timeout_ms)
 {
     const char *server = NULL;
     lrmd_key_value_t *tmp = NULL;
     int port = 0;
     int timeout_used = timeout_ms > MAX_START_TIMEOUT_MS ? MAX_START_TIMEOUT_MS : timeout_ms;
 
     for (tmp = cmd->params; tmp; tmp = tmp->next) {
         if (safe_str_eq(tmp->key, "addr") || safe_str_eq(tmp->key, "server")) {
             server = tmp->value;
         }
         if (safe_str_eq(tmp->key, "port")) {
             port = atoi(tmp->value);
         }
     }
 
     return lrm_state_remote_connect_async(lrm_state, server, port, timeout_used);
 }
 
 static gboolean
 handle_remote_ra_exec(gpointer user_data)
 {
     int rc = 0;
     lrm_state_t *lrm_state = user_data;
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
     remote_ra_cmd_t *cmd;
     GList *first = NULL;
 
     if (ra_data->cur_cmd) {
         /* still waiting on previous cmd */
         return TRUE;
     }
 
     while (ra_data->cmds) {
         first = ra_data->cmds;
         cmd = first->data;
         if (cmd->delay_id) {
             /* still waiting for start delay timer to trip */
             return TRUE;
         }
 
         ra_data->cmds = g_list_remove_link(ra_data->cmds, first);
         g_list_free_1(first);
 
         if (!strcmp(cmd->action, "start") || !strcmp(cmd->action, "migrate_from")) {
             ra_data->migrate_status = 0;
             rc = handle_remote_ra_start(lrm_state, cmd, cmd->timeout);
             if (rc == 0) {
                 /* take care of this later when we get async connection result */
                 crm_debug("began remote lrmd connect, waiting for connect event.");
                 ra_data->cur_cmd = cmd;
                 return TRUE;
             } else {
                 crm_debug("connect failed, not expecting to match any connection event later");
                 cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
                 cmd->op_status = PCMK_LRM_OP_ERROR;
             }
             report_remote_ra_result(cmd);
 
         } else if (!strcmp(cmd->action, "monitor")) {
 
             if (lrm_state_is_connected(lrm_state) == TRUE) {
                 rc = lrm_state_poke_connection(lrm_state);
                 if (rc < 0) {
                     cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
                     cmd->op_status = PCMK_LRM_OP_ERROR;
                 }
             } else {
                 rc = -1;
                 cmd->op_status = PCMK_LRM_OP_DONE;
                 cmd->rc = PCMK_OCF_NOT_RUNNING;
             }
 
             if (rc == 0) {
                 crm_debug("poked remote lrmd at node %s, waiting for async response.", cmd->rsc_id);
                 ra_data->cur_cmd = cmd;
                 cmd->monitor_timeout_id = g_timeout_add(cmd->timeout, monitor_timeout_cb, cmd);
                 return TRUE;
             }
             report_remote_ra_result(cmd);
 
         } else if (!strcmp(cmd->action, "stop")) {
 
             if (ra_data->migrate_status == expect_takeover) {
                 /* briefly wait on stop for the takeover event to occur. If the
                  * takeover event does not occur during the wait period, that's fine.
                  * It just means that the remote-node's lrm_status section is going to get
                  * cleared which will require all the resources running in the remote-node
                  * to be explicitly re-detected via probe actions.  If the takeover does occur
                  * successfully, then we can leave the status section intact. */
                 cmd->takeover_timeout_id = g_timeout_add((cmd->timeout/2), connection_takeover_timeout_cb, cmd);
                 ra_data->cur_cmd = cmd;
                 return TRUE;
             }
 
             handle_remote_ra_stop(lrm_state, cmd);
 
         } else if (!strcmp(cmd->action, "migrate_to")) {
             ra_data->migrate_status = expect_takeover;
             cmd->rc = PCMK_OCF_OK;
             cmd->op_status = PCMK_LRM_OP_DONE;
             report_remote_ra_result(cmd);
         } else if (!strcmp(cmd->action, "reload")) {
             /* reloads are a no-op right now, add logic here when they become important */
             cmd->rc = PCMK_OCF_OK;
             cmd->op_status = PCMK_LRM_OP_DONE;
             report_remote_ra_result(cmd);
         }
 
         free_cmd(cmd);
     }
 
     return TRUE;
 }
 
 static void
 remote_ra_data_init(lrm_state_t * lrm_state)
 {
     remote_ra_data_t *ra_data = NULL;
 
     if (lrm_state->remote_ra_data) {
         return;
     }
 
     ra_data = calloc(1, sizeof(remote_ra_data_t));
     ra_data->work = mainloop_add_trigger(G_PRIORITY_HIGH, handle_remote_ra_exec, lrm_state);
     lrm_state->remote_ra_data = ra_data;
 }
 
 void
 remote_ra_cleanup(lrm_state_t * lrm_state)
 {
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
     if (!ra_data) {
         return;
     }
 
     if (ra_data->cmds) {
         g_list_free_full(ra_data->cmds, free_cmd);
     }
 
     if (ra_data->recurring_cmds) {
         g_list_free_full(ra_data->recurring_cmds, free_cmd);
     }
     mainloop_destroy_trigger(ra_data->work);
     free(ra_data);
     lrm_state->remote_ra_data = NULL;
 }
 
 gboolean
 is_remote_lrmd_ra(const char *agent, const char *provider, const char *id)
 {
     if (agent && provider && !strcmp(agent, REMOTE_LRMD_RA) && !strcmp(provider, "pacemaker")) {
         return TRUE;
     }
     if (id && lrm_state_find(id) && safe_str_neq(id, fsa_our_uname)) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 lrmd_rsc_info_t *
 remote_ra_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id)
 {
     lrmd_rsc_info_t *info = NULL;
 
     if ((lrm_state_find(rsc_id))) {
         info = calloc(1, sizeof(lrmd_rsc_info_t));
 
         info->id = strdup(rsc_id);
         info->type = strdup(REMOTE_LRMD_RA);
         info->standard = strdup(PCMK_RESOURCE_CLASS_OCF);
         info->provider = strdup("pacemaker");
     }
 
     return info;
 }
 
 static gboolean
 is_remote_ra_supported_action(const char *action)
 {
     if (!action) {
         return FALSE;
     } else if (strcmp(action, "start") &&
                strcmp(action, "stop") &&
                strcmp(action, "reload") &&
                strcmp(action, "migrate_to") &&
                strcmp(action, "migrate_from") && strcmp(action, "monitor")) {
         return FALSE;
     }
 
     return TRUE;
 }
 
 static GList *
 fail_all_monitor_cmds(GList * list)
 {
     GList *rm_list = NULL;
     remote_ra_cmd_t *cmd = NULL;
     GListPtr gIter = NULL;
 
     for (gIter = list; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
         if ((cmd->interval_ms > 0) && safe_str_eq(cmd->action, "monitor")) {
             rm_list = g_list_append(rm_list, cmd);
         }
     }
 
     for (gIter = rm_list; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
 
         cmd->rc = PCMK_OCF_UNKNOWN_ERROR;
         cmd->op_status = PCMK_LRM_OP_ERROR;
         crm_trace("Pre-emptively failing %s %s (interval=%u, %s)",
                   cmd->action, cmd->rsc_id, cmd->interval_ms, cmd->userdata);
         report_remote_ra_result(cmd);
 
         list = g_list_remove(list, cmd);
         free_cmd(cmd);
     }
 
     /* frees only the list data, not the cmds */
     g_list_free(rm_list);
     return list;
 }
 
 static GList *
 remove_cmd(GList * list, const char *action, guint interval_ms)
 {
     remote_ra_cmd_t *cmd = NULL;
     GListPtr gIter = NULL;
 
     for (gIter = list; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
         if ((cmd->interval_ms == interval_ms)
             && safe_str_eq(cmd->action, action)) {
             break;
         }
         cmd = NULL;
     }
     if (cmd) {
         list = g_list_remove(list, cmd);
         free_cmd(cmd);
     }
     return list;
 }
 
 int
 remote_ra_cancel(lrm_state_t *lrm_state, const char *rsc_id,
                  const char *action, guint interval_ms)
 {
     lrm_state_t *connection_rsc = NULL;
     remote_ra_data_t *ra_data = NULL;
 
     connection_rsc = lrm_state_find(rsc_id);
     if (!connection_rsc || !connection_rsc->remote_ra_data) {
         return -EINVAL;
     }
 
     ra_data = connection_rsc->remote_ra_data;
     ra_data->cmds = remove_cmd(ra_data->cmds, action, interval_ms);
     ra_data->recurring_cmds = remove_cmd(ra_data->recurring_cmds, action,
                                          interval_ms);
     if (ra_data->cur_cmd &&
         (ra_data->cur_cmd->interval_ms == interval_ms) &&
         (safe_str_eq(ra_data->cur_cmd->action, action))) {
 
         ra_data->cur_cmd->cancel = TRUE;
     }
 
     return 0;
 }
 
 static remote_ra_cmd_t *
 handle_dup_monitor(remote_ra_data_t *ra_data, guint interval_ms,
                    const char *userdata)
 {
     GList *gIter = NULL;
     remote_ra_cmd_t *cmd = NULL;
 
     /* there are 3 places a potential duplicate monitor operation
      * could exist.
      * 1. recurring_cmds list. where the op is waiting for its next interval
      * 2. cmds list, where the op is queued to get executed immediately
      * 3. cur_cmd, which means the monitor op is in flight right now.
      */
     if (interval_ms == 0) {
         return NULL;
     }
 
     if (ra_data->cur_cmd &&
         ra_data->cur_cmd->cancel == FALSE &&
         (ra_data->cur_cmd->interval_ms == interval_ms) &&
         safe_str_eq(ra_data->cur_cmd->action, "monitor")) {
 
         cmd = ra_data->cur_cmd;
         goto handle_dup;
     }
 
     for (gIter = ra_data->recurring_cmds; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
         if ((cmd->interval_ms == interval_ms)
             && safe_str_eq(cmd->action, "monitor")) {
             goto handle_dup;
         }
     }
 
     for (gIter = ra_data->cmds; gIter != NULL; gIter = gIter->next) {
         cmd = gIter->data;
         if ((cmd->interval_ms == interval_ms)
             && safe_str_eq(cmd->action, "monitor")) {
             goto handle_dup;
         }
     }
 
     return NULL;
 
 handle_dup:
 
     crm_trace("merging duplicate monitor cmd " CRM_OP_FMT,
               cmd->rsc_id, "monitor", interval_ms);
 
     /* update the userdata */
     if (userdata) {
        free(cmd->userdata);
        cmd->userdata = strdup(userdata);
     }
 
     /* if we've already reported success, generate a new call id */
     if (cmd->reported_success) {
         cmd->start_time = time(NULL);
         cmd->call_id = generate_callid();
         cmd->reported_success = 0;
     }
 
     /* if we have an interval_id set, that means we are in the process of
      * waiting for this cmd's next interval. instead of waiting, cancel
      * the timer and execute the action immediately */
     if (cmd->interval_id) {
         g_source_remove(cmd->interval_id);
         cmd->interval_id = 0;
         recurring_helper(cmd);
     }
 
     return cmd;  
 }
 
 int
 remote_ra_exec(lrm_state_t *lrm_state, const char *rsc_id, const char *action,
                const char *userdata, guint interval_ms,
                int timeout,     /* ms */
                int start_delay, /* ms */
                lrmd_key_value_t * params)
 {
     int rc = 0;
     lrm_state_t *connection_rsc = NULL;
     remote_ra_cmd_t *cmd = NULL;
     remote_ra_data_t *ra_data = NULL;
 
     if (is_remote_ra_supported_action(action) == FALSE) {
         rc = -EINVAL;
         goto exec_done;
     }
 
     connection_rsc = lrm_state_find(rsc_id);
     if (!connection_rsc) {
         rc = -EINVAL;
         goto exec_done;
     }
 
     remote_ra_data_init(connection_rsc);
     ra_data = connection_rsc->remote_ra_data;
 
     cmd = handle_dup_monitor(ra_data, interval_ms, userdata);
     if (cmd) {
        return cmd->call_id;
     }
 
     cmd = calloc(1, sizeof(remote_ra_cmd_t));
     cmd->owner = strdup(lrm_state->node_name);
     cmd->rsc_id = strdup(rsc_id);
     cmd->action = strdup(action);
     cmd->userdata = strdup(userdata);
     cmd->interval_ms = interval_ms;
     cmd->timeout = timeout;
     cmd->start_delay = start_delay;
     cmd->params = params;
     cmd->start_time = time(NULL);
 
     cmd->call_id = generate_callid();
 
     if (cmd->start_delay) {
         cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
     }
 
     ra_data->cmds = g_list_append(ra_data->cmds, cmd);
     mainloop_set_trigger(ra_data->work);
 
     return cmd->call_id;
   exec_done:
 
     lrmd_key_value_freeall(params);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Immediately fail all monitors of a remote node, if proxied here
  *
  * \param[in] node_name  Name of pacemaker_remote node
  */
 void
 remote_ra_fail(const char *node_name)
 {
     lrm_state_t *lrm_state = lrm_state_find(node_name);
 
     if (lrm_state && lrm_state_is_connected(lrm_state)) {
         remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
         crm_info("Failing monitors on pacemaker_remote node %s", node_name);
         ra_data->recurring_cmds = fail_all_monitor_cmds(ra_data->recurring_cmds);
         ra_data->cmds = fail_all_monitor_cmds(ra_data->cmds);
     }
 }
 
 /* A guest node fencing implied by host fencing looks like:
  *
  *  <pseudo_event id="103" operation="stonith" operation_key="stonith-lxc1-off"
  *                on_node="lxc1" on_node_uuid="lxc1">
  *     <attributes CRM_meta_master_lxc_ms="10" CRM_meta_on_node="lxc1"
  *                 CRM_meta_on_node_uuid="lxc1" CRM_meta_stonith_action="off"
  *                 crm_feature_set="3.0.12"/>
  *     <downed>
  *       <node id="lxc1"/>
  *     </downed>
  *  </pseudo_event>
  */
 #define XPATH_PSEUDO_FENCE "//" XML_GRAPH_TAG_PSEUDO_EVENT \
     "[@" XML_LRM_ATTR_TASK "='stonith']/" XML_GRAPH_TAG_DOWNED \
     "/" XML_CIB_TAG_NODE
 
 /*!
  * \internal
  * \brief Check a pseudo-action for Pacemaker Remote node side effects
  *
  * \param[in] xml  XML of pseudo-action to check
  */
 void
 remote_ra_process_pseudo(xmlNode *xml)
 {
     xmlXPathObjectPtr search = xpath_search(xml, XPATH_PSEUDO_FENCE);
 
     if (numXpathResults(search) == 1) {
         xmlNode *result = getXpathResult(search, 0);
 
         /* Normally, we handle the necessary side effects of a guest node stop
          * action when reporting the remote agent's result. However, if the stop
          * is implied due to fencing, it will be a fencing pseudo-event, and
          * there won't be a result to report. Handle that case here.
          *
          * This will result in a duplicate call to remote_node_down() if the
          * guest stop was real instead of implied, but that shouldn't hurt.
          *
          * There is still one corner case that isn't handled: if a guest node
          * isn't running any resources when its host is fenced, it will appear
          * to be cleanly stopped, so there will be no pseudo-fence, and our
          * peer cache state will be incorrect unless and until the guest is
          * recovered.
          */
         if (result) {
             const char *remote = ID(result);
 
             if (remote) {
                 remote_node_down(remote, DOWN_ERASE_LRM);
             }
         }
     }
     freeXpathObject(search);
 }
 
 static void
 remote_ra_maintenance(lrm_state_t * lrm_state, gboolean maintenance)
 {
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
     xmlNode *update, *state;
     int call_opt, call_id = 0;
     crm_node_t *node;
 
     call_opt = crmd_cib_smart_opt();
     node = crm_remote_peer_get(lrm_state->node_name);
     CRM_CHECK(node != NULL, return);
     update = create_xml_node(NULL, XML_CIB_TAG_STATUS);
     state = create_node_state_update(node, node_update_none, update,
                                      __FUNCTION__);
     crm_xml_add(state, XML_NODE_IS_MAINTENANCE, maintenance?"1":"0");
     fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, call_id, NULL);
     if (call_id < 0) {
         crm_perror(LOG_WARNING, "%s CIB node state update failed", lrm_state->node_name);
     } else {
         /* TODO: still not 100% sure that async update will succeed ... */
         ra_data->is_maintenance = maintenance;
     }
     free_xml(update);
 }
 
 #define XPATH_PSEUDO_MAINTENANCE "//" XML_GRAPH_TAG_PSEUDO_EVENT \
     "[@" XML_LRM_ATTR_TASK "='" CRM_OP_MAINTENANCE_NODES "']/" \
     XML_GRAPH_TAG_MAINTENANCE
 
 /*!
  * \internal
  * \brief Check a pseudo-action holding updates for maintenance state
  *
  * \param[in] xml  XML of pseudo-action to check
  */
 
 void
 remote_ra_process_maintenance_nodes(xmlNode *xml)
 {
     xmlXPathObjectPtr search = xpath_search(xml, XPATH_PSEUDO_MAINTENANCE);
 
     if (numXpathResults(search) == 1) {
         xmlNode *node;
         int cnt = 0, cnt_remote = 0;
 
         for (node =
                 first_named_child(getXpathResult(search, 0), XML_CIB_TAG_NODE);
             node; node = __xml_next(node)) {
             lrm_state_t *lrm_state = lrm_state_find(ID(node));
 
             cnt++;
             if (lrm_state && lrm_state->remote_ra_data &&
                 ((remote_ra_data_t *) lrm_state->remote_ra_data)->active) {
                 cnt_remote++;
                 remote_ra_maintenance(lrm_state,
                                         crm_atoi(crm_element_value(node,
                                             XML_NODE_IS_MAINTENANCE), "0"));
 
             }
         }
         crm_trace("Action holds %d nodes (%d remotes found) "
                     "adjusting maintenance-mode", cnt, cnt_remote);
     }
     freeXpathObject(search);
 }
 
 gboolean
 remote_ra_is_in_maintenance(lrm_state_t * lrm_state)
 {
     remote_ra_data_t *ra_data = lrm_state->remote_ra_data;
 
     return ra_data->is_maintenance;
 }
diff --git a/include/crm/lrmd.h b/include/crm/lrmd.h
index cf85afb2a3..f6c534c6a6 100644
--- a/include/crm/lrmd.h
+++ b/include/crm/lrmd.h
@@ -1,533 +1,533 @@
 /*
  * Copyright (c) 2012 David Vossel <davidvossel@gmail.com>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  * 
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  *
  */
 
 #ifndef LRMD__H
 #  define LRMD__H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /**
  * \file
  * \brief Local Resource Manager
  * \ingroup lrmd
  */
 #include <stdbool.h>
 #include <crm_config.h>
 #include <crm/services.h>
 
 typedef struct lrmd_s lrmd_t;
 typedef struct lrmd_key_value_s {
     char *key;
     char *value;
     struct lrmd_key_value_s *next;
 } lrmd_key_value_t;
 
 /* This should be bumped every time there is an incompatible change that
  * prevents older clients from connecting to this version of the server.
  */
 #define LRMD_PROTOCOL_VERSION "1.1"
 
 /* This is the version that the client version will actually be compared
  * against. This should be identical to LRMD_PROTOCOL_VERSION. However, we
  * accidentally bumped LRMD_PROTOCOL_VERSION in 6424a647 (1.1.15) when we didn't
  * need to, so for now it's different. If we ever have a truly incompatible
  * bump, we can drop this and compare against LRMD_PROTOCOL_VERSION.
  */
 #define LRMD_MIN_PROTOCOL_VERSION "1.0"
 
 /* *INDENT-OFF* */
 #define DEFAULT_REMOTE_KEY_LOCATION PACEMAKER_CONFIG_DIR "/authkey"
 #define ALT_REMOTE_KEY_LOCATION "/etc/corosync/authkey"
 #define DEFAULT_REMOTE_PORT 3121
 #define DEFAULT_REMOTE_USERNAME "lrmd"
 
 #define F_LRMD_OPERATION        "lrmd_op"
 #define F_LRMD_CLIENTNAME       "lrmd_clientname"
 #define F_LRMD_IS_IPC_PROVIDER  "lrmd_is_ipc_provider"
 #define F_LRMD_CLIENTID         "lrmd_clientid"
 #define F_LRMD_PROTOCOL_VERSION "lrmd_protocol_version"
 #define F_LRMD_REMOTE_MSG_TYPE  "lrmd_remote_msg_type"
 #define F_LRMD_REMOTE_MSG_ID    "lrmd_remote_msg_id"
 #define F_LRMD_CALLBACK_TOKEN   "lrmd_async_id"
 #define F_LRMD_CALLID           "lrmd_callid"
 #define F_LRMD_CALLOPTS         "lrmd_callopt"
 #define F_LRMD_CALLDATA         "lrmd_calldata"
 #define F_LRMD_RC               "lrmd_rc"
 #define F_LRMD_EXEC_RC          "lrmd_exec_rc"
 #define F_LRMD_OP_STATUS        "lrmd_exec_op_status"
 #define F_LRMD_TIMEOUT          "lrmd_timeout"
 #define F_LRMD_WATCHDOG         "lrmd_watchdog"
 #define F_LRMD_CLASS            "lrmd_class"
 #define F_LRMD_PROVIDER         "lrmd_provider"
 #define F_LRMD_TYPE             "lrmd_type"
 #define F_LRMD_ORIGIN           "lrmd_origin"
 
 #define F_LRMD_RSC_RUN_TIME      "lrmd_run_time"
 #define F_LRMD_RSC_RCCHANGE_TIME "lrmd_rcchange_time"
 #define F_LRMD_RSC_EXEC_TIME     "lrmd_exec_time"
 #define F_LRMD_RSC_QUEUE_TIME    "lrmd_queue_time"
 
 #define F_LRMD_RSC_ID           "lrmd_rsc_id"
 #define F_LRMD_RSC_ACTION       "lrmd_rsc_action"
 #define F_LRMD_RSC_USERDATA_STR "lrmd_rsc_userdata_str"
 #define F_LRMD_RSC_OUTPUT       "lrmd_rsc_output"
 #define F_LRMD_RSC_EXIT_REASON  "lrmd_rsc_exit_reason"
 #define F_LRMD_RSC_START_DELAY  "lrmd_rsc_start_delay"
 #define F_LRMD_RSC_INTERVAL     "lrmd_rsc_interval"
 #define F_LRMD_RSC_DELETED      "lrmd_rsc_deleted"
 #define F_LRMD_RSC              "lrmd_rsc"
 
 #define F_LRMD_ALERT_ID           "lrmd_alert_id"
 #define F_LRMD_ALERT_PATH         "lrmd_alert_path"
 #define F_LRMD_ALERT              "lrmd_alert"
 
 #define LRMD_OP_RSC_REG           "lrmd_rsc_register"
 #define LRMD_OP_RSC_EXEC          "lrmd_rsc_exec"
 #define LRMD_OP_RSC_CANCEL        "lrmd_rsc_cancel"
 #define LRMD_OP_RSC_UNREG         "lrmd_rsc_unregister"
 #define LRMD_OP_RSC_INFO          "lrmd_rsc_info"
 #define LRMD_OP_RSC_METADATA      "lrmd_rsc_metadata"
 #define LRMD_OP_POKE              "lrmd_rsc_poke"
 #define LRMD_OP_NEW_CLIENT        "lrmd_rsc_new_client"
 #define LRMD_OP_CHECK             "lrmd_check"
 #define LRMD_OP_ALERT_EXEC        "lrmd_alert_exec"
 
 #define LRMD_IPC_OP_NEW           "new"
 #define LRMD_IPC_OP_DESTROY       "destroy"
 #define LRMD_IPC_OP_EVENT         "event"
 #define LRMD_IPC_OP_REQUEST       "request"
 #define LRMD_IPC_OP_RESPONSE      "response"
 #define LRMD_IPC_OP_SHUTDOWN_REQ  "shutdown_req"
 #define LRMD_IPC_OP_SHUTDOWN_ACK  "shutdown_ack"
 #define LRMD_IPC_OP_SHUTDOWN_NACK "shutdown_nack"
 
 #define F_LRMD_IPC_OP           "lrmd_ipc_op"
 #define F_LRMD_IPC_IPC_SERVER   "lrmd_ipc_server"
 #define F_LRMD_IPC_SESSION      "lrmd_ipc_session"
 #define F_LRMD_IPC_CLIENT       "lrmd_ipc_client"
 #define F_LRMD_IPC_USER         "lrmd_ipc_user"
 #define F_LRMD_IPC_MSG          "lrmd_ipc_msg"
 #define F_LRMD_IPC_MSG_ID       "lrmd_ipc_msg_id"
 #define F_LRMD_IPC_MSG_FLAGS    "lrmd_ipc_msg_flags"
 
 #define T_LRMD           "lrmd"
 #define T_LRMD_REPLY     "lrmd_reply"
 #define T_LRMD_NOTIFY    "lrmd_notify"
 #define T_LRMD_IPC_PROXY "lrmd_ipc_proxy"
 /* *INDENT-ON* */
 
 /*!
  * \brief Create a new local lrmd connection
  */
 lrmd_t *lrmd_api_new(void);
 
 /*!
  * \brief Create a new remote lrmd connection using tls backend
  *
  * \param nodename  name of remote node identified with this connection
  * \param server    name of server to connect to
  * \param port      port number to connect to
  *
  * \note nodename and server may be the same value.
  */
 lrmd_t *lrmd_remote_api_new(const char *nodename, const char *server, int port);
 
 /*!
  * \brief Use after lrmd_poll returns 1 to read and dispatch a message
  *
  * \param[in,out] lrmd  lrmd connection object
  *
  * \return TRUE if connection is still up, FALSE if disconnected
  */
 bool lrmd_dispatch(lrmd_t * lrmd);
 
 /*!
  * \brief Poll for a specified timeout period to determine if a message
  *        is ready for dispatch.
  * \retval 1 msg is ready
  * \retval 0 timeout occurred
  * \retval negative error code
  */
 int lrmd_poll(lrmd_t * lrmd, int timeout);
 
 /*!
  * \brief Destroy lrmd object
  */
 void lrmd_api_delete(lrmd_t * lrmd);
 lrmd_key_value_t *lrmd_key_value_add(lrmd_key_value_t * kvp, const char *key, const char *value);
 
 /* *INDENT-OFF* */
 /* Reserved for future use */
 enum lrmd_call_options {
     lrmd_opt_none = 0x00000000,
     /* lrmd_opt_sync_call = 0x00000001, //Not implemented, patches welcome. */
     /*! Only notify the client originating a exec() the results */
     lrmd_opt_notify_orig_only = 0x00000002,
     /*! Drop recurring operations initiated by a client when client disconnects.
      * This call_option is only valid when registering a resource. When used
      * remotely with the pacemaker_remote daemon, this option means that recurring
      * operations will be dropped once all the remote connections disconnect. */
     lrmd_opt_drop_recurring = 0x00000003,
     /*! Send notifications for recurring operations only when the result changes */
     lrmd_opt_notify_changes_only = 0x00000004,
 };
 
 enum lrmd_callback_event {
     lrmd_event_register,
     lrmd_event_unregister,
     lrmd_event_exec_complete,
     lrmd_event_disconnect,
     lrmd_event_connect,
     lrmd_event_poke,
     lrmd_event_new_client,
 };
 
 /* *INDENT-ON* */
 
 typedef struct lrmd_event_data_s {
     /*! Type of event, register, unregister, call_completed... */
     enum lrmd_callback_event type;
 
     /*! The resource this event occurred on. */
     const char *rsc_id;
     /*! The action performed, start, stop, monitor... */
     const char *op_type;
     /*! The userdata string given do exec() api function */
     const char *user_data;
 
     /*! The client api call id associated with this event */
     int call_id;
     /*! The operation's timeout period in ms. */
     int timeout;
     /*! The operation's recurring interval in ms. */
-    int interval;
+    guint interval_ms;
     /*! The operation's start delay value in ms. */
     int start_delay;
     /*! This operation that just completed is on a deleted rsc. */
     int rsc_deleted;
 
     /*! The executed ra return code mapped to OCF */
     enum ocf_exitcode rc;
     /*! The lrmd status returned for exec_complete events */
     int op_status;
     /*! stdout from resource agent operation */
     const char *output;
     /*! Timestamp of when op ran */
     unsigned int t_run;
     /*! Timestamp of last rc change */
     unsigned int t_rcchange;
     /*! Time in length op took to execute */
     unsigned int exec_time;
     /*! Time in length spent in queue */
     unsigned int queue_time;
 
     /*! int connection result. Used for connection and poke events */
     int connection_rc;
 
     /* This is a GHashTable containing the
      * parameters given to the operation */
     void *params;
 
     /*! client node name associated with this connection
      * (used to match actions to the proper client when there are multiple)
      */
     const char *remote_nodename;
 
     /*! exit failure reason string from resource agent operation */
     const char *exit_reason;
 } lrmd_event_data_t;
 
 lrmd_event_data_t *lrmd_copy_event(lrmd_event_data_t * event);
 void lrmd_free_event(lrmd_event_data_t * event);
 
 typedef struct lrmd_rsc_info_s {
     char *id;
     char *type;
     char *standard;
     char *provider;
 } lrmd_rsc_info_t;
 
 lrmd_rsc_info_t *lrmd_new_rsc_info(const char *rsc_id, const char *standard,
                                    const char *provider, const char *type);
 lrmd_rsc_info_t *lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info);
 void lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info);
 
 typedef void (*lrmd_event_callback) (lrmd_event_data_t * event);
 
 typedef struct lrmd_list_s {
     const char *val;
     struct lrmd_list_s *next;
 } lrmd_list_t;
 
 void lrmd_list_freeall(lrmd_list_t * head);
 void lrmd_key_value_freeall(lrmd_key_value_t * head);
 
 typedef struct lrmd_api_operations_s {
     /*!
      * \brief Connect from the lrmd.
      *
      * \retval 0, success
      * \retval negative error code on failure
      */
     int (*connect) (lrmd_t * lrmd, const char *client_name, int *fd);
 
     /*!
      * \brief Establish an connection to lrmd, don't block while connecting.
      * \note this function requires the use of mainloop.
      *
      * \note The is returned using the event callback.
      * \note When this function returns 0, the callback will be invoked
      *       to report the final result of the connect.
      * \retval 0, connect in progress, wait for event callback
      * \retval -1, failure.
      */
     int (*connect_async) (lrmd_t * lrmd, const char *client_name, int timeout /*ms */ );
 
     /*!
      * \brief Is connected to lrmd daemon?
      *
      * \retval 0, false
      * \retval 1, true
      */
     int (*is_connected) (lrmd_t * lrmd);
 
     /*!
      * \brief Poke lrmd connection to verify it is still capable of serving requests
      * \note The response comes in the form of a poke event to the callback. 
      *
      * \retval 0, wait for response in callback
      * \retval -1, connection failure, callback may not be invoked
      */
     int (*poke_connection) (lrmd_t * lrmd);
 
     /*!
      * \brief Disconnect from the lrmd.
      *
      * \retval 0, success
      * \retval negative error code on failure
      */
     int (*disconnect) (lrmd_t * lrmd);
 
     /*!
      * \brief Register a resource with the lrmd.
      *
      * \note Synchronous, guaranteed to occur in daemon before function returns.
      *
      * \retval 0, success
      * \retval negative error code on failure
      */
     int (*register_rsc) (lrmd_t * lrmd,
                          const char *rsc_id,
                          const char *standard,
                          const char *provider, const char *agent, enum lrmd_call_options options);
 
     /*!
      * \brief Retrieve registration info for a rsc
      *
      * \retval info on success
      * \retval NULL on failure
      */
     lrmd_rsc_info_t *(*get_rsc_info) (lrmd_t * lrmd,
                                       const char *rsc_id, enum lrmd_call_options options);
 
     /*!
      * \brief Unregister a resource from the lrmd.
      *
      * \note All pending and recurring operations will be cancelled
      *       automatically.
      *
      * \note Synchronous, guaranteed to occur in daemon before function returns.
      *
      * \retval 0, success
      * \retval -1, success, but operations are currently executing on the rsc which will
      *         return once they are completed.
      * \retval negative error code on failure
      *
      */
     int (*unregister_rsc) (lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options);
 
     /*!
      * \brief Sets the callback to receive lrmd events on.
      */
     void (*set_callback) (lrmd_t * lrmd, lrmd_event_callback callback);
 
     /*!
      * \brief Issue a command on a resource
      *
      * \note Asynchronous, command is queued in daemon on function return, but
      *       execution of command is not synced.
      *
      * \note Operations on individual resources are guaranteed to occur
      *       in the order the client api calls them in.
      *
      * \note Operations between different resources are not guaranteed
      *       to occur in any specific order in relation to one another
      *       regardless of what order the client api is called in.
      * \retval call_id to track async event result on success
      * \retval negative error code on failure
      */
     int (*exec) (lrmd_t * lrmd, const char *rsc_id, const char *action, const char *userdata,   /* userdata string given back in event notification */
                  guint interval_ms,
                  int timeout,   /* ms */
                  int start_delay,       /* ms */
                  enum lrmd_call_options options, lrmd_key_value_t * params);    /* ownership of params is given up to api here */
 
     /*!
      * \brief Cancel a recurring command.
      *
      * \note Synchronous, guaranteed to occur in daemon before function returns.
      *
      * \note The cancel is completed async from this call.
      *       We can be guaranteed the cancel has completed once
      *       the callback receives an exec_complete event with
      *       the lrmd_op_status signifying that the operation is
      *       cancelled.
      * \note For each resource, cancel operations and exec operations
      *       are processed in the order they are received.
      *       It is safe to assume that for a single resource, a cancel
      *       will occur in the lrmd before an exec if the client's cancel
      *       api call occurs before the exec api call.
      *
      *       It is not however safe to assume any operation on one resource will
      *       occur before an operation on another resource regardless of
      *       the order the client api is called in.
      *
      * \retval 0, cancel command sent.
      * \retval negative error code on failure
      */
     int (*cancel) (lrmd_t *lrmd, const char *rsc_id, const char *action,
                    guint interval_ms);
 
     /*!
      * \brief Get resource metadata for a specified resource agent
      *
      * \param[in]  lrmd      LRMD connection (unused)
      * \param[in]  class     Resource agent class
      * \param[in]  provider  Resource agent provider
      * \param[in]  agent     Resource agent type
      * \param[out] output    Metadata will be stored here (must not be NULL)
      * \param[in]  options   Options to use with any LRMD API calls (unused)
      *
      * \note Caller is responsible for freeing output. This call is currently
      *       always synchronous (blocking), and always done directly by the
      *       library (not via the LRMD connection). This means that it is based
      *       on the local host environment, even if the lrmd connection is to a
      *       remote node, so (for most resource agent classes) this will fail if
      *       the agent is not installed locally. This also means that, if an
      *       external agent must be executed, it will be executed by the
      *       caller's user, not the lrmd's.
      * \todo Add a metadata call to the LRMD API and let the server handle this.
      *
      * \retval lrmd_ok success
      * \retval negative error code on failure
      */
     int (*get_metadata) (lrmd_t * lrmd,
                          const char *standard,
                          const char *provider,
                          const char *agent, char **output, enum lrmd_call_options options);
 
     /*!
      * \brief Retrieve a list of installed resource agents.
      *
      * \note if class is not provided, all known agents will be returned
      * \note list must be freed using lrmd_list_freeall()
      *
      * \retval num items in list on success
      * \retval negative error code on failure
      */
     int (*list_agents) (lrmd_t * lrmd, lrmd_list_t ** agents,
                         const char *standard, const char *provider);
 
     /*!
      * \brief Retrieve a list of resource agent providers
      *
      * \note When the agent is provided, only the agent's provider will be returned
      * \note When no agent is supplied, all providers will be returned.
      * \note List must be freed using lrmd_list_freeall()
      *
      * \retval num items in list on success
      * \retval negative error code on failure
      */
     int (*list_ocf_providers) (lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers);
 
     /*!
      * \brief Retrieve a list of standards supported by this machine/installation
      *
      * \note List must be freed using lrmd_list_freeall()
      *
      * \retval num items in list on success
      * \retval negative error code on failure
      */
     int (*list_standards) (lrmd_t * lrmd, lrmd_list_t ** standards);
 
     /*!
      * \brief Execute an alert agent
      *
      * \note Asynchronous, command is queued in daemon on function return, but
      *       execution of command is not synced.
      *
      * \note Operations on individual alerts are guaranteed to occur
      *       in the order the client api calls them in.
      *
      * \note Operations between different alerts are not guaranteed
      *       to occur in any specific order in relation to one another
      *       regardless of what order the client api is called in.
      * \retval call_id to track async event result on success
      * \retval negative error code on failure
      */
     int (*exec_alert) (lrmd_t *lrmd, const char *alert_id,
                        const char *alert_path, int timeout, /* ms */
                        lrmd_key_value_t *params); /* ownership of params is given up to api here */
 
 } lrmd_api_operations_t;
 
 struct lrmd_s {
     lrmd_api_operations_t *cmds;
     void *lrmd_private;
 };
 
 static inline const char *
 lrmd_event_type2str(enum lrmd_callback_event type)
 {
     switch (type) {
         case lrmd_event_register:
             return "register";
         case lrmd_event_unregister:
             return "unregister";
         case lrmd_event_exec_complete:
             return "exec_complete";
         case lrmd_event_disconnect:
             return "disconnect";
         case lrmd_event_connect:
             return "connect";
         case lrmd_event_poke:
             return "poke";
         case lrmd_event_new_client:
             return "new_client";
     }
     return "unknown";
 }
 
 #ifdef __cplusplus
 }
 #endif
 
 #endif
diff --git a/include/crm/services.h b/include/crm/services.h
index 204c4cfb0c..d2cda8bd1d 100644
--- a/include/crm/services.h
+++ b/include/crm/services.h
@@ -1,415 +1,415 @@
 /*
  * Copyright (C) 2010 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #ifndef __PCMK_SERVICES__
 #  define __PCMK_SERVICES__
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /**
  * \file
  * \brief Services API
  * \ingroup core
  */
 
 #  include <glib.h>
 #  include <stdio.h>
 #  include <string.h>
 #  include <stdbool.h>
 #  include <sys/types.h>
 
 #  ifndef OCF_ROOT_DIR
 #    define OCF_ROOT_DIR "/usr/lib/ocf"
 #  endif
 
 #  ifndef LSB_ROOT_DIR
 #    define LSB_ROOT_DIR "/etc/init.d"
 #  endif
 
 /* TODO: Autodetect these two ?*/
 #  ifndef SYSTEMCTL
 #    define SYSTEMCTL "/bin/systemctl"
 #  endif
 
 /* Known resource classes */
 #define PCMK_RESOURCE_CLASS_OCF     "ocf"
 #define PCMK_RESOURCE_CLASS_SERVICE "service"
 #define PCMK_RESOURCE_CLASS_LSB     "lsb"
 #define PCMK_RESOURCE_CLASS_SYSTEMD "systemd"
 #define PCMK_RESOURCE_CLASS_UPSTART "upstart"
 #define PCMK_RESOURCE_CLASS_NAGIOS  "nagios"
 #define PCMK_RESOURCE_CLASS_STONITH "stonith"
 
 /* This is the string passed in the OCF_EXIT_REASON_PREFIX
  * environment variable. The stderr output that occurs
  * after this prefix is encountered is considered the exit
  * reason for a completed operationt */
 #define PCMK_OCF_REASON_PREFIX "ocf-exit-reason:"
 
 // Agent version to use if agent doesn't specify one
 #define PCMK_DEFAULT_AGENT_VERSION "0.1"
 
 enum lsb_exitcode {
     PCMK_LSB_OK                  = 0,
     PCMK_LSB_UNKNOWN_ERROR       = 1,
     PCMK_LSB_INVALID_PARAM       = 2,
     PCMK_LSB_UNIMPLEMENT_FEATURE = 3,
     PCMK_LSB_INSUFFICIENT_PRIV   = 4,
     PCMK_LSB_NOT_INSTALLED       = 5,
     PCMK_LSB_NOT_CONFIGURED      = 6,
     PCMK_LSB_NOT_RUNNING         = 7,
 };
 
 /* The return codes for the status operation are not the same for other
  * operatios - go figure
  */
 enum lsb_status_exitcode {
     PCMK_LSB_STATUS_OK             = 0,
     PCMK_LSB_STATUS_VAR_PID        = 1,
     PCMK_LSB_STATUS_VAR_LOCK       = 2,
     PCMK_LSB_STATUS_NOT_RUNNING    = 3,
     PCMK_LSB_STATUS_UNKNOWN        = 4,
 
     /* custom codes should be in the 150-199 range reserved for application use */
     PCMK_LSB_STATUS_NOT_INSTALLED      = 150,
     PCMK_LSB_STATUS_INSUFFICIENT_PRIV  = 151,
 };
 
 /* Uniform exit codes
  * Everything is mapped to its OCF equivalent so that Pacemaker only deals with one set of codes
  */
 enum ocf_exitcode {
     PCMK_OCF_OK                   = 0,
     PCMK_OCF_UNKNOWN_ERROR        = 1,
     PCMK_OCF_INVALID_PARAM        = 2,
     PCMK_OCF_UNIMPLEMENT_FEATURE  = 3,
     PCMK_OCF_INSUFFICIENT_PRIV    = 4,
     PCMK_OCF_NOT_INSTALLED        = 5,
     PCMK_OCF_NOT_CONFIGURED       = 6,
     PCMK_OCF_NOT_RUNNING          = 7,  /* End of overlap with LSB */
     PCMK_OCF_RUNNING_MASTER       = 8,
     PCMK_OCF_FAILED_MASTER        = 9,
 
 
     /* 150-199	reserved for application use */
     PCMK_OCF_CONNECTION_DIED = 189, /* Operation failure implied by disconnection of the LRM API to a local or remote node */
 
     PCMK_OCF_DEGRADED        = 190, /* Active resource that is no longer 100% functional */
     PCMK_OCF_DEGRADED_MASTER = 191, /* Promoted resource that is no longer 100% functional */
 
     PCMK_OCF_EXEC_ERROR    = 192, /* Generic problem invoking the agent */
     PCMK_OCF_UNKNOWN       = 193, /* State of the service is unknown - used for recording in-flight operations */
     PCMK_OCF_SIGNAL        = 194,
     PCMK_OCF_NOT_SUPPORTED = 195,
     PCMK_OCF_PENDING       = 196,
     PCMK_OCF_CANCELLED     = 197,
     PCMK_OCF_TIMEOUT       = 198,
     PCMK_OCF_OTHER_ERROR   = 199, /* Keep the same codes as PCMK_LSB */
 };
 
 enum op_status {
     PCMK_LRM_OP_UNKNOWN = -2,
     PCMK_LRM_OP_PENDING = -1,
     PCMK_LRM_OP_DONE,
     PCMK_LRM_OP_CANCELLED,
     PCMK_LRM_OP_TIMEOUT,
     PCMK_LRM_OP_NOTSUPPORTED,
     PCMK_LRM_OP_ERROR,
     PCMK_LRM_OP_ERROR_HARD,
     PCMK_LRM_OP_ERROR_FATAL,
     PCMK_LRM_OP_NOT_INSTALLED,
 };
 
 enum nagios_exitcode {
     NAGIOS_STATE_OK        = 0,
     NAGIOS_STATE_WARNING   = 1,
     NAGIOS_STATE_CRITICAL  = 2,
     NAGIOS_STATE_UNKNOWN   = 3,
     NAGIOS_STATE_DEPENDENT = 4,
 
     NAGIOS_INSUFFICIENT_PRIV = 100,
     NAGIOS_NOT_INSTALLED     = 101,
 };
 
 enum svc_action_flags {
     /* On timeout, only kill pid, do not kill entire pid group */
     SVC_ACTION_LEAVE_GROUP = 0x01,
 };
 
 typedef struct svc_action_private_s svc_action_private_t;
 typedef struct svc_action_s {
     char *id;
     char *rsc;
     char *action;
-    int interval;
+    guint interval_ms;
 
     char *standard;
     char *provider;
     char *agent;
 
     int timeout;
     GHashTable *params; /* used by OCF agents and alert agents */
 
     int rc;
     int pid;
     int cancel;
     int status;
     int sequence;
     int expected_rc;
     int synchronous;
     enum svc_action_flags flags;
 
     char *stderr_data;
     char *stdout_data;
 
     /*!
      * Data stored by the creator of the action.
      *
      * This may be used to hold data that is needed later on by a callback,
      * for example.
      */
     void *cb_data;
 
     svc_action_private_t *opaque;
 } svc_action_t;
 
 /**
  * \brief Get a list of files or directories in a given path
  *
  * \param[in] root       full path to a directory to read
  * \param[in] files      return list of files if TRUE or directories if FALSE
  * \param[in] executable if TRUE and files is TRUE, only return executable files
  *
  * \return a list of what was found.  The list items are char *.
  * \note It is the caller's responsibility to free the result with g_list_free_full(list, free).
  */
     GList *get_directory_list(const char *root, gboolean files, gboolean executable);
 
 /**
  * Get a list of services
  *
  * \return a list of services.  The list items are gchar *.  This list _must_
  *         be destroyed using g_list_free_full(list, free).
  */
     GList *services_list(void);
 
 /**
  * \brief Get a list of providers
  *
  * \param[in] standard  list providers of this standard (e.g. ocf, lsb, etc.)
  *
  * \return a list of providers as char * list items (or NULL if standard does not support providers)
  * \note The caller is responsible for freeing the result using g_list_free_full(list, free).
  */
     GList *resources_list_providers(const char *standard);
 
 /**
  * \brief Get a list of resource agents
  *
  * \param[in] standard  list agents using this standard (e.g. ocf, lsb, etc.) (or NULL for all)
  * \param[in] provider  list agents from this provider (or NULL for all)
  *
  * \return a list of resource agents.  The list items are char *.
  * \note The caller is responsible for freeing the result using g_list_free_full(list, free).
  */
     GList *resources_list_agents(const char *standard, const char *provider);
 
 /**
  * Get list of available standards
  *
  * \return a list of resource standards. The list items are char *. This list _must_
  *         be destroyed using g_list_free_full(list, free).
  */
     GList *resources_list_standards(void);
 
 svc_action_t *services_action_create(const char *name, const char *action,
                                      guint interval_ms, int timeout /* ms */);
 
 /**
  * \brief Create a new resource action
  *
  * \param[in] name        Name of resource
  * \param[in] standard    Resource agent standard (ocf, lsb, etc.)
  * \param[in] provider    Resource agent provider
  * \param[in] agent       Resource agent name
  * \param[in] action      action (start, stop, monitor, etc.)
  * \param[in] interval_ms How often to repeat this action (if 0, execute once)
  * \param[in] timeout     Consider action failed if it does not complete in this many milliseconds
  * \param[in] params      Action parameters
  *
  * \return newly allocated action instance
  *
  * \post After the call, 'params' is owned, and later free'd by the svc_action_t result
  * \note The caller is responsible for freeing the return value using
  *       services_action_free().
  */
 svc_action_t *resources_action_create(const char *name, const char *standard,
                                       const char *provider, const char *agent,
                                       const char *action, guint interval_ms,
                                       int timeout /* ms */, GHashTable *params,
                                       enum svc_action_flags flags);
 
 /**
  * Kick a recurring action so it is scheduled immediately for re-execution
  */
 gboolean services_action_kick(const char *name, const char *action,
                               guint interval_ms);
 
     const char *resources_find_service_class(const char *agent);
 
 /**
  * Utilize services API to execute an arbitrary command.
  *
  * This API has useful infrastructure in place to be able to run a command
  * in the background and get notified via a callback when the command finishes.
  *
  * \param[in] exec command to execute
  * \param[in] args arguments to the command, NULL terminated
  *
  * \return a svc_action_t object, used to pass to the execute function
  * (services_action_sync() or services_action_async()) and is
  * provided to the callback.
  */
     svc_action_t *services_action_create_generic(const char *exec, const char *args[]);
 
     void services_action_cleanup(svc_action_t * op);
     void services_action_free(svc_action_t * op);
     int services_action_user(svc_action_t *op, const char *user);
 
     gboolean services_action_sync(svc_action_t * op);
 
 /**
  * Run an action asynchronously.
  *
  * \param[in] op services action data
  * \param[in] action_callback callback for when the action completes
  *
  * \retval TRUE succesfully started execution
  * \retval FALSE failed to start execution, no callback will be received
  */
     gboolean services_action_async(svc_action_t * op, void (*action_callback) (svc_action_t *));
 
 gboolean services_action_cancel(const char *name, const char *action,
                                 guint interval_ms);
 
 /* functions for alert agents */
 svc_action_t *services_alert_create(const char *id, const char *exec,
                                    int timeout, GHashTable *params,
                                    int sequence, void *cb_data);
 gboolean services_alert_async(svc_action_t *action,
                               void (*cb)(svc_action_t *op));
 
     static inline const char *services_lrm_status_str(enum op_status status) {
         switch (status) {
             case PCMK_LRM_OP_PENDING:
                 return "pending";
                 case PCMK_LRM_OP_DONE:return "complete";
                 case PCMK_LRM_OP_CANCELLED:return "Cancelled";
                 case PCMK_LRM_OP_TIMEOUT:return "Timed Out";
                 case PCMK_LRM_OP_NOTSUPPORTED:return "NOT SUPPORTED";
                 case PCMK_LRM_OP_ERROR:return "Error";
                 case PCMK_LRM_OP_NOT_INSTALLED:return "Not installed";
                 default:return "UNKNOWN!";
         }
     }
 
     static inline const char *services_ocf_exitcode_str(enum ocf_exitcode code) {
         switch (code) {
             case PCMK_OCF_OK:
                 return "ok";
             case PCMK_OCF_UNKNOWN_ERROR:
                 return "unknown error";
             case PCMK_OCF_INVALID_PARAM:
                 return "invalid parameter";
             case PCMK_OCF_UNIMPLEMENT_FEATURE:
                 return "unimplemented feature";
             case PCMK_OCF_INSUFFICIENT_PRIV:
                 return "insufficient privileges";
             case PCMK_OCF_NOT_INSTALLED:
                 return "not installed";
             case PCMK_OCF_NOT_CONFIGURED:
                 return "not configured";
             case PCMK_OCF_NOT_RUNNING:
                 return "not running";
             case PCMK_OCF_RUNNING_MASTER:
                 return "master";
             case PCMK_OCF_FAILED_MASTER:
                 return "master (failed)";
             case PCMK_OCF_SIGNAL:
                 return "OCF_SIGNAL";
             case PCMK_OCF_NOT_SUPPORTED:
                 return "OCF_NOT_SUPPORTED";
             case PCMK_OCF_PENDING:
                 return "OCF_PENDING";
             case PCMK_OCF_CANCELLED:
                 return "OCF_CANCELLED";
             case PCMK_OCF_TIMEOUT:
                 return "OCF_TIMEOUT";
             case PCMK_OCF_OTHER_ERROR:
                 return "OCF_OTHER_ERROR";
             case PCMK_OCF_DEGRADED:
                 return "OCF_DEGRADED";
             case PCMK_OCF_DEGRADED_MASTER:
                 return "OCF_DEGRADED_MASTER";
             default:
                 return "unknown";
         }
     }
 
     /**
      * \brief Get OCF equivalent of LSB exit code
      *
      * \param[in] action        LSB action that produced exit code
      * \param[in] lsb_exitcode  Exit code of LSB action
      *
      * \return PCMK_OCF_* constant that corresponds to LSB exit code
      */
     static inline enum ocf_exitcode
     services_get_ocf_exitcode(const char *action, int lsb_exitcode)
     {
         /* For non-status actions, LSB and OCF share error code meaning <= 7 */
         if (action && strcmp(action, "status") && strcmp(action, "monitor")) {
             if ((lsb_exitcode < 0) || (lsb_exitcode > PCMK_LSB_NOT_RUNNING)) {
                 return PCMK_OCF_UNKNOWN_ERROR;
             }
             return (enum ocf_exitcode)lsb_exitcode;
         }
 
         /* status has different return codes */
         switch (lsb_exitcode) {
             case PCMK_LSB_STATUS_OK:
                 return PCMK_OCF_OK;
             case PCMK_LSB_STATUS_NOT_INSTALLED:
                 return PCMK_OCF_NOT_INSTALLED;
             case PCMK_LSB_STATUS_INSUFFICIENT_PRIV:
                 return PCMK_OCF_INSUFFICIENT_PRIV;
             case PCMK_LSB_STATUS_VAR_PID:
             case PCMK_LSB_STATUS_VAR_LOCK:
             case PCMK_LSB_STATUS_NOT_RUNNING:
                 return PCMK_OCF_NOT_RUNNING;
         }
         return PCMK_OCF_UNKNOWN_ERROR;
     }
 
 #  ifdef __cplusplus
 }
 #  endif
 
 #endif                          /* __PCMK_SERVICES__ */
diff --git a/include/crm/transition.h b/include/crm/transition.h
index 9c5614b48b..ad8219749c 100644
--- a/include/crm/transition.h
+++ b/include/crm/transition.h
@@ -1,151 +1,151 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 #ifndef CRM_TRANSITION__H
 #  define CRM_TRANSITION__H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 typedef enum {
     action_type_pseudo,
     action_type_rsc,
     action_type_crm
 } action_type_e;
 
 typedef struct te_timer_s crm_action_timer_t;
 typedef struct crm_graph_s crm_graph_t;
 
 typedef struct synapse_s {
     int id;
     int priority;
 
     gboolean ready;
     gboolean failed;
     gboolean executed;
     gboolean confirmed;
 
     GListPtr actions;           /* crm_action_t* */
     GListPtr inputs;            /* crm_action_t* */
 } synapse_t;
 
 typedef struct crm_action_s {
     int id;
     int timeout;
-    int interval;
+    guint interval_ms;
     GHashTable *params;
     action_type_e type;
 
     crm_action_timer_t *timer;
     synapse_t *synapse;
 
     gboolean sent_update;       /* sent to the CIB */
     gboolean executed;          /* sent to the CRM */
     gboolean confirmed;
 
     gboolean failed;
     gboolean can_fail;
 
     xmlNode *xml;
 
 } crm_action_t;
 
 struct te_timer_s {
     int source_id;
     int timeout;
     crm_action_t *action;
 };
 
 /* order matters here */
 enum transition_action {
     tg_done,
     tg_stop,
     tg_restart,
     tg_shutdown,
 };
 
 struct crm_graph_s {
     int id;
     char *source;
     int abort_priority;
 
     gboolean complete;
     const char *abort_reason;
     enum transition_action completion_action;
 
     int num_actions;
     int num_synapses;
 
     int batch_limit;
     int network_delay;
     int stonith_timeout;
     int transition_timeout;
 
     int fired;
     int pending;
     int skipped;
     int completed;
     int incomplete;
 
     GListPtr synapses;          /* synapse_t* */
 
     int migration_limit;
 };
 
 typedef struct crm_graph_functions_s {
     gboolean(*pseudo) (crm_graph_t * graph, crm_action_t * action);
     gboolean(*rsc) (crm_graph_t * graph, crm_action_t * action);
     gboolean(*crmd) (crm_graph_t * graph, crm_action_t * action);
     gboolean(*stonith) (crm_graph_t * graph, crm_action_t * action);
     gboolean(*allowed) (crm_graph_t * graph, crm_action_t * action);
 } crm_graph_functions_t;
 
 enum transition_status {
     transition_active,
     transition_pending,         /* active but no actions performed this time */
     transition_complete,
     transition_stopped,
     transition_terminated,
     transition_action_failed,
     transition_failed,
 };
 
 void set_default_graph_functions(void);
 void set_graph_functions(crm_graph_functions_t * fns);
 crm_graph_t *unpack_graph(xmlNode * xml_graph, const char *reference);
 int run_graph(crm_graph_t * graph);
 gboolean update_graph(crm_graph_t * graph, crm_action_t * action);
 void destroy_graph(crm_graph_t * graph);
 const char *transition_status(enum transition_status state);
 void print_graph(unsigned int log_level, crm_graph_t * graph);
 void print_action(int log_level, const char *prefix, crm_action_t * action);
 bool update_abort_priority(crm_graph_t * graph, int priority,
                            enum transition_action action, const char *abort_reason);
 const char *actiontype2text(action_type_e type);
 lrmd_event_data_t *convert_graph_action(xmlNode * resource, crm_action_t * action, int status,
                                         int rc);
 
 #ifdef __cplusplus
 }
 #endif
 
 #endif
diff --git a/lib/common/operations.c b/lib/common/operations.c
index 507adb2a4f..9eed251b2b 100644
--- a/lib/common/operations.c
+++ b/lib/common/operations.c
@@ -1,566 +1,566 @@
 /*
  * Copyright (C) 2004-2017 Andrew Beekhof <andrew@beekhof.net>
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #ifndef _GNU_SOURCE
 #  define _GNU_SOURCE
 #endif
 
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
 #include <ctype.h>
 
 #include <crm/crm.h>
 #include <crm/lrmd.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/util.h>
 
 /*!
  * \brief Generate an operation key
  *
  * \param[in] rsc_id       ID of resource being operated on
  * \param[in] op_type      Operation name
  * \param[in] interval_ms  Operation interval
  *
  * \return Newly allocated memory containing operation key as string
  *
  * \note It is the caller's responsibility to free() the result.
  */
 char *
 generate_op_key(const char *rsc_id, const char *op_type, guint interval_ms)
 {
     CRM_ASSERT(rsc_id != NULL);
     CRM_ASSERT(op_type != NULL);
     return crm_strdup_printf(CRM_OP_FMT, rsc_id, op_type, interval_ms);
 }
 
 gboolean
 parse_op_key(const char *key, char **rsc_id, char **op_type, guint *interval_ms)
 {
     char *notify = NULL;
     char *mutable_key = NULL;
     char *mutable_key_ptr = NULL;
     size_t len = 0, offset = 0;
     unsigned long long ch = 0;
 
     CRM_CHECK(key != NULL, return FALSE);
 
     // Parse interval at end of string
     *interval_ms = 0;
     len = strlen(key);
     offset = len - 1;
     while ((offset > 0) && isdigit(key[offset])) {
         ch = key[offset] - '0';
         for (int digits = len - offset; digits > 1; --digits) {
             ch = ch * 10;
         }
         *interval_ms += ch;
         offset--;
     }
     crm_trace("Operation key '%s' has interval %ums", key, *interval_ms);
 
     CRM_CHECK(key[offset] == '_', return FALSE);
 
     mutable_key = strdup(key);
     mutable_key[offset] = 0;
     offset--;
 
     while (offset > 0 && key[offset] != '_') {
         offset--;
     }
 
     CRM_CHECK(key[offset] == '_', free(mutable_key);
               return FALSE);
 
     mutable_key_ptr = mutable_key + offset + 1;
 
     crm_trace("  Action: %s", mutable_key_ptr);
 
     *op_type = strdup(mutable_key_ptr);
 
     mutable_key[offset] = 0;
     offset--;
 
     CRM_CHECK(mutable_key != mutable_key_ptr, free(mutable_key);
               return FALSE);
 
     notify = strstr(mutable_key, "_post_notify");
     if (notify && safe_str_eq(notify, "_post_notify")) {
         notify[0] = 0;
     }
 
     notify = strstr(mutable_key, "_pre_notify");
     if (notify && safe_str_eq(notify, "_pre_notify")) {
         notify[0] = 0;
     }
 
     crm_trace("  Resource: %s", mutable_key);
     *rsc_id = mutable_key;
 
     return TRUE;
 }
 
 char *
 generate_notify_key(const char *rsc_id, const char *notify_type, const char *op_type)
 {
     CRM_CHECK(rsc_id != NULL, return NULL);
     CRM_CHECK(op_type != NULL, return NULL);
     CRM_CHECK(notify_type != NULL, return NULL);
     return crm_strdup_printf("%s_%s_notify_%s_0",
                              rsc_id, notify_type, op_type);
 }
 
 char *
 generate_transition_magic(const char *transition_key, int op_status, int op_rc)
 {
     CRM_CHECK(transition_key != NULL, return NULL);
     return crm_strdup_printf("%d:%d;%s", op_status, op_rc, transition_key);
 }
 
 gboolean
 decode_transition_magic(const char *magic, char **uuid, int *transition_id, int *action_id,
                         int *op_status, int *op_rc, int *target_rc)
 {
     int res = 0;
     char *key = NULL;
     gboolean result = TRUE;
 
     CRM_CHECK(magic != NULL, return FALSE);
     CRM_CHECK(op_rc != NULL, return FALSE);
     CRM_CHECK(op_status != NULL, return FALSE);
 
     key = calloc(1, strlen(magic) + 1);
     res = sscanf(magic, "%d:%d;%s", op_status, op_rc, key);
     if (res != 3) {
         crm_warn("Only found %d items in: '%s'", res, magic);
         free(key);
         return FALSE;
     }
 
     CRM_CHECK(decode_transition_key(key, uuid, transition_id, action_id, target_rc), result = FALSE);
 
     free(key);
     return result;
 }
 
 char *
 generate_transition_key(int transition_id, int action_id, int target_rc, const char *node)
 {
     CRM_CHECK(node != NULL, return NULL);
     return crm_strdup_printf("%d:%d:%d:%-*s",
                              action_id, transition_id, target_rc, 36, node);
 }
 
 gboolean
 decode_transition_key(const char *key, char **uuid, int *transition_id, int *action_id,
                       int *target_rc)
 {
     CRM_CHECK(uuid != NULL, return FALSE);
     CRM_CHECK(target_rc != NULL, return FALSE);
     CRM_CHECK(action_id != NULL, return FALSE);
     CRM_CHECK(transition_id != NULL, return FALSE);
 
     *uuid = calloc(37, sizeof(char));
     if (sscanf(key, "%d:%d:%d:%36s",
                action_id, transition_id, target_rc, *uuid) != 4) {
         crm_err("Invalid transition key '%s'", key);
         free(*uuid);
         *uuid = NULL;
         *target_rc = -1;
         *action_id = -1;
         *transition_id = -1;
         return FALSE;
     }
     if (strlen(*uuid) != 36) {
         crm_warn("Invalid UUID '%s' in transition key '%s'", *uuid, key);
     }
     return TRUE;
 }
 
 void
 filter_action_parameters(xmlNode * param_set, const char *version)
 {
     char *key = NULL;
     char *timeout = NULL;
     char *interval_ms_s = NULL;
 
     const char *attr_filter[] = {
         XML_ATTR_ID,
         XML_ATTR_CRM_VERSION,
         XML_LRM_ATTR_OP_DIGEST,
         XML_LRM_ATTR_TARGET,
         XML_LRM_ATTR_TARGET_UUID,
         "pcmk_external_ip"
     };
 
     gboolean do_delete = FALSE;
     int lpc = 0;
     static int meta_len = 0;
 
     if (meta_len == 0) {
         meta_len = strlen(CRM_META);
     }
 
     if (param_set == NULL) {
         return;
     }
 
     for (lpc = 0; lpc < DIMOF(attr_filter); lpc++) {
         xml_remove_prop(param_set, attr_filter[lpc]);
     }
 
     key = crm_meta_name(XML_LRM_ATTR_INTERVAL_MS);
     interval_ms_s = crm_element_value_copy(param_set, key);
     free(key);
 
     key = crm_meta_name(XML_ATTR_TIMEOUT);
     timeout = crm_element_value_copy(param_set, key);
 
     if (param_set) {
         xmlAttrPtr xIter = param_set->properties;
 
         while (xIter) {
             const char *prop_name = (const char *)xIter->name;
 
             xIter = xIter->next;
             do_delete = FALSE;
             if (strncasecmp(prop_name, CRM_META, meta_len) == 0) {
                 do_delete = TRUE;
             }
 
             if (do_delete) {
                 xml_remove_prop(param_set, prop_name);
             }
         }
     }
 
     if (crm_get_msec(interval_ms_s) > 0) {
         /* Re-instate the operation's timeout value */
         if (timeout != NULL) {
             crm_xml_add(param_set, key, timeout);
         }
     }
 
     free(interval_ms_s);
     free(timeout);
     free(key);
 }
 
 #define FAKE_TE_ID	"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
 static void
 append_digest(lrmd_event_data_t * op, xmlNode * update, const char *version, const char *magic,
               int level)
 {
     /* this will enable us to later determine that the
      *   resource's parameters have changed and we should force
      *   a restart
      */
     char *digest = NULL;
     xmlNode *args_xml = NULL;
 
     if (op->params == NULL) {
         return;
     }
 
     args_xml = create_xml_node(NULL, XML_TAG_PARAMS);
     g_hash_table_foreach(op->params, hash2field, args_xml);
     filter_action_parameters(args_xml, version);
     digest = calculate_operation_digest(args_xml, version);
 
 #if 0
     if (level < get_crm_log_level()
-        && op->interval == 0 && crm_str_eq(op->op_type, CRMD_ACTION_START, TRUE)) {
+        && op->interval_ms == 0 && crm_str_eq(op->op_type, CRMD_ACTION_START, TRUE)) {
         char *digest_source = dump_xml_unformatted(args_xml);
 
         do_crm_log(level, "Calculated digest %s for %s (%s). Source: %s\n",
                    digest, ID(update), magic, digest_source);
         free(digest_source);
     }
 #endif
     crm_xml_add(update, XML_LRM_ATTR_OP_DIGEST, digest);
 
     free_xml(args_xml);
     free(digest);
 }
 
 int
 rsc_op_expected_rc(lrmd_event_data_t * op)
 {
     int rc = 0;
 
     if (op && op->user_data) {
         int dummy = 0;
         char *uuid = NULL;
 
         decode_transition_key(op->user_data, &uuid, &dummy, &dummy, &rc);
         free(uuid);
     }
     return rc;
 }
 
 gboolean
 did_rsc_op_fail(lrmd_event_data_t * op, int target_rc)
 {
     switch (op->op_status) {
         case PCMK_LRM_OP_CANCELLED:
         case PCMK_LRM_OP_PENDING:
             return FALSE;
             break;
 
         case PCMK_LRM_OP_NOTSUPPORTED:
         case PCMK_LRM_OP_TIMEOUT:
         case PCMK_LRM_OP_ERROR:
             return TRUE;
             break;
 
         default:
             if (target_rc != op->rc) {
                 return TRUE;
             }
     }
 
     return FALSE;
 }
 
 /*!
  * \brief Create a CIB XML element for an operation
  *
  * \param[in] parent         If not NULL, make new XML node a child of this one
  * \param[in] prefix         Generate an ID using this prefix
  * \param[in] task           Operation task to set
  * \param[in] interval_spec  Operation interval to set
  * \param[in] timeout        If not NULL, operation timeout to set
  *
  * \return New XML object on success, NULL otherwise
  */
 xmlNode *
 crm_create_op_xml(xmlNode *parent, const char *prefix, const char *task,
                   const char *interval_spec, const char *timeout)
 {
     xmlNode *xml_op;
 
     CRM_CHECK(prefix && task && interval_spec, return NULL);
 
     xml_op = create_xml_node(parent, XML_ATTR_OP);
     crm_xml_set_id(xml_op, "%s-%s-%s", prefix, task, interval_spec);
     crm_xml_add(xml_op, XML_LRM_ATTR_INTERVAL, interval_spec);
     crm_xml_add(xml_op, "name", task);
     if (timeout) {
         crm_xml_add(xml_op, XML_ATTR_TIMEOUT, timeout);
     }
     return xml_op;
 }
 
 xmlNode *
 create_operation_update(xmlNode * parent, lrmd_event_data_t * op, const char * caller_version,
                         int target_rc, const char * node, const char * origin, int level)
 {
     char *key = NULL;
     char *magic = NULL;
     char *op_id = NULL;
     char *op_id_additional = NULL;
     char *local_user_data = NULL;
     const char *exit_reason = NULL;
 
     xmlNode *xml_op = NULL;
     const char *task = NULL;
 
     CRM_CHECK(op != NULL, return NULL);
-    do_crm_log(level, "%s: Updating resource %s after %s op %s (interval=%d)",
+    do_crm_log(level, "%s: Updating resource %s after %s op %s (interval=%u)",
                origin, op->rsc_id, op->op_type, services_lrm_status_str(op->op_status),
-               op->interval);
+               op->interval_ms);
 
     crm_trace("DC version: %s", caller_version);
 
     task = op->op_type;
 
     /* Remap the task name under various scenarios, to make life easier for the
      * PE when determining the current state.
      */
     if (crm_str_eq(task, "reload", TRUE)) {
         if (op->op_status == PCMK_LRM_OP_DONE) {
             task = CRMD_ACTION_START;
         } else {
             task = CRMD_ACTION_STATUS;
         }
 
     } else if (crm_str_eq(task, CRMD_ACTION_MIGRATE, TRUE)) {
         /* if the migrate_from fails it will have enough info to do the right thing */
         if (op->op_status == PCMK_LRM_OP_DONE) {
             task = CRMD_ACTION_STOP;
         } else {
             task = CRMD_ACTION_STATUS;
         }
 
     } else if ((op->op_status == PCMK_LRM_OP_DONE)
                && crm_str_eq(task, CRMD_ACTION_MIGRATED, TRUE)) {
         task = CRMD_ACTION_START;
     }
 
-    key = generate_op_key(op->rsc_id, task, op->interval);
+    key = generate_op_key(op->rsc_id, task, op->interval_ms);
     if (crm_str_eq(task, CRMD_ACTION_NOTIFY, TRUE)) {
         const char *n_type = crm_meta_value(op->params, "notify_type");
         const char *n_task = crm_meta_value(op->params, "notify_operation");
 
         CRM_LOG_ASSERT(n_type != NULL);
         CRM_LOG_ASSERT(n_task != NULL);
         op_id = generate_notify_key(op->rsc_id, n_type, n_task);
 
         /* these are not yet allowed to fail */
         op->op_status = PCMK_LRM_OP_DONE;
         op->rc = 0;
 
     } else if (did_rsc_op_fail(op, target_rc)) {
         op_id = generate_op_key(op->rsc_id, "last_failure", 0);
-        if (op->interval == 0) {
+        if (op->interval_ms == 0) {
             /* Ensure 'last' gets updated too in case recording-pending="true" */
             op_id_additional = generate_op_key(op->rsc_id, "last", 0);
         }
         exit_reason = op->exit_reason;
 
-    } else if (op->interval > 0) {
+    } else if (op->interval_ms > 0) {
         op_id = strdup(key);
 
     } else {
         op_id = generate_op_key(op->rsc_id, "last", 0);
     }
 
   again:
     xml_op = find_entity(parent, XML_LRM_TAG_RSC_OP, op_id);
     if (xml_op == NULL) {
         xml_op = create_xml_node(parent, XML_LRM_TAG_RSC_OP);
     }
 
     if (op->user_data == NULL) {
-        crm_debug("Generating fake transition key for:"
-                  " %s_%s_%d %d from %s",
-                  op->rsc_id, op->op_type, op->interval, op->call_id, origin);
+        crm_debug("Generating fake transition key for: " CRM_OP_FMT " %d from %s",
+                  op->rsc_id, op->op_type, op->interval_ms,
+                  op->call_id, origin);
         local_user_data = generate_transition_key(-1, op->call_id, target_rc, FAKE_TE_ID);
         op->user_data = local_user_data;
     }
 
     if(magic == NULL) {
         magic = generate_transition_magic(op->user_data, op->op_status, op->rc);
     }
 
     crm_xml_add(xml_op, XML_ATTR_ID, op_id);
     crm_xml_add(xml_op, XML_LRM_ATTR_TASK_KEY, key);
     crm_xml_add(xml_op, XML_LRM_ATTR_TASK, task);
     crm_xml_add(xml_op, XML_ATTR_ORIGIN, origin);
     crm_xml_add(xml_op, XML_ATTR_CRM_VERSION, caller_version);
     crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY, op->user_data);
     crm_xml_add(xml_op, XML_ATTR_TRANSITION_MAGIC, magic);
     crm_xml_add(xml_op, XML_LRM_ATTR_EXIT_REASON, exit_reason == NULL ? "" : exit_reason);
     crm_xml_add(xml_op, XML_LRM_ATTR_TARGET, node); /* For context during triage */
 
     crm_xml_add_int(xml_op, XML_LRM_ATTR_CALLID, op->call_id);
     crm_xml_add_int(xml_op, XML_LRM_ATTR_RC, op->rc);
     crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS, op->op_status);
-    crm_xml_add_int(xml_op, XML_LRM_ATTR_INTERVAL_MS, op->interval);
+    crm_xml_add_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, op->interval_ms);
 
     if (compare_version("2.1", caller_version) <= 0) {
         if (op->t_run || op->t_rcchange || op->exec_time || op->queue_time) {
-            crm_trace("Timing data (%s_%s_%d): last=%u change=%u exec=%u queue=%u",
-                      op->rsc_id, op->op_type, op->interval,
+            crm_trace("Timing data (" CRM_OP_FMT "): last=%u change=%u exec=%u queue=%u",
+                      op->rsc_id, op->op_type, op->interval_ms,
                       op->t_run, op->t_rcchange, op->exec_time, op->queue_time);
 
-            if (op->interval == 0) {
+            if (op->interval_ms == 0) {
                 /* The values are the same for non-recurring ops */
                 crm_xml_add_int(xml_op, XML_RSC_OP_LAST_RUN, op->t_run);
                 crm_xml_add_int(xml_op, XML_RSC_OP_LAST_CHANGE, op->t_run);
 
             } else if(op->t_rcchange) {
                 /* last-run is not accurate for recurring ops */
                 crm_xml_add_int(xml_op, XML_RSC_OP_LAST_CHANGE, op->t_rcchange);
 
             } else {
                 /* ...but is better than nothing otherwise */
                 crm_xml_add_int(xml_op, XML_RSC_OP_LAST_CHANGE, op->t_run);
             }
 
             crm_xml_add_int(xml_op, XML_RSC_OP_T_EXEC, op->exec_time);
             crm_xml_add_int(xml_op, XML_RSC_OP_T_QUEUE, op->queue_time);
         }
     }
 
     if (crm_str_eq(op->op_type, CRMD_ACTION_MIGRATE, TRUE)
         || crm_str_eq(op->op_type, CRMD_ACTION_MIGRATED, TRUE)) {
         /*
          * Record migrate_source and migrate_target always for migrate ops.
          */
         const char *name = XML_LRM_ATTR_MIGRATE_SOURCE;
 
         crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
 
         name = XML_LRM_ATTR_MIGRATE_TARGET;
         crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
     }
 
     append_digest(op, xml_op, caller_version, magic, LOG_DEBUG);
 
     if (op_id_additional) {
         free(op_id);
         op_id = op_id_additional;
         op_id_additional = NULL;
         goto again;
     }
 
     if (local_user_data) {
         free(local_user_data);
         op->user_data = NULL;
     }
     free(magic);
     free(op_id);
     free(key);
     return xml_op;
 }
 
 /*!
  * \brief Check whether an operation requires resource agent meta-data
  *
  * \param[in] rsc_class  Resource agent class (or NULL to skip class check)
  * \param[in] op         Operation action (or NULL to skip op check)
  *
  * \return TRUE if operation needs meta-data, FALSE otherwise
  * \note At least one of rsc_class and op must be specified.
  */
 bool
 crm_op_needs_metadata(const char *rsc_class, const char *op)
 {
     /* Agent meta-data is used to determine whether a reload is possible, and to
      * evaluate versioned parameters -- so if this op is not relevant to those
      * features, we don't need the meta-data.
      */
 
     CRM_CHECK(rsc_class || op, return FALSE);
 
     if (rsc_class
         && strcmp(rsc_class, PCMK_RESOURCE_CLASS_OCF)
         && strcmp(rsc_class, PCMK_RESOURCE_CLASS_STONITH)) {
 
         /* Meta-data is only needed for resource classes that use parameters */
         return FALSE;
     }
 
     /* Meta-data is only needed for these actions */
     if (op
         && strcmp(op, CRMD_ACTION_START)
         && strcmp(op, CRMD_ACTION_STATUS)
         && strcmp(op, CRMD_ACTION_PROMOTE)
         && strcmp(op, CRMD_ACTION_DEMOTE)
         && strcmp(op, CRMD_ACTION_RELOAD)
         && strcmp(op, CRMD_ACTION_MIGRATE)
         && strcmp(op, CRMD_ACTION_MIGRATED)
         && strcmp(op, CRMD_ACTION_NOTIFY)) {
         return FALSE;
     }
 
     return TRUE;
 }
diff --git a/lib/lrmd/lrmd_alerts.c b/lib/lrmd/lrmd_alerts.c
index ae78c77c6c..43a7de41cf 100644
--- a/lib/lrmd/lrmd_alerts.c
+++ b/lib/lrmd/lrmd_alerts.c
@@ -1,367 +1,378 @@
 /*
  * Copyright (c) 2015 David Vossel <davidvossel@gmail.com>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  * 
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  *
  */
 
 #include <crm_internal.h>
 
 #include <glib.h>
 #include <unistd.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/services.h>
 #include <crm/common/mainloop.h>
 #include <crm/common/alerts_internal.h>
 #include <crm/common/iso8601_internal.h>
 #include <crm/lrmd_alerts_internal.h>
 
 #include <crm/pengine/status.h>
 #include <crm/cib.h>
 #include <crm/lrmd.h>
 
 static lrmd_key_value_t *
 alert_key2param(lrmd_key_value_t *head, enum crm_alert_keys_e name,
                 const char *value)
 {
     const char **key;
 
     if (value == NULL) {
         value = "";
     }
     for (key = crm_alert_keys[name]; *key; key++) {
         crm_trace("Setting alert key %s = '%s'", *key, value);
         head = lrmd_key_value_add(head, *key, value);
     }
     return head;
 }
 
 static lrmd_key_value_t *
 alert_key2param_int(lrmd_key_value_t *head, enum crm_alert_keys_e name,
                     int value)
 {
     char *value_s = crm_itoa(value);
 
     head = alert_key2param(head, name, value_s);
     free(value_s);
     return head;
 }
 
+static lrmd_key_value_t *
+alert_key2param_ms(lrmd_key_value_t *head, enum crm_alert_keys_e name,
+                   guint value)
+{
+    char *value_s = crm_strdup_printf("%u", value);
+
+    head = alert_key2param(head, name, value_s);
+    free(value_s);
+    return head;
+}
+
 static void
 set_ev_kv(gpointer key, gpointer value, gpointer user_data)
 {
     lrmd_key_value_t **head = (lrmd_key_value_t **) user_data;
 
     if (value) {
         crm_trace("Setting environment variable %s='%s'",
                   (char*)key, (char*)value);
         *head = lrmd_key_value_add(*head, key, value);
     }
 }
 
 static lrmd_key_value_t *
 alert_envvar2params(lrmd_key_value_t *head, crm_alert_entry_t *entry)
 {
     if (entry->envvars) {
         g_hash_table_foreach(entry->envvars, set_ev_kv, &head);
     }
     return head;
 }
 
 /*
  * We could use g_strv_contains() instead of this function,
  * but that has only been available since glib 2.43.2.
  */
 static gboolean
 is_target_alert(char **list, const char *value)
 {
     int target_list_num = 0;
     gboolean rc = FALSE;
 
     CRM_CHECK(value != NULL, return FALSE);
 
     if (list == NULL) {
         return TRUE;
     }
 
     target_list_num = g_strv_length(list);
 
     for (int cnt = 0; cnt < target_list_num; cnt++) {
         if (strcmp(list[cnt], value) == 0) {
             rc = TRUE;
             break;
         }
     }
     return rc;
 }
 
 /*!
  * \internal
  * \brief Execute alert agents for an event
  *
  * \param[in]     lrmd        LRMD connection to use
  * \param[in]     alert_list  Alerts to execute
  * \param[in]     kind        Type of event that is being alerted for
  * \param[in]     attr_name   If crm_alert_attribute, the attribute name
  * \param[in,out] params      Environment variables to pass to agents
  *
  * \retval pcmk_ok on success
  * \retval -1 if some alerts failed
  * \retval -2 if all alerts failed
  */
 static int
 exec_alert_list(lrmd_t *lrmd, GList *alert_list, enum crm_alert_flags kind,
                 const char *attr_name, lrmd_key_value_t *params)
 {
     bool any_success = FALSE, any_failure = FALSE;
     const char *kind_s = crm_alert_flag2text(kind);
     crm_time_hr_t *now = NULL;
 
     params = alert_key2param(params, CRM_alert_kind, kind_s);
     params = alert_key2param(params, CRM_alert_version, VERSION);
 
     for (GList *iter = g_list_first(alert_list); iter; iter = g_list_next(iter)) {
         crm_alert_entry_t *entry = (crm_alert_entry_t *)(iter->data);
         lrmd_key_value_t *copy_params = NULL;
         lrmd_key_value_t *head = NULL;
         int rc;
 
         if (is_not_set(entry->flags, kind)) {
             crm_trace("Filtering unwanted %s alert to %s via %s",
                       kind_s, entry->recipient, entry->id);
             continue;
         }
 
         if ((kind == crm_alert_attribute)
             && !is_target_alert(entry->select_attribute_name, attr_name)) {
 
             crm_trace("Filtering unwanted attribute '%s' alert to %s via %s",
                       attr_name, entry->recipient, entry->id);
             continue;
         }
 
         if (now == NULL) {
             now = crm_time_hr_new(NULL);
         }
         crm_info("Sending %s alert via %s to %s",
                  kind_s, entry->id, entry->recipient);
 
         /* Make a copy of the parameters, because each alert will be unique */
         for (head = params; head != NULL; head = head->next) {
             copy_params = lrmd_key_value_add(copy_params, head->key, head->value);
         }
 
         copy_params = alert_key2param(copy_params, CRM_alert_recipient,
                                       entry->recipient);
 
         if (now) {
             char *timestamp = crm_time_format_hr(entry->tstamp_format, now);
 
             if (timestamp) {
                 copy_params = alert_key2param(copy_params, CRM_alert_timestamp,
                                               timestamp);
                 free(timestamp);
             }
         }
 
         copy_params = alert_envvar2params(copy_params, entry);
 
         rc = lrmd->cmds->exec_alert(lrmd, entry->id, entry->path,
                                     entry->timeout, copy_params);
         if (rc < 0) {
             crm_err("Could not execute alert %s: %s " CRM_XS " rc=%d",
                     entry->id, pcmk_strerror(rc), rc);
             any_failure = TRUE;
         } else {
             any_success = TRUE;
         }
     }
 
     if (now) {
         free(now);
     }
 
     if (any_failure) {
         return (any_success? -1 : -2);
     }
     return pcmk_ok;
 }
 
 /*!
  * \internal
  * \brief Send an alert for a node attribute change
  *
  * \param[in] lrmd        LRMD connection to use
  * \param[in] alert_list  List of alert agents to execute
  * \param[in] node        Name of node with attribute change
  * \param[in] nodeid      Node ID of node with attribute change
  * \param[in] attr_name   Name of attribute that changed
  * \param[in] attr_value  New value of attribute that changed
  *
  * \retval pcmk_ok on success
  * \retval -1 if some alert agents failed
  * \retval -2 if all alert agents failed
  */
 int
 lrmd_send_attribute_alert(lrmd_t *lrmd, GList *alert_list,
                           const char *node, uint32_t nodeid,
                           const char *attr_name, const char *attr_value)
 {
     int rc = pcmk_ok;
     lrmd_key_value_t *params = NULL;
 
     if (lrmd == NULL) {
         return -2;
     }
 
     params = alert_key2param(params, CRM_alert_node, node);
     params = alert_key2param_int(params, CRM_alert_nodeid, nodeid);
     params = alert_key2param(params, CRM_alert_attribute_name, attr_name);
     params = alert_key2param(params, CRM_alert_attribute_value, attr_value);
 
     rc = exec_alert_list(lrmd, alert_list, crm_alert_attribute, attr_name,
                          params);
     lrmd_key_value_freeall(params);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Send an alert for a node membership event
  *
  * \param[in] lrmd        LRMD connection to use
  * \param[in] alert_list  List of alert agents to execute
  * \param[in] node        Name of node with change
  * \param[in] nodeid      Node ID of node with change
  * \param[in] state       New state of node with change
  *
  * \retval pcmk_ok on success
  * \retval -1 if some alert agents failed
  * \retval -2 if all alert agents failed
  */
 int
 lrmd_send_node_alert(lrmd_t *lrmd, GList *alert_list,
                      const char *node, uint32_t nodeid, const char *state)
 {
     int rc = pcmk_ok;
     lrmd_key_value_t *params = NULL;
 
     if (lrmd == NULL) {
         return -2;
     }
 
     params = alert_key2param(params, CRM_alert_node, node);
     params = alert_key2param(params, CRM_alert_desc, state);
     params = alert_key2param_int(params, CRM_alert_nodeid, nodeid);
 
     rc = exec_alert_list(lrmd, alert_list, crm_alert_node, NULL, params);
     lrmd_key_value_freeall(params);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Send an alert for a fencing event
  *
  * \param[in] lrmd        LRMD connection to use
  * \param[in] alert_list  List of alert agents to execute
  * \param[in] target      Name of fence target node
  * \param[in] task        Type of fencing event that occurred
  * \param[in] desc        Readable description of event
  * \param[in] op_rc       Result of fence action
  *
  * \retval pcmk_ok on success
  * \retval -1 if some alert agents failed
  * \retval -2 if all alert agents failed
  */
 int
 lrmd_send_fencing_alert(lrmd_t *lrmd, GList *alert_list,
                         const char *target, const char *task, const char *desc,
                         int op_rc)
 {
     int rc = pcmk_ok;
     lrmd_key_value_t *params = NULL;
 
     if (lrmd == NULL) {
         return -2;
     }
 
     params = alert_key2param(params, CRM_alert_node, target);
     params = alert_key2param(params, CRM_alert_task, task);
     params = alert_key2param(params, CRM_alert_desc, desc);
     params = alert_key2param_int(params, CRM_alert_rc, op_rc);
 
     rc = exec_alert_list(lrmd, alert_list, crm_alert_fencing, NULL, params);
     lrmd_key_value_freeall(params);
     return rc;
 }
 
 /*!
  * \internal
  * \brief Send an alert for a resource operation
  *
  * \param[in] lrmd        LRMD connection to use
  * \param[in] alert_list  List of alert agents to execute
  * \param[in] node        Name of node that executed operation
  * \param[in] op          Resource operation
  *
  * \retval pcmk_ok on success
  * \retval -1 if some alert agents failed
  * \retval -2 if all alert agents failed
  */
 int
 lrmd_send_resource_alert(lrmd_t *lrmd, GList *alert_list,
                          const char *node, lrmd_event_data_t *op)
 {
     int rc = pcmk_ok;
     int target_rc = pcmk_ok;
     lrmd_key_value_t *params = NULL;
 
     if (lrmd == NULL) {
         return -2;
     }
 
     target_rc = rsc_op_expected_rc(op);
-    if ((op->interval == 0) && (target_rc == op->rc)
+    if ((op->interval_ms == 0) && (target_rc == op->rc)
         && safe_str_eq(op->op_type, RSC_STATUS)) {
 
         /* Don't send alerts for probes with the expected result. Leave it up to
          * the agent whether to alert for 'failed' probes. (Even if we find a
          * resource running, it was probably because someone did a clean-up of
          * the status section.)
          */
         return pcmk_ok;
     }
 
     params = alert_key2param(params, CRM_alert_node, node);
     params = alert_key2param(params, CRM_alert_rsc, op->rsc_id);
     params = alert_key2param(params, CRM_alert_task, op->op_type);
-    params = alert_key2param_int(params, CRM_alert_interval, op->interval);
+    params = alert_key2param_ms(params, CRM_alert_interval, op->interval_ms);
     params = alert_key2param_int(params, CRM_alert_target_rc, target_rc);
     params = alert_key2param_int(params, CRM_alert_status, op->op_status);
     params = alert_key2param_int(params, CRM_alert_rc, op->rc);
 
     if (op->op_status == PCMK_LRM_OP_DONE) {
         params = alert_key2param(params, CRM_alert_desc, services_ocf_exitcode_str(op->rc));
     } else {
         params = alert_key2param(params, CRM_alert_desc, services_lrm_status_str(op->op_status));
     }
 
     rc = exec_alert_list(lrmd, alert_list, crm_alert_resource, NULL, params);
     lrmd_key_value_freeall(params);
     return rc;
 }
diff --git a/lib/lrmd/lrmd_client.c b/lib/lrmd/lrmd_client.c
index 63d2b4b92a..711afb094f 100644
--- a/lib/lrmd/lrmd_client.c
+++ b/lib/lrmd/lrmd_client.c
@@ -1,1918 +1,1918 @@
 /*
  * Copyright (c) 2012 David Vossel <davidvossel@gmail.com>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  *
  */
 
 #include <crm_internal.h>
 
 #include <unistd.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdarg.h>
 #include <string.h>
 #include <ctype.h>
 
 #include <sys/types.h>
 #include <sys/wait.h>
 
 #include <glib.h>
 #include <dirent.h>
 
 #include <crm/crm.h>
 #include <crm/lrmd.h>
 #include <crm/services.h>
 #include <crm/common/mainloop.h>
 #include <crm/common/ipcs.h>
 #include <crm/msg_xml.h>
 
 #include <crm/stonith-ng.h>
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 #  undef KEYFILE
 #  include <gnutls/gnutls.h>
 #endif
 
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <netinet/ip.h>
 #include <arpa/inet.h>
 #include <netdb.h>
 
 #define MAX_TLS_RECV_WAIT 10000
 
 CRM_TRACE_INIT_DATA(lrmd);
 
 static int lrmd_api_disconnect(lrmd_t * lrmd);
 static int lrmd_api_is_connected(lrmd_t * lrmd);
 
 /* IPC proxy functions */
 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
 static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 #  define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000    /* 5 seconds */
 gnutls_psk_client_credentials_t psk_cred_s;
 int lrmd_tls_set_key(gnutls_datum_t * key);
 static void lrmd_tls_disconnect(lrmd_t * lrmd);
 static int global_remote_msg_id = 0;
 int lrmd_tls_send_msg(crm_remote_t * session, xmlNode * msg, uint32_t id, const char *msg_type);
 static void lrmd_tls_connection_destroy(gpointer userdata);
 #endif
 
 typedef struct lrmd_private_s {
     enum client_type type;
     char *token;
     mainloop_io_t *source;
 
     /* IPC parameters */
     crm_ipc_t *ipc;
 
     crm_remote_t *remote;
 
     /* Extra TLS parameters */
     char *remote_nodename;
 #ifdef HAVE_GNUTLS_GNUTLS_H
     char *server;
     int port;
     gnutls_psk_client_credentials_t psk_cred_c;
 
     /* while the async connection is occurring, this is the id
      * of the connection timeout timer. */
     int async_timer;
     int sock;
     /* since tls requires a round trip across the network for a
      * request/reply, there are times where we just want to be able
      * to send a request from the client and not wait around (or even care
      * about) what the reply is. */
     int expected_late_replies;
     GList *pending_notify;
     crm_trigger_t *process_notify;
 #endif
 
     lrmd_event_callback callback;
 
     /* Internal IPC proxy msg passing for remote guests */
     void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
     void *proxy_callback_userdata;
     char *peer_version;
 } lrmd_private_t;
 
 static lrmd_list_t *
 lrmd_list_add(lrmd_list_t * head, const char *value)
 {
     lrmd_list_t *p, *end;
 
     p = calloc(1, sizeof(lrmd_list_t));
     p->val = strdup(value);
 
     end = head;
     while (end && end->next) {
         end = end->next;
     }
 
     if (end) {
         end->next = p;
     } else {
         head = p;
     }
 
     return head;
 }
 
 void
 lrmd_list_freeall(lrmd_list_t * head)
 {
     lrmd_list_t *p;
 
     while (head) {
         char *val = (char *)head->val;
 
         p = head->next;
         free(val);
         free(head);
         head = p;
     }
 }
 
 lrmd_key_value_t *
 lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
 {
     lrmd_key_value_t *p, *end;
 
     p = calloc(1, sizeof(lrmd_key_value_t));
     p->key = strdup(key);
     p->value = strdup(value);
 
     end = head;
     while (end && end->next) {
         end = end->next;
     }
 
     if (end) {
         end->next = p;
     } else {
         head = p;
     }
 
     return head;
 }
 
 void
 lrmd_key_value_freeall(lrmd_key_value_t * head)
 {
     lrmd_key_value_t *p;
 
     while (head) {
         p = head->next;
         free(head->key);
         free(head->value);
         free(head);
         head = p;
     }
 }
 
 lrmd_event_data_t *
 lrmd_copy_event(lrmd_event_data_t * event)
 {
     lrmd_event_data_t *copy = NULL;
 
     copy = calloc(1, sizeof(lrmd_event_data_t));
 
     /* This will get all the int values.
      * we just have to be careful not to leave any
      * dangling pointers to strings. */
     memcpy(copy, event, sizeof(lrmd_event_data_t));
 
     copy->rsc_id = event->rsc_id ? strdup(event->rsc_id) : NULL;
     copy->op_type = event->op_type ? strdup(event->op_type) : NULL;
     copy->user_data = event->user_data ? strdup(event->user_data) : NULL;
     copy->output = event->output ? strdup(event->output) : NULL;
     copy->exit_reason = event->exit_reason ? strdup(event->exit_reason) : NULL;
     copy->remote_nodename = event->remote_nodename ? strdup(event->remote_nodename) : NULL;
     copy->params = crm_str_table_dup(event->params);
 
     return copy;
 }
 
 void
 lrmd_free_event(lrmd_event_data_t * event)
 {
     if (!event) {
         return;
     }
 
     /* free gives me grief if i try to cast */
     free((char *)event->rsc_id);
     free((char *)event->op_type);
     free((char *)event->user_data);
     free((char *)event->output);
     free((char *)event->exit_reason);
     free((char *)event->remote_nodename);
     if (event->params) {
         g_hash_table_destroy(event->params);
     }
     free(event);
 }
 
 static int
 lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
 {
     const char *type;
     const char *proxy_session = crm_element_value(msg, F_LRMD_IPC_SESSION);
     lrmd_private_t *native = lrmd->lrmd_private;
     lrmd_event_data_t event = { 0, };
 
     if (proxy_session != NULL) {
         /* this is proxy business */
         lrmd_internal_proxy_dispatch(lrmd, msg);
         return 1;
     } else if (!native->callback) {
         /* no callback set */
         crm_trace("notify event received but client has not set callback");
         return 1;
     }
 
     event.remote_nodename = native->remote_nodename;
     type = crm_element_value(msg, F_LRMD_OPERATION);
     crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
     event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);
 
     if (crm_str_eq(type, LRMD_OP_RSC_REG, TRUE)) {
         event.type = lrmd_event_register;
     } else if (crm_str_eq(type, LRMD_OP_RSC_UNREG, TRUE)) {
         event.type = lrmd_event_unregister;
     } else if (crm_str_eq(type, LRMD_OP_RSC_EXEC, TRUE)) {
         crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
-        crm_element_value_int(msg, F_LRMD_RSC_INTERVAL, &event.interval);
+        crm_element_value_ms(msg, F_LRMD_RSC_INTERVAL, &event.interval_ms);
         crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
         crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
         crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
         crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);
 
         crm_element_value_int(msg, F_LRMD_RSC_RUN_TIME, (int *)&event.t_run);
         crm_element_value_int(msg, F_LRMD_RSC_RCCHANGE_TIME, (int *)&event.t_rcchange);
         crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
         crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);
 
         event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
         event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
         event.output = crm_element_value(msg, F_LRMD_RSC_OUTPUT);
         event.exit_reason = crm_element_value(msg, F_LRMD_RSC_EXIT_REASON);
         event.type = lrmd_event_exec_complete;
 
         event.params = xml2list(msg);
     } else if (crm_str_eq(type, LRMD_OP_NEW_CLIENT, TRUE)) {
         event.type = lrmd_event_new_client;
     } else if (crm_str_eq(type, LRMD_OP_POKE, TRUE)) {
         event.type = lrmd_event_poke;
     } else {
         return 1;
     }
 
     crm_trace("op %s notify event received", type);
     native->callback(&event);
 
     if (event.params) {
         g_hash_table_destroy(event.params);
     }
     return 1;
 }
 
 static int
 lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
 {
     lrmd_t *lrmd = userdata;
     lrmd_private_t *native = lrmd->lrmd_private;
     xmlNode *msg;
     int rc;
 
     if (!native->callback) {
         /* no callback set */
         return 1;
     }
 
     msg = string2xml(buffer);
     rc = lrmd_dispatch_internal(lrmd, msg);
     free_xml(msg);
     return rc;
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 static void
 lrmd_free_xml(gpointer userdata)
 {
     free_xml((xmlNode *) userdata);
 }
 
 static int
 lrmd_tls_connected(lrmd_t * lrmd)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     if (native->remote->tls_session) {
         return TRUE;
     }
 
     return FALSE;
 }
 
 static int
 lrmd_tls_dispatch(gpointer userdata)
 {
     lrmd_t *lrmd = userdata;
     lrmd_private_t *native = lrmd->lrmd_private;
     xmlNode *xml = NULL;
     int rc = 0;
     int disconnected = 0;
 
     if (lrmd_tls_connected(lrmd) == FALSE) {
         crm_trace("TLS dispatch triggered after disconnect");
         return 0;
     }
 
     crm_trace("TLS dispatch triggered");
 
     /* First check if there are any pending notifies to process that came
      * while we were waiting for replies earlier. */
     if (native->pending_notify) {
         GList *iter = NULL;
 
         crm_trace("Processing pending notifies");
         for (iter = native->pending_notify; iter; iter = iter->next) {
             lrmd_dispatch_internal(lrmd, iter->data);
         }
         g_list_free_full(native->pending_notify, lrmd_free_xml);
         native->pending_notify = NULL;
     }
 
     /* Next read the current buffer and see if there are any messages to handle. */
     rc = crm_remote_ready(native->remote, 0);
     if (rc == 0) {
         /* nothing to read, see if any full messages are already in buffer. */
         xml = crm_remote_parse_buffer(native->remote);
     } else if (rc < 0) {
         disconnected = 1;
     } else {
         crm_remote_recv(native->remote, -1, &disconnected);
         xml = crm_remote_parse_buffer(native->remote);
     }
     while (xml) {
         const char *msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
         if (safe_str_eq(msg_type, "notify")) {
             lrmd_dispatch_internal(lrmd, xml);
         } else if (safe_str_eq(msg_type, "reply")) {
             if (native->expected_late_replies > 0) {
                 native->expected_late_replies--;
             } else {
                 int reply_id = 0;
                 crm_element_value_int(xml, F_LRMD_CALLID, &reply_id);
                 /* if this happens, we want to know about it */
                 crm_err("Got outdated remote LRM reply %d", reply_id);
             }
         }
         free_xml(xml);
         xml = crm_remote_parse_buffer(native->remote);
     }
 
     if (disconnected) {
         crm_info("Lost %s LRM connection while reading data",
                  (native->remote_nodename? native->remote_nodename : "local"));
         lrmd_tls_disconnect(lrmd);
         return 0;
     }
     return 1;
 }
 #endif
 
 /* Not used with mainloop */
 int
 lrmd_poll(lrmd_t * lrmd, int timeout)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     switch (native->type) {
         case CRM_CLIENT_IPC:
             return crm_ipc_ready(native->ipc);
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             if (native->pending_notify) {
                 return 1;
             }
 
             return crm_remote_ready(native->remote, 0);
 #endif
         default:
             crm_err("Unsupported connection type: %d", native->type);
     }
 
     return 0;
 }
 
 /* Not used with mainloop */
 bool
 lrmd_dispatch(lrmd_t * lrmd)
 {
     lrmd_private_t *private = NULL;
 
     CRM_ASSERT(lrmd != NULL);
 
     private = lrmd->lrmd_private;
     switch (private->type) {
         case CRM_CLIENT_IPC:
             while (crm_ipc_ready(private->ipc)) {
                 if (crm_ipc_read(private->ipc) > 0) {
                     const char *msg = crm_ipc_buffer(private->ipc);
 
                     lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
                 }
             }
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             lrmd_tls_dispatch(lrmd);
             break;
 #endif
         default:
             crm_err("Unsupported connection type: %d", private->type);
     }
 
     if (lrmd_api_is_connected(lrmd) == FALSE) {
         crm_err("Connection closed");
         return FALSE;
     }
 
     return TRUE;
 }
 
 static xmlNode *
 lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
                enum lrmd_call_options options)
 {
     xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");
 
     CRM_CHECK(op_msg != NULL, return NULL);
     CRM_CHECK(token != NULL, return NULL);
 
     crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");
     crm_xml_add(op_msg, F_TYPE, T_LRMD);
     crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
     crm_xml_add(op_msg, F_LRMD_OPERATION, op);
     crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);
     crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);
 
     if (data != NULL) {
         add_message_xml(op_msg, F_LRMD_CALLDATA, data);
     }
 
     crm_trace("Created lrmd %s command with call options %.8lx (%d)",
               op, (long)options, options);
     return op_msg;
 }
 
 static void
 lrmd_ipc_connection_destroy(gpointer userdata)
 {
     lrmd_t *lrmd = userdata;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     crm_info("IPC connection destroyed");
 
     /* Prevent these from being cleaned up in lrmd_api_disconnect() */
     native->ipc = NULL;
     native->source = NULL;
 
     if (native->callback) {
         lrmd_event_data_t event = { 0, };
         event.type = lrmd_event_disconnect;
         event.remote_nodename = native->remote_nodename;
         native->callback(&event);
     }
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 static void
 lrmd_tls_connection_destroy(gpointer userdata)
 {
     lrmd_t *lrmd = userdata;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     crm_info("TLS connection destroyed");
 
     if (native->remote->tls_session) {
         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
         gnutls_deinit(*native->remote->tls_session);
         gnutls_free(native->remote->tls_session);
     }
     if (native->psk_cred_c) {
         gnutls_psk_free_client_credentials(native->psk_cred_c);
     }
     if (native->sock) {
         close(native->sock);
     }
     if (native->process_notify) {
         mainloop_destroy_trigger(native->process_notify);
         native->process_notify = NULL;
     }
     if (native->pending_notify) {
         g_list_free_full(native->pending_notify, lrmd_free_xml);
         native->pending_notify = NULL;
     }
 
     free(native->remote->buffer);
     native->remote->buffer = NULL;
     native->source = 0;
     native->sock = 0;
     native->psk_cred_c = NULL;
     native->remote->tls_session = NULL;
     native->sock = 0;
 
     if (native->callback) {
         lrmd_event_data_t event = { 0, };
         event.remote_nodename = native->remote_nodename;
         event.type = lrmd_event_disconnect;
         native->callback(&event);
     }
     return;
 }
 
 int
 lrmd_tls_send_msg(crm_remote_t * session, xmlNode * msg, uint32_t id, const char *msg_type)
 {
     crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id);
     crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type);
     return crm_remote_send(session, msg);
 }
 
 static xmlNode *
 lrmd_tls_recv_reply(lrmd_t * lrmd, int total_timeout, int expected_reply_id, int *disconnected)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
     xmlNode *xml = NULL;
     time_t start = time(NULL);
     const char *msg_type = NULL;
     int reply_id = 0;
     int remaining_timeout = 0;
 
     /* A timeout of 0 here makes no sense.  We have to wait a period of time
      * for the response to come back.  If -1 or 0, default to 10 seconds. */
     if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
         total_timeout = MAX_TLS_RECV_WAIT;
     }
 
     while (!xml) {
 
         xml = crm_remote_parse_buffer(native->remote);
         if (!xml) {
             /* read some more off the tls buffer if we still have time left. */
             if (remaining_timeout) {
                 remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
             } else {
                 remaining_timeout = total_timeout;
             }
             if (remaining_timeout <= 0) {
                 crm_err("Never received the expected reply during the timeout period, disconnecting.");
                 *disconnected = TRUE;
                 return NULL;
             }
 
             crm_remote_recv(native->remote, remaining_timeout, disconnected);
             xml = crm_remote_parse_buffer(native->remote);
             if (!xml) {
                 crm_err("Unable to receive expected reply, disconnecting.");
                 *disconnected = TRUE;
                 return NULL;
             } else if (*disconnected) {
                 return NULL;
             }
         }
 
         CRM_ASSERT(xml != NULL);
 
         crm_element_value_int(xml, F_LRMD_REMOTE_MSG_ID, &reply_id);
         msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
 
         if (!msg_type) {
             crm_err("Empty msg type received while waiting for reply");
             free_xml(xml);
             xml = NULL;
         } else if (safe_str_eq(msg_type, "notify")) {
             /* got a notify while waiting for reply, trigger the notify to be processed later */
             crm_info("queueing notify");
             native->pending_notify = g_list_append(native->pending_notify, xml);
             if (native->process_notify) {
                 crm_info("notify trigger set.");
                 mainloop_set_trigger(native->process_notify);
             }
             xml = NULL;
         } else if (safe_str_neq(msg_type, "reply")) {
             /* msg isn't a reply, make some noise */
             crm_err("Expected a reply, got %s", msg_type);
             free_xml(xml);
             xml = NULL;
         } else if (reply_id != expected_reply_id) {
             if (native->expected_late_replies > 0) {
                 native->expected_late_replies--;
             } else {
                 crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
             }
             free_xml(xml);
             xml = NULL;
         }
     }
 
     if (native->remote->buffer && native->process_notify) {
         mainloop_set_trigger(native->process_notify);
     }
 
     return xml;
 }
 
 static int
 lrmd_tls_send(lrmd_t * lrmd, xmlNode * msg)
 {
     int rc = 0;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     global_remote_msg_id++;
     if (global_remote_msg_id <= 0) {
         global_remote_msg_id = 1;
     }
 
     rc = lrmd_tls_send_msg(native->remote, msg, global_remote_msg_id, "request");
     if (rc <= 0) {
         crm_err("Remote lrmd send failed, disconnecting");
         lrmd_tls_disconnect(lrmd);
         return -ENOTCONN;
     }
     return pcmk_ok;
 }
 
 static int
 lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
 {
     int rc = 0;
     int disconnected = 0;
     xmlNode *xml = NULL;
 
     if (lrmd_tls_connected(lrmd) == FALSE) {
         return -1;
     }
 
     rc = lrmd_tls_send(lrmd, msg);
     if (rc < 0) {
         return rc;
     }
 
     xml = lrmd_tls_recv_reply(lrmd, timeout, global_remote_msg_id, &disconnected);
 
     if (disconnected) {
         crm_err("Remote lrmd server disconnected while waiting for reply with id %d. ",
                 global_remote_msg_id);
         lrmd_tls_disconnect(lrmd);
         rc = -ENOTCONN;
     } else if (!xml) {
         crm_err("Remote lrmd never received reply for request id %d. timeout: %dms ",
                 global_remote_msg_id, timeout);
         rc = -ECOMM;
     }
 
     if (reply) {
         *reply = xml;
     } else {
         free_xml(xml);
     }
 
     return rc;
 }
 #endif
 
 static int
 lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
 {
     int rc = -1;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     switch (native->type) {
         case CRM_CLIENT_IPC:
             rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
             break;
 #endif
         default:
             crm_err("Unsupported connection type: %d", native->type);
     }
 
     return rc;
 }
 
 static int
 lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
 {
     int rc = -1;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     switch (native->type) {
         case CRM_CLIENT_IPC:
             rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             rc = lrmd_tls_send(lrmd, msg);
             if (rc == pcmk_ok) {
                 /* we don't want to wait around for the reply, but
                  * since the request/reply protocol needs to behave the same
                  * as libqb, a reply will eventually come later anyway. */
                 native->expected_late_replies++;
             }
             break;
 #endif
         default:
             crm_err("Unsupported connection type: %d", native->type);
     }
 
     return rc;
 }
 
 static int
 lrmd_api_is_connected(lrmd_t * lrmd)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     switch (native->type) {
         case CRM_CLIENT_IPC:
             return crm_ipc_connected(native->ipc);
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             return lrmd_tls_connected(lrmd);
             break;
 #endif
         default:
             crm_err("Unsupported connection type: %d", native->type);
     }
 
     return 0;
 }
 
 /*!
  * \internal
  * \brief Send a prepared API command to the lrmd server
  *
  * \param[in]  lrmd          Existing connection to the lrmd server
  * \param[in]  op            Name of API command to send
  * \param[in]  data          Command data XML to add to the sent command
  * \param[out] output_data   If expecting a reply, it will be stored here
  * \param[in]  timeout       Timeout in milliseconds (if 0, defaults to 1000);
  *                           will be added to the command XML
  * \param[in]  call_options  Call options to pass to server when sending
  * \param[in]  expect_reply  If TRUE, wait for a reply from the server;
  *                           must be TRUE for IPC (as opposed to TLS) clients
  *
  * \return pcmk_ok on success, -errno on error
  */
 static int
 lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
                   xmlNode **output_data, int timeout,
                   enum lrmd_call_options options, gboolean expect_reply)
 {
     int rc = pcmk_ok;
     lrmd_private_t *native = lrmd->lrmd_private;
     xmlNode *op_msg = NULL;
     xmlNode *op_reply = NULL;
 
     if (!lrmd_api_is_connected(lrmd)) {
         return -ENOTCONN;
     }
 
     if (op == NULL) {
         crm_err("No operation specified");
         return -EINVAL;
     }
 
     CRM_CHECK(native->token != NULL,;
         );
     crm_trace("sending %s op to lrmd", op);
 
     op_msg = lrmd_create_op(native->token, op, data, timeout, options);
 
     if (op_msg == NULL) {
         return -EINVAL;
     }
 
     if (expect_reply) {
         rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
     } else {
         rc = lrmd_send_xml_no_reply(lrmd, op_msg);
         goto done;
     }
 
     if (rc < 0) {
         crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
         rc = -ECOMM;
         goto done;
 
     } else if(op_reply == NULL) {
         rc = -ENOMSG;
         goto done;
     }
 
     rc = pcmk_ok;
     crm_trace("%s op reply received", op);
     if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
         rc = -ENOMSG;
         goto done;
     }
 
     crm_log_xml_trace(op_reply, "Reply");
 
     if (output_data) {
         *output_data = op_reply;
         op_reply = NULL;        /* Prevent subsequent free */
     }
 
   done:
     if (lrmd_api_is_connected(lrmd) == FALSE) {
         crm_err("LRMD disconnected");
     }
 
     free_xml(op_msg);
     free_xml(op_reply);
     return rc;
 }
 
 static int
 lrmd_api_poke_connection(lrmd_t * lrmd)
 {
     int rc;
     lrmd_private_t *native = lrmd->lrmd_private;
     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 
     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
     rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0, native->type == CRM_CLIENT_IPC ? TRUE : FALSE);
     free_xml(data);
 
     return rc < 0 ? rc : pcmk_ok;
 }
 
 int
 remote_proxy_check(lrmd_t * lrmd, GHashTable *hash)
 {
     int rc;
     const char *value;
     lrmd_private_t *native = lrmd->lrmd_private;
     xmlNode *data = create_xml_node(NULL, F_LRMD_OPERATION);
 
     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
 
     value = g_hash_table_lookup(hash, "stonith-watchdog-timeout");
     crm_xml_add(data, F_LRMD_WATCHDOG, value);
 
     rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0, native->type == CRM_CLIENT_IPC ? TRUE : FALSE);
     free_xml(data);
 
     return rc < 0 ? rc : pcmk_ok;
 }
 
 static int
 lrmd_handshake(lrmd_t * lrmd, const char *name)
 {
     int rc = pcmk_ok;
     lrmd_private_t *native = lrmd->lrmd_private;
     xmlNode *reply = NULL;
     xmlNode *hello = create_xml_node(NULL, "lrmd_command");
 
     crm_xml_add(hello, F_TYPE, T_LRMD);
     crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
     crm_xml_add(hello, F_LRMD_CLIENTNAME, name);
     crm_xml_add(hello, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
 
     /* advertise that we are a proxy provider */
     if (native->proxy_callback) {
         crm_xml_add(hello, F_LRMD_IS_IPC_PROVIDER, "true");
     }
 
     rc = lrmd_send_xml(lrmd, hello, -1, &reply);
 
     if (rc < 0) {
         crm_perror(LOG_DEBUG, "Couldn't complete registration with the lrmd API: %d", rc);
         rc = -ECOMM;
     } else if (reply == NULL) {
         crm_err("Did not receive registration reply");
         rc = -EPROTO;
     } else {
         const char *version = crm_element_value(reply, F_LRMD_PROTOCOL_VERSION);
         const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
         const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);
 
         crm_element_value_int(reply, F_LRMD_RC, &rc);
 
         if (rc == -EPROTO) {
             crm_err("LRMD protocol mismatch client version %s, server version %s",
                 LRMD_PROTOCOL_VERSION, version);
             crm_log_xml_err(reply, "Protocol Error");
 
         } else if (safe_str_neq(msg_type, CRM_OP_REGISTER)) {
             crm_err("Invalid registration message: %s", msg_type);
             crm_log_xml_err(reply, "Bad reply");
             rc = -EPROTO;
         } else if (tmp_ticket == NULL) {
             crm_err("No registration token provided");
             crm_log_xml_err(reply, "Bad reply");
             rc = -EPROTO;
         } else {
             crm_trace("Obtained registration token: %s", tmp_ticket);
             native->token = strdup(tmp_ticket);
             native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
             rc = pcmk_ok;
         }
     }
 
     free_xml(reply);
     free_xml(hello);
 
     if (rc != pcmk_ok) {
         lrmd_api_disconnect(lrmd);
     }
     return rc;
 }
 
 static int
 lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
 {
     int rc = pcmk_ok;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     static struct ipc_client_callbacks lrmd_callbacks = {
         .dispatch = lrmd_ipc_dispatch,
         .destroy = lrmd_ipc_connection_destroy
     };
 
     crm_info("Connecting to lrmd");
 
     if (fd) {
         /* No mainloop */
         native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
         if (native->ipc && crm_ipc_connect(native->ipc)) {
             *fd = crm_ipc_get_fd(native->ipc);
         } else if (native->ipc) {
             crm_perror(LOG_ERR, "Connection to local resource manager failed");
             rc = -ENOTCONN;
         }
     } else {
         native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
         native->ipc = mainloop_get_ipc_client(native->source);
     }
 
     if (native->ipc == NULL) {
         crm_debug("Could not connect to the LRMD API");
         rc = -ENOTCONN;
     }
 
     return rc;
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 static int
 set_key(gnutls_datum_t * key, const char *location)
 {
     FILE *stream;
     int read_len = 256;
     int cur_len = 0;
     int buf_len = read_len;
     static char *key_cache = NULL;
     static size_t key_cache_len = 0;
     static time_t key_cache_updated;
 
     if (location == NULL) {
         return -1;
     }
 
     if (key_cache) {
         time_t now = time(NULL);
 
         if ((now - key_cache_updated) < 60) {
             key->data = gnutls_malloc(key_cache_len + 1);
             key->size = key_cache_len;
             memcpy(key->data, key_cache, key_cache_len);
 
             crm_debug("using cached LRMD key");
             return 0;
         } else {
             key_cache_len = 0;
             key_cache_updated = 0;
             free(key_cache);
             key_cache = NULL;
             crm_debug("clearing lrmd key cache");
         }
     }
 
     stream = fopen(location, "r");
     if (!stream) {
         return -1;
     }
 
     key->data = gnutls_malloc(read_len);
     while (!feof(stream)) {
         int next;
 
         if (cur_len == buf_len) {
             buf_len = cur_len + read_len;
             key->data = gnutls_realloc(key->data, buf_len);
         }
         next = fgetc(stream);
         if (next == EOF && feof(stream)) {
             break;
         }
 
         key->data[cur_len] = next;
         cur_len++;
     }
     fclose(stream);
 
     key->size = cur_len;
     if (!cur_len) {
         gnutls_free(key->data);
         key->data = 0;
         return -1;
     }
 
     if (!key_cache) {
         key_cache = calloc(1, key->size + 1);
         memcpy(key_cache, key->data, key->size);
 
         key_cache_len = key->size;
         key_cache_updated = time(NULL);
     }
 
     return 0;
 }
 
 int
 lrmd_tls_set_key(gnutls_datum_t * key)
 {
     const char *specific_location = getenv("PCMK_authkey_location");
 
     if (set_key(key, specific_location) == 0) {
         crm_debug("Using custom authkey location %s", specific_location);
         return pcmk_ok;
 
     } else if (specific_location) {
         crm_err("No valid lrmd remote key found at %s, trying default location", specific_location);
     }
 
     if ((set_key(key, DEFAULT_REMOTE_KEY_LOCATION) != 0)
         && (set_key(key, ALT_REMOTE_KEY_LOCATION) != 0)) {
         crm_err("No valid lrmd remote key found at %s", DEFAULT_REMOTE_KEY_LOCATION);
         return -ENOKEY;
     }
 
     return pcmk_ok;
 }
 
 static void
 lrmd_gnutls_global_init(void)
 {
     static int gnutls_init = 0;
 
     if (!gnutls_init) {
         crm_gnutls_global_init();
     }
     gnutls_init = 1;
 }
 #endif
 
 static void
 report_async_connection_result(lrmd_t * lrmd, int rc)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     if (native->callback) {
         lrmd_event_data_t event = { 0, };
         event.type = lrmd_event_connect;
         event.remote_nodename = native->remote_nodename;
         event.connection_rc = rc;
         native->callback(&event);
     }
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 static void
 lrmd_tcp_connect_cb(void *userdata, int sock)
 {
     lrmd_t *lrmd = userdata;
     lrmd_private_t *native = lrmd->lrmd_private;
     char *name;
     static struct mainloop_fd_callbacks lrmd_tls_callbacks = {
         .dispatch = lrmd_tls_dispatch,
         .destroy = lrmd_tls_connection_destroy,
     };
     int rc = sock;
     gnutls_datum_t psk_key = { NULL, 0 };
 
     native->async_timer = 0;
 
     if (rc < 0) {
         lrmd_tls_connection_destroy(lrmd);
         crm_info("Could not connect to remote LRMD at %s:%d",
                  native->server, native->port);
         report_async_connection_result(lrmd, rc);
         return;
     }
 
     /* The TCP connection was successful, so establish the TLS connection.
      * @TODO make this async to avoid blocking code in client
      */
 
     native->sock = sock;
 
     rc = lrmd_tls_set_key(&psk_key);
     if (rc != 0) {
         crm_warn("Could not set key for remote LRMD at %s:%d " CRM_XS " rc=%d",
                  native->server, native->port, rc);
         lrmd_tls_connection_destroy(lrmd);
         report_async_connection_result(lrmd, rc);
         return;
     }
 
     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
     gnutls_free(psk_key.data);
 
     native->remote->tls_session = create_psk_tls_session(sock, GNUTLS_CLIENT, native->psk_cred_c);
 
     if (crm_initiate_client_tls_handshake(native->remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT) != 0) {
         crm_warn("Disconnecting after TLS handshake with remote LRMD %s:%d failed",
                  native->server, native->port);
         gnutls_deinit(*native->remote->tls_session);
         gnutls_free(native->remote->tls_session);
         native->remote->tls_session = NULL;
         lrmd_tls_connection_destroy(lrmd);
         report_async_connection_result(lrmd, -EKEYREJECTED);
         return;
     }
 
     crm_info("TLS connection to remote LRMD %s:%d succeeded",
              native->server, native->port);
 
     name = crm_strdup_printf("remote-lrmd-%s:%d", native->server, native->port);
 
     native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_tls_dispatch, lrmd);
     native->source =
         mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd, &lrmd_tls_callbacks);
 
     rc = lrmd_handshake(lrmd, name);
     free(name);
 
     report_async_connection_result(lrmd, rc);
     return;
 }
 
 static int
 lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
 {
     int sock = 0;
     int timer_id = 0;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     lrmd_gnutls_global_init();
     sock = crm_remote_tcp_connect_async(native->server, native->port, timeout,
                                         &timer_id, lrmd, lrmd_tcp_connect_cb);
     if (sock < 0) {
         return sock;
     }
     native->sock = sock;
     native->async_timer = timer_id;
     return pcmk_ok;
 }
 
 static int
 lrmd_tls_connect(lrmd_t * lrmd, int *fd)
 {
     static struct mainloop_fd_callbacks lrmd_tls_callbacks = {
         .dispatch = lrmd_tls_dispatch,
         .destroy = lrmd_tls_connection_destroy,
     };
     int rc;
 
     lrmd_private_t *native = lrmd->lrmd_private;
     int sock;
     gnutls_datum_t psk_key = { NULL, 0 };
 
     lrmd_gnutls_global_init();
 
     sock = crm_remote_tcp_connect(native->server, native->port);
     if (sock < 0) {
         crm_warn("Could not establish remote lrmd connection to %s", native->server);
         lrmd_tls_connection_destroy(lrmd);
         return -ENOTCONN;
     }
 
     native->sock = sock;
 
     rc = lrmd_tls_set_key(&psk_key);
     if (rc < 0) {
         lrmd_tls_connection_destroy(lrmd);
         return rc;
     }
 
     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
     gnutls_free(psk_key.data);
 
     native->remote->tls_session = create_psk_tls_session(sock, GNUTLS_CLIENT, native->psk_cred_c);
 
     if (crm_initiate_client_tls_handshake(native->remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT) != 0) {
         crm_err("Session creation for %s:%d failed", native->server, native->port);
         gnutls_deinit(*native->remote->tls_session);
         gnutls_free(native->remote->tls_session);
         native->remote->tls_session = NULL;
         lrmd_tls_connection_destroy(lrmd);
         return -EKEYREJECTED;
     }
 
     crm_info("Remote lrmd client TLS connection established with server %s:%d", native->server,
              native->port);
 
     if (fd) {
         *fd = sock;
     } else {
         char *name = crm_strdup_printf("remote-lrmd-%s:%d",
                                        native->server, native->port);
 
         native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_tls_dispatch, lrmd);
         native->source =
             mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd, &lrmd_tls_callbacks);
         free(name);
     }
     return pcmk_ok;
 }
 #endif
 
 static int
 lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
 {
     int rc = -ENOTCONN;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     switch (native->type) {
         case CRM_CLIENT_IPC:
             rc = lrmd_ipc_connect(lrmd, fd);
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             rc = lrmd_tls_connect(lrmd, fd);
             break;
 #endif
         default:
             crm_err("Unsupported connection type: %d", native->type);
     }
 
     if (rc == pcmk_ok) {
         rc = lrmd_handshake(lrmd, name);
     }
 
     return rc;
 }
 
 static int
 lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
 {
     int rc = 0;
     lrmd_private_t *native = lrmd->lrmd_private;
 
     if (!native->callback) {
         crm_err("Async connect not possible, no lrmd client callback set.");
         return -1;
     }
 
     switch (native->type) {
         case CRM_CLIENT_IPC:
             /* fake async connection with ipc.  it should be fast
              * enough that we gain very little from async */
             rc = lrmd_api_connect(lrmd, name, NULL);
             if (!rc) {
                 report_async_connection_result(lrmd, rc);
             }
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             rc = lrmd_tls_connect_async(lrmd, timeout);
             if (rc) {
                 /* connection failed, report rc now */
                 report_async_connection_result(lrmd, rc);
             }
             break;
 #endif
         default:
             crm_err("Unsupported connection type: %d", native->type);
     }
 
     return rc;
 }
 
 static void
 lrmd_ipc_disconnect(lrmd_t * lrmd)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     if (native->source != NULL) {
         /* Attached to mainloop */
         mainloop_del_ipc_client(native->source);
         native->source = NULL;
         native->ipc = NULL;
 
     } else if (native->ipc) {
         /* Not attached to mainloop */
         crm_ipc_t *ipc = native->ipc;
 
         native->ipc = NULL;
         crm_ipc_close(ipc);
         crm_ipc_destroy(ipc);
     }
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 static void
 lrmd_tls_disconnect(lrmd_t * lrmd)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     if (native->remote->tls_session) {
         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
         gnutls_deinit(*native->remote->tls_session);
         gnutls_free(native->remote->tls_session);
         native->remote->tls_session = 0;
     }
 
     if (native->async_timer) {
         g_source_remove(native->async_timer);
         native->async_timer = 0;
     }
 
     if (native->source != NULL) {
         /* Attached to mainloop */
         mainloop_del_ipc_client(native->source);
         native->source = NULL;
 
     } else if (native->sock) {
         close(native->sock);
         native->sock = 0;
     }
 
     if (native->pending_notify) {
         g_list_free_full(native->pending_notify, lrmd_free_xml);
         native->pending_notify = NULL;
     }
 }
 #endif
 
 static int
 lrmd_api_disconnect(lrmd_t * lrmd)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     crm_info("Disconnecting %s LRM connection to %s",
              crm_client_type_text(native->type),
              (native->remote_nodename? native->remote_nodename : "local"));
     switch (native->type) {
         case CRM_CLIENT_IPC:
             lrmd_ipc_disconnect(lrmd);
             break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
         case CRM_CLIENT_TLS:
             lrmd_tls_disconnect(lrmd);
             break;
 #endif
         default:
             crm_err("Unsupported connection type: %d", native->type);
     }
 
     free(native->token);
     native->token = NULL;
 
     free(native->peer_version);
     native->peer_version = NULL;
     return 0;
 }
 
 static int
 lrmd_api_register_rsc(lrmd_t * lrmd,
                       const char *rsc_id,
                       const char *class,
                       const char *provider, const char *type, enum lrmd_call_options options)
 {
     int rc = pcmk_ok;
     xmlNode *data = NULL;
 
     if (!class || !type || !rsc_id) {
         return -EINVAL;
     }
     if (crm_provider_required(class) && !provider) {
         return -EINVAL;
     }
 
     data = create_xml_node(NULL, F_LRMD_RSC);
 
     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     crm_xml_add(data, F_LRMD_CLASS, class);
     crm_xml_add(data, F_LRMD_PROVIDER, provider);
     crm_xml_add(data, F_LRMD_TYPE, type);
     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
     free_xml(data);
 
     return rc;
 }
 
 static int
 lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
 {
     int rc = pcmk_ok;
     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 
     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
     free_xml(data);
 
     return rc;
 }
 
 lrmd_rsc_info_t *
 lrmd_new_rsc_info(const char *rsc_id, const char *standard,
                   const char *provider, const char *type)
 {
     lrmd_rsc_info_t *rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
 
     CRM_ASSERT(rsc_info);
     if (rsc_id) {
         rsc_info->id = strdup(rsc_id);
         CRM_ASSERT(rsc_info->id);
     }
     if (standard) {
         rsc_info->standard = strdup(standard);
         CRM_ASSERT(rsc_info->standard);
     }
     if (provider) {
         rsc_info->provider = strdup(provider);
         CRM_ASSERT(rsc_info->provider);
     }
     if (type) {
         rsc_info->type = strdup(type);
         CRM_ASSERT(rsc_info->type);
     }
     return rsc_info;
 }
 
 lrmd_rsc_info_t *
 lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
 {
     return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
                              rsc_info->provider, rsc_info->type);
 }
 
 void
 lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
 {
     if (!rsc_info) {
         return;
     }
     free(rsc_info->id);
     free(rsc_info->type);
     free(rsc_info->standard);
     free(rsc_info->provider);
     free(rsc_info);
 }
 
 static lrmd_rsc_info_t *
 lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
 {
     lrmd_rsc_info_t *rsc_info = NULL;
     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
     xmlNode *output = NULL;
     const char *class = NULL;
     const char *provider = NULL;
     const char *type = NULL;
 
     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
     free_xml(data);
 
     if (!output) {
         return NULL;
     }
 
     class = crm_element_value(output, F_LRMD_CLASS);
     provider = crm_element_value(output, F_LRMD_PROVIDER);
     type = crm_element_value(output, F_LRMD_TYPE);
 
     if (!class || !type) {
         free_xml(output);
         return NULL;
     } else if (crm_provider_required(class) && !provider) {
         free_xml(output);
         return NULL;
     }
 
     rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
     free_xml(output);
     return rsc_info;
 }
 
 static void
 lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     native->callback = callback;
 }
 
 void
 lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     native->proxy_callback = callback;
     native->proxy_callback_userdata = userdata;
 }
 
 void
 lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
 {
     lrmd_private_t *native = lrmd->lrmd_private;
 
     if (native->proxy_callback) {
         crm_log_xml_trace(msg, "PROXY_INBOUND");
         native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
     }
 }
 
 int
 lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
 {
     if (lrmd == NULL) {
         return -ENOTCONN;
     }
     crm_xml_add(msg, F_LRMD_OPERATION, CRM_OP_IPC_FWD);
 
     crm_log_xml_trace(msg, "PROXY_OUTBOUND");
     return lrmd_send_xml_no_reply(lrmd, msg);
 }
 
 static int
 stonith_get_metadata(const char *provider, const char *type, char **output)
 {
     int rc = pcmk_ok;
     stonith_t *stonith_api = stonith_api_new();
 
     if(stonith_api) {
         stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type, provider, output, 0);
         stonith_api->cmds->free(stonith_api);
     }
     if (*output == NULL) {
         rc = -EIO;
     }
     return rc;
 }
 
 static int
 lrmd_api_get_metadata(lrmd_t * lrmd,
                       const char *class,
                       const char *provider,
                       const char *type, char **output, enum lrmd_call_options options)
 {
     svc_action_t *action;
 
     if (!class || !type) {
         return -EINVAL;
     }
 
     if (safe_str_eq(class, PCMK_RESOURCE_CLASS_STONITH)) {
         return stonith_get_metadata(provider, type, output);
     }
 
     action = resources_action_create(type, class, provider, type,
                                      "meta-data", 0,
                                      CRMD_METADATA_CALL_TIMEOUT, NULL, 0);
     if (action == NULL) {
         crm_err("Unable to retrieve meta-data for %s:%s:%s", class, provider, type);
         services_action_free(action);
         return -EINVAL;
     }
 
     if (!(services_action_sync(action))) {
         crm_err("Failed to retrieve meta-data for %s:%s:%s", class, provider, type);
         services_action_free(action);
         return -EIO;
     }
 
     if (!action->stdout_data) {
         crm_err("Failed to receive meta-data for %s:%s:%s", class, provider, type);
         services_action_free(action);
         return -EIO;
     }
 
     *output = strdup(action->stdout_data);
     services_action_free(action);
 
     return pcmk_ok;
 }
 
 static int
 lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action,
               const char *userdata, guint interval_ms,
               int timeout,      /* ms */
               int start_delay,  /* ms */
               enum lrmd_call_options options, lrmd_key_value_t * params)
 {
     int rc = pcmk_ok;
     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
     lrmd_key_value_t *tmp = NULL;
 
     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
     crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
     crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);
 
     for (tmp = params; tmp; tmp = tmp->next) {
         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
     }
 
     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
     free_xml(data);
 
     lrmd_key_value_freeall(params);
     return rc;
 }
 
 /* timeout is in ms */
 static int
 lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
                     int timeout, lrmd_key_value_t *params)
 {
     int rc = pcmk_ok;
     xmlNode *data = create_xml_node(NULL, F_LRMD_ALERT);
     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
     lrmd_key_value_t *tmp = NULL;
 
     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
     crm_xml_add(data, F_LRMD_ALERT_ID, alert_id);
     crm_xml_add(data, F_LRMD_ALERT_PATH, alert_path);
     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
 
     for (tmp = params; tmp; tmp = tmp->next) {
         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
     }
 
     rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
                            lrmd_opt_notify_orig_only, TRUE);
     free_xml(data);
 
     lrmd_key_value_freeall(params);
     return rc;
 }
 
 static int
 lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action,
                 guint interval_ms)
 {
     int rc = pcmk_ok;
     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 
     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
     free_xml(data);
     return rc;
 }
 
 static int
 list_stonith_agents(lrmd_list_t ** resources)
 {
     int rc = 0;
     stonith_t *stonith_api = stonith_api_new();
     stonith_key_value_t *stonith_resources = NULL;
     stonith_key_value_t *dIter = NULL;
 
     if(stonith_api) {
         stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL, &stonith_resources, 0);
         stonith_api->cmds->free(stonith_api);
     }
 
     for (dIter = stonith_resources; dIter; dIter = dIter->next) {
         rc++;
         if (resources) {
             *resources = lrmd_list_add(*resources, dIter->value);
         }
     }
 
     stonith_key_value_freeall(stonith_resources, 1, 0);
     return rc;
 }
 
 static int
 lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
                      const char *provider)
 {
     int rc = 0;
 
     if (safe_str_eq(class, PCMK_RESOURCE_CLASS_STONITH)) {
         rc += list_stonith_agents(resources);
 
     } else {
         GListPtr gIter = NULL;
         GList *agents = resources_list_agents(class, provider);
 
         for (gIter = agents; gIter != NULL; gIter = gIter->next) {
             *resources = lrmd_list_add(*resources, (const char *)gIter->data);
             rc++;
         }
         g_list_free_full(agents, free);
 
         if (!class) {
             rc += list_stonith_agents(resources);
         }
     }
 
     if (rc == 0) {
         crm_notice("No agents found for class %s", class);
         rc = -EPROTONOSUPPORT;
     }
     return rc;
 }
 
 static int
 does_provider_have_agent(const char *agent, const char *provider, const char *class)
 {
     int found = 0;
     GList *agents = NULL;
     GListPtr gIter2 = NULL;
 
     agents = resources_list_agents(class, provider);
     for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
         if (safe_str_eq(agent, gIter2->data)) {
             found = 1;
         }
     }
     g_list_free_full(agents, free);
 
     return found;
 }
 
 static int
 lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
 {
     int rc = pcmk_ok;
     char *provider = NULL;
     GList *ocf_providers = NULL;
     GListPtr gIter = NULL;
 
     ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
 
     for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
         provider = gIter->data;
         if (!agent || does_provider_have_agent(agent, provider,
                                                PCMK_RESOURCE_CLASS_OCF)) {
             *providers = lrmd_list_add(*providers, (const char *)gIter->data);
             rc++;
         }
     }
 
     g_list_free_full(ocf_providers, free);
     return rc;
 }
 
 static int
 lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
 {
     int rc = 0;
     GList *standards = NULL;
     GListPtr gIter = NULL;
 
     standards = resources_list_standards();
 
     for (gIter = standards; gIter != NULL; gIter = gIter->next) {
         *supported = lrmd_list_add(*supported, (const char *)gIter->data);
         rc++;
     }
 
     if (list_stonith_agents(NULL) > 0) {
         *supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
         rc++;
     }
 
     g_list_free_full(standards, free);
     return rc;
 }
 
 lrmd_t *
 lrmd_api_new(void)
 {
     lrmd_t *new_lrmd = NULL;
     lrmd_private_t *pvt = NULL;
 
     new_lrmd = calloc(1, sizeof(lrmd_t));
     pvt = calloc(1, sizeof(lrmd_private_t));
     pvt->remote = calloc(1, sizeof(crm_remote_t));
     new_lrmd->cmds = calloc(1, sizeof(lrmd_api_operations_t));
 
     pvt->type = CRM_CLIENT_IPC;
     new_lrmd->lrmd_private = pvt;
 
     new_lrmd->cmds->connect = lrmd_api_connect;
     new_lrmd->cmds->connect_async = lrmd_api_connect_async;
     new_lrmd->cmds->is_connected = lrmd_api_is_connected;
     new_lrmd->cmds->poke_connection = lrmd_api_poke_connection;
     new_lrmd->cmds->disconnect = lrmd_api_disconnect;
     new_lrmd->cmds->register_rsc = lrmd_api_register_rsc;
     new_lrmd->cmds->unregister_rsc = lrmd_api_unregister_rsc;
     new_lrmd->cmds->get_rsc_info = lrmd_api_get_rsc_info;
     new_lrmd->cmds->set_callback = lrmd_api_set_callback;
     new_lrmd->cmds->get_metadata = lrmd_api_get_metadata;
     new_lrmd->cmds->exec = lrmd_api_exec;
     new_lrmd->cmds->cancel = lrmd_api_cancel;
     new_lrmd->cmds->list_agents = lrmd_api_list_agents;
     new_lrmd->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
     new_lrmd->cmds->list_standards = lrmd_api_list_standards;
     new_lrmd->cmds->exec_alert = lrmd_api_exec_alert;
 
     return new_lrmd;
 }
 
 lrmd_t *
 lrmd_remote_api_new(const char *nodename, const char *server, int port)
 {
 #ifdef HAVE_GNUTLS_GNUTLS_H
     lrmd_t *new_lrmd = lrmd_api_new();
     lrmd_private_t *native = new_lrmd->lrmd_private;
 
     if (!nodename && !server) {
         lrmd_api_delete(new_lrmd);
         return NULL;
     }
 
     native->type = CRM_CLIENT_TLS;
     native->remote_nodename = nodename ? strdup(nodename) : strdup(server);
     native->server = server ? strdup(server) : strdup(nodename);
     native->port = port;
     if (native->port == 0) {
         native->port = crm_default_remote_port();
     }
 
     return new_lrmd;
 #else
     crm_err("GNUTLS is not enabled for this build, remote LRMD client can not be created");
     return NULL;
 #endif
 
 }
 
 void
 lrmd_api_delete(lrmd_t * lrmd)
 {
     if (!lrmd) {
         return;
     }
     lrmd->cmds->disconnect(lrmd);       /* no-op if already disconnected */
     free(lrmd->cmds);
     if (lrmd->lrmd_private) {
         lrmd_private_t *native = lrmd->lrmd_private;
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
         free(native->server);
 #endif
         free(native->remote_nodename);
         free(native->remote);
         free(native->token);
         free(native->peer_version);
     }
 
     free(lrmd->lrmd_private);
     free(lrmd);
 }
diff --git a/lib/services/services.c b/lib/services/services.c
index 10fc76019e..b634165cdc 100644
--- a/lib/services/services.c
+++ b/lib/services/services.c
@@ -1,1320 +1,1320 @@
 /*
  * Copyright (C) 2010-2016 Andrew Beekhof <andrew@beekhof.net>
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #ifndef _GNU_SOURCE
 #  define _GNU_SOURCE
 #endif
 
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <stdio.h>
 
 #include <errno.h>
 #include <unistd.h>
 #include <dirent.h>
 #include <fcntl.h>
 
 #include <crm/crm.h>
 #include <crm/common/mainloop.h>
 #include <crm/services.h>
 #include <crm/msg_xml.h>
 #include "services_private.h"
 
 #if SUPPORT_UPSTART
 #  include <upstart.h>
 #endif
 
 #if SUPPORT_SYSTEMD
 #  include <systemd.h>
 #endif
 
 /* TODO: Develop a rollover strategy */
 
 static int operations = 0;
 static GHashTable *recurring_actions = NULL;
 
 /* ops waiting to run async because of conflicting active
  * pending ops */
 static GList *blocked_ops = NULL;
 
 /* ops currently active (in-flight) */
 static GList *inflight_ops = NULL;
 
 static void handle_blocked_ops(void);
 
 svc_action_t *
 services_action_create(const char *name, const char *action,
                        guint interval_ms, int timeout)
 {
     return resources_action_create(name, PCMK_RESOURCE_CLASS_LSB, NULL, name,
                                    action, interval_ms, timeout, NULL, 0);
 }
 
 /*!
  * \brief Find first service class that can provide a specified agent
  *
  * \param[in] agent  Name of agent to search for
  *
  * \return Service class if found, NULL otherwise
  *
  * \note The priority is LSB, then systemd, then upstart. It would be preferable
  *       to put systemd first, but LSB merely requires a file existence check,
  *       while systemd requires contacting D-Bus.
  */
 const char *
 resources_find_service_class(const char *agent)
 {
     /* Priority is:
      * - lsb
      * - systemd
      * - upstart
      */
     struct stat st;
     char *path = NULL;
 
 #ifdef LSB_ROOT_DIR
     path = crm_strdup_printf("%s/%s", LSB_ROOT_DIR, agent);
     if (stat(path, &st) == 0) {
         free(path);
         return PCMK_RESOURCE_CLASS_LSB;
     }
     free(path);
 #endif
 
 #if SUPPORT_SYSTEMD
     if (systemd_unit_exists(agent)) {
         return PCMK_RESOURCE_CLASS_SYSTEMD;
     }
 #endif
 
 #if SUPPORT_UPSTART
     if (upstart_job_exists(agent)) {
         return PCMK_RESOURCE_CLASS_UPSTART;
     }
 #endif
     return NULL;
 }
 
 static inline void
 init_recurring_actions(void)
 {
     if (recurring_actions == NULL) {
         recurring_actions = g_hash_table_new_full(g_str_hash, g_str_equal, NULL,
                                                   NULL);
     }
 }
 
 /*!
  * \internal
  * \brief Check whether op is in-flight systemd or upstart op
  *
  * \param[in] op  Operation to check
  *
  * \return TRUE if op is in-flight systemd or upstart op
  */
 static inline gboolean
 inflight_systemd_or_upstart(svc_action_t *op)
 {
     return (safe_str_eq(op->standard, PCMK_RESOURCE_CLASS_SYSTEMD)
             || safe_str_eq(op->standard, PCMK_RESOURCE_CLASS_UPSTART))
             && (g_list_find(inflight_ops, op) != NULL);
 }
 
 /*!
  * \internal
  * \brief Expand "service" alias to an actual resource class
  *
  * \param[in] rsc       Resource name (for logging only)
  * \param[in] standard  Resource class as configured
  * \param[in] agent     Agent name to look for
  *
  * \return Newly allocated string with actual resource class
  *
  * \note The caller is responsible for calling free() on the result.
  */
 static char *
 expand_resource_class(const char *rsc, const char *standard, const char *agent)
 {
     char *expanded_class = NULL;
 
     if (strcasecmp(standard, PCMK_RESOURCE_CLASS_SERVICE) == 0) {
         const char *found_class = resources_find_service_class(agent);
 
         if (found_class) {
             crm_debug("Found %s agent %s for %s", found_class, agent, rsc);
             expanded_class = strdup(found_class);
         } else {
             crm_info("Assuming resource class lsb for agent %s for %s",
                      agent, rsc);
             expanded_class = strdup(PCMK_RESOURCE_CLASS_LSB);
         }
     } else {
         expanded_class = strdup(standard);
     }
     CRM_ASSERT(expanded_class);
     return expanded_class;
 }
 
 /*!
  * \brief Duplicate a file path, inserting a prefix if not absolute
  *
  * \param[in] filename  File path to duplicate
  * \param[in] dirname   If filename is not absolute, prefix to add
  *
  * \return Newly allocated memory with full path
  */
 static char *
 dup_file_path(const char *filename, const char *dirname)
 {
     return (*filename == '/')? strdup(filename)
            : crm_strdup_printf("%s/%s", dirname, filename);
 }
 
 svc_action_t *
 resources_action_create(const char *name, const char *standard,
                         const char *provider, const char *agent,
                         const char *action, guint interval_ms, int timeout,
                         GHashTable *params, enum svc_action_flags flags)
 {
     svc_action_t *op = NULL;
 
     /*
      * Do some up front sanity checks before we go off and
      * build the svc_action_t instance.
      */
 
     if (crm_strlen_zero(name)) {
         crm_err("Cannot create operation without resource name");
         goto return_error;
     }
 
     if (crm_strlen_zero(standard)) {
         crm_err("Cannot create operation for %s without resource class", name);
         goto return_error;
     }
 
     if (crm_provider_required(standard) && crm_strlen_zero(provider)) {
         crm_err("Cannot create OCF operation for %s without provider", name);
         goto return_error;
     }
 
     if (crm_strlen_zero(agent)) {
         crm_err("Cannot create operation for %s without agent name", name);
         goto return_error;
     }
 
     if (crm_strlen_zero(action)) {
         crm_err("Cannot create operation for %s without operation name", name);
         goto return_error;
     }
 
     /*
      * Sanity checks passed, proceed!
      */
 
     op = calloc(1, sizeof(svc_action_t));
     op->opaque = calloc(1, sizeof(svc_action_private_t));
     op->rsc = strdup(name);
-    op->interval = interval_ms;
+    op->interval_ms = interval_ms;
     op->timeout = timeout;
     op->standard = expand_resource_class(name, standard, agent);
     op->agent = strdup(agent);
     op->sequence = ++operations;
     op->flags = flags;
     op->id = generate_op_key(name, action, interval_ms);
 
     if (safe_str_eq(action, "monitor")
         && safe_str_eq(op->standard, PCMK_RESOURCE_CLASS_LSB)) {
         action = "status";
     }
     op->action = strdup(action);
 
     if (crm_provider_required(op->standard)) {
         op->provider = strdup(provider);
         op->params = params;
         params = NULL;
 
         op->opaque->exec = crm_strdup_printf("%s/resource.d/%s/%s",
                                              OCF_ROOT_DIR, provider, agent);
         op->opaque->args[0] = strdup(op->opaque->exec);
         op->opaque->args[1] = strdup(action);
 
     } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_LSB) == 0) {
         op->opaque->exec = dup_file_path(op->agent, LSB_ROOT_DIR);
         op->opaque->args[0] = strdup(op->opaque->exec);
         op->opaque->args[1] = strdup(op->action);
         op->opaque->args[2] = NULL;
 #if SUPPORT_SYSTEMD
     } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_SYSTEMD) == 0) {
         op->opaque->exec = strdup("systemd-dbus");
 #endif
 #if SUPPORT_UPSTART
     } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_UPSTART) == 0) {
         op->opaque->exec = strdup("upstart-dbus");
 #endif
 #if SUPPORT_NAGIOS
     } else if (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_NAGIOS) == 0) {
         int index = 0;
 
         op->opaque->exec = dup_file_path(op->agent, NAGIOS_PLUGIN_DIR);
         op->opaque->args[0] = strdup(op->opaque->exec);
         index = 1;
 
-        if (safe_str_eq(op->action, "monitor") && op->interval == 0) {
+        if (safe_str_eq(op->action, "monitor") && (op->interval_ms == 0)) {
             /* Invoke --version for a nagios probe */
             op->opaque->args[index] = strdup("--version");
             index++;
 
         } else if (params) {
             GHashTableIter iter;
             char *key = NULL;
             char *value = NULL;
             static int args_size = sizeof(op->opaque->args) / sizeof(char *);
 
             g_hash_table_iter_init(&iter, params);
 
             while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value) &&
                    index <= args_size - 3) {
 
                 if (safe_str_eq(key, XML_ATTR_CRM_VERSION) || strstr(key, CRM_META "_")) {
                     continue;
                 }
                 op->opaque->args[index++] = crm_strdup_printf("--%s", key);
                 op->opaque->args[index++] = strdup(value);
             }
         }
         op->opaque->args[index] = NULL;
 #endif
     } else {
         crm_err("Unknown resource standard: %s", op->standard);
         services_action_free(op);
         op = NULL;
     }
 
     if(params) {
         g_hash_table_destroy(params);
     }
     return op;
 
   return_error:
     if(params) {
         g_hash_table_destroy(params);
     }
     services_action_free(op);
 
     return NULL;
 }
 
 svc_action_t *
 services_action_create_generic(const char *exec, const char *args[])
 {
     svc_action_t *op;
     unsigned int cur_arg;
 
     op = calloc(1, sizeof(*op));
     op->opaque = calloc(1, sizeof(svc_action_private_t));
 
     op->opaque->exec = strdup(exec);
     op->opaque->args[0] = strdup(exec);
 
     for (cur_arg = 1; args && args[cur_arg - 1]; cur_arg++) {
         op->opaque->args[cur_arg] = strdup(args[cur_arg - 1]);
 
         if (cur_arg == DIMOF(op->opaque->args) - 1) {
             crm_err("svc_action_t args list not long enough for '%s' execution request.", exec);
             break;
         }
     }
 
     return op;
 }
 
 /*!
  * \brief Create an alert agent action
  *
  * \param[in] id        Alert ID
  * \param[in] exec      Path to alert agent executable
  * \param[in] timeout   Action timeout
  * \param[in] params    Parameters to use with action
  * \param[in] sequence  Action sequence number
  * \param[in] cb_data   Data to pass to callback function
  *
  * \return New action on success, NULL on error
  * \note It is the caller's responsibility to free cb_data.
  *       The caller should not free params explicitly.
  */
 svc_action_t *
 services_alert_create(const char *id, const char *exec, int timeout,
                       GHashTable *params, int sequence, void *cb_data)
 {
     svc_action_t *action = services_action_create_generic(exec, NULL);
 
     CRM_ASSERT(action);
     action->timeout = timeout;
     action->id = strdup(id);
     action->params = params;
     action->sequence = sequence;
     action->cb_data = cb_data;
     return action;
 }
 
 /*!
  * \brief Set the user and group that an action will execute as
  *
  * \param[in,out] action  Action to modify
  * \param[in]     user    Name of user to execute action as
  * \param[in]     group   Name of group to execute action as
  *
  * \return pcmk_ok on success, -errno otherwise
  *
  * \note This will have no effect unless the process executing the action runs
  *       as root, and the action is not a systemd or upstart action.
  *       We could implement this for systemd by adding User= and Group= to
  *       [Service] in the override file, but that seems more likely to cause
  *       problems than be useful.
  */
 int
 services_action_user(svc_action_t *op, const char *user)
 {
     CRM_CHECK((op != NULL) && (user != NULL), return -EINVAL);
     return crm_user_lookup(user, &(op->opaque->uid), &(op->opaque->gid));
 }
 
 static void
 set_alert_env(gpointer key, gpointer value, gpointer user_data)
 {
     int rc;
 
     if (value) {
         rc = setenv(key, value, 1);
     } else {
         rc = unsetenv(key);
     }
 
     if (rc < 0) {
         crm_perror(LOG_ERR, "setenv %s=%s",
                   (char*)key, (value? (char*)value : ""));
     } else {
         crm_trace("setenv %s=%s", (char*)key, (value? (char*)value : ""));
     }
 }
 
 static void
 unset_alert_env(gpointer key, gpointer value, gpointer user_data)
 {
     if (unsetenv(key) < 0) {
         crm_perror(LOG_ERR, "unset %s", (char*)key);
     } else {
         crm_trace("unset %s", (char*)key);
     }
 }
 
 /*!
  * \brief Execute an alert agent action
  *
  * \param[in] action  Action to execute
  * \param[in] cb      Function to call when action completes
  *
  * \return TRUE if the library will free action, FALSE otherwise
  *
  * \note If this function returns FALSE, it is the caller's responsibility to
  *       free the action with services_action_free().
  */
 gboolean
 services_alert_async(svc_action_t *action, void (*cb)(svc_action_t *op))
 {
     gboolean responsible;
 
     action->synchronous = false;
     action->opaque->callback = cb;
     if (action->params) {
         g_hash_table_foreach(action->params, set_alert_env, NULL);
     }
     responsible = services_os_action_execute(action);
     if (action->params) {
         g_hash_table_foreach(action->params, unset_alert_env, NULL);
     }
     return responsible;
 }
 
 #if SUPPORT_DBUS
 /*!
  * \internal
  * \brief Update operation's pending DBus call, unreferencing old one if needed
  *
  * \param[in,out] op       Operation to modify
  * \param[in]     pending  Pending call to set
  */
 void
 services_set_op_pending(svc_action_t *op, DBusPendingCall *pending)
 {
     if (op->opaque->pending && (op->opaque->pending != pending)) {
         if (pending) {
             crm_info("Lost pending %s DBus call (%p)", op->id, op->opaque->pending);
         } else {
             crm_trace("Done with pending %s DBus call (%p)", op->id, op->opaque->pending);
         }
         dbus_pending_call_unref(op->opaque->pending);
     }
     op->opaque->pending = pending;
     if (pending) {
         crm_trace("Updated pending %s DBus call (%p)", op->id, pending);
     } else {
         crm_trace("Cleared pending %s DBus call", op->id);
     }
 }
 #endif
 
 void
 services_action_cleanup(svc_action_t * op)
 {
     if(op->opaque == NULL) {
         return;
     }
 
 #if SUPPORT_DBUS
     if(op->opaque->timerid != 0) {
         crm_trace("Removing timer for call %s to %s", op->action, op->rsc);
         g_source_remove(op->opaque->timerid);
         op->opaque->timerid = 0;
     }
 
     if(op->opaque->pending) {
         crm_trace("Cleaning up pending dbus call %p %s for %s", op->opaque->pending, op->action, op->rsc);
         if(dbus_pending_call_get_completed(op->opaque->pending)) {
             crm_warn("Pending dbus call %s for %s did not complete", op->action, op->rsc);
         }
         dbus_pending_call_cancel(op->opaque->pending);
         dbus_pending_call_unref(op->opaque->pending);
         op->opaque->pending = NULL;
     }
 #endif
 
     if (op->opaque->stderr_gsource) {
         mainloop_del_fd(op->opaque->stderr_gsource);
         op->opaque->stderr_gsource = NULL;
     }
 
     if (op->opaque->stdout_gsource) {
         mainloop_del_fd(op->opaque->stdout_gsource);
         op->opaque->stdout_gsource = NULL;
     }
 }
 
 void
 services_action_free(svc_action_t * op)
 {
     unsigned int i;
 
     if (op == NULL) {
         return;
     }
 
     /* The operation should be removed from all tracking lists by this point.
      * If it's not, we have a bug somewhere, so bail. That may lead to a
      * memory leak, but it's better than a use-after-free segmentation fault.
      */
     CRM_CHECK(g_list_find(inflight_ops, op) == NULL, return);
     CRM_CHECK(g_list_find(blocked_ops, op) == NULL, return);
     CRM_CHECK((recurring_actions == NULL)
               || (g_hash_table_lookup(recurring_actions, op->id) == NULL),
               return);
 
     services_action_cleanup(op);
 
     if (op->opaque->repeat_timer) {
         g_source_remove(op->opaque->repeat_timer);
         op->opaque->repeat_timer = 0;
     }
 
     free(op->id);
     free(op->opaque->exec);
 
     for (i = 0; i < DIMOF(op->opaque->args); i++) {
         free(op->opaque->args[i]);
     }
 
     free(op->opaque);
     free(op->rsc);
     free(op->action);
 
     free(op->standard);
     free(op->agent);
     free(op->provider);
 
     free(op->stdout_data);
     free(op->stderr_data);
 
     if (op->params) {
         g_hash_table_destroy(op->params);
         op->params = NULL;
     }
 
     free(op);
 }
 
 gboolean
 cancel_recurring_action(svc_action_t * op)
 {
     crm_info("Cancelling %s operation %s", op->standard, op->id);
 
     if (recurring_actions) {
         g_hash_table_remove(recurring_actions, op->id);
     }
 
     if (op->opaque->repeat_timer) {
         g_source_remove(op->opaque->repeat_timer);
         op->opaque->repeat_timer = 0;
     }
 
     return TRUE;
 }
 
 /*!
  * \brief Cancel a recurring action
  *
  * \param[in] name         Name of resource that operation is for
  * \param[in] action       Name of operation to cancel
  * \param[in] interval_ms  Interval of operation to cancel
  *
  * \return TRUE if action was successfully cancelled, FALSE otherwise
  */
 gboolean
 services_action_cancel(const char *name, const char *action, guint interval_ms)
 {
     gboolean cancelled = FALSE;
     char *id = generate_op_key(name, action, interval_ms);
     svc_action_t *op = NULL;
 
     /* We can only cancel a recurring action */
     init_recurring_actions();
     op = g_hash_table_lookup(recurring_actions, id);
     if (op == NULL) {
         goto done;
     }
 
     /* Tell operation_finalize() not to reschedule the operation */
     op->cancel = TRUE;
 
     /* Stop tracking it as a recurring operation, and stop its timer */
     cancel_recurring_action(op);
 
     /* If the op has a PID, it's an in-flight child process, so kill it.
      *
      * Whether the kill succeeds or fails, the main loop will send the op to
      * operation_finished() (and thus operation_finalize()) when the process
      * goes away.
      */
     if (op->pid != 0) {
         crm_info("Terminating in-flight op %s (pid %d) early because it was cancelled",
                  id, op->pid);
         cancelled = mainloop_child_kill(op->pid);
         if (cancelled == FALSE) {
             crm_err("Termination of %s (pid %d) failed", id, op->pid);
         }
         goto done;
     }
 
     /* In-flight systemd and upstart ops don't have a pid. The relevant handlers
      * will call operation_finalize() when the operation completes.
      * @TODO: Can we request early termination, maybe using
      * dbus_pending_call_cancel()?
      */
     if (inflight_systemd_or_upstart(op)) {
         crm_info("Will cancel %s op %s when in-flight instance completes",
                  op->standard, op->id);
         cancelled = FALSE;
         goto done;
     }
 
     /* Otherwise, operation is not in-flight, just report as cancelled */
     op->status = PCMK_LRM_OP_CANCELLED;
     if (op->opaque->callback) {
         op->opaque->callback(op);
     }
 
     blocked_ops = g_list_remove(blocked_ops, op);
     services_action_free(op);
     cancelled = TRUE;
 
 done:
     free(id);
     return cancelled;
 }
 
 gboolean
 services_action_kick(const char *name, const char *action, guint interval_ms)
 {
     svc_action_t * op = NULL;
     char *id = generate_op_key(name, action, interval_ms);
 
     init_recurring_actions();
     op = g_hash_table_lookup(recurring_actions, id);
     free(id);
 
     if (op == NULL) {
         return FALSE;
     }
 
 
     if (op->pid || inflight_systemd_or_upstart(op)) {
         return TRUE;
     } else {
         if (op->opaque->repeat_timer) {
             g_source_remove(op->opaque->repeat_timer);
             op->opaque->repeat_timer = 0;
         }
         recurring_action_timer(op);
         return TRUE;
     }
 
 }
 
 /*!
  * \internal
  * \brief Add a new recurring operation, checking for duplicates
  *
  * \param[in] op               Operation to add
  *
  * \return TRUE if duplicate found (and reschedule), FALSE otherwise
  */
 static gboolean
 handle_duplicate_recurring(svc_action_t * op)
 {
     svc_action_t * dup = NULL;
 
     /* check for duplicates */
     dup = g_hash_table_lookup(recurring_actions, op->id);
 
     if (dup && (dup != op)) {
         /* update user data */
         if (op->opaque->callback) {
             dup->opaque->callback = op->opaque->callback;
             dup->cb_data = op->cb_data;
             op->cb_data = NULL;
         }
         /* immediately execute the next interval */
         if (dup->pid != 0) {
             if (op->opaque->repeat_timer) {
                 g_source_remove(op->opaque->repeat_timer);
                 op->opaque->repeat_timer = 0;
             }
             recurring_action_timer(dup);
         }
         /* free the duplicate */
         services_action_free(op);
         return TRUE;
     }
 
     return FALSE;
 }
 
 inline static gboolean
 action_exec_helper(svc_action_t * op)
 {
     /* Whether a/synchronous must be decided (op->synchronous) beforehand. */
     if (op->standard
         && (strcasecmp(op->standard, PCMK_RESOURCE_CLASS_UPSTART) == 0)) {
 #if SUPPORT_UPSTART
         return upstart_job_exec(op);
 #endif
     } else if (op->standard && strcasecmp(op->standard,
                                           PCMK_RESOURCE_CLASS_SYSTEMD) == 0) {
 #if SUPPORT_SYSTEMD
         return systemd_unit_exec(op);
 #endif
     } else {
         return services_os_action_execute(op);
     }
     /* The 'op' has probably been freed if the execution functions return TRUE
        for the asynchronous 'op'. */
     /* Avoid using the 'op' in here. */
 
     return FALSE;
 }
 
 void
 services_add_inflight_op(svc_action_t * op)
 {
     if (op == NULL) {
         return;
     }
 
     CRM_ASSERT(op->synchronous == FALSE);
 
     /* keep track of ops that are in-flight to avoid collisions in the same namespace */
     if (op->rsc) {
         inflight_ops = g_list_append(inflight_ops, op);
     }
 }
 
 /*!
  * \internal
  * \brief Stop tracking an operation that completed
  *
  * \param[in] op  Operation to stop tracking
  */
 void
 services_untrack_op(svc_action_t *op)
 {
     /* Op is no longer in-flight or blocked */
     inflight_ops = g_list_remove(inflight_ops, op);
     blocked_ops = g_list_remove(blocked_ops, op);
 
     /* Op is no longer blocking other ops, so check if any need to run */
     handle_blocked_ops();
 }
 
 gboolean
 services_action_async(svc_action_t * op, void (*action_callback) (svc_action_t *))
 {
     op->synchronous = false;
     if (action_callback) {
         op->opaque->callback = action_callback;
     }
 
-    if (op->interval > 0) {
+    if (op->interval_ms > 0) {
         init_recurring_actions();
         if (handle_duplicate_recurring(op) == TRUE) {
             /* entry rescheduled, dup freed */
             /* exit early */
             return TRUE;
         }
         g_hash_table_replace(recurring_actions, op->id, op);
     }
 
     if (op->rsc && is_op_blocked(op->rsc)) {
         blocked_ops = g_list_append(blocked_ops, op);
         return TRUE;
     }
 
     return action_exec_helper(op);
 }
 
 
 static gboolean processing_blocked_ops = FALSE;
 
 gboolean
 is_op_blocked(const char *rsc)
 {
     GList *gIter = NULL;
     svc_action_t *op = NULL;
 
     for (gIter = inflight_ops; gIter != NULL; gIter = gIter->next) {
         op = gIter->data;
         if (safe_str_eq(op->rsc, rsc)) {
             return TRUE;
         }
     }
 
     return FALSE;
 }
 
 static void
 handle_blocked_ops(void)
 {
     GList *executed_ops = NULL;
     GList *gIter = NULL;
     svc_action_t *op = NULL;
     gboolean res = FALSE;
 
     if (processing_blocked_ops) {
         /* avoid nested calling of this function */
         return;
     }
 
     processing_blocked_ops = TRUE;
 
     /* n^2 operation here, but blocked ops are incredibly rare. this list
      * will be empty 99% of the time. */
     for (gIter = blocked_ops; gIter != NULL; gIter = gIter->next) {
         op = gIter->data;
         if (is_op_blocked(op->rsc)) {
             continue;
         }
         executed_ops = g_list_append(executed_ops, op);
         res = action_exec_helper(op);
         if (res == FALSE) {
             op->status = PCMK_LRM_OP_ERROR;
             /* this can cause this function to be called recursively
              * which is why we have processing_blocked_ops static variable */
             operation_finalize(op);
         }
     }
 
     for (gIter = executed_ops; gIter != NULL; gIter = gIter->next) {
         op = gIter->data;
         blocked_ops = g_list_remove(blocked_ops, op);
     }
     g_list_free(executed_ops);
 
     processing_blocked_ops = FALSE;
 }
 
 #define lsb_metadata_template  \
     "<?xml version='1.0'?>\n"                                           \
     "<!DOCTYPE resource-agent SYSTEM 'ra-api-1.dtd'>\n"                 \
     "<resource-agent name='%s' version='" PCMK_DEFAULT_AGENT_VERSION "'>\n" \
     "  <version>1.0</version>\n"                                        \
     "  <longdesc lang='en'>\n"                                          \
     "%s"                                                                \
     "  </longdesc>\n"                                                   \
     "  <shortdesc lang='en'>%s</shortdesc>\n"                           \
     "  <parameters>\n"                                                  \
     "  </parameters>\n"                                                 \
     "  <actions>\n"                                                     \
     "    <action name='meta-data'    timeout='5' />\n"                  \
     "    <action name='start'        timeout='15' />\n"                 \
     "    <action name='stop'         timeout='15' />\n"                 \
     "    <action name='status'       timeout='15' />\n"                 \
     "    <action name='restart'      timeout='15' />\n"                 \
     "    <action name='force-reload' timeout='15' />\n"                 \
     "    <action name='monitor'      timeout='15' interval='15' />\n"   \
     "  </actions>\n"                                                    \
     "  <special tag='LSB'>\n"                                           \
     "    <Provides>%s</Provides>\n"                                     \
     "    <Required-Start>%s</Required-Start>\n"                         \
     "    <Required-Stop>%s</Required-Stop>\n"                           \
     "    <Should-Start>%s</Should-Start>\n"                             \
     "    <Should-Stop>%s</Should-Stop>\n"                               \
     "    <Default-Start>%s</Default-Start>\n"                           \
     "    <Default-Stop>%s</Default-Stop>\n"                             \
     "  </special>\n"                                                    \
     "</resource-agent>\n"
 
 /* See "Comment Conventions for Init Scripts" in the LSB core specification at:
  * http://refspecs.linuxfoundation.org/lsb.shtml
  */
 #define LSB_INITSCRIPT_INFOBEGIN_TAG "### BEGIN INIT INFO"
 #define LSB_INITSCRIPT_INFOEND_TAG "### END INIT INFO"
 #define PROVIDES    "# Provides:"
 #define REQ_START   "# Required-Start:"
 #define REQ_STOP    "# Required-Stop:"
 #define SHLD_START  "# Should-Start:"
 #define SHLD_STOP   "# Should-Stop:"
 #define DFLT_START  "# Default-Start:"
 #define DFLT_STOP   "# Default-Stop:"
 #define SHORT_DSCR  "# Short-Description:"
 #define DESCRIPTION "# Description:"
 
 #define lsb_meta_helper_free_value(m)           \
     do {                                        \
         if ((m) != NULL) {                      \
             xmlFree(m);                         \
             (m) = NULL;                         \
         }                                       \
     } while(0)
 
 /*!
  * \internal
  * \brief Grab an LSB header value
  *
  * \param[in]     line    Line read from LSB init script
  * \param[in,out] value   If not set, will be set to XML-safe copy of value
  * \param[in]     prefix  Set value if line starts with this pattern
  *
  * \return TRUE if value was set, FALSE otherwise
  */
 static inline gboolean
 lsb_meta_helper_get_value(const char *line, char **value, const char *prefix)
 {
     if (!*value && crm_starts_with(line, prefix)) {
         *value = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST line+strlen(prefix));
         return TRUE;
     }
     return FALSE;
 }
 
 #define DESC_MAX 2048
 
 static int
 lsb_get_metadata(const char *type, char **output)
 {
     char ra_pathname[PATH_MAX] = { 0, };
     FILE *fp = NULL;
     char buffer[1024] = { 0, };
     char *provides = NULL;
     char *req_start = NULL;
     char *req_stop = NULL;
     char *shld_start = NULL;
     char *shld_stop = NULL;
     char *dflt_start = NULL;
     char *dflt_stop = NULL;
     char *s_dscrpt = NULL;
     char *xml_l_dscrpt = NULL;
     int offset = 0;
     bool in_header = FALSE;
     char description[DESC_MAX] = { 0, };
 
     if (type[0] == '/') {
         snprintf(ra_pathname, sizeof(ra_pathname), "%s", type);
     } else {
         snprintf(ra_pathname, sizeof(ra_pathname), "%s/%s",
                  LSB_ROOT_DIR, type);
     }
 
     crm_trace("Looking into %s", ra_pathname);
     fp = fopen(ra_pathname, "r");
     if (fp == NULL) {
         return -errno;
     }
 
     /* Enter into the LSB-compliant comment block */
     while (fgets(buffer, sizeof(buffer), fp)) {
 
         // Ignore lines up to and including the block delimiter
         if (crm_starts_with(buffer, LSB_INITSCRIPT_INFOBEGIN_TAG)) {
             in_header = TRUE;
             continue;
         }
         if (!in_header) {
             continue;
         }
 
         /* Assume each of the following eight arguments contain one line */
         if (lsb_meta_helper_get_value(buffer, &provides, PROVIDES)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &req_start, REQ_START)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &req_stop, REQ_STOP)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &shld_start, SHLD_START)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &shld_stop, SHLD_STOP)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &dflt_start, DFLT_START)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &dflt_stop, DFLT_STOP)) {
             continue;
         }
         if (lsb_meta_helper_get_value(buffer, &s_dscrpt, SHORT_DSCR)) {
             continue;
         }
 
         /* Long description may cross multiple lines */
         if ((offset == 0) // haven't already found long description
             && crm_starts_with(buffer, DESCRIPTION)) {
             bool processed_line = TRUE;
 
             // Get remainder of description line itself
             offset += snprintf(description, DESC_MAX, "%s",
                                buffer + strlen(DESCRIPTION));
 
             // Read any continuation lines of the description
             buffer[0] = '\0';
             while (fgets(buffer, sizeof(buffer), fp)) {
                 if (crm_starts_with(buffer, "#  ")
                     || crm_starts_with(buffer, "#\t")) {
                     /* '#' followed by a tab or more than one space indicates a
                      * continuation of the long description.
                      */
                     offset += snprintf(description + offset, DESC_MAX - offset,
                                        "%s", buffer + 1);
                 } else {
                     /* This line is not part of the long description,
                      * so continue with normal processing.
                      */
                     processed_line = FALSE;
                     break;
                 }
             }
 
             // Make long description safe to use in XML
             xml_l_dscrpt = (char *)xmlEncodeEntitiesReentrant(NULL, BAD_CAST(description));
 
             if (processed_line) {
                 // We grabbed the line into the long description
                 continue;
             }
         }
 
         // Stop if we leave the header block
         if (crm_starts_with(buffer, LSB_INITSCRIPT_INFOEND_TAG)) {
             break;
         }
         if (buffer[0] != '#') {
             break;
         }
     }
     fclose(fp);
 
     *output = crm_strdup_printf(lsb_metadata_template, type,
                                 (xml_l_dscrpt? xml_l_dscrpt : type),
                                 (s_dscrpt? s_dscrpt : type),
                                 (provides? provides : ""),
                                 (req_start? req_start : ""),
                                 (req_stop? req_stop : ""),
                                 (shld_start? shld_start : ""),
                                 (shld_stop? shld_stop : ""),
                                 (dflt_start? dflt_start : ""),
                                 (dflt_stop? dflt_stop : ""));
 
     lsb_meta_helper_free_value(xml_l_dscrpt);
     lsb_meta_helper_free_value(s_dscrpt);
     lsb_meta_helper_free_value(provides);
     lsb_meta_helper_free_value(req_start);
     lsb_meta_helper_free_value(req_stop);
     lsb_meta_helper_free_value(shld_start);
     lsb_meta_helper_free_value(shld_stop);
     lsb_meta_helper_free_value(dflt_start);
     lsb_meta_helper_free_value(dflt_stop);
 
     crm_trace("Created fake metadata: %llu",
               (unsigned long long) strlen(*output));
     return pcmk_ok;
 }
 
 #if SUPPORT_NAGIOS
 static int
 nagios_get_metadata(const char *type, char **output)
 {
     int rc = pcmk_ok;
     FILE *file_strm = NULL;
     int start = 0, length = 0, read_len = 0;
     char *metadata_file = crm_strdup_printf("%s/%s.xml",
                                             NAGIOS_METADATA_DIR, type);
 
     file_strm = fopen(metadata_file, "r");
     if (file_strm == NULL) {
         crm_err("Metadata file %s does not exist", metadata_file);
         free(metadata_file);
         return -EIO;
     }
 
     /* see how big the file is */
     start = ftell(file_strm);
     fseek(file_strm, 0L, SEEK_END);
     length = ftell(file_strm);
     fseek(file_strm, 0L, start);
 
     CRM_ASSERT(length >= 0);
     CRM_ASSERT(start == ftell(file_strm));
 
     if (length <= 0) {
         crm_info("%s was not valid", metadata_file);
         free(*output);
         *output = NULL;
         rc = -EIO;
 
     } else {
         crm_trace("Reading %d bytes from file", length);
         *output = calloc(1, (length + 1));
         read_len = fread(*output, 1, length, file_strm);
         if (read_len != length) {
             crm_err("Calculated and read bytes differ: %d vs. %d",
                     length, read_len);
             free(*output);
             *output = NULL;
             rc = -EIO;
         }
     }
 
     fclose(file_strm);
     free(metadata_file);
     return rc;
 }
 #endif
 
 static gboolean
 action_get_metadata(svc_action_t *op)
 {
     const char *class = op->standard;
 
     if (op->agent == NULL) {
         crm_err("meta-data requested without specifying agent");
         return FALSE;
     }
 
     if (class == NULL) {
         crm_err("meta-data requested for agent %s without specifying class",
                 op->agent);
         return FALSE;
     }
 
     if (!strcmp(class, PCMK_RESOURCE_CLASS_SERVICE)) {
         class = resources_find_service_class(op->agent);
     }
 
     if (class == NULL) {
         crm_err("meta-data requested for %s, but could not determine class",
                 op->agent);
         return FALSE;
     }
 
     if (safe_str_eq(class, PCMK_RESOURCE_CLASS_LSB)) {
         return (lsb_get_metadata(op->agent, &op->stdout_data) >= 0);
     }
 
 #if SUPPORT_NAGIOS
     if (safe_str_eq(class, PCMK_RESOURCE_CLASS_NAGIOS)) {
         return (nagios_get_metadata(op->agent, &op->stdout_data) >= 0);
     }
 #endif
 
     return action_exec_helper(op);
 }
 
 gboolean
 services_action_sync(svc_action_t * op)
 {
     gboolean rc = TRUE;
 
     if (op == NULL) {
         crm_trace("No operation to execute");
         return FALSE;
     }
 
     op->synchronous = true;
 
     if (safe_str_eq(op->action, "meta-data")) {
         /* Synchronous meta-data operations are handled specially. Since most
          * resource classes don't provide any meta-data, it has to be
          * synthesized from available information about the agent.
          *
          * services_action_async() doesn't treat meta-data actions specially, so
          * it will result in an error for classes that don't support the action.
          */
         rc = action_get_metadata(op);
     } else {
         rc = action_exec_helper(op);
     }
-    crm_trace(" > %s_%s_%d: %s = %d",
-              op->rsc, op->action, op->interval, op->opaque->exec, op->rc);
+    crm_trace(" > " CRM_OP_FMT ": %s = %d",
+              op->rsc, op->action, op->interval_ms, op->opaque->exec, op->rc);
     if (op->stdout_data) {
         crm_trace(" >  stdout: %s", op->stdout_data);
     }
     if (op->stderr_data) {
         crm_trace(" >  stderr: %s", op->stderr_data);
     }
     return rc;
 }
 
 GList *
 get_directory_list(const char *root, gboolean files, gboolean executable)
 {
     return services_os_get_directory_list(root, files, executable);
 }
 
 GList *
 services_list(void)
 {
     return resources_list_agents(PCMK_RESOURCE_CLASS_LSB, NULL);
 }
 
 GList *
 resources_list_standards(void)
 {
     GList *standards = NULL;
     GList *agents = NULL;
 
     standards = g_list_append(standards, strdup(PCMK_RESOURCE_CLASS_OCF));
     standards = g_list_append(standards, strdup(PCMK_RESOURCE_CLASS_LSB));
     standards = g_list_append(standards, strdup(PCMK_RESOURCE_CLASS_SERVICE));
 
 #if SUPPORT_SYSTEMD
     agents = systemd_unit_listall();
     if (agents) {
         standards = g_list_append(standards,
                                   strdup(PCMK_RESOURCE_CLASS_SYSTEMD));
         g_list_free_full(agents, free);
     }
 #endif
 
 #if SUPPORT_UPSTART
     agents = upstart_job_listall();
     if (agents) {
         standards = g_list_append(standards,
                                   strdup(PCMK_RESOURCE_CLASS_UPSTART));
         g_list_free_full(agents, free);
     }
 #endif
 
 #if SUPPORT_NAGIOS
     agents = resources_os_list_nagios_agents();
     if (agents) {
         standards = g_list_append(standards,
                                   strdup(PCMK_RESOURCE_CLASS_NAGIOS));
         g_list_free_full(agents, free);
     }
 #endif
 
     return standards;
 }
 
 GList *
 resources_list_providers(const char *standard)
 {
     if (crm_provider_required(standard)) {
         return resources_os_list_ocf_providers();
     }
 
     return NULL;
 }
 
 GList *
 resources_list_agents(const char *standard, const char *provider)
 {
     if ((standard == NULL)
         || (strcasecmp(standard, PCMK_RESOURCE_CLASS_SERVICE) == 0)) {
 
         GList *tmp1;
         GList *tmp2;
         GList *result = resources_os_list_lsb_agents();
 
         if (standard == NULL) {
             tmp1 = result;
             tmp2 = resources_os_list_ocf_agents(NULL);
             if (tmp2) {
                 result = g_list_concat(tmp1, tmp2);
             }
         }
 #if SUPPORT_SYSTEMD
         tmp1 = result;
         tmp2 = systemd_unit_listall();
         if (tmp2) {
             result = g_list_concat(tmp1, tmp2);
         }
 #endif
 
 #if SUPPORT_UPSTART
         tmp1 = result;
         tmp2 = upstart_job_listall();
         if (tmp2) {
             result = g_list_concat(tmp1, tmp2);
         }
 #endif
 
         return result;
 
     } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_OCF) == 0) {
         return resources_os_list_ocf_agents(provider);
     } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_LSB) == 0) {
         return resources_os_list_lsb_agents();
 #if SUPPORT_SYSTEMD
     } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_SYSTEMD) == 0) {
         return systemd_unit_listall();
 #endif
 #if SUPPORT_UPSTART
     } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_UPSTART) == 0) {
         return upstart_job_listall();
 #endif
 #if SUPPORT_NAGIOS
     } else if (strcasecmp(standard, PCMK_RESOURCE_CLASS_NAGIOS) == 0) {
         return resources_os_list_nagios_agents();
 #endif
     }
 
     return NULL;
 }
diff --git a/lib/services/services_linux.c b/lib/services/services_linux.c
index 169f43b45d..a0b676038a 100644
--- a/lib/services/services_linux.c
+++ b/lib/services/services_linux.c
@@ -1,938 +1,938 @@
 /*
  * Copyright (C) 2010-2016 Andrew Beekhof <andrew@beekhof.net>
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #ifndef _GNU_SOURCE
 #  define _GNU_SOURCE
 #endif
 
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/wait.h>
 #include <errno.h>
 #include <unistd.h>
 #include <dirent.h>
 #include <grp.h>
 #include <string.h>
 #include <sys/time.h>
 #include <sys/resource.h>
 
 #ifdef HAVE_SYS_SIGNALFD_H
 #include <sys/signalfd.h>
 #endif
 
 #include "crm/crm.h"
 #include "crm/common/mainloop.h"
 #include "crm/services.h"
 
 #include "services_private.h"
 
 #if SUPPORT_CIBSECRETS
 #  include "crm/common/cib_secrets.h"
 #endif
 
 static gboolean
 svc_read_output(int fd, svc_action_t * op, bool is_stderr)
 {
     char *data = NULL;
     int rc = 0, len = 0;
     char buf[500];
     static const size_t buf_read_len = sizeof(buf) - 1;
 
 
     if (fd < 0) {
         crm_trace("No fd for %s", op->id);
         return FALSE;
     }
 
     if (is_stderr && op->stderr_data) {
         len = strlen(op->stderr_data);
         data = op->stderr_data;
         crm_trace("Reading %s stderr into offset %d", op->id, len);
 
     } else if (is_stderr == FALSE && op->stdout_data) {
         len = strlen(op->stdout_data);
         data = op->stdout_data;
         crm_trace("Reading %s stdout into offset %d", op->id, len);
 
     } else {
         crm_trace("Reading %s %s into offset %d", op->id, is_stderr?"stderr":"stdout", len);
     }
 
     do {
         rc = read(fd, buf, buf_read_len);
         if (rc > 0) {
             crm_trace("Got %d chars: %.80s", rc, buf);
             buf[rc] = 0;
             data = realloc_safe(data, len + rc + 1);
             len += sprintf(data + len, "%s", buf);
 
         } else if (errno != EINTR) {
             /* error or EOF
              * Cleanup happens in pipe_done()
              */
             rc = FALSE;
             break;
         }
 
     } while (rc == buf_read_len || rc < 0);
 
     if (is_stderr) {
         op->stderr_data = data;
     } else {
         op->stdout_data = data;
     }
 
     return rc;
 }
 
 static int
 dispatch_stdout(gpointer userdata)
 {
     svc_action_t *op = (svc_action_t *) userdata;
 
     return svc_read_output(op->opaque->stdout_fd, op, FALSE);
 }
 
 static int
 dispatch_stderr(gpointer userdata)
 {
     svc_action_t *op = (svc_action_t *) userdata;
 
     return svc_read_output(op->opaque->stderr_fd, op, TRUE);
 }
 
 static void
 pipe_out_done(gpointer user_data)
 {
     svc_action_t *op = (svc_action_t *) user_data;
 
     crm_trace("%p", op);
 
     op->opaque->stdout_gsource = NULL;
     if (op->opaque->stdout_fd > STDOUT_FILENO) {
         close(op->opaque->stdout_fd);
     }
     op->opaque->stdout_fd = -1;
 }
 
 static void
 pipe_err_done(gpointer user_data)
 {
     svc_action_t *op = (svc_action_t *) user_data;
 
     op->opaque->stderr_gsource = NULL;
     if (op->opaque->stderr_fd > STDERR_FILENO) {
         close(op->opaque->stderr_fd);
     }
     op->opaque->stderr_fd = -1;
 }
 
 static struct mainloop_fd_callbacks stdout_callbacks = {
     .dispatch = dispatch_stdout,
     .destroy = pipe_out_done,
 };
 
 static struct mainloop_fd_callbacks stderr_callbacks = {
     .dispatch = dispatch_stderr,
     .destroy = pipe_err_done,
 };
 
 static void
 set_ocf_env(const char *key, const char *value, gpointer user_data)
 {
     if (setenv(key, value, 1) != 0) {
         crm_perror(LOG_ERR, "setenv failed for key:%s and value:%s", key, value);
     }
 }
 
 static void
 set_ocf_env_with_prefix(gpointer key, gpointer value, gpointer user_data)
 {
     char buffer[500];
 
     snprintf(buffer, sizeof(buffer), "OCF_RESKEY_%s", (char *)key);
     set_ocf_env(buffer, value, user_data);
 }
 
 /*!
  * \internal
  * \brief Add environment variables suitable for an action
  *
  * \param[in] op  Action to use
  */
 static void
 add_action_env_vars(const svc_action_t *op)
 {
     if (safe_str_eq(op->standard, PCMK_RESOURCE_CLASS_OCF) == FALSE) {
         return;
     }
 
     if (op->params) {
         g_hash_table_foreach(op->params, set_ocf_env_with_prefix, NULL);
     }
 
     set_ocf_env("OCF_RA_VERSION_MAJOR", "1", NULL);
     set_ocf_env("OCF_RA_VERSION_MINOR", "0", NULL);
     set_ocf_env("OCF_ROOT", OCF_ROOT_DIR, NULL);
     set_ocf_env("OCF_EXIT_REASON_PREFIX", PCMK_OCF_REASON_PREFIX, NULL);
 
     if (op->rsc) {
         set_ocf_env("OCF_RESOURCE_INSTANCE", op->rsc, NULL);
     }
 
     if (op->agent != NULL) {
         set_ocf_env("OCF_RESOURCE_TYPE", op->agent, NULL);
     }
 
     /* Notes: this is not added to specification yet. Sept 10,2004 */
     if (op->provider != NULL) {
         set_ocf_env("OCF_RESOURCE_PROVIDER", op->provider, NULL);
     }
 }
 
 gboolean
 recurring_action_timer(gpointer data)
 {
     svc_action_t *op = data;
 
     crm_debug("Scheduling another invocation of %s", op->id);
 
     /* Clean out the old result */
     free(op->stdout_data);
     op->stdout_data = NULL;
     free(op->stderr_data);
     op->stderr_data = NULL;
     op->opaque->repeat_timer = 0;
 
     services_action_async(op, NULL);
     return FALSE;
 }
 
 /* Returns FALSE if 'op' should be free'd by the caller */
 gboolean
 operation_finalize(svc_action_t * op)
 {
     int recurring = 0;
 
-    if (op->interval) {
+    if (op->interval_ms) {
         if (op->cancel) {
             op->status = PCMK_LRM_OP_CANCELLED;
             cancel_recurring_action(op);
         } else {
             recurring = 1;
-            op->opaque->repeat_timer = g_timeout_add(op->interval,
+            op->opaque->repeat_timer = g_timeout_add(op->interval_ms,
                                                      recurring_action_timer, (void *)op);
         }
     }
 
     if (op->opaque->callback) {
         op->opaque->callback(op);
     }
 
     op->pid = 0;
 
     services_untrack_op(op);
 
     if (!recurring && op->synchronous == FALSE) {
         /*
          * If this is a recurring action, do not free explicitly.
          * It will get freed whenever the action gets cancelled.
          */
         services_action_free(op);
         return TRUE;
     }
 
     services_action_cleanup(op);
     return FALSE;
 }
 
 static void
 operation_finished(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode)
 {
     svc_action_t *op = mainloop_child_userdata(p);
     char *prefix = crm_strdup_printf("%s:%d", op->id, op->pid);
 
     mainloop_clear_child_userdata(p);
     op->status = PCMK_LRM_OP_DONE;
     CRM_ASSERT(op->pid == pid);
 
     crm_trace("%s %p %p", prefix, op->opaque->stderr_gsource, op->opaque->stdout_gsource);
     if (op->opaque->stderr_gsource) {
         /* Make sure we have read everything from the buffer.
          * Depending on the priority mainloop gives the fd, operation_finished
          * could occur before all the reads are done.  Force the read now.*/
         crm_trace("%s dispatching stderr", prefix);
         dispatch_stderr(op);
         crm_trace("%s: %p", op->id, op->stderr_data);
         mainloop_del_fd(op->opaque->stderr_gsource);
         op->opaque->stderr_gsource = NULL;
     }
 
     if (op->opaque->stdout_gsource) {
         /* Make sure we have read everything from the buffer.
          * Depending on the priority mainloop gives the fd, operation_finished
          * could occur before all the reads are done.  Force the read now.*/
         crm_trace("%s dispatching stdout", prefix);
         dispatch_stdout(op);
         crm_trace("%s: %p", op->id, op->stdout_data);
         mainloop_del_fd(op->opaque->stdout_gsource);
         op->opaque->stdout_gsource = NULL;
     }
 
     if (signo) {
         if (mainloop_child_timeout(p)) {
             crm_warn("%s - timed out after %dms", prefix, op->timeout);
             op->status = PCMK_LRM_OP_TIMEOUT;
             op->rc = PCMK_OCF_TIMEOUT;
 
         } else {
             do_crm_log_unlikely((op->cancel) ? LOG_INFO : LOG_WARNING,
                                 "%s - terminated with signal %d", prefix, signo);
             op->status = PCMK_LRM_OP_ERROR;
             op->rc = PCMK_OCF_SIGNAL;
         }
 
     } else {
         op->rc = exitcode;
         crm_debug("%s - exited with rc=%d", prefix, exitcode);
     }
 
     free(prefix);
     prefix = crm_strdup_printf("%s:%d:stderr", op->id, op->pid);
     crm_log_output(LOG_NOTICE, prefix, op->stderr_data);
 
     free(prefix);
     prefix = crm_strdup_printf("%s:%d:stdout", op->id, op->pid);
     crm_log_output(LOG_DEBUG, prefix, op->stdout_data);
 
     free(prefix);
     operation_finalize(op);
 }
 
 /*!
  * \internal
  * \brief Set operation rc and status per errno from stat(), fork() or execvp()
  *
  * \param[in,out] op     Operation to set rc and status for
  * \param[in]     error  Value of errno after system call
  *
  * \return void
  */
 static void
 services_handle_exec_error(svc_action_t * op, int error)
 {
     int rc_not_installed, rc_insufficient_priv, rc_exec_error;
 
     /* Mimic the return codes for each standard as that's what we'll convert back from in get_uniform_rc() */
     if (safe_str_eq(op->standard, PCMK_RESOURCE_CLASS_LSB)
         && safe_str_eq(op->action, "status")) {
 
         rc_not_installed = PCMK_LSB_STATUS_NOT_INSTALLED;
         rc_insufficient_priv = PCMK_LSB_STATUS_INSUFFICIENT_PRIV;
         rc_exec_error = PCMK_LSB_STATUS_UNKNOWN;
 
 #if SUPPORT_NAGIOS
     } else if (safe_str_eq(op->standard, PCMK_RESOURCE_CLASS_NAGIOS)) {
         rc_not_installed = NAGIOS_NOT_INSTALLED;
         rc_insufficient_priv = NAGIOS_INSUFFICIENT_PRIV;
         rc_exec_error = PCMK_OCF_EXEC_ERROR;
 #endif
 
     } else {
         rc_not_installed = PCMK_OCF_NOT_INSTALLED;
         rc_insufficient_priv = PCMK_OCF_INSUFFICIENT_PRIV;
         rc_exec_error = PCMK_OCF_EXEC_ERROR;
     }
 
     switch (error) {   /* see execve(2), stat(2) and fork(2) */
         case ENOENT:   /* No such file or directory */
         case EISDIR:   /* Is a directory */
         case ENOTDIR:  /* Path component is not a directory */
         case EINVAL:   /* Invalid executable format */
         case ENOEXEC:  /* Invalid executable format */
             op->rc = rc_not_installed;
             op->status = PCMK_LRM_OP_NOT_INSTALLED;
             break;
         case EACCES:   /* permission denied (various errors) */
         case EPERM:    /* permission denied (various errors) */
             op->rc = rc_insufficient_priv;
             op->status = PCMK_LRM_OP_ERROR;
             break;
         default:
             op->rc = rc_exec_error;
             op->status = PCMK_LRM_OP_ERROR;
     }
 }
 
 static void
 action_launch_child(svc_action_t *op)
 {
     int lpc;
 
     /* SIGPIPE is ignored (which is different from signal blocking) by the gnutls library.
      * Depending on the libqb version in use, libqb may set SIGPIPE to be ignored as well. 
      * We do not want this to be inherited by the child process. By resetting this the signal
      * to the default behavior, we avoid some potential odd problems that occur during OCF
      * scripts when SIGPIPE is ignored by the environment. */
     signal(SIGPIPE, SIG_DFL);
 
 #if defined(HAVE_SCHED_SETSCHEDULER)
     if (sched_getscheduler(0) != SCHED_OTHER) {
         struct sched_param sp;
 
         memset(&sp, 0, sizeof(sp));
         sp.sched_priority = 0;
 
         if (sched_setscheduler(0, SCHED_OTHER, &sp) == -1) {
             crm_perror(LOG_ERR, "Could not reset scheduling policy to SCHED_OTHER for %s", op->id);
         }
     }
 #endif
     if (setpriority(PRIO_PROCESS, 0, 0) == -1) {
         crm_perror(LOG_ERR, "Could not reset process priority to 0 for %s", op->id);
     }
 
     /* Man: The call setpgrp() is equivalent to setpgid(0,0)
      * _and_ compiles on BSD variants too
      * need to investigate if it works the same too.
      */
     setpgid(0, 0);
 
     // Close all file descriptors except stdin/stdout/stderr
     for (lpc = getdtablesize() - 1; lpc > STDERR_FILENO; lpc--) {
         close(lpc);
     }
 
 #if SUPPORT_CIBSECRETS
     if (replace_secret_params(op->rsc, op->params) < 0) {
         /* replacing secrets failed! */
         if (safe_str_eq(op->action,"stop")) {
             /* don't fail on stop! */
             crm_info("proceeding with the stop operation for %s", op->rsc);
 
         } else {
             crm_err("failed to get secrets for %s, "
                     "considering resource not configured", op->rsc);
             _exit(PCMK_OCF_NOT_CONFIGURED);
         }
     }
 #endif
 
     add_action_env_vars(op);
 
     /* Become the desired user */
     if (op->opaque->uid && (geteuid() == 0)) {
 
         // If requested, set effective group
         if (op->opaque->gid && (setgid(op->opaque->gid) < 0)) {
             crm_perror(LOG_ERR, "Could not set child group to %d", op->opaque->gid);
             _exit(PCMK_OCF_NOT_CONFIGURED);
         }
 
         // Erase supplementary group list
         // (We could do initgroups() if we kept a copy of the username)
         if (setgroups(0, NULL) < 0) {
             crm_perror(LOG_ERR, "Could not set child groups");
             _exit(PCMK_OCF_NOT_CONFIGURED);
         }
 
         // Set effective user
         if (setuid(op->opaque->uid) < 0) {
             crm_perror(LOG_ERR, "setting user to %d", op->opaque->uid);
             _exit(PCMK_OCF_NOT_CONFIGURED);
         }
     }
 
     /* execute the RA */
     execvp(op->opaque->exec, op->opaque->args);
 
     /* Most cases should have been already handled by stat() */
     services_handle_exec_error(op, errno);
 
     _exit(op->rc);
 }
 
 #ifndef HAVE_SYS_SIGNALFD_H
 static int sigchld_pipe[2] = { -1, -1 };
 
 static void
 sigchld_handler()
 {
     if ((sigchld_pipe[1] >= 0) && (write(sigchld_pipe[1], "", 1) == -1)) {
         crm_perror(LOG_TRACE, "Could not poke SIGCHLD self-pipe");
     }
 }
 #endif
 
 static void
 action_synced_wait(svc_action_t * op, sigset_t *mask)
 {
     int status = 0;
     int timeout = op->timeout;
     int sfd = -1;
     time_t start = -1;
     struct pollfd fds[3];
     int wait_rc = 0;
 
 #ifdef HAVE_SYS_SIGNALFD_H
     sfd = signalfd(-1, mask, SFD_NONBLOCK);
     if (sfd < 0) {
         crm_perror(LOG_ERR, "signalfd() failed");
     }
 #else
     sfd = sigchld_pipe[0];
 #endif
 
     fds[0].fd = op->opaque->stdout_fd;
     fds[0].events = POLLIN;
     fds[0].revents = 0;
 
     fds[1].fd = op->opaque->stderr_fd;
     fds[1].events = POLLIN;
     fds[1].revents = 0;
 
     fds[2].fd = sfd;
     fds[2].events = POLLIN;
     fds[2].revents = 0;
 
     crm_trace("Waiting for %d", op->pid);
     start = time(NULL);
     do {
         int poll_rc = poll(fds, 3, timeout);
 
         if (poll_rc > 0) {
             if (fds[0].revents & POLLIN) {
                 svc_read_output(op->opaque->stdout_fd, op, FALSE);
             }
 
             if (fds[1].revents & POLLIN) {
                 svc_read_output(op->opaque->stderr_fd, op, TRUE);
             }
 
             if (fds[2].revents & POLLIN) {
 #ifdef HAVE_SYS_SIGNALFD_H
                 struct signalfd_siginfo fdsi;
                 ssize_t s;
 
                 s = read(sfd, &fdsi, sizeof(struct signalfd_siginfo));
                 if (s != sizeof(struct signalfd_siginfo)) {
                     crm_perror(LOG_ERR, "Read from signal fd %d failed", sfd);
 
                 } else if (fdsi.ssi_signo == SIGCHLD) {
 #else
                 if (1) {
                     /* Clear out the sigchld pipe. */
                     char ch;
                     while (read(sfd, &ch, 1) == 1) /*omit*/;
 #endif
                     wait_rc = waitpid(op->pid, &status, WNOHANG);
 
                     if (wait_rc > 0) {
                         break;
 
                     } else if (wait_rc < 0){
                         if (errno == ECHILD) {
                                 /* Here, don't dare to kill and bail out... */
                                 break;
 
                         } else {
                                 /* ...otherwise pretend process still runs. */
                                 wait_rc = 0;
                         }
                         crm_perror(LOG_ERR, "waitpid() for %d failed", op->pid);
                     }
                 }
             }
 
         } else if (poll_rc == 0) {
             timeout = 0;
             break;
 
         } else if (poll_rc < 0) {
             if (errno != EINTR) {
                 crm_perror(LOG_ERR, "poll() failed");
                 break;
             }
         }
 
         timeout = op->timeout - (time(NULL) - start) * 1000;
 
     } while ((op->timeout < 0 || timeout > 0));
 
     crm_trace("Child done: %d", op->pid);
     if (wait_rc <= 0) {
         op->rc = PCMK_OCF_UNKNOWN_ERROR;
 
         if (op->timeout > 0 && timeout <= 0) {
             op->status = PCMK_LRM_OP_TIMEOUT;
             crm_warn("%s:%d - timed out after %dms", op->id, op->pid, op->timeout);
 
         } else {
             op->status = PCMK_LRM_OP_ERROR;
         }
 
         /* If only child hasn't been successfully waited for, yet.
            This is to limit killing wrong target a bit more. */
         if (wait_rc == 0 && waitpid(op->pid, &status, WNOHANG) == 0) {
             if (kill(op->pid, SIGKILL)) {
                 crm_err("kill(%d, KILL) failed: %d", op->pid, errno);
             }
             /* Safe to skip WNOHANG here as we sent non-ignorable signal. */
             while (waitpid(op->pid, &status, 0) == (pid_t) -1 && errno == EINTR) /*omit*/;
         }
 
     } else if (WIFEXITED(status)) {
         op->status = PCMK_LRM_OP_DONE;
         op->rc = WEXITSTATUS(status);
         crm_info("Managed %s process %d exited with rc=%d", op->id, op->pid, op->rc);
 
     } else if (WIFSIGNALED(status)) {
         int signo = WTERMSIG(status);
 
         op->status = PCMK_LRM_OP_ERROR;
         crm_err("Managed %s process %d exited with signal=%d", op->id, op->pid, signo);
     }
 #ifdef WCOREDUMP
     if (WCOREDUMP(status)) {
         crm_err("Managed %s process %d dumped core", op->id, op->pid);
     }
 #endif
 
     svc_read_output(op->opaque->stdout_fd, op, FALSE);
     svc_read_output(op->opaque->stderr_fd, op, TRUE);
 
     close(op->opaque->stdout_fd);
     close(op->opaque->stderr_fd);
 
 #ifdef HAVE_SYS_SIGNALFD_H
     close(sfd);
 #endif
 }
 
 /* For an asynchronous 'op', returns FALSE if 'op' should be free'd by the caller */
 /* For a synchronous 'op', returns FALSE if 'op' fails */
 gboolean
 services_os_action_execute(svc_action_t * op)
 {
     int stdout_fd[2];
     int stderr_fd[2];
     int rc;
     struct stat st;
     sigset_t *pmask;
 
 #ifdef HAVE_SYS_SIGNALFD_H
     sigset_t mask;
     sigset_t old_mask;
 #define sigchld_cleanup() do {                                                \
     if (sigismember(&old_mask, SIGCHLD) == 0) {                               \
         if (sigprocmask(SIG_UNBLOCK, &mask, NULL) < 0) {                      \
             crm_perror(LOG_ERR, "sigprocmask() failed to unblock sigchld");   \
         }                                                                     \
     }                                                                         \
 } while (0)
 #else
     struct sigaction sa;
     struct sigaction old_sa;
 #define sigchld_cleanup() do {                                                \
     if (sigaction(SIGCHLD, &old_sa, NULL) < 0) {                              \
         crm_perror(LOG_ERR, "sigaction() failed to remove sigchld handler");  \
     }                                                                         \
     close(sigchld_pipe[0]);                                                   \
     close(sigchld_pipe[1]);                                                   \
     sigchld_pipe[0] = sigchld_pipe[1] = -1;                                   \
 } while(0)
 #endif
 
     /* Fail fast */
     if(stat(op->opaque->exec, &st) != 0) {
         rc = errno;
         crm_warn("Cannot execute '%s': %s (%d)", op->opaque->exec, pcmk_strerror(rc), rc);
         services_handle_exec_error(op, rc);
         if (!op->synchronous) {
             return operation_finalize(op);
         }
         return FALSE;
     }
 
     if (pipe(stdout_fd) < 0) {
         rc = errno;
 
         crm_err("pipe(stdout_fd) failed. '%s': %s (%d)", op->opaque->exec, pcmk_strerror(rc), rc);
 
         services_handle_exec_error(op, rc);
         if (!op->synchronous) {
             return operation_finalize(op);
         }
         return FALSE;
     }
 
     if (pipe(stderr_fd) < 0) {
         rc = errno;
 
         close(stdout_fd[0]);
         close(stdout_fd[1]);
 
         crm_err("pipe(stderr_fd) failed. '%s': %s (%d)", op->opaque->exec, pcmk_strerror(rc), rc);
 
         services_handle_exec_error(op, rc);
         if (!op->synchronous) {
             return operation_finalize(op);
         }
         return FALSE;
     }
 
     if (op->synchronous) {
 #ifdef HAVE_SYS_SIGNALFD_H
         sigemptyset(&mask);
         sigaddset(&mask, SIGCHLD);
         sigemptyset(&old_mask);
 
         if (sigprocmask(SIG_BLOCK, &mask, &old_mask) < 0) {
             crm_perror(LOG_ERR, "sigprocmask() failed to block sigchld");
         }
 
         pmask = &mask;
 #else
         if(pipe(sigchld_pipe) == -1) {
             crm_perror(LOG_ERR, "pipe() failed");
         }
 
         rc = crm_set_nonblocking(sigchld_pipe[0]);
         if (rc < 0) {
             crm_warn("Could not set pipe input non-blocking: %s " CRM_XS " rc=%d",
                      pcmk_strerror(rc), rc);
         }
         rc = crm_set_nonblocking(sigchld_pipe[1]);
         if (rc < 0) {
             crm_warn("Could not set pipe output non-blocking: %s " CRM_XS " rc=%d",
                      pcmk_strerror(rc), rc);
         }
 
         sa.sa_handler = sigchld_handler;
         sa.sa_flags = 0;
         sigemptyset(&sa.sa_mask);
         if (sigaction(SIGCHLD, &sa, &old_sa) < 0) {
             crm_perror(LOG_ERR, "sigaction() failed to set sigchld handler");
         }
 
         pmask = NULL;
 #endif
     }
 
     op->pid = fork();
     switch (op->pid) {
         case -1:
             rc = errno;
 
             close(stdout_fd[0]);
             close(stdout_fd[1]);
             close(stderr_fd[0]);
             close(stderr_fd[1]);
 
             crm_err("Could not execute '%s': %s (%d)", op->opaque->exec, pcmk_strerror(rc), rc);
             services_handle_exec_error(op, rc);
             if (!op->synchronous) {
                 return operation_finalize(op);
             }
 
             sigchld_cleanup();
             return FALSE;
 
         case 0:                /* Child */
             close(stdout_fd[0]);
             close(stderr_fd[0]);
             if (STDOUT_FILENO != stdout_fd[1]) {
                 if (dup2(stdout_fd[1], STDOUT_FILENO) != STDOUT_FILENO) {
                     crm_err("dup2() failed (stdout)");
                 }
                 close(stdout_fd[1]);
             }
             if (STDERR_FILENO != stderr_fd[1]) {
                 if (dup2(stderr_fd[1], STDERR_FILENO) != STDERR_FILENO) {
                     crm_err("dup2() failed (stderr)");
                 }
                 close(stderr_fd[1]);
             }
 
             if (op->synchronous) {
                 sigchld_cleanup();
             }
 
             action_launch_child(op);
             CRM_ASSERT(0);  /* action_launch_child is effectively noreturn */
     }
 
     /* Only the parent reaches here */
     close(stdout_fd[1]);
     close(stderr_fd[1]);
 
     op->opaque->stdout_fd = stdout_fd[0];
     rc = crm_set_nonblocking(op->opaque->stdout_fd);
     if (rc < 0) {
         crm_warn("Could not set child output non-blocking: %s "
                  CRM_XS " rc=%d",
                  pcmk_strerror(rc), rc);
     }
 
     op->opaque->stderr_fd = stderr_fd[0];
     rc = crm_set_nonblocking(op->opaque->stderr_fd);
     if (rc < 0) {
         crm_warn("Could not set child error output non-blocking: %s "
                  CRM_XS " rc=%d",
                  pcmk_strerror(rc), rc);
     }
 
     if (op->synchronous) {
         action_synced_wait(op, pmask);
         sigchld_cleanup();
     } else {
 
         crm_trace("Async waiting for %d - %s", op->pid, op->opaque->exec);
         mainloop_child_add_with_flags(op->pid,
                                       op->timeout,
                                       op->id,
                                       op,
                                       (op->flags & SVC_ACTION_LEAVE_GROUP) ? mainloop_leave_pid_group : 0,
                                       operation_finished);
 
 
         op->opaque->stdout_gsource = mainloop_add_fd(op->id,
                                                      G_PRIORITY_LOW,
                                                      op->opaque->stdout_fd, op, &stdout_callbacks);
 
         op->opaque->stderr_gsource = mainloop_add_fd(op->id,
                                                      G_PRIORITY_LOW,
                                                      op->opaque->stderr_fd, op, &stderr_callbacks);
 
         services_add_inflight_op(op);
     }
 
     return TRUE;
 }
 
 GList *
 services_os_get_directory_list(const char *root, gboolean files, gboolean executable)
 {
     GList *list = NULL;
     struct dirent **namelist;
     int entries = 0, lpc = 0;
     char buffer[PATH_MAX];
 
     entries = scandir(root, &namelist, NULL, alphasort);
     if (entries <= 0) {
         return list;
     }
 
     for (lpc = 0; lpc < entries; lpc++) {
         struct stat sb;
 
         if ('.' == namelist[lpc]->d_name[0]) {
             free(namelist[lpc]);
             continue;
         }
 
         snprintf(buffer, sizeof(buffer), "%s/%s", root, namelist[lpc]->d_name);
 
         if (stat(buffer, &sb)) {
             continue;
         }
 
         if (S_ISDIR(sb.st_mode)) {
             if (files) {
                 free(namelist[lpc]);
                 continue;
             }
 
         } else if (S_ISREG(sb.st_mode)) {
             if (files == FALSE) {
                 free(namelist[lpc]);
                 continue;
 
             } else if (executable
                        && (sb.st_mode & S_IXUSR) == 0
                        && (sb.st_mode & S_IXGRP) == 0 && (sb.st_mode & S_IXOTH) == 0) {
                 free(namelist[lpc]);
                 continue;
             }
         }
 
         list = g_list_append(list, strdup(namelist[lpc]->d_name));
 
         free(namelist[lpc]);
     }
 
     free(namelist);
     return list;
 }
 
 GList *
 resources_os_list_lsb_agents(void)
 {
     return get_directory_list(LSB_ROOT_DIR, TRUE, TRUE);
 }
 
 GList *
 resources_os_list_ocf_providers(void)
 {
     return get_directory_list(OCF_ROOT_DIR "/resource.d", FALSE, TRUE);
 }
 
 GList *
 resources_os_list_ocf_agents(const char *provider)
 {
     GList *gIter = NULL;
     GList *result = NULL;
     GList *providers = NULL;
 
     if (provider) {
         char buffer[500];
 
         snprintf(buffer, sizeof(buffer), "%s/resource.d/%s", OCF_ROOT_DIR, provider);
         return get_directory_list(buffer, TRUE, TRUE);
     }
 
     providers = resources_os_list_ocf_providers();
     for (gIter = providers; gIter != NULL; gIter = gIter->next) {
         GList *tmp1 = result;
         GList *tmp2 = resources_os_list_ocf_agents(gIter->data);
 
         if (tmp2) {
             result = g_list_concat(tmp1, tmp2);
         }
     }
     g_list_free_full(providers, free);
     return result;
 }
 
 #if SUPPORT_NAGIOS
 GList *
 resources_os_list_nagios_agents(void)
 {
     GList *plugin_list = NULL;
     GList *result = NULL;
     GList *gIter = NULL;
 
     plugin_list = get_directory_list(NAGIOS_PLUGIN_DIR, TRUE, TRUE);
 
     /* Make sure both the plugin and its metadata exist */
     for (gIter = plugin_list; gIter != NULL; gIter = gIter->next) {
         const char *plugin = gIter->data;
         char *metadata = crm_strdup_printf(NAGIOS_METADATA_DIR "/%s.xml", plugin);
         struct stat st;
 
         if (stat(metadata, &st) == 0) {
             result = g_list_append(result, strdup(plugin));
         }
 
         free(metadata);
     }
     g_list_free_full(plugin_list, free);
     return result;
 }
 #endif
diff --git a/lib/transition/unpack.c b/lib/transition/unpack.c
index 9f121f8e0e..31e39cb8b9 100644
--- a/lib/transition/unpack.c
+++ b/lib/transition/unpack.c
@@ -1,343 +1,343 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 
 #include <crm/common/xml.h>
 #include <crm/transition.h>
 #include <sys/stat.h>
 
 CRM_TRACE_INIT_DATA(transitioner);
 
 static crm_action_t *
 unpack_action(synapse_t * parent, xmlNode * xml_action)
 {
     crm_action_t *action = NULL;
     const char *value = crm_element_value(xml_action, XML_ATTR_ID);
 
     if (value == NULL) {
         crm_err("Actions must have an id!");
         crm_log_xml_trace(xml_action, "Action with missing id");
         return NULL;
     }
 
     action = calloc(1, sizeof(crm_action_t));
     if (action == NULL) {
         crm_perror(LOG_CRIT, "Cannot unpack action");
         crm_log_xml_trace(xml_action, "Lost action");
         return NULL;
     }
 
     action->id = crm_parse_int(value, NULL);
     action->type = action_type_rsc;
     action->xml = copy_xml(xml_action);
     action->synapse = parent;
 
     if (safe_str_eq(crm_element_name(action->xml), XML_GRAPH_TAG_RSC_OP)) {
         action->type = action_type_rsc;
 
     } else if (safe_str_eq(crm_element_name(action->xml), XML_GRAPH_TAG_PSEUDO_EVENT)) {
         action->type = action_type_pseudo;
 
     } else if (safe_str_eq(crm_element_name(action->xml), XML_GRAPH_TAG_CRM_EVENT)) {
         action->type = action_type_crm;
     }
 
     action->params = xml2list(action->xml);
 
     value = g_hash_table_lookup(action->params, "CRM_meta_timeout");
     if (value != NULL) {
         action->timeout = crm_parse_int(value, NULL);
     }
 
     /* Take start-delay into account for the timeout of the action timer */
     value = g_hash_table_lookup(action->params, "CRM_meta_start_delay");
     if (value != NULL) {
         action->timeout += crm_parse_int(value, NULL);
     }
 
     value = g_hash_table_lookup(action->params, "CRM_meta_interval");
     if (value != NULL) {
-        action->interval = crm_parse_int(value, NULL);
+        action->interval_ms = crm_parse_ms(value);
     }
 
     value = g_hash_table_lookup(action->params, "CRM_meta_can_fail");
     if (value != NULL) {
         crm_str_to_boolean(value, &(action->can_fail));
     }
 
     crm_trace("Action %d has timer set to %dms", action->id, action->timeout);
 
     return action;
 }
 
 static synapse_t *
 unpack_synapse(crm_graph_t * new_graph, xmlNode * xml_synapse)
 {
     const char *value = NULL;
     xmlNode *inputs = NULL;
     xmlNode *action_set = NULL;
     synapse_t *new_synapse = NULL;
 
     CRM_CHECK(xml_synapse != NULL, return NULL);
     crm_trace("looking in synapse %s", ID(xml_synapse));
 
     new_synapse = calloc(1, sizeof(synapse_t));
     new_synapse->id = crm_parse_int(ID(xml_synapse), NULL);
 
     value = crm_element_value(xml_synapse, XML_CIB_ATTR_PRIORITY);
     if (value != NULL) {
         new_synapse->priority = crm_parse_int(value, NULL);
     }
 
     CRM_CHECK(new_synapse->id >= 0, free(new_synapse);
               return NULL);
 
     new_graph->num_synapses++;
 
     crm_trace("look for actions in synapse %s", crm_element_value(xml_synapse, XML_ATTR_ID));
 
     for (action_set = __xml_first_child(xml_synapse); action_set != NULL;
          action_set = __xml_next(action_set)) {
         if (crm_str_eq((const char *)action_set->name, "action_set", TRUE)) {
             xmlNode *action = NULL;
 
             for (action = __xml_first_child(action_set); action != NULL;
                  action = __xml_next(action)) {
                 crm_action_t *new_action = unpack_action(new_synapse, action);
 
                 if (new_action == NULL) {
                     continue;
                 }
 
                 new_graph->num_actions++;
 
                 crm_trace("Adding action %d to synapse %d", new_action->id, new_synapse->id);
 
                 new_synapse->actions = g_list_append(new_synapse->actions, new_action);
             }
         }
     }
 
     crm_trace("look for inputs in synapse %s", ID(xml_synapse));
 
     for (inputs = __xml_first_child(xml_synapse); inputs != NULL; inputs = __xml_next(inputs)) {
         if (crm_str_eq((const char *)inputs->name, "inputs", TRUE)) {
             xmlNode *trigger = NULL;
 
             for (trigger = __xml_first_child(inputs); trigger != NULL;
                  trigger = __xml_next(trigger)) {
                 xmlNode *input = NULL;
 
                 for (input = __xml_first_child(trigger); input != NULL; input = __xml_next(input)) {
                     crm_action_t *new_input = unpack_action(new_synapse, input);
 
                     if (new_input == NULL) {
                         continue;
                     }
 
                     crm_trace("Adding input %d to synapse %d", new_input->id, new_synapse->id);
 
                     new_synapse->inputs = g_list_append(new_synapse->inputs, new_input);
                 }
             }
         }
     }
 
     return new_synapse;
 }
 
 static void destroy_action(crm_action_t * action);
 
 crm_graph_t *
 unpack_graph(xmlNode * xml_graph, const char *reference)
 {
 /*
   <transition_graph>
   <synapse>
   <action_set>
   <rsc_op id="2"
   ...
   <inputs>
   <rsc_op id="2"
   ...
 */
     crm_graph_t *new_graph = NULL;
     const char *t_id = NULL;
     const char *time = NULL;
     xmlNode *synapse = NULL;
 
     new_graph = calloc(1, sizeof(crm_graph_t));
 
     new_graph->id = -1;
     new_graph->abort_priority = 0;
     new_graph->network_delay = -1;
     new_graph->transition_timeout = -1;
     new_graph->stonith_timeout = -1;
     new_graph->completion_action = tg_done;
 
     if (reference) {
         new_graph->source = strdup(reference);
     } else {
         new_graph->source = strdup("unknown");
     }
 
     if (xml_graph != NULL) {
         t_id = crm_element_value(xml_graph, "transition_id");
         CRM_CHECK(t_id != NULL, free(new_graph);
                   return NULL);
         new_graph->id = crm_parse_int(t_id, "-1");
 
         time = crm_element_value(xml_graph, "cluster-delay");
         CRM_CHECK(time != NULL, free(new_graph);
                   return NULL);
         new_graph->network_delay = crm_get_msec(time);
 
         time = crm_element_value(xml_graph, "stonith-timeout");
         if (time == NULL) {
             new_graph->stonith_timeout = new_graph->network_delay;
         } else {
             new_graph->stonith_timeout = crm_get_msec(time);
         }
 
         t_id = crm_element_value(xml_graph, "batch-limit");
         new_graph->batch_limit = crm_parse_int(t_id, "0");
 
         t_id = crm_element_value(xml_graph, "migration-limit");
         new_graph->migration_limit = crm_parse_int(t_id, "-1");
     }
 
     for (synapse = __xml_first_child(xml_graph); synapse != NULL; synapse = __xml_next(synapse)) {
         if (crm_str_eq((const char *)synapse->name, "synapse", TRUE)) {
             synapse_t *new_synapse = unpack_synapse(new_graph, synapse);
 
             if (new_synapse != NULL) {
                 new_graph->synapses = g_list_append(new_graph->synapses, new_synapse);
             }
         }
     }
 
     crm_debug("Unpacked transition %d: %d actions in %d synapses",
               new_graph->id, new_graph->num_actions, new_graph->num_synapses);
 
     return new_graph;
 }
 
 static void
 destroy_action(crm_action_t * action)
 {
     if (action->timer && action->timer->source_id != 0) {
         crm_warn("Cancelling timer for action %d (src=%d)", action->id, action->timer->source_id);
         g_source_remove(action->timer->source_id);
     }
     if (action->params) {
         g_hash_table_destroy(action->params);
     }
     free_xml(action->xml);
     free(action->timer);
     free(action);
 }
 
 static void
 destroy_synapse(synapse_t * synapse)
 {
     while (g_list_length(synapse->actions) > 0) {
         crm_action_t *action = g_list_nth_data(synapse->actions, 0);
 
         synapse->actions = g_list_remove(synapse->actions, action);
         destroy_action(action);
     }
 
     while (g_list_length(synapse->inputs) > 0) {
         crm_action_t *action = g_list_nth_data(synapse->inputs, 0);
 
         synapse->inputs = g_list_remove(synapse->inputs, action);
         destroy_action(action);
     }
     free(synapse);
 }
 
 void
 destroy_graph(crm_graph_t * graph)
 {
     if (graph == NULL) {
         return;
     }
     while (g_list_length(graph->synapses) > 0) {
         synapse_t *synapse = g_list_nth_data(graph->synapses, 0);
 
         graph->synapses = g_list_remove(graph->synapses, synapse);
         destroy_synapse(synapse);
     }
 
     free(graph->source);
     free(graph);
 }
 
 lrmd_event_data_t *
 convert_graph_action(xmlNode * resource, crm_action_t * action, int status, int rc)
 {
     xmlNode *xop = NULL;
     lrmd_event_data_t *op = NULL;
     GHashTableIter iter;
     const char *name = NULL;
     const char *value = NULL;
     xmlNode *action_resource = NULL;
 
     CRM_CHECK(action != NULL, return NULL);
     CRM_CHECK(action->type == action_type_rsc, return NULL);
 
     action_resource = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
     CRM_CHECK(action_resource != NULL, crm_log_xml_warn(action->xml, "Bad");
               return NULL);
 
     op = calloc(1, sizeof(lrmd_event_data_t));
 
     op->rsc_id = strdup(ID(action_resource));
-    op->interval = action->interval;
+    op->interval_ms = action->interval_ms;
     op->op_type = strdup(crm_element_value(action->xml, XML_LRM_ATTR_TASK));
 
     op->rc = rc;
     op->op_status = status;
     op->t_run = time(NULL);
     op->t_rcchange = op->t_run;
 
     op->params = g_hash_table_new_full(crm_str_hash, g_str_equal, free, free);
 
     g_hash_table_iter_init(&iter, action->params);
     while (g_hash_table_iter_next(&iter, (void **)&name, (void **)&value)) {
         g_hash_table_insert(op->params, strdup(name), strdup(value));
     }
 
     for (xop = __xml_first_child(resource); xop != NULL; xop = __xml_next(xop)) {
         int tmp = 0;
 
         crm_element_value_int(xop, XML_LRM_ATTR_CALLID, &tmp);
         crm_debug("Got call_id=%d for %s", tmp, ID(resource));
         if (tmp > op->call_id) {
             op->call_id = tmp;
         }
     }
 
     op->call_id++;
     return op;
 }
diff --git a/tools/fake_transition.c b/tools/fake_transition.c
index 7b010745f7..8816363bdd 100644
--- a/tools/fake_transition.c
+++ b/tools/fake_transition.c
@@ -1,839 +1,840 @@
 /*
  * Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <unistd.h>
 #include <stdlib.h>
 
 #include <sys/stat.h>
 #include <sys/param.h>
 #include <sys/types.h>
 #include <dirent.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/common/util.h>
 #include <crm/transition.h>
 #include <crm/common/iso8601.h>
 #include <crm/pengine/status.h>
 #include <allocate.h>
 #include "fake_transition.h"
 
 static bool fake_quiet = FALSE;
 static cib_t *fake_cib = NULL;
 static GListPtr fake_resource_list = NULL;
 static GListPtr fake_op_fail_list = NULL;
 gboolean bringing_nodes_online = FALSE;
 
 #define STATUS_PATH_MAX 512
 
 #define quiet_log(fmt, args...) do {              \
               if(fake_quiet) {                         \
                   crm_trace(fmt, ##args);         \
               } else {                            \
                   printf(fmt , ##args);           \
               }                                   \
     } while(0)
 
 #define new_node_template "//"XML_CIB_TAG_NODE"[@uname='%s']"
 #define node_template "//"XML_CIB_TAG_STATE"[@uname='%s']"
 #define rsc_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']"
 #define op_template  "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s']"
 /* #define op_template  "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s' and @"XML_LRM_ATTR_CALLID"='%d']" */
 
 
 static void
 inject_transient_attr(xmlNode * cib_node, const char *name, const char *value)
 {
     xmlNode *attrs = NULL;
     xmlNode *instance_attrs = NULL;
     xmlChar *node_path;
     const char *node_uuid = ID(cib_node);
 
     node_path = xmlGetNodePath(cib_node);
     quiet_log(" + Injecting attribute %s=%s into %s '%s'\n",
               name, value, node_path, ID(cib_node));
     free(node_path);
 
     attrs = first_named_child(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
     if (attrs == NULL) {
         attrs = create_xml_node(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
         crm_xml_add(attrs, XML_ATTR_ID, node_uuid);
     }
 
     instance_attrs = first_named_child(attrs, XML_TAG_ATTR_SETS);
     if (instance_attrs == NULL) {
         instance_attrs = create_xml_node(attrs, XML_TAG_ATTR_SETS);
         crm_xml_add(instance_attrs, XML_ATTR_ID, node_uuid);
     }
 
     crm_create_nvpair_xml(instance_attrs, NULL, name, value);
 }
 
 static void
 update_failcounts(xmlNode * cib_node, const char *resource, const char *task,
                   guint interval_ms, int rc)
 {
     if (rc == 0) {
         return;
 
     } else if ((rc == 7) && (interval_ms == 0)) {
         return;
 
     } else {
         char *name = NULL;
         char *now = crm_itoa(time(NULL));
 
         name = crm_failcount_name(resource, task, interval_ms);
         inject_transient_attr(cib_node, name, "value++");
         free(name);
 
         name = crm_lastfailure_name(resource, task, interval_ms);
         inject_transient_attr(cib_node, name, now);
         free(name);
         free(now);
     }
 }
 
 static void
 create_node_entry(cib_t * cib_conn, const char *node)
 {
     int rc = pcmk_ok;
     char *xpath = crm_strdup_printf(new_node_template, node);
 
     rc = cib_conn->cmds->query(cib_conn, xpath, NULL, cib_xpath | cib_sync_call | cib_scope_local);
 
     if (rc == -ENXIO) {
         xmlNode *cib_object = create_xml_node(NULL, XML_CIB_TAG_NODE);
 
         crm_xml_add(cib_object, XML_ATTR_ID, node); // Use node name as ID
         crm_xml_add(cib_object, XML_ATTR_UNAME, node);
         cib_conn->cmds->create(cib_conn, XML_CIB_TAG_NODES, cib_object,
                                cib_sync_call | cib_scope_local);
         /* Not bothering with subsequent query to see if it exists,
            we'll bomb out later in the call to query_node_uuid()... */
 
         free_xml(cib_object);
     }
 
     free(xpath);
 }
 
 static lrmd_event_data_t *
 create_op(xmlNode *cib_resource, const char *task, guint interval_ms,
           int outcome)
 {
     lrmd_event_data_t *op = NULL;
     xmlNode *xop = NULL;
 
     op = calloc(1, sizeof(lrmd_event_data_t));
 
     op->rsc_id = strdup(ID(cib_resource));
-    op->interval = interval_ms;
+    op->interval_ms = interval_ms;
     op->op_type = strdup(task);
 
     op->rc = outcome;
     op->op_status = 0;
     op->params = NULL;          /* TODO: Fill me in */
     op->t_run = time(NULL);
     op->t_rcchange = op->t_run;
 
     op->call_id = 0;
     for (xop = __xml_first_child(cib_resource); xop != NULL; xop = __xml_next(xop)) {
         int tmp = 0;
 
         crm_element_value_int(xop, XML_LRM_ATTR_CALLID, &tmp);
         if (tmp > op->call_id) {
             op->call_id = tmp;
         }
     }
     op->call_id++;
 
     return op;
 }
 
 static xmlNode *
 inject_op(xmlNode * cib_resource, lrmd_event_data_t * op, int target_rc)
 {
     return create_operation_update(cib_resource, op, CRM_FEATURE_SET, target_rc,
                                    NULL, crm_system_name, LOG_TRACE);
 }
 
 static xmlNode *
 inject_node_state(cib_t * cib_conn, const char *node, const char *uuid)
 {
     int rc = pcmk_ok;
     xmlNode *cib_object = NULL;
     char *xpath = crm_strdup_printf(node_template, node);
 
     if (bringing_nodes_online) {
         create_node_entry(cib_conn, node);
     }
 
     rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
                                cib_xpath | cib_sync_call | cib_scope_local);
 
     if (cib_object && ID(cib_object) == NULL) {
         crm_err("Detected multiple node_state entries for xpath=%s, bailing", xpath);
         crm_log_xml_warn(cib_object, "Duplicates");
         free(xpath);
         crm_exit(CRM_EX_SOFTWARE);
         return NULL; // not reached, but makes static analysis happy
     }
 
     if (rc == -ENXIO) {
         char *found_uuid = NULL;
 
         if (uuid == NULL) {
             query_node_uuid(cib_conn, node, &found_uuid, NULL);
         } else {
             found_uuid = strdup(uuid);
         }
 
         cib_object = create_xml_node(NULL, XML_CIB_TAG_STATE);
         crm_xml_add(cib_object, XML_ATTR_UUID, found_uuid);
         crm_xml_add(cib_object, XML_ATTR_UNAME, node);
         cib_conn->cmds->create(cib_conn, XML_CIB_TAG_STATUS, cib_object,
                                cib_sync_call | cib_scope_local);
         free_xml(cib_object);
         free(found_uuid);
 
         rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
                                    cib_xpath | cib_sync_call | cib_scope_local);
         crm_trace("injecting node state for %s. rc is %d", node, rc);
     }
 
     free(xpath);
     CRM_ASSERT(rc == pcmk_ok);
     return cib_object;
 }
 
 static xmlNode *
 modify_node(cib_t * cib_conn, char *node, gboolean up)
 {
     xmlNode *cib_node = inject_node_state(cib_conn, node, NULL);
 
     if (up) {
         crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_YES);
         crm_xml_add(cib_node, XML_NODE_IS_PEER, ONLINESTATUS);
         crm_xml_add(cib_node, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_MEMBER);
         crm_xml_add(cib_node, XML_NODE_EXPECTED, CRMD_JOINSTATE_MEMBER);
 
     } else {
         crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO);
         crm_xml_add(cib_node, XML_NODE_IS_PEER, OFFLINESTATUS);
         crm_xml_add(cib_node, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_DOWN);
         crm_xml_add(cib_node, XML_NODE_EXPECTED, CRMD_JOINSTATE_DOWN);
     }
 
     crm_xml_add(cib_node, XML_ATTR_ORIGIN, crm_system_name);
     return cib_node;
 }
 
 static xmlNode *
 find_resource_xml(xmlNode * cib_node, const char *resource)
 {
     xmlNode *match = NULL;
     const char *node = crm_element_value(cib_node, XML_ATTR_UNAME);
     char *xpath = crm_strdup_printf(rsc_template, node, resource);
 
     match = get_xpath_object(xpath, cib_node, LOG_TRACE);
     free(xpath);
     return match;
 }
 
 
 static xmlNode *
 inject_resource(xmlNode * cib_node, const char *resource, const char *rclass, const char *rtype,
                 const char *rprovider)
 {
     xmlNode *lrm = NULL;
     xmlNode *container = NULL;
     xmlNode *cib_resource = NULL;
     char *xpath = NULL;
 
     cib_resource = find_resource_xml(cib_node, resource);
     if (cib_resource != NULL) {
         return cib_resource;
     }
 
     /* One day, add query for class, provider, type */
 
     if (rclass == NULL || rtype == NULL) {
         fprintf(stderr, "Resource %s not found in the status section of %s."
                 "  Please supply the class and type to continue\n", resource, ID(cib_node));
         return NULL;
 
     } else if (safe_str_neq(rclass, PCMK_RESOURCE_CLASS_OCF)
                && safe_str_neq(rclass, PCMK_RESOURCE_CLASS_STONITH)
                && safe_str_neq(rclass, PCMK_RESOURCE_CLASS_SERVICE)
                && safe_str_neq(rclass, PCMK_RESOURCE_CLASS_UPSTART)
                && safe_str_neq(rclass, PCMK_RESOURCE_CLASS_SYSTEMD)
                && safe_str_neq(rclass, PCMK_RESOURCE_CLASS_LSB)) {
         fprintf(stderr, "Invalid class for %s: %s\n", resource, rclass);
         return NULL;
 
     } else if (crm_provider_required(rclass) && (rprovider == NULL)) {
         fprintf(stderr, "Please specify the provider for resource %s\n", resource);
         return NULL;
     }
 
     xpath = (char *)xmlGetNodePath(cib_node);
     crm_info("Injecting new resource %s into %s '%s'", resource, xpath, ID(cib_node));
     free(xpath);
 
     lrm = first_named_child(cib_node, XML_CIB_TAG_LRM);
     if (lrm == NULL) {
         const char *node_uuid = ID(cib_node);
 
         lrm = create_xml_node(cib_node, XML_CIB_TAG_LRM);
         crm_xml_add(lrm, XML_ATTR_ID, node_uuid);
     }
 
     container = first_named_child(lrm, XML_LRM_TAG_RESOURCES);
     if (container == NULL) {
         container = create_xml_node(lrm, XML_LRM_TAG_RESOURCES);
     }
 
     cib_resource = create_xml_node(container, XML_LRM_TAG_RESOURCE);
     crm_xml_add(cib_resource, XML_ATTR_ID, resource);
 
     crm_xml_add(cib_resource, XML_AGENT_ATTR_CLASS, rclass);
     crm_xml_add(cib_resource, XML_AGENT_ATTR_PROVIDER, rprovider);
     crm_xml_add(cib_resource, XML_ATTR_TYPE, rtype);
 
     return cib_resource;
 }
 
 #define XPATH_MAX 1024
 
 static int
 find_ticket_state(cib_t * the_cib, const char *ticket_id, xmlNode ** ticket_state_xml)
 {
     int offset = 0;
     int rc = pcmk_ok;
     xmlNode *xml_search = NULL;
 
     char *xpath_string = NULL;
 
     CRM_ASSERT(ticket_state_xml != NULL);
     *ticket_state_xml = NULL;
 
     xpath_string = calloc(1, XPATH_MAX);
     offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "%s", "/cib/status/tickets");
 
     if (ticket_id) {
         offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "/%s[@id=\"%s\"]",
                            XML_CIB_TAG_TICKET_STATE, ticket_id);
     }
     CRM_LOG_ASSERT(offset > 0);
     rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
                               cib_sync_call | cib_scope_local | cib_xpath);
 
     if (rc != pcmk_ok) {
         goto bail;
     }
 
     crm_log_xml_debug(xml_search, "Match");
     if (xml_has_children(xml_search)) {
         if (ticket_id) {
             fprintf(stdout, "Multiple ticket_states match ticket_id=%s\n", ticket_id);
         }
         *ticket_state_xml = xml_search;
     } else {
         *ticket_state_xml = xml_search;
     }
 
   bail:
     free(xpath_string);
     return rc;
 }
 
 static int
 set_ticket_state_attr(const char *ticket_id, const char *attr_name,
                       const char *attr_value, cib_t * cib, int cib_options)
 {
     int rc = pcmk_ok;
     xmlNode *xml_top = NULL;
     xmlNode *ticket_state_xml = NULL;
 
     rc = find_ticket_state(cib, ticket_id, &ticket_state_xml);
     if (rc == pcmk_ok) {
         crm_debug("Found a match state for ticket: id=%s", ticket_id);
         xml_top = ticket_state_xml;
 
     } else if (rc != -ENXIO) {
         return rc;
 
     } else {
         xmlNode *xml_obj = NULL;
 
         xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
         xml_obj = create_xml_node(xml_top, XML_CIB_TAG_TICKETS);
         ticket_state_xml = create_xml_node(xml_obj, XML_CIB_TAG_TICKET_STATE);
         crm_xml_add(ticket_state_xml, XML_ATTR_ID, ticket_id);
     }
 
     crm_xml_add(ticket_state_xml, attr_name, attr_value);
 
     crm_log_xml_debug(xml_top, "Update");
 
     rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, xml_top, cib_options);
 
     free_xml(xml_top);
 
     return rc;
 }
 
 void
 modify_configuration(pe_working_set_t * data_set, cib_t *cib,
                      const char *quorum, const char *watchdog, GListPtr node_up, GListPtr node_down, GListPtr node_fail,
                      GListPtr op_inject, GListPtr ticket_grant, GListPtr ticket_revoke,
                      GListPtr ticket_standby, GListPtr ticket_activate)
 {
     int rc = pcmk_ok;
     GListPtr gIter = NULL;
 
     xmlNode *cib_op = NULL;
     xmlNode *cib_node = NULL;
     xmlNode *cib_resource = NULL;
 
     lrmd_event_data_t *op = NULL;
 
     if (quorum) {
         xmlNode *top = create_xml_node(NULL, XML_TAG_CIB);
 
         quiet_log(" + Setting quorum: %s\n", quorum);
         /* crm_xml_add(top, XML_ATTR_DC_UUID, dc_uuid);      */
         crm_xml_add(top, XML_ATTR_HAVE_QUORUM, quorum);
 
         rc = cib->cmds->modify(cib, NULL, top, cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     if (watchdog) {
         quiet_log(" + Setting watchdog: %s\n", watchdog);
 
         rc = update_attr_delegate(cib, cib_sync_call | cib_scope_local,
                              XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
                              XML_ATTR_HAVE_WATCHDOG, watchdog, FALSE, NULL, NULL);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = node_up; gIter != NULL; gIter = gIter->next) {
         char *node = (char *)gIter->data;
 
         quiet_log(" + Bringing node %s online\n", node);
         cib_node = modify_node(cib, node, TRUE);
         CRM_ASSERT(cib_node != NULL);
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                                       cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
         free_xml(cib_node);
     }
 
     for (gIter = node_down; gIter != NULL; gIter = gIter->next) {
         char xpath[STATUS_PATH_MAX];
         char *node = (char *)gIter->data;
 
         quiet_log(" + Taking node %s offline\n", node);
         cib_node = modify_node(cib, node, FALSE);
         CRM_ASSERT(cib_node != NULL);
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                                       cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
         free_xml(cib_node);
 
         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", node, XML_CIB_TAG_LRM);
         cib->cmds->remove(cib, xpath, NULL,
                                       cib_xpath | cib_sync_call | cib_scope_local);
 
         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", node,
                  XML_TAG_TRANSIENT_NODEATTRS);
         cib->cmds->remove(cib, xpath, NULL,
                                       cib_xpath | cib_sync_call | cib_scope_local);
 
     }
 
     for (gIter = node_fail; gIter != NULL; gIter = gIter->next) {
         char *node = (char *)gIter->data;
 
         quiet_log(" + Failing node %s\n", node);
         cib_node = modify_node(cib, node, TRUE);
         crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO);
         CRM_ASSERT(cib_node != NULL);
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                                       cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
         free_xml(cib_node);
     }
 
     for (gIter = ticket_grant; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         quiet_log(" + Granting ticket %s\n", ticket_id);
         rc = set_ticket_state_attr(ticket_id, "granted", "true",
                                    cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = ticket_revoke; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         quiet_log(" + Revoking ticket %s\n", ticket_id);
         rc = set_ticket_state_attr(ticket_id, "granted", "false",
                                    cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = ticket_standby; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         quiet_log(" + Making ticket %s standby\n", ticket_id);
         rc = set_ticket_state_attr(ticket_id, "standby", "true",
                                    cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = ticket_activate; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         quiet_log(" + Activating ticket %s\n", ticket_id);
         rc = set_ticket_state_attr(ticket_id, "standby", "false",
                                    cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = op_inject; gIter != NULL; gIter = gIter->next) {
         char *spec = (char *)gIter->data;
 
         int rc = 0;
         int outcome = 0;
         guint interval_ms = 0;
 
         char *key = NULL;
         char *node = NULL;
         char *task = NULL;
         char *resource = NULL;
 
         const char *rtype = NULL;
         const char *rclass = NULL;
         const char *rprovider = NULL;
 
         resource_t *rsc = NULL;
 
         quiet_log(" + Injecting %s into the configuration\n", spec);
 
         key = calloc(1, strlen(spec) + 1);
         node = calloc(1, strlen(spec) + 1);
         rc = sscanf(spec, "%[^@]@%[^=]=%d", key, node, &outcome);
         if (rc != 3) {
             fprintf(stderr, "Invalid operation spec: %s.  Only found %d fields\n", spec, rc);
             free(key);
             free(node);
             continue;
         }
 
         parse_op_key(key, &resource, &task, &interval_ms);
 
         rsc = pe_find_resource(data_set->resources, resource);
         if (rsc == NULL) {
             fprintf(stderr, " - Invalid resource name: %s\n", resource);
         } else {
             rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
             rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
             rprovider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
 
             cib_node = inject_node_state(cib, node, NULL);
             CRM_ASSERT(cib_node != NULL);
 
             update_failcounts(cib_node, resource, task, interval_ms, outcome);
 
             cib_resource = inject_resource(cib_node, resource, rclass, rtype, rprovider);
             CRM_ASSERT(cib_resource != NULL);
 
             op = create_op(cib_resource, task, interval_ms, outcome);
             CRM_ASSERT(op != NULL);
 
             cib_op = inject_op(cib_resource, op, 0);
             CRM_ASSERT(cib_op != NULL);
             lrmd_free_event(op);
 
             rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                                           cib_sync_call | cib_scope_local);
             CRM_ASSERT(rc == pcmk_ok);
         }
         free(task);
         free(node);
         free(key);
     }
 }
 
 static gboolean
 exec_pseudo_action(crm_graph_t * graph, crm_action_t * action)
 {
     const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 
     action->confirmed = TRUE;
 
     quiet_log(" * Pseudo action:   %s%s%s\n", task, node ? " on " : "", node ? node : "");
     update_graph(graph, action);
     return TRUE;
 }
 
 static gboolean
 exec_rsc_action(crm_graph_t * graph, crm_action_t * action)
 {
     int rc = 0;
     GListPtr gIter = NULL;
     lrmd_event_data_t *op = NULL;
     int target_outcome = 0;
 
     const char *rtype = NULL;
     const char *rclass = NULL;
     const char *resource = NULL;
     const char *rprovider = NULL;
     const char *operation = crm_element_value(action->xml, "operation");
     const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);
 
     xmlNode *cib_node = NULL;
     xmlNode *cib_resource = NULL;
     xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
 
     char *node = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
     char *uuid = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET_UUID);
     const char *router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 
     if (safe_str_eq(operation, CRM_OP_PROBED)
         || safe_str_eq(operation, CRM_OP_REPROBE)) {
         crm_info("Skipping %s op for %s", operation, node);
         goto done;
     }
 
     if (action_rsc == NULL) {
         crm_log_xml_err(action->xml, "Bad");
         free(node); free(uuid);
         return FALSE;
     }
 
     /* Look for the preferred name
      * If not found, try the expected 'local' name
      * If not found use the preferred name anyway
      */
     resource = crm_element_value(action_rsc, XML_ATTR_ID);
     if (pe_find_resource(fake_resource_list, resource) == NULL) {
         const char *longname = crm_element_value(action_rsc, XML_ATTR_ID_LONG);
 
         if (pe_find_resource(fake_resource_list, longname)) {
             resource = longname;
         }
     }
 
     if (safe_str_eq(operation, "delete") || safe_str_eq(operation, RSC_METADATA)) {
         quiet_log(" * Resource action: %-15s %s on %s\n", resource, operation, node);
         goto done;
     }
 
     rclass = crm_element_value(action_rsc, XML_AGENT_ATTR_CLASS);
     rtype = crm_element_value(action_rsc, XML_ATTR_TYPE);
     rprovider = crm_element_value(action_rsc, XML_AGENT_ATTR_PROVIDER);
 
     if (target_rc_s != NULL) {
         target_outcome = crm_parse_int(target_rc_s, "0");
     }
 
     CRM_ASSERT(fake_cib->cmds->query(fake_cib, NULL, NULL, cib_sync_call | cib_scope_local) ==
                pcmk_ok);
 
     cib_node = inject_node_state(fake_cib, node, (router_node? node : uuid));
     CRM_ASSERT(cib_node != NULL);
 
     cib_resource = inject_resource(cib_node, resource, rclass, rtype, rprovider);
     if (cib_resource == NULL) {
         crm_err("invalid resource in transition");
         return FALSE;
     }
 
     op = convert_graph_action(cib_resource, action, 0, target_outcome);
-    if (op->interval) {
-        quiet_log(" * Resource action: %-15s %s=%d on %s\n", resource, op->op_type, op->interval,
-                  node);
+    if (op->interval_ms) {
+        quiet_log(" * Resource action: %-15s %s=%u on %s\n",
+                  resource, op->op_type, op->interval_ms, node);
     } else {
         quiet_log(" * Resource action: %-15s %s on %s\n", resource, op->op_type, node);
     }
 
     for (gIter = fake_op_fail_list; gIter != NULL; gIter = gIter->next) {
         char *spec = (char *)gIter->data;
-        char *key = crm_strdup_printf("%s_%s_%d@%s=", resource, op->op_type,
-                                      op->interval, node);
+        char *key = crm_strdup_printf(CRM_OP_FMT "@%s=", resource, op->op_type,
+                                      op->interval_ms, node);
 
         if (strncasecmp(key, spec, strlen(key)) == 0) {
             rc = sscanf(spec, "%*[^=]=%d", (int *) &op->rc);
             // ${resource}_${task}_${interval_in_ms}@${node}=${rc}
 
             if (rc != 1) {
                 fprintf(stderr,
                         "Invalid failed operation spec: %s. Result code must be integer\n",
                         spec);
                 free(key);
                 continue;
             }
             action->failed = TRUE;
             graph->abort_priority = INFINITY;
             printf("\tPretending action %d failed with rc=%d\n", action->id, op->rc);
-            update_failcounts(cib_node, resource, op->op_type, op->interval, op->rc);
+            update_failcounts(cib_node, resource, op->op_type, op->interval_ms,
+                              op->rc);
             free(key);
             break;
         }
         free(key);
     }
 
     inject_op(cib_resource, op, target_outcome);
     lrmd_free_event(op);
 
     rc = fake_cib->cmds->modify(fake_cib, XML_CIB_TAG_STATUS, cib_node,
                                   cib_sync_call | cib_scope_local);
     CRM_ASSERT(rc == pcmk_ok);
 
   done:
     free(node); free(uuid);
     free_xml(cib_node);
     action->confirmed = TRUE;
     update_graph(graph, action);
     return TRUE;
 }
 
 static gboolean
 exec_crmd_action(crm_graph_t * graph, crm_action_t * action)
 {
     const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     xmlNode *rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
 
     action->confirmed = TRUE;
 
     if(rsc) {
         quiet_log(" * Cluster action:  %s for %s on %s\n", task, ID(rsc), node);
     } else {
         quiet_log(" * Cluster action:  %s on %s\n", task, node);
     }
     update_graph(graph, action);
     return TRUE;
 }
 
 static gboolean
 exec_stonith_action(crm_graph_t * graph, crm_action_t * action)
 {
     const char *op = crm_meta_value(action->params, "stonith_action");
     char *target = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
 
     quiet_log(" * Fencing %s (%s)\n", target, op);
     if(safe_str_neq(op, "on")) {
         int rc = 0;
         char xpath[STATUS_PATH_MAX];
         xmlNode *cib_node = modify_node(fake_cib, target, FALSE);
 
         crm_xml_add(cib_node, XML_ATTR_ORIGIN, __FUNCTION__);
         CRM_ASSERT(cib_node != NULL);
 
         rc = fake_cib->cmds->replace(fake_cib, XML_CIB_TAG_STATUS, cib_node,
                                    cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
 
         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target, XML_CIB_TAG_LRM);
         fake_cib->cmds->remove(fake_cib, xpath, NULL,
                                       cib_xpath | cib_sync_call | cib_scope_local);
 
         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target,
                  XML_TAG_TRANSIENT_NODEATTRS);
         fake_cib->cmds->remove(fake_cib, xpath, NULL,
                                       cib_xpath | cib_sync_call | cib_scope_local);
 
         free_xml(cib_node);
     }
 
     action->confirmed = TRUE;
     update_graph(graph, action);
     free(target);
     return TRUE;
 }
 
 int
 run_simulation(pe_working_set_t * data_set, cib_t *cib, GListPtr op_fail_list, bool quiet)
 {
     crm_graph_t *transition = NULL;
     enum transition_status graph_rc = -1;
 
     crm_graph_functions_t exec_fns = {
         exec_pseudo_action,
         exec_rsc_action,
         exec_crmd_action,
         exec_stonith_action,
     };
 
     fake_cib = cib;
     fake_quiet = quiet;
     fake_op_fail_list = op_fail_list;
 
     quiet_log("\nExecuting cluster transition:\n");
 
     set_graph_functions(&exec_fns);
     transition = unpack_graph(data_set->graph, crm_system_name);
     print_graph(LOG_DEBUG, transition);
 
     fake_resource_list = data_set->resources;
     do {
         graph_rc = run_graph(transition);
 
     } while (graph_rc == transition_active);
     fake_resource_list = NULL;
 
     if (graph_rc != transition_complete) {
         fprintf(stdout, "Transition failed: %s\n", transition_status(graph_rc));
         print_graph(LOG_ERR, transition);
     }
     destroy_graph(transition);
     if (graph_rc != transition_complete) {
         fprintf(stdout, "An invalid transition was produced\n");
     }
 
     if (quiet == FALSE) {
         xmlNode *cib_object = NULL;
         int rc = fake_cib->cmds->query(fake_cib, NULL, &cib_object, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
         cleanup_alloc_calculations(data_set);
         data_set->input = cib_object;
     }
 
     if (graph_rc != transition_complete) {
         return graph_rc;
     }
     return 0;
 }