diff --git a/crmd/messages.c b/crmd/messages.c
index 990a7a1893..2f285f03a7 100644
--- a/crmd/messages.c
+++ b/crmd/messages.c
@@ -1,1050 +1,1050 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <string.h>
 #include <time.h>
 #include <crmd_fsa.h>
 
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crm/cluster/internal.h>
 #include <crm/cib.h>
 #include <crm/common/ipcs.h>
 
 #include <crmd.h>
 #include <crmd_messages.h>
 #include <crmd_lrm.h>
 #include <tengine.h>
 #include <throttle.h>
 
 GListPtr fsa_message_queue = NULL;
 extern void crm_shutdown(int nsig);
 
 extern crm_ipc_t *attrd_ipc;
 void handle_response(xmlNode * stored_msg);
 enum crmd_fsa_input handle_request(xmlNode * stored_msg, enum crmd_fsa_cause cause);
 enum crmd_fsa_input handle_shutdown_request(xmlNode * stored_msg);
 
 #define ROUTER_RESULT(x)	crm_trace("Router result: %s", x)
 
 /* debug only, can wrap all it likes */
 int last_data_id = 0;
 
 void
 register_fsa_error_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
                        fsa_data_t * cur_data, void *new_data, const char *raised_from)
 {
     /* save the current actions if any */
     if (fsa_actions != A_NOTHING) {
         register_fsa_input_adv(cur_data ? cur_data->fsa_cause : C_FSA_INTERNAL,
                                I_NULL, cur_data ? cur_data->data : NULL,
                                fsa_actions, TRUE, __FUNCTION__);
     }
 
     /* reset the action list */
     crm_info("Resetting the current action list");
     fsa_dump_actions(fsa_actions, "Drop");
     fsa_actions = A_NOTHING;
 
     /* register the error */
     register_fsa_input_adv(cause, input, new_data, A_NOTHING, TRUE, raised_from);
 }
 
 int
 register_fsa_input_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
                        void *data, long long with_actions,
                        gboolean prepend, const char *raised_from)
 {
     unsigned old_len = g_list_length(fsa_message_queue);
     fsa_data_t *fsa_data = NULL;
 
     CRM_CHECK(raised_from != NULL, raised_from = "<unknown>");
 
     if (input == I_NULL && with_actions == A_NOTHING /* && data == NULL */ ) {
         /* no point doing anything */
         crm_err("Cannot add entry to queue: no input and no action");
         return 0;
     }
 
     if (input == I_WAIT_FOR_EVENT) {
         do_fsa_stall = TRUE;
         crm_debug("Stalling the FSA pending further input: source=%s cause=%s data=%p queue=%d",
                   raised_from, fsa_cause2string(cause), data, old_len);
 
         if (old_len > 0) {
             fsa_dump_queue(LOG_TRACE);
             prepend = FALSE;
         }
 
         if (data == NULL) {
             fsa_actions |= with_actions;
             fsa_dump_actions(with_actions, "Restored");
             return 0;
         }
 
         /* Store everything in the new event and reset fsa_actions */
         with_actions |= fsa_actions;
         fsa_actions = A_NOTHING;
     }
 
     last_data_id++;
     crm_trace("%s %s FSA input %d (%s) (cause=%s) %s data",
               raised_from, prepend ? "prepended" : "appended", last_data_id,
               fsa_input2string(input), fsa_cause2string(cause), data ? "with" : "without");
 
     fsa_data = calloc(1, sizeof(fsa_data_t));
     fsa_data->id = last_data_id;
     fsa_data->fsa_input = input;
     fsa_data->fsa_cause = cause;
     fsa_data->origin = raised_from;
     fsa_data->data = NULL;
     fsa_data->data_type = fsa_dt_none;
     fsa_data->actions = with_actions;
 
     if (with_actions != A_NOTHING) {
         crm_trace("Adding actions %.16llx to input", with_actions);
     }
 
     if (data != NULL) {
         switch (cause) {
             case C_FSA_INTERNAL:
             case C_CRMD_STATUS_CALLBACK:
             case C_IPC_MESSAGE:
             case C_HA_MESSAGE:
                 crm_trace("Copying %s data from %s as a HA msg",
                           fsa_cause2string(cause), raised_from);
                 CRM_CHECK(((ha_msg_input_t *) data)->msg != NULL,
                           crm_err("Bogus data from %s", raised_from));
                 fsa_data->data = copy_ha_msg_input(data);
                 fsa_data->data_type = fsa_dt_ha_msg;
                 break;
 
             case C_LRM_OP_CALLBACK:
                 crm_trace("Copying %s data from %s as lrmd_event_data_t",
                           fsa_cause2string(cause), raised_from);
                 fsa_data->data = lrmd_copy_event((lrmd_event_data_t *) data);
                 fsa_data->data_type = fsa_dt_lrm;
                 break;
 
             case C_CCM_CALLBACK:
             case C_SUBSYSTEM_CONNECT:
             case C_LRM_MONITOR_CALLBACK:
             case C_TIMER_POPPED:
             case C_SHUTDOWN:
             case C_HEARTBEAT_FAILED:
             case C_HA_DISCONNECT:
             case C_ILLEGAL:
             case C_UNKNOWN:
             case C_STARTUP:
                 crm_err("Copying %s data (from %s)"
                         " not yet implemented", fsa_cause2string(cause), raised_from);
                 crmd_exit(pcmk_err_generic);
                 break;
         }
         crm_trace("%s data copied", fsa_cause2string(fsa_data->fsa_cause));
     }
 
     /* make sure to free it properly later */
     if (prepend) {
         crm_trace("Prepending input");
         fsa_message_queue = g_list_prepend(fsa_message_queue, fsa_data);
     } else {
         fsa_message_queue = g_list_append(fsa_message_queue, fsa_data);
     }
 
     crm_trace("Queue len: %d", g_list_length(fsa_message_queue));
 
     /* fsa_dump_queue(LOG_DEBUG_2); */
 
     if (old_len == g_list_length(fsa_message_queue)) {
         crm_err("Couldn't add message to the queue");
     }
 
     if (fsa_source && input != I_WAIT_FOR_EVENT) {
         crm_trace("Triggering FSA: %s", __FUNCTION__);
         mainloop_set_trigger(fsa_source);
     }
     return last_data_id;
 }
 
 void
 fsa_dump_queue(int log_level)
 {
     int offset = 0;
     GListPtr lpc = NULL;
 
     for (lpc = fsa_message_queue; lpc != NULL; lpc = lpc->next) {
         fsa_data_t *data = (fsa_data_t *) lpc->data;
 
         do_crm_log_unlikely(log_level,
                             "queue[%d.%d]: input %s raised by %s(%p.%d)\t(cause=%s)",
                             offset++, data->id, fsa_input2string(data->fsa_input),
                             data->origin, data->data, data->data_type,
                             fsa_cause2string(data->fsa_cause));
     }
 }
 
 ha_msg_input_t *
 copy_ha_msg_input(ha_msg_input_t * orig)
 {
     ha_msg_input_t *copy = NULL;
     xmlNodePtr data = NULL;
 
     if (orig != NULL) {
         crm_trace("Copy msg");
         data = copy_xml(orig->msg);
 
     } else {
         crm_trace("No message to copy");
     }
     copy = new_ha_msg_input(data);
     if (orig && orig->msg != NULL) {
         CRM_CHECK(copy->msg != NULL, crm_err("copy failed"));
     }
     return copy;
 }
 
 void
 delete_fsa_input(fsa_data_t * fsa_data)
 {
     lrmd_event_data_t *op = NULL;
     xmlNode *foo = NULL;
 
     if (fsa_data == NULL) {
         return;
     }
     crm_trace("About to free %s data", fsa_cause2string(fsa_data->fsa_cause));
 
     if (fsa_data->data != NULL) {
         switch (fsa_data->data_type) {
             case fsa_dt_ha_msg:
                 delete_ha_msg_input(fsa_data->data);
                 break;
 
             case fsa_dt_xml:
                 foo = fsa_data->data;
                 free_xml(foo);
                 break;
 
             case fsa_dt_lrm:
                 op = (lrmd_event_data_t *) fsa_data->data;
                 lrmd_free_event(op);
                 break;
 
             case fsa_dt_none:
                 if (fsa_data->data != NULL) {
                     crm_err("Don't know how to free %s data from %s",
                             fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
                     crmd_exit(pcmk_err_generic);
                 }
                 break;
         }
         crm_trace("%s data freed", fsa_cause2string(fsa_data->fsa_cause));
     }
 
     free(fsa_data);
 }
 
 /* returns the next message */
 fsa_data_t *
 get_message(void)
 {
     fsa_data_t *message = g_list_nth_data(fsa_message_queue, 0);
 
     fsa_message_queue = g_list_remove(fsa_message_queue, message);
     crm_trace("Processing input %d", message->id);
     return message;
 }
 
 /* returns the current head of the FIFO queue */
 gboolean
 is_message(void)
 {
     return (g_list_length(fsa_message_queue) > 0);
 }
 
 void *
 fsa_typed_data_adv(fsa_data_t * fsa_data, enum fsa_data_type a_type, const char *caller)
 {
     void *ret_val = NULL;
 
     if (fsa_data == NULL) {
         crm_err("%s: No FSA data available", caller);
 
     } else if (fsa_data->data == NULL) {
         crm_err("%s: No message data available. Origin: %s", caller, fsa_data->origin);
 
     } else if (fsa_data->data_type != a_type) {
         crm_crit("%s: Message data was the wrong type! %d vs. requested=%d.  Origin: %s",
                  caller, fsa_data->data_type, a_type, fsa_data->origin);
         CRM_ASSERT(fsa_data->data_type == a_type);
     } else {
         ret_val = fsa_data->data;
     }
 
     return ret_val;
 }
 
 /*	A_MSG_ROUTE	*/
 void
 do_msg_route(long long action,
              enum crmd_fsa_cause cause,
              enum crmd_fsa_state cur_state,
              enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
 
     route_message(msg_data->fsa_cause, input->msg);
 }
 
 void
 route_message(enum crmd_fsa_cause cause, xmlNode * input)
 {
     ha_msg_input_t fsa_input;
     enum crmd_fsa_input result = I_NULL;
 
     fsa_input.msg = input;
     CRM_CHECK(cause == C_IPC_MESSAGE || cause == C_HA_MESSAGE, return);
 
     /* try passing the buck first */
     if (relay_message(input, cause == C_IPC_MESSAGE)) {
         return;
     }
 
     /* handle locally */
     result = handle_message(input, cause);
 
     /* done or process later? */
     switch (result) {
         case I_NULL:
         case I_CIB_OP:
         case I_ROUTER:
         case I_NODE_JOIN:
         case I_JOIN_REQUEST:
         case I_JOIN_RESULT:
             break;
         default:
             /* Defering local processing of message */
             register_fsa_input_later(cause, result, &fsa_input);
             return;
     }
 
     if (result != I_NULL) {
         /* add to the front of the queue */
         register_fsa_input(cause, result, &fsa_input);
     }
 }
 
 gboolean
 relay_message(xmlNode * msg, gboolean originated_locally)
 {
     int dest = 1;
     int is_for_dc = 0;
     int is_for_dcib = 0;
     int is_for_te = 0;
     int is_for_crm = 0;
     int is_for_cib = 0;
     int is_local = 0;
     gboolean processing_complete = FALSE;
     const char *host_to = crm_element_value(msg, F_CRM_HOST_TO);
     const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
     const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
     const char *type = crm_element_value(msg, F_TYPE);
     const char *msg_error = NULL;
 
     crm_trace("Routing message %s", crm_element_value(msg, XML_ATTR_REFERENCE));
 
     if (msg == NULL) {
         msg_error = "Cannot route empty message";
 
     } else if (safe_str_eq(CRM_OP_HELLO, crm_element_value(msg, F_CRM_TASK))) {
         /* quietly ignore */
         processing_complete = TRUE;
 
     } else if (safe_str_neq(type, T_CRM)) {
         msg_error = "Bad message type";
 
     } else if (sys_to == NULL) {
         msg_error = "Bad message destination: no subsystem";
     }
 
     if (msg_error != NULL) {
         processing_complete = TRUE;
         crm_err("%s", msg_error);
         crm_log_xml_warn(msg, "bad msg");
     }
 
     if (processing_complete) {
         return TRUE;
     }
 
     processing_complete = TRUE;
 
     is_for_dc = (strcasecmp(CRM_SYSTEM_DC, sys_to) == 0);
     is_for_dcib = (strcasecmp(CRM_SYSTEM_DCIB, sys_to) == 0);
     is_for_te = (strcasecmp(CRM_SYSTEM_TENGINE, sys_to) == 0);
     is_for_cib = (strcasecmp(CRM_SYSTEM_CIB, sys_to) == 0);
     is_for_crm = (strcasecmp(CRM_SYSTEM_CRMD, sys_to) == 0);
 
     is_local = 0;
     if (host_to == NULL || strlen(host_to) == 0) {
         if (is_for_dc || is_for_te) {
             is_local = 0;
 
         } else if (is_for_crm && originated_locally) {
             is_local = 0;
 
         } else {
             is_local = 1;
         }
 
     } else if (safe_str_eq(fsa_our_uname, host_to)) {
         is_local = 1;
     }
 
     if (is_for_dc || is_for_dcib || is_for_te) {
         if (AM_I_DC && is_for_te) {
             ROUTER_RESULT("Message result: Local relay");
             send_msg_via_ipc(msg, sys_to);
 
         } else if (AM_I_DC) {
             ROUTER_RESULT("Message result: DC/CRMd process");
             processing_complete = FALSE;        /* more to be done by caller */
         } else if (originated_locally && safe_str_neq(sys_from, CRM_SYSTEM_PENGINE)
                    && safe_str_neq(sys_from, CRM_SYSTEM_TENGINE)) {
 
             /* Neither the TE or PE should be sending messages
              *   to DC's on other nodes
              *
              * By definition, if we are no longer the DC, then
              *   the PE or TE's data should be discarded
              */
 
 #if SUPPORT_COROSYNC
             if (is_openais_cluster()) {
                 dest = text2msg_type(sys_to);
             }
 #endif
             ROUTER_RESULT("Message result: External relay to DC");
             send_cluster_message(host_to ? crm_get_peer(0, host_to) : NULL, dest, msg, TRUE);
 
         } else {
             /* discard */
             ROUTER_RESULT("Message result: Discard, not DC");
         }
 
     } else if (is_local && (is_for_crm || is_for_cib)) {
         ROUTER_RESULT("Message result: CRMd process");
         processing_complete = FALSE;    /* more to be done by caller */
 
     } else if (is_local) {
         ROUTER_RESULT("Message result: Local relay");
         send_msg_via_ipc(msg, sys_to);
 
     } else {
         crm_node_t *node_to = NULL;
 
 #if SUPPORT_COROSYNC
         if (is_openais_cluster()) {
             dest = text2msg_type(sys_to);
 
             if (dest == crm_msg_none || dest > crm_msg_stonith_ng) {
                 dest = crm_msg_crmd;
             }
         }
 #endif
 
         if (host_to) {
             node_to = crm_find_peer(0, host_to);
             if (node_to == NULL) {
                crm_err("Cannot route message to unknown node %s", host_to);
                return TRUE;
             }
         }
 
         ROUTER_RESULT("Message result: External relay");
         send_cluster_message(host_to ? node_to : NULL, dest, msg, TRUE);
     }
 
     return processing_complete;
 }
 
 static gboolean
 process_hello_message(xmlNode * hello,
                       char **client_name, char **major_version, char **minor_version)
 {
     const char *local_client_name;
     const char *local_major_version;
     const char *local_minor_version;
 
     *client_name = NULL;
     *major_version = NULL;
     *minor_version = NULL;
 
     if (hello == NULL) {
         return FALSE;
     }
 
     local_client_name = crm_element_value(hello, "client_name");
     local_major_version = crm_element_value(hello, "major_version");
     local_minor_version = crm_element_value(hello, "minor_version");
 
     if (local_client_name == NULL || strlen(local_client_name) == 0) {
         crm_err("Hello message was not valid (field %s not found)", "client name");
         return FALSE;
 
     } else if (local_major_version == NULL || strlen(local_major_version) == 0) {
         crm_err("Hello message was not valid (field %s not found)", "major version");
         return FALSE;
 
     } else if (local_minor_version == NULL || strlen(local_minor_version) == 0) {
         crm_err("Hello message was not valid (field %s not found)", "minor version");
         return FALSE;
     }
 
     *client_name = strdup(local_client_name);
     *major_version = strdup(local_major_version);
     *minor_version = strdup(local_minor_version);
 
     crm_trace("Hello message ok");
     return TRUE;
 }
 
 gboolean
 crmd_authorize_message(xmlNode * client_msg, crm_client_t * curr_client, const char *proxy_session)
 {
     char *client_name = NULL;
     char *major_version = NULL;
     char *minor_version = NULL;
     gboolean auth_result = FALSE;
 
     xmlNode *xml = NULL;
     const char *op = crm_element_value(client_msg, F_CRM_TASK);
     const char *uuid = curr_client ? curr_client->id : proxy_session;
 
     if (uuid == NULL) {
         crm_warn("Message [%s] not authorized", crm_element_value(client_msg, XML_ATTR_REFERENCE));
         return FALSE;
 
     } else if (safe_str_neq(CRM_OP_HELLO, op)) {
         return TRUE;
     }
 
     xml = get_message_xml(client_msg, F_CRM_DATA);
     auth_result = process_hello_message(xml, &client_name, &major_version, &minor_version);
 
     if (auth_result == TRUE) {
         if (client_name == NULL) {
             crm_err("Bad client details (client_name=%s, uuid=%s)",
                     crm_str(client_name), uuid);
             auth_result = FALSE;
         }
     }
 
     if (auth_result == TRUE) {
         /* check version */
         int mav = atoi(major_version);
         int miv = atoi(minor_version);
 
         crm_trace("Checking client version number");
         if (mav < 0 || miv < 0) {
             crm_err("Client version (%d:%d) is not acceptable", mav, miv);
             auth_result = FALSE;
         }
     }
 
     if (auth_result == TRUE) {
         crm_trace("Accepted client %s", client_name);
         if (curr_client) {
             curr_client->userdata = strdup(client_name);
         }
 
         crm_trace("Triggering FSA: %s", __FUNCTION__);
         mainloop_set_trigger(fsa_source);
 
     } else {
         crm_warn("Rejected client logon request");
         if (curr_client) {
             qb_ipcs_disconnect(curr_client->ipcs);
         }
     }
 
     free(minor_version);
     free(major_version);
     free(client_name);
 
     /* hello messages should never be processed further */
     return FALSE;
 }
 
 enum crmd_fsa_input
 handle_message(xmlNode * msg, enum crmd_fsa_cause cause)
 {
     const char *type = NULL;
 
     CRM_CHECK(msg != NULL, return I_NULL);
 
     type = crm_element_value(msg, F_CRM_MSG_TYPE);
     if (crm_str_eq(type, XML_ATTR_REQUEST, TRUE)) {
         return handle_request(msg, cause);
 
     } else if (crm_str_eq(type, XML_ATTR_RESPONSE, TRUE)) {
         handle_response(msg);
         return I_NULL;
     }
 
     crm_err("Unknown message type: %s", type);
     return I_NULL;
 }
 
 static enum crmd_fsa_input
 handle_failcount_op(xmlNode * stored_msg)
 {
     const char *rsc = NULL;
     const char *uname = NULL;
     gboolean is_remote_node = FALSE;
     xmlNode *xml_rsc = get_xpath_object("//" XML_CIB_TAG_RESOURCE, stored_msg, LOG_ERR);
 
     if (xml_rsc) {
         rsc = ID(xml_rsc);
     }
 
     uname = crm_element_value(stored_msg, XML_LRM_ATTR_TARGET);
     if (crm_element_value(stored_msg, XML_LRM_ATTR_ROUTER_NODE)) {
         is_remote_node = TRUE;
     }
 
     if (rsc) {
         char *attr = NULL;
 
         crm_info("Removing failcount for %s", rsc);
 
-        attr = crm_concat("fail-count", rsc, '-');
+        attr = crm_failcount_name(rsc);
         update_attrd(uname, attr, NULL, NULL, is_remote_node);
         free(attr);
 
-        attr = crm_concat("last-failure", rsc, '-');
+        attr = crm_lastfailure_name(rsc);
         update_attrd(uname, attr, NULL, NULL, is_remote_node);
         free(attr);
 
         lrm_clear_last_failure(rsc, uname);
     } else {
         crm_log_xml_warn(stored_msg, "invalid failcount op");
     }
 
     return I_NULL;
 }
 
 /*!
  * \brief Handle a CRM_OP_REMOTE_STATE message by updating remote peer cache
  *
  * \param[in] msg  Message XML
  *
  * \return Next FSA input
  */
 static enum crmd_fsa_input
 handle_remote_state(xmlNode *msg)
 {
     const char *remote_uname = ID(msg);
     const char *remote_is_up = crm_element_value(msg, XML_NODE_IN_CLUSTER);
     crm_node_t *remote_peer;
 
     CRM_CHECK(remote_uname && remote_is_up, return I_NULL);
 
     remote_peer = crm_remote_peer_get(remote_uname);
     CRM_CHECK(remote_peer, return I_NULL);
 
     crm_update_peer_state(__FUNCTION__, remote_peer,
                           crm_is_true(remote_is_up)?
                           CRM_NODE_MEMBER : CRM_NODE_LOST, 0);
     return I_NULL;
 }
 
 enum crmd_fsa_input
 handle_request(xmlNode * stored_msg, enum crmd_fsa_cause cause)
 {
     xmlNode *msg = NULL;
     const char *op = crm_element_value(stored_msg, F_CRM_TASK);
 
     /* Optimize this for the DC - it has the most to do */
 
     if (op == NULL) {
         crm_log_xml_err(stored_msg, "Bad message");
         return I_NULL;
     }
 
     if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
         const char *from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
         crm_node_t *node = crm_find_peer(0, from);
 
         crm_update_peer_expected(__FUNCTION__, node, CRMD_JOINSTATE_DOWN);
         if(AM_I_DC == FALSE) {
             return I_NULL; /* Done */
         }
     }
 
     /*========== DC-Only Actions ==========*/
     if (AM_I_DC) {
         if (strcmp(op, CRM_OP_JOIN_ANNOUNCE) == 0) {
             return I_NODE_JOIN;
 
         } else if (strcmp(op, CRM_OP_JOIN_REQUEST) == 0) {
             return I_JOIN_REQUEST;
 
         } else if (strcmp(op, CRM_OP_JOIN_CONFIRM) == 0) {
             return I_JOIN_RESULT;
 
         } else if (strcmp(op, CRM_OP_SHUTDOWN) == 0) {
             const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
             gboolean dc_match = safe_str_eq(host_from, fsa_our_dc);
 
             if (is_set(fsa_input_register, R_SHUTDOWN)) {
                 crm_info("Shutting ourselves down (DC)");
                 return I_STOP;
 
             } else if (dc_match) {
                 crm_err("We didn't ask to be shut down, yet our"
                         " TE is telling us to. Better get out now!");
                 return I_TERMINATE;
 
             } else if (fsa_state != S_STOPPING) {
                 crm_err("Another node is asking us to shutdown" " but we think we're ok.");
                 return I_ELECTION;
             }
 
         } else if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
             /* a slave wants to shut down */
             /* create cib fragment and add to message */
             return handle_shutdown_request(stored_msg);
 
         } else if (strcmp(op, CRM_OP_REMOTE_STATE) == 0) {
             /* a remote connection host is letting us know the node state */
             return handle_remote_state(stored_msg);
         }
     }
 
     /*========== common actions ==========*/
     if (strcmp(op, CRM_OP_NOVOTE) == 0) {
         ha_msg_input_t fsa_input;
 
         fsa_input.msg = stored_msg;
         register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
                                A_ELECTION_COUNT | A_ELECTION_CHECK, FALSE, __FUNCTION__);
 
     } else if (strcmp(op, CRM_OP_THROTTLE) == 0) {
         throttle_update(stored_msg);
         if (AM_I_DC && transition_graph != NULL) {
             if (transition_graph->complete == FALSE) {
                 crm_debug("The throttle changed. Trigger a graph.");
                 trigger_graph();
             }
         }
         return I_NULL;
 
     } else if (strcmp(op, CRM_OP_CLEAR_FAILCOUNT) == 0) {
         return handle_failcount_op(stored_msg);
 
     } else if (strcmp(op, CRM_OP_VOTE) == 0) {
         /* count the vote and decide what to do after that */
         ha_msg_input_t fsa_input;
 
         fsa_input.msg = stored_msg;
         register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
                                A_ELECTION_COUNT | A_ELECTION_CHECK, FALSE, __FUNCTION__);
 
         /* Sometimes we _must_ go into S_ELECTION */
         if (fsa_state == S_HALT) {
             crm_debug("Forcing an election from S_HALT");
             return I_ELECTION;
 #if 0
         } else if (AM_I_DC) {
             /* This is the old way of doing things but what is gained? */
             return I_ELECTION;
 #endif
         }
 
     } else if (strcmp(op, CRM_OP_JOIN_OFFER) == 0) {
         crm_debug("Raising I_JOIN_OFFER: join-%s", crm_element_value(stored_msg, F_CRM_JOIN_ID));
         return I_JOIN_OFFER;
 
     } else if (strcmp(op, CRM_OP_JOIN_ACKNAK) == 0) {
         crm_debug("Raising I_JOIN_RESULT: join-%s", crm_element_value(stored_msg, F_CRM_JOIN_ID));
         return I_JOIN_RESULT;
 
     } else if (strcmp(op, CRM_OP_LRM_DELETE) == 0
                || strcmp(op, CRM_OP_LRM_FAIL) == 0
                || strcmp(op, CRM_OP_LRM_REFRESH) == 0 || strcmp(op, CRM_OP_REPROBE) == 0) {
 
         crm_xml_add(stored_msg, F_CRM_SYS_TO, CRM_SYSTEM_LRMD);
         return I_ROUTER;
 
     } else if (strcmp(op, CRM_OP_NOOP) == 0) {
         return I_NULL;
 
     } else if (strcmp(op, CRM_OP_LOCAL_SHUTDOWN) == 0) {
 
         crm_shutdown(SIGTERM);
         /*return I_SHUTDOWN; */
         return I_NULL;
 
         /*========== (NOT_DC)-Only Actions ==========*/
     } else if (AM_I_DC == FALSE && strcmp(op, CRM_OP_SHUTDOWN) == 0) {
 
         const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
         gboolean dc_match = safe_str_eq(host_from, fsa_our_dc);
 
         if (dc_match || fsa_our_dc == NULL) {
             if (is_set(fsa_input_register, R_SHUTDOWN) == FALSE) {
                 crm_err("We didn't ask to be shut down, yet our DC is telling us to.");
                 set_bit(fsa_input_register, R_STAYDOWN);
                 return I_STOP;
             }
             crm_info("Shutting down");
             return I_STOP;
 
         } else {
             crm_warn("Discarding %s op from %s", op, host_from);
         }
 
     } else if (strcmp(op, CRM_OP_PING) == 0) {
         /* eventually do some stuff to figure out
          * if we /are/ ok
          */
         const char *sys_to = crm_element_value(stored_msg, F_CRM_SYS_TO);
         xmlNode *ping = create_xml_node(NULL, XML_CRM_TAG_PING);
 
         crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok");
         crm_xml_add(ping, XML_PING_ATTR_SYSFROM, sys_to);
         crm_xml_add(ping, "crmd_state", fsa_state2string(fsa_state));
 
         /* Ok, so technically not so interesting, but CTS needs to see this */
         crm_notice("Current ping state: %s", fsa_state2string(fsa_state));
 
         msg = create_reply(stored_msg, ping);
         if (msg) {
             (void)relay_message(msg, TRUE);
         }
 
         free_xml(ping);
         free_xml(msg);
 
     } else if (strcmp(op, CRM_OP_RM_NODE_CACHE) == 0) {
         int id = 0;
         const char *name = NULL;
 
         crm_element_value_int(stored_msg, XML_ATTR_ID, &id);
         name = crm_element_value(stored_msg, XML_ATTR_UNAME);
 
         if(cause == C_IPC_MESSAGE) {
             msg = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
             if (send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) {
                 crm_err("Could not instruct peers to remove references to node %s/%u", name, id);
             } else {
                 crm_notice("Instructing peers to remove references to node %s/%u", name, id);
             }
             free_xml(msg);
 
         } else {
             reap_crm_member(id, name);
         }
 
     } else {
         crm_err("Unexpected request (%s) sent to %s", op, AM_I_DC ? "the DC" : "non-DC node");
         crm_log_xml_err(stored_msg, "Unexpected");
     }
 
     return I_NULL;
 }
 
 void
 handle_response(xmlNode * stored_msg)
 {
     const char *op = crm_element_value(stored_msg, F_CRM_TASK);
 
     if (op == NULL) {
         crm_log_xml_err(stored_msg, "Bad message");
 
     } else if (AM_I_DC && strcmp(op, CRM_OP_PECALC) == 0) {
         /* Check if the PE answer been superseded by a subsequent request? */
         const char *msg_ref = crm_element_value(stored_msg, XML_ATTR_REFERENCE);
 
         if (msg_ref == NULL) {
             crm_err("%s - Ignoring calculation with no reference", op);
 
         } else if (safe_str_eq(msg_ref, fsa_pe_ref)) {
             ha_msg_input_t fsa_input;
 
             fsa_input.msg = stored_msg;
             register_fsa_input_later(C_IPC_MESSAGE, I_PE_SUCCESS, &fsa_input);
             crm_trace("Completed: %s...", fsa_pe_ref);
 
         } else {
             crm_info("%s calculation %s is obsolete", op, msg_ref);
         }
 
     } else if (strcmp(op, CRM_OP_VOTE) == 0
                || strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0 || strcmp(op, CRM_OP_SHUTDOWN) == 0) {
 
     } else {
         const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
 
         crm_err("Unexpected response (op=%s, src=%s) sent to the %s",
                 op, host_from, AM_I_DC ? "DC" : "CRMd");
     }
 }
 
 enum crmd_fsa_input
 handle_shutdown_request(xmlNode * stored_msg)
 {
     /* handle here to avoid potential version issues
      *   where the shutdown message/procedure may have
      *   been changed in later versions.
      *
      * This way the DC is always in control of the shutdown
      */
 
     char *now_s = NULL;
     time_t now = time(NULL);
     const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
 
     if (host_from == NULL) {
         /* we're shutting down and the DC */
         host_from = fsa_our_uname;
     }
 
     crm_info("Creating shutdown request for %s (state=%s)", host_from, fsa_state2string(fsa_state));
     crm_log_xml_trace(stored_msg, "message");
 
     now_s = crm_itoa(now);
     update_attrd(host_from, XML_CIB_ATTR_SHUTDOWN, now_s, NULL, FALSE);
     free(now_s);
 
     /* will be picked up by the TE as long as its running */
     return I_NULL;
 }
 
 /* msg is deleted by the time this returns */
 extern gboolean process_te_message(xmlNode * msg, xmlNode * xml_data);
 
 gboolean
 send_msg_via_ipc(xmlNode * msg, const char *sys)
 {
     gboolean send_ok = TRUE;
     crm_client_t *client_channel = crm_client_get_by_id(sys);
 
     if (crm_element_value(msg, F_CRM_HOST_FROM) == NULL) {
         crm_xml_add(msg, F_CRM_HOST_FROM, fsa_our_uname);
     }
 
     if (client_channel != NULL) {
         /* Transient clients such as crmadmin */
         send_ok = crm_ipcs_send(client_channel, 0, msg, crm_ipc_server_event);
 
     } else if (sys != NULL && strcmp(sys, CRM_SYSTEM_TENGINE) == 0) {
         xmlNode *data = get_message_xml(msg, F_CRM_DATA);
 
         process_te_message(msg, data);
 
     } else if (sys != NULL && strcmp(sys, CRM_SYSTEM_LRMD) == 0) {
         fsa_data_t fsa_data;
         ha_msg_input_t fsa_input;
 
         fsa_input.msg = msg;
         fsa_input.xml = get_message_xml(msg, F_CRM_DATA);
 
         fsa_data.id = 0;
         fsa_data.actions = 0;
         fsa_data.data = &fsa_input;
         fsa_data.fsa_input = I_MESSAGE;
         fsa_data.fsa_cause = C_IPC_MESSAGE;
         fsa_data.origin = __FUNCTION__;
         fsa_data.data_type = fsa_dt_ha_msg;
 
 #ifdef FSA_TRACE
         crm_trace("Invoking action A_LRM_INVOKE (%.16llx)", A_LRM_INVOKE);
 #endif
         do_lrm_invoke(A_LRM_INVOKE, C_IPC_MESSAGE, fsa_state, I_MESSAGE, &fsa_data);
 
     } else if (sys != NULL && crmd_is_proxy_session(sys)) {
         crmd_proxy_send(sys, msg);
 
     } else {
         crm_debug("Unknown Sub-system (%s)... discarding message.", crm_str(sys));
         send_ok = FALSE;
     }
 
     return send_ok;
 }
 
 ha_msg_input_t *
 new_ha_msg_input(xmlNode * orig)
 {
     ha_msg_input_t *input_copy = NULL;
 
     input_copy = calloc(1, sizeof(ha_msg_input_t));
     input_copy->msg = orig;
     input_copy->xml = get_message_xml(input_copy->msg, F_CRM_DATA);
     return input_copy;
 }
 
 void
 delete_ha_msg_input(ha_msg_input_t * orig)
 {
     if (orig == NULL) {
         return;
     }
     free_xml(orig->msg);
     free(orig);
 }
 
 /*!
  * \internal
  * \brief Notify the DC of a remote node state change
  *
  * \param[in] node_name  Node's name
  * \param[in] node_up    TRUE if node is up, FALSE if down
  */
 void
 send_remote_state_message(const char *node_name, gboolean node_up)
 {
     /* If we don't have a DC, or the message fails, we have a failsafe:
      * the DC will eventually pick up the change via the CIB node state.
      * The message allows it to happen sooner if possible.
      */
     if (fsa_our_dc) {
         xmlNode *msg = create_request(CRM_OP_REMOTE_STATE, NULL, fsa_our_dc,
                                       CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
 
         crm_info("Notifying DC %s of pacemaker_remote node %s %s",
                  fsa_our_dc, node_name, (node_up? "coming up" : "going down"));
         crm_xml_add(msg, XML_ATTR_ID, node_name);
         crm_xml_add_boolean(msg, XML_NODE_IN_CLUSTER, node_up);
         send_cluster_message(crm_get_peer(0, fsa_our_dc), crm_msg_crmd, msg,
                              TRUE);
         free_xml(msg);
     } else {
         crm_debug("No DC to notify of pacemaker_remote node %s %s",
                   node_name, (node_up? "coming up" : "going down"));
     }
 }
 
diff --git a/crmd/te_events.c b/crmd/te_events.c
index e42e63d288..13801e2724 100644
--- a/crmd/te_events.c
+++ b/crmd/te_events.c
@@ -1,535 +1,535 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 
 #include <crm/common/xml.h>
 #include <tengine.h>
 
 #include <crmd_fsa.h>
 
 char *failed_stop_offset = NULL;
 char *failed_start_offset = NULL;
 
 gboolean
 fail_incompletable_actions(crm_graph_t * graph, const char *down_node)
 {
     const char *target_uuid = NULL;
     const char *router = NULL;
     const char *router_uuid = NULL;
     xmlNode *last_action = NULL;
 
     GListPtr gIter = NULL;
     GListPtr gIter2 = NULL;
 
     if (graph == NULL || graph->complete) {
         return FALSE;
     }
 
     gIter = graph->synapses;
     for (; gIter != NULL; gIter = gIter->next) {
         synapse_t *synapse = (synapse_t *) gIter->data;
 
         if (synapse->confirmed || synapse->failed) {
             /* We've already been here */
             continue;
         }
 
         gIter2 = synapse->actions;
         for (; gIter2 != NULL; gIter2 = gIter2->next) {
             crm_action_t *action = (crm_action_t *) gIter2->data;
 
             if (action->type == action_type_pseudo || action->confirmed) {
                 continue;
             } else if (action->type == action_type_crm) {
                 const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 
                 if (safe_str_eq(task, CRM_OP_FENCE)) {
                     continue;
                 }
             }
 
             target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
             router = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
             if (router) {
                 crm_node_t *node = crm_get_peer(0, router);
                 if (node) {
                     router_uuid = node->uuid;
                 }
             }
 
             if (safe_str_eq(target_uuid, down_node) || safe_str_eq(router_uuid, down_node)) {
                 action->failed = TRUE;
                 synapse->failed = TRUE;
                 last_action = action->xml;
                 stop_te_timer(action->timer);
                 update_graph(graph, action);
 
                 if (synapse->executed) {
                     crm_notice("Action %d (%s) was pending on %s (offline)",
                                action->id, crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY), down_node);
                 } else {
                     crm_info("Action %d (%s) is scheduled for %s (offline)",
                              action->id, crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY), down_node);
                 }
             }
         }
     }
 
     if (last_action != NULL) {
         crm_info("Node %s shutdown resulted in un-runnable actions", down_node);
         abort_transition(INFINITY, tg_restart, "Node failure", last_action);
         return TRUE;
     }
 
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Update failure-related node attributes if warranted
  *
  * \param[in] event            XML describing operation that (maybe) failed
  * \param[in] event_node_uuid  Node that event occurred on
  * \param[in] rc               Actual operation return code
  * \param[in] target_rc        Expected operation return code
  * \param[in] do_update        If TRUE, do update regardless of operation type
  * \param[in] ignore_failures  If TRUE, update last failure but not fail count
  *
  * \return TRUE if this was not a direct nack, success or lrm status refresh
  */
 static gboolean
 update_failcount(xmlNode * event, const char *event_node_uuid, int rc,
                  int target_rc, gboolean do_update, gboolean ignore_failures)
 {
     int interval = 0;
 
     char *task = NULL;
     char *rsc_id = NULL;
 
     const char *value = NULL;
     const char *id = crm_element_value(event, XML_LRM_ATTR_TASK_KEY);
     const char *on_uname = crm_peer_uname(event_node_uuid);
     const char *origin = crm_element_value(event, XML_ATTR_ORIGIN);
 
     /* Nothing needs to be done for success, lrm status refresh,
      * or direct nack (internal code for "busy, try again")
      */
     if ((rc == CRM_DIRECT_NACK_RC) || (rc == target_rc)) {
         return FALSE;
     } else if (safe_str_eq(origin, "build_active_RAs")) {
         crm_debug("No update for %s (rc=%d) on %s: Old failure from lrm status refresh",
                   id, rc, on_uname);
         return FALSE;
     }
 
     /* Sanity check */
     CRM_CHECK(on_uname != NULL, return TRUE);
     CRM_CHECK(parse_op_key(id, &rsc_id, &task, &interval),
               crm_err("Couldn't parse: %s", ID(event)); goto bail);
     CRM_CHECK(task != NULL, goto bail);
     CRM_CHECK(rsc_id != NULL, goto bail);
 
     /* Decide whether update is necessary and what value to use */
     if ((interval > 0) || safe_str_eq(task, CRMD_ACTION_PROMOTE)
         || safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
         do_update = TRUE;
 
     } else if (safe_str_eq(task, CRMD_ACTION_START)) {
         do_update = TRUE;
         if (failed_start_offset == NULL) {
             failed_start_offset = strdup(INFINITY_S);
         }
         value = failed_start_offset;
 
     } else if (safe_str_eq(task, CRMD_ACTION_STOP)) {
         do_update = TRUE;
         if (failed_stop_offset == NULL) {
             failed_stop_offset = strdup(INFINITY_S);
         }
         value = failed_stop_offset;
     }
 
     /* Fail count will be either incremented or set to infinity */
     if (value == NULL || safe_str_neq(value, INFINITY_S)) {
         value = XML_NVPAIR_ATTR_VALUE "++";
     }
 
     if (do_update) {
         char *now = crm_itoa(time(NULL));
         char *attr_name = NULL;
         gboolean is_remote_node = FALSE;
 
         if (g_hash_table_lookup(crm_remote_peer_cache, event_node_uuid)) {
             is_remote_node = TRUE;
         }
 
         crm_info("Updating %s for %s on %s after failed %s: rc=%d (update=%s, time=%s)",
                  (ignore_failures? "last failure" : "failcount"),
                  rsc_id, on_uname, task, rc, value, now);
 
         /* Update the fail count, if we're not ignoring failures */
         if (!ignore_failures) {
-            attr_name = crm_concat("fail-count", rsc_id, '-');
+            attr_name = crm_failcount_name(rsc_id);
             update_attrd(on_uname, attr_name, value, NULL, is_remote_node);
             free(attr_name);
         }
 
         /* Update the last failure time (even if we're ignoring failures,
          * so that failure can still be detected and shown, e.g. by crm_mon)
          */
-        attr_name = crm_concat("last-failure", rsc_id, '-');
+        attr_name = crm_lastfailure_name(rsc_id);
         update_attrd(on_uname, attr_name, now, NULL, is_remote_node);
         free(attr_name);
 
         free(now);
     }
 
   bail:
     free(rsc_id);
     free(task);
     return TRUE;
 }
 
 /*!
  * \internal
  * \brief Return simplified operation status based on operation return code
  *
  * \param[in] action       CRM action instance of operation
  * \param[in] orig_status  Original reported operation status
  * \param[in] rc           Actual operation return code
  * \param[in] target_rc    Expected operation return code
  *
  * \return PCMK_LRM_OP_DONE if rc equals target_rc, PCMK_LRM_OP_ERROR otherwise
  *
  * \note This assumes that PCMK_LRM_OP_PENDING operations have already been
  *       filtered (otherwise they will get simplified as well).
  */
 static int
 status_from_rc(crm_action_t * action, int orig_status, int rc, int target_rc)
 {
     if (target_rc == rc) {
         crm_trace("Target rc: == %d", rc);
         if (orig_status != PCMK_LRM_OP_DONE) {
             crm_trace("Re-mapping op status to PCMK_LRM_OP_DONE for rc=%d", rc);
         }
         return PCMK_LRM_OP_DONE;
     }
 
     if (rc != CRM_DIRECT_NACK_RC) {
         const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
         const char *uname = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 
         crm_warn("Action %d (%s) on %s failed (target: %d vs. rc: %d): %s",
                  action->id, task, uname, target_rc, rc,
                  services_lrm_status_str(PCMK_LRM_OP_ERROR));
     }
     return PCMK_LRM_OP_ERROR;
 }
 
 /*!
  * \internal
  * \brief Confirm action and update transition graph, aborting transition on failures
  *
  * \param[in/out] action           CRM action instance of this operation
  * \param[in]     event            Event instance of this operation
  * \param[in]     orig_status      Original reported operation status
  * \param[in]     op_rc            Actual operation return code
  * \param[in]     target_rc        Expected operation return code
  * \param[in]     ignore_failures  Whether to ignore operation failures
  *
  * \note This assumes that PCMK_LRM_OP_PENDING operations have already been
  *       filtered (otherwise they may be treated as failures).
  */
 static void
 match_graph_event(crm_action_t *action, xmlNode *event, int op_status,
                   int op_rc, int target_rc, gboolean ignore_failures)
 {
     const char *target = NULL;
     const char *this_event = NULL;
     const char *ignore_s = "";
 
     /* Remap operation status based on return code */
     op_status = status_from_rc(action, op_status, op_rc, target_rc);
 
     /* Process OP status */
     switch (op_status) {
         case PCMK_LRM_OP_DONE:
             break;
         case PCMK_LRM_OP_ERROR:
         case PCMK_LRM_OP_TIMEOUT:
         case PCMK_LRM_OP_NOTSUPPORTED:
             if (ignore_failures) {
                 ignore_s = ", ignoring failure";
             } else {
                 action->failed = TRUE;
             }
             break;
         case PCMK_LRM_OP_CANCELLED:
             /* do nothing?? */
             crm_err("Don't know what to do for cancelled ops yet");
             break;
         default:
             /*
              PCMK_LRM_OP_ERROR_HARD,
              PCMK_LRM_OP_ERROR_FATAL,
              PCMK_LRM_OP_NOT_INSTALLED
              */
             action->failed = TRUE;
             crm_err("Unsupported action result: %d", op_status);
     }
 
     /* stop this event's timer if it had one */
     stop_te_timer(action->timer);
     te_action_confirmed(action);
 
     update_graph(transition_graph, action);
     trigger_graph();
 
     if (action->failed) {
         abort_transition(action->synapse->priority + 1, tg_restart, "Event failed", event);
     }
 
     this_event = crm_element_value(event, XML_LRM_ATTR_TASK_KEY);
     target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     crm_info("Action %s (%d) confirmed on %s (rc=%d%s)",
              crm_str(this_event), action->id, crm_str(target), op_rc, ignore_s);
 }
 
 crm_action_t *
 get_action(int id, gboolean confirmed)
 {
     GListPtr gIter = NULL;
     GListPtr gIter2 = NULL;
 
     gIter = transition_graph->synapses;
     for (; gIter != NULL; gIter = gIter->next) {
         synapse_t *synapse = (synapse_t *) gIter->data;
 
         gIter2 = synapse->actions;
         for (; gIter2 != NULL; gIter2 = gIter2->next) {
             crm_action_t *action = (crm_action_t *) gIter2->data;
 
             if (action->id == id) {
                 if (confirmed) {
                     stop_te_timer(action->timer);
                     te_action_confirmed(action);
                 }
                 return action;
             }
         }
     }
 
     return NULL;
 }
 
 crm_action_t *
 get_cancel_action(const char *id, const char *node)
 {
     GListPtr gIter = NULL;
     GListPtr gIter2 = NULL;
 
     gIter = transition_graph->synapses;
     for (; gIter != NULL; gIter = gIter->next) {
         synapse_t *synapse = (synapse_t *) gIter->data;
 
         gIter2 = synapse->actions;
         for (; gIter2 != NULL; gIter2 = gIter2->next) {
             const char *task = NULL;
             const char *target = NULL;
             crm_action_t *action = (crm_action_t *) gIter2->data;
 
             task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
             if (safe_str_neq(CRMD_ACTION_CANCEL, task)) {
                 continue;
             }
 
             task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
             if (safe_str_neq(task, id)) {
                 crm_trace("Wrong key %s for %s on %s", task, id, node);
                 continue;
             }
 
             target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
             if (node && safe_str_neq(target, node)) {
                 crm_trace("Wrong node %s for %s on %s", target, id, node);
                 continue;
             }
 
             crm_trace("Found %s on %s", id, node);
             return action;
         }
     }
 
     return NULL;
 }
 
 /* downed nodes are listed like: <downed> <node id="UUID1" /> ... </downed> */
 #define XPATH_DOWNED "//" XML_GRAPH_TAG_DOWNED \
                      "/" XML_CIB_TAG_NODE "[@" XML_ATTR_UUID "='%s']"
 
 /*!
  * \brief Find a transition event that would have made a specified node down
  *
  * \param[in] target  UUID of node to match
  * \param[in] quiet   If FALSE, log a warning if no match found
  *
  * \return Matching event if found, NULL otherwise
  */
 crm_action_t *
 match_down_event(const char *target, bool quiet)
 {
     crm_action_t *match = NULL;
     xmlXPathObjectPtr xpath_ret = NULL;
     GListPtr gIter, gIter2;
 
     char *xpath = crm_strdup_printf(XPATH_DOWNED, target);
 
     for (gIter = transition_graph->synapses;
          gIter != NULL && match == NULL;
          gIter = gIter->next) {
 
         for (gIter2 = ((synapse_t*)gIter->data)->actions;
              gIter2 != NULL && match == NULL;
              gIter2 = gIter2->next) {
 
             match = (crm_action_t*)gIter2->data;
             xpath_ret = xpath_search(match->xml, xpath);
             if (numXpathResults(xpath_ret) < 1) {
                 match = NULL;
             }
             freeXpathObject(xpath_ret);
         }
     }
 
     free(xpath);
 
     if (match != NULL) {
         crm_debug("Shutdown action found for node %s: action %d (%s)",
                   target, match->id,
                   crm_element_value(match->xml, XML_LRM_ATTR_TASK_KEY));
 
     } else if(quiet == FALSE) {
         crm_warn("No reason to expect node %s to be down", target);
     }
 
     return match;
 }
 
 gboolean
 process_graph_event(xmlNode * event, const char *event_node)
 {
     int rc = -1;
     int status = -1;
     int callid = -1;
 
     int action_num = -1;
     crm_action_t *action = NULL;
 
     int target_rc = -1;
     int transition_num = -1;
     char *update_te_uuid = NULL;
 
     gboolean stop_early = FALSE;
     gboolean ignore_failures = FALSE;
     const char *id = NULL;
     const char *desc = NULL;
     const char *magic = NULL;
 
     CRM_ASSERT(event != NULL);
 
 /*
 <lrm_rsc_op id="rsc_east-05_last_0" operation_key="rsc_east-05_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="9:2:7:be2e97d9-05e2-439d-863e-48f7aecab2aa" transition-magic="0:7;9:2:7:be2e97d9-05e2-439d-863e-48f7aecab2aa" call-id="17" rc-code="7" op-status="0" interval="0" last-run="1355361636" last-rc-change="1355361636" exec-time="128" queue-time="0" op-digest="c81f5f40b1c9e859c992e800b1aa6972"/>
 */
 
     id = crm_element_value(event, XML_LRM_ATTR_TASK_KEY);
     crm_element_value_int(event, XML_LRM_ATTR_RC, &rc);
     crm_element_value_int(event, XML_LRM_ATTR_OPSTATUS, &status);
     crm_element_value_int(event, XML_LRM_ATTR_CALLID, &callid);
 
     magic = crm_element_value(event, XML_ATTR_TRANSITION_KEY);
     if (magic == NULL) {
         /* non-change */
         return FALSE;
     }
 
     if (decode_transition_key(magic, &update_te_uuid, &transition_num,
                               &action_num, &target_rc) == FALSE) {
         crm_err("Invalid event %s.%d detected: %s", id, callid, magic);
         abort_transition(INFINITY, tg_restart, "Bad event", event);
         return FALSE;
     }
 
     if (status == PCMK_LRM_OP_PENDING) {
         goto bail;
     }
 
     if (transition_num == -1) {
         desc = "initiated outside of the cluster";
         abort_transition(INFINITY, tg_restart, "Unexpected event", event);
 
     } else if ((action_num < 0) || (crm_str_eq(update_te_uuid, te_uuid, TRUE) == FALSE)) {
         desc = "initiated by a different node";
         abort_transition(INFINITY, tg_restart, "Foreign event", event);
         stop_early = TRUE;      /* This could be an lrm status refresh */
 
     } else if (transition_graph->id != transition_num) {
         desc = "arrived really late";
         abort_transition(INFINITY, tg_restart, "Old event", event);
         stop_early = TRUE;      /* This could be an lrm status refresh */
 
     } else if (transition_graph->complete) {
         desc = "arrived late";
         abort_transition(INFINITY, tg_restart, "Inactive graph", event);
 
     } else {
         action = get_action(action_num, FALSE);
 
         if (action == NULL) {
             desc = "unknown";
             abort_transition(INFINITY, tg_restart, "Unknown event", event);
 
         } else {
             ignore_failures = safe_str_eq(
                 crm_meta_value(action->params, XML_OP_ATTR_ON_FAIL), "ignore");
             match_graph_event(action, event, status, rc, target_rc, ignore_failures);
         }
     }
 
     if (action && (rc == target_rc)) {
         crm_trace("Processed update to %s: %s", id, magic);
     } else {
         if (update_failcount(event, event_node, rc, target_rc,
                              (transition_num == -1), ignore_failures)) {
             /* Turns out this wasn't an lrm status refresh update afterall */
             stop_early = FALSE;
             desc = "failed";
         }
         crm_info("Detected action (%d.%d) %s.%d=%s: %s", transition_num,
                  action_num, id, callid, services_ocf_exitcode_str(rc), desc);
     }
 
   bail:
     free(update_te_uuid);
     return stop_early;
 }
diff --git a/lib/pengine/failcounts.c b/lib/pengine/failcounts.c
index 2fd10faf18..6579228ecd 100644
--- a/lib/pengine/failcounts.c
+++ b/lib/pengine/failcounts.c
@@ -1,304 +1,303 @@
 /*
  * Copyright (C) 2008-2017 Andrew Beekhof <andrew@beekhof.net>
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <glib.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/util.h>
 #include <crm/pengine/internal.h>
 
 struct fail_search {
     resource_t *rsc;
     pe_working_set_t *data_set;
     int count;
     long long last;
     char *key;
 };
 
 static void
 get_failcount_by_prefix(gpointer key_p, gpointer value, gpointer user_data)
 {
     struct fail_search *search = user_data;
     const char *attr_id = key_p;
     const char *match = strstr(attr_id, search->key);
     resource_t *parent = NULL;
 
     if (match == NULL) {
         return;
     }
 
     /* we are only incrementing the failcounts here if the rsc
      * that matches our prefix has the same uber parent as the rsc we're
      * calculating the failcounts for. This prevents false positive matches
      * where unrelated resources may have similar prefixes in their names.
      *
      * search->rsc is already set to be the uber parent. */
     parent = uber_parent(pe_find_resource(search->data_set->resources, match));
     if (parent == NULL || parent != search->rsc) {
         return;
     }
-    if (strstr(attr_id, "last-failure-") == attr_id) {
+    if (strstr(attr_id, CRM_LAST_FAILURE_PREFIX "-") == attr_id) {
         search->last = crm_int_helper(value, NULL);
 
-    } else if (strstr(attr_id, "fail-count-") == attr_id) {
+    } else if (strstr(attr_id, CRM_FAIL_COUNT_PREFIX "-") == attr_id) {
         search->count += char2score(value);
     }
 }
 
 int
 get_failcount(node_t *node, resource_t *rsc, time_t *last_failure,
               pe_working_set_t *data_set)
 {
     return get_failcount_full(node, rsc, last_failure, TRUE, NULL, data_set);
 }
 
 static gboolean
 is_matched_failure(const char *rsc_id, xmlNode *conf_op_xml,
                    xmlNode *lrm_op_xml)
 {
     gboolean matched = FALSE;
     const char *conf_op_name = NULL;
     int conf_op_interval = 0;
     const char *lrm_op_task = NULL;
     int lrm_op_interval = 0;
     const char *lrm_op_id = NULL;
     char *last_failure_key = NULL;
 
     if (rsc_id == NULL || conf_op_xml == NULL || lrm_op_xml == NULL) {
         return FALSE;
     }
 
     conf_op_name = crm_element_value(conf_op_xml, "name");
     conf_op_interval = crm_get_msec(crm_element_value(conf_op_xml, "interval"));
     lrm_op_task = crm_element_value(lrm_op_xml, XML_LRM_ATTR_TASK);
     crm_element_value_int(lrm_op_xml, XML_LRM_ATTR_INTERVAL, &lrm_op_interval);
 
     if (safe_str_eq(conf_op_name, lrm_op_task) == FALSE
         || conf_op_interval != lrm_op_interval) {
         return FALSE;
     }
 
     lrm_op_id = ID(lrm_op_xml);
     last_failure_key = generate_op_key(rsc_id, "last_failure", 0);
 
     if (safe_str_eq(last_failure_key, lrm_op_id)) {
         matched = TRUE;
 
     } else {
         char *expected_op_key = generate_op_key(rsc_id, conf_op_name,
                                                 conf_op_interval);
 
         if (safe_str_eq(expected_op_key, lrm_op_id)) {
             int rc = 0;
             int target_rc = get_target_rc(lrm_op_xml);
 
             crm_element_value_int(lrm_op_xml, XML_LRM_ATTR_RC, &rc);
             if (rc != target_rc) {
                 matched = TRUE;
             }
         }
         free(expected_op_key);
     }
 
     free(last_failure_key);
     return matched;
 }
 
 static gboolean
 block_failure(node_t *node, resource_t *rsc, xmlNode *xml_op,
               pe_working_set_t *data_set)
 {
     char *xml_name = clone_strip(rsc->id);
     char *xpath = crm_strdup_printf("//primitive[@id='%s']//op[@on-fail='block']",
                                     xml_name);
     xmlXPathObject *xpathObj = xpath_search(rsc->xml, xpath);
     gboolean should_block = FALSE;
 
     free(xpath);
 
     if (xpathObj) {
         int max = numXpathResults(xpathObj);
         int lpc = 0;
 
         for (lpc = 0; lpc < max; lpc++) {
             xmlNode *pref = getXpathResult(xpathObj, lpc);
 
             if (xml_op) {
                 should_block = is_matched_failure(xml_name, pref, xml_op);
                 if (should_block) {
                     break;
                 }
 
             } else {
                 const char *conf_op_name = NULL;
                 int conf_op_interval = 0;
                 char *lrm_op_xpath = NULL;
                 xmlXPathObject *lrm_op_xpathObj = NULL;
 
                 conf_op_name = crm_element_value(pref, "name");
                 conf_op_interval = crm_get_msec(crm_element_value(pref, "interval"));
 
                 lrm_op_xpath = crm_strdup_printf("//node_state[@uname='%s']"
                                                "//lrm_resource[@id='%s']"
                                                "/lrm_rsc_op[@operation='%s'][@interval='%d']",
                                                node->details->uname, xml_name,
                                                conf_op_name, conf_op_interval);
                 lrm_op_xpathObj = xpath_search(data_set->input, lrm_op_xpath);
 
                 free(lrm_op_xpath);
 
                 if (lrm_op_xpathObj) {
                     int max2 = numXpathResults(lrm_op_xpathObj);
                     int lpc2 = 0;
 
                     for (lpc2 = 0; lpc2 < max2; lpc2++) {
                         xmlNode *lrm_op_xml = getXpathResult(lrm_op_xpathObj,
                                                              lpc2);
 
                         should_block = is_matched_failure(xml_name, pref,
                                                           lrm_op_xml);
                         if (should_block) {
                             break;
                         }
                     }
                 }
                 freeXpathObject(lrm_op_xpathObj);
 
                 if (should_block) {
                     break;
                 }
             }
         }
     }
 
     free(xml_name);
     freeXpathObject(xpathObj);
 
     return should_block;
 }
 
 int
 get_failcount_full(node_t *node, resource_t *rsc, time_t *last_failure,
                    bool effective, xmlNode *xml_op, pe_working_set_t *data_set)
 {
     char *key = NULL;
     const char *value = NULL;
     struct fail_search search = { rsc, data_set, 0, 0, NULL };
 
     /* Optimize the "normal" case */
-    key = crm_concat("fail-count", rsc->clone_name ? rsc->clone_name : rsc->id, '-');
+    key = crm_failcount_name(rsc->clone_name? rsc->clone_name : rsc->id);
     value = g_hash_table_lookup(node->details->attrs, key);
     search.count = char2score(value);
     crm_trace("%s = %s", key, value);
     free(key);
 
     if (value) {
-        key = crm_concat("last-failure",
-                         rsc->clone_name ? rsc->clone_name : rsc->id, '-');
+        key = crm_lastfailure_name(rsc->clone_name? rsc->clone_name : rsc->id);
         value = g_hash_table_lookup(node->details->attrs, key);
         search.last = crm_int_helper(value, NULL);
         free(key);
 
         /* This block is still relevant once we omit anonymous instance numbers
          * because stopped clones won't have clone_name set
          */
     } else if (is_not_set(rsc->flags, pe_rsc_unique)) {
         search.rsc = uber_parent(rsc);
         search.key = clone_strip(rsc->id);
 
         g_hash_table_foreach(node->details->attrs, get_failcount_by_prefix,
                              &search);
         free(search.key);
         search.key = NULL;
     }
 
     if (search.count != 0 && search.last != 0 && last_failure) {
         *last_failure = search.last;
     }
 
     if (search.count && rsc->failure_timeout) {
         /* Never time-out if blocking failures are configured */
         if (block_failure(node, rsc, xml_op, data_set)) {
             pe_warn("Setting %s.failure-timeout=%d conflicts with on-fail=block: ignoring timeout",
                     rsc->id, rsc->failure_timeout);
             rsc->failure_timeout = 0;
 #if 0
             /* A good idea? */
         } else if (rsc->container == NULL && is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
             /* In this case, stop.on-fail defaults to block in unpack_operation() */
             rsc->failure_timeout = 0;
 #endif
         }
     }
 
     if (effective && (search.count != 0) && (search.last != 0)
         && rsc->failure_timeout) {
 
         if (search.last > 0) {
             time_t now = get_effective_time(data_set);
 
             if (now > (search.last + rsc->failure_timeout)) {
                 crm_debug("Failcount for %s on %s has expired (limit was %ds)",
                           search.rsc->id, node->details->uname,
                           rsc->failure_timeout);
                 search.count = 0;
             }
         }
     }
 
     if (search.count != 0) {
         char *score = score2char(search.count);
 
         crm_info("%s has failed %s times on %s",
                  search.rsc->id, score, node->details->uname);
         free(score);
     }
 
     return search.count;
 }
 
 /* If it's a resource container, get its failcount plus all the failcounts of
  * the resources within it
  */
 int
 get_failcount_all(node_t *node, resource_t *rsc, time_t *last_failure,
                   pe_working_set_t *data_set)
 {
     int failcount_all = 0;
 
     failcount_all = get_failcount(node, rsc, last_failure, data_set);
 
     if (rsc->fillers) {
         GListPtr gIter = NULL;
 
         for (gIter = rsc->fillers; gIter != NULL; gIter = gIter->next) {
             resource_t *filler = (resource_t *) gIter->data;
             time_t filler_last_failure = 0;
 
             failcount_all += get_failcount(node, filler, &filler_last_failure,
                                            data_set);
 
             if (last_failure && filler_last_failure > *last_failure) {
                 *last_failure = filler_last_failure;
             }
         }
 
         if (failcount_all != 0) {
             char *score = score2char(failcount_all);
 
             crm_info("Container %s and the resources within it have failed %s times on %s",
                      rsc->id, score, node->details->uname);
             free(score);
         }
     }
 
     return failcount_all;
 }
diff --git a/tools/crm_mon.c b/tools/crm_mon.c
index 80872122a7..776aea8b31 100644
--- a/tools/crm_mon.c
+++ b/tools/crm_mon.c
@@ -1,4218 +1,4221 @@
 /*
  * Copyright (C) 2004-2015 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <libgen.h>
 #include <sys/utsname.h>
 
 #include <crm/msg_xml.h>
 #include <crm/services.h>
 #include <crm/lrmd.h>
 #include <crm/common/util.h>
 #include <crm/common/xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/mainloop.h>
 
 #include <crm/cib/internal.h>
 #include <crm/pengine/status.h>
 #include <crm/pengine/internal.h>
 #include <../lib/pengine/unpack.h>
 #include <../pengine/pengine.h>
 #include <crm/stonith-ng.h>
 
 extern void cleanup_alloc_calculations(pe_working_set_t * data_set);
 
 void clean_up(int rc);
 void crm_diff_update(const char *event, xmlNode * msg);
 gboolean mon_refresh_display(gpointer user_data);
 int cib_connect(gboolean full);
 void mon_st_callback(stonith_t * st, stonith_event_t * e);
 static char *get_node_display_name(node_t *node);
 
 /*
  * Definitions indicating which items to print
  */
 
 #define mon_show_times      (0x0001U)
 #define mon_show_stack      (0x0002U)
 #define mon_show_dc         (0x0004U)
 #define mon_show_count      (0x0008U)
 #define mon_show_nodes      (0x0010U)
 #define mon_show_resources  (0x0020U)
 #define mon_show_attributes (0x0040U)
 #define mon_show_failcounts (0x0080U)
 #define mon_show_operations (0x0100U)
 #define mon_show_tickets    (0x0200U)
 #define mon_show_bans       (0x0400U)
 
 #define mon_show_headers    (mon_show_times | mon_show_stack | mon_show_dc | mon_show_count)
 #define mon_show_default    (mon_show_headers | mon_show_nodes | mon_show_resources)
 #define mon_show_all        (mon_show_default | mon_show_attributes | mon_show_failcounts \
                      | mon_show_operations | mon_show_tickets | mon_show_bans)
 
 unsigned int show = mon_show_default;
 
 /*
  * Definitions indicating how to output
  */
 
 enum mon_output_format_e {
     mon_output_none,
     mon_output_monitor,
     mon_output_plain,
     mon_output_console,
     mon_output_xml,
     mon_output_html,
     mon_output_cgi
 } output_format = mon_output_console;
 
 char *output_filename = NULL;   /* if sending output to a file, its name */
 
 /* other globals */
 char *xml_file = NULL;
 char *pid_file = NULL;
 char *snmp_target = NULL;
 char *snmp_community = NULL;
 
 gboolean group_by_node = FALSE;
 gboolean inactive_resources = FALSE;
 int reconnect_msec = 5000;
 gboolean daemonize = FALSE;
 GMainLoop *mainloop = NULL;
 guint timer_id = 0;
 GList *attr_list = NULL;
 
 const char *crm_mail_host = NULL;
 const char *crm_mail_prefix = NULL;
 const char *crm_mail_from = NULL;
 const char *crm_mail_to = NULL;
 const char *external_agent = NULL;
 const char *external_recipient = NULL;
 
 cib_t *cib = NULL;
 stonith_t *st = NULL;
 xmlNode *current_cib = NULL;
 
 gboolean one_shot = FALSE;
 gboolean has_warnings = FALSE;
 gboolean print_timing = FALSE;
 gboolean watch_fencing = FALSE;
 gboolean print_brief = FALSE;
 gboolean print_pending = TRUE;
 gboolean print_clone_detail = FALSE;
 
 /* FIXME allow, detect, and correctly interpret glob pattern or regex? */
 const char *print_neg_location_prefix = "";
 
 /* Never display node attributes whose name starts with one of these prefixes */
-#define FILTER_STR { "shutdown", "terminate", "standby", "fail-count",  \
-                     "last-failure", "probe_complete", "#", NULL }
+#define FILTER_STR { CRM_FAIL_COUNT_PREFIX, CRM_LAST_FAILURE_PREFIX,       \
+                     "shutdown", "terminate", "standby", "probe_complete", \
+                     "#", NULL }
 
 long last_refresh = 0;
 crm_trigger_t *refresh_trigger = NULL;
 
 /*
  * 1.3.6.1.4.1.32723 has been assigned to the project by IANA
  * http://www.iana.org/assignments/enterprise-numbers
  */
 #define PACEMAKER_PREFIX "1.3.6.1.4.1.32723"
 #define PACEMAKER_TRAP_PREFIX PACEMAKER_PREFIX ".1"
 
 #define snmp_crm_trap_oid   PACEMAKER_TRAP_PREFIX
 #define snmp_crm_oid_node   PACEMAKER_TRAP_PREFIX ".1"
 #define snmp_crm_oid_rsc    PACEMAKER_TRAP_PREFIX ".2"
 #define snmp_crm_oid_task   PACEMAKER_TRAP_PREFIX ".3"
 #define snmp_crm_oid_desc   PACEMAKER_TRAP_PREFIX ".4"
 #define snmp_crm_oid_status PACEMAKER_TRAP_PREFIX ".5"
 #define snmp_crm_oid_rc     PACEMAKER_TRAP_PREFIX ".6"
 #define snmp_crm_oid_trc    PACEMAKER_TRAP_PREFIX ".7"
 
 /* Define exit codes for monitoring-compatible output */
 #define MON_STATUS_OK   (0)
 #define MON_STATUS_WARN (1)
 
 /* Convenience macro for prettifying output (e.g. "node" vs "nodes") */
 #define s_if_plural(i) (((i) == 1)? "" : "s")
 
 #if CURSES_ENABLED
 #  define print_dot() if (output_format == mon_output_console) { \
 	printw(".");				\
 	clrtoeol();				\
 	refresh();				\
     } else {					\
 	fprintf(stdout, ".");			\
     }
 #else
 #  define print_dot() fprintf(stdout, ".");
 #endif
 
 #if CURSES_ENABLED
 #  define print_as(fmt, args...) if (output_format == mon_output_console) { \
 	printw(fmt, ##args);				\
 	clrtoeol();					\
 	refresh();					\
     } else {						\
 	fprintf(stdout, fmt, ##args);			\
     }
 #else
 #  define print_as(fmt, args...) fprintf(stdout, fmt, ##args);
 #endif
 
 static void
 blank_screen(void)
 {
 #if CURSES_ENABLED
     int lpc = 0;
 
     for (lpc = 0; lpc < LINES; lpc++) {
         move(lpc, 0);
         clrtoeol();
     }
     move(0, 0);
     refresh();
 #endif
 }
 
 static gboolean
 mon_timer_popped(gpointer data)
 {
     int rc = pcmk_ok;
 
 #if CURSES_ENABLED
     if (output_format == mon_output_console) {
         clear();
         refresh();
     }
 #endif
 
     if (timer_id > 0) {
         g_source_remove(timer_id);
     }
 
     print_as("Reconnecting...\n");
     rc = cib_connect(TRUE);
 
     if (rc != pcmk_ok) {
         timer_id = g_timeout_add(reconnect_msec, mon_timer_popped, NULL);
     }
     return FALSE;
 }
 
 static void
 mon_cib_connection_destroy(gpointer user_data)
 {
     print_as("Connection to the CIB terminated\n");
     if (cib) {
         cib->cmds->signoff(cib);
         timer_id = g_timeout_add(reconnect_msec, mon_timer_popped, NULL);
     }
     return;
 }
 
 /*
  * Mainloop signal handler.
  */
 static void
 mon_shutdown(int nsig)
 {
     clean_up(EX_OK);
 }
 
 #if ON_DARWIN
 #  define sighandler_t sig_t
 #endif
 
 #if CURSES_ENABLED
 #  ifndef HAVE_SIGHANDLER_T
 typedef void (*sighandler_t) (int);
 #  endif
 static sighandler_t ncurses_winch_handler;
 static void
 mon_winresize(int nsig)
 {
     static int not_done;
     int lines = 0, cols = 0;
 
     if (!not_done++) {
         if (ncurses_winch_handler)
             /* the original ncurses WINCH signal handler does the
              * magic of retrieving the new window size;
              * otherwise, we'd have to use ioctl or tgetent */
             (*ncurses_winch_handler) (SIGWINCH);
         getmaxyx(stdscr, lines, cols);
         resizeterm(lines, cols);
         mainloop_set_trigger(refresh_trigger);
     }
     not_done--;
 }
 #endif
 
 int
 cib_connect(gboolean full)
 {
     int rc = pcmk_ok;
     static gboolean need_pass = TRUE;
 
     CRM_CHECK(cib != NULL, return -EINVAL);
 
     if (getenv("CIB_passwd") != NULL) {
         need_pass = FALSE;
     }
 
     if (watch_fencing && st == NULL) {
         st = stonith_api_new();
     }
 
     if (watch_fencing && st->state == stonith_disconnected) {
         crm_trace("Connecting to stonith");
         rc = st->cmds->connect(st, crm_system_name, NULL);
         if (rc == pcmk_ok) {
             crm_trace("Setting up stonith callbacks");
             st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, mon_st_callback);
         }
     }
 
     if (cib->state != cib_connected_query && cib->state != cib_connected_command) {
         crm_trace("Connecting to the CIB");
         if ((output_format == mon_output_console) && need_pass && (cib->variant == cib_remote)) {
             need_pass = FALSE;
             print_as("Password:");
         }
 
         rc = cib->cmds->signon(cib, crm_system_name, cib_query);
 
         if (rc != pcmk_ok) {
             return rc;
         }
 
         rc = cib->cmds->query(cib, NULL, &current_cib, cib_scope_local | cib_sync_call);
         if (rc == pcmk_ok) {
             mon_refresh_display(NULL);
         }
 
         if (rc == pcmk_ok && full) {
             if (rc == pcmk_ok) {
                 rc = cib->cmds->set_connection_dnotify(cib, mon_cib_connection_destroy);
                 if (rc == -EPROTONOSUPPORT) {
                     print_as
                         ("Notification setup not supported, won't be able to reconnect after failure");
                     if (output_format == mon_output_console) {
                         sleep(2);
                     }
                     rc = pcmk_ok;
                 }
 
             }
 
             if (rc == pcmk_ok) {
                 cib->cmds->del_notify_callback(cib, T_CIB_DIFF_NOTIFY, crm_diff_update);
                 rc = cib->cmds->add_notify_callback(cib, T_CIB_DIFF_NOTIFY, crm_diff_update);
             }
 
             if (rc != pcmk_ok) {
                 print_as("Notification setup failed, could not monitor CIB actions");
                 if (output_format == mon_output_console) {
                     sleep(2);
                 }
                 clean_up(-rc);
             }
         }
     }
     return rc;
 }
 
 /* *INDENT-OFF* */
 static struct crm_option long_options[] = {
     /* Top-level Options */
     {"help",           0, 0, '?', "\tThis text"},
     {"version",        0, 0, '$', "\tVersion information"  },
     {"verbose",        0, 0, 'V', "\tIncrease debug output"},
     {"quiet",          0, 0, 'Q', "\tDisplay only essential output" },
 
     {"-spacer-",	1, 0, '-', "\nModes:"},
     {"as-html",        1, 0, 'h', "\tWrite cluster status to the named html file"},
     {"as-xml",         0, 0, 'X', "\t\tWrite cluster status as xml to stdout. This will enable one-shot mode."},
     {"web-cgi",        0, 0, 'w', "\t\tWeb mode with output suitable for cgi"},
     {"simple-status",  0, 0, 's', "\tDisplay the cluster status once as a simple one line output (suitable for nagios)"},
     {"snmp-traps",     1, 0, 'S', "\tSend SNMP traps to this station", !ENABLE_SNMP},
     {"snmp-community", 1, 0, 'C', "Specify community for SNMP traps(default is NULL)", !ENABLE_SNMP},
     {"mail-to",        1, 0, 'T', "\tSend Mail alerts to this user.  See also --mail-from, --mail-host, --mail-prefix", !ENABLE_ESMTP},
 
     {"-spacer-",	1, 0, '-', "\nDisplay Options:"},
     {"group-by-node",  0, 0, 'n', "\tGroup resources by node"     },
     {"inactive",       0, 0, 'r', "\t\tDisplay inactive resources"  },
     {"failcounts",     0, 0, 'f', "\tDisplay resource fail counts"},
     {"operations",     0, 0, 'o', "\tDisplay resource operation history" },
     {"timing-details", 0, 0, 't', "\tDisplay resource operation history with timing details" },
     {"tickets",        0, 0, 'c', "\t\tDisplay cluster tickets"},
     {"watch-fencing",  0, 0, 'W', "\tListen for fencing events. For use with --external-agent, --mail-to and/or --snmp-traps where supported"},
     {"neg-locations",  2, 0, 'L', "Display negative location constraints [optionally filtered by id prefix]"},
     {"show-node-attributes", 0, 0, 'A', "Display node attributes" },
     {"hide-headers",   0, 0, 'D', "\tHide all headers" },
     {"show-detail",    0, 0, 'R', "\tShow more details (node IDs, individual clone instances)" },
     {"brief",          0, 0, 'b', "\t\tBrief output" },
     {"pending",        0, 0, 'j', "\t\tDisplay pending state if 'record-pending' is enabled", pcmk_option_hidden},
 
     {"-spacer-",	1, 0, '-', "\nAdditional Options:"},
     {"interval",       1, 0, 'i', "\tUpdate frequency in seconds" },
     {"one-shot",       0, 0, '1', "\t\tDisplay the cluster status once on the console and exit"},
     {"disable-ncurses",0, 0, 'N', "\tDisable the use of ncurses", !CURSES_ENABLED},
     {"daemonize",      0, 0, 'd', "\tRun in the background as a daemon"},
     {"pid-file",       1, 0, 'p', "\t(Advanced) Daemon pid file location"},
     {"mail-from",      1, 0, 'F', "\tMail alerts should come from the named user", !ENABLE_ESMTP},
     {"mail-host",      1, 0, 'H', "\tMail alerts should be sent via the named host", !ENABLE_ESMTP},
     {"mail-prefix",    1, 0, 'P', "Subjects for mail alerts should start with this string", !ENABLE_ESMTP},
     {"external-agent",    1, 0, 'E', "A program to run when resource operations take place."},
     {"external-recipient",1, 0, 'e', "A recipient for your program (assuming you want the program to send something to someone)."},
 
 
     {"xml-file",       1, 0, 'x', NULL, pcmk_option_hidden},
 
     {"-spacer-",	1, 0, '-', "\nExamples:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', "Display the cluster status on the console with updates as they occur:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_mon", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Display the cluster status on the console just once then exit:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_mon -1", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Display your cluster status, group resources by node, and include inactive resources in the list:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_mon --group-by-node --inactive", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Start crm_mon as a background daemon and have it write the cluster status to an HTML file:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_mon --daemonize --as-html /path/to/docroot/filename.html", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Start crm_mon and export the current cluster status as xml to stdout, then exit.:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_mon --as-xml", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Start crm_mon as a background daemon and have it send email alerts:", pcmk_option_paragraph|!ENABLE_ESMTP},
     {"-spacer-",	1, 0, '-', " crm_mon --daemonize --mail-to user@example.com --mail-host mail.example.com", pcmk_option_example|!ENABLE_ESMTP},
     {"-spacer-",	1, 0, '-', "Start crm_mon as a background daemon and have it send SNMP alerts:", pcmk_option_paragraph|!ENABLE_SNMP},
     {"-spacer-",	1, 0, '-', " crm_mon --daemonize --snmp-traps snmptrapd.example.com", pcmk_option_example|!ENABLE_SNMP},
 
     {NULL, 0, 0, 0}
 };
 /* *INDENT-ON* */
 
 #if CURSES_ENABLED
 static const char *
 get_option_desc(char c)
 {
     int lpc;
 
     for (lpc = 0; long_options[lpc].name != NULL; lpc++) {
 
         if (long_options[lpc].name[0] == '-')
             continue;
 
         if (long_options[lpc].val == c) {
             const char * tab = NULL;
             tab = strrchr(long_options[lpc].desc, '\t');
             return tab ? ++tab : long_options[lpc].desc;
         }
     }
 
     return NULL;
 }
 
 #define print_option_help(option, condition) \
     print_as("%c %c: \t%s\n", ((condition)? '*': ' '), option, get_option_desc(option));
 
 static gboolean
 detect_user_input(GIOChannel *channel, GIOCondition condition, gpointer unused)
 {
     int c;
     gboolean config_mode = FALSE;
 
     while (1) {
 
         /* Get user input */
         c = getchar();
 
         switch (c) {
             case 'c':
                 show ^= mon_show_tickets;
                 break;
             case 'f':
                 show ^= mon_show_failcounts;
                 break;
             case 'n':
                 group_by_node = ! group_by_node;
                 break;
             case 'o':
                 show ^= mon_show_operations;
                 if ((show & mon_show_operations) == 0) {
                     print_timing = 0;
                 }
                 break;
             case 'r':
                 inactive_resources = ! inactive_resources;
                 break;
             case 'R':
                 print_clone_detail = ! print_clone_detail;
                 break;
             case 't':
                 print_timing = ! print_timing;
                 if (print_timing) {
                     show |= mon_show_operations;
                 }
                 break;
             case 'A':
                 show ^= mon_show_attributes;
                 break;
             case 'L':
                 show ^= mon_show_bans;
                 break;
             case 'D':
                 /* If any header is shown, clear them all, otherwise set them all */
                 if (show & mon_show_headers) {
                     show &= ~mon_show_headers;
                 } else {
                     show |= mon_show_headers;
                 }
                 break;
             case 'b':
                 print_brief = ! print_brief;
                 break;
             case 'j':
                 print_pending = ! print_pending;
                 break;
             case '?':
                 config_mode = TRUE;
                 break;
             default:
                 goto refresh;
         }
 
         if (!config_mode)
             goto refresh;
 
         blank_screen();
 
         print_as("Display option change mode\n");
         print_as("\n");
         print_option_help('c', show & mon_show_tickets);
         print_option_help('f', show & mon_show_failcounts);
         print_option_help('n', group_by_node);
         print_option_help('o', show & mon_show_operations);
         print_option_help('r', inactive_resources);
         print_option_help('t', print_timing);
         print_option_help('A', show & mon_show_attributes);
         print_option_help('L', show & mon_show_bans);
         print_option_help('D', (show & mon_show_headers) == 0);
         print_option_help('R', print_clone_detail);
         print_option_help('b', print_brief);
         print_option_help('j', print_pending);
         print_as("\n");
         print_as("Toggle fields via field letter, type any other key to return");
     }
 
 refresh:
     mon_refresh_display(NULL);
     return TRUE;
 }
 #endif
 
 int
 main(int argc, char **argv)
 {
     int flag;
     int argerr = 0;
     int exit_code = 0;
     int option_index = 0;
 
     pid_file = strdup("/tmp/ClusterMon.pid");
     crm_log_cli_init("crm_mon");
     crm_set_options(NULL, "mode [options]", long_options,
                     "Provides a summary of cluster's current state."
                     "\n\nOutputs varying levels of detail in a number of different formats.\n");
 
 #if !defined (ON_DARWIN) && !defined (ON_BSD)
     /* prevent zombies */
     signal(SIGCLD, SIG_IGN);
 #endif
 
     if (strcmp(crm_system_name, "crm_mon.cgi") == 0) {
         output_format = mon_output_cgi;
         one_shot = TRUE;
     }
 
     while (1) {
         flag = crm_get_option(argc, argv, &option_index);
         if (flag == -1)
             break;
 
         switch (flag) {
             case 'V':
                 crm_bump_log_level(argc, argv);
                 break;
             case 'Q':
                 show &= ~mon_show_times;
                 break;
             case 'i':
                 reconnect_msec = crm_get_msec(optarg);
                 break;
             case 'n':
                 group_by_node = TRUE;
                 break;
             case 'r':
                 inactive_resources = TRUE;
                 break;
             case 'W':
                 watch_fencing = TRUE;
                 break;
             case 'd':
                 daemonize = TRUE;
                 break;
             case 't':
                 print_timing = TRUE;
                 show |= mon_show_operations;
                 break;
             case 'o':
                 show |= mon_show_operations;
                 break;
             case 'f':
                 show |= mon_show_failcounts;
                 break;
             case 'A':
                 show |= mon_show_attributes;
                 break;
             case 'L':
                 show |= mon_show_bans;
                 print_neg_location_prefix = optarg? optarg : "";
                 break;
             case 'D':
                 show &= ~mon_show_headers;
                 break;
             case 'b':
                 print_brief = TRUE;
                 break;
             case 'j':
                 print_pending = TRUE;
                 break;
             case 'R':
                 print_clone_detail = TRUE;
                 break;
             case 'c':
                 show |= mon_show_tickets;
                 break;
             case 'p':
                 free(pid_file);
                 if(optarg == NULL) {
                     return crm_help(flag, EX_USAGE);
                 }
                 pid_file = strdup(optarg);
                 break;
             case 'x':
                 if(optarg == NULL) {
                     return crm_help(flag, EX_USAGE);
                 }
                 xml_file = strdup(optarg);
                 one_shot = TRUE;
                 break;
             case 'h':
                 if(optarg == NULL) {
                     return crm_help(flag, EX_USAGE);
                 }
                 output_format = mon_output_html;
                 output_filename = strdup(optarg);
                 umask(S_IWGRP | S_IWOTH);
                 break;
             case 'X':
                 output_format = mon_output_xml;
                 one_shot = TRUE;
                 break;
             case 'w':
                 output_format = mon_output_cgi;
                 one_shot = TRUE;
                 break;
             case 's':
                 output_format = mon_output_monitor;
                 one_shot = TRUE;
                 break;
             case 'S':
                 snmp_target = optarg;
                 break;
             case 'T':
                 crm_mail_to = optarg;
                 break;
             case 'F':
                 crm_mail_from = optarg;
                 break;
             case 'H':
                 crm_mail_host = optarg;
                 break;
             case 'P':
                 crm_mail_prefix = optarg;
                 break;
             case 'E':
                 external_agent = optarg;
                 break;
             case 'e':
                 external_recipient = optarg;
                 break;
             case '1':
                 one_shot = TRUE;
                 break;
             case 'N':
                 if (output_format == mon_output_console) {
                     output_format = mon_output_plain;
                 }
                 break;
             case 'C':
                 snmp_community = optarg;
                 break;
             case '$':
             case '?':
                 return crm_help(flag, EX_OK);
                 break;
             default:
                 printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
                 ++argerr;
                 break;
         }
     }
 
     if (optind < argc) {
         printf("non-option ARGV-elements: ");
         while (optind < argc)
             printf("%s ", argv[optind++]);
         printf("\n");
     }
     if (argerr) {
         return crm_help('?', EX_USAGE);
     }
 
     /* XML output always prints everything */
     if (output_format == mon_output_xml) {
         show = mon_show_all;
         print_timing = TRUE;
     }
 
     if (one_shot) {
         if (output_format == mon_output_console) {
             output_format = mon_output_plain;
         }
 
     } else if (daemonize) {
         if ((output_format == mon_output_console) || (output_format == mon_output_plain)) {
             output_format = mon_output_none;
         }
         crm_enable_stderr(FALSE);
 
         if ((output_format != mon_output_html) && (output_format != mon_output_xml)
             && !snmp_target && !crm_mail_to && !external_agent) {
             printf
                 ("Looks like you forgot to specify one or more of: --as-html, --as-xml, --mail-to, --snmp-target, --external-agent\n");
             return crm_help('?', EX_USAGE);
         }
 
         crm_make_daemon(crm_system_name, TRUE, pid_file);
 
     } else if (output_format == mon_output_console) {
 #if CURSES_ENABLED
         initscr();
         cbreak();
         noecho();
         crm_enable_stderr(FALSE);
 #else
         one_shot = TRUE;
         output_format = mon_output_plain;
         printf("Defaulting to one-shot mode\n");
         printf("You need to have curses available at compile time to enable console mode\n");
 #endif
     }
 
     crm_info("Starting %s", crm_system_name);
     if (xml_file != NULL) {
         current_cib = filename2xml(xml_file);
         mon_refresh_display(NULL);
         return exit_code;
     }
 
     if (current_cib == NULL) {
         cib = cib_new();
 
         do {
             if (!one_shot) {
                 print_as("Attempting connection to the cluster...\n");
             }
             exit_code = cib_connect(!one_shot);
 
             if (one_shot) {
                 break;
 
             } else if (exit_code != pcmk_ok) {
                 sleep(reconnect_msec / 1000);
 #if CURSES_ENABLED
                 if (output_format == mon_output_console) {
                     clear();
                     refresh();
                 }
 #endif
             }
 
         } while (exit_code == -ENOTCONN);
 
         if (exit_code != pcmk_ok) {
             if (output_format == mon_output_monitor) {
                 printf("CLUSTER WARN: Connection to cluster failed: %s\n", pcmk_strerror(exit_code));
                 clean_up(MON_STATUS_WARN);
             } else {
                 print_as("\nConnection to cluster failed: %s\n", pcmk_strerror(exit_code));
             }
             if (output_format == mon_output_console) {
                 sleep(2);
             }
             clean_up(-exit_code);
         }
     }
 
     if (one_shot) {
         return exit_code;
     }
 
     mainloop = g_main_new(FALSE);
 
     mainloop_add_signal(SIGTERM, mon_shutdown);
     mainloop_add_signal(SIGINT, mon_shutdown);
 #if CURSES_ENABLED
     if (output_format == mon_output_console) {
         ncurses_winch_handler = signal(SIGWINCH, mon_winresize);
         if (ncurses_winch_handler == SIG_DFL ||
             ncurses_winch_handler == SIG_IGN || ncurses_winch_handler == SIG_ERR)
             ncurses_winch_handler = NULL;
         g_io_add_watch(g_io_channel_unix_new(STDIN_FILENO), G_IO_IN, detect_user_input, NULL);
     }
 #endif
     refresh_trigger = mainloop_add_trigger(G_PRIORITY_LOW, mon_refresh_display, NULL);
 
     g_main_run(mainloop);
     g_main_destroy(mainloop);
 
     crm_info("Exiting %s", crm_system_name);
 
     clean_up(0);
     return 0;                   /* never reached */
 }
 
 #define mon_warn(fmt...) do {			\
 	if (!has_warnings) {			\
 	    print_as("CLUSTER WARN:");		\
 	} else {				\
 	    print_as(",");			\
 	}					\
 	print_as(fmt);				\
 	has_warnings = TRUE;			\
     } while(0)
 
 static int
 count_resources(pe_working_set_t * data_set, resource_t * rsc)
 {
     int count = 0;
     GListPtr gIter = NULL;
 
     if (rsc == NULL) {
         gIter = data_set->resources;
     } else if (rsc->children) {
         gIter = rsc->children;
     } else {
         return is_not_set(rsc->flags, pe_rsc_orphan);
     }
 
     for (; gIter != NULL; gIter = gIter->next) {
         count += count_resources(data_set, gIter->data);
     }
     return count;
 }
 
 /*!
  * \internal
  * \brief Print one-line status suitable for use with monitoring software
  *
  * \param[in] data_set  Working set of CIB state
  *
  * \note This function's output (and the return code when the program exits)
  *       should conform to https://www.monitoring-plugins.org/doc/guidelines.html
  */
 static void
 print_simple_status(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     int nodes_online = 0;
     int nodes_standby = 0;
     int nodes_maintenance = 0;
 
     if (data_set->dc_node == NULL) {
         mon_warn(" No DC");
     }
 
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
 
         if (node->details->standby && node->details->online) {
             nodes_standby++;
         } else if (node->details->maintenance && node->details->online) {
             nodes_maintenance++;
         } else if (node->details->online) {
             nodes_online++;
         } else {
             mon_warn(" offline node: %s", node->details->uname);
         }
     }
 
     if (!has_warnings) {
         int nresources = count_resources(data_set, NULL);
 
         print_as("CLUSTER OK: %d node%s online", nodes_online, s_if_plural(nodes_online));
         if (nodes_standby > 0) {
             print_as(", %d standby node%s", nodes_standby, s_if_plural(nodes_standby));
         }
         if (nodes_maintenance > 0) {
             print_as(", %d maintenance node%s", nodes_maintenance, s_if_plural(nodes_maintenance));
         }
         print_as(", %d resource%s configured", nresources, s_if_plural(nresources));
     }
 
     print_as("\n");
 }
 
 /*!
  * \internal
  * \brief Print a [name]=[value][units] pair, optionally using time string
  *
  * \param[in] stream      File stream to display output to
  * \param[in] name        Name to display
  * \param[in] value       Value to display (or NULL to convert time instead)
  * \param[in] units       Units to display (or NULL for no units)
  * \param[in] epoch_time  Epoch time to convert if value is NULL
  */
 static void
 print_nvpair(FILE *stream, const char *name, const char *value,
              const char *units, time_t epoch_time)
 {
     /* print name= */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as(" %s=", name);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
         case mon_output_xml:
             fprintf(stream, " %s=", name);
             break;
 
         default:
             break;
     }
 
     /* If we have a value (and optionally units), print it */
     if (value) {
         switch (output_format) {
             case mon_output_plain:
             case mon_output_console:
                 print_as("%s%s", value, (units? units : ""));
                 break;
 
             case mon_output_html:
             case mon_output_cgi:
                 fprintf(stream, "%s%s", value, (units? units : ""));
                 break;
 
             case mon_output_xml:
                 fprintf(stream, "\"%s%s\"", value, (units? units : ""));
                 break;
 
             default:
                 break;
         }
 
     /* Otherwise print user-friendly time string */
     } else {
         char *date_str, *c;
 
         date_str = asctime(localtime(&epoch_time));
         for (c = date_str; c != '\0'; ++c) {
             if (*c == '\n') {
                 *c = '\0';
                 break;
             }
         }
         switch (output_format) {
             case mon_output_plain:
             case mon_output_console:
                 print_as("'%s'", date_str);
                 break;
 
             case mon_output_html:
             case mon_output_cgi:
             case mon_output_xml:
                 fprintf(stream, "\"%s\"", date_str);
                 break;
 
             default:
                 break;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Print whatever is needed to start a node section
  *
  * \param[in] stream     File stream to display output to
  * \param[in] node       Node to print
  */
 static void
 print_node_start(FILE *stream, node_t *node)
 {
     char *node_name;
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             node_name = get_node_display_name(node);
             print_as("* Node %s:\n", node_name);
             free(node_name);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             node_name = get_node_display_name(node);
             fprintf(stream, "  <h3>Node: %s</h3>\n  <ul>\n", node_name);
             free(node_name);
             break;
 
         case mon_output_xml:
             fprintf(stream, "        <node name=\"%s\">\n", node->details->uname);
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print whatever is needed to end a node section
  *
  * \param[in] stream     File stream to display output to
  */
 static void
 print_node_end(FILE *stream)
 {
     switch (output_format) {
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "  </ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "        </node>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print resources section heading appropriate to options
  *
  * \param[in] stream      File stream to display output to
  */
 static void
 print_resources_heading(FILE *stream)
 {
     const char *heading;
 
     if (group_by_node) {
 
         /* Active resources have already been printed by node */
         heading = (inactive_resources? "Inactive resources" : NULL);
 
     } else if (inactive_resources) {
         heading = "Full list of resources";
 
     } else {
         heading = "Active resources";
     }
 
     /* Print section heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n%s:\n\n", heading);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <hr />\n <h2>%s</h2>\n", heading);
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <resources>\n");
             break;
 
         default:
             break;
     }
 
 }
 
 /*!
  * \internal
  * \brief Print whatever resource section closing is appropriate
  *
  * \param[in] stream     File stream to display output to
  */
 static void
 print_resources_closing(FILE *stream, gboolean printed_heading)
 {
     const char *heading;
 
     /* What type of resources we did or did not display */
     if (group_by_node) {
         heading = "inactive ";
     } else if (inactive_resources) {
         heading = "";
     } else {
         heading = "active ";
     }
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             if (!printed_heading) {
                 print_as("\nNo %sresources\n\n", heading);
             }
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             if (!printed_heading) {
                 fprintf(stream, " <hr />\n <h2>No %sresources</h2>\n", heading);
             }
             break;
 
         case mon_output_xml:
             fprintf(stream, "    %s\n",
                     (printed_heading? "</resources>" : "<resources/>"));
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print whatever resource section(s) are appropriate
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Cluster state to display
  * \param[in] print_opts  Bitmask of pe_print_options
  */
 static void
 print_resources(FILE *stream, pe_working_set_t *data_set, int print_opts)
 {
     GListPtr rsc_iter;
     const char *prefix = NULL;
     gboolean printed_heading = FALSE;
     gboolean brief_output = print_brief;
 
     /* If we already showed active resources by node, and
      * we're not showing inactive resources, we have nothing to do
      */
     if (group_by_node && !inactive_resources) {
         return;
     }
 
     /* XML uses an indent, and ignores brief option for resources */
     if (output_format == mon_output_xml) {
         prefix = "        ";
         brief_output = FALSE;
     }
 
     /* If we haven't already printed resources grouped by node,
      * and brief output was requested, print resource summary */
     if (brief_output && !group_by_node) {
         print_resources_heading(stream);
         printed_heading = TRUE;
         print_rscs_brief(data_set->resources, NULL, print_opts, stream,
                          inactive_resources);
     }
 
     /* For each resource, display it if appropriate */
     for (rsc_iter = data_set->resources; rsc_iter != NULL; rsc_iter = rsc_iter->next) {
         resource_t *rsc = (resource_t *) rsc_iter->data;
 
         /* Complex resources may have some sub-resources active and some inactive */
         gboolean is_active = rsc->fns->active(rsc, TRUE);
         gboolean partially_active = rsc->fns->active(rsc, FALSE);
 
         /* Skip inactive orphans (deleted but still in CIB) */
         if (is_set(rsc->flags, pe_rsc_orphan) && !is_active) {
             continue;
 
         /* Skip active resources if we already displayed them by node */
         } else if (group_by_node) {
             if (is_active) {
                 continue;
             }
 
         /* Skip primitives already counted in a brief summary */
         } else if (brief_output && (rsc->variant == pe_native)) {
             continue;
 
         /* Skip resources that aren't at least partially active,
          * unless we're displaying inactive resources
          */
         } else if (!partially_active && !inactive_resources) {
             continue;
         }
 
         /* Print this resource */
         if (printed_heading == FALSE) {
             print_resources_heading(stream);
             printed_heading = TRUE;
         }
         rsc->fns->print(rsc, prefix, print_opts, stream);
     }
 
     print_resources_closing(stream, printed_heading);
 }
 
 /*!
  * \internal
  * \brief Print heading for resource history
  *
  * \param[in] stream      File stream to display output to
  * \param[in] data_set    Current state of CIB
  * \param[in] node        Node that ran this resource
  * \param[in] rsc         Resource to print
  * \param[in] rsc_id      ID of resource to print
  * \param[in] all         Whether to print every resource or just failed ones
  */
 static void
 print_rsc_history_start(FILE *stream, pe_working_set_t *data_set, node_t *node,
                         resource_t *rsc, const char *rsc_id, gboolean all)
 {
     time_t last_failure = 0;
     int failcount = rsc? get_failcount_full(node, rsc, &last_failure, FALSE, NULL, data_set) : 0;
 
     if (!all && !failcount && (last_failure <= 0)) {
         return;
     }
 
     /* Print resource ID */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("   %s:", rsc_id);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "   <li>%s:", rsc_id);
             break;
 
         case mon_output_xml:
             fprintf(stream, "            <resource_history id=\"%s\"", rsc_id);
             break;
 
         default:
             break;
     }
 
     /* If resource is an orphan, that's all we can say about it */
     if (rsc == NULL) {
         switch (output_format) {
             case mon_output_plain:
             case mon_output_console:
                 print_as(" orphan");
                 break;
 
             case mon_output_html:
             case mon_output_cgi:
                 fprintf(stream, " orphan");
                 break;
 
             case mon_output_xml:
                 fprintf(stream, " orphan=\"true\"");
                 break;
 
             default:
                 break;
         }
 
     /* If resource is not an orphan, print some details */
     } else if (all || failcount || (last_failure > 0)) {
 
         /* Print migration threshold */
         switch (output_format) {
             case mon_output_plain:
             case mon_output_console:
                 print_as(" migration-threshold=%d", rsc->migration_threshold);
                 break;
 
             case mon_output_html:
             case mon_output_cgi:
                 fprintf(stream, " migration-threshold=%d", rsc->migration_threshold);
                 break;
 
             case mon_output_xml:
                 fprintf(stream, " orphan=\"false\" migration-threshold=\"%d\"",
                         rsc->migration_threshold);
                 break;
 
             default:
                 break;
         }
 
         /* Print fail count if any */
         if (failcount > 0) {
             switch (output_format) {
                 case mon_output_plain:
                 case mon_output_console:
-                    print_as(" fail-count=%d", failcount);
+                    print_as(" " CRM_FAIL_COUNT_PREFIX "=%d", failcount);
                     break;
 
                 case mon_output_html:
                 case mon_output_cgi:
-                    fprintf(stream, " fail-count=%d", failcount);
+                    fprintf(stream, " " CRM_FAIL_COUNT_PREFIX "=%d", failcount);
                     break;
 
                 case mon_output_xml:
-                    fprintf(stream, " fail-count=\"%d\"", failcount);
+                    fprintf(stream, " " CRM_FAIL_COUNT_PREFIX "=\"%d\"",
+                            failcount);
                     break;
 
                 default:
                     break;
             }
         }
 
         /* Print last failure time if any */
         if (last_failure > 0) {
-            print_nvpair(stream, "last-failure", NULL, NULL, last_failure);
+            print_nvpair(stream, CRM_LAST_FAILURE_PREFIX, NULL, NULL,
+                         last_failure);
         }
     }
 
     /* End the heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "\n    <ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, ">\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print closing for resource history
  *
  * \param[in] stream      File stream to display output to
  */
 static void
 print_rsc_history_end(FILE *stream)
 {
     switch (output_format) {
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "    </ul>\n   </li>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "            </resource_history>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print operation history
  *
  * \param[in] stream      File stream to display output to
  * \param[in] data_set    Current state of CIB
  * \param[in] node        Node this operation is for
  * \param[in] xml_op      Root of XML tree describing this operation
  * \param[in] task        Task parsed from this operation's XML
  * \param[in] interval    Interval parsed from this operation's XML
  * \param[in] rc          Return code parsed from this operation's XML
  */
 static void
 print_op_history(FILE *stream, pe_working_set_t *data_set, node_t *node,
                  xmlNode *xml_op, const char *task, const char *interval, int rc)
 {
     const char *value = NULL;
     const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
 
     /* Begin the operation description */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("    + (%s) %s:", call, task);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "     <li>(%s) %s:", call, task);
             break;
 
         case mon_output_xml:
             fprintf(stream, "                <operation_history call=\"%s\" task=\"%s\"",
                     call, task);
             break;
 
         default:
             break;
     }
 
     /* Add name=value pairs as appropriate */
     if (safe_str_neq(interval, "0")) {
         print_nvpair(stream, "interval", interval, "ms", 0);
     }
     if (print_timing) {
         int int_value;
         const char *attr;
 
         attr = XML_RSC_OP_LAST_CHANGE;
         value = crm_element_value(xml_op, attr);
         if (value) {
             int_value = crm_parse_int(value, NULL);
             if (int_value > 0) {
                 print_nvpair(stream, attr, NULL, NULL, int_value);
             }
         }
 
         attr = XML_RSC_OP_LAST_RUN;
         value = crm_element_value(xml_op, attr);
         if (value) {
             int_value = crm_parse_int(value, NULL);
             if (int_value > 0) {
                 print_nvpair(stream, attr, NULL, NULL, int_value);
             }
         }
 
         attr = XML_RSC_OP_T_EXEC;
         value = crm_element_value(xml_op, attr);
         if (value) {
             print_nvpair(stream, attr, value, "ms", 0);
         }
 
         attr = XML_RSC_OP_T_QUEUE;
         value = crm_element_value(xml_op, attr);
         if (value) {
             print_nvpair(stream, attr, value, "ms", 0);
         }
     }
 
     /* End the operation description */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as(" rc=%d (%s)\n", rc, services_ocf_exitcode_str(rc));
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " rc=%d (%s)</li>\n", rc, services_ocf_exitcode_str(rc));
             break;
 
         case mon_output_xml:
             fprintf(stream, " rc=\"%d\" rc_text=\"%s\" />\n", rc, services_ocf_exitcode_str(rc));
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print resource operation/failure history
  *
  * \param[in] stream      File stream to display output to
  * \param[in] data_set    Current state of CIB
  * \param[in] node        Node that ran this resource
  * \param[in] rsc_entry   Root of XML tree describing resource status
  * \param[in] operations  Whether to print operations or just failcounts
  */
 static void
 print_rsc_history(FILE *stream, pe_working_set_t *data_set, node_t *node,
                   xmlNode *rsc_entry, gboolean operations)
 {
     GListPtr gIter = NULL;
     GListPtr op_list = NULL;
     gboolean printed = FALSE;
     const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
     resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
     xmlNode *rsc_op = NULL;
 
     /* If we're not showing operations, just print the resource failure summary */
     if (operations == FALSE) {
         print_rsc_history_start(stream, data_set, node, rsc, rsc_id, FALSE);
         print_rsc_history_end(stream);
         return;
     }
 
     /* Create a list of this resource's operations */
     for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next(rsc_op)) {
         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
             op_list = g_list_append(op_list, rsc_op);
         }
     }
     op_list = g_list_sort(op_list, sort_op_by_callid);
 
     /* Print each operation */
     for (gIter = op_list; gIter != NULL; gIter = gIter->next) {
         xmlNode *xml_op = (xmlNode *) gIter->data;
         const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
         const char *interval = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
         const char *op_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC);
         int rc = crm_parse_int(op_rc, "0");
 
         /* Display 0-interval monitors as "probe" */
         if (safe_str_eq(task, CRMD_ACTION_STATUS) && safe_str_eq(interval, "0")) {
             task = "probe";
         }
 
         /* Ignore notifies and some probes */
         if (safe_str_eq(task, CRMD_ACTION_NOTIFY) || (safe_str_eq(task, "probe") && (rc == 7))) {
             continue;
         }
 
         /* If this is the first printed operation, print heading for resource */
         if (printed == FALSE) {
             printed = TRUE;
             print_rsc_history_start(stream, data_set, node, rsc, rsc_id, TRUE);
         }
 
         /* Print the operation */
         print_op_history(stream, data_set, node, xml_op, task, interval, rc);
     }
 
     /* Free the list we created (no need to free the individual items) */
     g_list_free(op_list);
 
     /* If we printed anything, close the resource */
     if (printed) {
         print_rsc_history_end(stream);
     }
 }
 
 /*!
  * \internal
  * \brief Print node operation/failure history
  *
  * \param[in] stream      File stream to display output to
  * \param[in] data_set    Current state of CIB
  * \param[in] node_state  Root of XML tree describing node status
  * \param[in] operations  Whether to print operations or just failcounts
  */
 static void
 print_node_history(FILE *stream, pe_working_set_t *data_set,
                    xmlNode *node_state, gboolean operations)
 {
     node_t *node = pe_find_node_id(data_set->nodes, ID(node_state));
     xmlNode *lrm_rsc = NULL;
     xmlNode *rsc_entry = NULL;
 
     if (node && node->details && node->details->online) {
         print_node_start(stream, node);
 
         lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
         lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
 
         /* Print history of each of the node's resources */
         for (rsc_entry = __xml_first_child(lrm_rsc); rsc_entry != NULL;
              rsc_entry = __xml_next(rsc_entry)) {
 
             if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
                 print_rsc_history(stream, data_set, node, rsc_entry, operations);
             }
         }
 
         print_node_end(stream);
     }
 }
 
 /*!
  * \internal
  * \brief Print extended information about an attribute if appropriate
  *
  * \param[in] data_set  Working set of CIB state
  *
  * \return TRUE if extended information was printed, FALSE otherwise
  * \note Currently, extended information is only supported for ping/pingd
  *       resources, for which a message will be printed if connectivity is lost
  *       or degraded.
  */
 static gboolean
 print_attr_msg(FILE *stream, node_t * node, GListPtr rsc_list, const char *attrname, const char *attrvalue)
 {
     GListPtr gIter = NULL;
 
     for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) {
         resource_t *rsc = (resource_t *) gIter->data;
         const char *type = g_hash_table_lookup(rsc->meta, "type");
 
         if (rsc->children != NULL) {
             if (print_attr_msg(stream, node, rsc->children, attrname, attrvalue)) {
                 return TRUE;
             }
         }
 
         if (safe_str_eq(type, "ping") || safe_str_eq(type, "pingd")) {
             const char *name = g_hash_table_lookup(rsc->parameters, "name");
 
             if (name == NULL) {
                 name = "pingd";
             }
 
             /* To identify the resource with the attribute name. */
             if (safe_str_eq(name, attrname)) {
                 int host_list_num = 0;
                 int expected_score = 0;
                 int value = crm_parse_int(attrvalue, "0");
                 const char *hosts = g_hash_table_lookup(rsc->parameters, "host_list");
                 const char *multiplier = g_hash_table_lookup(rsc->parameters, "multiplier");
 
                 if(hosts) {
                     char **host_list = g_strsplit(hosts, " ", 0);
                     host_list_num = g_strv_length(host_list);
                     g_strfreev(host_list);
                 }
 
                 /* pingd multiplier is the same as the default value. */
                 expected_score = host_list_num * crm_parse_int(multiplier, "1");
 
                 switch (output_format) {
                     case mon_output_plain:
                     case mon_output_console:
                         if (value <= 0) {
                             print_as("\t: Connectivity is lost");
                         } else if (value < expected_score) {
                             print_as("\t: Connectivity is degraded (Expected=%d)", expected_score);
                         }
                         break;
 
                     case mon_output_html:
                     case mon_output_cgi:
                         if (value <= 0) {
                             fprintf(stream, " <b>(connectivity is lost)</b>");
                         } else if (value < expected_score) {
                             fprintf(stream, " <b>(connectivity is degraded -- expected %d)</b>",
                                     expected_score);
                         }
                         break;
 
                     case mon_output_xml:
                         fprintf(stream, " expected=\"%d\"", expected_score);
                         break;
 
                     default:
                         break;
                 }
                 return TRUE;
             }
         }
     }
     return FALSE;
 }
 
 static int
 compare_attribute(gconstpointer a, gconstpointer b)
 {
     int rc;
 
     rc = strcmp((const char *)a, (const char *)b);
 
     return rc;
 }
 
 static void
 create_attr_list(gpointer name, gpointer value, gpointer data)
 {
     int i;
     const char *filt_str[] = FILTER_STR;
 
     CRM_CHECK(name != NULL, return);
 
     /* filtering automatic attributes */
     for (i = 0; filt_str[i] != NULL; i++) {
         if (g_str_has_prefix(name, filt_str[i])) {
             return;
         }
     }
 
     attr_list = g_list_insert_sorted(attr_list, name, compare_attribute);
 }
 
 /* structure for passing multiple user data to g_list_foreach() */
 struct mon_attr_data {
     FILE *stream;
     node_t *node;
 };
 
 static void
 print_node_attribute(gpointer name, gpointer user_data)
 {
     const char *value = NULL;
     struct mon_attr_data *data = (struct mon_attr_data *) user_data;
 
     value = g_hash_table_lookup(data->node->details->attrs, name);
 
     /* Print attribute name and value */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("    + %-32s\t: %-10s", (char *)name, value);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(data->stream, "   <li>%s: %s",
                     (char *)name, value);
             break;
 
         case mon_output_xml:
             fprintf(data->stream,
                     "            <attribute name=\"%s\" value=\"%s\"",
                     (char *)name, value);
             break;
 
         default:
             break;
     }
 
     /* Print extended information if appropriate */
     print_attr_msg(data->stream, data->node, data->node->details->running_rsc,
                    name, value);
 
     /* Close out the attribute */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(data->stream, "</li>\n");
             break;
 
         case mon_output_xml:
             fprintf(data->stream, " />\n");
             break;
 
         default:
             break;
     }
 }
 
 static void
 print_node_summary(FILE *stream, pe_working_set_t * data_set, gboolean operations)
 {
     xmlNode *node_state = NULL;
     xmlNode *cib_status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);
 
     /* Print heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             if (operations) {
                 print_as("\nOperations:\n");
             } else {
                 print_as("\nMigration Summary:\n");
             }
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             if (operations) {
                 fprintf(stream, " <hr />\n <h2>Operations</h2>\n");
             } else {
                 fprintf(stream, " <hr />\n <h2>Migration Summary</h2>\n");
             }
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <node_history>\n");
             break;
 
         default:
             break;
     }
 
     /* Print each node in the CIB status */
     for (node_state = __xml_first_child(cib_status); node_state != NULL;
          node_state = __xml_next(node_state)) {
         if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
             print_node_history(stream, data_set, node_state, operations);
         }
     }
 
     /* Close section */
     switch (output_format) {
         case mon_output_xml:
             fprintf(stream, "    </node_history>\n");
             break;
 
         default:
             break;
     }
 }
 
 static void
 print_ticket(gpointer name, gpointer value, gpointer data)
 {
     ticket_t *ticket = (ticket_t *) value;
     FILE *stream = (FILE *) data;
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("* %s:\t%s%s", ticket->id,
                      (ticket->granted? "granted" : "revoked"),
                      (ticket->standby? " [standby]" : ""));
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "  <li>%s: %s%s", ticket->id,
                     (ticket->granted? "granted" : "revoked"),
                     (ticket->standby? " [standby]" : ""));
             break;
 
         case mon_output_xml:
             fprintf(stream, "        <ticket id=\"%s\" status=\"%s\" standby=\"%s\"",
                     ticket->id, (ticket->granted? "granted" : "revoked"),
                     (ticket->standby? "true" : "false"));
             break;
 
         default:
             break;
     }
     if (ticket->last_granted > -1) {
         print_nvpair(stdout, "last-granted", NULL, NULL, ticket->last_granted);
     }
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "</li>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, " />\n");
             break;
 
         default:
             break;
     }
 }
 
 static void
 print_cluster_tickets(FILE *stream, pe_working_set_t * data_set)
 {
     /* Print section heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\nTickets:\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <hr />\n <h2>Tickets</h2>\n <ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <tickets>\n");
             break;
 
         default:
             break;
     }
 
     /* Print each ticket */
     g_hash_table_foreach(data_set->tickets, print_ticket, stream);
 
     /* Close section */
     switch (output_format) {
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " </ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    </tickets>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Return human-friendly string representing node name
  *
  * The returned string will be in the format
  *    uname[@hostUname] [(nodeID)]
  * "@hostUname" will be printed if the node is a guest node.
  * "(nodeID)" will be printed if the node ID is different from the node uname,
  *  and detailed output has been requested.
  *
  * \param[in] node  Node to represent
  * \return Newly allocated string with representation of node name
  * \note It is the caller's responsibility to free the result with free().
  */
 static char *
 get_node_display_name(node_t *node)
 {
     char *node_name;
     const char *node_host = NULL;
     const char *node_id = NULL;
     int name_len;
 
     CRM_ASSERT((node != NULL) && (node->details != NULL) && (node->details->uname != NULL));
 
     /* Host is displayed only if this is a guest node */
     if (is_container_remote_node(node)) {
         if (node->details->remote_rsc->running_on) {
             /* running_on is a list, but guest nodes will have exactly one entry
              * unless they are in the process of migrating, in which case they
              * will have two; either way, we can use the first item in the list
              */
             node_t *host_node = (node_t *) node->details->remote_rsc->running_on->data;
 
             if (host_node && host_node->details) {
                 node_host = host_node->details->uname;
             }
         }
         if (node_host == NULL) {
             node_host = ""; /* so we at least get "uname@" to indicate guest */
         }
     }
 
     /* Node ID is displayed if different from uname and detail is requested */
     if (print_clone_detail && safe_str_neq(node->details->uname, node->details->id)) {
         node_id = node->details->id;
     }
 
     /* Determine name length */
     name_len = strlen(node->details->uname) + 1;
     if (node_host) {
         name_len += strlen(node_host) + 1; /* "@node_host" */
     }
     if (node_id) {
         name_len += strlen(node_id) + 3; /* + " (node_id)" */
     }
 
     /* Allocate and populate display name */
     node_name = malloc(name_len);
     CRM_ASSERT(node_name != NULL);
     strcpy(node_name, node->details->uname);
     if (node_host) {
         strcat(node_name, "@");
         strcat(node_name, node_host);
     }
     if (node_id) {
         strcat(node_name, " (");
         strcat(node_name, node_id);
         strcat(node_name, ")");
     }
     return node_name;
 }
 
 /*!
  * \internal
  * \brief Print a negative location constraint
  *
  * \param[in] stream     File stream to display output to
  * \param[in] node       Node affected by constraint
  * \param[in] location   Constraint to print
  */
 static void print_ban(FILE *stream, node_t *node, rsc_to_node_t *location)
 {
     char *node_name = NULL;
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             node_name = get_node_display_name(node);
             print_as(" %s\tprevents %s from running %son %s\n",
                      location->id, location->rsc_lh->id,
                      ((location->role_filter == RSC_ROLE_MASTER)? "as Master " : ""),
                      node_name);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             node_name = get_node_display_name(node);
             fprintf(stream, "  <li>%s prevents %s from running %son %s</li>\n",
                      location->id, location->rsc_lh->id,
                      ((location->role_filter == RSC_ROLE_MASTER)? "as Master " : ""),
                      node_name);
             break;
 
         case mon_output_xml:
             fprintf(stream,
                     "        <ban id=\"%s\" resource=\"%s\" node=\"%s\" weight=\"%d\" master_only=\"%s\" />\n",
                     location->id, location->rsc_lh->id, node->details->uname, node->weight,
                     ((location->role_filter == RSC_ROLE_MASTER)? "true" : "false"));
             break;
 
         default:
             break;
     }
     free(node_name);
 }
 
 /*!
  * \internal
  * \brief Print section for negative location constraints
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set corresponding to CIB status to display
  */
 static void print_neg_locations(FILE *stream, pe_working_set_t *data_set)
 {
     GListPtr gIter, gIter2;
 
     /* Print section heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\nNegative Location Constraints:\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <hr />\n <h2>Negative Location Constraints</h2>\n <ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <bans>\n");
             break;
 
         default:
             break;
     }
 
     /* Print each ban */
     for (gIter = data_set->placement_constraints; gIter != NULL; gIter = gIter->next) {
         rsc_to_node_t *location = (rsc_to_node_t *) gIter->data;
         if (!g_str_has_prefix(location->id, print_neg_location_prefix))
             continue;
         for (gIter2 = location->node_list_rh; gIter2 != NULL; gIter2 = gIter2->next) {
             node_t *node = (node_t *) gIter2->data;
 
             if (node->weight < 0) {
                 print_ban(stream, node, location);
             }
         }
     }
 
     /* Close section */
     switch (output_format) {
         case mon_output_cgi:
         case mon_output_html:
             fprintf(stream, " </ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    </bans>\n");
             break;
 
         default:
             break;
     }
 }
 
 static void
 crm_mon_get_parameters(resource_t *rsc, pe_working_set_t * data_set)
 {
     get_rsc_attributes(rsc->parameters, rsc, NULL, data_set);
     crm_trace("Beekhof: unpacked params for %s (%d)", rsc->id, g_hash_table_size(rsc->parameters));
     if(rsc->children) {
         GListPtr gIter = NULL;
 
         for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
             crm_mon_get_parameters(gIter->data, data_set);
         }
     }
 }
 
 /*!
  * \internal
  * \brief Print node attributes section
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_node_attributes(FILE *stream, pe_working_set_t *data_set)
 {
     GListPtr gIter = NULL;
 
     /* Print section heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\nNode Attributes:\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <hr />\n <h2>Node Attributes</h2>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <node_attributes>\n");
             break;
 
         default:
             break;
     }
 
     /* Unpack all resource parameters (it would be more efficient to do this
      * only when needed for the first time in print_attr_msg())
      */
     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
         crm_mon_get_parameters(gIter->data, data_set);
     }
 
     /* Display each node's attributes */
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         struct mon_attr_data data;
 
         data.stream = stream;
         data.node = (node_t *) gIter->data;
 
         if (data.node && data.node->details && data.node->details->online) {
             print_node_start(stream, data.node);
             g_hash_table_foreach(data.node->details->attrs, create_attr_list, NULL);
             g_list_foreach(attr_list, print_node_attribute, &data);
             g_list_free(attr_list);
             attr_list = NULL;
             print_node_end(stream);
         }
     }
 
     /* Print section footer */
     switch (output_format) {
         case mon_output_xml:
             fprintf(stream, "    </node_attributes>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Return resource display options corresponding to command-line choices
  *
  * \return Bitmask of pe_print_options suitable for resource print functions
  */
 static int
 get_resource_display_options(void)
 {
     int print_opts;
 
     /* Determine basic output format */
     switch (output_format) {
         case mon_output_console:
             print_opts = pe_print_ncurses;
             break;
         case mon_output_html:
         case mon_output_cgi:
             print_opts = pe_print_html;
             break;
         case mon_output_xml:
             print_opts = pe_print_xml;
             break;
         default:
             print_opts = pe_print_printf;
             break;
     }
 
     /* Add optional display elements */
     if (print_pending) {
         print_opts |= pe_print_pending;
     }
     if (print_clone_detail) {
         print_opts |= pe_print_clone_details;
     }
     if (!inactive_resources) {
         print_opts |= pe_print_clone_active;
     }
     if (print_brief) {
         print_opts |= pe_print_brief;
     }
     return print_opts;
 }
 
 /*!
  * \internal
  * \brief Return human-friendly string representing current time
  *
  * \return Current time as string (as by ctime() but without newline) on success
  *         or "Could not determine current time" on error
  * \note The return value points to a statically allocated string which might be
  *       overwritten by subsequent calls to any of the C library date and time functions.
  */
 static const char *
 crm_now_string(void)
 {
     time_t a_time = time(NULL);
     char *since_epoch = ctime(&a_time);
 
     if ((a_time == (time_t) -1) || (since_epoch == NULL)) {
         return "Could not determine current time";
     }
     since_epoch[strlen(since_epoch) - 1] = EOS; /* trim newline */
     return (since_epoch);
 }
 
 /*!
  * \internal
  * \brief Print header for cluster summary if needed
  *
  * \param[in] stream     File stream to display output to
  */
 static void
 print_cluster_summary_header(FILE *stream)
 {
     switch (output_format) {
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <h2>Cluster Summary</h2>\n <p>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <summary>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print footer for cluster summary if needed
  *
  * \param[in] stream     File stream to display output to
  */
 static void
 print_cluster_summary_footer(FILE *stream)
 {
     switch (output_format) {
         case mon_output_cgi:
         case mon_output_html:
             fprintf(stream, " </p>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    </summary>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print times the display was last updated and CIB last changed
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_cluster_times(FILE *stream, pe_working_set_t *data_set)
 {
     const char *last_written = crm_element_value(data_set->input, XML_CIB_ATTR_WRITTEN);
     const char *user = crm_element_value(data_set->input, XML_ATTR_UPDATE_USER);
     const char *client = crm_element_value(data_set->input, XML_ATTR_UPDATE_CLIENT);
     const char *origin = crm_element_value(data_set->input, XML_ATTR_UPDATE_ORIG);
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("Last updated: %s", crm_now_string());
             print_as((user || client || origin)? "\n" : "\t\t");
             print_as("Last change: %s", last_written ? last_written : "");
             if (user) {
                 print_as(" by %s", user);
             }
             if (client) {
                 print_as(" via %s", client);
             }
             if (origin) {
                 print_as(" on %s", origin);
             }
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <b>Last updated:</b> %s<br/>\n", crm_now_string());
             fprintf(stream, " <b>Last change:</b> %s", last_written ? last_written : "");
             if (user) {
                 fprintf(stream, " by %s", user);
             }
             if (client) {
                 fprintf(stream, " via %s", client);
             }
             if (origin) {
                 fprintf(stream, " on %s", origin);
             }
             fprintf(stream, "<br/>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "        <last_update time=\"%s\" />\n", crm_now_string());
             fprintf(stream, "        <last_change time=\"%s\" user=\"%s\" client=\"%s\" origin=\"%s\" />\n",
                     last_written ? last_written : "", user ? user : "",
                     client ? client : "", origin ? origin : "");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print cluster stack
  *
  * \param[in] stream     File stream to display output to
  * \param[in] stack_s    Stack name
  */
 static void
 print_cluster_stack(FILE *stream, const char *stack_s)
 {
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("Stack: %s\n", stack_s);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <b>Stack:</b> %s<br/>\n", stack_s);
             break;
 
         case mon_output_xml:
             fprintf(stream, "        <stack type=\"%s\" />\n", stack_s);
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print current DC and its version
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_cluster_dc(FILE *stream, pe_working_set_t *data_set)
 {
     node_t *dc = data_set->dc_node;
     xmlNode *dc_version = get_xpath_object("//nvpair[@name='dc-version']",
                                            data_set->input, LOG_DEBUG);
     const char *dc_version_s = dc_version?
                                crm_element_value(dc_version, XML_NVPAIR_ATTR_VALUE)
                                : NULL;
     const char *quorum = crm_element_value(data_set->input, XML_ATTR_HAVE_QUORUM);
     char *dc_name = dc? get_node_display_name(dc) : NULL;
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("Current DC: ");
             if (dc) {
                 print_as("%s (version %s) - partition %s quorum\n",
                          dc_name, (dc_version_s? dc_version_s : "unknown"),
                          (crm_is_true(quorum) ? "with" : "WITHOUT"));
             } else {
                 print_as("NONE\n");
             }
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <b>Current DC:</b> ");
             if (dc) {
                 fprintf(stream, "%s (version %s) - partition %s quorum",
                         dc_name, (dc_version_s? dc_version_s : "unknown"),
                         (crm_is_true(quorum)? "with" : "<font color=\"red\"><b>WITHOUT</b></font>"));
             } else {
                 fprintf(stream, "<font color=\"red\"><b>NONE</b></font>");
             }
             fprintf(stream, "<br/>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream,  "        <current_dc ");
             if (dc) {
                 fprintf(stream,
                         "present=\"true\" version=\"%s\" name=\"%s\" id=\"%s\" with_quorum=\"%s\"",
                         (dc_version_s? dc_version_s : ""), dc->details->uname, dc->details->id,
                         (crm_is_true(quorum) ? "true" : "false"));
             } else {
                 fprintf(stream, "present=\"false\"");
             }
             fprintf(stream, " />\n");
             break;
 
         default:
             break;
     }
     free(dc_name);
 }
 
 /*!
  * \internal
  * \brief Print counts of configured nodes and resources
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  * \param[in] stack_s    Stack name
  */
 static void
 print_cluster_counts(FILE *stream, pe_working_set_t *data_set, const char *stack_s)
 {
     int nnodes = g_list_length(data_set->nodes);
     int nresources = count_resources(data_set, NULL);
     xmlNode *quorum_node = get_xpath_object("//nvpair[@name='" XML_ATTR_EXPECTED_VOTES "']",
                                             data_set->input, LOG_DEBUG);
     const char *quorum_votes = quorum_node?
                                crm_element_value(quorum_node, XML_NVPAIR_ATTR_VALUE)
                                : "unknown";
 
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
 
             print_as("\n%d node%s configured", nnodes, s_if_plural(nnodes));
             if (stack_s && strstr(stack_s, "classic openais") != NULL) {
                 print_as(" (%s expected votes)", quorum_votes);
             }
             print_as("\n");
 
             print_as("%d resource%s configured",
                      nresources, s_if_plural(nresources));
             if(data_set->disabled_resources || data_set->blocked_resources) {
                 print_as(" (");
                 if (data_set->disabled_resources) {
                     print_as("%d DISABLED", data_set->disabled_resources);
                 }
                 if (data_set->disabled_resources && data_set->blocked_resources) {
                     print_as(", ");
                 }
                 if (data_set->blocked_resources) {
                     print_as("%d BLOCKED from starting due to failure",
                              data_set->blocked_resources);
                 }
                 print_as(")");
             }
             print_as("\n");
 
             break;
 
         case mon_output_html:
         case mon_output_cgi:
 
             fprintf(stream, " %d node%s configured", nnodes, s_if_plural(nnodes));
             if (stack_s && strstr(stack_s, "classic openais") != NULL) {
                 fprintf(stream, " (%s expected votes)", quorum_votes);
             }
             fprintf(stream, "<br/>\n");
 
             fprintf(stream, " %d resource%s configured",
                     nresources, s_if_plural(nresources));
             if (data_set->disabled_resources || data_set->blocked_resources) {
                 fprintf(stream, " (");
                 if (data_set->disabled_resources) {
                     fprintf(stream, "%d <strong>DISABLED</strong>",
                             data_set->disabled_resources);
                 }
                 if (data_set->disabled_resources && data_set->blocked_resources) {
                     fprintf(stream, ", ");
                 }
                 if (data_set->blocked_resources) {
                     fprintf(stream,
                             "%d <strong>BLOCKED</strong> from starting due to failure",
                             data_set->blocked_resources);
                 }
                 fprintf(stream, ")");
             }
             fprintf(stream, "<br/>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream,
                     "        <nodes_configured number=\"%d\" expected_votes=\"%s\" />\n",
                     g_list_length(data_set->nodes), quorum_votes);
             fprintf(stream,
                     "        <resources_configured number=\"%d\" disabled=\"%d\" blocked=\"%d\" />\n",
                     count_resources(data_set, NULL),
                     data_set->disabled_resources, data_set->blocked_resources);
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print cluster-wide options
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  *
  * \note Currently this is only implemented for HTML and XML output, and
  *       prints only a few options. If there is demand, more could be added.
  */
 static void
 print_cluster_options(FILE *stream, pe_working_set_t *data_set)
 {
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
                 print_as("\n              *** Resource management is DISABLED ***");
                 print_as("\n  The cluster will not attempt to start, stop or recover services");
                 print_as("\n");
             }
             break;
 
         case mon_output_html:
             fprintf(stream, " </p>\n <h3>Config Options</h3>\n");
             fprintf(stream, " <table>\n");
             fprintf(stream, "  <tr><th>STONITH of failed nodes</th><td>%s</td></tr>\n",
                     is_set(data_set->flags, pe_flag_stonith_enabled)? "enabled" : "disabled");
 
             fprintf(stream, "  <tr><th>Cluster is</th><td>%ssymmetric</td></tr>\n",
                     is_set(data_set->flags, pe_flag_symmetric_cluster)? "" : "a");
 
             fprintf(stream, "  <tr><th>No Quorum Policy</th><td>");
             switch (data_set->no_quorum_policy) {
                 case no_quorum_freeze:
                     fprintf(stream, "Freeze resources");
                     break;
                 case no_quorum_stop:
                     fprintf(stream, "Stop ALL resources");
                     break;
                 case no_quorum_ignore:
                     fprintf(stream, "Ignore");
                     break;
                 case no_quorum_suicide:
                     fprintf(stream, "Suicide");
                     break;
             }
             fprintf(stream, "</td></tr>\n");
 
             fprintf(stream, "  <tr><th>Resource management</th><td>");
             if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
                 fprintf(stream, "<strong>DISABLED</strong> (the cluster will "
                                 "not attempt to start, stop or recover services)");
             } else {
                 fprintf(stream, "enabled");
             }
             fprintf(stream, "</td></tr>\n");
 
             fprintf(stream, "</table>\n <p>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "        <cluster_options");
             fprintf(stream, " stonith-enabled=\"%s\"",
                     is_set(data_set->flags, pe_flag_stonith_enabled)?
                     "true" : "false");
             fprintf(stream, " symmetric-cluster=\"%s\"",
                     is_set(data_set->flags, pe_flag_symmetric_cluster)?
                     "true" : "false");
             fprintf(stream, " no-quorum-policy=\"");
             switch (data_set->no_quorum_policy) {
                 case no_quorum_freeze:
                     fprintf(stream, "freeze");
                     break;
                 case no_quorum_stop:
                     fprintf(stream, "stop");
                     break;
                 case no_quorum_ignore:
                     fprintf(stream, "ignore");
                     break;
                 case no_quorum_suicide:
                     fprintf(stream, "suicide");
                     break;
             }
             fprintf(stream, "\"");
             fprintf(stream, " maintenance-mode=\"%s\"",
                     is_set(data_set->flags, pe_flag_maintenance_mode)?
                     "true" : "false");
             fprintf(stream, " />\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Get the name of the stack in use (or "unknown" if not available)
  *
  * \param[in] data_set   Working set of CIB state
  *
  * \return String representing stack name
  */
 static const char *
 get_cluster_stack(pe_working_set_t *data_set)
 {
     xmlNode *stack = get_xpath_object("//nvpair[@name='cluster-infrastructure']",
                                       data_set->input, LOG_DEBUG);
     return stack? crm_element_value(stack, XML_NVPAIR_ATTR_VALUE) : "unknown";
 }
 
 /*!
  * \internal
  * \brief Print a summary of cluster-wide information
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_cluster_summary(FILE *stream, pe_working_set_t *data_set)
 {
     const char *stack_s = get_cluster_stack(data_set);
     gboolean header_printed = FALSE;
 
     if (show & mon_show_stack) {
         if (header_printed == FALSE) {
             print_cluster_summary_header(stream);
             header_printed = TRUE;
         }
         print_cluster_stack(stream, stack_s);
     }
 
     /* Always print DC if none, even if not requested */
     if ((data_set->dc_node == NULL) || (show & mon_show_dc)) {
         if (header_printed == FALSE) {
             print_cluster_summary_header(stream);
             header_printed = TRUE;
         }
         print_cluster_dc(stream, data_set);
     }
 
     if (show & mon_show_times) {
         if (header_printed == FALSE) {
             print_cluster_summary_header(stream);
             header_printed = TRUE;
         }
         print_cluster_times(stream, data_set);
     }
 
     if (is_set(data_set->flags, pe_flag_maintenance_mode)
         || data_set->disabled_resources
         || data_set->blocked_resources
         || is_set(show, mon_show_count)) {
         if (header_printed == FALSE) {
             print_cluster_summary_header(stream);
             header_printed = TRUE;
         }
         print_cluster_counts(stream, data_set, stack_s);
     }
 
     /* There is not a separate option for showing cluster options, so show with
      * stack for now; a separate option could be added if there is demand
      */
     if (show & mon_show_stack) {
         print_cluster_options(stream, data_set);
     }
 
     if (header_printed) {
         print_cluster_summary_footer(stream);
     }
 }
 
 /*!
  * \internal
  * \brief Print a failed action
  *
  * \param[in] stream     File stream to display output to
  * \param[in] xml_op     Root of XML tree describing failed action
  */
 static void
 print_failed_action(FILE *stream, xmlNode *xml_op)
 {
     const char *op_key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
     const char *op_key_attr = "op_key";
     const char *last = crm_element_value(xml_op, XML_RSC_OP_LAST_CHANGE);
     const char *node = crm_element_value(xml_op, XML_ATTR_UNAME);
     const char *call = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
     const char *exit_reason = crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON);
     int rc = crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_RC), "0");
     int status = crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS), "0");
     char *exit_reason_cleaned;
 
     /* If no op_key was given, use id instead */
     if (op_key == NULL) {
         op_key = ID(xml_op);
         op_key_attr = "id";
     }
 
     /* If no exit reason was given, use "none" */
     if (exit_reason == NULL) {
         exit_reason = "none";
     }
 
     /* Print common action information */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("* %s on %s '%s' (%d): call=%s, status=%s, exitreason='%s'",
                      op_key, node, services_ocf_exitcode_str(rc), rc,
                      call, services_lrm_status_str(status), exit_reason);
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "  <li>%s on %s '%s' (%d): call=%s, status=%s, exitreason='%s'",
                      op_key, node, services_ocf_exitcode_str(rc), rc,
                      call, services_lrm_status_str(status), exit_reason);
             break;
 
         case mon_output_xml:
             exit_reason_cleaned = crm_xml_escape(exit_reason);
             fprintf(stream, "        <failure %s=\"%s\" node=\"%s\"",
                     op_key_attr, op_key, node);
             fprintf(stream, " exitstatus=\"%s\" exitreason=\"%s\" exitcode=\"%d\"",
                     services_ocf_exitcode_str(rc), exit_reason_cleaned, rc);
             fprintf(stream, " call=\"%s\" status=\"%s\"",
                     call, services_lrm_status_str(status));
             free(exit_reason_cleaned);
             break;
 
         default:
             break;
     }
 
     /* If last change was given, print timing information as well */
     if (last) {
         time_t run_at = crm_parse_int(last, "0");
         char *run_at_s = ctime(&run_at);
 
         if (run_at_s) {
             run_at_s[24] = 0; /* Overwrite the newline */
         }
 
         switch (output_format) {
             case mon_output_plain:
             case mon_output_console:
                 print_as(",\n    last-rc-change='%s', queued=%sms, exec=%sms",
                          run_at_s? run_at_s : "",
                          crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
                          crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
                 break;
 
             case mon_output_html:
             case mon_output_cgi:
                 fprintf(stream, " last-rc-change='%s', queued=%sms, exec=%sms",
                         run_at_s? run_at_s : "",
                         crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
                         crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
                 break;
 
             case mon_output_xml:
                 fprintf(stream,
                         " last-rc-change=\"%s\" queued=\"%s\" exec=\"%s\" interval=\"%d\" task=\"%s\"",
                         run_at_s? run_at_s : "",
                         crm_element_value(xml_op, XML_RSC_OP_T_QUEUE),
                         crm_element_value(xml_op, XML_RSC_OP_T_EXEC),
                         crm_parse_int(crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL), "0"),
                         crm_element_value(xml_op, XML_LRM_ATTR_TASK));
                 break;
 
             default:
                 break;
         }
     }
 
     /* End the action listing */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, "</li>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, " />\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print a section for failed actions
  *
  * \param[in] stream     File stream to display output to
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_failed_actions(FILE *stream, pe_working_set_t *data_set)
 {
     xmlNode *xml_op = NULL;
 
     /* Print section heading */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\nFailed Actions:\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " <hr />\n <h2>Failed Actions</h2>\n <ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    <failures>\n");
             break;
 
         default:
             break;
     }
 
     /* Print each failed action */
     for (xml_op = __xml_first_child(data_set->failed); xml_op != NULL;
          xml_op = __xml_next(xml_op)) {
         print_failed_action(stream, xml_op);
     }
 
     /* End section */
     switch (output_format) {
         case mon_output_plain:
         case mon_output_console:
             print_as("\n");
             break;
 
         case mon_output_html:
         case mon_output_cgi:
             fprintf(stream, " </ul>\n");
             break;
 
         case mon_output_xml:
             fprintf(stream, "    </failures>\n");
             break;
 
         default:
             break;
     }
 }
 
 /*!
  * \internal
  * \brief Print cluster status to screen
  *
  * This uses the global display preferences set by command-line options
  * to display cluster status in a human-friendly way.
  *
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_status(pe_working_set_t * data_set)
 {
     GListPtr gIter = NULL;
     int print_opts = get_resource_display_options();
 
     /* space-separated lists of node names */
     char *online_nodes = NULL;
     char *online_remote_nodes = NULL;
     char *online_guest_nodes = NULL;
     char *offline_nodes = NULL;
     char *offline_remote_nodes = NULL;
 
     if (output_format == mon_output_console) {
         blank_screen();
     }
     print_cluster_summary(stdout, data_set);
     print_as("\n");
 
     /* Gather node information (and print if in bad state or grouping by node) */
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
         const char *node_mode = NULL;
         char *node_name = get_node_display_name(node);
 
         /* Get node mode */
         if (node->details->unclean) {
             if (node->details->online) {
                 node_mode = "UNCLEAN (online)";
 
             } else if (node->details->pending) {
                 node_mode = "UNCLEAN (pending)";
 
             } else {
                 node_mode = "UNCLEAN (offline)";
             }
 
         } else if (node->details->pending) {
             node_mode = "pending";
 
         } else if (node->details->standby_onfail && node->details->online) {
             node_mode = "standby (on-fail)";
 
         } else if (node->details->standby) {
             if (node->details->online) {
                 node_mode = "standby";
             } else {
                 node_mode = "OFFLINE (standby)";
             }
 
         } else if (node->details->maintenance) {
             if (node->details->online) {
                 node_mode = "maintenance";
             } else {
                 node_mode = "OFFLINE (maintenance)";
             }
 
         } else if (node->details->online) {
             node_mode = "online";
             if (group_by_node == FALSE) {
                 if (is_container_remote_node(node)) {
                     online_guest_nodes = add_list_element(online_guest_nodes, node_name);
                 } else if (is_baremetal_remote_node(node)) {
                     online_remote_nodes = add_list_element(online_remote_nodes, node_name);
                 } else {
                     online_nodes = add_list_element(online_nodes, node_name);
                 }
                 free(node_name);
                 continue;
             }
         } else {
             node_mode = "OFFLINE";
             if (group_by_node == FALSE) {
                 if (is_baremetal_remote_node(node)) {
                     offline_remote_nodes = add_list_element(offline_remote_nodes, node_name);
                 } else if (is_container_remote_node(node)) {
                     /* ignore offline guest nodes */
                 } else {
                     offline_nodes = add_list_element(offline_nodes, node_name);
                 }
                 free(node_name);
                 continue;
             }
         }
 
         /* If we get here, node is in bad state, or we're grouping by node */
 
         /* Print the node name and status */
         if (is_container_remote_node(node)) {
             print_as("Guest");
         } else if (is_baremetal_remote_node(node)) {
             print_as("Remote");
         }
         print_as("Node %s: %s\n", node_name, node_mode);
 
         /* If we're grouping by node, print its resources */
         if (group_by_node) {
             if (print_brief) {
                 print_rscs_brief(node->details->running_rsc, "\t", print_opts | pe_print_rsconly,
                                  stdout, FALSE);
             } else {
                 GListPtr gIter2 = NULL;
 
                 for (gIter2 = node->details->running_rsc; gIter2 != NULL; gIter2 = gIter2->next) {
                     resource_t *rsc = (resource_t *) gIter2->data;
 
                     rsc->fns->print(rsc, "\t", print_opts | pe_print_rsconly, stdout);
                 }
             }
         }
         free(node_name);
     }
 
     /* If we're not grouping by node, summarize nodes by status */
     if (online_nodes) {
         print_as("Online: [%s ]\n", online_nodes);
         free(online_nodes);
     }
     if (offline_nodes) {
         print_as("OFFLINE: [%s ]\n", offline_nodes);
         free(offline_nodes);
     }
     if (online_remote_nodes) {
         print_as("RemoteOnline: [%s ]\n", online_remote_nodes);
         free(online_remote_nodes);
     }
     if (offline_remote_nodes) {
         print_as("RemoteOFFLINE: [%s ]\n", offline_remote_nodes);
         free(offline_remote_nodes);
     }
     if (online_guest_nodes) {
         print_as("GuestOnline: [%s ]\n", online_guest_nodes);
         free(online_guest_nodes);
     }
 
     /* Print resources section, if needed */
     print_resources(stdout, data_set, print_opts);
 
     /* print Node Attributes section if requested */
     if (show & mon_show_attributes) {
         print_node_attributes(stdout, data_set);
     }
 
     /* If requested, print resource operations (which includes failcounts)
      * or just failcounts
      */
     if (show & (mon_show_operations | mon_show_failcounts)) {
         print_node_summary(stdout, data_set,
                            ((show & mon_show_operations)? TRUE : FALSE));
     }
 
     /* If there were any failed actions, print them */
     if (xml_has_children(data_set->failed)) {
         print_failed_actions(stdout, data_set);
     }
 
     /* Print tickets if requested */
     if (show & mon_show_tickets) {
         print_cluster_tickets(stdout, data_set);
     }
 
     /* Print negative location constraints if requested */
     if (show & mon_show_bans) {
         print_neg_locations(stdout, data_set);
     }
 
 #if CURSES_ENABLED
     if (output_format == mon_output_console) {
         refresh();
     }
 #endif
 }
 
 /*!
  * \internal
  * \brief Print cluster status in XML format
  *
  * \param[in] data_set   Working set of CIB state
  */
 static void
 print_xml_status(pe_working_set_t * data_set)
 {
     FILE *stream = stdout;
     GListPtr gIter = NULL;
     int print_opts = get_resource_display_options();
 
     fprintf(stream, "<?xml version=\"1.0\"?>\n");
     fprintf(stream, "<crm_mon version=\"%s\">\n", VERSION);
 
     print_cluster_summary(stream, data_set);
 
     /*** NODES ***/
     fprintf(stream, "    <nodes>\n");
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
         const char *node_type = "unknown";
 
         switch (node->details->type) {
             case node_member:
                 node_type = "member";
                 break;
             case node_remote:
                 node_type = "remote";
                 break;
             case node_ping:
                 node_type = "ping";
                 break;
         }
 
         fprintf(stream, "        <node name=\"%s\" ", node->details->uname);
         fprintf(stream, "id=\"%s\" ", node->details->id);
         fprintf(stream, "online=\"%s\" ", node->details->online ? "true" : "false");
         fprintf(stream, "standby=\"%s\" ", node->details->standby ? "true" : "false");
         fprintf(stream, "standby_onfail=\"%s\" ", node->details->standby_onfail ? "true" : "false");
         fprintf(stream, "maintenance=\"%s\" ", node->details->maintenance ? "true" : "false");
         fprintf(stream, "pending=\"%s\" ", node->details->pending ? "true" : "false");
         fprintf(stream, "unclean=\"%s\" ", node->details->unclean ? "true" : "false");
         fprintf(stream, "shutdown=\"%s\" ", node->details->shutdown ? "true" : "false");
         fprintf(stream, "expected_up=\"%s\" ", node->details->expected_up ? "true" : "false");
         fprintf(stream, "is_dc=\"%s\" ", node->details->is_dc ? "true" : "false");
         fprintf(stream, "resources_running=\"%d\" ", g_list_length(node->details->running_rsc));
         fprintf(stream, "type=\"%s\" ", node_type);
         if (is_container_remote_node(node)) {
             fprintf(stream, "id_as_resource=\"%s\" ", node->details->remote_rsc->container->id);
         }
 
         if (group_by_node) {
             GListPtr lpc2 = NULL;
 
             fprintf(stream, ">\n");
             for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
                 resource_t *rsc = (resource_t *) lpc2->data;
 
                 rsc->fns->print(rsc, "            ", print_opts | pe_print_rsconly, stream);
             }
             fprintf(stream, "        </node>\n");
         } else {
             fprintf(stream, "/>\n");
         }
     }
     fprintf(stream, "    </nodes>\n");
 
     /* Print resources section, if needed */
     print_resources(stream, data_set, print_opts);
 
     /* print Node Attributes section if requested */
     if (show & mon_show_attributes) {
         print_node_attributes(stream, data_set);
     }
 
     /* If requested, print resource operations (which includes failcounts)
      * or just failcounts
      */
     if (show & (mon_show_operations | mon_show_failcounts)) {
         print_node_summary(stream, data_set,
                            ((show & mon_show_operations)? TRUE : FALSE));
     }
 
     /* If there were any failed actions, print them */
     if (xml_has_children(data_set->failed)) {
         print_failed_actions(stream, data_set);
     }
 
     /* Print tickets if requested */
     if (show & mon_show_tickets) {
         print_cluster_tickets(stream, data_set);
     }
 
     /* Print negative location constraints if requested */
     if (show & mon_show_bans) {
         print_neg_locations(stream, data_set);
     }
 
     fprintf(stream, "</crm_mon>\n");
     fflush(stream);
     fclose(stream);
 }
 
 /*!
  * \internal
  * \brief Print cluster status in HTML format (with HTTP headers if CGI)
  *
  * \param[in] data_set   Working set of CIB state
  * \param[in] filename   Name of file to write HTML to (ignored if CGI)
  *
  * \return 0 on success, -1 on error
  */
 static int
 print_html_status(pe_working_set_t * data_set, const char *filename)
 {
     FILE *stream;
     GListPtr gIter = NULL;
     char *filename_tmp = NULL;
     int print_opts = get_resource_display_options();
 
     if (output_format == mon_output_cgi) {
         stream = stdout;
         fprintf(stream, "Content-type: text/html\n\n");
 
     } else {
         filename_tmp = crm_concat(filename, "tmp", '.');
         stream = fopen(filename_tmp, "w");
         if (stream == NULL) {
             crm_perror(LOG_ERR, "Cannot open %s for writing", filename_tmp);
             free(filename_tmp);
             return -1;
         }
     }
 
     fprintf(stream, "<html>\n");
     fprintf(stream, " <head>\n");
     fprintf(stream, "  <title>Cluster status</title>\n");
     fprintf(stream, "  <meta http-equiv=\"refresh\" content=\"%d\">\n", reconnect_msec / 1000);
     fprintf(stream, " </head>\n");
     fprintf(stream, "<body>\n");
 
     print_cluster_summary(stream, data_set);
 
     /*** NODE LIST ***/
 
     fprintf(stream, " <hr />\n <h2>Node List</h2>\n");
     fprintf(stream, "<ul>\n");
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
         node_t *node = (node_t *) gIter->data;
         char *node_name = get_node_display_name(node);
 
         fprintf(stream, "<li>Node: %s: ", node_name);
         if (node->details->standby_onfail && node->details->online) {
             fprintf(stream, "<font color=\"orange\">standby (on-fail)</font>\n");
         } else if (node->details->standby && node->details->online) {
             fprintf(stream, "<font color=\"orange\">standby</font>\n");
         } else if (node->details->standby) {
             fprintf(stream, "<font color=\"red\">OFFLINE (standby)</font>\n");
         } else if (node->details->maintenance && node->details->online) {
             fprintf(stream, "<font color=\"blue\">maintenance</font>\n");
         } else if (node->details->maintenance) {
             fprintf(stream, "<font color=\"red\">OFFLINE (maintenance)</font>\n");
         } else if (node->details->online) {
             fprintf(stream, "<font color=\"green\">online</font>\n");
         } else {
             fprintf(stream, "<font color=\"red\">OFFLINE</font>\n");
         }
         if (print_brief && group_by_node) {
             fprintf(stream, "<ul>\n");
             print_rscs_brief(node->details->running_rsc, NULL, print_opts | pe_print_rsconly,
                              stream, FALSE);
             fprintf(stream, "</ul>\n");
 
         } else if (group_by_node) {
             GListPtr lpc2 = NULL;
 
             fprintf(stream, "<ul>\n");
             for (lpc2 = node->details->running_rsc; lpc2 != NULL; lpc2 = lpc2->next) {
                 resource_t *rsc = (resource_t *) lpc2->data;
 
                 fprintf(stream, "<li>");
                 rsc->fns->print(rsc, NULL, print_opts | pe_print_rsconly, stream);
                 fprintf(stream, "</li>\n");
             }
             fprintf(stream, "</ul>\n");
         }
         fprintf(stream, "</li>\n");
         free(node_name);
     }
     fprintf(stream, "</ul>\n");
 
     /* Print resources section, if needed */
     print_resources(stream, data_set, print_opts);
 
     /* print Node Attributes section if requested */
     if (show & mon_show_attributes) {
         print_node_attributes(stream, data_set);
     }
 
     /* If requested, print resource operations (which includes failcounts)
      * or just failcounts
      */
     if (show & (mon_show_operations | mon_show_failcounts)) {
         print_node_summary(stream, data_set,
                            ((show & mon_show_operations)? TRUE : FALSE));
     }
 
     /* If there were any failed actions, print them */
     if (xml_has_children(data_set->failed)) {
         print_failed_actions(stream, data_set);
     }
 
     /* Print tickets if requested */
     if (show & mon_show_tickets) {
         print_cluster_tickets(stream, data_set);
     }
 
     /* Print negative location constraints if requested */
     if (show & mon_show_bans) {
         print_neg_locations(stream, data_set);
     }
 
     fprintf(stream, "</body>\n");
     fprintf(stream, "</html>\n");
     fflush(stream);
     fclose(stream);
 
     if (output_format != mon_output_cgi) {
         if (rename(filename_tmp, filename) != 0) {
             crm_perror(LOG_ERR, "Unable to rename %s->%s", filename_tmp, filename);
         }
         free(filename_tmp);
     }
     return 0;
 }
 
 #if ENABLE_SNMP
 #  include <net-snmp/net-snmp-config.h>
 #  include <net-snmp/snmpv3_api.h>
 #  include <net-snmp/agent/agent_trap.h>
 #  include <net-snmp/library/snmp_client.h>
 #  include <net-snmp/library/mib.h>
 #  include <net-snmp/library/snmp_debug.h>
 
 #  define add_snmp_field(list, oid_string, value) do {			\
 	oid name[MAX_OID_LEN];						\
         size_t name_length = MAX_OID_LEN;				\
 	if (snmp_parse_oid(oid_string, name, &name_length)) {		\
 	    int s_rc = snmp_add_var(list, name, name_length, 's', (value)); \
 	    if(s_rc != 0) {						\
 		crm_err("Could not add %s=%s rc=%d", oid_string, value, s_rc); \
 	    } else {							\
 		crm_trace("Added %s=%s", oid_string, value);		\
 	    }								\
 	} else {							\
 	    crm_err("Could not parse OID: %s", oid_string);		\
 	}								\
     } while(0)								\
 
 #  define add_snmp_field_int(list, oid_string, value) do {		\
 	oid name[MAX_OID_LEN];						\
         size_t name_length = MAX_OID_LEN;				\
 	if (snmp_parse_oid(oid_string, name, &name_length)) {		\
 	    if(NULL == snmp_pdu_add_variable(				\
 		   list, name, name_length, ASN_INTEGER,		\
 		   (u_char *) & value, sizeof(value))) {		\
 		crm_err("Could not add %s=%d", oid_string, value);	\
 	    } else {							\
 		crm_trace("Added %s=%d", oid_string, value);		\
 	    }								\
 	} else {							\
 	    crm_err("Could not parse OID: %s", oid_string);		\
 	}								\
     } while(0)								\
 
 static int
 snmp_input(int operation, netsnmp_session * session, int reqid, netsnmp_pdu * pdu, void *magic)
 {
     return 1;
 }
 
 static netsnmp_session *
 crm_snmp_init(const char *target, char *community)
 {
     static netsnmp_session *session = NULL;
 
 #  ifdef NETSNMPV53
     char target53[128];
 
     snprintf(target53, sizeof(target53), "%s:162", target);
 #  endif
 
     if (session) {
         return session;
     }
 
     if (target == NULL) {
         return NULL;
     }
 
     if (get_crm_log_level() > LOG_INFO) {
         char *debug_tokens = strdup("run:shell,snmptrap,tdomain");
 
         debug_register_tokens(debug_tokens);
         snmp_set_do_debugging(1);
     }
 
     session = calloc(1, sizeof(netsnmp_session));
     snmp_sess_init(session);
     session->version = SNMP_VERSION_2c;
     session->callback = snmp_input;
     session->callback_magic = NULL;
 
     if (community) {
         session->community_len = strlen(community);
         session->community = (unsigned char *)community;
     }
 
     session = snmp_add(session,
 #  ifdef NETSNMPV53
                        netsnmp_tdomain_transport(target53, 0, "udp"),
 #  else
                        netsnmp_transport_open_client("snmptrap", target),
 #  endif
                        NULL, NULL);
 
     if (session == NULL) {
         snmp_sess_perror("Could not create snmp transport", session);
     }
     return session;
 }
 
 #endif
 
 static int
 send_snmp_trap(const char *node, const char *rsc, const char *task, int target_rc, int rc,
                int status, const char *desc)
 {
     int ret = 1;
 
 #if ENABLE_SNMP
     static oid snmptrap_oid[] = { 1, 3, 6, 1, 6, 3, 1, 1, 4, 1, 0 };
     static oid sysuptime_oid[] = { 1, 3, 6, 1, 2, 1, 1, 3, 0 };
 
     netsnmp_pdu *trap_pdu;
     netsnmp_session *session = crm_snmp_init(snmp_target, snmp_community);
 
     trap_pdu = snmp_pdu_create(SNMP_MSG_TRAP2);
     if (!trap_pdu) {
         crm_err("Failed to create SNMP notification");
         return SNMPERR_GENERR;
     }
 
     if (1) {
         /* send uptime */
         char csysuptime[20];
         time_t now = time(NULL);
 
         sprintf(csysuptime, "%lld", (long long) now);
         snmp_add_var(trap_pdu, sysuptime_oid, sizeof(sysuptime_oid) / sizeof(oid), 't', csysuptime);
     }
 
     /* Indicate what the trap is by setting snmpTrapOid.0 */
     ret =
         snmp_add_var(trap_pdu, snmptrap_oid, sizeof(snmptrap_oid) / sizeof(oid), 'o',
                      snmp_crm_trap_oid);
     if (ret != 0) {
         crm_err("Failed set snmpTrapOid.0=%s", snmp_crm_trap_oid);
         return ret;
     }
 
     /* Add extries to the trap */
     if (rsc) {
         add_snmp_field(trap_pdu, snmp_crm_oid_rsc, rsc);
     }
     add_snmp_field(trap_pdu, snmp_crm_oid_node, node);
     add_snmp_field(trap_pdu, snmp_crm_oid_task, task);
     add_snmp_field(trap_pdu, snmp_crm_oid_desc, desc);
 
     add_snmp_field_int(trap_pdu, snmp_crm_oid_rc, rc);
     add_snmp_field_int(trap_pdu, snmp_crm_oid_trc, target_rc);
     add_snmp_field_int(trap_pdu, snmp_crm_oid_status, status);
 
     /* Send and cleanup */
     ret = snmp_send(session, trap_pdu);
     if (ret == 0) {
         /* error */
         snmp_sess_perror("Could not send SNMP trap", session);
         snmp_free_pdu(trap_pdu);
         ret = SNMPERR_GENERR;
     } else {
         ret = SNMPERR_SUCCESS;
     }
 #else
     crm_err("Sending SNMP traps is not supported by this installation");
 #endif
     return ret;
 }
 
 #if ENABLE_ESMTP
 #  include <auth-client.h>
 #  include <libesmtp.h>
 
 static void
 print_recipient_status(smtp_recipient_t recipient, const char *mailbox, void *arg)
 {
     const smtp_status_t *status;
 
     status = smtp_recipient_status(recipient);
     printf("%s: %d %s", mailbox, status->code, status->text);
 }
 
 static void
 event_cb(smtp_session_t session, int event_no, void *arg, ...)
 {
     int *ok;
     va_list alist;
 
     va_start(alist, arg);
     switch (event_no) {
         case SMTP_EV_CONNECT:
         case SMTP_EV_MAILSTATUS:
         case SMTP_EV_RCPTSTATUS:
         case SMTP_EV_MESSAGEDATA:
         case SMTP_EV_MESSAGESENT:
         case SMTP_EV_DISCONNECT:
             break;
 
         case SMTP_EV_WEAK_CIPHER:{
                 int bits = va_arg(alist, long);
                 ok = va_arg(alist, int *);
 
                 crm_debug("SMTP_EV_WEAK_CIPHER, bits=%d - accepted.", bits);
                 *ok = 1;
                 break;
             }
         case SMTP_EV_STARTTLS_OK:
             crm_debug("SMTP_EV_STARTTLS_OK - TLS started here.");
             break;
 
         case SMTP_EV_INVALID_PEER_CERTIFICATE:{
                 long vfy_result = va_arg(alist, long);
                 ok = va_arg(alist, int *);
 
                 /* There is a table in handle_invalid_peer_certificate() of mail-file.c */
                 crm_err("SMTP_EV_INVALID_PEER_CERTIFICATE: %ld", vfy_result);
                 *ok = 1;
                 break;
             }
         case SMTP_EV_NO_PEER_CERTIFICATE:
             ok = va_arg(alist, int *);
 
             crm_debug("SMTP_EV_NO_PEER_CERTIFICATE - accepted.");
             *ok = 1;
             break;
         case SMTP_EV_WRONG_PEER_CERTIFICATE:
             ok = va_arg(alist, int *);
 
             crm_debug("SMTP_EV_WRONG_PEER_CERTIFICATE - accepted.");
             *ok = 1;
             break;
         case SMTP_EV_NO_CLIENT_CERTIFICATE:
             ok = va_arg(alist, int *);
 
             crm_debug("SMTP_EV_NO_CLIENT_CERTIFICATE - accepted.");
             *ok = 1;
             break;
         default:
             crm_debug("Got event: %d - ignored.", event_no);
     }
     va_end(alist);
 }
 #endif
 
 #define BODY_MAX 2048
 
 #if ENABLE_ESMTP
 static void
 crm_smtp_debug(const char *buf, int buflen, int writing, void *arg)
 {
     char type = 0;
     int lpc = 0, last = 0, level = *(int *)arg;
 
     if (writing == SMTP_CB_HEADERS) {
         type = 'H';
     } else if (writing) {
         type = 'C';
     } else {
         type = 'S';
     }
 
     for (; lpc < buflen; lpc++) {
         switch (buf[lpc]) {
             case 0:
             case '\n':
                 if (last > 0) {
                     do_crm_log(level, "   %.*s", lpc - last, buf + last);
                 } else {
                     do_crm_log(level, "%c: %.*s", type, lpc - last, buf + last);
                 }
                 last = lpc + 1;
                 break;
         }
     }
 }
 #endif
 
 static int
 send_custom_trap(const char *node, const char *rsc, const char *task, int target_rc, int rc,
                  int status, const char *desc)
 {
     pid_t pid;
 
     /*setenv needs chars, these are ints */
     char *rc_s = crm_itoa(rc);
     char *status_s = crm_itoa(status);
     char *target_rc_s = crm_itoa(target_rc);
 
     crm_debug("Sending external notification to '%s' via '%s'", external_recipient, external_agent);
 
     if(rsc) {
         setenv("CRM_notify_rsc", rsc, 1);
     }
     if (external_recipient) {
         setenv("CRM_notify_recipient", external_recipient, 1);
     }
     setenv("CRM_notify_node", node, 1);
     setenv("CRM_notify_task", task, 1);
     setenv("CRM_notify_desc", desc, 1);
     setenv("CRM_notify_rc", rc_s, 1);
     setenv("CRM_notify_target_rc", target_rc_s, 1);
     setenv("CRM_notify_status", status_s, 1);
 
     pid = fork();
     if (pid == -1) {
         crm_perror(LOG_ERR, "notification fork() failed.");
     }
     if (pid == 0) {
         /* crm_debug("notification: I am the child. Executing the nofitication program."); */
         execl(external_agent, external_agent, NULL);
     }
 
     crm_trace("Finished running custom notification program '%s'.", external_agent);
     free(target_rc_s);
     free(status_s);
     free(rc_s);
     return 0;
 }
 
 static int
 send_smtp_trap(const char *node, const char *rsc, const char *task, int target_rc, int rc,
                int status, const char *desc)
 {
 #if ENABLE_ESMTP
     smtp_session_t session;
     smtp_message_t message;
     auth_context_t authctx;
     struct sigaction sa;
 
     int len = 25; /* Note: Check extra padding on the Subject line below */
     int noauth = 1;
     int smtp_debug = LOG_DEBUG;
     char crm_mail_body[BODY_MAX];
     char *crm_mail_subject = NULL;
 
     memset(&sa, 0, sizeof(struct sigaction));
 
     if (node == NULL) {
         node = "-";
     }
     if (rsc == NULL) {
         rsc = "-";
     }
     if (desc == NULL) {
         desc = "-";
     }
 
     if (crm_mail_to == NULL) {
         return 1;
     }
 
     if (crm_mail_host == NULL) {
         crm_mail_host = "localhost:25";
     }
 
     if (crm_mail_prefix == NULL) {
         crm_mail_prefix = "Cluster notification";
     }
 
     crm_debug("Sending '%s' mail to %s via %s", crm_mail_prefix, crm_mail_to, crm_mail_host);
 
     len += strlen(crm_mail_prefix);
     len += strlen(task);
     len += strlen(rsc);
     len += strlen(node);
     len += strlen(desc);
     len++;
 
     crm_mail_subject = calloc(1, len);
     /* If you edit this line, ensure you allocate enough memory for it by altering 'len' above */
     snprintf(crm_mail_subject, len, "%s - %s event for %s on %s: %s\r\n", crm_mail_prefix, task,
              rsc, node, desc);
 
     len = 0;
     len += snprintf(crm_mail_body + len, BODY_MAX - len, "\r\n%s\r\n", crm_mail_prefix);
     len += snprintf(crm_mail_body + len, BODY_MAX - len, "====\r\n\r\n");
     if (rc == target_rc) {
         len += snprintf(crm_mail_body + len, BODY_MAX - len,
                         "Completed operation %s for resource %s on %s\r\n", task, rsc, node);
     } else {
         len += snprintf(crm_mail_body + len, BODY_MAX - len,
                         "Operation %s for resource %s on %s failed: %s\r\n", task, rsc, node, desc);
     }
 
     len += snprintf(crm_mail_body + len, BODY_MAX - len, "\r\nDetails:\r\n");
     len += snprintf(crm_mail_body + len, BODY_MAX - len,
                     "\toperation status: (%d) %s\r\n", status, services_lrm_status_str(status));
     if (status == PCMK_LRM_OP_DONE) {
         len += snprintf(crm_mail_body + len, BODY_MAX - len,
                         "\tscript returned: (%d) %s\r\n", rc, services_ocf_exitcode_str(rc));
         len += snprintf(crm_mail_body + len, BODY_MAX - len,
                         "\texpected return value: (%d) %s\r\n", target_rc,
                         services_ocf_exitcode_str(target_rc));
     }
 
     auth_client_init();
     session = smtp_create_session();
     message = smtp_add_message(session);
 
     smtp_starttls_enable(session, Starttls_ENABLED);
 
     sa.sa_handler = SIG_IGN;
     sigemptyset(&sa.sa_mask);
     sa.sa_flags = 0;
     sigaction(SIGPIPE, &sa, NULL);
 
     smtp_set_server(session, crm_mail_host);
 
     authctx = auth_create_context();
     auth_set_mechanism_flags(authctx, AUTH_PLUGIN_PLAIN, 0);
 
     smtp_set_eventcb(session, event_cb, NULL);
 
     /* Now tell libESMTP it can use the SMTP AUTH extension.
      */
     if (!noauth) {
         crm_debug("Adding authentication context");
         smtp_auth_set_context(session, authctx);
     }
 
     if (crm_mail_from == NULL) {
         struct utsname us;
         char auto_from[BODY_MAX];
 
         CRM_ASSERT(uname(&us) == 0);
         snprintf(auto_from, BODY_MAX, "crm_mon@%s", us.nodename);
         smtp_set_reverse_path(message, auto_from);
 
     } else {
         /* NULL is ok */
         smtp_set_reverse_path(message, crm_mail_from);
     }
 
     smtp_set_header(message, "To", NULL /*phrase */ , NULL /*addr */ ); /* "Phrase" <addr> */
     smtp_add_recipient(message, crm_mail_to);
 
     /* Set the Subject: header and override any subject line in the message headers. */
     smtp_set_header(message, "Subject", crm_mail_subject);
     smtp_set_header_option(message, "Subject", Hdr_OVERRIDE, 1);
 
     smtp_set_message_str(message, crm_mail_body);
     smtp_set_monitorcb(session, crm_smtp_debug, &smtp_debug, 1);
 
     if (smtp_start_session(session)) {
         char buf[128];
         int rc = smtp_errno();
 
         crm_err("SMTP server problem: %s (%d)", smtp_strerror(rc, buf, sizeof buf), rc);
 
     } else {
         char buf[128];
         int rc = smtp_errno();
         const smtp_status_t *smtp_status = smtp_message_transfer_status(message);
 
         if (rc != 0) {
             crm_err("SMTP server problem: %s (%d)", smtp_strerror(rc, buf, sizeof buf), rc);
         }
         crm_info("Send status: %d %s", smtp_status->code, crm_str(smtp_status->text));
         smtp_enumerate_recipients(message, print_recipient_status, NULL);
     }
 
     smtp_destroy_session(session);
     auth_destroy_context(authctx);
     auth_client_exit();
 #endif
     return 0;
 }
 
 static void
 handle_rsc_op(xmlNode * xml, const char *node_id)
 {
     int rc = -1;
     int status = -1;
     int action = -1;
     int interval = 0;
     int target_rc = -1;
     int transition_num = -1;
     gboolean notify = TRUE;
 
     char *rsc = NULL;
     char *task = NULL;
     const char *desc = NULL;
     const char *magic = NULL;
     const char *id = NULL;
     char *update_te_uuid = NULL;
     const char *node = NULL;
 
     xmlNode *n = xml;
     xmlNode * rsc_op = xml;
 
     if(strcmp((const char*)xml->name, XML_LRM_TAG_RSC_OP) != 0) {
         xmlNode *cIter;
 
         for(cIter = xml->children; cIter; cIter = cIter->next) {
             handle_rsc_op(cIter, node_id);
         }
 
         return;
     }
 
     id = crm_element_value(rsc_op, XML_LRM_ATTR_TASK_KEY);
     if (id == NULL) {
         /* Compatibility with <= 1.1.5 */
         id = ID(rsc_op);
     }
 
     magic = crm_element_value(rsc_op, XML_ATTR_TRANSITION_MAGIC);
     if (magic == NULL) {
         /* non-change */
         return;
     }
 
     if (FALSE == decode_transition_magic(magic, &update_te_uuid, &transition_num, &action,
                                          &status, &rc, &target_rc)) {
         crm_err("Invalid event %s detected for %s", magic, id);
         return;
     }
 
     if (parse_op_key(id, &rsc, &task, &interval) == FALSE) {
         crm_err("Invalid event detected for %s", id);
         goto bail;
     }
 
     node = crm_element_value(rsc_op, XML_LRM_ATTR_TARGET);
 
     while (n != NULL && safe_str_neq(XML_CIB_TAG_STATE, TYPE(n))) {
         n = n->parent;
     }
 
     if(node == NULL && n) {
         node = crm_element_value(n, XML_ATTR_UNAME);
     }
 
     if (node == NULL && n) {
         node = ID(n);
     }
 
     if (node == NULL) {
         node = node_id;
     }
 
     if (node == NULL) {
         crm_err("No node detected for event %s (%s)", magic, id);
         goto bail;
     }
 
     /* look up where we expected it to be? */
     desc = pcmk_strerror(pcmk_ok);
     if (status == PCMK_LRM_OP_DONE && target_rc == rc) {
         crm_notice("%s of %s on %s completed: %s", task, rsc, node, desc);
         if (rc == PCMK_OCF_NOT_RUNNING) {
             notify = FALSE;
         }
 
     } else if (status == PCMK_LRM_OP_DONE) {
         desc = services_ocf_exitcode_str(rc);
         crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
 
     } else {
         desc = services_lrm_status_str(status);
         crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
     }
 
     if (notify && snmp_target) {
         send_snmp_trap(node, rsc, task, target_rc, rc, status, desc);
     }
     if (notify && crm_mail_to) {
         send_smtp_trap(node, rsc, task, target_rc, rc, status, desc);
     }
     if (notify && external_agent) {
         send_custom_trap(node, rsc, task, target_rc, rc, status, desc);
     }
   bail:
     free(update_te_uuid);
     free(rsc);
     free(task);
 }
 
 static gboolean
 mon_trigger_refresh(gpointer user_data)
 {
     mainloop_set_trigger(refresh_trigger);
     return FALSE;
 }
 
 #define NODE_PATT "/lrm[@id="
 static char *get_node_from_xpath(const char *xpath) 
 {
     char *nodeid = NULL;
     char *tmp = strstr(xpath, NODE_PATT);
 
     if(tmp) {
         tmp += strlen(NODE_PATT);
         tmp += 1;
 
         nodeid = strdup(tmp);
         tmp = strstr(nodeid, "\'");
         CRM_ASSERT(tmp);
         tmp[0] = 0;
     }
     return nodeid;
 }
 
 static void crm_diff_update_v2(const char *event, xmlNode * msg) 
 {
     xmlNode *change = NULL;
     xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
 
     for (change = __xml_first_child(diff); change != NULL; change = __xml_next(change)) {
         const char *name = NULL;
         const char *op = crm_element_value(change, XML_DIFF_OP);
         const char *xpath = crm_element_value(change, XML_DIFF_PATH);
         xmlNode *match = NULL;
         const char *node = NULL;
 
         if(op == NULL) {
             continue;
 
         } else if(strcmp(op, "create") == 0) {
             match = change->children;
 
         } else if(strcmp(op, "move") == 0) {
             continue;
 
         } else if(strcmp(op, "delete") == 0) {
             continue;
 
         } else if(strcmp(op, "modify") == 0) {
             match = first_named_child(change, XML_DIFF_RESULT);
             if(match) {
                 match = match->children;
             }
         }
 
         if(match) {
             name = (const char *)match->name;
         }
 
         crm_trace("Handling %s operation for %s %p, %s", op, xpath, match, name);
         if(xpath == NULL) {
             /* Version field, ignore */
 
         } else if(name == NULL) {
             crm_debug("No result for %s operation to %s", op, xpath);
             CRM_ASSERT(strcmp(op, "delete") == 0 || strcmp(op, "move") == 0);
 
         } else if(strcmp(name, XML_TAG_CIB) == 0) {
             xmlNode *state = NULL;
             xmlNode *status = first_named_child(match, XML_CIB_TAG_STATUS);
 
             for (state = __xml_first_child(status); state != NULL; state = __xml_next(state)) {
                 node = crm_element_value(state, XML_ATTR_UNAME);
                 if (node == NULL) {
                     node = ID(state);
                 }
                 handle_rsc_op(state, node);
             }
 
         } else if(strcmp(name, XML_CIB_TAG_STATUS) == 0) {
             xmlNode *state = NULL;
 
             for (state = __xml_first_child(match); state != NULL; state = __xml_next(state)) {
                 node = crm_element_value(state, XML_ATTR_UNAME);
                 if (node == NULL) {
                     node = ID(state);
                 }
                 handle_rsc_op(state, node);
             }
 
         } else if(strcmp(name, XML_CIB_TAG_STATE) == 0) {
             node = crm_element_value(match, XML_ATTR_UNAME);
             if (node == NULL) {
                 node = ID(match);
             }
             handle_rsc_op(match, node);
 
         } else if(strcmp(name, XML_CIB_TAG_LRM) == 0) {
             node = ID(match);
             handle_rsc_op(match, node);
 
         } else if(strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
             char *local_node = get_node_from_xpath(xpath);
 
             handle_rsc_op(match, local_node);
             free(local_node);
 
         } else if(strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
             char *local_node = get_node_from_xpath(xpath);
 
             handle_rsc_op(match, local_node);
             free(local_node);
 
         } else if(strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
             char *local_node = get_node_from_xpath(xpath);
 
             handle_rsc_op(match, local_node);
             free(local_node);
 
         } else {
             crm_trace("Ignoring %s operation for %s %p, %s", op, xpath, match, name);
         }
     }
 }
 
 static void crm_diff_update_v1(const char *event, xmlNode * msg) 
 {
     /* Process operation updates */
     xmlXPathObject *xpathObj = xpath_search(msg,
                                             "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED
                                             "//" XML_LRM_TAG_RSC_OP);
     int lpc = 0, max = numXpathResults(xpathObj);
 
     for (lpc = 0; lpc < max; lpc++) {
         xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
 
         handle_rsc_op(rsc_op, NULL);
     }
     freeXpathObject(xpathObj);
 }
 
 void
 crm_diff_update(const char *event, xmlNode * msg)
 {
     int rc = -1;
     long now = time(NULL);
     static bool stale = FALSE;
     static int updates = 0;
     static mainloop_timer_t *refresh_timer = NULL;
     xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
 
     print_dot();
 
     if(refresh_timer == NULL) {
         refresh_timer = mainloop_timer_add("refresh", 2000, FALSE, mon_trigger_refresh, NULL);
     }
 
     if (current_cib != NULL) {
         rc = xml_apply_patchset(current_cib, diff, TRUE);
 
         switch (rc) {
             case -pcmk_err_diff_resync:
             case -pcmk_err_diff_failed:
                 crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
                 free_xml(current_cib); current_cib = NULL;
                 break;
             case pcmk_ok:
                 updates++;
                 break;
             default:
                 crm_notice("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
                 free_xml(current_cib); current_cib = NULL;
         }
     }
 
     if (current_cib == NULL) {
         crm_trace("Re-requesting the full cib");
         cib->cmds->query(cib, NULL, &current_cib, cib_scope_local | cib_sync_call);
     }
 
     if (crm_mail_to || snmp_target || external_agent) {
         int format = 0;
         crm_element_value_int(diff, "format", &format);
         switch(format) {
             case 1:
                 crm_diff_update_v1(event, msg);
                 break;
             case 2:
                 crm_diff_update_v2(event, msg);
                 break;
             default:
                 crm_err("Unknown patch format: %d", format);
         }
     }
 
     if (current_cib == NULL) {
         if(!stale) {
             print_as("--- Stale data ---");
         }
         stale = TRUE;
         return;
     }
 
     stale = FALSE;
     /* Refresh
      * - immediately if the last update was more than 5s ago
      * - every 10 updates
      * - at most 2s after the last update
      */
     if ((now - last_refresh) > (reconnect_msec / 1000)) {
         mainloop_set_trigger(refresh_trigger);
         mainloop_timer_stop(refresh_timer);
         updates = 0;
 
     } else if(updates > 10) {
         mainloop_set_trigger(refresh_trigger);
         mainloop_timer_stop(refresh_timer);
         updates = 0;
 
     } else {
         mainloop_timer_start(refresh_timer);
     }
 }
 
 gboolean
 mon_refresh_display(gpointer user_data)
 {
     xmlNode *cib_copy = copy_xml(current_cib);
     pe_working_set_t data_set;
 
     last_refresh = time(NULL);
 
     if (cli_config_update(&cib_copy, NULL, FALSE) == FALSE) {
         if (cib) {
             cib->cmds->signoff(cib);
         }
         print_as("Upgrade failed: %s", pcmk_strerror(-pcmk_err_schema_validation));
         if (output_format == mon_output_console) {
             sleep(2);
         }
         clean_up(EX_USAGE);
         return FALSE;
     }
 
     set_working_set_defaults(&data_set);
     data_set.input = cib_copy;
     cluster_status(&data_set);
 
     /* Unpack constraints if any section will need them
      * (tickets may be referenced in constraints but not granted yet,
      * and bans need negative location constraints) */
     if (show & (mon_show_bans | mon_show_tickets)) {
         xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set.input);
         unpack_constraints(cib_constraints, &data_set);
     }
 
     switch (output_format) {
         case mon_output_html:
         case mon_output_cgi:
             if (print_html_status(&data_set, output_filename) != 0) {
                 fprintf(stderr, "Critical: Unable to output html file\n");
                 clean_up(EX_USAGE);
             }
             break;
 
         case mon_output_xml:
             print_xml_status(&data_set);
             break;
 
         case mon_output_monitor:
             print_simple_status(&data_set);
             if (has_warnings) {
                 clean_up(MON_STATUS_WARN);
             }
             break;
 
         case mon_output_plain:
         case mon_output_console:
             print_status(&data_set);
             break;
 
         case mon_output_none:
             break;
     }
 
     cleanup_alloc_calculations(&data_set);
     return TRUE;
 }
 
 void
 mon_st_callback(stonith_t * st, stonith_event_t * e)
 {
     char *desc = crm_strdup_printf("Operation %s requested by %s for peer %s: %s (ref=%s)",
                                  e->operation, e->origin, e->target, pcmk_strerror(e->result),
                                  e->id);
 
     if (snmp_target) {
         send_snmp_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
     }
     if (crm_mail_to) {
         send_smtp_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
     }
     if (external_agent) {
         send_custom_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
     }
     free(desc);
 }
 
 /*
  * De-init ncurses, signoff from the CIB and deallocate memory.
  */
 void
 clean_up(int rc)
 {
 #if ENABLE_SNMP
     netsnmp_session *session = crm_snmp_init(NULL, NULL);
 
     if (session) {
         snmp_close(session);
         snmp_shutdown("snmpapp");
     }
 #endif
 
 #if CURSES_ENABLED
     if (output_format == mon_output_console) {
         output_format = mon_output_plain;
         echo();
         nocbreak();
         endwin();
     }
 #endif
 
     if (cib != NULL) {
         cib->cmds->signoff(cib);
         cib_delete(cib);
         cib = NULL;
     }
 
     free(output_filename);
     free(xml_file);
     free(pid_file);
 
     if (rc >= 0) {
         crm_exit(rc);
     }
     return;
 }
diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index 4fde33ece7..c4ee78e621 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1,1064 +1,1065 @@
 
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_resource.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <libgen.h>
 #include <time.h>
 
 bool BE_QUIET = FALSE;
 bool scope_master = FALSE;
 int cib_options = cib_sync_call;
 
 GMainLoop *mainloop = NULL;
 
 #define message_timeout_ms 60*1000
 
 static gboolean
 resource_ipc_timeout(gpointer data)
 {
     fprintf(stderr, "No messages received in %d seconds.. aborting\n",
             (int)message_timeout_ms / 1000);
     crm_err("No messages received in %d seconds", (int)message_timeout_ms / 1000);
     return crm_exit(-1);
 }
 
 static void
 resource_ipc_connection_destroy(gpointer user_data)
 {
     crm_info("Connection to CRMd was terminated");
     crm_exit(1);
 }
 
 static void
 start_mainloop(void)
 {
     if (crmd_replies_needed == 0) {
         return;
     }
 
     mainloop = g_main_new(FALSE);
     fprintf(stderr, "Waiting for %d replies from the CRMd", crmd_replies_needed);
     crm_debug("Waiting for %d replies from the CRMd", crmd_replies_needed);
 
     g_timeout_add(message_timeout_ms, resource_ipc_timeout, NULL);
     g_main_run(mainloop);
 }
 
 static int
 resource_ipc_callback(const char *buffer, ssize_t length, gpointer userdata)
 {
     xmlNode *msg = string2xml(buffer);
 
     fprintf(stderr, ".");
     crm_log_xml_trace(msg, "[inbound]");
 
     crmd_replies_needed--;
     if (crmd_replies_needed == 0) {
         fprintf(stderr, " OK\n");
         crm_debug("Got all the replies we expected");
         return crm_exit(pcmk_ok);
     }
 
     free_xml(msg);
     return 0;
 }
 
 struct ipc_client_callbacks crm_callbacks = {
     .dispatch = resource_ipc_callback,
     .destroy = resource_ipc_connection_destroy,
 };
 
 
 /* *INDENT-OFF* */
 static struct crm_option long_options[] = {
     /* Top-level Options */
     {"help",    0, 0, '?', "\t\tThis text"},
     {"version", 0, 0, '$', "\t\tVersion information"  },
     {"verbose", 0, 0, 'V', "\t\tIncrease debug output"},
     {"quiet",   0, 0, 'Q', "\t\tPrint only the value on stdout\n"},
 
     {"resource",   1, 0, 'r', "\tResource ID" },
 
     {"-spacer-",1, 0, '-', "\nQueries:"},
     {"list",       0, 0, 'L', "\t\tList all cluster resources"},
     {"list-raw",   0, 0, 'l', "\tList the IDs of all instantiated resources (no groups/clones/...)"},
     {"list-cts",   0, 0, 'c', NULL, pcmk_option_hidden},
     {"list-operations", 0, 0, 'O', "\tList active resource operations.  Optionally filtered by resource (-r) and/or node (-N)"},
     {"list-all-operations", 0, 0, 'o', "List all resource operations.  Optionally filtered by resource (-r) and/or node (-N)\n"},
     {"pending",    0, 0, 'j', "\t\tDisplay pending state if 'record-pending' is enabled\n", pcmk_option_hidden},
 
     {"list-standards",        0, 0, 0, "\tList supported standards"},
     {"list-ocf-providers",    0, 0, 0, "List all available OCF providers"},
     {"list-agents",           1, 0, 0, "List all agents available for the named standard and/or provider."},
     {"list-ocf-alternatives", 1, 0, 0, "List all available providers for the named OCF agent\n"},
     {"show-metadata",         1, 0, 0, "Show the metadata for the named class:provider:agent"},
 
     {"query-xml",  0, 0, 'q', "\tQuery the definition of a resource (template expanded)"},
     {"query-xml-raw",  0, 0, 'w', "\tQuery the definition of a resource (raw xml)"},
     {"locate",     0, 0, 'W', "\t\tDisplay the current location(s) of a resource"},
     {"stack",      0, 0, 'A', "\t\tDisplay the prerequisites and dependents of a resource"},
     {"constraints",0, 0, 'a', "\tDisplay the (co)location constraints that apply to a resource"},
 
     {"-spacer-",	1, 0, '-', "\nCommands:"},
     {"cleanup",         0, 0, 'C', "\t\tDelete the resource history and re-check the current state. Optional: --resource"},
     {"set-parameter",   1, 0, 'p', "Set the named parameter for a resource. See also -m, --meta"},
     {"get-parameter",   1, 0, 'g', "Display the named parameter for a resource. See also -m, --meta"},
     {"delete-parameter",1, 0, 'd', "Delete the named parameter for a resource. See also -m, --meta"},
     {"get-property",    1, 0, 'G', "Display the 'class', 'type' or 'provider' of a resource", pcmk_option_hidden},
     {"set-property",    1, 0, 'S',
         "(Advanced) Set resource property (e.g. 'class', 'type', or 'provider'). Required: -r, -t, -v",
         pcmk_option_hidden},
 
     {"-spacer-",	1, 0, '-', "\nResource location:"},
     {
         "move",    0, 0, 'M',
         "\t\tMove a resource from its current location to the named destination.\n  "
         "\t\t\t\tRequires: --host. Optional: --lifetime, --master\n\n"
         "\t\t\t\tNOTE: This may prevent the resource from running on the previous location node until the implicit constraints expire or are removed with --unban\n"
     },
     {
         "ban",    0, 0, 'B',
         "\t\tPrevent the named resource from running on the named --host.  \n"
         "\t\t\t\tRequires: --resource. Optional: --host, --lifetime, --master\n\n"
         "\t\t\t\tIf --host is not specified, it defaults to:\n"
         "\t\t\t\t * the current location for primitives and groups, or\n\n"
         "\t\t\t\t * the current location of the master for m/s resources with master-max=1\n\n"
         "\t\t\t\tAll other situations result in an error as there is no sane default.\n\n"
         "\t\t\t\tNOTE: This will prevent the resource from running on this node until the constraint expires or is removed with --clear\n"
     },
     {
         "clear", 0, 0, 'U', "\t\tRemove all constraints created by the --ban and/or --move commands.  \n"
         "\t\t\t\tRequires: --resource. Optional: --host, --master\n\n"
         "\t\t\t\tIf --host is not specified, all constraints created by --ban and --move will be removed for the named resource.\n"
     },
     {"lifetime",   1, 0, 'u', "\tLifespan of constraints created by the --ban and --move commands"},
     {
         "master",  0, 0,  0,
         "\t\tLimit the scope of the --ban, --move and --clear  commands to the Master role.\n"
         "\t\t\t\tFor --ban and --move, the previous master can still remain active in the Slave role."
     },
 
     {"-spacer-",   1, 0, '-', "\nAdvanced Commands:"},
     {"delete",     0, 0, 'D', "\t\t(Advanced) Delete a resource from the CIB"},
     {"fail",       0, 0, 'F', "\t\t(Advanced) Tell the cluster this resource has failed"},
     {"restart",    0, 0,  0,  "\t\t(Advanced) Tell the cluster to restart this resource and anything that depends on it"},
     {"wait",       0, 0,  0,  "\t\t(Advanced) Wait until the cluster settles into a stable state"},
     {"force-demote",0,0,  0,  "\t(Advanced) Bypass the cluster and demote a resource on the local node. Additional detail with -V"},
     {"force-stop", 0, 0,  0,  "\t(Advanced) Bypass the cluster and stop a resource on the local node. Additional detail with -V"},
     {"force-start",0, 0,  0,  "\t(Advanced) Bypass the cluster and start a resource on the local node. Additional detail with -V"},
     {"force-promote",0,0, 0,  "\t(Advanced) Bypass the cluster and promote a resource on the local node. Additional detail with -V"},
     {"force-check",0, 0,  0,  "\t(Advanced) Bypass the cluster and check the state of a resource on the local node. Additional detail with -V\n"},
 
     {"-spacer-",	1, 0, '-', "\nAdditional Options:"},
     {"node",		1, 0, 'N', "\tHost uname"},
     {"recursive",       0, 0,  0,  "\tFollow colocation chains when using --set-parameter"},
     {"resource-type",	1, 0, 't', "Resource type (primitive, clone, group, ...)"},
     {"parameter-value", 1, 0, 'v', "Value to use with -p"},
     {"meta",		0, 0, 'm', "\t\tModify a resource's configuration option rather than one which is passed to the resource agent script. For use with -p, -g, -d"},
     {"utilization",	0, 0, 'z', "\tModify a resource's utilization attribute. For use with -p, -g, -d"},
     {"set-name",        1, 0, 's', "\t(Advanced) ID of the instance_attributes object to change"},
     {"nvpair",          1, 0, 'i', "\t(Advanced) ID of the nvpair object to change/delete"},
     {"timeout",         1, 0, 'T',  "\t(Advanced) Abort if command does not finish in this time (with --restart or --wait)"},
     {"force",		0, 0, 'f', "\n" /* Is this actually true anymore?
 					   "\t\tForce the resource to move by creating a rule for the current location and a score of -INFINITY"
 					   "\n\t\tThis should be used if the resource's stickiness and constraint scores total more than INFINITY (Currently 100,000)"
 					   "\n\t\tNOTE: This will prevent the resource from running on this node until the constraint is removed with -U or the --lifetime duration expires\n"*/ },
 
     {"xml-file", 1, 0, 'x', NULL, pcmk_option_hidden},\
 
     /* legacy options */
     {"host-uname", 1, 0, 'H', NULL, pcmk_option_hidden},
     {"migrate",    0, 0, 'M', NULL, pcmk_option_hidden},
     {"un-migrate", 0, 0, 'U', NULL, pcmk_option_hidden},
     {"un-move",    0, 0, 'U', NULL, pcmk_option_hidden},
 
     {"refresh",    0, 0, 'R', NULL, pcmk_option_hidden},
     {"reprobe",    0, 0, 'P', NULL, pcmk_option_hidden},
 
     {"-spacer-",	1, 0, '-', "\nExamples:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', "List the configured resources:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --list", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "List the available OCF agents:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --list-agents ocf", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "List the available OCF agents from the linux-ha project:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --list-agents ocf:heartbeat", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Display the current location of 'myResource':", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --locate", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Move 'myResource' to another machine:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --move", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Move 'myResource' to a specific machine:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --move --node altNode", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Allow (but not force) 'myResource' to move back to its original location:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --un-move", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Tell the cluster that 'myResource' failed:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --fail", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Stop 'myResource' (and anything that depends on it):", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --set-parameter target-role --meta --parameter-value Stopped", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Tell the cluster not to manage 'myResource':", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', "The cluster will not attempt to start or stop the resource under any circumstances."},
     {"-spacer-",	1, 0, '-', "Useful when performing maintenance tasks on a resource.", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --set-parameter is-managed --meta --parameter-value false", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Erase the operation history of 'myResource' on 'aNode':", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', "The cluster will 'forget' the existing resource state (including any errors) and attempt to recover the resource."},
     {"-spacer-",	1, 0, '-', "Useful when a resource had failed permanently and has been repaired by an administrator.", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --cleanup --node aNode", pcmk_option_example},
 
     {0, 0, 0, 0}
 };
 /* *INDENT-ON* */
 
 int
 main(int argc, char **argv)
 {
     char rsc_cmd = 'L';
 
     const char *rsc_id = NULL;
     const char *host_uname = NULL;
     const char *prop_name = NULL;
     const char *prop_value = NULL;
     const char *rsc_type = NULL;
     const char *prop_id = NULL;
     const char *prop_set = NULL;
     const char *rsc_long_cmd = NULL;
     const char *longname = NULL;
     GHashTable *override_params = NULL;
 
     char *xml_file = NULL;
     crm_ipc_t *crmd_channel = NULL;
     pe_working_set_t data_set;
     cib_t *cib_conn = NULL;
     bool recursive = FALSE;
     char *our_pid = NULL;
 
     /* Not all commands set these appropriately, but the defaults here are
      * sufficient to get the logic right. */
     bool require_resource = TRUE; /* whether command requires that resource be specified */
     bool require_dataset = TRUE;  /* whether command requires populated dataset instance */
     bool require_crmd = FALSE;    /* whether command requires connection to CRMd */
 
     int rc = pcmk_ok;
     int option_index = 0;
     int timeout_ms = 0;
     int argerr = 0;
     int flag;
 
     crm_log_cli_init("crm_resource");
     crm_set_options(NULL, "(query|command) [options]", long_options,
                     "Perform tasks related to cluster resources.\nAllows resources to be queried (definition and location), modified, and moved around the cluster.\n");
 
     if (argc < 2) {
         crm_help('?', EX_USAGE);
     }
 
     while (1) {
         flag = crm_get_option_long(argc, argv, &option_index, &longname);
         if (flag == -1)
             break;
 
         switch (flag) {
             case 0: /* long options with no short equivalent */
                 if (safe_str_eq("master", longname)) {
                     scope_master = TRUE;
 
                 } else if(safe_str_eq(longname, "recursive")) {
                     recursive = TRUE;
 
                 } else if (safe_str_eq("wait", longname)) {
                     rsc_cmd = flag;
                     rsc_long_cmd = longname;
                     require_resource = FALSE;
                     require_dataset = FALSE;
 
                 } else if (
                     safe_str_eq("restart", longname)
                     || safe_str_eq("force-demote",  longname)
                     || safe_str_eq("force-stop",    longname)
                     || safe_str_eq("force-start",   longname)
                     || safe_str_eq("force-promote", longname)
                     || safe_str_eq("force-check",   longname)) {
                     rsc_cmd = flag;
                     rsc_long_cmd = longname;
                     crm_log_args(argc, argv);
 
                 } else if (safe_str_eq("list-ocf-providers", longname)
                            || safe_str_eq("list-ocf-alternatives", longname)
                            || safe_str_eq("list-standards", longname)) {
                     const char *text = NULL;
                     lrmd_list_t *list = NULL;
                     lrmd_list_t *iter = NULL;
                     lrmd_t *lrmd_conn = lrmd_api_new();
 
                     if (safe_str_eq("list-ocf-providers", longname)
                         || safe_str_eq("list-ocf-alternatives", longname)) {
                         rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, optarg, &list);
                         text = "OCF providers";
 
                     } else if (safe_str_eq("list-standards", longname)) {
                         rc = lrmd_conn->cmds->list_standards(lrmd_conn, &list);
                         text = "standards";
                     }
 
                     if (rc > 0) {
                         rc = 0;
                         for (iter = list; iter != NULL; iter = iter->next) {
                             rc++;
                             printf("%s\n", iter->val);
                         }
                         lrmd_list_freeall(list);
 
                     } else if (optarg) {
                         fprintf(stderr, "No %s found for %s\n", text, optarg);
                     } else {
                         fprintf(stderr, "No %s found\n", text);
                     }
 
                     lrmd_api_delete(lrmd_conn);
                     return crm_exit(rc);
 
                 } else if (safe_str_eq("show-metadata", longname)) {
                     char standard[512];
                     char provider[512];
                     char type[512];
                     char *metadata = NULL;
                     lrmd_t *lrmd_conn = lrmd_api_new();
 
                     rc = sscanf(optarg, "%[^:]:%[^:]:%s", standard, provider, type);
                     if (rc == 3) {
                         rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard, provider, type,
                                                            &metadata, 0);
 
                     } else if (rc == 2) {
                         rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard, NULL, provider,
                                                            &metadata, 0);
 
                     } else if (rc < 2) {
                         fprintf(stderr,
                                 "Please specify standard:type or standard:provider:type, not %s\n",
                                 optarg);
                         rc = -EINVAL;
                     }
 
                     if (metadata) {
                         printf("%s\n", metadata);
                     } else {
                         fprintf(stderr, "Metadata query for %s failed: %d\n", optarg, rc);
                     }
                     lrmd_api_delete(lrmd_conn);
                     return crm_exit(rc);
 
                 } else if (safe_str_eq("list-agents", longname)) {
                     lrmd_list_t *list = NULL;
                     lrmd_list_t *iter = NULL;
                     char *provider = strchr (optarg, ':');
                     lrmd_t *lrmd_conn = lrmd_api_new();
 
                     if (provider) {
                         *provider++ = 0;
                     }
                     rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, optarg, provider);
 
                     if (rc > 0) {
                         rc = 0;
                         for (iter = list; iter != NULL; iter = iter->next) {
                             printf("%s\n", iter->val);
                             rc++;
                         }
                         lrmd_list_freeall(list);
                         rc = 0;
                     } else {
                         fprintf(stderr, "No agents found for standard=%s, provider=%s\n",
                                 optarg, (provider? provider : "*"));
                         rc = -1;
                     }
                     lrmd_api_delete(lrmd_conn);
                     return crm_exit(rc);
 
                 } else {
                     crm_err("Unhandled long option: %s", longname);
                 }
                 break;
             case 'V':
                 do_trace = TRUE;
                 crm_bump_log_level(argc, argv);
                 break;
             case '$':
             case '?':
                 crm_help(flag, EX_OK);
                 break;
             case 'x':
                 xml_file = strdup(optarg);
                 break;
             case 'Q':
                 BE_QUIET = TRUE;
                 break;
             case 'm':
                 attr_set_type = XML_TAG_META_SETS;
                 break;
             case 'z':
                 attr_set_type = XML_TAG_UTILIZATION;
                 break;
             case 'u':
                 move_lifetime = strdup(optarg);
                 break;
             case 'f':
                 do_force = TRUE;
                 crm_log_args(argc, argv);
                 break;
             case 'i':
                 prop_id = optarg;
                 break;
             case 's':
                 prop_set = optarg;
                 break;
             case 'r':
                 rsc_id = optarg;
                 break;
             case 'v':
                 prop_value = optarg;
                 break;
             case 't':
                 rsc_type = optarg;
                 break;
             case 'T':
                 timeout_ms = crm_get_msec(optarg);
                 break;
 
             case 'C':
             case 'R':
             case 'P':
                 crm_log_args(argc, argv);
                 require_resource = FALSE;
                 require_crmd = TRUE;
                 rsc_cmd = 'C';
                 break;
 
             case 'F':
                 crm_log_args(argc, argv);
                 require_crmd = TRUE;
                 rsc_cmd = flag;
                 break;
 
             case 'U':
             case 'B':
             case 'M':
             case 'D':
                 crm_log_args(argc, argv);
                 rsc_cmd = flag;
                 break;
 
             case 'L':
             case 'c':
             case 'l':
             case 'q':
             case 'w':
             case 'W':
             case 'O':
             case 'o':
             case 'A':
             case 'a':
                 rsc_cmd = flag;
                 break;
 
             case 'j':
                 print_pending = TRUE;
                 break;
             case 'p':
             case 'd':
             case 'S':
                 crm_log_args(argc, argv);
                 prop_name = optarg;
                 rsc_cmd = flag;
                 break;
             case 'G':
             case 'g':
                 prop_name = optarg;
                 rsc_cmd = flag;
                 break;
             case 'h':
             case 'H':
             case 'N':
                 crm_trace("Option %c => %s", flag, optarg);
                 host_uname = optarg;
                 break;
 
             default:
                 CMD_ERR("Argument code 0%o (%c) is not (?yet?) supported", flag, flag);
                 ++argerr;
                 break;
         }
     }
 
     if (optind < argc
         && argv[optind] != NULL
         && rsc_cmd == 0
         && rsc_long_cmd) {
 
         override_params = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
         while (optind < argc && argv[optind] != NULL) {
             char *name = calloc(1, strlen(argv[optind]));
             char *value = calloc(1, strlen(argv[optind]));
             int rc = sscanf(argv[optind], "%[^=]=%s", name, value);
 
             if(rc == 2) {
                 g_hash_table_replace(override_params, name, value);
 
             } else {
                 CMD_ERR("Error parsing '%s' as a name=value pair for --%s", argv[optind], rsc_long_cmd);
                 free(value);
                 free(name);
                 argerr++;
             }
             optind++;
         }
 
     } else if (optind < argc && argv[optind] != NULL && rsc_cmd == 0) {
         CMD_ERR("non-option ARGV-elements: ");
         while (optind < argc && argv[optind] != NULL) {
             CMD_ERR("[%d of %d] %s ", optind, argc, argv[optind]);
             optind++;
             argerr++;
         }
     }
 
     if (optind > argc) {
         ++argerr;
     }
 
     if (argerr) {
         CMD_ERR("Invalid option(s) supplied, use --help for valid usage");
         return crm_exit(EX_USAGE);
     }
 
     our_pid = calloc(1, 11);
     if (our_pid != NULL) {
         snprintf(our_pid, 10, "%d", getpid());
         our_pid[10] = '\0';
     }
 
     if (do_force) {
         crm_debug("Forcing...");
         cib_options |= cib_quorum_override;
     }
 
     data_set.input = NULL; /* make clean-up easier */
 
     /* If user specified resource, look for it, even if it's optional for command */
     if (rsc_id) {
         require_resource = TRUE;
     }
 
     /* We need a dataset to find a resource, even if command doesn't need it */
     if (require_resource) {
         require_dataset = TRUE;
     }
 
     /* Establish a connection to the CIB */
     cib_conn = cib_new();
     rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
     if (rc != pcmk_ok) {
         CMD_ERR("Error signing on to the CIB service: %s", pcmk_strerror(rc));
         return crm_exit(rc);
     }
 
     /* Populate working set from XML file if specified or CIB query otherwise */
     if (require_dataset) {
         xmlNode *cib_xml_copy = NULL;
 
         if (xml_file != NULL) {
             cib_xml_copy = filename2xml(xml_file);
 
         } else {
             rc = cib_conn->cmds->query(cib_conn, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
         }
 
         if(rc != pcmk_ok) {
             goto bail;
         }
 
         /* Populate the working set instance */
         set_working_set_defaults(&data_set);
         rc = update_working_set_xml(&data_set, &cib_xml_copy);
         if (rc != pcmk_ok) {
             goto bail;
         }
         cluster_status(&data_set);
 
         /* Set rc to -ENXIO if no resource matching rsc_id is found.
          * This does not bail, but is handled later for certain commands.
          * That handling could be done here instead if all flags above set
          * require_resource appropriately. */
         if (require_resource && rsc_id && (find_rsc_or_clone(rsc_id, &data_set) == NULL)) {
             rc = -ENXIO;
         }
     }
 
     /* Establish a connection to the CRMd if needed */
     if (require_crmd) {
         xmlNode *xml = NULL;
         mainloop_io_t *source =
             mainloop_add_ipc_client(CRM_SYSTEM_CRMD, G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks);
         crmd_channel = mainloop_get_ipc_client(source);
 
         if (crmd_channel == NULL) {
             CMD_ERR("Error signing on to the CRMd service");
             rc = -ENOTCONN;
             goto bail;
         }
 
         xml = create_hello_message(our_pid, crm_system_name, "0", "1");
         crm_ipc_send(crmd_channel, xml, 0, 0, NULL);
         free_xml(xml);
     }
 
     /* Handle rsc_cmd appropriately */
     if (rsc_cmd == 'L') {
         rc = pcmk_ok;
         cli_resource_print_list(&data_set, FALSE);
 
     } else if (rsc_cmd == 'l') {
         int found = 0;
         GListPtr lpc = NULL;
 
         rc = pcmk_ok;
         for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
             resource_t *rsc = (resource_t *) lpc->data;
 
             found++;
             cli_resource_print_raw(rsc);
         }
 
         if (found == 0) {
             printf("NO resources configured\n");
             rc = -ENXIO;
         }
 
     } else if (rsc_cmd == 0 && rsc_long_cmd && safe_str_eq(rsc_long_cmd, "restart")) {
         resource_t *rsc = NULL;
 
         rc = -ENXIO;
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r");
             goto bail;
         }
 
         rsc = pe_find_resource(data_set.resources, rsc_id);
 
         rc = -EINVAL;
         if (rsc == NULL) {
             CMD_ERR("Resource '%s' not restarted: unknown", rsc_id);
             goto bail;
         }
 
         rc = cli_resource_restart(rsc, host_uname, timeout_ms, cib_conn);
 
     } else if (rsc_cmd == 0 && rsc_long_cmd && safe_str_eq(rsc_long_cmd, "wait")) {
         rc = wait_till_stable(timeout_ms, cib_conn);
 
     } else if (rsc_cmd == 0 && rsc_long_cmd) { /* force-(stop|start|check) */
         rc = cli_resource_execute(rsc_id, rsc_long_cmd, override_params, cib_conn, &data_set);
 
     } else if (rsc_cmd == 'A' || rsc_cmd == 'a') {
         GListPtr lpc = NULL;
         resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
         xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set.input);
 
         if (rsc == NULL) {
             CMD_ERR("Must supply a resource id with -r");
             rc = -ENXIO;
             goto bail;
         }
 
         unpack_constraints(cib_constraints, &data_set);
 
         for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
             resource_t *r = (resource_t *) lpc->data;
 
             clear_bit(r->flags, pe_rsc_allocating);
         }
 
         cli_resource_print_colocation(rsc, TRUE, rsc_cmd == 'A', 1);
 
         fprintf(stdout, "* %s\n", rsc->id);
         cli_resource_print_location(rsc, NULL);
 
         for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
             resource_t *r = (resource_t *) lpc->data;
 
             clear_bit(r->flags, pe_rsc_allocating);
         }
 
         cli_resource_print_colocation(rsc, FALSE, rsc_cmd == 'A', 1);
 
     } else if (rsc_cmd == 'c') {
         int found = 0;
         GListPtr lpc = NULL;
 
         rc = pcmk_ok;
         for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
             resource_t *rsc = (resource_t *) lpc->data;
 
             cli_resource_print_cts(rsc);
             found++;
         }
         cli_resource_print_cts_constraints(&data_set);
 
     } else if (rsc_cmd == 'F') {
         rc = cli_resource_fail(crmd_channel, host_uname, rsc_id, &data_set);
         if (rc == pcmk_ok) {
             start_mainloop();
         }
 
     } else if (rsc_cmd == 'O') {
         rc = cli_resource_print_operations(rsc_id, host_uname, TRUE, &data_set);
 
     } else if (rsc_cmd == 'o') {
         rc = cli_resource_print_operations(rsc_id, host_uname, FALSE, &data_set);
 
     /* All remaining commands require that resource exist */
     } else if (rc == -ENXIO) {
         CMD_ERR("Resource '%s' not found: %s", crm_str(rsc_id), pcmk_strerror(rc));
 
     } else if (rsc_cmd == 'W') {
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r");
             rc = -ENXIO;
             goto bail;
         }
         rc = cli_resource_search(rsc_id, &data_set);
         if (rc >= 0) {
             rc = pcmk_ok;
         }
 
     } else if (rsc_cmd == 'q') {
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r");
             rc = -ENXIO;
             goto bail;
         }
         rc = cli_resource_print(rsc_id, &data_set, TRUE);
 
     } else if (rsc_cmd == 'w') {
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r");
             rc = -ENXIO;
             goto bail;
         }
         rc = cli_resource_print(rsc_id, &data_set, FALSE);
 
     } else if (rsc_cmd == 'U') {
         node_t *dest = NULL;
 
         if (rsc_id == NULL) {
             CMD_ERR("No value specified for --resource");
             rc = -ENXIO;
             goto bail;
         }
 
         if (host_uname) {
             dest = pe_find_node(data_set.nodes, host_uname);
             if (dest == NULL) {
                 CMD_ERR("Unknown node: %s", host_uname);
                 rc = -ENXIO;
                 goto bail;
             }
             rc = cli_resource_clear(rsc_id, dest->details->uname, NULL, cib_conn);
 
         } else {
             rc = cli_resource_clear(rsc_id, NULL, data_set.nodes, cib_conn);
         }
 
     } else if (rsc_cmd == 'M' && host_uname) {
         rc = cli_resource_move(rsc_id, host_uname, cib_conn, &data_set);
 
     } else if (rsc_cmd == 'B' && host_uname) {
         resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
         node_t *dest = pe_find_node(data_set.nodes, host_uname);
 
         rc = -ENXIO;
         if (rsc_id == NULL) {
             CMD_ERR("No value specified for --resource");
             goto bail;
         } else if(rsc == NULL) {
             CMD_ERR("Resource '%s' not moved: unknown", rsc_id);
 
         } else if (dest == NULL) {
             CMD_ERR("Error performing operation: node '%s' is unknown", host_uname);
             goto bail;
         }
         rc = cli_resource_ban(rsc_id, dest->details->uname, NULL, cib_conn);
 
     } else if (rsc_cmd == 'B' || rsc_cmd == 'M') {
         resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
 
         rc = -ENXIO;
         if (rsc_id == NULL) {
             CMD_ERR("No value specified for --resource");
             goto bail;
         }
 
         rc = -EINVAL;
         if(rsc == NULL) {
             CMD_ERR("Resource '%s' not moved: unknown", rsc_id);
 
         } else if(g_list_length(rsc->running_on) == 1) {
             node_t *current = rsc->running_on->data;
             rc = cli_resource_ban(rsc_id, current->details->uname, NULL, cib_conn);
 
         } else if(rsc->variant == pe_master) {
             int count = 0;
             GListPtr iter = NULL;
             node_t *current = NULL;
 
             for(iter = rsc->children; iter; iter = iter->next) {
                 resource_t *child = (resource_t *)iter->data;
                 enum rsc_role_e child_role = child->fns->state(child, TRUE);
 
                 if(child_role == RSC_ROLE_MASTER) {
                     count++;
                     current = child->running_on->data;
                 }
             }
 
             if(count == 1 && current) {
                 rc = cli_resource_ban(rsc_id, current->details->uname, NULL, cib_conn);
 
             } else {
                 CMD_ERR("Resource '%s' not moved: active in %d locations (promoted in %d).", rsc_id, g_list_length(rsc->running_on), count);
                 CMD_ERR("You can prevent '%s' from running on a specific location with: --ban --host <name>", rsc_id);
                 CMD_ERR("You can prevent '%s' from being promoted at a specific location with:"
                         " --ban --master --host <name>", rsc_id);
             }
 
         } else {
             CMD_ERR("Resource '%s' not moved: active in %d locations.", rsc_id, g_list_length(rsc->running_on));
             CMD_ERR("You can prevent '%s' from running on a specific location with: --ban --host <name>", rsc_id);
         }
 
     } else if (rsc_cmd == 'G') {
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r");
             rc = -ENXIO;
             goto bail;
         }
         rc = cli_resource_print_property(rsc_id, prop_name, &data_set);
 
     } else if (rsc_cmd == 'S') {
         xmlNode *msg_data = NULL;
 
         if ((rsc_id == NULL) || !strlen(rsc_id)) {
             CMD_ERR("Must specify -r with resource id");
             rc = -ENXIO;
             goto bail;
 
         } else if ((rsc_type == NULL) || !strlen(rsc_type)) {
             CMD_ERR("Must specify -t with resource type");
             rc = -ENXIO;
             goto bail;
 
         } else if ((prop_value == NULL) || !strlen(prop_value)) {
             CMD_ERR("Must supply -v with new value");
             rc = -EINVAL;
             goto bail;
 
         } else if (cib_conn == NULL) {
             rc = -ENOTCONN;
             goto bail;
         }
 
         CRM_LOG_ASSERT(prop_name != NULL);
 
         msg_data = create_xml_node(NULL, rsc_type);
         crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);
         crm_xml_add(msg_data, prop_name, prop_value);
 
         rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
         free_xml(msg_data);
 
     } else if (rsc_cmd == 'g') {
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r");
             rc = -ENXIO;
             goto bail;
         }
 
         rc = cli_resource_print_attribute(rsc_id, prop_name, &data_set);
 
     } else if (rsc_cmd == 'p') {
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r");
             rc = -ENXIO;
             goto bail;
         }
         if (prop_value == NULL || strlen(prop_value) == 0) {
             CMD_ERR("You need to supply a value with the -v option");
             rc = -EINVAL;
             goto bail;
         }
 
         /* coverity[var_deref_model] False positive */
         rc = cli_resource_update_attribute(rsc_id, prop_set, prop_id, prop_name,
                                prop_value, recursive, cib_conn, &data_set);
 
     } else if (rsc_cmd == 'd') {
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r");
             rc = -ENXIO;
             goto bail;
         }
         /* coverity[var_deref_model] False positive */
         rc = cli_resource_delete_attribute(rsc_id, prop_set, prop_id, prop_name, cib_conn, &data_set);
 
     } else if (rsc_cmd == 'C' && rsc_id) {
         resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
 
         if(do_force == FALSE) {
             rsc = uber_parent(rsc);
         }
 
         crm_debug("Re-checking the state of %s for %s on %s", rsc->id, rsc_id, host_uname);
         if(rsc) {
             crmd_replies_needed = 0;
             rc = cli_resource_delete(cib_conn, crmd_channel, host_uname, rsc, &data_set);
         } else {
             rc = -ENODEV;
         }
 
         if(rc == pcmk_ok && BE_QUIET == FALSE) {
             /* Now check XML_RSC_ATTR_TARGET_ROLE and XML_RSC_ATTR_MANAGED */
             cli_resource_check(cib_conn, rsc);
         }
 
         if (rc == pcmk_ok) {
             start_mainloop();
         }
 
     } else if (rsc_cmd == 'C') {
 #if HAVE_ATOMIC_ATTRD
         const char *router_node = host_uname;
         xmlNode *msg_data = NULL;
         xmlNode *cmd = NULL;
 
         if (host_uname) {
             node_t *node = pe_find_node(data_set.nodes, host_uname);
 
             if (node && is_remote_node(node)) {
                 if (node->details->remote_rsc == NULL || node->details->remote_rsc->running_on == NULL) {
                     CMD_ERR("No lrmd connection detected to remote node %s", host_uname);
                     return -ENXIO;
                 }
                 node = node->details->remote_rsc->running_on->data;
                 router_node = node->details->uname;
 
             }
         }
 
         msg_data = create_xml_node(NULL, "crm-resource-reprobe-op");
         crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, host_uname);
         if (safe_str_neq(router_node, host_uname)) {
             crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
         }
 
         cmd = create_request(CRM_OP_REPROBE, msg_data, router_node,
                              CRM_SYSTEM_CRMD, crm_system_name, our_pid);
         free_xml(msg_data);
 
         crm_debug("Re-checking the state of all resources on %s", host_uname?host_uname:"all nodes");
 
-        rc = attrd_update_delegate(NULL, 'u', host_uname, "^fail-count-", NULL,
+        rc = attrd_update_delegate(NULL, 'u', host_uname,
+                                   "^" CRM_FAIL_COUNT_PREFIX "-", NULL,
                                    XML_CIB_TAG_STATUS, NULL, NULL, NULL,
                                    attrd_opt_none);
 
         if (crm_ipc_send(crmd_channel, cmd, 0, 0, NULL) > 0) {
             start_mainloop();
         }
 
         free_xml(cmd);
 #else
         GListPtr rIter = NULL;
 
         crmd_replies_needed = 0;
         for (rIter = data_set.resources; rIter; rIter = rIter->next) {
             resource_t *rsc = rIter->data;
             cli_resource_delete(cib_conn, crmd_channel, host_uname, rsc, &data_set);
         }
 
         start_mainloop();
 #endif
 
     } else if (rsc_cmd == 'D') {
         xmlNode *msg_data = NULL;
 
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r");
             rc = -ENXIO;
             goto bail;
 
         }
         if (rsc_type == NULL) {
             CMD_ERR("You need to specify a resource type with -t");
             rc = -ENXIO;
             goto bail;
 
         } else if (cib_conn == NULL) {
             rc = -ENOTCONN;
             goto bail;
         }
 
         msg_data = create_xml_node(NULL, rsc_type);
         crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);
 
         rc = cib_conn->cmds->delete(cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
         free_xml(msg_data);
 
     } else {
         CMD_ERR("Unknown command: %c", rsc_cmd);
     }
 
   bail:
 
     if (data_set.input != NULL) {
         cleanup_alloc_calculations(&data_set);
     }
     if (cib_conn != NULL) {
         cib_conn->cmds->signoff(cib_conn);
         cib_delete(cib_conn);
     }
 
     if (rc == -pcmk_err_no_quorum) {
         CMD_ERR("Error performing operation: %s", pcmk_strerror(rc));
         CMD_ERR("Try using -f");
 
     } else if (rc != pcmk_ok) {
         CMD_ERR("Error performing operation: %s", pcmk_strerror(rc));
     }
 
     return crm_exit(rc);
 }
diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
index 49815f734e..4f3a2fc660 100644
--- a/tools/crm_resource_runtime.c
+++ b/tools/crm_resource_runtime.c
@@ -1,1690 +1,1690 @@
 
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_resource.h>
 
 bool do_trace = FALSE;
 bool do_force = FALSE;
 int crmd_replies_needed = 1; /* The welcome message */
 
 const char *attr_set_type = XML_TAG_ATTR_SETS;
 
 static int
 do_find_resource(const char *rsc, resource_t * the_rsc, pe_working_set_t * data_set)
 {
     int found = 0;
     GListPtr lpc = NULL;
 
     for (lpc = the_rsc->running_on; lpc != NULL; lpc = lpc->next) {
         node_t *node = (node_t *) lpc->data;
 
         crm_trace("resource %s is running on: %s", rsc, node->details->uname);
         if (BE_QUIET) {
             fprintf(stdout, "%s\n", node->details->uname);
         } else {
             const char *state = "";
 
             if (the_rsc->variant < pe_clone && the_rsc->fns->state(the_rsc, TRUE) == RSC_ROLE_MASTER) {
                 state = "Master";
             }
             fprintf(stdout, "resource %s is running on: %s %s\n", rsc, node->details->uname, state);
         }
 
         found++;
     }
 
     if (BE_QUIET == FALSE && found == 0) {
         fprintf(stderr, "resource %s is NOT running\n", rsc);
     }
 
     return found;
 }
 
 int
 cli_resource_search(const char *rsc, pe_working_set_t * data_set)
 {
     int found = 0;
     resource_t *the_rsc = NULL;
     resource_t *parent = NULL;
 
     if (the_rsc == NULL) {
         the_rsc = pe_find_resource(data_set->resources, rsc);
     }
 
     if (the_rsc == NULL) {
         return -ENXIO;
     }
 
     if (the_rsc->variant >= pe_clone) {
         GListPtr gIter = the_rsc->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             found += do_find_resource(rsc, gIter->data, data_set);
         }
 
     /* The anonymous clone children's common ID is supplied */
     } else if ((parent = uber_parent(the_rsc)) != NULL
                && parent->variant >= pe_clone
                && is_not_set(the_rsc->flags, pe_rsc_unique)
                && the_rsc->clone_name
                && safe_str_eq(rsc, the_rsc->clone_name)
                && safe_str_neq(rsc, the_rsc->id)) {
         GListPtr gIter = parent->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             found += do_find_resource(rsc, gIter->data, data_set);
         }
 
     } else {
         found += do_find_resource(rsc, the_rsc, data_set);
     }
 
     return found;
 }
 
 resource_t *
 find_rsc_or_clone(const char *rsc, pe_working_set_t * data_set)
 {
     resource_t *the_rsc = pe_find_resource(data_set->resources, rsc);
 
     if (the_rsc == NULL) {
         char *as_clone = crm_concat(rsc, "0", ':');
 
         the_rsc = pe_find_resource(data_set->resources, as_clone);
         free(as_clone);
     }
     return the_rsc;
 }
 
 
 static int
 find_resource_attr(cib_t * the_cib, const char *attr, const char *rsc, const char *set_type,
                    const char *set_name, const char *attr_id, const char *attr_name, char **value)
 {
     int offset = 0;
     static int xpath_max = 1024;
     int rc = pcmk_ok;
     xmlNode *xml_search = NULL;
     char *xpath_string = NULL;
 
     if(value) {
         *value = NULL;
     }
 
     if(the_cib == NULL) {
         return -ENOTCONN;
     }
 
     xpath_string = calloc(1, xpath_max);
     offset +=
         snprintf(xpath_string + offset, xpath_max - offset, "%s", get_object_path("resources"));
 
     offset += snprintf(xpath_string + offset, xpath_max - offset, "//*[@id=\"%s\"]", rsc);
 
     if (set_type) {
         offset += snprintf(xpath_string + offset, xpath_max - offset, "/%s", set_type);
         if (set_name) {
             offset += snprintf(xpath_string + offset, xpath_max - offset, "[@id=\"%s\"]", set_name);
         }
     }
 
     offset += snprintf(xpath_string + offset, xpath_max - offset, "//nvpair[");
     if (attr_id) {
         offset += snprintf(xpath_string + offset, xpath_max - offset, "@id=\"%s\"", attr_id);
     }
 
     if (attr_name) {
         if (attr_id) {
             offset += snprintf(xpath_string + offset, xpath_max - offset, " and ");
         }
         offset += snprintf(xpath_string + offset, xpath_max - offset, "@name=\"%s\"", attr_name);
     }
     offset += snprintf(xpath_string + offset, xpath_max - offset, "]");
     CRM_LOG_ASSERT(offset > 0);
 
     rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
                               cib_sync_call | cib_scope_local | cib_xpath);
 
     if (rc != pcmk_ok) {
         goto bail;
     }
 
     crm_log_xml_debug(xml_search, "Match");
     if (xml_has_children(xml_search)) {
         xmlNode *child = NULL;
 
         rc = -EINVAL;
         printf("Multiple attributes match name=%s\n", attr_name);
 
         for (child = __xml_first_child(xml_search); child != NULL; child = __xml_next(child)) {
             printf("  Value: %s \t(id=%s)\n",
                    crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
         }
 
     } else if(value) {
         const char *tmp = crm_element_value(xml_search, attr);
 
         if (tmp) {
             *value = strdup(tmp);
         }
     }
 
   bail:
     free(xpath_string);
     free_xml(xml_search);
     return rc;
 }
 
 static resource_t *
 find_matching_attr_resource(resource_t * rsc, const char * rsc_id, const char * attr_set, const char * attr_id,
                             const char * attr_name, cib_t * cib, const char * cmd)
 {
     int rc = pcmk_ok;
     char *lookup_id = NULL;
     char *local_attr_id = NULL;
 
     if(do_force == TRUE) {
         return rsc;
 
     } else if(rsc->parent) {
         switch(rsc->parent->variant) {
             case pe_group:
                 if (BE_QUIET == FALSE) {
                     printf("Performing %s of '%s' for '%s' will not apply to its peers in '%s'\n", cmd, attr_name, rsc_id, rsc->parent->id);
                 }
                 break;
             case pe_master:
             case pe_clone:
 
                 rc = find_resource_attr(cib, XML_ATTR_ID, rsc_id, attr_set_type, attr_set, attr_id, attr_name, &local_attr_id);
                 free(local_attr_id);
 
                 if(rc != pcmk_ok) {
                     rsc = rsc->parent;
                     if (BE_QUIET == FALSE) {
                         printf("Performing %s of '%s' on '%s', the parent of '%s'\n", cmd, attr_name, rsc->id, rsc_id);
                     }
                 }
                 break;
             default:
                 break;
         }
 
     } else if (rsc->parent && BE_QUIET == FALSE) {
         printf("Forcing %s of '%s' for '%s' instead of '%s'\n", cmd, attr_name, rsc_id, rsc->parent->id);
 
     } else if(rsc->parent == NULL && rsc->children) {
         resource_t *child = rsc->children->data;
 
         if(child->variant == pe_native) {
             lookup_id = clone_strip(child->id); /* Could be a cloned group! */
             rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name, &local_attr_id);
 
             if(rc == pcmk_ok) {
                 rsc = child;
                 if (BE_QUIET == FALSE) {
                     printf("A value for '%s' already exists in child '%s', performing %s on that instead of '%s'\n", attr_name, lookup_id, cmd, rsc_id);
                 }
             }
 
             free(local_attr_id);
             free(lookup_id);
         }
     }
 
     return rsc;
 }
 
 int
 cli_resource_update_attribute(const char *rsc_id, const char *attr_set, const char *attr_id,
                   const char *attr_name, const char *attr_value, bool recursive,
                   cib_t * cib, pe_working_set_t * data_set)
 {
     int rc = pcmk_ok;
     static bool need_init = TRUE;
 
     char *lookup_id = NULL;
     char *local_attr_id = NULL;
     char *local_attr_set = NULL;
 
     xmlNode *xml_top = NULL;
     xmlNode *xml_obj = NULL;
 
     bool use_attributes_tag = FALSE;
     resource_t *rsc = find_rsc_or_clone(rsc_id, data_set);
 
     if (rsc == NULL) {
         return -ENXIO;
     }
 
     if(attr_id == NULL
        && do_force == FALSE
        && pcmk_ok != find_resource_attr(
            cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL, NULL, NULL, attr_name, NULL)) {
         printf("\n");
     }
 
     if (safe_str_eq(attr_set_type, XML_TAG_ATTR_SETS)) {
         if (do_force == FALSE) {
             rc = find_resource_attr(cib, XML_ATTR_ID, uber_parent(rsc)->id,
                                     XML_TAG_META_SETS, attr_set, attr_id,
                                     attr_name, &local_attr_id);
             if (rc == pcmk_ok && BE_QUIET == FALSE) {
                 printf("WARNING: There is already a meta attribute for '%s' called '%s' (id=%s)\n",
                        uber_parent(rsc)->id, attr_name, local_attr_id);
                 printf("         Delete '%s' first or use --force to override\n", local_attr_id);
             }
             free(local_attr_id);
             if (rc == pcmk_ok) {
                 return -ENOTUNIQ;
             }
         }
 
     } else {
         rsc = find_matching_attr_resource(rsc, rsc_id, attr_set, attr_id, attr_name, cib, "update");
     }
 
     lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
     rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name,
                             &local_attr_id);
 
     if (rc == pcmk_ok) {
         crm_debug("Found a match for name=%s: id=%s", attr_name, local_attr_id);
         attr_id = local_attr_id;
 
     } else if (rc != -ENXIO) {
         free(lookup_id);
         free(local_attr_id);
         return rc;
 
     } else {
         const char *value = NULL;
         xmlNode *cib_top = NULL;
         const char *tag = crm_element_name(rsc->xml);
 
         cib->cmds->query(cib, "/cib", &cib_top,
                               cib_sync_call | cib_scope_local | cib_xpath | cib_no_children);
         value = crm_element_value(cib_top, "ignore_dtd");
         if (value != NULL) {
             use_attributes_tag = TRUE;
 
         } else {
             value = crm_element_value(cib_top, XML_ATTR_VALIDATION);
             if (crm_ends_with(value, "-0.6")) {
                 use_attributes_tag = TRUE;
             }
         }
         free_xml(cib_top);
 
         if (attr_set == NULL) {
             local_attr_set = crm_concat(lookup_id, attr_set_type, '-');
             attr_set = local_attr_set;
         }
         if (attr_id == NULL) {
             local_attr_id = crm_concat(attr_set, attr_name, '-');
             attr_id = local_attr_id;
         }
 
         if (use_attributes_tag && safe_str_eq(tag, XML_CIB_TAG_MASTER)) {
             tag = "master_slave";       /* use the old name */
         }
 
         xml_top = create_xml_node(NULL, tag);
         crm_xml_add(xml_top, XML_ATTR_ID, lookup_id);
 
         xml_obj = create_xml_node(xml_top, attr_set_type);
         crm_xml_add(xml_obj, XML_ATTR_ID, attr_set);
 
         if (use_attributes_tag) {
             xml_obj = create_xml_node(xml_obj, XML_TAG_ATTRS);
         }
     }
 
     xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_NVPAIR);
     if (xml_top == NULL) {
         xml_top = xml_obj;
     }
 
     crm_xml_add(xml_obj, XML_ATTR_ID, attr_id);
     crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, attr_name);
     crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, attr_value);
 
     crm_log_xml_debug(xml_top, "Update");
 
     rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top, cib_options);
     if (rc == pcmk_ok && BE_QUIET == FALSE) {
         printf("Set '%s' option: id=%s%s%s%s%s=%s\n", lookup_id, local_attr_id,
                attr_set ? " set=" : "", attr_set ? attr_set : "",
                attr_name ? " name=" : "", attr_name ? attr_name : "", attr_value);
     }
 
     free_xml(xml_top);
 
     free(lookup_id);
     free(local_attr_id);
     free(local_attr_set);
 
     if(recursive && safe_str_eq(attr_set_type, XML_TAG_META_SETS)) {
         GListPtr lpc = NULL;
 
         if(need_init) {
             xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);
 
             need_init = FALSE;
             unpack_constraints(cib_constraints, data_set);
 
             for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
                 resource_t *r = (resource_t *) lpc->data;
 
                 clear_bit(r->flags, pe_rsc_allocating);
             }
         }
 
         crm_debug("Looking for dependencies %p", rsc->rsc_cons_lhs);
         set_bit(rsc->flags, pe_rsc_allocating);
         for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
             rsc_colocation_t *cons = (rsc_colocation_t *) lpc->data;
             resource_t *peer = cons->rsc_lh;
 
             crm_debug("Checking %s %d", cons->id, cons->score);
             if (cons->score > 0 && is_not_set(peer->flags, pe_rsc_allocating)) {
                 /* Don't get into colocation loops */
                 crm_debug("Setting %s=%s for dependent resource %s", attr_name, attr_value, peer->id);
                 cli_resource_update_attribute(peer->id, NULL, NULL, attr_name, attr_value, recursive, cib, data_set);
             }
         }
     }
 
     return rc;
 }
 
 int
 cli_resource_delete_attribute(const char *rsc_id, const char *attr_set, const char *attr_id,
                      const char *attr_name, cib_t * cib, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
 
     int rc = pcmk_ok;
     char *lookup_id = NULL;
     char *local_attr_id = NULL;
     resource_t *rsc = find_rsc_or_clone(rsc_id, data_set);
 
     if (rsc == NULL) {
         return -ENXIO;
     }
 
     if(attr_id == NULL
        && do_force == FALSE
        && find_resource_attr(
            cib, XML_ATTR_ID, uber_parent(rsc)->id, NULL, NULL, NULL, attr_name, NULL) != pcmk_ok) {
         printf("\n");
     }
 
     if(safe_str_eq(attr_set_type, XML_TAG_META_SETS)) {
         rsc = find_matching_attr_resource(rsc, rsc_id, attr_set, attr_id, attr_name, cib, "delete");
     }
 
     lookup_id = clone_strip(rsc->id);
     rc = find_resource_attr(cib, XML_ATTR_ID, lookup_id, attr_set_type, attr_set, attr_id, attr_name,
                             &local_attr_id);
 
     if (rc == -ENXIO) {
         free(lookup_id);
         return pcmk_ok;
 
     } else if (rc != pcmk_ok) {
         free(lookup_id);
         return rc;
     }
 
     if (attr_id == NULL) {
         attr_id = local_attr_id;
     }
 
     xml_obj = create_xml_node(NULL, XML_CIB_TAG_NVPAIR);
     crm_xml_add(xml_obj, XML_ATTR_ID, attr_id);
     crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, attr_name);
 
     crm_log_xml_debug(xml_obj, "Delete");
 
     CRM_ASSERT(cib);
     rc = cib->cmds->delete(cib, XML_CIB_TAG_RESOURCES, xml_obj, cib_options);
 
     if (rc == pcmk_ok && BE_QUIET == FALSE) {
         printf("Deleted '%s' option: id=%s%s%s%s%s\n", lookup_id, local_attr_id,
                attr_set ? " set=" : "", attr_set ? attr_set : "",
                attr_name ? " name=" : "", attr_name ? attr_name : "");
     }
 
     free(lookup_id);
     free_xml(xml_obj);
     free(local_attr_id);
     return rc;
 }
 
 static int
 send_lrm_rsc_op(crm_ipc_t * crmd_channel, const char *op,
                 const char *host_uname, const char *rsc_id,
                 bool only_failed, pe_working_set_t * data_set)
 {
     char *our_pid = NULL;
     char *key = NULL;
     int rc = -ECOMM;
     xmlNode *cmd = NULL;
     xmlNode *xml_rsc = NULL;
     const char *value = NULL;
     const char *router_node = host_uname;
     xmlNode *params = NULL;
     xmlNode *msg_data = NULL;
     resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
 
     if (rsc == NULL) {
         CMD_ERR("Resource %s not found", rsc_id);
         return -ENXIO;
 
     } else if (rsc->variant != pe_native) {
         CMD_ERR("We can only process primitive resources, not %s", rsc_id);
         return -EINVAL;
 
     } else if (host_uname == NULL) {
         CMD_ERR("Please supply a hostname with -H");
         return -EINVAL;
     } else {
         node_t *node = pe_find_node(data_set->nodes, host_uname);
 
         if (node && is_remote_node(node)) {
             if (node->details->remote_rsc == NULL || node->details->remote_rsc->running_on == NULL) {
                 CMD_ERR("No lrmd connection detected to remote node %s", host_uname);
                 return -ENXIO;
             }
             node = node->details->remote_rsc->running_on->data;
             router_node = node->details->uname;
         }
     }
 
     key = generate_transition_key(0, getpid(), 0, "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");
 
     msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
     crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
     free(key);
 
     crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, host_uname);
     if (safe_str_neq(router_node, host_uname)) {
         crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
     }
 
     xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
     if (rsc->clone_name) {
         crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->clone_name);
         crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc->id);
 
     } else {
         crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->id);
     }
 
     value = crm_copy_xml_element(rsc->xml, xml_rsc, XML_ATTR_TYPE);
     if (value == NULL) {
         CMD_ERR("%s has no type!  Aborting...", rsc_id);
         return -ENXIO;
     }
 
     value = crm_copy_xml_element(rsc->xml, xml_rsc, XML_AGENT_ATTR_CLASS);
     if (value == NULL) {
         CMD_ERR("%s has no class!  Aborting...", rsc_id);
         return -ENXIO;
     }
 
     crm_copy_xml_element(rsc->xml, xml_rsc, XML_AGENT_ATTR_PROVIDER);
 
     params = create_xml_node(msg_data, XML_TAG_ATTRS);
     crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
 
     key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
     crm_xml_add(params, key, "60000");  /* 1 minute */
     free(key);
 
     our_pid = calloc(1, 11);
     if (our_pid != NULL) {
         snprintf(our_pid, 10, "%d", getpid());
         our_pid[10] = '\0';
     }
     cmd = create_request(op, msg_data, router_node, CRM_SYSTEM_CRMD, crm_system_name, our_pid);
 
 /* 	crm_log_xml_warn(cmd, "send_lrm_rsc_op"); */
     free_xml(msg_data);
 
     if (crm_ipc_send(crmd_channel, cmd, 0, 0, NULL) > 0) {
         rc = 0;
 
     } else {
         CMD_ERR("Could not send %s op to the crmd", op);
         rc = -ENOTCONN;
     }
 
     free_xml(cmd);
     return rc;
 }
 
 static int
 cli_delete_attr(cib_t * cib_conn, const char * host_uname, const char * attr_name,
                 pe_working_set_t * data_set)
 {
     node_t *node = pe_find_node(data_set->nodes, host_uname);
     int attr_options = attrd_opt_none;
 
     if (node && is_remote_node(node)) {
 #if HAVE_ATOMIC_ATTRD
         set_bit(attr_options, attrd_opt_remote);
 #else
         /* Talk directly to cib for remote nodes if it's legacy attrd */
         return delete_attr_delegate(cib_conn, cib_sync_call, XML_CIB_TAG_STATUS, node->details->id, NULL, NULL,
                                     NULL, attr_name, NULL, FALSE, NULL);
 #endif
     }
     return attrd_update_delegate(NULL, 'D', host_uname, attr_name, NULL,
                                  XML_CIB_TAG_STATUS, NULL, NULL, NULL,
                                  attr_options);
 }
 
 int
 cli_resource_delete(cib_t *cib_conn, crm_ipc_t * crmd_channel, const char *host_uname,
                resource_t * rsc, pe_working_set_t * data_set)
 {
     int rc = pcmk_ok;
     node_t *node = NULL;
 
     if (rsc == NULL) {
         return -ENXIO;
 
     } else if (rsc->children) {
         GListPtr lpc = NULL;
 
         for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
             resource_t *child = (resource_t *) lpc->data;
 
             rc = cli_resource_delete(cib_conn, crmd_channel, host_uname, child, data_set);
             if(rc != pcmk_ok
                || (rsc->variant >= pe_clone && is_not_set(rsc->flags, pe_rsc_unique))) {
                 return rc;
             }
         }
         return pcmk_ok;
 
     } else if (host_uname == NULL) {
         GListPtr lpc = NULL;
 
         for (lpc = data_set->nodes; lpc != NULL; lpc = lpc->next) {
             node = (node_t *) lpc->data;
 
             if (node->details->online) {
                 cli_resource_delete(cib_conn, crmd_channel, node->details->uname, rsc, data_set);
             }
         }
 
         return pcmk_ok;
     }
 
     node = pe_find_node(data_set->nodes, host_uname);
 
     if (node && node->details->rsc_discovery_enabled) {
         printf("Cleaning up %s on %s", rsc->id, host_uname);
         rc = send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_DELETE, host_uname, rsc->id, TRUE, data_set);
     } else {
         printf("Resource discovery disabled on %s. Unable to delete lrm state.\n", host_uname);
         rc = -EOPNOTSUPP;
     }
 
     if (rc == pcmk_ok) {
         char *attr_name = NULL;
 
         if(node && node->details->remote_rsc == NULL && node->details->rsc_discovery_enabled) {
             crmd_replies_needed++;
         }
 
         if(is_not_set(rsc->flags, pe_rsc_unique)) {
             char *id = clone_strip(rsc->id);
-            attr_name = crm_strdup_printf("fail-count-%s", id);
+            attr_name = crm_failcount_name(id);
             free(id);
 
         } else if (rsc->clone_name) {
-            attr_name = crm_strdup_printf("fail-count-%s", rsc->clone_name);
+            attr_name = crm_failcount_name(rsc->clone_name);
 
         } else {
-            attr_name = crm_strdup_printf("fail-count-%s", rsc->id);
+            attr_name = crm_failcount_name(rsc->id);
         }
 
         printf(", removing %s\n", attr_name);
         rc = cli_delete_attr(cib_conn, host_uname, attr_name, data_set);
         free(attr_name);
 
     } else if(rc != -EOPNOTSUPP) {
         printf(" - FAILED\n");
     }
 
     return rc;
 }
 
 void
 cli_resource_check(cib_t * cib_conn, resource_t *rsc)
 {
     int need_nl = 0;
     char *role_s = NULL;
     char *managed = NULL;
     resource_t *parent = uber_parent(rsc);
 
     find_resource_attr(cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
                        NULL, NULL, NULL, XML_RSC_ATTR_MANAGED, &managed);
 
     find_resource_attr(cib_conn, XML_NVPAIR_ATTR_VALUE, parent->id,
                        NULL, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, &role_s);
 
     if(role_s) {
         enum rsc_role_e role = text2role(role_s);
         if(role == RSC_ROLE_UNKNOWN) {
             // Treated as if unset
 
         } else if(role == RSC_ROLE_STOPPED) {
             printf("\n  * The configuration specifies that '%s' should remain stopped\n", parent->id);
             need_nl++;
 
         } else if(parent->variant > pe_clone && role == RSC_ROLE_SLAVE) {
             printf("\n  * The configuration specifies that '%s' should not be promoted\n", parent->id);
             need_nl++;
         }
     }
 
     if(managed && crm_is_true(managed) == FALSE) {
         printf("%s  * The configuration prevents the cluster from stopping or starting '%s' (unmanaged)\n", need_nl == 0?"\n":"", parent->id);
         need_nl++;
     }
 
     if(need_nl) {
         printf("\n");
     }
 }
 
 int
 cli_resource_fail(crm_ipc_t * crmd_channel, const char *host_uname,
              const char *rsc_id, pe_working_set_t * data_set)
 {
     crm_warn("Failing: %s", rsc_id);
     return send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_FAIL, host_uname, rsc_id, FALSE, data_set);
 }
 
 static GHashTable *
 generate_resource_params(resource_t * rsc, pe_working_set_t * data_set)
 {
     GHashTable *params = NULL;
     GHashTable *meta = NULL;
     GHashTable *combined = NULL;
     GHashTableIter iter;
 
     if (!rsc) {
         crm_err("Resource does not exist in config");
         return NULL;
     }
 
     params =
         g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
     meta = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
     combined =
         g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
 
     get_rsc_attributes(params, rsc, NULL /* TODO: Pass in local node */ , data_set);
     get_meta_attributes(meta, rsc, NULL /* TODO: Pass in local node */ , data_set);
 
     if (params) {
         char *key = NULL;
         char *value = NULL;
 
         g_hash_table_iter_init(&iter, params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             g_hash_table_insert(combined, strdup(key), strdup(value));
         }
         g_hash_table_destroy(params);
     }
 
     if (meta) {
         char *key = NULL;
         char *value = NULL;
 
         g_hash_table_iter_init(&iter, meta);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             char *crm_name = crm_meta_name(key);
 
             g_hash_table_insert(combined, crm_name, strdup(value));
         }
         g_hash_table_destroy(meta);
     }
 
     return combined;
 }
 
 static bool resource_is_running_on(resource_t *rsc, const char *host) 
 {
     bool found = TRUE;
     GListPtr hIter = NULL;
     GListPtr hosts = NULL;
 
     if(rsc == NULL) {
         return FALSE;
     }
 
     rsc->fns->location(rsc, &hosts, TRUE);
     for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
         pe_node_t *node = (pe_node_t *) hIter->data;
 
         if(strcmp(host, node->details->uname) == 0) {
             crm_trace("Resource %s is running on %s\n", rsc->id, host);
             goto done;
         } else if(strcmp(host, node->details->id) == 0) {
             crm_trace("Resource %s is running on %s\n", rsc->id, host);
             goto done;
         }
     }
 
     if(host != NULL) {
         crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
         found = FALSE;
 
     } else if(host == NULL && hosts == NULL) {
         crm_trace("Resource %s is not running\n", rsc->id);
         found = FALSE;
     }
 
   done:
 
     g_list_free(hosts);
     return found;
 }
 
 /*!
  * \internal
  * \brief Create a list of all resources active on host from a given list
  *
  * \param[in] host      Name of host to check whether resources are active
  * \param[in] rsc_list  List of resources to check
  *
  * \return New list of resources from list that are active on host
  */
 static GList *
 get_active_resources(const char *host, GList *rsc_list)
 {
     GList *rIter = NULL;
     GList *active = NULL;
 
     for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
         resource_t *rsc = (resource_t *) rIter->data;
 
         /* Expand groups to their members, because if we're restarting a member
          * other than the first, we can't otherwise tell which resources are
          * stopping and starting.
          */
         if (rsc->variant == pe_group) {
             active = g_list_concat(active,
                                    get_active_resources(host, rsc->children));
         } else if (resource_is_running_on(rsc, host)) {
             active = g_list_append(active, strdup(rsc->id));
         }
     }
     return active;
 }
 
 static GList *subtract_lists(GList *from, GList *items) 
 {
     GList *item = NULL;
     GList *result = g_list_copy(from);
 
     for (item = items; item != NULL; item = item->next) {
         GList *candidate = NULL;
         for (candidate = from; candidate != NULL; candidate = candidate->next) {
             crm_info("Comparing %s with %s", candidate->data, item->data);
             if(strcmp(candidate->data, item->data) == 0) {
                 result = g_list_remove(result, candidate->data);
                 break;
             }
         }
     }
 
     return result;
 }
 
 static void dump_list(GList *items, const char *tag) 
 {
     int lpc = 0;
     GList *item = NULL;
 
     for (item = items; item != NULL; item = item->next) {
         crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
         lpc++;
     }
 }
 
 static void display_list(GList *items, const char *tag) 
 {
     GList *item = NULL;
 
     for (item = items; item != NULL; item = item->next) {
         fprintf(stdout, "%s%s\n", tag, (const char *)item->data);
     }
 }
 
 /*!
  * \internal
  * \brief Upgrade XML to latest schema version and use it as working set input
  *
  * This also updates the working set timestamp to the current time.
  *
  * \param[in] data_set   Working set instance to update
  * \param[in] xml        XML to use as input
  *
  * \return pcmk_ok on success, -ENOKEY if unable to upgrade XML
  * \note On success, caller is responsible for freeing memory allocated for
  *       data_set->now.
  * \todo This follows the example of other callers of cli_config_update()
  *       and returns -ENOKEY ("Required key not available") if that fails,
  *       but perhaps -pcmk_err_schema_validation would be better in that case.
  */
 int
 update_working_set_xml(pe_working_set_t *data_set, xmlNode **xml)
 {
     if (cli_config_update(xml, NULL, FALSE) == FALSE) {
         return -ENOKEY;
     }
     data_set->input = *xml;
     data_set->now = crm_time_new(NULL);
     return pcmk_ok;
 }
 
 /*!
  * \internal
  * \brief Update a working set's XML input based on a CIB query
  *
  * \param[in] data_set   Data set instance to initialize
  * \param[in] cib        Connection to the CIB
  *
  * \return pcmk_ok on success, -errno on failure
  * \note On success, caller is responsible for freeing memory allocated for
  *       data_set->input and data_set->now.
  */
 static int
 update_working_set_from_cib(pe_working_set_t * data_set, cib_t *cib)
 {
     xmlNode *cib_xml_copy = NULL;
     int rc;
 
     rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
     if (rc != pcmk_ok) {
         fprintf(stderr, "Could not obtain the current CIB: %s (%d)\n", pcmk_strerror(rc), rc);
         return rc;
     }
     rc = update_working_set_xml(data_set, &cib_xml_copy);
     if (rc != pcmk_ok) {
         fprintf(stderr, "Could not upgrade the current CIB XML\n");
         free_xml(cib_xml_copy);
         return rc;
     }
     return pcmk_ok;
 }
 
 static int
 update_dataset(cib_t *cib, pe_working_set_t * data_set, bool simulate)
 {
     char *pid = NULL;
     char *shadow_file = NULL;
     cib_t *shadow_cib = NULL;
     int rc;
 
     cleanup_alloc_calculations(data_set);
     rc = update_working_set_from_cib(data_set, cib);
     if (rc != pcmk_ok) {
         return rc;
     }
 
     if(simulate) {
         pid = crm_itoa(getpid());
         shadow_cib = cib_shadow_new(pid);
         shadow_file = get_shadow_file(pid);
 
         if (shadow_cib == NULL) {
             fprintf(stderr, "Could not create shadow cib: '%s'\n", pid);
             rc = -ENXIO;
             goto cleanup;
         }
 
         rc = write_xml_file(data_set->input, shadow_file, FALSE);
 
         if (rc < 0) {
             fprintf(stderr, "Could not populate shadow cib: %s (%d)\n", pcmk_strerror(rc), rc);
             goto cleanup;
         }
 
         rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
         if(rc != pcmk_ok) {
             fprintf(stderr, "Could not connect to shadow cib: %s (%d)\n", pcmk_strerror(rc), rc);
             goto cleanup;
         }
 
         do_calculations(data_set, data_set->input, NULL);
         run_simulation(data_set, shadow_cib, NULL, TRUE);
         rc = update_dataset(shadow_cib, data_set, FALSE);
 
     } else {
         cluster_status(data_set);
     }
 
   cleanup:
     /* Do not free data_set->input here, we need rsc->xml to be valid later on */
     cib_delete(shadow_cib);
     free(pid);
 
     if(shadow_file) {
         unlink(shadow_file);
         free(shadow_file);
     }
 
     return rc;
 }
 
 static int
 max_delay_for_resource(pe_working_set_t * data_set, resource_t *rsc) 
 {
     int delay = 0;
     int max_delay = 0;
 
     if(rsc && rsc->children) {
         GList *iter = NULL;
 
         for(iter = rsc->children; iter; iter = iter->next) {
             resource_t *child = (resource_t *)iter->data;
 
             delay = max_delay_for_resource(data_set, child);
             if(delay > max_delay) {
                 double seconds = delay / 1000.0;
                 crm_trace("Calculated new delay of %.1fs due to %s", seconds, child->id);
                 max_delay = delay;
             }
         }
 
     } else if(rsc) {
         char *key = crm_strdup_printf("%s_%s_0", rsc->id, RSC_STOP);
         action_t *stop = custom_action(rsc, key, RSC_STOP, NULL, TRUE, FALSE, data_set);
         const char *value = g_hash_table_lookup(stop->meta, XML_ATTR_TIMEOUT);
 
         max_delay = crm_int_helper(value, NULL);
         pe_free_action(stop);
     }
 
 
     return max_delay;
 }
 
 static int
 max_delay_in(pe_working_set_t * data_set, GList *resources) 
 {
     int max_delay = 0;
     GList *item = NULL;
 
     for (item = resources; item != NULL; item = item->next) {
         int delay = 0;
         resource_t *rsc = pe_find_resource(data_set->resources, (const char *)item->data);
 
         delay = max_delay_for_resource(data_set, rsc);
 
         if(delay > max_delay) {
             double seconds = delay / 1000.0;
             crm_trace("Calculated new delay of %.1fs due to %s", seconds, rsc->id);
             max_delay = delay;
         }
     }
 
     return 5 + (max_delay / 1000);
 }
 
 #define waiting_for_starts(d, r, h) ((g_list_length(d) > 0) || \
                                     (resource_is_running_on((r), (h)) == FALSE))
 
 /*!
  * \internal
  * \brief Restart a resource (on a particular host if requested).
  *
  * \param[in] rsc        The resource to restart
  * \param[in] host       The host to restart the resource on (or NULL for all)
  * \param[in] timeout_ms Consider failed if actions do not complete in this time
  *                       (specified in milliseconds, but a two-second
  *                       granularity is actually used; if 0, a timeout will be
  *                       calculated based on the resource timeout)
  * \param[in] cib        Connection to the CIB for modifying/checking resource
  *
  * \return pcmk_ok on success, -errno on failure (exits on certain failures)
  */
 int
 cli_resource_restart(resource_t * rsc, const char *host, int timeout_ms, cib_t * cib)
 {
     int rc = 0;
     int lpc = 0;
     int before = 0;
     int step_timeout_s = 0;
     int sleep_interval = 2;
     int timeout = timeout_ms / 1000;
 
     bool is_clone = FALSE;
     char *rsc_id = NULL;
     char *orig_target_role = NULL;
 
     GList *list_delta = NULL;
     GList *target_active = NULL;
     GList *current_active = NULL;
     GList *restart_target_active = NULL;
 
     pe_working_set_t data_set;
 
     if(resource_is_running_on(rsc, host) == FALSE) {
         const char *id = rsc->clone_name?rsc->clone_name:rsc->id;
         if(host) {
             printf("%s is not running on %s and so cannot be restarted\n", id, host);
         } else {
             printf("%s is not running anywhere and so cannot be restarted\n", id);
         }
         return -ENXIO;
     }
 
     /* We might set the target-role meta-attribute */
     attr_set_type = XML_TAG_META_SETS;
 
     rsc_id = strdup(rsc->id);
     if(rsc->variant > pe_group) {
         is_clone = TRUE;
     }
 
     /*
       grab full cib
       determine originally active resources
       disable or ban
       poll cib and watch for affected resources to get stopped
       without --timeout, calculate the stop timeout for each step and wait for that
       if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
       if everything stopped, re-enable or un-ban
       poll cib and watch for affected resources to get started
       without --timeout, calculate the start timeout for each step and wait for that
       if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
       report success
 
       Optimizations:
       - use constraints to determine ordered list of affected resources
       - Allow a --no-deps option (aka. --force-restart)
     */
 
 
     set_working_set_defaults(&data_set);
     rc = update_dataset(cib, &data_set, FALSE);
     if(rc != pcmk_ok) {
         fprintf(stdout, "Could not get new resource list: %s (%d)\n", pcmk_strerror(rc), rc);
         free(rsc_id);
         return rc;
     }
 
     restart_target_active = get_active_resources(host, data_set.resources);
     current_active = get_active_resources(host, data_set.resources);
 
     dump_list(current_active, "Origin");
 
     if(is_clone && host) {
         /* Stop the clone instance by banning it from the host */
         BE_QUIET = TRUE;
         rc = cli_resource_ban(rsc_id, host, NULL, cib);
 
     } else {
         /* Stop the resource by setting target-role to Stopped.
          * Remember any existing target-role so we can restore it later
          * (though it only makes any difference if it's Slave).
          */
         char *lookup_id = clone_strip(rsc->id);
 
         find_resource_attr(cib, XML_NVPAIR_ATTR_VALUE, lookup_id, NULL, NULL,
                            NULL, XML_RSC_ATTR_TARGET_ROLE, &orig_target_role);
         free(lookup_id);
         rc = cli_resource_update_attribute(rsc_id, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, RSC_STOPPED, FALSE, cib, &data_set);
     }
     if(rc != pcmk_ok) {
         fprintf(stderr, "Could not set target-role for %s: %s (%d)\n", rsc_id, pcmk_strerror(rc), rc);
         if (current_active) {
             g_list_free_full(current_active, free);
         }
         if (restart_target_active) {
             g_list_free_full(restart_target_active, free);
         }
         free(rsc_id);
         return crm_exit(rc);
     }
 
     rc = update_dataset(cib, &data_set, TRUE);
     if(rc != pcmk_ok) {
         fprintf(stderr, "Could not determine which resources would be stopped\n");
         goto failure;
     }
 
     target_active = get_active_resources(host, data_set.resources);
     dump_list(target_active, "Target");
 
     list_delta = subtract_lists(current_active, target_active);
     fprintf(stdout, "Waiting for %d resources to stop:\n", g_list_length(list_delta));
     display_list(list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
     while(g_list_length(list_delta) > 0) {
         before = g_list_length(list_delta);
         if(timeout_ms == 0) {
             step_timeout_s = max_delay_in(&data_set, list_delta) / sleep_interval;
         }
 
         /* We probably don't need the entire step timeout */
         for(lpc = 0; lpc < step_timeout_s && g_list_length(list_delta) > 0; lpc++) {
             sleep(sleep_interval);
             if(timeout) {
                 timeout -= sleep_interval;
                 crm_trace("%ds remaining", timeout);
             }
             rc = update_dataset(cib, &data_set, FALSE);
             if(rc != pcmk_ok) {
                 fprintf(stderr, "Could not determine which resources were stopped\n");
                 goto failure;
             }
 
             if (current_active) {
                 g_list_free_full(current_active, free);
             }
             current_active = get_active_resources(host, data_set.resources);
             g_list_free(list_delta);
             list_delta = subtract_lists(current_active, target_active);
             dump_list(current_active, "Current");
             dump_list(list_delta, "Delta");
         }
 
         crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
         if(before == g_list_length(list_delta)) {
             /* aborted during stop phase, print the contents of list_delta */
             fprintf(stderr, "Could not complete shutdown of %s, %d resources remaining\n", rsc_id, g_list_length(list_delta));
             display_list(list_delta, " * ");
             rc = -ETIME;
             goto failure;
         }
 
     }
 
     if(is_clone && host) {
         rc = cli_resource_clear(rsc_id, host, NULL, cib);
 
     } else if (orig_target_role) {
         rc = cli_resource_update_attribute(rsc_id, NULL, NULL,
                                            XML_RSC_ATTR_TARGET_ROLE,
                                            orig_target_role, FALSE, cib,
                                            &data_set);
         free(orig_target_role);
         orig_target_role = NULL;
     } else {
         rc = cli_resource_delete_attribute(rsc_id, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, cib, &data_set);
     }
 
     if(rc != pcmk_ok) {
         fprintf(stderr, "Could not unset target-role for %s: %s (%d)\n", rsc_id, pcmk_strerror(rc), rc);
         free(rsc_id);
         return crm_exit(rc);
     }
 
     if (target_active) {
         g_list_free_full(target_active, free);
     }
     target_active = restart_target_active;
     if (list_delta) {
         g_list_free(list_delta);
     }
     list_delta = subtract_lists(target_active, current_active);
     fprintf(stdout, "Waiting for %d resources to start again:\n", g_list_length(list_delta));
     display_list(list_delta, " * ");
 
     step_timeout_s = timeout / sleep_interval;
     while (waiting_for_starts(list_delta, rsc, host)) {
         before = g_list_length(list_delta);
         if(timeout_ms == 0) {
             step_timeout_s = max_delay_in(&data_set, list_delta) / sleep_interval;
         }
 
         /* We probably don't need the entire step timeout */
         for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {
 
             sleep(sleep_interval);
             if(timeout) {
                 timeout -= sleep_interval;
                 crm_trace("%ds remaining", timeout);
             }
 
             rc = update_dataset(cib, &data_set, FALSE);
             if(rc != pcmk_ok) {
                 fprintf(stderr, "Could not determine which resources were started\n");
                 goto failure;
             }
 
             if (current_active) {
                 g_list_free_full(current_active, free);
             }
 
             /* It's OK if dependent resources moved to a different node,
              * so we check active resources on all nodes.
              */
             current_active = get_active_resources(NULL, data_set.resources);
             g_list_free(list_delta);
             list_delta = subtract_lists(target_active, current_active);
             dump_list(current_active, "Current");
             dump_list(list_delta, "Delta");
         }
 
         if(before == g_list_length(list_delta)) {
             /* aborted during start phase, print the contents of list_delta */
             fprintf(stdout, "Could not complete restart of %s, %d resources remaining\n", rsc_id, g_list_length(list_delta));
             display_list(list_delta, " * ");
             rc = -ETIME;
             goto failure;
         }
 
     }
 
     rc = pcmk_ok;
     goto done;
 
   failure:
     if(is_clone && host) {
         cli_resource_clear(rsc_id, host, NULL, cib);
     } else if (orig_target_role) {
         cli_resource_update_attribute(rsc_id, NULL, NULL,
                                       XML_RSC_ATTR_TARGET_ROLE,
                                       orig_target_role, FALSE, cib, &data_set);
         free(orig_target_role);
     } else {
         cli_resource_delete_attribute(rsc_id, NULL, NULL, XML_RSC_ATTR_TARGET_ROLE, cib, &data_set);
     }
 
 done:
     if (list_delta) {
         g_list_free(list_delta);
     }
     if (current_active) {
         g_list_free_full(current_active, free);
     }
     if (target_active && (target_active != restart_target_active)) {
         g_list_free_full(target_active, free);
     }
     if (restart_target_active) {
         g_list_free_full(restart_target_active, free);
     }
     cleanup_alloc_calculations(&data_set);
     free(rsc_id);
     return rc;
 }
 
 #define action_is_pending(action) \
     ((is_set((action)->flags, pe_action_optional) == FALSE) \
     && (is_set((action)->flags, pe_action_runnable) == TRUE) \
     && (is_set((action)->flags, pe_action_pseudo) == FALSE))
 
 /*!
  * \internal
  * \brief Return TRUE if any actions in a list are pending
  *
  * \param[in] actions   List of actions to check
  *
  * \return TRUE if any actions in the list are pending, FALSE otherwise
  */
 static bool
 actions_are_pending(GListPtr actions)
 {
     GListPtr action;
 
     for (action = actions; action != NULL; action = action->next) {
         if (action_is_pending((action_t *) action->data)) {
             return TRUE;
         }
     }
     return FALSE;
 }
 
 /*!
  * \internal
  * \brief Print pending actions to stderr
  *
  * \param[in] actions   List of actions to check
  *
  * \return void
  */
 static void
 print_pending_actions(GListPtr actions)
 {
     GListPtr action;
 
     fprintf(stderr, "Pending actions:\n");
     for (action = actions; action != NULL; action = action->next) {
         action_t *a = (action_t *) action->data;
 
         if (action_is_pending(a)) {
             fprintf(stderr, "\tAction %d: %s", a->id, a->uuid);
             if (a->node) {
                 fprintf(stderr, "\ton %s", a->node->details->uname);
             }
             fprintf(stderr, "\n");
         }
     }
 }
 
 /* For --wait, timeout (in seconds) to use if caller doesn't specify one */
 #define WAIT_DEFAULT_TIMEOUT_S (60 * 60)
 
 /* For --wait, how long to sleep between cluster state checks */
 #define WAIT_SLEEP_S (2)
 
 /*!
  * \internal
  * \brief Wait until all pending cluster actions are complete
  *
  * This waits until either the CIB's transition graph is idle or a timeout is
  * reached.
  *
  * \param[in] timeout_ms Consider failed if actions do not complete in this time
  *                       (specified in milliseconds, but one-second granularity
  *                       is actually used; if 0, a default will be used)
  * \param[in] cib        Connection to the CIB
  *
  * \return pcmk_ok on success, -errno on failure
  */
 int
 wait_till_stable(int timeout_ms, cib_t * cib)
 {
     pe_working_set_t data_set;
     int rc = -1;
     int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
     time_t expire_time = time(NULL) + timeout_s;
     time_t time_diff;
 
     set_working_set_defaults(&data_set);
     do {
 
         /* Abort if timeout is reached */
         time_diff = expire_time - time(NULL);
         if (time_diff > 0) {
             crm_info("Waiting up to %d seconds for cluster actions to complete", time_diff);
         } else {
             print_pending_actions(data_set.actions);
             cleanup_alloc_calculations(&data_set);
             return -ETIME;
         }
         if (rc == pcmk_ok) { /* this avoids sleep on first loop iteration */
             sleep(WAIT_SLEEP_S);
         }
 
         /* Get latest transition graph */
         cleanup_alloc_calculations(&data_set);
         rc = update_working_set_from_cib(&data_set, cib);
         if (rc != pcmk_ok) {
             cleanup_alloc_calculations(&data_set);
             return rc;
         }
         do_calculations(&data_set, data_set.input, NULL);
 
     } while (actions_are_pending(data_set.actions));
 
     return pcmk_ok;
 }
 
 int
 cli_resource_execute(const char *rsc_id, const char *rsc_action, GHashTable *override_hash, cib_t * cib, pe_working_set_t *data_set)
 {
     int rc = pcmk_ok;
     svc_action_t *op = NULL;
     const char *rtype = NULL;
     const char *rprov = NULL;
     const char *rclass = NULL;
     const char *action = NULL;
     GHashTable *params = NULL;
     resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
 
     if (rsc == NULL) {
         CMD_ERR("Must supply a resource id with -r");
         return -ENXIO;
     }
 
     if (safe_str_eq(rsc_action, "force-check")) {
         action = "monitor";
 
     } else if (safe_str_eq(rsc_action, "force-stop")) {
         action = rsc_action+6;
 
     } else if (safe_str_eq(rsc_action, "force-start")
                || safe_str_eq(rsc_action, "force-demote")
                || safe_str_eq(rsc_action, "force-promote")) {
         action = rsc_action+6;
 
         if(rsc->variant >= pe_clone) {
             rc = cli_resource_search(rsc_id, data_set);
             if(rc > 0 && do_force == FALSE) {
                 CMD_ERR("It is not safe to %s %s here: the cluster claims it is already active", action, rsc_id);
                 CMD_ERR("Try setting target-role=stopped first or specifying --force");
                 crm_exit(EPERM);
             }
         }
     }
 
     if(rsc->variant == pe_clone || rsc->variant == pe_master) {
         /* Grab the first child resource in the hope it's not a group */
         rsc = rsc->children->data;
     }
 
     if(rsc->variant == pe_group) {
         CMD_ERR("Sorry, --%s doesn't support group resources", rsc_action);
         crm_exit(EOPNOTSUPP);
     }
 
     rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
     rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
     rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
 
     if(safe_str_eq(rclass, "stonith")){
         CMD_ERR("Sorry, --%s doesn't support %s resources yet", rsc_action, rclass);
         crm_exit(EOPNOTSUPP);
     }
 
     params = generate_resource_params(rsc, data_set);
     op = resources_action_create(rsc->id, rclass, rprov, rtype, action, 0, -1, params, 0);
 
     if(do_trace) {
         setenv("OCF_TRACE_RA", "1", 1);
     }
 
     if(op && override_hash) {
         GHashTableIter iter;
         char *name = NULL;
         char *value = NULL;
 
         g_hash_table_iter_init(&iter, override_hash);
         while (g_hash_table_iter_next(&iter, (gpointer *) & name, (gpointer *) & value)) {
             printf("Overriding the cluster configuration for '%s' with '%s' = '%s'\n",
                    rsc->id, name, value);
             g_hash_table_replace(op->params, strdup(name), strdup(value));
         }
     }
 
     if(op == NULL) {
         /* Re-run but with stderr enabled so we can display a sane error message */
         crm_enable_stderr(TRUE);
         resources_action_create(rsc->id, rclass, rprov, rtype, action, 0, -1, params, 0);
         return crm_exit(EINVAL);
 
     } else if (services_action_sync(op)) {
         int more, lpc, last;
         char *local_copy = NULL;
 
         if (op->status == PCMK_LRM_OP_DONE) {
             printf("Operation %s for %s (%s:%s:%s) returned %d\n",
                    action, rsc->id, rclass, rprov ? rprov : "", rtype, op->rc);
         } else {
             printf("Operation %s for %s (%s:%s:%s) failed: %d\n",
                    action, rsc->id, rclass, rprov ? rprov : "", rtype, op->status);
         }
 
         if (op->stdout_data) {
             local_copy = strdup(op->stdout_data);
             more = strlen(local_copy);
             last = 0;
 
             for (lpc = 0; lpc < more; lpc++) {
                 if (local_copy[lpc] == '\n' || local_copy[lpc] == 0) {
                     local_copy[lpc] = 0;
                     printf(" >  stdout: %s\n", local_copy + last);
                     last = lpc + 1;
                 }
             }
             free(local_copy);
         }
         if (op->stderr_data) {
             local_copy = strdup(op->stderr_data);
             more = strlen(local_copy);
             last = 0;
 
             for (lpc = 0; lpc < more; lpc++) {
                 if (local_copy[lpc] == '\n' || local_copy[lpc] == 0) {
                     local_copy[lpc] = 0;
                     printf(" >  stderr: %s\n", local_copy + last);
                     last = lpc + 1;
                 }
             }
             free(local_copy);
         }
     }
     rc = op->rc;
     services_action_free(op);
     return rc;
 }
 
 int
 cli_resource_move(const char *rsc_id, const char *host_name, cib_t * cib, pe_working_set_t *data_set)
 {
     int rc = -EINVAL;
     int count = 0;
     node_t *current = NULL;
     node_t *dest = pe_find_node(data_set->nodes, host_name);
     resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
     bool cur_is_dest = FALSE;
 
     if (rsc == NULL) {
         CMD_ERR("Resource '%s' not moved: not found", rsc_id);
         return -ENXIO;
 
     } else if (scope_master && rsc->variant < pe_master) {
         resource_t *p = uber_parent(rsc);
         if(p->variant == pe_master) {
             CMD_ERR("Using parent '%s' for --move command instead of '%s'.", rsc->id, rsc_id);
             rsc_id = p->id;
             rsc = p;
 
         } else {
             CMD_ERR("Ignoring '--master' option: not valid for %s resources.",
                     get_resource_typename(rsc->variant));
             scope_master = FALSE;
         }
     }
 
     if(rsc->variant == pe_master) {
         GListPtr iter = NULL;
 
         for(iter = rsc->children; iter; iter = iter->next) {
             resource_t *child = (resource_t *)iter->data;
             enum rsc_role_e child_role = child->fns->state(child, TRUE);
 
             if(child_role == RSC_ROLE_MASTER) {
                 rsc = child;
                 count++;
             }
         }
 
         if(scope_master == FALSE && count == 0) {
             count = g_list_length(rsc->running_on);
         }
 
     } else if (rsc->variant > pe_group) {
         count = g_list_length(rsc->running_on);
 
     } else if (g_list_length(rsc->running_on) > 1) {
         CMD_ERR("Resource '%s' not moved: active on multiple nodes", rsc_id);
         return rc;
     }
 
     if(dest == NULL) {
         CMD_ERR("Error performing operation: node '%s' is unknown", host_name);
         return -ENXIO;
     }
 
     if(g_list_length(rsc->running_on) == 1) {
         current = rsc->running_on->data;
     }
 
     if(current == NULL) {
         /* Nothing to check */
 
     } else if(scope_master && rsc->fns->state(rsc, TRUE) != RSC_ROLE_MASTER) {
         crm_trace("%s is already active on %s but not in correct state", rsc_id, dest->details->uname);
     } else if (safe_str_eq(current->details->uname, dest->details->uname)) {
         cur_is_dest = TRUE;
         if (do_force) {
             crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
                      rsc_id, scope_master?"promoted":"active", dest->details->uname);
         } else {
             CMD_ERR("Error performing operation: %s is already %s on %s",
                     rsc_id, scope_master?"promoted":"active", dest->details->uname);
             return rc;
         }
     }
 
     /* Clear any previous constraints for 'dest' */
     cli_resource_clear(rsc_id, dest->details->uname, data_set->nodes, cib);
 
     /* Record an explicit preference for 'dest' */
     rc = cli_resource_prefer(rsc_id, dest->details->uname, cib);
 
     crm_trace("%s%s now prefers node %s%s",
               rsc->id, scope_master?" (master)":"", dest->details->uname, do_force?"(forced)":"");
 
     /* only ban the previous location if current location != destination location.
      * it is possible to use -M to enforce a location without regard of where the
      * resource is currently located */
     if(do_force && (cur_is_dest == FALSE)) {
         /* Ban the original location if possible */
         if(current) {
             (void)cli_resource_ban(rsc_id, current->details->uname, NULL, cib);
 
         } else if(count > 1) {
             CMD_ERR("Resource '%s' is currently %s in %d locations.  One may now move one to %s",
                     rsc_id, scope_master?"promoted":"active", count, dest->details->uname);
             CMD_ERR("You can prevent '%s' from being %s at a specific location with:"
                     " --ban %s--host <name>", rsc_id, scope_master?"promoted":"active", scope_master?"--master ":"");
 
         } else {
             crm_trace("Not banning %s from its current location: not active", rsc_id);
         }
     }
 
     return rc;
 }
diff --git a/tools/fake_transition.c b/tools/fake_transition.c
index d4e61892c1..c45349f545 100644
--- a/tools/fake_transition.c
+++ b/tools/fake_transition.c
@@ -1,852 +1,852 @@
 /*
  * Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <unistd.h>
 #include <stdlib.h>
 
 #include <sys/stat.h>
 #include <sys/param.h>
 #include <sys/types.h>
 #include <dirent.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/common/util.h>
 #include <crm/transition.h>
 #include <crm/common/iso8601.h>
 #include <crm/pengine/status.h>
 #include <allocate.h>
 #include "fake_transition.h"
 
 static bool fake_quiet = FALSE;
 static cib_t *fake_cib = NULL;
 static GListPtr fake_resource_list = NULL;
 static GListPtr fake_op_fail_list = NULL;
 gboolean bringing_nodes_online = FALSE;
 
 #define STATUS_PATH_MAX 512
 
 #define quiet_log(fmt, args...) do {              \
               if(fake_quiet) {                         \
                   crm_trace(fmt, ##args);         \
               } else {                            \
                   printf(fmt , ##args);           \
               }                                   \
     } while(0)
 
 #define new_node_template "//"XML_CIB_TAG_NODE"[@uname='%s']"
 #define node_template "//"XML_CIB_TAG_STATE"[@uname='%s']"
 #define rsc_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']"
 #define op_template  "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s']"
 /* #define op_template  "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s' and @"XML_LRM_ATTR_CALLID"='%d']" */
 
 
 static void
 inject_transient_attr(xmlNode * cib_node, const char *name, const char *value)
 {
     xmlNode *attrs = NULL;
     xmlNode *container = NULL;
     xmlNode *nvp = NULL;
     xmlChar *node_path;
     const char *node_uuid = ID(cib_node);
     char *nvp_id = crm_concat(name, node_uuid, '-');
 
     node_path = xmlGetNodePath(cib_node);
     quiet_log("Injecting attribute %s=%s into %s '%s'", name, value, node_path,
              ID(cib_node));
     free(node_path);
 
     attrs = first_named_child(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
     if (attrs == NULL) {
         attrs = create_xml_node(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
         crm_xml_add(attrs, XML_ATTR_ID, node_uuid);
     }
 
     container = first_named_child(attrs, XML_TAG_ATTR_SETS);
     if (container == NULL) {
         container = create_xml_node(attrs, XML_TAG_ATTR_SETS);
         crm_xml_add(container, XML_ATTR_ID, node_uuid);
     }
 
     nvp = create_xml_node(container, XML_CIB_TAG_NVPAIR);
     crm_xml_add(nvp, XML_ATTR_ID, nvp_id);
     crm_xml_add(nvp, XML_NVPAIR_ATTR_NAME, name);
     crm_xml_add(nvp, XML_NVPAIR_ATTR_VALUE, value);
 
     free(nvp_id);
 }
 
 static void
 update_failcounts(xmlNode * cib_node, const char *resource, int interval, int rc)
 {
     if (rc == 0) {
         return;
 
     } else if (rc == 7 && interval == 0) {
         return;
 
     } else {
         char *name = NULL;
         char *now = crm_itoa(time(NULL));
 
-        name = crm_concat("fail-count", resource, '-');
+        name = crm_failcount_name(resource);
         inject_transient_attr(cib_node, name, "value++");
+        free(name);
 
-        name = crm_concat("last-failure", resource, '-');
+        name = crm_lastfailure_name(resource);
         inject_transient_attr(cib_node, name, now);
-
         free(name);
         free(now);
     }
 }
 
 static void
 create_node_entry(cib_t * cib_conn, const char *node)
 {
     int rc = pcmk_ok;
     int max = strlen(new_node_template) + strlen(node) + 1;
     char *xpath = NULL;
 
     xpath = calloc(1, max);
 
     snprintf(xpath, max, new_node_template, node);
     rc = cib_conn->cmds->query(cib_conn, xpath, NULL, cib_xpath | cib_sync_call | cib_scope_local);
 
     if (rc == -ENXIO) {
         xmlNode *cib_object = create_xml_node(NULL, XML_CIB_TAG_NODE);
 
         /* Using node uname as uuid ala corosync/openais */
         crm_xml_add(cib_object, XML_ATTR_ID, node);
         crm_xml_add(cib_object, XML_ATTR_UNAME, node);
         cib_conn->cmds->create(cib_conn, XML_CIB_TAG_NODES, cib_object,
                                cib_sync_call | cib_scope_local);
         /* Not bothering with subsequent query to see if it exists,
            we'll bomb out later in the call to query_node_uuid()... */
 
         free_xml(cib_object);
     }
 
     free(xpath);
 }
 
 static lrmd_event_data_t *
 create_op(xmlNode * cib_resource, const char *task, int interval, int outcome)
 {
     lrmd_event_data_t *op = NULL;
     xmlNode *xop = NULL;
 
     op = calloc(1, sizeof(lrmd_event_data_t));
 
     op->rsc_id = strdup(ID(cib_resource));
     op->interval = interval;
     op->op_type = strdup(task);
 
     op->rc = outcome;
     op->op_status = 0;
     op->params = NULL;          /* TODO: Fill me in */
     op->t_run = time(NULL);
     op->t_rcchange = op->t_run;
 
     op->call_id = 0;
     for (xop = __xml_first_child(cib_resource); xop != NULL; xop = __xml_next(xop)) {
         int tmp = 0;
 
         crm_element_value_int(xop, XML_LRM_ATTR_CALLID, &tmp);
         if (tmp > op->call_id) {
             op->call_id = tmp;
         }
     }
     op->call_id++;
 
     return op;
 }
 
 static xmlNode *
 inject_op(xmlNode * cib_resource, lrmd_event_data_t * op, int target_rc)
 {
     return create_operation_update(cib_resource, op, CRM_FEATURE_SET, target_rc, NULL, crm_system_name,
                                    LOG_DEBUG_2);
 }
 
 static xmlNode *
 inject_node_state(cib_t * cib_conn, const char *node, const char *uuid)
 {
     int rc = pcmk_ok;
     int max = strlen(rsc_template) + strlen(node) + 1;
     char *xpath = NULL;
     xmlNode *cib_object = NULL;
 
     xpath = calloc(1, max);
 
     if (bringing_nodes_online) {
         create_node_entry(cib_conn, node);
     }
 
     snprintf(xpath, max, node_template, node);
     rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
                                cib_xpath | cib_sync_call | cib_scope_local);
 
     if (cib_object && ID(cib_object) == NULL) {
         crm_err("Detected multiple node_state entries for xpath=%s, bailing", xpath);
         crm_log_xml_warn(cib_object, "Duplicates");
         crm_exit(ENOTUNIQ);
     }
 
     if (rc == -ENXIO) {
         char *found_uuid = NULL;
 
         if (uuid == NULL) {
             query_node_uuid(cib_conn, node, &found_uuid, NULL);
         } else {
             found_uuid = strdup(uuid);
         }
 
         cib_object = create_xml_node(NULL, XML_CIB_TAG_STATE);
         crm_xml_add(cib_object, XML_ATTR_UUID, found_uuid);
         crm_xml_add(cib_object, XML_ATTR_UNAME, node);
         cib_conn->cmds->create(cib_conn, XML_CIB_TAG_STATUS, cib_object,
                                cib_sync_call | cib_scope_local);
         free_xml(cib_object);
         free(found_uuid);
 
         rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
                                    cib_xpath | cib_sync_call | cib_scope_local);
         crm_trace("injecting node state for %s. rc is %d", node, rc);
     }
 
     free(xpath);
     CRM_ASSERT(rc == pcmk_ok);
     return cib_object;
 }
 
 static xmlNode *
 modify_node(cib_t * cib_conn, char *node, gboolean up)
 {
     xmlNode *cib_node = inject_node_state(cib_conn, node, NULL);
 
     if (up) {
         crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_YES);
         crm_xml_add(cib_node, XML_NODE_IS_PEER, ONLINESTATUS);
         crm_xml_add(cib_node, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_MEMBER);
         crm_xml_add(cib_node, XML_NODE_EXPECTED, CRMD_JOINSTATE_MEMBER);
 
     } else {
         crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO);
         crm_xml_add(cib_node, XML_NODE_IS_PEER, OFFLINESTATUS);
         crm_xml_add(cib_node, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_DOWN);
         crm_xml_add(cib_node, XML_NODE_EXPECTED, CRMD_JOINSTATE_DOWN);
     }
 
     crm_xml_add(cib_node, XML_ATTR_ORIGIN, crm_system_name);
     return cib_node;
 }
 
 static xmlNode *
 find_resource_xml(xmlNode * cib_node, const char *resource)
 {
     char *xpath = NULL;
     xmlNode *match = NULL;
     const char *node = crm_element_value(cib_node, XML_ATTR_UNAME);
     int max = strlen(rsc_template) + strlen(resource) + strlen(node) + 1;
 
     xpath = calloc(1, max);
 
     snprintf(xpath, max, rsc_template, node, resource);
     match = get_xpath_object(xpath, cib_node, LOG_DEBUG_2);
 
     free(xpath);
     return match;
 }
 
 
 static xmlNode *
 inject_resource(xmlNode * cib_node, const char *resource, const char *rclass, const char *rtype,
                 const char *rprovider)
 {
     xmlNode *lrm = NULL;
     xmlNode *container = NULL;
     xmlNode *cib_resource = NULL;
     char *xpath = NULL;
 
     cib_resource = find_resource_xml(cib_node, resource);
     if (cib_resource != NULL) {
         return cib_resource;
     }
 
     /* One day, add query for class, provider, type */
 
     if (rclass == NULL || rtype == NULL) {
         fprintf(stderr, "Resource %s not found in the status section of %s."
                 "  Please supply the class and type to continue\n", resource, ID(cib_node));
         return NULL;
 
     } else if (safe_str_neq(rclass, "ocf")
                && safe_str_neq(rclass, "stonith")
                && safe_str_neq(rclass, "heartbeat")
                && safe_str_neq(rclass, "service")
                && safe_str_neq(rclass, "upstart")
                && safe_str_neq(rclass, "systemd")
                && safe_str_neq(rclass, "lsb")) {
         fprintf(stderr, "Invalid class for %s: %s\n", resource, rclass);
         return NULL;
 
     } else if (safe_str_eq(rclass, "ocf") && rprovider == NULL) {
         fprintf(stderr, "Please specify the provider for resource %s\n", resource);
         return NULL;
     }
 
     xpath = (char *)xmlGetNodePath(cib_node);
     crm_info("Injecting new resource %s into %s '%s'", resource, xpath, ID(cib_node));
     free(xpath);
 
     lrm = first_named_child(cib_node, XML_CIB_TAG_LRM);
     if (lrm == NULL) {
         const char *node_uuid = ID(cib_node);
 
         lrm = create_xml_node(cib_node, XML_CIB_TAG_LRM);
         crm_xml_add(lrm, XML_ATTR_ID, node_uuid);
     }
 
     container = first_named_child(lrm, XML_LRM_TAG_RESOURCES);
     if (container == NULL) {
         container = create_xml_node(lrm, XML_LRM_TAG_RESOURCES);
     }
 
     cib_resource = create_xml_node(container, XML_LRM_TAG_RESOURCE);
     crm_xml_add(cib_resource, XML_ATTR_ID, resource);
 
     crm_xml_add(cib_resource, XML_AGENT_ATTR_CLASS, rclass);
     crm_xml_add(cib_resource, XML_AGENT_ATTR_PROVIDER, rprovider);
     crm_xml_add(cib_resource, XML_ATTR_TYPE, rtype);
 
     return cib_resource;
 }
 
 static int
 find_ticket_state(cib_t * the_cib, const char *ticket_id, xmlNode ** ticket_state_xml)
 {
     int offset = 0;
     static int xpath_max = 1024;
     int rc = pcmk_ok;
     xmlNode *xml_search = NULL;
 
     char *xpath_string = NULL;
 
     CRM_ASSERT(ticket_state_xml != NULL);
     *ticket_state_xml = NULL;
 
     xpath_string = calloc(1, xpath_max);
     offset += snprintf(xpath_string + offset, xpath_max - offset, "%s", "/cib/status/tickets");
 
     if (ticket_id) {
         offset += snprintf(xpath_string + offset, xpath_max - offset, "/%s[@id=\"%s\"]",
                            XML_CIB_TAG_TICKET_STATE, ticket_id);
     }
     CRM_LOG_ASSERT(offset > 0);
     rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
                               cib_sync_call | cib_scope_local | cib_xpath);
 
     if (rc != pcmk_ok) {
         goto bail;
     }
 
     crm_log_xml_debug(xml_search, "Match");
     if (xml_has_children(xml_search)) {
         if (ticket_id) {
             fprintf(stdout, "Multiple ticket_states match ticket_id=%s\n", ticket_id);
         }
         *ticket_state_xml = xml_search;
     } else {
         *ticket_state_xml = xml_search;
     }
 
   bail:
     free(xpath_string);
     return rc;
 }
 
 static int
 set_ticket_state_attr(const char *ticket_id, const char *attr_name,
                       const char *attr_value, cib_t * cib, int cib_options)
 {
     int rc = pcmk_ok;
     xmlNode *xml_top = NULL;
     xmlNode *ticket_state_xml = NULL;
 
     rc = find_ticket_state(cib, ticket_id, &ticket_state_xml);
     if (rc == pcmk_ok) {
         crm_debug("Found a match state for ticket: id=%s", ticket_id);
         xml_top = ticket_state_xml;
 
     } else if (rc != -ENXIO) {
         return rc;
 
     } else {
         xmlNode *xml_obj = NULL;
 
         xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
         xml_obj = create_xml_node(xml_top, XML_CIB_TAG_TICKETS);
         ticket_state_xml = create_xml_node(xml_obj, XML_CIB_TAG_TICKET_STATE);
         crm_xml_add(ticket_state_xml, XML_ATTR_ID, ticket_id);
     }
 
     crm_xml_add(ticket_state_xml, attr_name, attr_value);
 
     crm_log_xml_debug(xml_top, "Update");
 
     rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, xml_top, cib_options);
 
     free_xml(xml_top);
 
     return rc;
 }
 
 void
 modify_configuration(pe_working_set_t * data_set, cib_t *cib,
                      const char *quorum, const char *watchdog, GListPtr node_up, GListPtr node_down, GListPtr node_fail,
                      GListPtr op_inject, GListPtr ticket_grant, GListPtr ticket_revoke,
                      GListPtr ticket_standby, GListPtr ticket_activate)
 {
     int rc = pcmk_ok;
     GListPtr gIter = NULL;
 
     xmlNode *cib_op = NULL;
     xmlNode *cib_node = NULL;
     xmlNode *cib_resource = NULL;
 
     lrmd_event_data_t *op = NULL;
 
     if (quorum) {
         xmlNode *top = create_xml_node(NULL, XML_TAG_CIB);
 
         quiet_log(" + Setting quorum: %s\n", quorum);
         /* crm_xml_add(top, XML_ATTR_DC_UUID, dc_uuid);      */
         crm_xml_add(top, XML_ATTR_HAVE_QUORUM, quorum);
 
         rc = cib->cmds->modify(cib, NULL, top, cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     if (watchdog) {
         quiet_log(" + Setting watchdog: %s\n", watchdog);
 
         rc = update_attr_delegate(cib, cib_sync_call | cib_scope_local,
                              XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
                              XML_ATTR_HAVE_WATCHDOG, watchdog, FALSE, NULL, NULL);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = node_up; gIter != NULL; gIter = gIter->next) {
         char *node = (char *)gIter->data;
 
         quiet_log(" + Bringing node %s online\n", node);
         cib_node = modify_node(cib, node, TRUE);
         CRM_ASSERT(cib_node != NULL);
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                                       cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
         free_xml(cib_node);
     }
 
     for (gIter = node_down; gIter != NULL; gIter = gIter->next) {
         char xpath[STATUS_PATH_MAX];
         char *node = (char *)gIter->data;
 
         quiet_log(" + Taking node %s offline\n", node);
         cib_node = modify_node(cib, node, FALSE);
         CRM_ASSERT(cib_node != NULL);
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                                       cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
         free_xml(cib_node);
 
         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", node, XML_CIB_TAG_LRM);
         cib->cmds->delete(cib, xpath, NULL,
                                       cib_xpath | cib_sync_call | cib_scope_local);
 
         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", node,
                  XML_TAG_TRANSIENT_NODEATTRS);
         cib->cmds->delete(cib, xpath, NULL,
                                       cib_xpath | cib_sync_call | cib_scope_local);
 
     }
 
     for (gIter = node_fail; gIter != NULL; gIter = gIter->next) {
         char *node = (char *)gIter->data;
 
         quiet_log(" + Failing node %s\n", node);
         cib_node = modify_node(cib, node, TRUE);
         crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO);
         CRM_ASSERT(cib_node != NULL);
 
         rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                                       cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
         free_xml(cib_node);
     }
 
     for (gIter = ticket_grant; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         quiet_log(" + Granting ticket %s\n", ticket_id);
         rc = set_ticket_state_attr(ticket_id, "granted", "true",
                                    cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = ticket_revoke; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         quiet_log(" + Revoking ticket %s\n", ticket_id);
         rc = set_ticket_state_attr(ticket_id, "granted", "false",
                                    cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = ticket_standby; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         quiet_log(" + Making ticket %s standby\n", ticket_id);
         rc = set_ticket_state_attr(ticket_id, "standby", "true",
                                    cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = ticket_activate; gIter != NULL; gIter = gIter->next) {
         char *ticket_id = (char *)gIter->data;
 
         quiet_log(" + Activating ticket %s\n", ticket_id);
         rc = set_ticket_state_attr(ticket_id, "standby", "false",
                                    cib, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
     }
 
     for (gIter = op_inject; gIter != NULL; gIter = gIter->next) {
         char *spec = (char *)gIter->data;
 
         int rc = 0;
         int outcome = 0;
         int interval = 0;
 
         char *key = NULL;
         char *node = NULL;
         char *task = NULL;
         char *resource = NULL;
 
         const char *rtype = NULL;
         const char *rclass = NULL;
         const char *rprovider = NULL;
 
         resource_t *rsc = NULL;
 
         quiet_log(" + Injecting %s into the configuration\n", spec);
 
         key = calloc(1, strlen(spec) + 1);
         node = calloc(1, strlen(spec) + 1);
         rc = sscanf(spec, "%[^@]@%[^=]=%d", key, node, &outcome);
         if (rc != 3) {
             fprintf(stderr, "Invalid operation spec: %s.  Only found %d fields\n", spec, rc);
             free(key);
             free(node);
             continue;
         }
 
         parse_op_key(key, &resource, &task, &interval);
 
         rsc = pe_find_resource(data_set->resources, resource);
         if (rsc == NULL) {
             fprintf(stderr, " - Invalid resource name: %s\n", resource);
         } else {
             rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
             rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
             rprovider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
 
             cib_node = inject_node_state(cib, node, NULL);
             CRM_ASSERT(cib_node != NULL);
 
             update_failcounts(cib_node, resource, interval, outcome);
 
             cib_resource = inject_resource(cib_node, resource, rclass, rtype, rprovider);
             CRM_ASSERT(cib_resource != NULL);
 
             op = create_op(cib_resource, task, interval, outcome);
             CRM_ASSERT(op != NULL);
 
             cib_op = inject_op(cib_resource, op, 0);
             CRM_ASSERT(cib_op != NULL);
             lrmd_free_event(op);
 
             rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
                                           cib_sync_call | cib_scope_local);
             CRM_ASSERT(rc == pcmk_ok);
         }
         free(task);
         free(node);
         free(key);
     }
 }
 
 static gboolean
 exec_pseudo_action(crm_graph_t * graph, crm_action_t * action)
 {
     const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 
     action->confirmed = TRUE;
 
     quiet_log(" * Pseudo action:   %s%s%s\n", task, node ? " on " : "", node ? node : "");
     update_graph(graph, action);
     return TRUE;
 }
 
 static gboolean
 exec_rsc_action(crm_graph_t * graph, crm_action_t * action)
 {
     int rc = 0;
     GListPtr gIter = NULL;
     lrmd_event_data_t *op = NULL;
     int target_outcome = 0;
     gboolean uname_is_uuid = FALSE;
 
     const char *rtype = NULL;
     const char *rclass = NULL;
     const char *resource = NULL;
     const char *rprovider = NULL;
     const char *operation = crm_element_value(action->xml, "operation");
     const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);
 
     xmlNode *cib_node = NULL;
     xmlNode *cib_resource = NULL;
     xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
 
     char *node = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
     char *uuid = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET_UUID);
     const char *router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 
     if (safe_str_eq(operation, CRM_OP_PROBED)
         || safe_str_eq(operation, CRM_OP_REPROBE)) {
         crm_info("Skipping %s op for %s", operation, node);
         goto done;
     }
 
     if (action_rsc == NULL) {
         crm_log_xml_err(action->xml, "Bad");
         free(node); free(uuid);
         return FALSE;
     }
 
     /* Look for the preferred name
      * If not found, try the expected 'local' name
      * If not found use the preferred name anyway
      */
     resource = crm_element_value(action_rsc, XML_ATTR_ID);
     if (pe_find_resource(fake_resource_list, resource) == NULL) {
         const char *longname = crm_element_value(action_rsc, XML_ATTR_ID_LONG);
 
         if (pe_find_resource(fake_resource_list, longname)) {
             resource = longname;
         }
     }
 
     if (safe_str_eq(operation, "delete")) {
         quiet_log(" * Resource action: %-15s delete on %s\n", resource, node);
         goto done;
     }
 
     rclass = crm_element_value(action_rsc, XML_AGENT_ATTR_CLASS);
     rtype = crm_element_value(action_rsc, XML_ATTR_TYPE);
     rprovider = crm_element_value(action_rsc, XML_AGENT_ATTR_PROVIDER);
 
     if (target_rc_s != NULL) {
         target_outcome = crm_parse_int(target_rc_s, "0");
     }
 
     CRM_ASSERT(fake_cib->cmds->query(fake_cib, NULL, NULL, cib_sync_call | cib_scope_local) ==
                pcmk_ok);
 
     if (router_node) {
         uname_is_uuid = TRUE;
     }
 
     cib_node = inject_node_state(fake_cib, node, uname_is_uuid ? node : uuid);
     CRM_ASSERT(cib_node != NULL);
 
     cib_resource = inject_resource(cib_node, resource, rclass, rtype, rprovider);
     CRM_ASSERT(cib_resource != NULL);
 
     op = convert_graph_action(cib_resource, action, 0, target_outcome);
     if (op->interval) {
         quiet_log(" * Resource action: %-15s %s=%d on %s\n", resource, op->op_type, op->interval,
                   node);
     } else {
         quiet_log(" * Resource action: %-15s %s on %s\n", resource, op->op_type, node);
     }
 
     for (gIter = fake_op_fail_list; gIter != NULL; gIter = gIter->next) {
         char *spec = (char *)gIter->data;
         char *key = NULL;
 
         key = calloc(1, 1 + strlen(spec));
         snprintf(key, strlen(spec), "%s_%s_%d@%s=", resource, op->op_type, op->interval, node);
 
         if (strncasecmp(key, spec, strlen(key)) == 0) {
             sscanf(spec, "%*[^=]=%d", (int *)&op->rc);
 
             action->failed = TRUE;
             graph->abort_priority = INFINITY;
             printf("\tPretending action %d failed with rc=%d\n", action->id, op->rc);
             update_failcounts(cib_node, resource, op->interval, op->rc);
             free(key);
             break;
         }
         free(key);
     }
 
     inject_op(cib_resource, op, target_outcome);
     lrmd_free_event(op);
 
     rc = fake_cib->cmds->modify(fake_cib, XML_CIB_TAG_STATUS, cib_node,
                                   cib_sync_call | cib_scope_local);
     CRM_ASSERT(rc == pcmk_ok);
 
   done:
     free(node); free(uuid);
     free_xml(cib_node);
     action->confirmed = TRUE;
     update_graph(graph, action);
     return TRUE;
 }
 
 static gboolean
 exec_crmd_action(crm_graph_t * graph, crm_action_t * action)
 {
     const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
     xmlNode *rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
 
     action->confirmed = TRUE;
 
     if(rsc) {
         quiet_log(" * Cluster action:  %s for %s on %s\n", task, ID(rsc), node);
     } else {
         quiet_log(" * Cluster action:  %s on %s\n", task, node);
     }
     update_graph(graph, action);
     return TRUE;
 }
 
 static gboolean
 exec_stonith_action(crm_graph_t * graph, crm_action_t * action)
 {
     const char *op = crm_meta_value(action->params, "stonith_action");
     char *target = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
 
     quiet_log(" * Fencing %s (%s)\n", target, op);
     if(safe_str_neq(op, "on")) {
         int rc = 0;
         char xpath[STATUS_PATH_MAX];
         xmlNode *cib_node = modify_node(fake_cib, target, FALSE);
 
         crm_xml_add(cib_node, XML_ATTR_ORIGIN, __FUNCTION__);
         CRM_ASSERT(cib_node != NULL);
 
         rc = fake_cib->cmds->replace(fake_cib, XML_CIB_TAG_STATUS, cib_node,
                                    cib_sync_call | cib_scope_local);
         CRM_ASSERT(rc == pcmk_ok);
 
         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target, XML_CIB_TAG_LRM);
         fake_cib->cmds->delete(fake_cib, xpath, NULL,
                                       cib_xpath | cib_sync_call | cib_scope_local);
 
         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target,
                  XML_TAG_TRANSIENT_NODEATTRS);
         fake_cib->cmds->delete(fake_cib, xpath, NULL,
                                       cib_xpath | cib_sync_call | cib_scope_local);
 
         free_xml(cib_node);
     }
 
     action->confirmed = TRUE;
     update_graph(graph, action);
     free(target);
     return TRUE;
 }
 
 int
 run_simulation(pe_working_set_t * data_set, cib_t *cib, GListPtr op_fail_list, bool quiet)
 {
     crm_graph_t *transition = NULL;
     enum transition_status graph_rc = -1;
 
     crm_graph_functions_t exec_fns = {
         exec_pseudo_action,
         exec_rsc_action,
         exec_crmd_action,
         exec_stonith_action,
     };
 
     fake_cib = cib;
     fake_quiet = quiet;
     fake_op_fail_list = op_fail_list;
 
     quiet_log("\nExecuting cluster transition:\n");
 
     set_graph_functions(&exec_fns);
     transition = unpack_graph(data_set->graph, crm_system_name);
     print_graph(LOG_DEBUG, transition);
 
     fake_resource_list = data_set->resources;
     do {
         graph_rc = run_graph(transition);
 
     } while (graph_rc == transition_active);
     fake_resource_list = NULL;
 
     if (graph_rc != transition_complete) {
         fprintf(stdout, "Transition failed: %s\n", transition_status(graph_rc));
         print_graph(LOG_ERR, transition);
     }
     destroy_graph(transition);
     if (graph_rc != transition_complete) {
         fprintf(stdout, "An invalid transition was produced\n");
     }
 
     if (quiet == FALSE) {
         xmlNode *cib_object = NULL;
         int rc = fake_cib->cmds->query(fake_cib, NULL, &cib_object, cib_sync_call | cib_scope_local);
 
         CRM_ASSERT(rc == pcmk_ok);
         cleanup_alloc_calculations(data_set);
         data_set->input = cib_object;
     }
 
     if (graph_rc != transition_complete) {
         return graph_rc;
     }
     return 0;
 }