diff --git a/lib/common/output_text.c b/lib/common/output_text.c
index 4408425620..3454a8cbdb 100644
--- a/lib/common/output_text.c
+++ b/lib/common/output_text.c
@@ -1,433 +1,433 @@
 /*
  * Copyright 2019-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdarg.h>
 #include <stdlib.h>
 #include <glib.h>
 #include <termios.h>
 
 static gboolean fancy = FALSE;
 
 GOptionEntry pcmk__text_output_entries[] = {
     { "text-fancy", 0, 0, G_OPTION_ARG_NONE, &fancy,
       "Use more highly formatted output (requires --output-as=text)",
       NULL },
 
     { NULL }
 };
 
 typedef struct text_list_data_s {
     unsigned int len;
     char *singular_noun;
     char *plural_noun;
 } text_list_data_t;
 
 typedef struct private_data_s {
     GQueue *parent_q;
 } private_data_t;
 
 static void
 text_free_priv(pcmk__output_t *out) {
     private_data_t *priv = out->priv;
 
     if (priv == NULL) {
         return;
     }
 
     g_queue_free(priv->parent_q);
     free(priv);
     out->priv = NULL;
 }
 
 static bool
 text_init(pcmk__output_t *out) {
     private_data_t *priv = NULL;
 
     /* If text_init was previously called on this output struct, just return. */
     if (out->priv != NULL) {
         return true;
     } else {
         out->priv = calloc(1, sizeof(private_data_t));
         if (out->priv == NULL) {
             return false;
         }
 
         priv = out->priv;
     }
 
     priv->parent_q = g_queue_new();
     return true;
 }
 
 static void
 text_finish(pcmk__output_t *out, crm_exit_t exit_status, bool print, void **copy_dest) {
     fflush(out->dest);
 }
 
 static void
 text_reset(pcmk__output_t *out) {
     CRM_ASSERT(out != NULL);
 
     if (out->dest != stdout) {
         out->dest = freopen(NULL, "w", out->dest);
     }
 
     CRM_ASSERT(out->dest != NULL);
 
     text_free_priv(out);
     text_init(out);
 }
 
 static void
 text_subprocess_output(pcmk__output_t *out, int exit_status,
                        const char *proc_stdout, const char *proc_stderr) {
     CRM_ASSERT(out != NULL);
 
     if (proc_stdout != NULL) {
         fprintf(out->dest, "%s\n", proc_stdout);
     }
 
     if (proc_stderr != NULL) {
         fprintf(out->dest, "%s\n", proc_stderr);
     }
 }
 
 static void
 text_version(pcmk__output_t *out, bool extended) {
     CRM_ASSERT(out != NULL);
 
     if (extended) {
         fprintf(out->dest, "Pacemaker %s (Build: %s): %s\n", PACEMAKER_VERSION, BUILD_VERSION, CRM_FEATURES);
     } else {
         fprintf(out->dest, "Pacemaker %s\n", PACEMAKER_VERSION);
         fprintf(out->dest, "Written by Andrew Beekhof\n");
     }
 }
 
 G_GNUC_PRINTF(2, 3)
 static void
 text_err(pcmk__output_t *out, const char *format, ...) {
     va_list ap;
     int len = 0;
 
     CRM_ASSERT(out != NULL);
 
     va_start(ap, format);
 
     /* Informational output does not get indented, to separate it from other
      * potentially indented list output.
      */
     len = vfprintf(stderr, format, ap);
     CRM_ASSERT(len >= 0);
     va_end(ap);
 
     /* Add a newline. */
     fprintf(stderr, "\n");
 }
 
 G_GNUC_PRINTF(2, 3)
 static int
 text_info(pcmk__output_t *out, const char *format, ...) {
     va_list ap;
     int len = 0;
 
     CRM_ASSERT(out != NULL);
 
     if (out->is_quiet(out)) {
         return pcmk_rc_no_output;
     }
 
     va_start(ap, format);
 
     /* Informational output does not get indented, to separate it from other
      * potentially indented list output.
      */
     len = vfprintf(out->dest, format, ap);
     CRM_ASSERT(len >= 0);
     va_end(ap);
 
     /* Add a newline. */
     fprintf(out->dest, "\n");
     return pcmk_rc_ok;
 }
 
 static void
 text_output_xml(pcmk__output_t *out, const char *name, const char *buf) {
     CRM_ASSERT(out != NULL);
     pcmk__indented_printf(out, "%s", buf);
 }
 
 G_GNUC_PRINTF(4, 5)
 static void
 text_begin_list(pcmk__output_t *out, const char *singular_noun, const char *plural_noun,
                 const char *format, ...) {
     private_data_t *priv = NULL;
     text_list_data_t *new_list = NULL;
     va_list ap;
 
     CRM_ASSERT(out != NULL && out->priv != NULL);
     priv = out->priv;
 
     va_start(ap, format);
 
     if (fancy && format) {
         pcmk__indented_vprintf(out, format, ap);
         fprintf(out->dest, ":\n");
     }
 
     va_end(ap);
 
     new_list = calloc(1, sizeof(text_list_data_t));
     new_list->len = 0;
     new_list->singular_noun = singular_noun == NULL ? NULL : strdup(singular_noun);
     new_list->plural_noun = plural_noun == NULL ? NULL : strdup(plural_noun);
 
     g_queue_push_tail(priv->parent_q, new_list);
 }
 
 G_GNUC_PRINTF(3, 4)
 static void
 text_list_item(pcmk__output_t *out, const char *id, const char *format, ...) {
     va_list ap;
 
     CRM_ASSERT(out != NULL);
 
     va_start(ap, format);
 
     if (fancy) {
         if (id != NULL) {
             /* Not really a good way to do this all in one call, so make it two.
              * The first handles the indentation and list styling.  The second
              * just prints right after that one.
              */
             pcmk__indented_printf(out, "%s: ", id);
             vfprintf(out->dest, format, ap);
         } else {
             pcmk__indented_vprintf(out, format, ap);
         }
     } else {
         pcmk__indented_vprintf(out, format, ap);
     }
 
     fputc('\n', out->dest);
     fflush(out->dest);
     va_end(ap);
 
     out->increment_list(out);
 }
 
 static void
 text_increment_list(pcmk__output_t *out) {
     private_data_t *priv = NULL;
     gpointer tail;
 
     CRM_ASSERT(out != NULL && out->priv != NULL);
     priv = out->priv;
 
     tail = g_queue_peek_tail(priv->parent_q);
     CRM_ASSERT(tail != NULL);
     ((text_list_data_t *) tail)->len++;
 }
 
 static void
 text_end_list(pcmk__output_t *out) {
     private_data_t *priv = NULL;
     text_list_data_t *node = NULL;
 
     CRM_ASSERT(out != NULL && out->priv != NULL);
     priv = out->priv;
 
     node = g_queue_pop_tail(priv->parent_q);
 
     if (node->singular_noun != NULL && node->plural_noun != NULL) {
         if (node->len == 1) {
             pcmk__indented_printf(out, "%d %s found\n", node->len, node->singular_noun);
         } else {
             pcmk__indented_printf(out, "%d %s found\n", node->len, node->plural_noun);
         }
     }
 
     free(node);
 }
 
 static bool
 text_is_quiet(pcmk__output_t *out) {
     CRM_ASSERT(out != NULL);
     return out->quiet;
 }
 
 static void
 text_spacer(pcmk__output_t *out) {
     CRM_ASSERT(out != NULL);
     fprintf(out->dest, "\n");
 }
 
 static void
 text_progress(pcmk__output_t *out, bool end) {
     CRM_ASSERT(out != NULL);
 
     if (out->dest == stdout) {
         fprintf(out->dest, ".");
 
         if (end) {
             fprintf(out->dest, "\n");
         }
     }
 }
 
 pcmk__output_t *
 pcmk__mk_text_output(char **argv) {
     pcmk__output_t *retval = calloc(1, sizeof(pcmk__output_t));
 
     if (retval == NULL) {
         return NULL;
     }
 
     retval->fmt_name = "text";
     retval->request = argv == NULL ? NULL : g_strjoinv(" ", argv);
 
     retval->init = text_init;
     retval->free_priv = text_free_priv;
     retval->finish = text_finish;
     retval->reset = text_reset;
 
     retval->register_message = pcmk__register_message;
     retval->message = pcmk__call_message;
 
     retval->subprocess_output = text_subprocess_output;
     retval->version = text_version;
     retval->info = text_info;
     retval->err = text_err;
     retval->output_xml = text_output_xml;
 
     retval->begin_list = text_begin_list;
     retval->list_item = text_list_item;
     retval->increment_list = text_increment_list;
     retval->end_list = text_end_list;
 
     retval->is_quiet = text_is_quiet;
     retval->spacer = text_spacer;
     retval->progress = text_progress;
     retval->prompt = pcmk__text_prompt;
 
     return retval;
 }
 
 G_GNUC_PRINTF(2, 0)
 void
 pcmk__formatted_vprintf(pcmk__output_t *out, const char *format, va_list args) {
     int len = 0;
 
     CRM_ASSERT(out != NULL);
 
     len = vfprintf(out->dest, format, args);
     CRM_ASSERT(len >= 0);
 }
 
 G_GNUC_PRINTF(2, 3)
 void
 pcmk__formatted_printf(pcmk__output_t *out, const char *format, ...) {
     va_list ap;
 
     CRM_ASSERT(out != NULL);
 
     va_start(ap, format);
     pcmk__formatted_vprintf(out, format, ap);
     va_end(ap);
 }
 
 G_GNUC_PRINTF(2, 0)
 void
 pcmk__indented_vprintf(pcmk__output_t *out, const char *format, va_list args) {
     CRM_ASSERT(out != NULL);
 
     if (!pcmk__str_eq(out->fmt_name, "text", pcmk__str_none)) {
         return;
     }
 
     if (fancy) {
         int level = 0;
         private_data_t *priv = out->priv;
 
         CRM_ASSERT(priv != NULL);
 
         level = g_queue_get_length(priv->parent_q);
 
         for (int i = 0; i < level; i++) {
             fprintf(out->dest, "  ");
         }
 
         if (level > 0) {
             fprintf(out->dest, "* ");
         }
     }
 
     pcmk__formatted_vprintf(out, format, args);
 }
 
 G_GNUC_PRINTF(2, 3)
 void
 pcmk__indented_printf(pcmk__output_t *out, const char *format, ...) {
     va_list ap;
 
     CRM_ASSERT(out != NULL);
 
     va_start(ap, format);
     pcmk__indented_vprintf(out, format, ap);
     va_end(ap);
 }
 
 void
 pcmk__text_prompt(const char *prompt, bool echo, char **dest)
 {
     int rc = 0;
     struct termios settings;
     tcflag_t orig_c_lflag = 0;
 
     CRM_ASSERT(prompt != NULL);
     CRM_ASSERT(dest != NULL);
 
     if (!echo) {
         rc = tcgetattr(0, &settings);
         if (rc == 0) {
             orig_c_lflag = settings.c_lflag;
             settings.c_lflag &= ~ECHO;
             rc = tcsetattr(0, TCSANOW, &settings);
         }
     }
 
     if (rc == 0) {
         fprintf(stderr, "%s: ", prompt);
 
         if (*dest != NULL) {
             free(*dest);
             *dest = NULL;
         }
 
 #if SSCANF_HAS_M
         rc = scanf("%ms", dest);
 #else
         *dest = calloc(1, 1024);
         rc = scanf("%1023s", *dest);
 #endif
         fprintf(stderr, "\n");
     }
 
     if (rc < 1) {
         free(*dest);
         *dest = NULL;
     }
 
     if (orig_c_lflag != 0) {
         settings.c_lflag = orig_c_lflag;
-        rc = tcsetattr(0, TCSANOW, &settings);
+        /* rc = */ tcsetattr(0, TCSANOW, &settings);
     }
 }
diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c
index 5a1d2bd2f4..b6ea12022a 100644
--- a/lib/fencing/st_client.c
+++ b/lib/fencing/st_client.c
@@ -1,2708 +1,2708 @@
 /*
  * Copyright 2004-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU Lesser General Public License
  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdbool.h>
 #include <string.h>
 #include <ctype.h>
 #include <libgen.h>
 #include <inttypes.h>
 
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 
 #include <glib.h>
 
 #include <crm/crm.h>
 #include <crm/stonith-ng.h>
 #include <crm/fencing/internal.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/xml_internal.h>
 
 #include <crm/common/mainloop.h>
 
 CRM_TRACE_INIT_DATA(stonith);
 
 struct stonith_action_s {
     /*! user defined data */
     char *agent;
     char *action;
     char *victim;
     GHashTable *args;
     int timeout;
     int async;
     void *userdata;
     void (*done_cb) (GPid pid, gint status, const char *output, gpointer user_data);
     void (*fork_cb) (GPid pid, gpointer user_data);
 
     svc_action_t *svc_action;
 
     /*! internal timing information */
     time_t initial_start_time;
     int tries;
     int remaining_timeout;
     int max_retries;
 
     /* device output data */
     GPid pid;
     int rc;
     char *output;
     char *error;
 };
 
 typedef struct stonith_private_s {
     char *token;
     crm_ipc_t *ipc;
     mainloop_io_t *source;
     GHashTable *stonith_op_callback_table;
     GList *notify_list;
     int notify_refcnt;
     bool notify_deletes;
 
     void (*op_callback) (stonith_t * st, stonith_callback_data_t * data);
 
 } stonith_private_t;
 
 typedef struct stonith_notify_client_s {
     const char *event;
     const char *obj_id;         /* implement one day */
     const char *obj_type;       /* implement one day */
     void (*notify) (stonith_t * st, stonith_event_t * e);
     bool delete;
 
 } stonith_notify_client_t;
 
 typedef struct stonith_callback_client_s {
     void (*callback) (stonith_t * st, stonith_callback_data_t * data);
     const char *id;
     void *user_data;
     gboolean only_success;
     gboolean allow_timeout_updates;
     struct timer_rec_s *timer;
 
 } stonith_callback_client_t;
 
 struct notify_blob_s {
     stonith_t *stonith;
     xmlNode *xml;
 };
 
 struct timer_rec_s {
     int call_id;
     int timeout;
     guint ref;
     stonith_t *stonith;
 };
 
 typedef int (*stonith_op_t) (const char *, int, const char *, xmlNode *,
                              xmlNode *, xmlNode *, xmlNode **, xmlNode **);
 
 bool stonith_dispatch(stonith_t * st);
 xmlNode *stonith_create_op(int call_id, const char *token, const char *op, xmlNode * data,
                            int call_options);
 static int stonith_send_command(stonith_t *stonith, const char *op,
                                 xmlNode *data, xmlNode **output_data,
                                 int call_options, int timeout);
 
 static void stonith_connection_destroy(gpointer user_data);
 static void stonith_send_notification(gpointer data, gpointer user_data);
 static int internal_stonith_action_execute(stonith_action_t * action);
 static void log_action(stonith_action_t *action, pid_t pid);
 
 /*!
  * \brief Get agent namespace by name
  *
  * \param[in] namespace_s  Name of namespace as string
  *
  * \return Namespace as enum value
  */
 enum stonith_namespace
 stonith_text2namespace(const char *namespace_s)
 {
-    if (pcmk__str_eq(namespace_s, "any", pcmk__str_null_matches)) {
+    if ((namespace_s == NULL) || !strcmp(namespace_s, "any")) {
         return st_namespace_any;
 
     } else if (!strcmp(namespace_s, "redhat")
                || !strcmp(namespace_s, "stonith-ng")) {
         return st_namespace_rhcs;
 
     } else if (!strcmp(namespace_s, "internal")) {
         return st_namespace_internal;
 
     } else if (!strcmp(namespace_s, "heartbeat")) {
         return st_namespace_lha;
     }
     return st_namespace_invalid;
 }
 
 /*!
  * \brief Get agent namespace name
  *
  * \param[in] namespace  Namespace as enum value
  *
  * \return Namespace name as string
  */
 const char *
 stonith_namespace2text(enum stonith_namespace st_namespace)
 {
     switch (st_namespace) {
         case st_namespace_any:      return "any";
         case st_namespace_rhcs:     return "stonith-ng";
         case st_namespace_internal: return "internal";
         case st_namespace_lha:      return "heartbeat";
         default:                    break;
     }
     return "unsupported";
 }
 
 /*!
  * \brief Determine namespace of a fence agent
  *
  * \param[in] agent        Fence agent type
  * \param[in] namespace_s  Name of agent namespace as string, if known
  *
  * \return Namespace of specified agent, as enum value
  */
 enum stonith_namespace
 stonith_get_namespace(const char *agent, const char *namespace_s)
 {
     if (pcmk__str_eq(namespace_s, "internal", pcmk__str_casei)) {
         return st_namespace_internal;
     }
 
     if (stonith__agent_is_rhcs(agent)) {
         return st_namespace_rhcs;
     }
 
 #if HAVE_STONITH_STONITH_H
     if (stonith__agent_is_lha(agent)) {
         return st_namespace_lha;
     }
 #endif
 
     crm_err("Unknown fence agent: %s", agent);
     return st_namespace_invalid;
 }
 
 static void
 log_action(stonith_action_t *action, pid_t pid)
 {
     if (action->output) {
         /* Logging the whole string confuses syslog when the string is xml */
         char *prefix = crm_strdup_printf("%s[%d] stdout:", action->agent, pid);
 
         crm_log_output(LOG_TRACE, prefix, action->output);
         free(prefix);
     }
 
     if (action->error) {
         /* Logging the whole string confuses syslog when the string is xml */
         char *prefix = crm_strdup_printf("%s[%d] stderr:", action->agent, pid);
 
         crm_log_output(LOG_WARNING, prefix, action->error);
         free(prefix);
     }
 }
 
 /* when cycling through the list we don't want to delete items
    so just mark them and when we know nobody is using the list
    loop over it to remove the marked items
  */
 static void
 foreach_notify_entry (stonith_private_t *private,
                 GFunc func,
                 gpointer user_data)
 {
     private->notify_refcnt++;
     g_list_foreach(private->notify_list, func, user_data);
     private->notify_refcnt--;
     if ((private->notify_refcnt == 0) &&
         private->notify_deletes) {
         GList *list_item = private->notify_list;
 
         private->notify_deletes = FALSE;
         while (list_item != NULL)
         {
             stonith_notify_client_t *list_client = list_item->data;
             GList *next = g_list_next(list_item);
 
             if (list_client->delete) {
                 free(list_client);
                 private->notify_list =
                     g_list_delete_link(private->notify_list, list_item);
             }
             list_item = next;
         }
     }
 }
 
 static void
 stonith_connection_destroy(gpointer user_data)
 {
     stonith_t *stonith = user_data;
     stonith_private_t *native = NULL;
     struct notify_blob_s blob;
 
     crm_trace("Sending destroyed notification");
     blob.stonith = stonith;
     blob.xml = create_xml_node(NULL, "notify");
 
     native = stonith->st_private;
     native->ipc = NULL;
     native->source = NULL;
 
     free(native->token); native->token = NULL;
     stonith->state = stonith_disconnected;
     crm_xml_add(blob.xml, F_TYPE, T_STONITH_NOTIFY);
     crm_xml_add(blob.xml, F_SUBTYPE, T_STONITH_NOTIFY_DISCONNECT);
 
     foreach_notify_entry(native, stonith_send_notification, &blob);
     free_xml(blob.xml);
 }
 
 xmlNode *
 create_device_registration_xml(const char *id, enum stonith_namespace namespace,
                                const char *agent, stonith_key_value_t *params,
                                const char *rsc_provides)
 {
     xmlNode *data = create_xml_node(NULL, F_STONITH_DEVICE);
     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
 
 #if HAVE_STONITH_STONITH_H
     if (namespace == st_namespace_any) {
         namespace = stonith_get_namespace(agent, NULL);
     }
     if (namespace == st_namespace_lha) {
         hash2field((gpointer) "plugin", (gpointer) agent, args);
         agent = "fence_legacy";
     }
 #endif
 
     crm_xml_add(data, XML_ATTR_ID, id);
     crm_xml_add(data, F_STONITH_ORIGIN, __func__);
     crm_xml_add(data, "agent", agent);
     if ((namespace != st_namespace_any) && (namespace != st_namespace_invalid)) {
         crm_xml_add(data, "namespace", stonith_namespace2text(namespace));
     }
     if (rsc_provides) {
         crm_xml_add(data, "rsc_provides", rsc_provides);
     }
 
     for (; params; params = params->next) {
         hash2field((gpointer) params->key, (gpointer) params->value, args);
     }
 
     return data;
 }
 
 static int
 stonith_api_register_device(stonith_t * st, int call_options,
                             const char *id, const char *namespace, const char *agent,
                             stonith_key_value_t * params)
 {
     int rc = 0;
     xmlNode *data = NULL;
 
     data = create_device_registration_xml(id, stonith_text2namespace(namespace),
                                           agent, params, NULL);
 
     rc = stonith_send_command(st, STONITH_OP_DEVICE_ADD, data, NULL, call_options, 0);
     free_xml(data);
 
     return rc;
 }
 
 static int
 stonith_api_remove_device(stonith_t * st, int call_options, const char *name)
 {
     int rc = 0;
     xmlNode *data = NULL;
 
     data = create_xml_node(NULL, F_STONITH_DEVICE);
     crm_xml_add(data, F_STONITH_ORIGIN, __func__);
     crm_xml_add(data, XML_ATTR_ID, name);
     rc = stonith_send_command(st, STONITH_OP_DEVICE_DEL, data, NULL, call_options, 0);
     free_xml(data);
 
     return rc;
 }
 
 static int
 stonith_api_remove_level_full(stonith_t *st, int options,
                               const char *node, const char *pattern,
                               const char *attr, const char *value, int level)
 {
     int rc = 0;
     xmlNode *data = NULL;
 
     CRM_CHECK(node || pattern || (attr && value), return -EINVAL);
 
     data = create_xml_node(NULL, XML_TAG_FENCING_LEVEL);
     crm_xml_add(data, F_STONITH_ORIGIN, __func__);
 
     if (node) {
         crm_xml_add(data, XML_ATTR_STONITH_TARGET, node);
 
     } else if (pattern) {
         crm_xml_add(data, XML_ATTR_STONITH_TARGET_PATTERN, pattern);
 
     } else {
         crm_xml_add(data, XML_ATTR_STONITH_TARGET_ATTRIBUTE, attr);
         crm_xml_add(data, XML_ATTR_STONITH_TARGET_VALUE, value);
     }
 
     crm_xml_add_int(data, XML_ATTR_STONITH_INDEX, level);
     rc = stonith_send_command(st, STONITH_OP_LEVEL_DEL, data, NULL, options, 0);
     free_xml(data);
 
     return rc;
 }
 
 static int
 stonith_api_remove_level(stonith_t * st, int options, const char *node, int level)
 {
     return stonith_api_remove_level_full(st, options, node,
                                          NULL, NULL, NULL, level);
 }
 
 /*!
  * \internal
  * \brief Create XML for fence topology level registration request
  *
  * \param[in] node        If not NULL, target level by this node name
  * \param[in] pattern     If not NULL, target by node name using this regex
  * \param[in] attr        If not NULL, target by this node attribute
  * \param[in] value       If not NULL, target by this node attribute value
  * \param[in] level       Index number of level to register
  * \param[in] device_list List of devices in level
  *
  * \return Newly allocated XML tree on success, NULL otherwise
  *
  * \note The caller should set only one of node, pattern or attr/value.
  */
 xmlNode *
 create_level_registration_xml(const char *node, const char *pattern,
                               const char *attr, const char *value,
                               int level, stonith_key_value_t *device_list)
 {
     size_t len = 0;
     char *list = NULL;
     xmlNode *data;
 
     CRM_CHECK(node || pattern || (attr && value), return NULL);
 
     data = create_xml_node(NULL, XML_TAG_FENCING_LEVEL);
     CRM_CHECK(data, return NULL);
 
     crm_xml_add(data, F_STONITH_ORIGIN, __func__);
     crm_xml_add_int(data, XML_ATTR_ID, level);
     crm_xml_add_int(data, XML_ATTR_STONITH_INDEX, level);
 
     if (node) {
         crm_xml_add(data, XML_ATTR_STONITH_TARGET, node);
 
     } else if (pattern) {
         crm_xml_add(data, XML_ATTR_STONITH_TARGET_PATTERN, pattern);
 
     } else {
         crm_xml_add(data, XML_ATTR_STONITH_TARGET_ATTRIBUTE, attr);
         crm_xml_add(data, XML_ATTR_STONITH_TARGET_VALUE, value);
     }
 
     // cppcheck seems not to understand the abort logic behind pcmk__realloc
     // cppcheck-suppress memleak
     for (; device_list; device_list = device_list->next) {
         pcmk__add_separated_word(&list, &len, device_list->value, ",");
     }
 
     crm_xml_add(data, XML_ATTR_STONITH_DEVICES, list);
 
     free(list);
     return data;
 }
 
 static int
 stonith_api_register_level_full(stonith_t * st, int options, const char *node,
                                 const char *pattern,
                                 const char *attr, const char *value,
                                 int level, stonith_key_value_t *device_list)
 {
     int rc = 0;
     xmlNode *data = create_level_registration_xml(node, pattern, attr, value,
                                                   level, device_list);
     CRM_CHECK(data != NULL, return -EINVAL);
 
     rc = stonith_send_command(st, STONITH_OP_LEVEL_ADD, data, NULL, options, 0);
     free_xml(data);
 
     return rc;
 }
 
 static int
 stonith_api_register_level(stonith_t * st, int options, const char *node, int level,
                            stonith_key_value_t * device_list)
 {
     return stonith_api_register_level_full(st, options, node, NULL, NULL, NULL,
                                            level, device_list);
 }
 
 static void
 append_config_arg(gpointer key, gpointer value, gpointer user_data)
 {
     /* The fencer will filter "action" out when it registers the device,
      * but ignore it here in case any external API users don't.
      *
      * Also filter out parameters handled directly by Pacemaker.
      */
     if (!pcmk__str_eq(key, STONITH_ATTR_ACTION_OP, pcmk__str_casei)
         && !pcmk_stonith_param(key)
         && (strstr(key, CRM_META) == NULL)
         && !pcmk__str_eq(key, "crm_feature_set", pcmk__str_casei)) {
 
         crm_trace("Passing %s=%s with fence action",
                   (const char *) key, (const char *) (value? value : ""));
         g_hash_table_insert((GHashTable *) user_data,
                             strdup(key), strdup(value? value : ""));
     }
 }
 
 static GHashTable *
 make_args(const char *agent, const char *action, const char *victim,
           uint32_t victim_nodeid, GHashTable * device_args,
           GHashTable * port_map, const char *host_arg)
 {
     GHashTable *arg_list = NULL;
     const char *value = NULL;
 
     CRM_CHECK(action != NULL, return NULL);
 
     arg_list = pcmk__strkey_table(free, free);
 
     // Add action to arguments (using an alias if requested)
     if (device_args) {
         char buffer[512];
 
         snprintf(buffer, sizeof(buffer), "pcmk_%s_action", action);
         value = g_hash_table_lookup(device_args, buffer);
         if (value) {
             crm_debug("Substituting '%s' for fence action %s targeting %s",
                       value, action, victim);
             action = value;
         }
     }
     g_hash_table_insert(arg_list, strdup(STONITH_ATTR_ACTION_OP),
                         strdup(action));
 
     /* If this is a fencing operation against another node, add more standard
      * arguments.
      */
     if (victim && device_args) {
         const char *param = NULL;
 
         /* Always pass the target's name, per
          * https://github.com/ClusterLabs/fence-agents/blob/master/doc/FenceAgentAPI.md
          */
         g_hash_table_insert(arg_list, strdup("nodename"), strdup(victim));
 
         // If the target's node ID was specified, pass it, too
         if (victim_nodeid) {
             char *nodeid = crm_strdup_printf("%" PRIu32, victim_nodeid);
 
             // cts-fencing looks for this log message
             crm_info("Passing '%s' as nodeid with fence action '%s' targeting %s",
                      nodeid, action, victim);
             g_hash_table_insert(arg_list, strdup("nodeid"), nodeid);
         }
 
         // Check whether target must be specified in some other way
         param = g_hash_table_lookup(device_args, PCMK_STONITH_HOST_ARGUMENT);
         if (!pcmk__str_eq(agent, "fence_legacy", pcmk__str_none)
             && !pcmk__str_eq(param, "none", pcmk__str_casei)) {
 
             if (param == NULL) {
                 /* Use the caller's default for pcmk_host_argument, or "port" if
                  * none was given
                  */
                 param = (host_arg == NULL)? "port" : host_arg;
             }
             value = g_hash_table_lookup(device_args, param);
 
             if (pcmk__str_eq(value, "dynamic",
                              pcmk__str_casei|pcmk__str_null_matches)) {
                 /* If the host argument was "dynamic" or not explicitly specified,
                  * add it with the target
                  */
                 const char *alias = NULL;
 
                 if (port_map) {
                     alias = g_hash_table_lookup(port_map, victim);
                 }
                 if (alias == NULL) {
                     alias = victim;
                 }
                 crm_debug("Passing %s='%s' with fence action %s targeting %s",
                           param, alias, action, victim);
                 g_hash_table_insert(arg_list, strdup(param), strdup(alias));
             }
         }
     }
 
     if (device_args) {
         g_hash_table_foreach(device_args, append_config_arg, arg_list);
     }
 
     return arg_list;
 }
 
 /*!
  * \internal
  * \brief Free all memory used by a stonith action
  *
  * \param[in,out] action  Action to free
  */
 void
 stonith__destroy_action(stonith_action_t *action)
 {
     if (action) {
         free(action->agent);
         if (action->args) {
             g_hash_table_destroy(action->args);
         }
         free(action->action);
         free(action->victim);
         if (action->svc_action) {
             services_action_free(action->svc_action);
         }
         free(action->output);
         free(action->error);
         free(action);
     }
 }
 
 /*!
  * \internal
  * \brief Get the result of an executed stonith action
  *
  * \param[in,out] action        Executed action
  * \param[out]    rc            Where to store result code (or NULL)
  * \param[out]    output        Where to store standard output (or NULL)
  * \param[out]    error_output  Where to store standard error output (or NULL)
  *
  * \note If output or error_output is not NULL, the caller is responsible for
  *       freeing the memory.
  */
 void
 stonith__action_result(stonith_action_t *action, int *rc, char **output,
                        char **error_output)
 {
     if (rc) {
         *rc = pcmk_ok;
     }
     if (output) {
         *output = NULL;
     }
     if (error_output) {
         *error_output = NULL;
     }
     if (action != NULL) {
         if (rc) {
             *rc = action->rc;
         }
         if (output && action->output) {
             *output = action->output;
             action->output = NULL; // hand off memory management to caller
         }
         if (error_output && action->error) {
             *error_output = action->error;
             action->error = NULL; // hand off memory management to caller
         }
     }
 }
 
 #define FAILURE_MAX_RETRIES 2
 stonith_action_t *
 stonith_action_create(const char *agent,
                       const char *_action,
                       const char *victim,
                       uint32_t victim_nodeid,
                       int timeout, GHashTable * device_args,
                       GHashTable * port_map, const char *host_arg)
 {
     stonith_action_t *action;
 
     action = calloc(1, sizeof(stonith_action_t));
     action->args = make_args(agent, _action, victim, victim_nodeid,
                              device_args, port_map, host_arg);
     crm_debug("Preparing '%s' action for %s using agent %s",
               _action, (victim? victim : "no target"), agent);
     action->agent = strdup(agent);
     action->action = strdup(_action);
     if (victim) {
         action->victim = strdup(victim);
     }
     action->timeout = action->remaining_timeout = timeout;
     action->max_retries = FAILURE_MAX_RETRIES;
 
     if (device_args) {
         char buffer[512];
         const char *value = NULL;
 
         snprintf(buffer, sizeof(buffer), "pcmk_%s_retries", _action);
         value = g_hash_table_lookup(device_args, buffer);
 
         if (value) {
             action->max_retries = atoi(value);
         }
     }
 
     return action;
 }
 
 static gboolean
 update_remaining_timeout(stonith_action_t * action)
 {
     int diff = time(NULL) - action->initial_start_time;
 
     if (action->tries >= action->max_retries) {
         crm_info("Attempted to execute agent %s (%s) the maximum number of times (%d) allowed",
                  action->agent, action->action, action->max_retries);
         action->remaining_timeout = 0;
     } else if ((action->rc != -ETIME) && diff < (action->timeout * 0.7)) {
         /* only set remaining timeout period if there is 30%
          * or greater of the original timeout period left */
         action->remaining_timeout = action->timeout - diff;
     } else {
         action->remaining_timeout = 0;
     }
     return action->remaining_timeout ? TRUE : FALSE;
 }
 
 static int
 svc_action_to_errno(svc_action_t *svc_action) {
     int rv = pcmk_ok;
 
     if (svc_action->rc > 0) {
         /* Try to provide a useful error code based on the fence agent's
             * error output.
             */
         if (svc_action->rc == PCMK_OCF_TIMEOUT) {
             rv = -ETIME;
 
         } else if (svc_action->stderr_data == NULL) {
             rv = -ENODATA;
 
         } else if (strstr(svc_action->stderr_data, "imed out")) {
             /* Some agents have their own internal timeouts */
             rv = -ETIME;
 
         } else if (strstr(svc_action->stderr_data, "Unrecognised action")) {
             rv = -EOPNOTSUPP;
 
         } else {
             rv = -pcmk_err_generic;
         }
     }
     return rv;
 }
 
 static void
 stonith_action_async_done(svc_action_t *svc_action)
 {
     stonith_action_t *action = (stonith_action_t *) svc_action->cb_data;
 
     action->rc = svc_action_to_errno(svc_action);
     action->output = svc_action->stdout_data;
     svc_action->stdout_data = NULL;
     action->error = svc_action->stderr_data;
     svc_action->stderr_data = NULL;
 
     svc_action->params = NULL;
 
     crm_debug("Child process %d performing action '%s' exited with rc %d",
                 action->pid, action->action, svc_action->rc);
 
     log_action(action, action->pid);
 
     if (action->rc != pcmk_ok && update_remaining_timeout(action)) {
         int rc = internal_stonith_action_execute(action);
         if (rc == pcmk_ok) {
             return;
         }
     }
 
     if (action->done_cb) {
         action->done_cb(action->pid, action->rc, action->output, action->userdata);
     }
 
     action->svc_action = NULL; // don't remove our caller
     stonith__destroy_action(action);
 }
 
 static void
 stonith_action_async_forked(svc_action_t *svc_action)
 {
     stonith_action_t *action = (stonith_action_t *) svc_action->cb_data;
 
     action->pid = svc_action->pid;
     action->svc_action = svc_action;
 
     if (action->fork_cb) {
         (action->fork_cb) (svc_action->pid, action->userdata);
     }
 
     crm_trace("Child process %d performing action '%s' successfully forked",
               action->pid, action->action);
 }
 
 static int
 internal_stonith_action_execute(stonith_action_t * action)
 {
     int rc = -EPROTO;
     int is_retry = 0;
     svc_action_t *svc_action = NULL;
     static int stonith_sequence = 0;
     char *buffer = NULL;
 
     if (!action->tries) {
         action->initial_start_time = time(NULL);
     }
     action->tries++;
 
     if (action->tries > 1) {
         crm_info("Attempt %d to execute %s (%s). remaining timeout is %d",
                  action->tries, action->agent, action->action, action->remaining_timeout);
         is_retry = 1;
     }
 
     if (action->args == NULL || action->agent == NULL)
         goto fail;
 
     buffer = crm_strdup_printf(PCMK__FENCE_BINDIR "/%s",
                                basename(action->agent));
     svc_action = services_action_create_generic(buffer, NULL);
     free(buffer);
     svc_action->timeout = 1000 * action->remaining_timeout;
     svc_action->standard = strdup(PCMK_RESOURCE_CLASS_STONITH);
     svc_action->id = crm_strdup_printf("%s_%s_%d", basename(action->agent),
                                        action->action, action->tries);
     svc_action->agent = strdup(action->agent);
     svc_action->sequence = stonith_sequence++;
     svc_action->params = action->args;
     svc_action->cb_data = (void *) action;
     svc_action->flags = pcmk__set_flags_as(__func__, __LINE__,
                                            LOG_TRACE, "Action",
                                            svc_action->id, svc_action->flags,
                                            SVC_ACTION_NON_BLOCKED,
                                            "SVC_ACTION_NON_BLOCKED");
 
     /* keep retries from executing out of control and free previous results */
     if (is_retry) {
         free(action->output);
         action->output = NULL;
         free(action->error);
         action->error = NULL;
         sleep(1);
     }
 
     if (action->async) {
         /* async */
         if(services_action_async_fork_notify(svc_action,
             &stonith_action_async_done,
             &stonith_action_async_forked) == FALSE) {
             services_action_free(svc_action);
             svc_action = NULL;
         } else {
             rc = 0;
         }
 
     } else {
         /* sync */
         if (services_action_sync(svc_action)) {
             rc = 0;
             action->rc = svc_action_to_errno(svc_action);
             action->output = svc_action->stdout_data;
             svc_action->stdout_data = NULL;
             action->error = svc_action->stderr_data;
             svc_action->stderr_data = NULL;
         } else {
             action->rc = -ECONNABORTED;
             rc = action->rc;
         }
 
         svc_action->params = NULL;
         services_action_free(svc_action);
     }
 
   fail:
     return rc;
 }
 
 /*!
  * \internal
  * \brief Kick off execution of an async stonith action
  *
  * \param[in,out] action        Action to be executed
  * \param[in,out] userdata      Datapointer to be passed to callbacks
  * \param[in]     done          Callback to notify action has failed/succeeded
  * \param[in]     fork_callback Callback to notify successful fork of child
  *
  * \return pcmk_ok if ownership of action has been taken, -errno otherwise
  */
 int
 stonith_action_execute_async(stonith_action_t * action,
                              void *userdata,
                              void (*done) (GPid pid, int rc, const char *output,
                                            gpointer user_data),
                              void (*fork_cb) (GPid pid, gpointer user_data))
 {
     if (!action) {
         return -EINVAL;
     }
 
     action->userdata = userdata;
     action->done_cb = done;
     action->fork_cb = fork_cb;
     action->async = 1;
 
     return internal_stonith_action_execute(action);
 }
 
 /*!
  * \internal
  * \brief Execute a stonith action
  *
  * \param[in,out] action  Action to execute
  *
  * \return pcmk_ok on success, -errno otherwise
  */
 int
 stonith__execute(stonith_action_t *action)
 {
     int rc = pcmk_ok;
 
     CRM_CHECK(action != NULL, return -EINVAL);
 
     // Keep trying until success, max retries, or timeout
     do {
         rc = internal_stonith_action_execute(action);
     } while ((rc != pcmk_ok) && update_remaining_timeout(action));
 
     return rc;
 }
 
 static int
 stonith_api_device_list(stonith_t * stonith, int call_options, const char *namespace,
                         stonith_key_value_t ** devices, int timeout)
 {
     int count = 0;
     enum stonith_namespace ns = stonith_text2namespace(namespace);
 
     if (devices == NULL) {
         crm_err("Parameter error: stonith_api_device_list");
         return -EFAULT;
     }
 
 #if HAVE_STONITH_STONITH_H
     // Include Linux-HA agents if requested
     if ((ns == st_namespace_any) || (ns == st_namespace_lha)) {
         count += stonith__list_lha_agents(devices);
     }
 #endif
 
     // Include Red Hat agents if requested
     if ((ns == st_namespace_any) || (ns == st_namespace_rhcs)) {
         count += stonith__list_rhcs_agents(devices);
     }
 
     return count;
 }
 
 static int
 stonith_api_device_metadata(stonith_t * stonith, int call_options, const char *agent,
                             const char *namespace, char **output, int timeout)
 {
     /* By executing meta-data directly, we can get it from stonith_admin when
      * the cluster is not running, which is important for higher-level tools.
      */
 
     enum stonith_namespace ns = stonith_get_namespace(agent, namespace);
 
     crm_trace("Looking up metadata for %s agent %s",
               stonith_namespace2text(ns), agent);
 
     switch (ns) {
         case st_namespace_rhcs:
             return stonith__rhcs_metadata(agent, timeout, output);
 
 #if HAVE_STONITH_STONITH_H
         case st_namespace_lha:
             return stonith__lha_metadata(agent, timeout, output);
 #endif
 
         default:
             crm_err("Can't get fence agent '%s' meta-data: No such agent",
                     agent);
             break;
     }
     return -ENODEV;
 }
 
 static int
 stonith_api_query(stonith_t * stonith, int call_options, const char *target,
                   stonith_key_value_t ** devices, int timeout)
 {
     int rc = 0, lpc = 0, max = 0;
 
     xmlNode *data = NULL;
     xmlNode *output = NULL;
     xmlXPathObjectPtr xpathObj = NULL;
 
     CRM_CHECK(devices != NULL, return -EINVAL);
 
     data = create_xml_node(NULL, F_STONITH_DEVICE);
     crm_xml_add(data, F_STONITH_ORIGIN, __func__);
     crm_xml_add(data, F_STONITH_TARGET, target);
     crm_xml_add(data, F_STONITH_ACTION, "off");
     rc = stonith_send_command(stonith, STONITH_OP_QUERY, data, &output, call_options, timeout);
 
     if (rc < 0) {
         return rc;
     }
 
     xpathObj = xpath_search(output, "//@agent");
     if (xpathObj) {
         max = numXpathResults(xpathObj);
 
         for (lpc = 0; lpc < max; lpc++) {
             xmlNode *match = getXpathResult(xpathObj, lpc);
 
             CRM_LOG_ASSERT(match != NULL);
             if(match != NULL) {
                 xmlChar *match_path = xmlGetNodePath(match);
 
                 crm_info("%s[%d] = %s", "//@agent", lpc, match_path);
                 free(match_path);
                 *devices = stonith_key_value_add(*devices, NULL, crm_element_value(match, XML_ATTR_ID));
             }
         }
 
         freeXpathObject(xpathObj);
     }
 
     free_xml(output);
     free_xml(data);
     return max;
 }
 
 static int
 stonith_api_call(stonith_t * stonith,
                  int call_options,
                  const char *id,
                  const char *action, const char *victim, int timeout, xmlNode ** output)
 {
     int rc = 0;
     xmlNode *data = NULL;
 
     data = create_xml_node(NULL, F_STONITH_DEVICE);
     crm_xml_add(data, F_STONITH_ORIGIN, __func__);
     crm_xml_add(data, F_STONITH_DEVICE, id);
     crm_xml_add(data, F_STONITH_ACTION, action);
     crm_xml_add(data, F_STONITH_TARGET, victim);
 
     rc = stonith_send_command(stonith, STONITH_OP_EXEC, data, output, call_options, timeout);
     free_xml(data);
 
     return rc;
 }
 
 static int
 stonith_api_list(stonith_t * stonith, int call_options, const char *id, char **list_info,
                  int timeout)
 {
     int rc;
     xmlNode *output = NULL;
 
     rc = stonith_api_call(stonith, call_options, id, "list", NULL, timeout, &output);
 
     if (output && list_info) {
         const char *list_str;
 
         list_str = crm_element_value(output, "st_output");
 
         if (list_str) {
             *list_info = strdup(list_str);
         }
     }
 
     if (output) {
         free_xml(output);
     }
 
     return rc;
 }
 
 static int
 stonith_api_monitor(stonith_t * stonith, int call_options, const char *id, int timeout)
 {
     return stonith_api_call(stonith, call_options, id, "monitor", NULL, timeout, NULL);
 }
 
 static int
 stonith_api_status(stonith_t * stonith, int call_options, const char *id, const char *port,
                    int timeout)
 {
     return stonith_api_call(stonith, call_options, id, "status", port, timeout, NULL);
 }
 
 static int
 stonith_api_fence_with_delay(stonith_t * stonith, int call_options, const char *node,
                              const char *action, int timeout, int tolerance, int delay)
 {
     int rc = 0;
     xmlNode *data = NULL;
 
     data = create_xml_node(NULL, __func__);
     crm_xml_add(data, F_STONITH_TARGET, node);
     crm_xml_add(data, F_STONITH_ACTION, action);
     crm_xml_add_int(data, F_STONITH_TIMEOUT, timeout);
     crm_xml_add_int(data, F_STONITH_TOLERANCE, tolerance);
     crm_xml_add_int(data, F_STONITH_DELAY, delay);
 
     rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, timeout);
     free_xml(data);
 
     return rc;
 }
 
 static int
 stonith_api_fence(stonith_t * stonith, int call_options, const char *node, const char *action,
                   int timeout, int tolerance)
 {
     return stonith_api_fence_with_delay(stonith, call_options, node, action,
                                         timeout, tolerance, 0);
 }
 
 static int
 stonith_api_confirm(stonith_t * stonith, int call_options, const char *target)
 {
     stonith__set_call_options(call_options, target, st_opt_manual_ack);
     return stonith_api_fence(stonith, call_options, target, "off", 0, 0);
 }
 
 static int
 stonith_api_history(stonith_t * stonith, int call_options, const char *node,
                     stonith_history_t ** history, int timeout)
 {
     int rc = 0;
     xmlNode *data = NULL;
     xmlNode *output = NULL;
     stonith_history_t *last = NULL;
 
     *history = NULL;
 
     if (node) {
         data = create_xml_node(NULL, __func__);
         crm_xml_add(data, F_STONITH_TARGET, node);
     }
 
     stonith__set_call_options(call_options, node, st_opt_sync_call);
     rc = stonith_send_command(stonith, STONITH_OP_FENCE_HISTORY, data, &output,
                               call_options, timeout);
     free_xml(data);
 
     if (rc == 0) {
         xmlNode *op = NULL;
         xmlNode *reply = get_xpath_object("//" F_STONITH_HISTORY_LIST, output,
                                           LOG_NEVER);
 
         for (op = pcmk__xml_first_child(reply); op != NULL;
              op = pcmk__xml_next(op)) {
             stonith_history_t *kvp;
             long long completed;
 
             kvp = calloc(1, sizeof(stonith_history_t));
             kvp->target = crm_element_value_copy(op, F_STONITH_TARGET);
             kvp->action = crm_element_value_copy(op, F_STONITH_ACTION);
             kvp->origin = crm_element_value_copy(op, F_STONITH_ORIGIN);
             kvp->delegate = crm_element_value_copy(op, F_STONITH_DELEGATE);
             kvp->client = crm_element_value_copy(op, F_STONITH_CLIENTNAME);
             crm_element_value_ll(op, F_STONITH_DATE, &completed);
             kvp->completed = (time_t) completed;
             crm_element_value_int(op, F_STONITH_STATE, &kvp->state);
 
             if (last) {
                 last->next = kvp;
             } else {
                 *history = kvp;
             }
             last = kvp;
         }
     }
 
     free_xml(output);
 
     return rc;
 }
 
 void stonith_history_free(stonith_history_t *history)
 {
     stonith_history_t *hp, *hp_old;
 
     for (hp = history; hp; hp_old = hp, hp = hp->next, free(hp_old)) {
         free(hp->target);
         free(hp->action);
         free(hp->origin);
         free(hp->delegate);
         free(hp->client);
     }
 }
 
 static gint
 stonithlib_GCompareFunc(gconstpointer a, gconstpointer b)
 {
     int rc = 0;
     const stonith_notify_client_t *a_client = a;
     const stonith_notify_client_t *b_client = b;
 
     if (a_client->delete || b_client->delete) {
         /* make entries marked for deletion not findable */
         return -1;
     }
     CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0);
     rc = strcmp(a_client->event, b_client->event);
     if (rc == 0) {
         if (a_client->notify == NULL || b_client->notify == NULL) {
             return 0;
 
         } else if (a_client->notify == b_client->notify) {
             return 0;
 
         } else if (((long)a_client->notify) < ((long)b_client->notify)) {
             crm_err("callbacks for %s are not equal: %p vs. %p",
                     a_client->event, a_client->notify, b_client->notify);
             return -1;
         }
         crm_err("callbacks for %s are not equal: %p vs. %p",
                 a_client->event, a_client->notify, b_client->notify);
         return 1;
     }
     return rc;
 }
 
 xmlNode *
 stonith_create_op(int call_id, const char *token, const char *op, xmlNode * data, int call_options)
 {
     xmlNode *op_msg = create_xml_node(NULL, "stonith_command");
 
     CRM_CHECK(op_msg != NULL, return NULL);
     CRM_CHECK(token != NULL, return NULL);
 
     crm_xml_add(op_msg, F_XML_TAGNAME, "stonith_command");
 
     crm_xml_add(op_msg, F_TYPE, T_STONITH_NG);
     crm_xml_add(op_msg, F_STONITH_CALLBACK_TOKEN, token);
     crm_xml_add(op_msg, F_STONITH_OPERATION, op);
     crm_xml_add_int(op_msg, F_STONITH_CALLID, call_id);
     crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options);
     crm_xml_add_int(op_msg, F_STONITH_CALLOPTS, call_options);
 
     if (data != NULL) {
         add_message_xml(op_msg, F_STONITH_CALLDATA, data);
     }
 
     return op_msg;
 }
 
 static void
 stonith_destroy_op_callback(gpointer data)
 {
     stonith_callback_client_t *blob = data;
 
     if (blob->timer && blob->timer->ref > 0) {
         g_source_remove(blob->timer->ref);
     }
     free(blob->timer);
     free(blob);
 }
 
 static int
 stonith_api_signoff(stonith_t * stonith)
 {
     stonith_private_t *native = stonith->st_private;
 
     crm_debug("Disconnecting from the fencer");
 
     if (native->source != NULL) {
         /* Attached to mainloop */
         mainloop_del_ipc_client(native->source);
         native->source = NULL;
         native->ipc = NULL;
 
     } else if (native->ipc) {
         /* Not attached to mainloop */
         crm_ipc_t *ipc = native->ipc;
 
         native->ipc = NULL;
         crm_ipc_close(ipc);
         crm_ipc_destroy(ipc);
     }
 
     free(native->token); native->token = NULL;
     stonith->state = stonith_disconnected;
     return pcmk_ok;
 }
 
 static int
 stonith_api_del_callback(stonith_t * stonith, int call_id, bool all_callbacks)
 {
     stonith_private_t *private = stonith->st_private;
 
     if (all_callbacks) {
         private->op_callback = NULL;
         g_hash_table_destroy(private->stonith_op_callback_table);
         private->stonith_op_callback_table = pcmk__intkey_table(stonith_destroy_op_callback);
 
     } else if (call_id == 0) {
         private->op_callback = NULL;
 
     } else {
         pcmk__intkey_table_remove(private->stonith_op_callback_table, call_id);
     }
     return pcmk_ok;
 }
 
 static void
 invoke_callback(stonith_t * st, int call_id, int rc, void *userdata,
                 void (*callback) (stonith_t * st, stonith_callback_data_t * data))
 {
     stonith_callback_data_t data = { 0, };
 
     data.call_id = call_id;
     data.rc = rc;
     data.userdata = userdata;
 
     callback(st, &data);
 }
 
 static void
 stonith_perform_callback(stonith_t * stonith, xmlNode * msg, int call_id, int rc)
 {
     stonith_private_t *private = NULL;
     stonith_callback_client_t *blob = NULL;
     stonith_callback_client_t local_blob;
 
     CRM_CHECK(stonith != NULL, return);
     CRM_CHECK(stonith->st_private != NULL, return);
 
     private = stonith->st_private;
 
     local_blob.id = NULL;
     local_blob.callback = NULL;
     local_blob.user_data = NULL;
     local_blob.only_success = FALSE;
 
     if (msg != NULL) {
         crm_element_value_int(msg, F_STONITH_RC, &rc);
         crm_element_value_int(msg, F_STONITH_CALLID, &call_id);
     }
 
     CRM_CHECK(call_id > 0, crm_log_xml_err(msg, "Bad result"));
 
     blob = pcmk__intkey_table_lookup(private->stonith_op_callback_table,
                                      call_id);
     if (blob != NULL) {
         local_blob = *blob;
         blob = NULL;
 
         stonith_api_del_callback(stonith, call_id, FALSE);
 
     } else {
         crm_trace("No callback found for call %d", call_id);
         local_blob.callback = NULL;
     }
 
     if (local_blob.callback != NULL && (rc == pcmk_ok || local_blob.only_success == FALSE)) {
         crm_trace("Invoking callback %s for call %d", crm_str(local_blob.id), call_id);
         invoke_callback(stonith, call_id, rc, local_blob.user_data, local_blob.callback);
 
     } else if (private->op_callback == NULL && rc != pcmk_ok) {
         crm_warn("Fencing command failed: %s", pcmk_strerror(rc));
         crm_log_xml_debug(msg, "Failed fence update");
     }
 
     if (private->op_callback != NULL) {
         crm_trace("Invoking global callback for call %d", call_id);
         invoke_callback(stonith, call_id, rc, NULL, private->op_callback);
     }
     crm_trace("OP callback activated.");
 }
 
 static gboolean
 stonith_async_timeout_handler(gpointer data)
 {
     struct timer_rec_s *timer = data;
 
     crm_err("Async call %d timed out after %dms", timer->call_id, timer->timeout);
     stonith_perform_callback(timer->stonith, NULL, timer->call_id, -ETIME);
 
     /* Always return TRUE, never remove the handler
      * We do that in stonith_del_callback()
      */
     return TRUE;
 }
 
 static void
 set_callback_timeout(stonith_callback_client_t * callback, stonith_t * stonith, int call_id,
                      int timeout)
 {
     struct timer_rec_s *async_timer = callback->timer;
 
     if (timeout <= 0) {
         return;
     }
 
     if (!async_timer) {
         async_timer = calloc(1, sizeof(struct timer_rec_s));
         callback->timer = async_timer;
     }
 
     async_timer->stonith = stonith;
     async_timer->call_id = call_id;
     /* Allow a fair bit of grace to allow the server to tell us of a timeout
      * This is only a fallback
      */
     async_timer->timeout = (timeout + 60) * 1000;
     if (async_timer->ref) {
         g_source_remove(async_timer->ref);
     }
     async_timer->ref =
         g_timeout_add(async_timer->timeout, stonith_async_timeout_handler, async_timer);
 }
 
 static void
 update_callback_timeout(int call_id, int timeout, stonith_t * st)
 {
     stonith_callback_client_t *callback = NULL;
     stonith_private_t *private = st->st_private;
 
     callback = pcmk__intkey_table_lookup(private->stonith_op_callback_table,
                                          call_id);
     if (!callback || !callback->allow_timeout_updates) {
         return;
     }
 
     set_callback_timeout(callback, st, call_id, timeout);
 }
 
 static int
 stonith_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata)
 {
     const char *type = NULL;
     struct notify_blob_s blob;
 
     stonith_t *st = userdata;
     stonith_private_t *private = NULL;
 
     CRM_ASSERT(st != NULL);
     private = st->st_private;
 
     blob.stonith = st;
     blob.xml = string2xml(buffer);
     if (blob.xml == NULL) {
         crm_warn("Received malformed message from fencer: %s", buffer);
         return 0;
     }
 
     /* do callbacks */
     type = crm_element_value(blob.xml, F_TYPE);
     crm_trace("Activating %s callbacks...", type);
 
     if (pcmk__str_eq(type, T_STONITH_NG, pcmk__str_casei)) {
         stonith_perform_callback(st, blob.xml, 0, 0);
 
     } else if (pcmk__str_eq(type, T_STONITH_NOTIFY, pcmk__str_casei)) {
         foreach_notify_entry(private, stonith_send_notification, &blob);
     } else if (pcmk__str_eq(type, T_STONITH_TIMEOUT_VALUE, pcmk__str_casei)) {
         int call_id = 0;
         int timeout = 0;
 
         crm_element_value_int(blob.xml, F_STONITH_TIMEOUT, &timeout);
         crm_element_value_int(blob.xml, F_STONITH_CALLID, &call_id);
 
         update_callback_timeout(call_id, timeout, st);
     } else {
         crm_err("Unknown message type: %s", type);
         crm_log_xml_warn(blob.xml, "BadReply");
     }
 
     free_xml(blob.xml);
     return 1;
 }
 
 static int
 stonith_api_signon(stonith_t * stonith, const char *name, int *stonith_fd)
 {
     int rc = pcmk_ok;
     stonith_private_t *native = NULL;
     const char *display_name = name? name : "client";
 
     struct ipc_client_callbacks st_callbacks = {
         .dispatch = stonith_dispatch_internal,
         .destroy = stonith_connection_destroy
     };
 
     CRM_CHECK(stonith != NULL, return -EINVAL);
 
     native = stonith->st_private;
     CRM_ASSERT(native != NULL);
 
     crm_debug("Attempting fencer connection by %s with%s mainloop",
               display_name, (stonith_fd? "out" : ""));
 
     stonith->state = stonith_connected_command;
     if (stonith_fd) {
         /* No mainloop */
         native->ipc = crm_ipc_new("stonith-ng", 0);
 
         if (native->ipc && crm_ipc_connect(native->ipc)) {
             *stonith_fd = crm_ipc_get_fd(native->ipc);
         } else if (native->ipc) {
             crm_ipc_close(native->ipc);
             crm_ipc_destroy(native->ipc);
             native->ipc = NULL;
         }
 
     } else {
         /* With mainloop */
         native->source =
             mainloop_add_ipc_client("stonith-ng", G_PRIORITY_MEDIUM, 0, stonith, &st_callbacks);
         native->ipc = mainloop_get_ipc_client(native->source);
     }
 
     if (native->ipc == NULL) {
         rc = -ENOTCONN;
     } else {
         xmlNode *reply = NULL;
         xmlNode *hello = create_xml_node(NULL, "stonith_command");
 
         crm_xml_add(hello, F_TYPE, T_STONITH_NG);
         crm_xml_add(hello, F_STONITH_OPERATION, CRM_OP_REGISTER);
         crm_xml_add(hello, F_STONITH_CLIENTNAME, name);
         rc = crm_ipc_send(native->ipc, hello, crm_ipc_client_response, -1, &reply);
 
         if (rc < 0) {
             crm_debug("Couldn't register with the fencer: %s "
                       CRM_XS " rc=%d", pcmk_strerror(rc), rc);
             rc = -ECOMM;
 
         } else if (reply == NULL) {
             crm_debug("Couldn't register with the fencer: no reply");
             rc = -EPROTO;
 
         } else {
             const char *msg_type = crm_element_value(reply, F_STONITH_OPERATION);
 
             native->token = crm_element_value_copy(reply, F_STONITH_CLIENTID);
             if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
                 crm_debug("Couldn't register with the fencer: invalid reply type '%s'",
                           (msg_type? msg_type : "(missing)"));
                 crm_log_xml_debug(reply, "Invalid fencer reply");
                 rc = -EPROTO;
 
             } else if (native->token == NULL) {
                 crm_debug("Couldn't register with the fencer: no token in reply");
                 crm_log_xml_debug(reply, "Invalid fencer reply");
                 rc = -EPROTO;
 
             } else {
 #if HAVE_MSGFROMIPC_TIMEOUT
                 stonith->call_timeout = PCMK__IPC_TIMEOUT;
 #endif
                 crm_debug("Connection to fencer by %s succeeded (registration token: %s)",
                           display_name, native->token);
                 rc = pcmk_ok;
             }
         }
 
         free_xml(reply);
         free_xml(hello);
     }
 
     if (rc != pcmk_ok) {
         crm_debug("Connection attempt to fencer by %s failed: %s "
                   CRM_XS " rc=%d", display_name, pcmk_strerror(rc), rc);
         stonith->cmds->disconnect(stonith);
     }
     return rc;
 }
 
 static int
 stonith_set_notification(stonith_t * stonith, const char *callback, int enabled)
 {
     int rc = pcmk_ok;
     xmlNode *notify_msg = create_xml_node(NULL, __func__);
     stonith_private_t *native = stonith->st_private;
 
     if (stonith->state != stonith_disconnected) {
 
         crm_xml_add(notify_msg, F_STONITH_OPERATION, T_STONITH_NOTIFY);
         if (enabled) {
             crm_xml_add(notify_msg, F_STONITH_NOTIFY_ACTIVATE, callback);
         } else {
             crm_xml_add(notify_msg, F_STONITH_NOTIFY_DEACTIVATE, callback);
         }
 
         rc = crm_ipc_send(native->ipc, notify_msg, crm_ipc_client_response, -1, NULL);
         if (rc < 0) {
             crm_perror(LOG_DEBUG, "Couldn't register for fencing notifications: %d", rc);
             rc = -ECOMM;
         } else {
             rc = pcmk_ok;
         }
     }
 
     free_xml(notify_msg);
     return rc;
 }
 
 static int
 stonith_api_add_notification(stonith_t * stonith, const char *event,
                              void (*callback) (stonith_t * stonith, stonith_event_t * e))
 {
     GList *list_item = NULL;
     stonith_notify_client_t *new_client = NULL;
     stonith_private_t *private = NULL;
 
     private = stonith->st_private;
     crm_trace("Adding callback for %s events (%d)", event, g_list_length(private->notify_list));
 
     new_client = calloc(1, sizeof(stonith_notify_client_t));
     new_client->event = event;
     new_client->notify = callback;
 
     list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc);
 
     if (list_item != NULL) {
         crm_warn("Callback already present");
         free(new_client);
         return -ENOTUNIQ;
 
     } else {
         private->notify_list = g_list_append(private->notify_list, new_client);
 
         stonith_set_notification(stonith, event, 1);
 
         crm_trace("Callback added (%d)", g_list_length(private->notify_list));
     }
     return pcmk_ok;
 }
 
 static int
 stonith_api_del_notification(stonith_t * stonith, const char *event)
 {
     GList *list_item = NULL;
     stonith_notify_client_t *new_client = NULL;
     stonith_private_t *private = NULL;
 
     crm_debug("Removing callback for %s events", event);
 
     private = stonith->st_private;
     new_client = calloc(1, sizeof(stonith_notify_client_t));
     new_client->event = event;
     new_client->notify = NULL;
 
     list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc);
 
     stonith_set_notification(stonith, event, 0);
 
     if (list_item != NULL) {
         stonith_notify_client_t *list_client = list_item->data;
 
         if (private->notify_refcnt) {
             list_client->delete = TRUE;
             private->notify_deletes = TRUE;
         } else {
             private->notify_list = g_list_remove(private->notify_list, list_client);
             free(list_client);
         }
 
         crm_trace("Removed callback");
 
     } else {
         crm_trace("Callback not present");
     }
     free(new_client);
     return pcmk_ok;
 }
 
 static int
 stonith_api_add_callback(stonith_t * stonith, int call_id, int timeout, int options,
                          void *user_data, const char *callback_name,
                          void (*callback) (stonith_t * st, stonith_callback_data_t * data))
 {
     stonith_callback_client_t *blob = NULL;
     stonith_private_t *private = NULL;
 
     CRM_CHECK(stonith != NULL, return -EINVAL);
     CRM_CHECK(stonith->st_private != NULL, return -EINVAL);
     private = stonith->st_private;
 
     if (call_id == 0) {
         private->op_callback = callback;
 
     } else if (call_id < 0) {
         if (!(options & st_opt_report_only_success)) {
             crm_trace("Call failed, calling %s: %s", callback_name, pcmk_strerror(call_id));
             invoke_callback(stonith, call_id, call_id, user_data, callback);
         } else {
             crm_warn("Fencer call failed: %s", pcmk_strerror(call_id));
         }
         return FALSE;
     }
 
     blob = calloc(1, sizeof(stonith_callback_client_t));
     blob->id = callback_name;
     blob->only_success = (options & st_opt_report_only_success) ? TRUE : FALSE;
     blob->user_data = user_data;
     blob->callback = callback;
     blob->allow_timeout_updates = (options & st_opt_timeout_updates) ? TRUE : FALSE;
 
     if (timeout > 0) {
         set_callback_timeout(blob, stonith, call_id, timeout);
     }
 
     pcmk__intkey_table_insert(private->stonith_op_callback_table, call_id,
                               blob);
     crm_trace("Added callback to %s for call %d", callback_name, call_id);
 
     return TRUE;
 }
 
 static void
 stonith_dump_pending_op(gpointer key, gpointer value, gpointer user_data)
 {
     int call = GPOINTER_TO_INT(key);
     stonith_callback_client_t *blob = value;
 
     crm_debug("Call %d (%s): pending", call, crm_str(blob->id));
 }
 
 void
 stonith_dump_pending_callbacks(stonith_t * stonith)
 {
     stonith_private_t *private = stonith->st_private;
 
     if (private->stonith_op_callback_table == NULL) {
         return;
     }
     return g_hash_table_foreach(private->stonith_op_callback_table, stonith_dump_pending_op, NULL);
 }
 
 /*
  <notify t="st_notify" subt="st_device_register" st_op="st_device_register" st_rc="0" >
    <st_calldata >
      <stonith_command t="stonith-ng" st_async_id="088fb640-431a-48b9-b2fc-c4ff78d0a2d9" st_op="st_device_register" st_callid="2" st_callopt="4096" st_timeout="0" st_clientid="088fb640-431a-48b9-b2fc-c4ff78d0a2d9" st_clientname="cts-fence-helper" >
        <st_calldata >
          <st_device_id id="test-id" origin="create_device_registration_xml" agent="fence_virsh" namespace="stonith-ng" >
            <attributes ipaddr="localhost" pcmk-portmal="some-host=pcmk-1 pcmk-3=3,4" login="root" identity_file="/root/.ssh/id_dsa" />
          </st_device_id>
        </st_calldata>
      </stonith_command>
    </st_calldata>
  </notify>
 
  <notify t="st_notify" subt="st_notify_fence" st_op="st_notify_fence" st_rc="0" >
    <st_calldata >
      <st_notify_fence st_rc="0" st_target="some-host" st_op="st_fence" st_delegate="test-id" st_origin="61dd7759-e229-4be7-b1f8-ef49dd14d9f0" />
    </st_calldata>
  </notify>
 */
 static stonith_event_t *
 xml_to_event(xmlNode * msg)
 {
     stonith_event_t *event = calloc(1, sizeof(stonith_event_t));
     const char *ntype = crm_element_value(msg, F_SUBTYPE);
     char *data_addr = crm_strdup_printf("//%s", ntype);
     xmlNode *data = get_xpath_object(data_addr, msg, LOG_DEBUG);
 
     crm_log_xml_trace(msg, "stonith_notify");
 
     crm_element_value_int(msg, F_STONITH_RC, &(event->result));
 
     if (pcmk__str_eq(ntype, T_STONITH_NOTIFY_FENCE, pcmk__str_casei)) {
         event->operation = crm_element_value_copy(msg, F_STONITH_OPERATION);
 
         if (data) {
             event->origin = crm_element_value_copy(data, F_STONITH_ORIGIN);
             event->action = crm_element_value_copy(data, F_STONITH_ACTION);
             event->target = crm_element_value_copy(data, F_STONITH_TARGET);
             event->executioner = crm_element_value_copy(data, F_STONITH_DELEGATE);
             event->id = crm_element_value_copy(data, F_STONITH_REMOTE_OP_ID);
             event->client_origin = crm_element_value_copy(data, F_STONITH_CLIENTNAME);
             event->device = crm_element_value_copy(data, F_STONITH_DEVICE);
 
         } else {
             crm_err("No data for %s event", ntype);
             crm_log_xml_notice(msg, "BadEvent");
         }
     }
 
     free(data_addr);
     return event;
 }
 
 static void
 event_free(stonith_event_t * event)
 {
     free(event->id);
     free(event->type);
     free(event->message);
     free(event->operation);
     free(event->origin);
     free(event->action);
     free(event->target);
     free(event->executioner);
     free(event->device);
     free(event->client_origin);
     free(event);
 }
 
 static void
 stonith_send_notification(gpointer data, gpointer user_data)
 {
     struct notify_blob_s *blob = user_data;
     stonith_notify_client_t *entry = data;
     stonith_event_t *st_event = NULL;
     const char *event = NULL;
 
     if (blob->xml == NULL) {
         crm_warn("Skipping callback - NULL message");
         return;
     }
 
     event = crm_element_value(blob->xml, F_SUBTYPE);
 
     if (entry == NULL) {
         crm_warn("Skipping callback - NULL callback client");
         return;
 
     } else if (entry->delete) {
         crm_trace("Skipping callback - marked for deletion");
         return;
 
     } else if (entry->notify == NULL) {
         crm_warn("Skipping callback - NULL callback");
         return;
 
     } else if (!pcmk__str_eq(entry->event, event, pcmk__str_casei)) {
         crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event);
         return;
     }
 
     st_event = xml_to_event(blob->xml);
 
     crm_trace("Invoking callback for %p/%s event...", entry, event);
     entry->notify(blob->stonith, st_event);
     crm_trace("Callback invoked...");
 
     event_free(st_event);
 }
 
 /*!
  * \internal
  * \brief Create and send an API request
  *
  * \param[in]  stonith       Stonith connection
  * \param[in]  op            API operation to request
  * \param[in]  data          Data to attach to request
  * \param[out] output_data   If not NULL, will be set to reply if synchronous
  * \param[in]  call_options  Bitmask of stonith_call_options to use
  * \param[in]  timeout       Error if not completed within this many seconds
  *
  * \return pcmk_ok (for synchronous requests) or positive call ID
  *         (for asynchronous requests) on success, -errno otherwise
  */
 static int
 stonith_send_command(stonith_t * stonith, const char *op, xmlNode * data, xmlNode ** output_data,
                      int call_options, int timeout)
 {
     int rc = 0;
     int reply_id = -1;
 
     xmlNode *op_msg = NULL;
     xmlNode *op_reply = NULL;
     stonith_private_t *native = NULL;
 
     CRM_ASSERT(stonith && stonith->st_private && op);
     native = stonith->st_private;
 
     if (output_data != NULL) {
         *output_data = NULL;
     }
 
     if ((stonith->state == stonith_disconnected) || (native->token == NULL)) {
         return -ENOTCONN;
     }
 
     /* Increment the call ID, which must be positive to avoid conflicting with
      * error codes. This shouldn't be a problem unless the client mucked with
      * it or the counter wrapped around.
      */
     stonith->call_id++;
     if (stonith->call_id < 1) {
         stonith->call_id = 1;
     }
 
     op_msg = stonith_create_op(stonith->call_id, native->token, op, data, call_options);
     if (op_msg == NULL) {
         return -EINVAL;
     }
 
     crm_xml_add_int(op_msg, F_STONITH_TIMEOUT, timeout);
     crm_trace("Sending %s message to fencer with timeout %ds", op, timeout);
 
     if (data) {
         const char *delay_s = crm_element_value(data, F_STONITH_DELAY);
 
         if (delay_s) {
             crm_xml_add(op_msg, F_STONITH_DELAY, delay_s);
         }
     }
 
     {
         enum crm_ipc_flags ipc_flags = crm_ipc_flags_none;
 
         if (call_options & st_opt_sync_call) {
             pcmk__set_ipc_flags(ipc_flags, "stonith command",
                                 crm_ipc_client_response);
         }
         rc = crm_ipc_send(native->ipc, op_msg, ipc_flags,
                           1000 * (timeout + 60), &op_reply);
     }
     free_xml(op_msg);
 
     if (rc < 0) {
         crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%ds): %d", op, timeout, rc);
         rc = -ECOMM;
         goto done;
     }
 
     crm_log_xml_trace(op_reply, "Reply");
 
     if (!(call_options & st_opt_sync_call)) {
         crm_trace("Async call %d, returning", stonith->call_id);
         free_xml(op_reply);
         return stonith->call_id;
     }
 
     rc = pcmk_ok;
     crm_element_value_int(op_reply, F_STONITH_CALLID, &reply_id);
 
     if (reply_id == stonith->call_id) {
         crm_trace("Synchronous reply %d received", reply_id);
 
         if (crm_element_value_int(op_reply, F_STONITH_RC, &rc) != 0) {
             rc = -ENOMSG;
         }
 
         if ((call_options & st_opt_discard_reply) || output_data == NULL) {
             crm_trace("Discarding reply");
 
         } else {
             *output_data = op_reply;
             op_reply = NULL;    /* Prevent subsequent free */
         }
 
     } else if (reply_id <= 0) {
         crm_err("Received bad reply: No id set");
         crm_log_xml_err(op_reply, "Bad reply");
         free_xml(op_reply);
         rc = -ENOMSG;
 
     } else {
         crm_err("Received bad reply: %d (wanted %d)", reply_id, stonith->call_id);
         crm_log_xml_err(op_reply, "Old reply");
         free_xml(op_reply);
         rc = -ENOMSG;
     }
 
   done:
     if (crm_ipc_connected(native->ipc) == FALSE) {
         crm_err("Fencer disconnected");
         free(native->token); native->token = NULL;
         stonith->state = stonith_disconnected;
     }
 
     free_xml(op_reply);
     return rc;
 }
 
 /* Not used with mainloop */
 bool
 stonith_dispatch(stonith_t * st)
 {
     gboolean stay_connected = TRUE;
     stonith_private_t *private = NULL;
 
     CRM_ASSERT(st != NULL);
     private = st->st_private;
 
     while (crm_ipc_ready(private->ipc)) {
 
         if (crm_ipc_read(private->ipc) > 0) {
             const char *msg = crm_ipc_buffer(private->ipc);
 
             stonith_dispatch_internal(msg, strlen(msg), st);
         }
 
         if (crm_ipc_connected(private->ipc) == FALSE) {
             crm_err("Connection closed");
             stay_connected = FALSE;
         }
     }
 
     return stay_connected;
 }
 
 static int
 stonith_api_free(stonith_t * stonith)
 {
     int rc = pcmk_ok;
 
     crm_trace("Destroying %p", stonith);
 
     if (stonith->state != stonith_disconnected) {
         crm_trace("Disconnecting %p first", stonith);
         rc = stonith->cmds->disconnect(stonith);
     }
 
     if (stonith->state == stonith_disconnected) {
         stonith_private_t *private = stonith->st_private;
 
         crm_trace("Removing %d callbacks", g_hash_table_size(private->stonith_op_callback_table));
         g_hash_table_destroy(private->stonith_op_callback_table);
 
         crm_trace("Destroying %d notification clients", g_list_length(private->notify_list));
         g_list_free_full(private->notify_list, free);
 
         free(stonith->st_private);
         free(stonith->cmds);
         free(stonith);
 
     } else {
         crm_err("Not free'ing active connection: %s (%d)", pcmk_strerror(rc), rc);
     }
 
     return rc;
 }
 
 void
 stonith_api_delete(stonith_t * stonith)
 {
     crm_trace("Destroying %p", stonith);
     if(stonith) {
         stonith->cmds->free(stonith);
     }
 }
 
 static int
 stonith_api_validate(stonith_t *st, int call_options, const char *rsc_id,
                      const char *namespace_s, const char *agent,
                      stonith_key_value_t *params, int timeout, char **output,
                      char **error_output)
 {
     /* Validation should be done directly via the agent, so we can get it from
      * stonith_admin when the cluster is not running, which is important for
      * higher-level tools.
      */
 
     int rc = pcmk_ok;
 
     /* Use a dummy node name in case the agent requires a target. We assume the
      * actual target doesn't matter for validation purposes (if in practice,
      * that is incorrect, we will need to allow the caller to pass the target).
      */
     const char *target = "node1";
     const char *host_arg = NULL;
 
     GHashTable *params_table = pcmk__strkey_table(free, free);
 
     // Convert parameter list to a hash table
     for (; params; params = params->next) {
         if (pcmk__str_eq(params->key, PCMK_STONITH_HOST_ARGUMENT,
                          pcmk__str_casei)) {
             host_arg = params->value;
         }
         if (!pcmk_stonith_param(params->key)) {
             g_hash_table_insert(params_table, strdup(params->key),
                                 strdup(params->value));
         }
     }
 
 #if SUPPORT_CIBSECRETS
     rc = pcmk__substitute_secrets(rsc_id, params_table);
     if (rc != pcmk_rc_ok) {
         crm_warn("Could not replace secret parameters for validation of %s: %s",
                  agent, pcmk_rc_str(rc));
         // rc is standard return value, don't return it in this function
     }
 #endif
 
     if (output) {
         *output = NULL;
     }
     if (error_output) {
         *error_output = NULL;
     }
 
     switch (stonith_get_namespace(agent, namespace_s)) {
         case st_namespace_rhcs:
             rc = stonith__rhcs_validate(st, call_options, target, agent,
                                         params_table, host_arg, timeout,
                                         output, error_output);
             break;
 
 #if HAVE_STONITH_STONITH_H
         case st_namespace_lha:
             rc = stonith__lha_validate(st, call_options, target, agent,
                                        params_table, timeout, output,
                                        error_output);
             break;
 #endif
 
         default:
             rc = -EINVAL;
             errno = EINVAL;
             crm_perror(LOG_ERR,
                        "Agent %s not found or does not support validation",
                        agent);
             break;
     }
     g_hash_table_destroy(params_table);
     return rc;
 }
 
 stonith_t *
 stonith_api_new(void)
 {
     stonith_t *new_stonith = NULL;
     stonith_private_t *private = NULL;
 
     new_stonith = calloc(1, sizeof(stonith_t));
     if (new_stonith == NULL) {
         return NULL;
     }
 
     private = calloc(1, sizeof(stonith_private_t));
     if (private == NULL) {
         free(new_stonith);
         return NULL;
     }
     new_stonith->st_private = private;
 
     private->stonith_op_callback_table = pcmk__intkey_table(stonith_destroy_op_callback);
     private->notify_list = NULL;
     private->notify_refcnt = 0;
     private->notify_deletes = FALSE;
 
     new_stonith->call_id = 1;
     new_stonith->state = stonith_disconnected;
 
     new_stonith->cmds = calloc(1, sizeof(stonith_api_operations_t));
     if (new_stonith->cmds == NULL) {
         free(new_stonith->st_private);
         free(new_stonith);
         return NULL;
     }
 
 /* *INDENT-OFF* */
     new_stonith->cmds->free       = stonith_api_free;
     new_stonith->cmds->connect    = stonith_api_signon;
     new_stonith->cmds->disconnect = stonith_api_signoff;
 
     new_stonith->cmds->list       = stonith_api_list;
     new_stonith->cmds->monitor    = stonith_api_monitor;
     new_stonith->cmds->status     = stonith_api_status;
     new_stonith->cmds->fence      = stonith_api_fence;
     new_stonith->cmds->fence_with_delay = stonith_api_fence_with_delay;
     new_stonith->cmds->confirm    = stonith_api_confirm;
     new_stonith->cmds->history    = stonith_api_history;
 
     new_stonith->cmds->list_agents  = stonith_api_device_list;
     new_stonith->cmds->metadata     = stonith_api_device_metadata;
 
     new_stonith->cmds->query           = stonith_api_query;
     new_stonith->cmds->remove_device   = stonith_api_remove_device;
     new_stonith->cmds->register_device = stonith_api_register_device;
 
     new_stonith->cmds->remove_level          = stonith_api_remove_level;
     new_stonith->cmds->remove_level_full     = stonith_api_remove_level_full;
     new_stonith->cmds->register_level        = stonith_api_register_level;
     new_stonith->cmds->register_level_full   = stonith_api_register_level_full;
 
     new_stonith->cmds->remove_callback       = stonith_api_del_callback;
     new_stonith->cmds->register_callback     = stonith_api_add_callback;
     new_stonith->cmds->remove_notification   = stonith_api_del_notification;
     new_stonith->cmds->register_notification = stonith_api_add_notification;
 
     new_stonith->cmds->validate              = stonith_api_validate;
 /* *INDENT-ON* */
 
     return new_stonith;
 }
 
 /*!
  * \brief Make a blocking connection attempt to the fencer
  *
  * \param[in,out] st            Fencer API object
  * \param[in]     name          Client name to use with fencer
  * \param[in]     max_attempts  Return error if this many attempts fail
  *
  * \return pcmk_ok on success, result of last attempt otherwise
  */
 int
 stonith_api_connect_retry(stonith_t *st, const char *name, int max_attempts)
 {
     int rc = -EINVAL; // if max_attempts is not positive
 
     for (int attempt = 1; attempt <= max_attempts; attempt++) {
         rc = st->cmds->connect(st, name, NULL);
         if (rc == pcmk_ok) {
             return pcmk_ok;
         } else if (attempt < max_attempts) {
             crm_notice("Fencer connection attempt %d of %d failed (retrying in 2s): %s "
                        CRM_XS " rc=%d",
                        attempt, max_attempts, pcmk_strerror(rc), rc);
             sleep(2);
         }
     }
     crm_notice("Could not connect to fencer: %s " CRM_XS " rc=%d",
                pcmk_strerror(rc), rc);
     return rc;
 }
 
 stonith_key_value_t *
 stonith_key_value_add(stonith_key_value_t * head, const char *key, const char *value)
 {
     stonith_key_value_t *p, *end;
 
     p = calloc(1, sizeof(stonith_key_value_t));
     if (key) {
         p->key = strdup(key);
     }
     if (value) {
         p->value = strdup(value);
     }
 
     end = head;
     while (end && end->next) {
         end = end->next;
     }
 
     if (end) {
         end->next = p;
     } else {
         head = p;
     }
 
     return head;
 }
 
 void
 stonith_key_value_freeall(stonith_key_value_t * head, int keys, int values)
 {
     stonith_key_value_t *p;
 
     while (head) {
         p = head->next;
         if (keys) {
             free(head->key);
         }
         if (values) {
             free(head->value);
         }
         free(head);
         head = p;
     }
 }
 
 #define api_log_open() openlog("stonith-api", LOG_CONS | LOG_NDELAY | LOG_PID, LOG_DAEMON)
 #define api_log(level, fmt, args...) syslog(level, "%s: "fmt, __func__, args)
 
 int
 stonith_api_kick(uint32_t nodeid, const char *uname, int timeout, bool off)
 {
     int rc = pcmk_ok;
     stonith_t *st = stonith_api_new();
     const char *action = off? "off" : "reboot";
 
     api_log_open();
     if (st == NULL) {
         api_log(LOG_ERR, "API initialization failed, could not kick (%s) node %u/%s",
                 action, nodeid, uname);
         return -EPROTO;
     }
 
     rc = st->cmds->connect(st, "stonith-api", NULL);
     if (rc != pcmk_ok) {
         api_log(LOG_ERR, "Connection failed, could not kick (%s) node %u/%s : %s (%d)",
                 action, nodeid, uname, pcmk_strerror(rc), rc);
     } else {
         char *name = (uname == NULL)? pcmk__itoa(nodeid) : strdup(uname);
         int opts = 0;
 
         stonith__set_call_options(opts, name,
                                   st_opt_sync_call|st_opt_allow_suicide);
         if ((uname == NULL) && (nodeid > 0)) {
             stonith__set_call_options(opts, name, st_opt_cs_nodeid);
         }
         rc = st->cmds->fence(st, opts, name, action, timeout, 0);
         free(name);
 
         if (rc != pcmk_ok) {
             api_log(LOG_ERR, "Could not kick (%s) node %u/%s : %s (%d)",
                     action, nodeid, uname, pcmk_strerror(rc), rc);
         } else {
             api_log(LOG_NOTICE, "Node %u/%s kicked: %s", nodeid, uname, action);
         }
     }
 
     stonith_api_delete(st);
     return rc;
 }
 
 time_t
 stonith_api_time(uint32_t nodeid, const char *uname, bool in_progress)
 {
     int rc = pcmk_ok;
     time_t when = 0;
     stonith_t *st = stonith_api_new();
     stonith_history_t *history = NULL, *hp = NULL;
 
     if (st == NULL) {
         api_log(LOG_ERR, "Could not retrieve fence history for %u/%s: "
                 "API initialization failed", nodeid, uname);
         return when;
     }
 
     rc = st->cmds->connect(st, "stonith-api", NULL);
     if (rc != pcmk_ok) {
         api_log(LOG_NOTICE, "Connection failed: %s (%d)", pcmk_strerror(rc), rc);
     } else {
         int entries = 0;
         int progress = 0;
         int completed = 0;
         int opts = 0;
         char *name = (uname == NULL)? pcmk__itoa(nodeid) : strdup(uname);
 
         stonith__set_call_options(opts, name, st_opt_sync_call);
         if ((uname == NULL) && (nodeid > 0)) {
             stonith__set_call_options(opts, name, st_opt_cs_nodeid);
         }
         rc = st->cmds->history(st, opts, name, &history, 120);
         free(name);
 
         for (hp = history; hp; hp = hp->next) {
             entries++;
             if (in_progress) {
                 progress++;
                 if (hp->state != st_done && hp->state != st_failed) {
                     when = time(NULL);
                 }
 
             } else if (hp->state == st_done) {
                 completed++;
                 if (hp->completed > when) {
                     when = hp->completed;
                 }
             }
         }
 
         stonith_history_free(history);
 
         if(rc == pcmk_ok) {
             api_log(LOG_INFO, "Found %d entries for %u/%s: %d in progress, %d completed", entries, nodeid, uname, progress, completed);
         } else {
             api_log(LOG_ERR, "Could not retrieve fence history for %u/%s: %s (%d)", nodeid, uname, pcmk_strerror(rc), rc);
         }
     }
 
     stonith_api_delete(st);
 
     if(when) {
         api_log(LOG_INFO, "Node %u/%s last kicked at: %ld", nodeid, uname, (long int)when);
     }
     return when;
 }
 
 bool
 stonith_agent_exists(const char *agent, int timeout)
 {
     stonith_t *st = NULL;
     stonith_key_value_t *devices = NULL;
     stonith_key_value_t *dIter = NULL;
     bool rc = FALSE;
 
     if (agent == NULL) {
         return rc;
     }
 
     st = stonith_api_new();
     if (st == NULL) {
         crm_err("Could not list fence agents: API memory allocation failed");
         return FALSE;
     }
     st->cmds->list_agents(st, st_opt_sync_call, NULL, &devices, timeout == 0 ? 120 : timeout);
 
     for (dIter = devices; dIter != NULL; dIter = dIter->next) {
         if (pcmk__str_eq(dIter->value, agent, pcmk__str_none)) {
             rc = TRUE;
             break;
         }
     }
 
     stonith_key_value_freeall(devices, 1, 1);
     stonith_api_delete(st);
     return rc;
 }
 
 const char *
 stonith_action_str(const char *action)
 {
     if (action == NULL) {
         return "fencing";
     } else if (!strcmp(action, "on")) {
         return "unfencing";
     } else if (!strcmp(action, "off")) {
         return "turning off";
     } else {
         return action;
     }
 }
 
 /*!
  * \internal
  * \brief Parse a target name from one line of a target list string
  *
  * \param[in]     line    One line of a target list string
  * \parma[in]     len     String length of line
  * \param[in,out] output  List to add newly allocated target name to
  */
 static void
 parse_list_line(const char *line, int len, GList **output)
 {
     size_t i = 0;
     size_t entry_start = 0;
 
     /* Skip complaints about additional parameters device doesn't understand
      *
      * @TODO Document or eliminate the implied restriction of target names
      */
     if (strstr(line, "invalid") || strstr(line, "variable")) {
         crm_debug("Skipping list output line: %s", line);
         return;
     }
 
     // Process line content, character by character
     for (i = 0; i <= len; i++) {
 
         if (isspace(line[i]) || (line[i] == ',') || (line[i] == ';')
             || (line[i] == '\0')) {
             // We've found a separator (i.e. the end of an entry)
 
             int rc = 0;
             char *entry = NULL;
 
             if (i == entry_start) {
                 // Skip leading and sequential separators
                 entry_start = i + 1;
                 continue;
             }
 
             entry = calloc(i - entry_start + 1, sizeof(char));
             CRM_ASSERT(entry != NULL);
 
             /* Read entry, stopping at first separator
              *
              * @TODO Document or eliminate these character restrictions
              */
             rc = sscanf(line + entry_start, "%[a-zA-Z0-9_-.]", entry);
             if (rc != 1) {
                 crm_warn("Could not parse list output entry: %s "
                          CRM_XS " entry_start=%d position=%d",
                          line + entry_start, entry_start, i);
                 free(entry);
 
             } else if (pcmk__strcase_any_of(entry, "on", "off", NULL)) {
                 /* Some agents print the target status in the list output,
                  * though none are known now (the separate list-status command
                  * is used for this, but it can also print "UNKNOWN"). To handle
                  * this possibility, skip such entries.
                  *
                  * @TODO Document or eliminate the implied restriction of target
                  * names.
                  */
                 free(entry);
 
             } else {
                 // We have a valid entry
                 *output = g_list_append(*output, entry);
             }
             entry_start = i + 1;
         }
     }
 }
 
 /*!
  * \internal
  * \brief Parse a list of targets from a string
  *
  * \param[in] list_output  Target list as a string
  *
  * \return List of target names
  * \note The target list string format is flexible, to allow for user-specified
  *       lists such pcmk_host_list and the output of an agent's list action
  *       (whether direct or via the API, which escapes newlines). There may be
  *       multiple lines, separated by either a newline or an escaped newline
  *       (backslash n). Each line may have one or more target names, separated
  *       by any combination of whitespace, commas, and semi-colons. Lines
  *       containing "invalid" or "variable" will be ignored entirely. Target
  *       names "on" or "off" (case-insensitive) will be ignored. Target names
  *       may contain only alphanumeric characters, underbars (_), dashes (-),
  *       and dots (.) (if any other character occurs in the name, it and all
  *       subsequent characters in the name will be ignored).
  * \note The caller is responsible for freeing the result with
  *       g_list_free_full(result, free).
  */
 GList *
 stonith__parse_targets(const char *target_spec)
 {
     GList *targets = NULL;
 
     if (target_spec != NULL) {
         size_t out_len = strlen(target_spec);
         size_t line_start = 0; // Starting index of line being processed
 
         for (size_t i = 0; i <= out_len; ++i) {
             if ((target_spec[i] == '\n') || (target_spec[i] == '\0')
                 || ((target_spec[i] == '\\') && (target_spec[i + 1] == 'n'))) {
                 // We've reached the end of one line of output
 
                 int len = i - line_start;
 
                 if (len > 0) {
                     char *line = strndup(target_spec + line_start, len);
 
                     line[len] = '\0'; // Because it might be a newline
                     parse_list_line(line, len, &targets);
                     free(line);
                 }
                 if (target_spec[i] == '\\') {
                     ++i; // backslash-n takes up two positions
                 }
                 line_start = i + 1;
             }
         }
     }
     return targets;
 }
 
 /*!
  * \internal
  * \brief Determine if a later stonith event succeeded.
  *
  * \note Before calling this function, use stonith__sort_history() to sort the
  *       top_history argument.
  */
 gboolean
 stonith__later_succeeded(stonith_history_t *event, stonith_history_t *top_history)
 {
      gboolean ret = FALSE;
 
      for (stonith_history_t *prev_hp = top_history; prev_hp; prev_hp = prev_hp->next) {
         if (prev_hp == event) {
             break;
         }
 
          if ((prev_hp->state == st_done) &&
             pcmk__str_eq(event->target, prev_hp->target, pcmk__str_casei) &&
             pcmk__str_eq(event->action, prev_hp->action, pcmk__str_casei) &&
             pcmk__str_eq(event->delegate, prev_hp->delegate, pcmk__str_casei) &&
             (event->completed < prev_hp->completed)) {
             ret = TRUE;
             break;
         }
     }
     return ret;
 }
 
 /*!
  * \internal
  * \brief Sort the stonith-history
  *        sort by competed most current on the top
  *        pending actions lacking a completed-stamp are gathered at the top
  *
  * \param[in] history    List of stonith actions
  *
  */
 stonith_history_t *
 stonith__sort_history(stonith_history_t *history)
 {
     stonith_history_t *new = NULL, *pending = NULL, *hp, *np, *tmp;
 
     for (hp = history; hp; ) {
         tmp = hp->next;
         if ((hp->state == st_done) || (hp->state == st_failed)) {
             /* sort into new */
             if ((!new) || (hp->completed > new->completed)) {
                 hp->next = new;
                 new = hp;
             } else {
                 np = new;
                 do {
                     if ((!np->next) || (hp->completed > np->next->completed)) {
                         hp->next = np->next;
                         np->next = hp;
                         break;
                     }
                     np = np->next;
                 } while (1);
             }
         } else {
             /* put into pending */
             hp->next = pending;
             pending = hp;
         }
         hp = tmp;
     }
 
     /* pending actions don't have a completed-stamp so make them go front */
     if (pending) {
         stonith_history_t *last_pending = pending;
 
         while (last_pending->next) {
             last_pending = last_pending->next;
         }
 
         last_pending->next = new;
         new = pending;
     }
     return new;
 }
 
 /*!
  * \brief Return string equivalent of an operation state value
  *
  * \param[in] state  Fencing operation state value
  *
  * \return Human-friendly string equivalent of state
  */
 const char *
 stonith_op_state_str(enum op_state state)
 {
     switch (state) {
         case st_query:      return "querying";
         case st_exec:       return "executing";
         case st_done:       return "completed";
         case st_duplicate:  return "duplicate";
         case st_failed:     return "failed";
     }
     return "unknown";
 }
 
 stonith_history_t *
 stonith__first_matching_event(stonith_history_t *history,
                               bool (*matching_fn)(stonith_history_t *, void *),
                               void *user_data)
 {
     for (stonith_history_t *hp = history; hp; hp = hp->next) {
         if (matching_fn(hp, user_data)) {
             return hp;
         }
     }
 
     return NULL;
 }
 
 bool
 stonith__event_state_pending(stonith_history_t *history, void *user_data)
 {
     return history->state != st_failed && history->state != st_done;
 }
 
 bool
 stonith__event_state_eq(stonith_history_t *history, void *user_data)
 {
     return history->state == GPOINTER_TO_INT(user_data);
 }
 
 bool
 stonith__event_state_neq(stonith_history_t *history, void *user_data)
 {
     return history->state != GPOINTER_TO_INT(user_data);
 }
 
 void
 stonith__device_parameter_flags(uint32_t *device_flags, const char *device_name,
                                 xmlNode *metadata)
 {
     xmlXPathObjectPtr xpath = NULL;
     int max = 0;
     int lpc = 0;
 
     CRM_CHECK((device_flags != NULL) && (metadata != NULL), return);
 
     xpath = xpath_search(metadata, "//parameter");
     max = numXpathResults(xpath);
 
     if (max <= 0) {
         freeXpathObject(xpath);
         return;
     }
 
     for (lpc = 0; lpc < max; lpc++) {
         const char *parameter = NULL;
         xmlNode *match = getXpathResult(xpath, lpc);
 
         CRM_LOG_ASSERT(match != NULL);
         if (match == NULL) {
             continue;
         }
 
         parameter = crm_element_value(match, "name");
 
         if (pcmk__str_eq(parameter, "plug", pcmk__str_casei)) {
             stonith__set_device_flags(*device_flags, device_name,
                                       st_device_supports_parameter_plug);
 
         } else if (pcmk__str_eq(parameter, "port", pcmk__str_casei)) {
             stonith__set_device_flags(*device_flags, device_name,
                                       st_device_supports_parameter_port);
         }
     }
 
     freeXpathObject(xpath);
 }
 
 // Deprecated functions kept only for backward API compatibility
 
 #include <crm/fencing/compat.h>
 
 const char *
 get_stonith_provider(const char *agent, const char *provider)
 {
     return stonith_namespace2text(stonith_get_namespace(agent, provider));
 }
 
 // End deprecated API
diff --git a/tools/crm_simulate.c b/tools/crm_simulate.c
index 371c5bde03..1b6b9ef12d 100644
--- a/tools/crm_simulate.c
+++ b/tools/crm_simulate.c
@@ -1,1188 +1,1187 @@
 /*
  * Copyright 2009-2021 the Pacemaker project contributors
  *
  * The version control history for this file may have further details.
  *
  * This source code is licensed under the GNU General Public License version 2
  * or later (GPLv2+) WITHOUT ANY WARRANTY.
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <time.h>
 
 #include <sys/stat.h>
 #include <sys/param.h>
 #include <sys/types.h>
 #include <dirent.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/common/cmdline_internal.h>
 #include <crm/common/output_internal.h>
 #include <crm/common/util.h>
 #include <crm/common/iso8601.h>
 #include <crm/pengine/status.h>
 #include <pacemaker-internal.h>
 
 #define SUMMARY "crm_simulate - simulate a Pacemaker cluster's response to events"
 
 struct {
     gboolean all_actions;
     char *dot_file;
     char *graph_file;
     gchar *input_file;
     guint modified;
     GList *node_up;
     GList *node_down;
     GList *node_fail;
     GList *op_fail;
     GList *op_inject;
     gchar *output_file;
     gboolean print_pending;
     gboolean process;
     char *quorum;
     long long repeat;
     gboolean show_attrs;
     gboolean show_failcounts;
     gboolean show_scores;
     gboolean show_utilization;
     gboolean simulate;
     gboolean store;
     gchar *test_dir;
     GList *ticket_grant;
     GList *ticket_revoke;
     GList *ticket_standby;
     GList *ticket_activate;
     char *use_date;
     char *watchdog;
     char *xml_file;
 } options = {
     .print_pending = TRUE,
     .repeat = 1
 };
 
 cib_t *global_cib = NULL;
 bool action_numbers = FALSE;
 char *temp_shadow = NULL;
 extern gboolean bringing_nodes_online;
 crm_exit_t exit_code = CRM_EX_OK;
 
 #define INDENT "                                   "
 
 static pcmk__supported_format_t formats[] = {
     PCMK__SUPPORTED_FORMAT_NONE,
     PCMK__SUPPORTED_FORMAT_TEXT,
     PCMK__SUPPORTED_FORMAT_XML,
     { NULL, NULL, NULL }
 };
 
 static gboolean
 in_place_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.store = TRUE;
     options.process = TRUE;
     options.simulate = TRUE;
     return TRUE;
 }
 
 static gboolean
 live_check_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (options.xml_file) {
         free(options.xml_file);
     }
 
     options.xml_file = NULL;
     return TRUE;
 }
 
 static gboolean
 node_down_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.modified++;
     options.node_down = g_list_append(options.node_down, (gchar *) g_strdup(optarg));
     return TRUE;
 }
 
 static gboolean
 node_fail_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.modified++;
     options.node_fail = g_list_append(options.node_fail, (gchar *) g_strdup(optarg));
     return TRUE;
 }
 
 static gboolean
 node_up_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.modified++;
     bringing_nodes_online = TRUE;
     options.node_up = g_list_append(options.node_up, (gchar *) g_strdup(optarg));
     return TRUE;
 }
 
 static gboolean
 op_fail_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.process = TRUE;
     options.simulate = TRUE;
     options.op_fail = g_list_append(options.op_fail, (gchar *) g_strdup(optarg));
     return TRUE;
 }
 
 static gboolean
 op_inject_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.modified++;
     options.op_inject = g_list_append(options.op_inject, (gchar *) g_strdup(optarg));
     return TRUE;
 }
 
 static gboolean
 quorum_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (options.quorum) {
         free(options.quorum);
     }
 
     options.modified++;
     options.quorum = strdup(optarg);
     return TRUE;
 }
 
 static gboolean
 save_dotfile_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (options.dot_file) {
         free(options.dot_file);
     }
 
     options.process = TRUE;
     options.dot_file = strdup(optarg);
     return TRUE;
 }
 
 static gboolean
 save_graph_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (options.graph_file) {
         free(options.graph_file);
     }
 
     options.process = TRUE;
     options.graph_file = strdup(optarg);
     return TRUE;
 }
 
 static gboolean
 show_scores_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.process = TRUE;
     options.show_scores = TRUE;
     return TRUE;
 }
 
 static gboolean
 simulate_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.process = TRUE;
     options.simulate = TRUE;
     return TRUE;
 }
 
 static gboolean
 ticket_activate_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.modified++;
     options.ticket_activate = g_list_append(options.ticket_activate, (gchar *) g_strdup(optarg));
     return TRUE;
 }
 
 static gboolean
 ticket_grant_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.modified++;
     options.ticket_grant = g_list_append(options.ticket_grant, (gchar *) g_strdup(optarg));
     return TRUE;
 }
 
 static gboolean
 ticket_revoke_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.modified++;
     options.ticket_revoke = g_list_append(options.ticket_revoke, (gchar *) g_strdup(optarg));
     return TRUE;
 }
 
 static gboolean
 ticket_standby_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.modified++;
     options.ticket_standby = g_list_append(options.ticket_standby, (gchar *) g_strdup(optarg));
     return TRUE;
 }
 
 static gboolean
 utilization_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     options.process = TRUE;
     options.show_utilization = TRUE;
     return TRUE;
 }
 
 static gboolean
 watchdog_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (options.watchdog) {
         free(options.watchdog);
     }
 
     options.modified++;
     options.watchdog = strdup(optarg);
     return TRUE;
 }
 
 static gboolean
 xml_file_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (options.xml_file) {
         free(options.xml_file);
     }
 
     options.xml_file = strdup(optarg);
     return TRUE;
 }
 
 static gboolean
 xml_pipe_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **error) {
     if (options.xml_file) {
         free(options.xml_file);
     }
 
     options.xml_file = strdup("-");
     return TRUE;
 }
 
 static GOptionEntry operation_entries[] = {
     { "run", 'R', 0, G_OPTION_ARG_NONE, &options.process,
       "Process the supplied input and show what actions the cluster will take in response",
       NULL },
     { "simulate", 'S', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, simulate_cb,
       "Like --run, but also simulate taking those actions and show the resulting new status",
       NULL },
     { "in-place", 'X', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, in_place_cb,
       "Like --simulate, but also store the results back to the input file",
       NULL },
     { "show-attrs", 'A', 0, G_OPTION_ARG_NONE, &options.show_attrs,
       "Show node attributes",
       NULL },
     { "show-failcounts", 'c', 0, G_OPTION_ARG_NONE, &options.show_failcounts,
       "Show resource fail counts",
       NULL },
     { "show-scores", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_scores_cb,
       "Show allocation scores",
       NULL },
     { "show-utilization", 'U', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, utilization_cb,
       "Show utilization information",
       NULL },
     { "profile", 'P', 0, G_OPTION_ARG_FILENAME, &options.test_dir,
       "Process all the XML files in the named directory to create profiling data",
       "DIR" },
     { "repeat", 'N', 0, G_OPTION_ARG_INT, &options.repeat,
       "With --profile, repeat each test N times and print timings",
       "N" },
     /* Deprecated */
     { "pending", 'j', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &options.print_pending,
       "Display pending state if 'record-pending' is enabled",
       NULL },
 
     { NULL }
 };
 
 static GOptionEntry synthetic_entries[] = {
     { "node-up", 'u', 0, G_OPTION_ARG_CALLBACK, node_up_cb,
       "Simulate bringing a node online",
       "NODE" },
     { "node-down", 'd', 0, G_OPTION_ARG_CALLBACK, node_down_cb,
       "Simulate taking a node offline",
       "NODE" },
     { "node-fail", 'f', 0, G_OPTION_ARG_CALLBACK, node_fail_cb,
       "Simulate a node failing",
       "NODE" },
     { "op-inject", 'i', 0, G_OPTION_ARG_CALLBACK, op_inject_cb,
       "Generate a failure for the cluster to react to in the simulation.\n"
       INDENT "See `Operation Specification` help for more information.",
       "OPSPEC" },
     { "op-fail", 'F', 0, G_OPTION_ARG_CALLBACK, op_fail_cb,
       "If the specified task occurs during the simulation, have it fail with return code ${rc}.\n"
       INDENT "The transition will normally stop at the failed action.\n"
       INDENT "Save the result with --save-output and re-run with --xml-file.\n"
       INDENT "See `Operation Specification` help for more information.",
       "OPSPEC" },
     { "set-datetime", 't', 0, G_OPTION_ARG_STRING, &options.use_date,
       "Set date/time (ISO 8601 format, see https://en.wikipedia.org/wiki/ISO_8601)",
       "DATETIME" },
     { "quorum", 'q', 0, G_OPTION_ARG_CALLBACK, quorum_cb,
       "Set to '1' (or 'true') to indicate cluster has quorum",
       "QUORUM" },
     { "watchdog", 'w', 0, G_OPTION_ARG_CALLBACK, watchdog_cb,
       "Set to '1' (or 'true') to indicate cluster has an active watchdog device",
       "DEVICE" },
     { "ticket-grant", 'g', 0, G_OPTION_ARG_CALLBACK, ticket_grant_cb,
       "Simulate granting a ticket",
       "TICKET" },
     { "ticket-revoke", 'r', 0, G_OPTION_ARG_CALLBACK, ticket_revoke_cb,
       "Simulate revoking a ticket",
       "TICKET" },
     { "ticket-standby", 'b', 0, G_OPTION_ARG_CALLBACK, ticket_standby_cb,
       "Simulate making a ticket standby",
       "TICKET" },
     { "ticket-activate", 'e', 0, G_OPTION_ARG_CALLBACK, ticket_activate_cb,
       "Simulate activating a ticket",
       "TICKET" },
 
     { NULL }
 };
 
 static GOptionEntry artifact_entries[] = {
     { "save-input", 'I', 0, G_OPTION_ARG_FILENAME, &options.input_file,
       "Save the input configuration to the named file",
       "FILE" },
     { "save-output", 'O', 0, G_OPTION_ARG_FILENAME, &options.output_file,
       "Save the output configuration to the named file",
       "FILE" },
     { "save-graph", 'G', 0, G_OPTION_ARG_CALLBACK, save_graph_cb,
       "Save the transition graph (XML format) to the named file",
       "FILE" },
     { "save-dotfile", 'D', 0, G_OPTION_ARG_CALLBACK, save_dotfile_cb,
       "Save the transition graph (DOT format) to the named file",
       "FILE" },
     { "all-actions", 'a', 0, G_OPTION_ARG_NONE, &options.all_actions,
       "Display all possible actions in DOT graph (even if not part of transition)",
       NULL },
 
     { NULL }
 };
 
 static GOptionEntry source_entries[] = {
     { "live-check", 'L', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, live_check_cb,
       "Connect to CIB manager and use the current CIB contents as input",
       NULL },
     { "xml-file", 'x', 0, G_OPTION_ARG_CALLBACK, xml_file_cb,
       "Retrieve XML from the named file",
       "FILE" },
     { "xml-pipe", 'p', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, xml_pipe_cb,
       "Retrieve XML from stdin",
       NULL },
 
     { NULL }
 };
 
 static void
 get_date(pe_working_set_t *data_set, bool print_original, char *use_date)
 {
     pcmk__output_t *out = data_set->priv;
     time_t original_date = 0;
 
     crm_element_value_epoch(data_set->input, "execution-date", &original_date);
 
     if (use_date) {
         data_set->now = crm_time_new(use_date);
         out->info(out, "Setting effective cluster time: %s", use_date);
         crm_time_log(LOG_NOTICE, "Pretending 'now' is", data_set->now,
                      crm_time_log_date | crm_time_log_timeofday);
 
 
     } else if (original_date) {
 
         data_set->now = crm_time_new(NULL);
         crm_time_set_timet(data_set->now, &original_date);
 
         if (print_original) {
             char *when = crm_time_as_string(data_set->now,
                             crm_time_log_date|crm_time_log_timeofday);
 
             out->info(out, "Using the original execution date of: %s", when);
             free(when);
         }
     }
 }
 
 static void
 print_cluster_status(pe_working_set_t * data_set, unsigned int print_opts)
 {
     pcmk__output_t *out = data_set->priv;
     int rc = pcmk_rc_no_output;
     GList *all = NULL;
 
     all = g_list_prepend(all, strdup("*"));
 
     rc = out->message(out, "node-list", data_set->nodes, all, all, print_opts,
                       FALSE, FALSE, FALSE);
     PCMK__OUTPUT_SPACER_IF(out, rc == pcmk_rc_ok);
     rc = out->message(out, "resource-list", data_set, print_opts, FALSE, TRUE,
                       FALSE, FALSE, all, all, FALSE);
 
     if (options.show_attrs) {
         out->message(out, "node-attribute-list", data_set,
                      0, rc == pcmk_rc_ok, FALSE, FALSE, FALSE, all, all);
     }
 
     if (options.show_failcounts) {
         out->message(out, "failed-action-list", data_set, all, all,
                      rc == pcmk_rc_ok);
     }
 
     g_list_free_full(all, free);
 }
 
 static char *
 create_action_name(pe_action_t *action)
 {
     char *action_name = NULL;
     const char *prefix = "";
     const char *action_host = NULL;
     const char *clone_name = NULL;
     const char *task = action->task;
 
     if (action->node) {
         action_host = action->node->details->uname;
     } else if (!pcmk_is_set(action->flags, pe_action_pseudo)) {
         action_host = "<none>";
     }
 
     if (pcmk__str_eq(action->task, RSC_CANCEL, pcmk__str_casei)) {
         prefix = "Cancel ";
         task = action->cancel_task;
     }
 
     if (action->rsc && action->rsc->clone_name) {
         clone_name = action->rsc->clone_name;
     }
 
     if (clone_name) {
         char *key = NULL;
         guint interval_ms = 0;
 
         if (pcmk__guint_from_hash(action->meta,
                                   XML_LRM_ATTR_INTERVAL_MS, 0,
                                   &interval_ms) != pcmk_rc_ok) {
             interval_ms = 0;
         }
 
         if (pcmk__strcase_any_of(action->task, RSC_NOTIFY, RSC_NOTIFIED, NULL)) {
             const char *n_type = g_hash_table_lookup(action->meta, "notify_key_type");
             const char *n_task = g_hash_table_lookup(action->meta, "notify_key_operation");
 
             CRM_ASSERT(n_type != NULL);
             CRM_ASSERT(n_task != NULL);
             key = pcmk__notify_key(clone_name, n_type, n_task);
 
         } else {
             key = pcmk__op_key(clone_name, task, interval_ms);
         }
 
         if (action_host) {
             action_name = crm_strdup_printf("%s%s %s", prefix, key, action_host);
         } else {
             action_name = crm_strdup_printf("%s%s", prefix, key);
         }
         free(key);
 
     } else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {
         const char *op = g_hash_table_lookup(action->meta, "stonith_action");
 
         action_name = crm_strdup_printf("%s%s '%s' %s", prefix, action->task, op, action_host);
 
     } else if (action->rsc && action_host) {
         action_name = crm_strdup_printf("%s%s %s", prefix, action->uuid, action_host);
 
     } else if (action_host) {
         action_name = crm_strdup_printf("%s%s %s", prefix, action->task, action_host);
 
     } else {
         action_name = crm_strdup_printf("%s", action->uuid);
     }
 
     if (action_numbers) { // i.e. verbose
         char *with_id = crm_strdup_printf("%s (%d)", action_name, action->id);
 
         free(action_name);
         action_name = with_id;
     }
     return action_name;
 }
 
 static bool
 create_dotfile(pe_working_set_t * data_set, const char *dot_file, gboolean all_actions,
                GError **error)
 {
     GList *gIter = NULL;
     FILE *dot_strm = fopen(dot_file, "w");
 
     if (dot_strm == NULL) {
         g_set_error(error, PCMK__RC_ERROR, errno,
                     "Could not open %s for writing: %s", dot_file,
                     pcmk_rc_str(errno));
         return false;
     }
 
     fprintf(dot_strm, " digraph \"g\" {\n");
     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
         const char *style = "dashed";
         const char *font = "black";
         const char *color = "black";
         char *action_name = create_action_name(action);
 
         crm_trace("Action %d: %s %s %p", action->id, action_name, action->uuid, action);
 
         if (pcmk_is_set(action->flags, pe_action_pseudo)) {
             font = "orange";
         }
 
         if (pcmk_is_set(action->flags, pe_action_dumped)) {
             style = "bold";
             color = "green";
 
         } else if ((action->rsc != NULL)
                    && !pcmk_is_set(action->rsc->flags, pe_rsc_managed)) {
             color = "red";
             font = "purple";
             if (all_actions == FALSE) {
                 goto do_not_write;
             }
 
         } else if (pcmk_is_set(action->flags, pe_action_optional)) {
             color = "blue";
             if (all_actions == FALSE) {
                 goto do_not_write;
             }
 
         } else {
             color = "red";
             CRM_CHECK(!pcmk_is_set(action->flags, pe_action_runnable), ;);
         }
 
         pe__set_action_flags(action, pe_action_dumped);
         crm_trace("\"%s\" [ style=%s color=\"%s\" fontcolor=\"%s\"]",
                 action_name, style, color, font);
         fprintf(dot_strm, "\"%s\" [ style=%s color=\"%s\" fontcolor=\"%s\"]\n",
                 action_name, style, color, font);
   do_not_write:
         free(action_name);
     }
 
     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
         pe_action_t *action = (pe_action_t *) gIter->data;
 
         GList *gIter2 = NULL;
 
         for (gIter2 = action->actions_before; gIter2 != NULL; gIter2 = gIter2->next) {
             pe_action_wrapper_t *before = (pe_action_wrapper_t *) gIter2->data;
 
             char *before_name = NULL;
             char *after_name = NULL;
             const char *style = "dashed";
             gboolean optional = TRUE;
 
             if (before->state == pe_link_dumped) {
                 optional = FALSE;
                 style = "bold";
             } else if (pcmk_is_set(action->flags, pe_action_pseudo)
                        && (before->type & pe_order_stonith_stop)) {
                 continue;
             } else if (before->type == pe_order_none) {
                 continue;
             } else if (pcmk_is_set(before->action->flags, pe_action_dumped)
                        && pcmk_is_set(action->flags, pe_action_dumped)
                        && before->type != pe_order_load) {
                 optional = FALSE;
             }
 
             if (all_actions || optional == FALSE) {
                 before_name = create_action_name(before->action);
                 after_name = create_action_name(action);
                 crm_trace("\"%s\" -> \"%s\" [ style = %s]",
                         before_name, after_name, style);
                 fprintf(dot_strm, "\"%s\" -> \"%s\" [ style = %s]\n",
                         before_name, after_name, style);
                 free(before_name);
                 free(after_name);
             }
         }
     }
 
     fprintf(dot_strm, "}\n");
     fflush(dot_strm);
     fclose(dot_strm);
     return true;
 }
 
 static int
 setup_input(const char *input, const char *output, GError **error)
 {
     int rc = pcmk_rc_ok;
     cib_t *cib_conn = NULL;
     xmlNode *cib_object = NULL;
     char *local_output = NULL;
 
     if (input == NULL) {
         /* Use live CIB */
         cib_conn = cib_new();
         rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
         rc = pcmk_legacy2rc(rc);
 
         if (rc == pcmk_rc_ok) {
             rc = cib_conn->cmds->query(cib_conn, NULL, &cib_object, cib_scope_local | cib_sync_call);
         }
 
         cib_conn->cmds->signoff(cib_conn);
         cib_delete(cib_conn);
         cib_conn = NULL;
 
         if (rc != pcmk_rc_ok) {
             rc = pcmk_legacy2rc(rc);
             g_set_error(error, PCMK__RC_ERROR, rc,
                         "Live CIB query failed: %s (%d)", pcmk_rc_str(rc), rc);
             return rc;
 
         } else if (cib_object == NULL) {
             g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_NOINPUT,
                         "Live CIB query failed: empty result");
             return pcmk_rc_no_input;
         }
 
     } else if (pcmk__str_eq(input, "-", pcmk__str_casei)) {
         cib_object = filename2xml(NULL);
 
     } else {
         cib_object = filename2xml(input);
     }
 
     if (get_object_root(XML_CIB_TAG_STATUS, cib_object) == NULL) {
         create_xml_node(cib_object, XML_CIB_TAG_STATUS);
     }
 
     if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
         free_xml(cib_object);
         return pcmk_rc_transform_failed;
     }
 
     if (validate_xml(cib_object, NULL, FALSE) != TRUE) {
         free_xml(cib_object);
         return pcmk_rc_schema_validation;
     }
 
     if (output == NULL) {
         char *pid = pcmk__getpid_s();
 
         local_output = get_shadow_file(pid);
         temp_shadow = strdup(local_output);
         output = local_output;
         free(pid);
     }
 
     rc = write_xml_file(cib_object, output, FALSE);
     free_xml(cib_object);
     cib_object = NULL;
 
     if (rc < 0) {
         rc = pcmk_legacy2rc(rc);
         g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_CANTCREAT,
                     "Could not create '%s': %s", output, pcmk_rc_str(rc));
         return rc;
     } else {
         setenv("CIB_file", output, 1);
         free(local_output);
         return pcmk_rc_ok;
     }
 }
 
 static void
 profile_one(const char *xml_file, long long repeat, pe_working_set_t *data_set, char *use_date)
 {
     pcmk__output_t *out = data_set->priv;
     xmlNode *cib_object = NULL;
     clock_t start = 0;
     clock_t end;
 
     cib_object = filename2xml(xml_file);
     start = clock();
 
     if (get_object_root(XML_CIB_TAG_STATUS, cib_object) == NULL) {
         create_xml_node(cib_object, XML_CIB_TAG_STATUS);
     }
 
 
     if (cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
         free_xml(cib_object);
         return;
     }
 
     if (validate_xml(cib_object, NULL, FALSE) != TRUE) {
         free_xml(cib_object);
         return;
     }
 
     for (int i = 0; i < repeat; ++i) {
         xmlNode *input = (repeat == 1)? cib_object : copy_xml(cib_object);
 
         data_set->input = input;
         get_date(data_set, false, use_date);
         pcmk__schedule_actions(data_set, input, NULL);
         pe_reset_working_set(data_set);
     }
 
     end = clock();
     out->message(out, "profile", xml_file, start, end);
 }
 
 #ifndef FILENAME_MAX
 #  define FILENAME_MAX 512
 #endif
 
 static void
 profile_all(const char *dir, long long repeat, pe_working_set_t *data_set, char *use_date)
 {
     pcmk__output_t *out = data_set->priv;
     struct dirent **namelist;
 
     int file_num = scandir(dir, &namelist, 0, alphasort);
 
     if (file_num > 0) {
         struct stat prop;
         char buffer[FILENAME_MAX];
 
         out->begin_list(out, NULL, NULL, "Timings");
 
         while (file_num--) {
             if ('.' == namelist[file_num]->d_name[0]) {
                 free(namelist[file_num]);
                 continue;
 
             } else if (!pcmk__ends_with_ext(namelist[file_num]->d_name,
                                             ".xml")) {
                 free(namelist[file_num]);
                 continue;
             }
             snprintf(buffer, sizeof(buffer), "%s/%s", dir, namelist[file_num]->d_name);
             if (stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) {
                 profile_one(buffer, repeat, data_set, use_date);
             }
             free(namelist[file_num]);
         }
         free(namelist);
 
         out->end_list(out);
     }
 }
 
 PCMK__OUTPUT_ARGS("profile", "const char *", "clock_t", "clock_t")
 static int
 profile_default(pcmk__output_t *out, va_list args) {
     const char *xml_file = va_arg(args, const char *);
     clock_t start = va_arg(args, clock_t);
     clock_t end = va_arg(args, clock_t);
 
     out->list_item(out, NULL, "Testing %s ... %.2f secs", xml_file,
                    (end - start) / (float) CLOCKS_PER_SEC);
 
     return pcmk_rc_ok;
 }
 
 PCMK__OUTPUT_ARGS("profile", "const char *", "clock_t", "clock_t")
 static int
 profile_xml(pcmk__output_t *out, va_list args) {
     const char *xml_file = va_arg(args, const char *);
     clock_t start = va_arg(args, clock_t);
     clock_t end = va_arg(args, clock_t);
 
     char *duration = pcmk__ftoa((end - start) / (float) CLOCKS_PER_SEC);
 
     pcmk__output_create_xml_node(out, "timing",
                                  "file", xml_file,
                                  "duration", duration,
                                  NULL);
 
     free(duration);
     return pcmk_rc_ok;
 }
 
 static pcmk__message_entry_t fmt_functions[] = {
     { "profile", "default", profile_default, },
     { "profile", "xml", profile_xml },
 
     { NULL }
 };
 
 static void
 crm_simulate_register_messages(pcmk__output_t *out) {
     pcmk__register_messages(out, fmt_functions);
 }
 
 static GOptionContext *
 build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
     GOptionContext *context = NULL;
 
     GOptionEntry extra_prog_entries[] = {
         { "quiet", 'Q', 0, G_OPTION_ARG_NONE, &(args->quiet),
           "Display only essential output",
           NULL },
 
         { NULL }
     };
 
     const char *description = "Operation Specification:\n\n"
                               "The OPSPEC in any command line option is of the form\n"
                               "${resource}_${task}_${interval_in_ms}@${node}=${rc}\n"
                               "(memcached_monitor_20000@bart.example.com=7, for example).\n"
                               "${rc} is an OCF return code.  For more information on these\n"
                               "return codes, refer to https://clusterlabs.org/pacemaker/doc/en-US/Pacemaker/2.0/html/Pacemaker_Administration/s-ocf-return-codes.html\n\n"
                               "Examples:\n\n"
                               "Pretend a recurring monitor action found memcached stopped on node\n"
                               "fred.example.com and, during recovery, that the memcached stop\n"
                               "action failed:\n\n"
                               "\tcrm_simulate -LS --op-inject memcached:0_monitor_20000@bart.example.com=7 "
                               "--op-fail memcached:0_stop_0@fred.example.com=1 --save-output /tmp/memcached-test.xml\n\n"
                               "Now see what the reaction to the stop failed would be:\n\n"
                               "\tcrm_simulate -S --xml-file /tmp/memcached-test.xml\n\n";
 
     context = pcmk__build_arg_context(args, "text (default), xml", group, NULL);
     pcmk__add_main_args(context, extra_prog_entries);
     g_option_context_set_description(context, description);
 
     pcmk__add_arg_group(context, "operations", "Operations:",
                         "Show operations options", operation_entries);
     pcmk__add_arg_group(context, "synthetic", "Synthetic Cluster Events:",
                         "Show synthetic cluster event options", synthetic_entries);
     pcmk__add_arg_group(context, "artifact", "Artifact Options:",
                         "Show artifact options", artifact_entries);
     pcmk__add_arg_group(context, "source", "Data Source:",
                         "Show data source options", source_entries);
 
     return context;
 }
 
 int
 main(int argc, char **argv)
 {
     int printed = pcmk_rc_no_output;
     int rc = pcmk_rc_ok;
     pe_working_set_t *data_set = NULL;
     pcmk__output_t *out = NULL;
     xmlNode *input = NULL;
 
     GError *error = NULL;
 
     GOptionGroup *output_group = NULL;
     pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
     gchar **processed_args = pcmk__cmdline_preproc(argv, "bdefgiqrtuwxDFGINO");
     GOptionContext *context = build_arg_context(args, &output_group);
 
     /* This must come before g_option_context_parse_strv. */
     options.xml_file = strdup("-");
 
     pcmk__register_formats(output_group, formats);
     if (!g_option_context_parse_strv(context, &processed_args, &error)) {
         exit_code = CRM_EX_USAGE;
         goto done;
     }
 
     pcmk__cli_init_logging("crm_simulate", args->verbosity);
 
     rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
     if (rc != pcmk_rc_ok) {
         fprintf(stderr, "Error creating output format %s: %s\n",
                 args->output_ty, pcmk_rc_str(rc));
         exit_code = CRM_EX_ERROR;
         goto done;
     }
 
     if (pcmk__str_eq(args->output_ty, "text", pcmk__str_null_matches) &&
         !options.show_scores && !options.show_utilization) {
         pcmk__force_args(context, &error, "%s --text-fancy", g_get_prgname());
     } else if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_none)) {
         pcmk__force_args(context, &error, "%s --xml-simple-list --xml-substitute", g_get_prgname());
     }
 
     crm_simulate_register_messages(out);
     pe__register_messages(out);
     pcmk__register_lib_messages(out);
 
     out->quiet = args->quiet;
 
     if (args->version) {
         out->version(out, false);
         goto done;
     }
 
     if (args->verbosity > 0) {
 #ifdef PCMK__COMPAT_2_0
         /* Redirect stderr to stdout so we can grep the output */
         close(STDERR_FILENO);
         dup2(STDOUT_FILENO, STDERR_FILENO);
 #endif
         action_numbers = TRUE;
     }
 
     data_set = pe_new_working_set();
     if (data_set == NULL) {
         rc = ENOMEM;
         g_set_error(&error, PCMK__RC_ERROR, rc, "Could not allocate working set");
         goto done;
     }
 
     if (options.show_scores) {
         pe__set_working_set_flags(data_set, pe_flag_show_scores);
     }
     if (options.show_utilization) {
         pe__set_working_set_flags(data_set, pe_flag_show_utilization);
     }
     pe__set_working_set_flags(data_set, pe_flag_no_compat);
 
     if (options.test_dir != NULL) {
         data_set->priv = out;
         profile_all(options.test_dir, options.repeat, data_set, options.use_date);
         rc = pcmk_rc_ok;
         goto done;
     }
 
     rc = setup_input(options.xml_file, options.store ? options.xml_file : options.output_file, &error);
     if (rc != pcmk_rc_ok) {
         goto done;
     }
 
     global_cib = cib_new();
     rc = global_cib->cmds->signon(global_cib, crm_system_name, cib_command);
     if (rc != pcmk_rc_ok) {
         rc = pcmk_legacy2rc(rc);
         g_set_error(&error, PCMK__RC_ERROR, rc,
                     "Could not connect to the CIB: %s", pcmk_rc_str(rc));
         goto done;
     }
 
     rc = global_cib->cmds->query(global_cib, NULL, &input, cib_sync_call | cib_scope_local);
     if (rc != pcmk_rc_ok) {
         rc = pcmk_legacy2rc(rc);
         g_set_error(&error, PCMK__RC_ERROR, rc,
                     "Could not get local CIB: %s", pcmk_rc_str(rc));
         goto done;
     }
 
     data_set->input = input;
     data_set->priv = out;
     get_date(data_set, true, options.use_date);
     if(options.xml_file) {
         pe__set_working_set_flags(data_set, pe_flag_sanitized);
     }
     if (options.show_scores) {
         pe__set_working_set_flags(data_set, pe_flag_show_scores);
     }
     if (options.show_utilization) {
         pe__set_working_set_flags(data_set, pe_flag_show_utilization);
     }
     cluster_status(data_set);
 
     if (!out->is_quiet(out)) {
         unsigned int opts = options.print_pending ? pe_print_pending : 0;
 
         if (pcmk_is_set(data_set->flags, pe_flag_maintenance_mode)) {
             printed = out->message(out, "maint-mode", data_set->flags);
         }
 
         if (data_set->disabled_resources || data_set->blocked_resources) {
             PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
             printed = out->info(out, "%d of %d resource instances DISABLED and %d BLOCKED "
                                 "from further action due to failure",
                                 data_set->disabled_resources, data_set->ninstances,
                                 data_set->blocked_resources);
         }
 
         PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
         /* Most formatted output headers use caps for each word, but this one
          * only has the first word capitalized for compatibility with pcs.
          */
         out->begin_list(out, NULL, NULL, "Current cluster status");
         print_cluster_status(data_set, opts);
         out->end_list(out);
         printed = pcmk_rc_ok;
     }
 
     if (options.modified) {
         PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
         modify_configuration(data_set, global_cib, options.quorum, options.watchdog, options.node_up,
                              options.node_down, options.node_fail, options.op_inject,
                              options.ticket_grant, options.ticket_revoke, options.ticket_standby,
                              options.ticket_activate);
         printed = pcmk_rc_ok;
 
         rc = global_cib->cmds->query(global_cib, NULL, &input, cib_sync_call);
         if (rc != pcmk_rc_ok) {
             rc = pcmk_legacy2rc(rc);
             g_set_error(&error, PCMK__RC_ERROR, rc,
                         "Could not get modified CIB: %s", pcmk_rc_str(rc));
             goto done;
         }
 
         cleanup_calculations(data_set);
         data_set->input = input;
         data_set->priv = out;
         get_date(data_set, true, options.use_date);
 
         if(options.xml_file) {
             pe__set_working_set_flags(data_set, pe_flag_sanitized);
         }
         if (options.show_scores) {
             pe__set_working_set_flags(data_set, pe_flag_show_scores);
         }
         if (options.show_utilization) {
             pe__set_working_set_flags(data_set, pe_flag_show_utilization);
         }
         cluster_status(data_set);
     }
 
     if (options.input_file != NULL) {
         rc = write_xml_file(input, options.input_file, FALSE);
         if (rc < 0) {
             rc = pcmk_legacy2rc(rc);
             g_set_error(&error, PCMK__RC_ERROR, rc,
                         "Could not create '%s': %s", options.input_file, pcmk_rc_str(rc));
             goto done;
         }
     }
 
     if (options.process || options.simulate) {
         crm_time_t *local_date = NULL;
         pcmk__output_t *logger_out = NULL;
 
         if (pcmk_all_flags_set(data_set->flags, pe_flag_show_scores|pe_flag_show_utilization)) {
             PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
             out->begin_list(out, NULL, NULL, "Allocation Scores and Utilization Information");
             printed = pcmk_rc_ok;
         } else if (pcmk_is_set(data_set->flags, pe_flag_show_scores)) {
             PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
             out->begin_list(out, NULL, NULL, "Allocation Scores");
             printed = pcmk_rc_ok;
         } else if (pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
             PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
             out->begin_list(out, NULL, NULL, "Utilization Information");
             printed = pcmk_rc_ok;
         } else {
             logger_out = pcmk__new_logger();
             if (logger_out == NULL) {
                 goto done;
             }
 
             data_set->priv = logger_out;
         }
 
         pcmk__schedule_actions(data_set, input, local_date);
 
         if (logger_out == NULL) {
             out->end_list(out);
         } else {
             logger_out->finish(logger_out, CRM_EX_OK, true, NULL);
             pcmk__output_free(logger_out);
             data_set->priv = out;
         }
 
         input = NULL;           /* Don't try and free it twice */
 
         if (options.graph_file != NULL) {
             write_xml_file(data_set->graph, options.graph_file, FALSE);
         }
 
         if (options.dot_file != NULL) {
             if (!create_dotfile(data_set, options.dot_file, options.all_actions, &error)) {
                 goto done;
             }
         }
 
         if (!out->is_quiet(out)) {
             GList *gIter = NULL;
 
             PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
             out->begin_list(out, NULL, NULL, "Transition Summary");
 
             LogNodeActions(data_set);
             for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
                 pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 
                 LogActions(rsc, data_set);
             }
 
             out->end_list(out);
             printed = pcmk_rc_ok;
         }
     }
 
     rc = pcmk_rc_ok;
 
     if (options.simulate) {
         PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
         if (run_simulation(data_set, global_cib, options.op_fail) != pcmk_rc_ok) {
             rc = pcmk_rc_error;
         }
 
         printed = pcmk_rc_ok;
 
         if (!out->is_quiet(out)) {
             get_date(data_set, true, options.use_date);
 
             PCMK__OUTPUT_SPACER_IF(out, printed == pcmk_rc_ok);
             out->begin_list(out, NULL, NULL, "Revised Cluster Status");
-            printed = pcmk_rc_ok;
 
             if (options.show_scores) {
                 pe__set_working_set_flags(data_set, pe_flag_show_scores);
             }
             if (options.show_utilization) {
                 pe__set_working_set_flags(data_set, pe_flag_show_utilization);
             }
 
             cluster_status(data_set);
             print_cluster_status(data_set, 0);
 
             out->end_list(out);
         }
     }
 
   done:
     pcmk__output_and_clear_error(error, NULL);
 
     /* There sure is a lot to free in options. */
     free(options.dot_file);
     free(options.graph_file);
     g_free(options.input_file);
     g_list_free_full(options.node_up, g_free);
     g_list_free_full(options.node_down, g_free);
     g_list_free_full(options.node_fail, g_free);
     g_list_free_full(options.op_fail, g_free);
     g_list_free_full(options.op_inject, g_free);
     g_free(options.output_file);
     free(options.quorum);
     g_free(options.test_dir);
     g_list_free_full(options.ticket_grant, g_free);
     g_list_free_full(options.ticket_revoke, g_free);
     g_list_free_full(options.ticket_standby, g_free);
     g_list_free_full(options.ticket_activate, g_free);
     free(options.use_date);
     free(options.watchdog);
     free(options.xml_file);
 
     pcmk__free_arg_context(context);
     g_strfreev(processed_args);
 
     if (data_set) {
         pe_free_working_set(data_set);
     }
 
     if (global_cib) {
         global_cib->cmds->signoff(global_cib);
         cib_delete(global_cib);
     }
 
     fflush(stderr);
 
     if (temp_shadow) {
         unlink(temp_shadow);
         free(temp_shadow);
     }
 
     if (rc != pcmk_rc_ok) {
         exit_code = pcmk_rc2exitc(rc);
     }
 
     if (out != NULL) {
         out->finish(out, exit_code, true, NULL);
         pcmk__output_free(out);
     }
 
     crm_exit(exit_code);
 }