diff --git a/cib/callbacks.c b/cib/callbacks.c
index f9f3f04113..91c654e7d6 100644
--- a/cib/callbacks.c
+++ b/cib/callbacks.c
@@ -1,1572 +1,1577 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/cluster/internal.h>
 
 #include <crm/common/xml.h>
 
 #include <cibio.h>
 #include <callbacks.h>
 #include <cibmessages.h>
 #include <notify.h>
 #include "common.h"
 
 extern GMainLoop *mainloop;
 extern gboolean cib_shutdown_flag;
 extern gboolean stand_alone;
 extern const char *cib_root;
 
 typedef struct cib_local_notify_s {
     xmlNode *notify_src;
     char *client_id;
     gboolean from_peer;
     gboolean sync_reply;
 } cib_local_notify_t;
 
 qb_ipcs_service_t *ipcs_ro = NULL;
 qb_ipcs_service_t *ipcs_rw = NULL;
 qb_ipcs_service_t *ipcs_shm = NULL;
 
 extern crm_cluster_t crm_cluster;
 
 extern int cib_update_counter(xmlNode * xml_obj, const char *field, gboolean reset);
 
 extern void GHFunc_count_peers(gpointer key, gpointer value, gpointer user_data);
 
 gint cib_GCompareFunc(gconstpointer a, gconstpointer b);
 gboolean can_write(int flags);
 void send_cib_replace(const xmlNode * sync_request, const char *host);
 void cib_process_request(xmlNode * request, gboolean privileged, gboolean force_synchronous,
                          gboolean from_peer, crm_client_t * cib_client);
 
 extern GHashTable *local_notify_queue;
 
 int next_client_id = 0;
 extern const char *cib_our_uname;
 extern unsigned long cib_num_ops, cib_num_local, cib_num_updates, cib_num_fail;
 extern unsigned long cib_bad_connects, cib_num_timeouts;
 extern int cib_status;
 
 int cib_process_command(xmlNode * request, xmlNode ** reply,
                         xmlNode ** cib_diff, gboolean privileged);
 
 gboolean cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size,
                              gboolean privileged);
 
 static gboolean cib_legacy_mode(void)
 {
     static gboolean init = TRUE;
     static gboolean legacy = FALSE;
 
     if(init) {
         init = FALSE;
         legacy = daemon_option_enabled("legacy", "cib");
         if(legacy) {
             crm_notice("Enabled legacy mode");
         }
     }
     return legacy;
 }
 
 static int32_t
 cib_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
 {
     crm_trace("Connection %p", c);
     if (cib_shutdown_flag) {
         crm_info("Ignoring new client [%d] during shutdown", crm_ipcs_client_pid(c));
         return -EPERM;
     }
 
     if (crm_client_new(c, uid, gid) == NULL) {
         return -EIO;
     }
     return 0;
 }
 
 static void
 cib_ipc_created(qb_ipcs_connection_t * c)
 {
     crm_trace("Connection %p", c);
 }
 
 static int32_t
 cib_ipc_dispatch_rw(qb_ipcs_connection_t * c, void *data, size_t size)
 {
     crm_client_t *client = crm_client_get(c);
 
     crm_trace("%p message from %s", c, client->id);
     return cib_common_callback(c, data, size, TRUE);
 }
 
 static int32_t
 cib_ipc_dispatch_ro(qb_ipcs_connection_t * c, void *data, size_t size)
 {
     crm_client_t *client = crm_client_get(c);
 
     crm_trace("%p message from %s", c, client->id);
     return cib_common_callback(c, data, size, FALSE);
 }
 
 /* Error code means? */
 static int32_t
 cib_ipc_closed(qb_ipcs_connection_t * c)
 {
     crm_client_t *client = crm_client_get(c);
 
     if (client == NULL) {
         return 0;
     }
     crm_trace("Connection %p", c);
     crm_client_destroy(client);
     return 0;
 }
 
 static void
 cib_ipc_destroy(qb_ipcs_connection_t * c)
 {
     crm_trace("Connection %p", c);
     cib_ipc_closed(c);
     if (cib_shutdown_flag) {
         cib_shutdown(0);
     }
 }
 
 struct qb_ipcs_service_handlers ipc_ro_callbacks = {
     .connection_accept = cib_ipc_accept,
     .connection_created = cib_ipc_created,
     .msg_process = cib_ipc_dispatch_ro,
     .connection_closed = cib_ipc_closed,
     .connection_destroyed = cib_ipc_destroy
 };
 
 struct qb_ipcs_service_handlers ipc_rw_callbacks = {
     .connection_accept = cib_ipc_accept,
     .connection_created = cib_ipc_created,
     .msg_process = cib_ipc_dispatch_rw,
     .connection_closed = cib_ipc_closed,
     .connection_destroyed = cib_ipc_destroy
 };
 
 void
 cib_common_callback_worker(uint32_t id, uint32_t flags, xmlNode * op_request,
                            crm_client_t * cib_client, gboolean privileged)
 {
     const char *op = crm_element_value(op_request, F_CIB_OPERATION);
 
     if (crm_str_eq(op, CRM_OP_REGISTER, TRUE)) {
         if (flags & crm_ipc_client_response) {
             xmlNode *ack = create_xml_node(NULL, __FUNCTION__);
 
             crm_xml_add(ack, F_CIB_OPERATION, CRM_OP_REGISTER);
             crm_xml_add(ack, F_CIB_CLIENTID, cib_client->id);
             crm_ipcs_send(cib_client, id, ack, flags);
             cib_client->request_id = 0;
             free_xml(ack);
         }
         return;
 
     } else if (crm_str_eq(op, T_CIB_NOTIFY, TRUE)) {
         /* Update the notify filters for this client */
         int on_off = 0;
         long long bit = 0;
         const char *type = crm_element_value(op_request, F_CIB_NOTIFY_TYPE);
 
         crm_element_value_int(op_request, F_CIB_NOTIFY_ACTIVATE, &on_off);
 
         crm_debug("Setting %s callbacks for %s (%s): %s",
                   type, cib_client->name, cib_client->id, on_off ? "on" : "off");
 
         if (safe_str_eq(type, T_CIB_POST_NOTIFY)) {
             bit = cib_notify_post;
 
         } else if (safe_str_eq(type, T_CIB_PRE_NOTIFY)) {
             bit = cib_notify_pre;
 
         } else if (safe_str_eq(type, T_CIB_UPDATE_CONFIRM)) {
             bit = cib_notify_confirm;
 
         } else if (safe_str_eq(type, T_CIB_DIFF_NOTIFY)) {
             bit = cib_notify_diff;
 
         } else if (safe_str_eq(type, T_CIB_REPLACE_NOTIFY)) {
             bit = cib_notify_replace;
         }
 
         if (on_off) {
             set_bit(cib_client->options, bit);
         } else {
             clear_bit(cib_client->options, bit);
         }
 
         if (flags & crm_ipc_client_response) {
             /* TODO - include rc */
             crm_ipcs_send_ack(cib_client, id, flags, "ack", __FUNCTION__, __LINE__);
         }
         return;
     }
 
     cib_process_request(op_request, FALSE, privileged, FALSE, cib_client);
 }
 
 int32_t
 cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean privileged)
 {
     uint32_t id = 0;
     uint32_t flags = 0;
     int call_options = 0;
     crm_client_t *cib_client = crm_client_get(c);
     xmlNode *op_request = crm_ipcs_recv(cib_client, data, size, &id, &flags);
 
     if (op_request) {
         crm_element_value_int(op_request, F_CIB_CALLOPTS, &call_options);
     }
 
     if (op_request == NULL) {
         crm_trace("Invalid message from %p", c);
         crm_ipcs_send_ack(cib_client, id, flags, "nack", __FUNCTION__, __LINE__);
         return 0;
 
     } else if(cib_client == NULL) {
         crm_trace("Invalid client %p", c);
         return 0;
     }
 
     if (is_set(call_options, cib_sync_call)) {
         CRM_ASSERT(flags & crm_ipc_client_response);
         CRM_LOG_ASSERT(cib_client->request_id == 0);    /* This means the client has two synchronous events in-flight */
         cib_client->request_id = id;    /* Reply only to the last one */
     }
 
     if (cib_client->name == NULL) {
         const char *value = crm_element_value(op_request, F_CIB_CLIENTNAME);
 
         if (value == NULL) {
             cib_client->name = crm_itoa(cib_client->pid);
         } else {
             cib_client->name = strdup(value);
         }
     }
 
     crm_xml_add(op_request, F_CIB_CLIENTID, cib_client->id);
     crm_xml_add(op_request, F_CIB_CLIENTNAME, cib_client->name);
 
 #if ENABLE_ACL
     determine_request_user(cib_client->user, op_request, F_CIB_USER);
 #endif
 
     crm_log_xml_trace(op_request, "Client[inbound]");
 
     cib_common_callback_worker(id, flags, op_request, cib_client, privileged);
     free_xml(op_request);
 
     return 0;
 }
 
 static uint64_t ping_seq = 0;
 static char *ping_digest = NULL;
 static bool ping_modified_since = FALSE;
 int sync_our_cib(xmlNode * request, gboolean all);
 
 static gboolean
 cib_digester_cb(gpointer data)
 {
     if (cib_is_master) {
         char buffer[32];
         xmlNode *ping = create_xml_node(NULL, "ping");
 
         ping_seq++;
         free(ping_digest);
         ping_digest = NULL;
         ping_modified_since = FALSE;
         snprintf(buffer, 32, U64T, ping_seq);
         crm_trace("Requesting peer digests (%s)", buffer);
 
         crm_xml_add(ping, F_TYPE, "cib");
         crm_xml_add(ping, F_CIB_OPERATION, CRM_OP_PING);
         crm_xml_add(ping, F_CIB_PING_ID, buffer);
 
         crm_xml_add(ping, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
         send_cluster_message(NULL, crm_msg_cib, ping, TRUE);
 
         free_xml(ping);
     }
     return FALSE;
 }
 
 static void
 process_ping_reply(xmlNode *reply) 
 {
     uint64_t seq = 0;
     const char *host = crm_element_value(reply, F_ORIG);
 
     xmlNode *pong = get_message_xml(reply, F_CIB_CALLDATA);
     const char *seq_s = crm_element_value(pong, F_CIB_PING_ID);
     const char *digest = crm_element_value(pong, XML_ATTR_DIGEST);
 
     if (seq_s) {
         seq = crm_int_helper(seq_s, NULL);
     }
 
     if(digest == NULL) {
         crm_trace("Ignoring ping reply %s from %s with no digest", seq_s, host);
 
     } else if(seq != ping_seq) {
         crm_trace("Ignoring out of sequence ping reply %s from %s", seq_s, host);
 
     } else if(ping_modified_since) {
         crm_trace("Ignoring ping reply %s from %s: cib updated since", seq_s, host);
 
     } else {
         const char *version = crm_element_value(pong, XML_ATTR_CRM_VERSION);
 
         if(ping_digest == NULL) {
             crm_trace("Calculating new digest");
             ping_digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, version);
         }
 
         crm_trace("Processing ping reply %s from %s (%s)", seq_s, host, digest);
         if(safe_str_eq(ping_digest, digest) == FALSE) {
             xmlNode *remote_cib = get_message_xml(pong, F_CIB_CALLDATA);
 
             crm_notice("Local CIB %s.%s.%s.%s differs from %s: %s.%s.%s.%s %p",
                        crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN),
                        crm_element_value(the_cib, XML_ATTR_GENERATION),
                        crm_element_value(the_cib, XML_ATTR_NUMUPDATES),
                        ping_digest, host,
                        remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION_ADMIN):"0",
                        remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION):"0",
                        remote_cib?crm_element_value(remote_cib, XML_ATTR_NUMUPDATES):"0",
                        digest, remote_cib);
             if(remote_cib) {
                 /* Additional debug */
                 xml_calculate_changes(the_cib, remote_cib);
                 xml_log_changes(LOG_INFO, __FUNCTION__, remote_cib);
                 free_xml(remote_cib);
             }
             crm_trace("End of differences");
             sync_our_cib(reply, FALSE);
         }
     }
 }
 
 static void
 do_local_notify(xmlNode * notify_src, const char *client_id,
                 gboolean sync_reply, gboolean from_peer)
 {
     /* send callback to originating child */
     crm_client_t *client_obj = NULL;
     int local_rc = pcmk_ok;
 
     if (client_id != NULL) {
         client_obj = crm_client_get_by_id(client_id);
     }
 
     if (client_obj == NULL) {
         local_rc = -ECONNRESET;
         crm_trace("No client to sent the response to. F_CIB_CLIENTID not set.");
 
     } else {
         int rid = 0;
 
         if (sync_reply) {
             if (client_obj->ipcs) {
                 CRM_LOG_ASSERT(client_obj->request_id);
 
                 rid = client_obj->request_id;
                 client_obj->request_id = 0;
 
                 crm_trace("Sending response %d to %s %s",
                           rid, client_obj->name,
                           from_peer ? "(originator of delegated request)" : "");
             } else {
                 crm_trace("Sending response to %s %s",
                           client_obj->name, from_peer ? "(originator of delegated request)" : "");
             }
 
         } else {
             crm_trace("Sending an event to %s %s",
                       client_obj->name, from_peer ? "(originator of delegated request)" : "");
         }
 
         switch (client_obj->kind) {
             case CRM_CLIENT_IPC:
                 if (crm_ipcs_send(client_obj, rid, notify_src, sync_reply?crm_ipc_flags_none:crm_ipc_server_event) < 0) {
                     local_rc = -ENOMSG;
                 }
                 break;
 #ifdef HAVE_GNUTLS_GNUTLS_H
             case CRM_CLIENT_TLS:
 #endif
             case CRM_CLIENT_TCP:
                 crm_remote_send(client_obj->remote, notify_src);
                 break;
             default:
                 crm_err("Unknown transport %d for %s", client_obj->kind, client_obj->name);
         }
     }
 
     if (local_rc != pcmk_ok && client_obj != NULL) {
         crm_warn("%sSync reply to %s failed: %s",
                  sync_reply ? "" : "A-",
                  client_obj ? client_obj->name : "<unknown>", pcmk_strerror(local_rc));
     }
 }
 
 static void
 parse_local_options_v1(crm_client_t * cib_client, int call_type, int call_options, const char *host,
                     const char *op, gboolean * local_notify, gboolean * needs_reply,
                     gboolean * process, gboolean * needs_forward)
 {
     if (cib_op_modifies(call_type)
         && !(call_options & cib_inhibit_bcast)) {
         /* we need to send an update anyway */
         *needs_reply = TRUE;
     } else {
         *needs_reply = FALSE;
     }
 
     if (host == NULL && (call_options & cib_scope_local)) {
         crm_trace("Processing locally scoped %s op from %s", op, cib_client->name);
         *local_notify = TRUE;
 
     } else if (host == NULL && cib_is_master) {
         crm_trace("Processing master %s op locally from %s", op, cib_client->name);
         *local_notify = TRUE;
 
     } else if (safe_str_eq(host, cib_our_uname)) {
         crm_trace("Processing locally addressed %s op from %s", op, cib_client->name);
         *local_notify = TRUE;
 
     } else if (stand_alone) {
         *needs_forward = FALSE;
         *local_notify = TRUE;
         *process = TRUE;
 
     } else {
         crm_trace("%s op from %s needs to be forwarded to %s",
                   op, cib_client->name, host ? host : "the master instance");
         *needs_forward = TRUE;
         *process = FALSE;
     }
 }
 
 static void
 parse_local_options_v2(crm_client_t * cib_client, int call_type, int call_options, const char *host,
                     const char *op, gboolean * local_notify, gboolean * needs_reply,
                     gboolean * process, gboolean * needs_forward)
 {
     if (cib_op_modifies(call_type)
         && safe_str_neq(op, CIB_OP_MASTER)
         && safe_str_neq(op, CIB_OP_SLAVE)) {
         /* we need to send an update anyway */
         *needs_reply = TRUE;
         *needs_forward = TRUE;
         *process = FALSE;
         crm_trace("%s op from %s needs to be forwarded to %s",
                   op, cib_client->name, host ? host : "the master instance");
         return;
     }
 
     *process = TRUE;
     *needs_reply = FALSE;
     *local_notify = TRUE;
     *needs_forward = FALSE;
 
     if (stand_alone) {
         crm_trace("Processing %s op from %s (stand-alone)", op, cib_client->name);
 
     } else if (host == NULL) {
         crm_trace("Processing unaddressed %s op from %s", op, cib_client->name);
 
     } else if (safe_str_eq(host, cib_our_uname)) {
         crm_trace("Processing locally addressed %s op from %s", op, cib_client->name);
 
     } else {
         crm_trace("%s op from %s needs to be forwarded to %s", op, cib_client->name, host);
         *needs_forward = TRUE;
         *process = FALSE;
     }
 }
 
 static void
 parse_local_options(crm_client_t * cib_client, int call_type, int call_options, const char *host,
                     const char *op, gboolean * local_notify, gboolean * needs_reply,
                     gboolean * process, gboolean * needs_forward)
 {
     if(cib_legacy_mode()) {
         parse_local_options_v1(cib_client, call_type, call_options, host,
                                op, local_notify, needs_reply, process, needs_forward);
     } else {
         parse_local_options_v2(cib_client, call_type, call_options, host,
                                op, local_notify, needs_reply, process, needs_forward);
     }
 }
 
 static gboolean
 parse_peer_options_v1(int call_type, xmlNode * request,
                    gboolean * local_notify, gboolean * needs_reply, gboolean * process,
                    gboolean * needs_forward)
 {
     const char *op = NULL;
     const char *host = NULL;
     const char *delegated = NULL;
     const char *originator = crm_element_value(request, F_ORIG);
     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
     const char *update = crm_element_value(request, F_CIB_GLOBAL_UPDATE);
 
     gboolean is_reply = safe_str_eq(reply_to, cib_our_uname);
 
     if (crm_is_true(update)) {
         *needs_reply = FALSE;
         if (is_reply) {
             *local_notify = TRUE;
             crm_trace("Processing global/peer update from %s"
                       " that originated from us", originator);
         } else {
             crm_trace("Processing global/peer update from %s", originator);
         }
         return TRUE;
     }
 
     crm_trace("Processing %s request sent by %s", op, originator);
     op = crm_element_value(request, F_CIB_OPERATION);
     if (safe_str_eq(op, "cib_shutdown_req")) {
         /* Always process these */
         *local_notify = FALSE;
         if (reply_to == NULL || is_reply) {
             *process = TRUE;
         }
         if (is_reply) {
             *needs_reply = FALSE;
         }
         return *process;
     }
 
     if (is_reply && safe_str_eq(op, CRM_OP_PING)) {
         process_ping_reply(request);
         return FALSE;
     }
 
     if (is_reply) {
         crm_trace("Forward reply sent from %s to local clients", originator);
         *process = FALSE;
         *needs_reply = FALSE;
         *local_notify = TRUE;
         return TRUE;
     }
 
     host = crm_element_value(request, F_CIB_HOST);
     if (host != NULL && safe_str_eq(host, cib_our_uname)) {
         crm_trace("Processing %s request sent to us from %s", op, originator);
         return TRUE;
 
     } else if(is_reply == FALSE && safe_str_eq(op, CRM_OP_PING)) {
         crm_trace("Processing %s request sent to %s by %s", op, host?host:"everyone", originator);
         *needs_reply = TRUE;
         return TRUE;
 
     } else if (host == NULL && cib_is_master == TRUE) {
         crm_trace("Processing %s request sent to master instance from %s", op, originator);
         return TRUE;
     }
 
     delegated = crm_element_value(request, F_CIB_DELEGATED);
     if (delegated != NULL) {
         crm_trace("Ignoring msg for master instance");
 
     } else if (host != NULL) {
         /* this is for a specific instance and we're not it */
         crm_trace("Ignoring msg for instance on %s", crm_str(host));
 
     } else if (reply_to == NULL && cib_is_master == FALSE) {
         /* this is for the master instance and we're not it */
         crm_trace("Ignoring reply to %s", crm_str(reply_to));
 
     } else if (safe_str_eq(op, "cib_shutdown_req")) {
         if (reply_to != NULL) {
             crm_debug("Processing %s from %s", op, host);
             *needs_reply = FALSE;
 
         } else {
             crm_debug("Processing %s reply from %s", op, host);
         }
         return TRUE;
 
     } else {
         crm_err("Nothing for us to do?");
         crm_log_xml_err(request, "Peer[inbound]");
     }
 
     return FALSE;
 }
 
 static gboolean
 parse_peer_options_v2(int call_type, xmlNode * request,
                    gboolean * local_notify, gboolean * needs_reply, gboolean * process,
                    gboolean * needs_forward)
 {
     const char *host = NULL;
     const char *delegated = crm_element_value(request, F_CIB_DELEGATED);
     const char *op = crm_element_value(request, F_CIB_OPERATION);
     const char *originator = crm_element_value(request, F_ORIG);
     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
     const char *update = crm_element_value(request, F_CIB_GLOBAL_UPDATE);
 
     gboolean is_reply = safe_str_eq(reply_to, cib_our_uname);
 
     if(safe_str_eq(op, CIB_OP_REPLACE)) {
         /* sync_our_cib() sets F_CIB_ISREPLY */
         delegated = reply_to;
         goto skip_is_reply;
 
     } else if(safe_str_eq(op, CIB_OP_SYNC)) {
 
     } else if (is_reply && safe_str_eq(op, CRM_OP_PING)) {
         process_ping_reply(request);
         return FALSE;
 
     } else if (crm_is_true(update)) {
         crm_trace("Ingoring legacy %s global update from %s", op, originator);
         return FALSE;
 
     } else if (is_reply && cib_op_modifies(call_type)) {
         crm_trace("Ignoring legacy %s reply sent from %s to local clients", op, originator);
         return FALSE;
 
     } else if (safe_str_eq(op, "cib_shutdown_req")) {
         /* Legacy handling */
         crm_debug("Legacy handling of %s message from %s", op, originator);
         *local_notify = FALSE;
         if (reply_to == NULL) {
             *process = TRUE;
         }
         return *process;
     }
 
     if(is_reply) {
         crm_trace("Handling %s reply sent from %s to local clients", op, originator);
         *process = FALSE;
         *needs_reply = FALSE;
         *local_notify = TRUE;
         return TRUE;
     }
 
   skip_is_reply:
     *process = TRUE;
     *needs_reply = FALSE;
 
     if(safe_str_eq(delegated, cib_our_uname)) {
         *local_notify = TRUE;
     } else {
         *local_notify = FALSE;
     }
 
     host = crm_element_value(request, F_CIB_HOST);
     if (host != NULL && safe_str_eq(host, cib_our_uname)) {
         crm_trace("Processing %s request sent to us from %s", op, originator);
         *needs_reply = TRUE;
         return TRUE;
 
     } else if (host != NULL) {
         /* this is for a specific instance and we're not it */
         crm_trace("Ignoring %s operation for instance on %s", op, crm_str(host));
         return FALSE;
 
     } else if(is_reply == FALSE && safe_str_eq(op, CRM_OP_PING)) {
         *needs_reply = TRUE;
     }
 
     crm_trace("Processing %s request sent to everyone by %s/%s on %s %s", op,
               crm_element_value(request, F_CIB_CLIENTNAME),
               crm_element_value(request, F_CIB_CALLID),
               originator, (*local_notify)?"(notify)":"");
     return TRUE;
 }
 
 static gboolean
 parse_peer_options(int call_type, xmlNode * request,
                    gboolean * local_notify, gboolean * needs_reply, gboolean * process,
                    gboolean * needs_forward)
 {
     /* TODO: What happens when an update comes in after node A
      * requests the CIB from node B, but before it gets the reply (and
      * sends out the replace operation)
      */
     if(cib_legacy_mode()) {
         return parse_peer_options_v1(
             call_type, request, local_notify, needs_reply, process, needs_forward);
     } else {
         return parse_peer_options_v2(
             call_type, request, local_notify, needs_reply, process, needs_forward);
     }
 }
 
 static void
 forward_request(xmlNode * request, crm_client_t * cib_client, int call_options)
 {
     const char *op = crm_element_value(request, F_CIB_OPERATION);
     const char *host = crm_element_value(request, F_CIB_HOST);
 
     crm_xml_add(request, F_CIB_DELEGATED, cib_our_uname);
 
     if (host != NULL) {
         crm_trace("Forwarding %s op to %s", op, host);
         send_cluster_message(crm_get_peer(0, host), crm_msg_cib, request, FALSE);
 
     } else {
         crm_trace("Forwarding %s op to master instance", op);
         send_cluster_message(NULL, crm_msg_cib, request, FALSE);
     }
 
     /* Return the request to its original state */
     xml_remove_prop(request, F_CIB_DELEGATED);
 
     if (call_options & cib_discard_reply) {
         crm_trace("Client not interested in reply");
     }
 }
 
 static gboolean
 send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gboolean broadcast)
 {
     CRM_ASSERT(msg != NULL);
 
     if (broadcast) {
         /* this (successful) call modified the CIB _and_ the
          * change needs to be broadcast...
          *   send via HA to other nodes
          */
         int diff_add_updates = 0;
         int diff_add_epoch = 0;
         int diff_add_admin_epoch = 0;
 
         int diff_del_updates = 0;
         int diff_del_epoch = 0;
         int diff_del_admin_epoch = 0;
 
         const char *digest = NULL;
 
         CRM_LOG_ASSERT(result_diff != NULL);
         digest = crm_element_value(result_diff, XML_ATTR_DIGEST);
         cib_diff_version_details(result_diff,
                                  &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
                                  &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
 
         crm_trace("Sending update diff %d.%d.%d -> %d.%d.%d %s",
                   diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
                   diff_add_admin_epoch, diff_add_epoch, diff_add_updates, digest);
 
         crm_xml_add(msg, F_CIB_ISREPLY, originator);
         crm_xml_add(msg, F_CIB_GLOBAL_UPDATE, XML_BOOLEAN_TRUE);
         crm_xml_add(msg, F_CIB_OPERATION, CIB_OP_APPLY_DIFF);
 
         CRM_ASSERT(digest != NULL);
 
         add_message_xml(msg, F_CIB_UPDATE_DIFF, result_diff);
         crm_log_xml_explicit(msg, "copy");
         return send_cluster_message(NULL, crm_msg_cib, msg, TRUE);
 
     } else if (originator != NULL) {
         /* send reply via HA to originating node */
         crm_trace("Sending request result to %s only", originator);
         crm_xml_add(msg, F_CIB_ISREPLY, originator);
         return send_cluster_message(crm_get_peer(0, originator), crm_msg_cib, msg, FALSE);
     }
 
     return FALSE;
 }
 
 void
 cib_process_request(xmlNode * request, gboolean force_synchronous, gboolean privileged,
                     gboolean unused, crm_client_t * cib_client)
 {
     int call_type = 0;
     int call_options = 0;
 
     gboolean process = TRUE;
     gboolean is_update = TRUE;
     gboolean from_peer = TRUE;
     gboolean needs_reply = TRUE;
     gboolean local_notify = FALSE;
     gboolean needs_forward = FALSE;
     gboolean global_update = crm_is_true(crm_element_value(request, F_CIB_GLOBAL_UPDATE));
 
     xmlNode *op_reply = NULL;
     xmlNode *result_diff = NULL;
 
     int rc = pcmk_ok;
     const char *op = crm_element_value(request, F_CIB_OPERATION);
     const char *originator = crm_element_value(request, F_ORIG);
     const char *host = crm_element_value(request, F_CIB_HOST);
     const char *target = NULL;
     const char *client_id = crm_element_value(request, F_CIB_CLIENTID);
     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
 
     if (cib_client) {
         from_peer = FALSE;
     }
 
     cib_num_ops++;
     if (cib_num_ops == 0) {
         cib_num_fail = 0;
         cib_num_local = 0;
         cib_num_updates = 0;
         crm_info("Stats wrapped around");
     }
 
     crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
     if (force_synchronous) {
         call_options |= cib_sync_call;
     }
 
     if (host != NULL && strlen(host) == 0) {
         host = NULL;
     }
 
     if (host) {
         target = host;
 
     } else if (call_options & cib_scope_local) {
         target = "local host";
 
     } else {
         target = "master";
     }
 
     if (from_peer) {
         crm_trace("Processing peer %s operation from %s on %s intended for %s (reply=%s)",
                   op, client_id, originator, target, reply_to);
     } else {
         crm_xml_add(request, F_ORIG, cib_our_uname);
         crm_trace("Processing local %s operation from %s intended for %s", op, client_id, target);
     }
 
     rc = cib_get_operation_id(op, &call_type);
     if (rc != pcmk_ok) {
         /* TODO: construct error reply? */
         crm_err("Pre-processing of command failed: %s", pcmk_strerror(rc));
         return;
     }
 
     if (from_peer == FALSE) {
         parse_local_options(cib_client, call_type, call_options, host, op,
                             &local_notify, &needs_reply, &process, &needs_forward);
 
     } else if (parse_peer_options(call_type, request, &local_notify,
                                   &needs_reply, &process, &needs_forward) == FALSE) {
         return;
     }
 
     is_update = cib_op_modifies(call_type);
     if (is_update) {
         cib_num_updates++;
     }
 
     if (call_options & cib_discard_reply) {
         needs_reply = is_update;
         local_notify = FALSE;
     }
 
     if (needs_forward) {
         const char *host = crm_element_value(request, F_CIB_HOST);
         const char *section = crm_element_value(request, F_CIB_SECTION);
 
         crm_info("Forwarding %s operation for section %s to %s (origin=%s/%s/%s)",
                  op,
                  section ? section : "'all'",
                  host ? host : "master",
                  originator ? originator : "local",
                  crm_element_value(request, F_CIB_CLIENTNAME),
                  crm_element_value(request, F_CIB_CALLID));
 
         forward_request(request, cib_client, call_options);
         return;
     }
 
     if (cib_status != pcmk_ok) {
         const char *call = crm_element_value(request, F_CIB_CALLID);
 
         rc = cib_status;
         crm_err("Operation ignored, cluster configuration is invalid."
                 " Please repair and restart: %s", pcmk_strerror(cib_status));
 
         op_reply = create_xml_node(NULL, "cib-reply");
         crm_xml_add(op_reply, F_TYPE, T_CIB);
         crm_xml_add(op_reply, F_CIB_OPERATION, op);
         crm_xml_add(op_reply, F_CIB_CALLID, call);
         crm_xml_add(op_reply, F_CIB_CLIENTID, client_id);
         crm_xml_add_int(op_reply, F_CIB_CALLOPTS, call_options);
         crm_xml_add_int(op_reply, F_CIB_RC, rc);
 
         crm_trace("Attaching reply output");
         add_message_xml(op_reply, F_CIB_CALLDATA, the_cib);
 
         crm_log_xml_explicit(op_reply, "cib:reply");
 
     } else if (process) {
         time_t finished = 0;
 
         int now = time(NULL);
         int level = LOG_INFO;
         const char *section = crm_element_value(request, F_CIB_SECTION);
 
         cib_num_local++;
         rc = cib_process_command(request, &op_reply, &result_diff, privileged);
 
         if (global_update) {
             switch (rc) {
                 case pcmk_ok:
                     level = LOG_INFO;
                     break;
                 case -pcmk_err_old_data:
                 case -pcmk_err_diff_resync:
                 case -pcmk_err_diff_failed:
                     level = LOG_TRACE;
                     break;
                 default:
                     level = LOG_ERR;
             }
 
         } else if (rc != pcmk_ok && is_update) {
             cib_num_fail++;
             level = LOG_WARNING;
         }
 
         do_crm_log(level,
                    "Completed %s operation for section %s: %s (rc=%d, origin=%s/%s/%s, version=%s.%s.%s)",
                    op, section ? section : "'all'", pcmk_strerror(rc), rc,
                    originator ? originator : "local",
                    crm_element_value(request, F_CIB_CLIENTNAME),
                    crm_element_value(request, F_CIB_CALLID),
                    the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN) : "0",
                    the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION) : "0",
                    the_cib ? crm_element_value(the_cib, XML_ATTR_NUMUPDATES) : "0");
 
         finished = time(NULL);
         if (finished - now > 3) {
             crm_trace("%s operation took %ds to complete", op, finished - now);
             crm_write_blackbox(0, NULL);
         }
 
         if (op_reply == NULL && (needs_reply || local_notify)) {
             crm_err("Unexpected NULL reply to message");
             crm_log_xml_err(request, "null reply");
             needs_reply = FALSE;
             local_notify = FALSE;
         }
     }
 
     /* from now on we are the server */
     if(is_update && cib_legacy_mode() == FALSE) {
         crm_trace("Completed pre-sync update from %s/%s/%s%s",
                   originator ? originator : "local",
                   crm_element_value(request, F_CIB_CLIENTNAME),
                   crm_element_value(request, F_CIB_CALLID),
                   local_notify?" with local notification":"");
 
     } else if (needs_reply == FALSE || stand_alone) {
         /* nothing more to do...
          * this was a non-originating slave update
          */
         crm_trace("Completed slave update");
 
     } else if (call_options & cib_discard_reply) {
         crm_trace("Caller isn't interested in reply");
 
     } else if (from_peer) {
         if (is_update == FALSE || result_diff == NULL) {
             crm_trace("Request not broadcast: R/O call");
 
         } else if (call_options & cib_inhibit_bcast) {
             crm_trace("Request not broadcast: inhibited");
 
         } else if (rc != pcmk_ok) {
             crm_trace("Request not broadcast: call failed: %s", pcmk_strerror(rc));
 
         } else {
             crm_trace("Directing reply to %s", originator);
         }
 
         send_peer_reply(op_reply, result_diff, originator, FALSE);
     }
 
     if (local_notify && client_id) {
         crm_trace("Performing local %ssync notification for %s",
                   (call_options & cib_sync_call) ? "" : "a-", client_id);
         if (process == FALSE) {
             do_local_notify(request, client_id, call_options & cib_sync_call, from_peer);
         } else {
             do_local_notify(op_reply, client_id, call_options & cib_sync_call, from_peer);
         }
     }
 
     free_xml(op_reply);
     free_xml(result_diff);
 
     return;
 }
 
 int
 cib_process_command(xmlNode * request, xmlNode ** reply, xmlNode ** cib_diff, gboolean privileged)
 {
     xmlNode *input = NULL;
     xmlNode *output = NULL;
     xmlNode *result_cib = NULL;
     xmlNode *current_cib = NULL;
 
     int call_type = 0;
     int call_options = 0;
 
     const char *op = NULL;
     const char *section = NULL;
     const char *call_id = crm_element_value(request, F_CIB_CALLID);
 
     int rc = pcmk_ok;
     int rc2 = pcmk_ok;
 
     gboolean send_r_notify = FALSE;
     gboolean global_update = FALSE;
     gboolean config_changed = FALSE;
     gboolean manage_counters = TRUE;
 
     static mainloop_timer_t *digest_timer = NULL;
 
     CRM_ASSERT(cib_status == pcmk_ok);
 
     if(digest_timer == NULL) {
         digest_timer = mainloop_timer_add("digester", 5000, FALSE, cib_digester_cb, NULL);
     }
 
     *reply = NULL;
     *cib_diff = NULL;
     current_cib = the_cib;
 
     /* Start processing the request... */
     op = crm_element_value(request, F_CIB_OPERATION);
     crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
     rc = cib_get_operation_id(op, &call_type);
 
     if (rc == pcmk_ok && privileged == FALSE) {
         rc = cib_op_can_run(call_type, call_options, privileged, global_update);
     }
 
     rc2 = cib_op_prepare(call_type, request, &input, &section);
     if (rc == pcmk_ok) {
         rc = rc2;
     }
 
     if (rc != pcmk_ok) {
         crm_trace("Call setup failed: %s", pcmk_strerror(rc));
         goto done;
 
     } else if (cib_op_modifies(call_type) == FALSE) {
         rc = cib_perform_op(op, call_options, cib_op_func(call_type), TRUE,
                             section, request, input, FALSE, &config_changed,
                             current_cib, &result_cib, NULL, &output);
 
         CRM_CHECK(result_cib == NULL, free_xml(result_cib));
         goto done;
     }
 
     /* Handle a valid write action */
     global_update = crm_is_true(crm_element_value(request, F_CIB_GLOBAL_UPDATE));
     if (global_update) {
         /* legacy code */
         manage_counters = FALSE;
         call_options |= cib_force_diff;
         crm_trace("Global update detected");
 
         CRM_CHECK(call_type == 3 || call_type == 4, crm_err("Call type: %d", call_type);
                   crm_log_xml_err(request, "bad op"));
     }
 
     if (rc == pcmk_ok) {
         ping_modified_since = TRUE;
         if (call_options & cib_inhibit_bcast) {
             /* skip */
             crm_trace("Skipping update: inhibit broadcast");
             manage_counters = FALSE;
         }
 
         if (is_not_set(call_options, cib_dryrun) && safe_str_eq(section, XML_CIB_TAG_STATUS)) {
             /* Copying large CIBs accounts for a huge percentage of our CIB usage */
             call_options |= cib_zero_copy;
         } else {
             clear_bit(call_options, cib_zero_copy);
         }
 
         /* result_cib must not be modified after cib_perform_op() returns */
         rc = cib_perform_op(op, call_options, cib_op_func(call_type), FALSE,
                             section, request, input, manage_counters, &config_changed,
                             current_cib, &result_cib, cib_diff, &output);
 
         if (manage_counters == FALSE) {
             /* Legacy code
              * If the diff is NULL at this point, its because nothing changed
              */
             config_changed = cib_config_changed(NULL, NULL, cib_diff);
         }
 
         /* Always write to disk for replace ops,
          * this also negates the need to detect ordering changes
          */
         if (crm_str_eq(CIB_OP_REPLACE, op, TRUE)) {
             config_changed = TRUE;
         }
     }
 
     if (rc == pcmk_ok && is_not_set(call_options, cib_dryrun)) {
         if(is_not_set(call_options, cib_zero_copy)) {
             rc = activateCibXml(result_cib, config_changed, op);
         }
 
         if (rc == pcmk_ok && cib_internal_config_changed(*cib_diff)) {
             cib_read_config(config_hash, result_cib);
         }
 
         if (crm_str_eq(CIB_OP_REPLACE, op, TRUE)) {
             if (section == NULL) {
                 send_r_notify = TRUE;
 
             } else if (safe_str_eq(section, XML_TAG_CIB)) {
                 send_r_notify = TRUE;
 
             } else if (safe_str_eq(section, XML_CIB_TAG_NODES)) {
                 send_r_notify = TRUE;
 
             } else if (safe_str_eq(section, XML_CIB_TAG_STATUS)) {
                 send_r_notify = TRUE;
             }
 
         } else if (crm_str_eq(CIB_OP_ERASE, op, TRUE)) {
             send_r_notify = TRUE;
         }
 
         mainloop_timer_stop(digest_timer);
         mainloop_timer_start(digest_timer);
 
     } else if (rc == -pcmk_err_dtd_validation) {
         CRM_ASSERT(is_not_set(call_options, cib_zero_copy));
 
         if (output != NULL) {
             crm_log_xml_info(output, "cib:output");
             free_xml(output);
         }
 
         output = result_cib;
 
     } else {
         CRM_ASSERT(is_not_set(call_options, cib_zero_copy));
         free_xml(result_cib);
     }
 
     if ((call_options & cib_inhibit_notify) == 0) {
         const char *client = crm_element_value(request, F_CIB_CLIENTNAME);
 
         crm_trace("Sending notifications");
         cib_diff_notify(call_options, client, call_id, op, input, rc, *cib_diff);
     }
 
     if (send_r_notify) {
         const char *origin = crm_element_value(request, F_ORIG);
 
         cib_replace_notify(origin, the_cib, rc, *cib_diff);
     }
 
     if (rc != pcmk_ok) {
         xml_log_patchset(LOG_DEBUG, "cib:diff", *cib_diff);
 
     } else {
         xml_log_patchset(LOG_INFO, "cib:diff", *cib_diff);
     }
 
   done:
     if ((call_options & cib_discard_reply) == 0) {
         const char *caller = crm_element_value(request, F_CIB_CLIENTID);
 
         *reply = create_xml_node(NULL, "cib-reply");
         crm_xml_add(*reply, F_TYPE, T_CIB);
         crm_xml_add(*reply, F_CIB_OPERATION, op);
         crm_xml_add(*reply, F_CIB_CALLID, call_id);
         crm_xml_add(*reply, F_CIB_CLIENTID, caller);
         crm_xml_add_int(*reply, F_CIB_CALLOPTS, call_options);
         crm_xml_add_int(*reply, F_CIB_RC, rc);
 
         if (output != NULL) {
             crm_trace("Attaching reply output");
             add_message_xml(*reply, F_CIB_CALLDATA, output);
         }
 
         crm_log_xml_explicit(*reply, "cib:reply");
     }
 
     crm_trace("cleanup");
 
+    if (cib_op_modifies(call_type) == FALSE && output != current_cib) {
+        free_xml(output);
+    }
+
     if (call_type >= 0) {
         cib_op_cleanup(call_type, call_options, &input, &output);
     }
+
     crm_trace("done");
     return rc;
 }
 
 gint
 cib_GCompareFunc(gconstpointer a, gconstpointer b)
 {
     const xmlNode *a_msg = a;
     const xmlNode *b_msg = b;
 
     int msg_a_id = 0;
     int msg_b_id = 0;
     const char *value = NULL;
 
     value = crm_element_value_const(a_msg, F_CIB_CALLID);
     msg_a_id = crm_parse_int(value, NULL);
 
     value = crm_element_value_const(b_msg, F_CIB_CALLID);
     msg_b_id = crm_parse_int(value, NULL);
 
     if (msg_a_id == msg_b_id) {
         return 0;
     } else if (msg_a_id < msg_b_id) {
         return -1;
     }
     return 1;
 }
 
 #if SUPPORT_HEARTBEAT
 void
 cib_ha_peer_callback(HA_Message * msg, void *private_data)
 {
     xmlNode *xml = convert_ha_message(NULL, msg, __FUNCTION__);
 
     cib_peer_callback(xml, private_data);
     free_xml(xml);
 }
 #endif
 
 void
 cib_peer_callback(xmlNode * msg, void *private_data)
 {
     const char *reason = NULL;
     const char *originator = crm_element_value(msg, F_ORIG);
 
     if (cib_legacy_mode() && (originator == NULL || crm_str_eq(originator, cib_our_uname, TRUE))) {
         /* message is from ourselves */
         return;
 
     } else if (crm_peer_cache == NULL) {
         reason = "membership not established";
         goto bail;
     }
 
     if (crm_element_value(msg, F_CIB_CLIENTNAME) == NULL) {
         crm_xml_add(msg, F_CIB_CLIENTNAME, originator);
     }
 
     /* crm_log_xml_trace("Peer[inbound]", msg); */
     cib_process_request(msg, FALSE, TRUE, TRUE, NULL);
     return;
 
   bail:
     if (reason) {
         const char *seq = crm_element_value(msg, F_SEQ);
         const char *op = crm_element_value(msg, F_CIB_OPERATION);
 
         crm_warn("Discarding %s message (%s) from %s: %s", op, seq, originator, reason);
     }
 }
 
 #if SUPPORT_HEARTBEAT
 extern oc_ev_t *cib_ev_token;
 static void *ccm_library = NULL;
 int (*ccm_api_callback_done) (void *cookie) = NULL;
 int (*ccm_api_handle_event) (const oc_ev_t * token) = NULL;
 
 void
 cib_client_status_callback(const char *node, const char *client, const char *status, void *private)
 {
     crm_node_t *peer = NULL;
 
     if (safe_str_eq(client, CRM_SYSTEM_CIB)) {
         crm_info("Status update: Client %s/%s now has status [%s]", node, client, status);
 
         if (safe_str_eq(status, JOINSTATUS)) {
             status = ONLINESTATUS;
 
         } else if (safe_str_eq(status, LEAVESTATUS)) {
             status = OFFLINESTATUS;
         }
 
         peer = crm_get_peer(0, node);
         crm_update_peer_proc(__FUNCTION__, peer, crm_proc_cib, status);
     }
     return;
 }
 
 int
 cib_ccm_dispatch(gpointer user_data)
 {
     int rc = 0;
     oc_ev_t *ccm_token = (oc_ev_t *) user_data;
 
     crm_trace("received callback");
 
     if (ccm_api_handle_event == NULL) {
         ccm_api_handle_event =
             find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_handle_event", 1);
     }
 
     rc = (*ccm_api_handle_event) (ccm_token);
     if (0 == rc) {
         return 0;
     }
 
     crm_err("CCM connection appears to have failed: rc=%d.", rc);
 
     /* eventually it might be nice to recover and reconnect... but until then... */
     crm_err("Exiting to recover from CCM connection failure");
     return crm_exit(ENOTCONN);
 }
 
 int current_instance = 0;
 void
 cib_ccm_msg_callback(oc_ed_t event, void *cookie, size_t size, const void *data)
 {
     gboolean update_id = FALSE;
     const oc_ev_membership_t *membership = data;
 
     CRM_ASSERT(membership != NULL);
 
     crm_info("Processing CCM event=%s (id=%d)", ccm_event_name(event), membership->m_instance);
 
     if (current_instance > membership->m_instance) {
         crm_err("Membership instance ID went backwards! %d->%d",
                 current_instance, membership->m_instance);
         CRM_ASSERT(current_instance <= membership->m_instance);
     }
 
     switch (event) {
         case OC_EV_MS_NEW_MEMBERSHIP:
         case OC_EV_MS_INVALID:
             update_id = TRUE;
             break;
         case OC_EV_MS_PRIMARY_RESTORED:
             update_id = TRUE;
             break;
         case OC_EV_MS_NOT_PRIMARY:
             crm_trace("Ignoring transitional CCM event: %s", ccm_event_name(event));
             break;
         case OC_EV_MS_EVICTED:
             crm_err("Evicted from CCM: %s", ccm_event_name(event));
             break;
         default:
             crm_err("Unknown CCM event: %d", event);
     }
 
     if (update_id) {
         unsigned int lpc = 0;
 
         CRM_CHECK(membership != NULL, return);
 
         current_instance = membership->m_instance;
 
         for (lpc = 0; lpc < membership->m_n_out; lpc++) {
             crm_update_ccm_node(membership, lpc + membership->m_out_idx, CRM_NODE_LOST,
                                 current_instance);
         }
 
         for (lpc = 0; lpc < membership->m_n_member; lpc++) {
             crm_update_ccm_node(membership, lpc + membership->m_memb_idx, CRM_NODE_ACTIVE,
                                 current_instance);
         }
     }
 
     if (ccm_api_callback_done == NULL) {
         ccm_api_callback_done =
             find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_callback_done", 1);
     }
     (*ccm_api_callback_done) (cookie);
     return;
 }
 #endif
 
 gboolean
 can_write(int flags)
 {
     return TRUE;
 }
 
 static gboolean
 cib_force_exit(gpointer data)
 {
     crm_notice("Forcing exit!");
     terminate_cib(__FUNCTION__, TRUE);
     return FALSE;
 }
 
 static void
 disconnect_remote_client(gpointer key, gpointer value, gpointer user_data)
 {
     crm_client_t *a_client = value;
 
     crm_err("Disconnecting %s... Not implemented", crm_str(a_client->name));
 }
 
 void
 cib_shutdown(int nsig)
 {
     struct qb_ipcs_stats srv_stats;
 
     if (cib_shutdown_flag == FALSE) {
         int disconnects = 0;
         qb_ipcs_connection_t *c = NULL;
 
         cib_shutdown_flag = TRUE;
 
         c = qb_ipcs_connection_first_get(ipcs_rw);
         while (c != NULL) {
             qb_ipcs_connection_t *last = c;
 
             c = qb_ipcs_connection_next_get(ipcs_rw, last);
 
             crm_debug("Disconnecting r/w client %p...", last);
             qb_ipcs_disconnect(last);
             qb_ipcs_connection_unref(last);
             disconnects++;
         }
 
         c = qb_ipcs_connection_first_get(ipcs_ro);
         while (c != NULL) {
             qb_ipcs_connection_t *last = c;
 
             c = qb_ipcs_connection_next_get(ipcs_ro, last);
 
             crm_debug("Disconnecting r/o client %p...", last);
             qb_ipcs_disconnect(last);
             qb_ipcs_connection_unref(last);
             disconnects++;
         }
 
         c = qb_ipcs_connection_first_get(ipcs_shm);
         while (c != NULL) {
             qb_ipcs_connection_t *last = c;
 
             c = qb_ipcs_connection_next_get(ipcs_shm, last);
 
             crm_debug("Disconnecting non-blocking r/w client %p...", last);
             qb_ipcs_disconnect(last);
             qb_ipcs_connection_unref(last);
             disconnects++;
         }
 
         disconnects += crm_hash_table_size(client_connections);
 
         crm_debug("Disconnecting %d remote clients", crm_hash_table_size(client_connections));
         g_hash_table_foreach(client_connections, disconnect_remote_client, NULL);
         crm_info("Disconnected %d clients", disconnects);
     }
 
     qb_ipcs_stats_get(ipcs_rw, &srv_stats, QB_FALSE);
 
     if (crm_hash_table_size(client_connections) == 0) {
         crm_info("All clients disconnected (%d)", srv_stats.active_connections);
         initiate_exit();
 
     } else {
         crm_info("Waiting on %d clients to disconnect (%d)",
                  crm_hash_table_size(client_connections), srv_stats.active_connections);
     }
 }
 
 void
 initiate_exit(void)
 {
     int active = 0;
     xmlNode *leaving = NULL;
 
     active = crm_active_peers();
     if (active < 2) {
         terminate_cib(__FUNCTION__, FALSE);
         return;
     }
 
     crm_info("Sending disconnect notification to %d peers...", active);
 
     leaving = create_xml_node(NULL, "exit-notification");
     crm_xml_add(leaving, F_TYPE, "cib");
     crm_xml_add(leaving, F_CIB_OPERATION, "cib_shutdown_req");
 
     send_cluster_message(NULL, crm_msg_cib, leaving, TRUE);
     free_xml(leaving);
 
     g_timeout_add(crm_get_msec("5s"), cib_force_exit, NULL);
 }
 
 extern int remote_fd;
 extern int remote_tls_fd;
 
 void
 terminate_cib(const char *caller, gboolean fast)
 {
     if (remote_fd > 0) {
         close(remote_fd);
         remote_fd = 0;
     }
     if (remote_tls_fd > 0) {
         close(remote_tls_fd);
         remote_tls_fd = 0;
     }
 
     if (!fast) {
         crm_info("%s: Disconnecting from cluster infrastructure", caller);
         crm_cluster_disconnect(&crm_cluster);
     }
 
     uninitializeCib();
 
     crm_info("%s: Exiting%s...", caller, fast ? " fast" : mainloop ? " from mainloop" : "");
 
     if (fast == FALSE && mainloop != NULL && g_main_is_running(mainloop)) {
         g_main_quit(mainloop);
 
     } else {
         qb_ipcs_destroy(ipcs_ro);
         qb_ipcs_destroy(ipcs_rw);
         qb_ipcs_destroy(ipcs_shm);
 
         if (fast) {
             crm_exit(EINVAL);
         } else {
             crm_exit(pcmk_ok);
         }
     }
 }
diff --git a/lib/cib/cib_file.c b/lib/cib/cib_file.c
index 5da84c938c..4e4ada8341 100644
--- a/lib/cib/cib_file.c
+++ b/lib/cib/cib_file.c
@@ -1,341 +1,342 @@
 /*
  * Copyright (c) 2004 International Business Machines
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  * 
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  *
  */
 #include <crm_internal.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdarg.h>
 #include <string.h>
 
 #include <sys/stat.h>
 #include <glib.h>
 
 #include <crm/crm.h>
 #include <crm/cib/internal.h>
 #include <crm/msg_xml.h>
 #include <crm/common/ipc.h>
 
 typedef struct cib_file_opaque_s {
     int flags;
     char *filename;
 
 } cib_file_opaque_t;
 
 int cib_file_perform_op(cib_t * cib, const char *op, const char *host, const char *section,
                         xmlNode * data, xmlNode ** output_data, int call_options);
 
 int cib_file_perform_op_delegate(cib_t * cib, const char *op, const char *host, const char *section,
                                  xmlNode * data, xmlNode ** output_data, int call_options,
                                  const char *user_name);
 
 int cib_file_signon(cib_t * cib, const char *name, enum cib_conn_type type);
 int cib_file_signoff(cib_t * cib);
 int cib_file_free(cib_t * cib);
 
 static int
 cib_file_inputfd(cib_t * cib)
 {
     return -EPROTONOSUPPORT;
 }
 
 static int
 cib_file_set_connection_dnotify(cib_t * cib, void (*dnotify) (gpointer user_data))
 {
     return -EPROTONOSUPPORT;
 }
 
 static int
 cib_file_register_notification(cib_t * cib, const char *callback, int enabled)
 {
     return -EPROTONOSUPPORT;
 }
 
 cib_t *
 cib_file_new(const char *cib_location)
 {
     cib_file_opaque_t *private = NULL;
     cib_t *cib = cib_new_variant();
 
     private = calloc(1, sizeof(cib_file_opaque_t));
 
     cib->variant = cib_file;
     cib->variant_opaque = private;
 
     if (cib_location == NULL) {
         cib_location = getenv("CIB_file");
     }
     private->filename = strdup(cib_location);
 
     /* assign variant specific ops */
     cib->delegate_fn = cib_file_perform_op_delegate;
     cib->cmds->signon = cib_file_signon;
     cib->cmds->signoff = cib_file_signoff;
     cib->cmds->free = cib_file_free;
     cib->cmds->inputfd = cib_file_inputfd;
 
     cib->cmds->register_notification = cib_file_register_notification;
     cib->cmds->set_connection_dnotify = cib_file_set_connection_dnotify;
 
     return cib;
 }
 
 static xmlNode *in_mem_cib = NULL;
 static int
 load_file_cib(const char *filename)
 {
     int rc = pcmk_ok;
     struct stat buf;
     xmlNode *root = NULL;
     gboolean dtd_ok = TRUE;
     const char *ignore_dtd = NULL;
     xmlNode *status = NULL;
 
     rc = stat(filename, &buf);
     if (rc == 0) {
         root = filename2xml(filename);
         if (root == NULL) {
             return -pcmk_err_dtd_validation;
         }
 
     } else {
         return -ENXIO;
     }
 
     rc = 0;
 
     status = find_xml_node(root, XML_CIB_TAG_STATUS, FALSE);
     if (status == NULL) {
         create_xml_node(root, XML_CIB_TAG_STATUS);
     }
 
     ignore_dtd = crm_element_value(root, XML_ATTR_VALIDATION);
     dtd_ok = validate_xml(root, NULL, TRUE);
     if (dtd_ok == FALSE) {
         crm_err("CIB does not validate against %s", ignore_dtd);
         rc = -pcmk_err_dtd_validation;
         goto bail;
     }
 
     in_mem_cib = root;
     return rc;
 
   bail:
     free_xml(root);
     root = NULL;
     return rc;
 }
 
 int
 cib_file_signon(cib_t * cib, const char *name, enum cib_conn_type type)
 {
     int rc = pcmk_ok;
     cib_file_opaque_t *private = cib->variant_opaque;
 
     if (private->filename == FALSE) {
         rc = -EINVAL;
     } else {
         rc = load_file_cib(private->filename);
     }
 
     if (rc == pcmk_ok) {
         crm_debug("%s: Opened connection to local file '%s'", name, private->filename);
         cib->state = cib_connected_command;
         cib->type = cib_command;
 
     } else {
         fprintf(stderr, "%s: Connection to local file '%s' failed: %s\n",
                 name, private->filename, pcmk_strerror(rc));
     }
 
     return rc;
 }
 
 int
 cib_file_signoff(cib_t * cib)
 {
     int rc = pcmk_ok;
     cib_file_opaque_t *private = cib->variant_opaque;
 
     crm_debug("Signing out of the CIB Service");
 
     if (strstr(private->filename, ".bz2") != NULL) {
         rc = write_xml_file(in_mem_cib, private->filename, TRUE);
 
     } else {
         rc = write_xml_file(in_mem_cib, private->filename, FALSE);
     }
 
     if (rc > 0) {
         crm_info("Wrote CIB to %s", private->filename);
         rc = pcmk_ok;
 
     } else {
         crm_err("Could not write CIB to %s: %s (%d)", private->filename, pcmk_strerror(rc), rc);
     }
     free_xml(in_mem_cib);
 
     cib->state = cib_disconnected;
     cib->type = cib_no_connection;
 
     return rc;
 }
 
 int
 cib_file_free(cib_t * cib)
 {
     int rc = pcmk_ok;
 
     if (cib->state != cib_disconnected) {
         rc = cib_file_signoff(cib);
     }
 
     if (rc == pcmk_ok) {
         cib_file_opaque_t *private = cib->variant_opaque;
 
         free(private->filename);
         free(cib->cmds);
         free(private);
         free(cib);
 
     } else {
         fprintf(stderr, "Couldn't sign off: %d\n", rc);
     }
 
     return rc;
 }
 
 struct cib_func_entry {
     const char *op;
     gboolean read_only;
     cib_op_t fn;
 };
 
 /* *INDENT-OFF* */
 static struct cib_func_entry cib_file_ops[] = {
     {CIB_OP_QUERY,      TRUE,  cib_process_query},
     {CIB_OP_MODIFY,     FALSE, cib_process_modify},
     {CIB_OP_APPLY_DIFF, FALSE, cib_process_diff},
     {CIB_OP_BUMP,       FALSE, cib_process_bump},
     {CIB_OP_REPLACE,    FALSE, cib_process_replace},
     {CIB_OP_CREATE,     FALSE, cib_process_create},
     {CIB_OP_DELETE,     FALSE, cib_process_delete},
     {CIB_OP_ERASE,      FALSE, cib_process_erase},
     {CIB_OP_UPGRADE,    FALSE, cib_process_upgrade},
 };
 /* *INDENT-ON* */
 
 int
 cib_file_perform_op(cib_t * cib, const char *op, const char *host, const char *section,
                     xmlNode * data, xmlNode ** output_data, int call_options)
 {
     return cib_file_perform_op_delegate(cib, op, host, section, data, output_data, call_options,
                                         NULL);
 }
 
 int
 cib_file_perform_op_delegate(cib_t * cib, const char *op, const char *host, const char *section,
                              xmlNode * data, xmlNode ** output_data, int call_options,
                              const char *user_name)
 {
     int rc = pcmk_ok;
     char *effective_user = NULL;
     gboolean query = FALSE;
     gboolean changed = FALSE;
     xmlNode *request = NULL;
     xmlNode *output = NULL;
     xmlNode *cib_diff = NULL;
     xmlNode *result_cib = NULL;
     cib_op_t *fn = NULL;
     int lpc = 0;
     static int max_msg_types = DIMOF(cib_file_ops);
 
     crm_info("%s on %s", op, section);
     call_options |= (cib_no_mtime | cib_inhibit_bcast | cib_scope_local);
 
     if (cib->state == cib_disconnected) {
         return -ENOTCONN;
     }
 
     if (output_data != NULL) {
         *output_data = NULL;
     }
 
     if (op == NULL) {
         return -EINVAL;
     }
 
     for (lpc = 0; lpc < max_msg_types; lpc++) {
         if (safe_str_eq(op, cib_file_ops[lpc].op)) {
             fn = &(cib_file_ops[lpc].fn);
             query = cib_file_ops[lpc].read_only;
             break;
         }
     }
 
     if (fn == NULL) {
         return -EPROTONOSUPPORT;
     }
 
     cib->call_id++;
     request = cib_create_op(cib->call_id, "dummy-token", op, host, section, data, call_options, user_name);
 #if ENABLE_ACL
     if(user_name != NULL) {
         effective_user = uid2username(geteuid());
         crm_trace("Checking if %s can impersonate %s", effective_user, user_name);
         determine_request_user(effective_user, request, F_CIB_USER);
     }
     crm_trace("Performing %s operation as %s", op, crm_element_value(request, F_CIB_USER));
 #endif
     rc = cib_perform_op(op, call_options, fn, query,
                         section, request, data, TRUE, &changed, in_mem_cib, &result_cib, &cib_diff,
                         &output);
 
     free_xml(request);
     if (rc == -pcmk_err_dtd_validation) {
         validate_xml_verbose(result_cib);
     }
 
     if (rc != pcmk_ok) {
         free_xml(result_cib);
 
     } else if (query == FALSE) {
         xml_log_patchset(LOG_DEBUG, "cib:diff", cib_diff);
         free_xml(in_mem_cib);
         in_mem_cib = result_cib;
     }
 
     free_xml(cib_diff);
 
     if (cib->op_callback != NULL) {
         cib->op_callback(NULL, cib->call_id, rc, output);
     }
 
     if (output_data && output) {
-        *output_data = copy_xml(output);
-    }
+        if(output == in_mem_cib) {
+            *output_data = copy_xml(output);
+        } else {
+            *output_data = output;
+        }
 
-    if (query == FALSE || (call_options & cib_no_children)) {
-        free_xml(output);
-    } else if (safe_str_eq(crm_element_name(output), "xpath-query")) {
+    } else if(output != in_mem_cib) {
         free_xml(output);
     }
 
     free(effective_user);
     return rc;
 }
diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c
index f1326d505d..fe5413f118 100644
--- a/lib/cib/cib_utils.c
+++ b/lib/cib/cib_utils.c
@@ -1,840 +1,850 @@
 /*
  * Copyright (c) 2004 International Business Machines
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  *
  */
 #include <crm_internal.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdarg.h>
 #include <string.h>
 #include <sys/utsname.h>
 
 #include <glib.h>
 
 #include <crm/crm.h>
 #include <crm/cib/internal.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/pengine/rules.h>
 
 struct config_root_s {
     const char *name;
     const char *parent;
     const char *path;
 };
 
  /*
   * "//crm_config" will also work in place of "/cib/configuration/crm_config"
   * The / prefix means find starting from the root, whereas the // prefix means
   * find anywhere and risks multiple matches
   */
 /* *INDENT-OFF* */
 struct config_root_s known_paths[] = {
     { NULL,			NULL,                 "//cib" },
     { XML_TAG_CIB,		NULL,                 "//cib" },
     { XML_CIB_TAG_STATUS,       "/cib",               "//cib/status" },
     { XML_CIB_TAG_CONFIGURATION,"/cib",               "//cib/configuration" },
     { XML_CIB_TAG_CRMCONFIG,    "/cib/configuration", "//cib/configuration/crm_config" },
     { XML_CIB_TAG_NODES,        "/cib/configuration", "//cib/configuration/nodes" },
     { XML_CIB_TAG_DOMAINS,      "/cib/configuration", "//cib/configuration/domains" },
     { XML_CIB_TAG_RESOURCES,    "/cib/configuration", "//cib/configuration/resources" },
     { XML_CIB_TAG_CONSTRAINTS,  "/cib/configuration", "//cib/configuration/constraints" },
     { XML_CIB_TAG_OPCONFIG,	"/cib/configuration", "//cib/configuration/op_defaults" },
     { XML_CIB_TAG_RSCCONFIG,	"/cib/configuration", "//cib/configuration/rsc_defaults" },
     { XML_CIB_TAG_ACLS,		"/cib/configuration", "//cib/configuration/acls" },
     { XML_TAG_FENCING_TOPOLOGY,	"/cib/configuration", "//cib/configuration/fencing-topology" },
     { XML_CIB_TAG_SECTION_ALL,  NULL,                 "//cib" },
 };
 /* *INDENT-ON* */
 
 int
 cib_compare_generation(xmlNode * left, xmlNode * right)
 {
     int lpc = 0;
 
     const char *attributes[] = {
         XML_ATTR_GENERATION_ADMIN,
         XML_ATTR_GENERATION,
         XML_ATTR_NUMUPDATES,
     };
 
     crm_log_xml_trace(left, "left");
     crm_log_xml_trace(right, "right");
 
     for (lpc = 0; lpc < DIMOF(attributes); lpc++) {
         int int_elem_l = -1;
         int int_elem_r = -1;
         const char *elem_r = NULL;
         const char *elem_l = crm_element_value(left, attributes[lpc]);
 
         if (right != NULL) {
             elem_r = crm_element_value(right, attributes[lpc]);
         }
 
         if (elem_l != NULL) {
             int_elem_l = crm_parse_int(elem_l, NULL);
         }
         if (elem_r != NULL) {
             int_elem_r = crm_parse_int(elem_r, NULL);
         }
 
         if (int_elem_l < int_elem_r) {
             crm_trace("%s (%s < %s)", attributes[lpc], crm_str(elem_l), crm_str(elem_r));
             return -1;
 
         } else if (int_elem_l > int_elem_r) {
             crm_trace("%s (%s > %s)", attributes[lpc], crm_str(elem_l), crm_str(elem_r));
             return 1;
         }
     }
 
     return 0;
 }
 
 xmlNode *
 get_cib_copy(cib_t * cib)
 {
     xmlNode *xml_cib;
     int options = cib_scope_local | cib_sync_call;
 
     if (cib->cmds->query(cib, NULL, &xml_cib, options) != pcmk_ok) {
         crm_err("Couldnt retrieve the CIB");
+        free_xml(xml_cib);
         return NULL;
+
     } else if (xml_cib == NULL) {
         crm_err("The CIB result was empty");
+        free_xml(xml_cib);
         return NULL;
     }
 
     if (safe_str_eq(crm_element_name(xml_cib), XML_TAG_CIB)) {
         return xml_cib;
     }
     free_xml(xml_cib);
     return NULL;
 }
 
 xmlNode *
 cib_get_generation(cib_t * cib)
 {
     xmlNode *the_cib = get_cib_copy(cib);
     xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);
 
     if (the_cib != NULL) {
         copy_in_properties(generation, the_cib);
         free_xml(the_cib);
     }
 
     return generation;
 }
 
 gboolean
 cib_version_details(xmlNode * cib, int *admin_epoch, int *epoch, int *updates)
 {
     *epoch = -1;
     *updates = -1;
     *admin_epoch = -1;
 
     if (cib == NULL) {
         return FALSE;
 
     } else {
         crm_element_value_int(cib, XML_ATTR_GENERATION, epoch);
         crm_element_value_int(cib, XML_ATTR_NUMUPDATES, updates);
         crm_element_value_int(cib, XML_ATTR_GENERATION_ADMIN, admin_epoch);
     }
     return TRUE;
 }
 
 gboolean
 cib_diff_version_details(xmlNode * diff, int *admin_epoch, int *epoch, int *updates,
                          int *_admin_epoch, int *_epoch, int *_updates)
 {
     int add[3];
     int del[3];
 
     xml_patch_versions(diff, add, del);
 
     *admin_epoch = add[0];
     *epoch = add[1];
     *updates = add[2];
 
     *_admin_epoch = del[0];
     *_epoch = del[1];
     *_updates = del[2];
 
     return TRUE;
 }
 
 /*
  * The caller should never free the return value
  */
 
 const char *
 get_object_path(const char *object_type)
 {
     int lpc = 0;
     int max = DIMOF(known_paths);
 
     for (; lpc < max; lpc++) {
         if ((object_type == NULL && known_paths[lpc].name == NULL)
             || safe_str_eq(object_type, known_paths[lpc].name)) {
             return known_paths[lpc].path;
         }
     }
     return NULL;
 }
 
 const char *
 get_object_parent(const char *object_type)
 {
     int lpc = 0;
     int max = DIMOF(known_paths);
 
     for (; lpc < max; lpc++) {
         if (safe_str_eq(object_type, known_paths[lpc].name)) {
             return known_paths[lpc].parent;
         }
     }
     return NULL;
 }
 
 xmlNode *
 get_object_root(const char *object_type, xmlNode * the_root)
 {
     const char *xpath = get_object_path(object_type);
 
     if (xpath == NULL) {
         return the_root;        /* or return NULL? */
     }
 
     return get_xpath_object(xpath, the_root, LOG_DEBUG_4);
 }
 
 xmlNode *
 create_cib_fragment_adv(xmlNode * update, const char *update_section, const char *source)
 {
     xmlNode *cib = NULL;
     gboolean whole_cib = FALSE;
     xmlNode *object_root = NULL;
     char *local_section = NULL;
 
 /* 	crm_debug("Creating a blank fragment: %s", update_section); */
 
     if (update == NULL && update_section == NULL) {
         crm_trace("Creating a blank fragment");
         update = createEmptyCib();
         crm_xml_add(cib, XML_ATTR_ORIGIN, source);
         return update;
 
     } else if (update == NULL) {
         crm_err("No update to create a fragment for");
         return NULL;
 
     }
 
     CRM_CHECK(update_section != NULL, return NULL);
     if (safe_str_eq(crm_element_name(update), XML_TAG_CIB)) {
         whole_cib = TRUE;
     }
 
     if (whole_cib == FALSE) {
         cib = createEmptyCib();
         crm_xml_add(cib, XML_ATTR_ORIGIN, source);
         object_root = get_object_root(update_section, cib);
         add_node_copy(object_root, update);
 
     } else {
         cib = copy_xml(update);
         crm_xml_add(cib, XML_ATTR_ORIGIN, source);
     }
 
     free(local_section);
     crm_trace("Verifying created fragment");
     return cib;
 }
 
 /*
  * It is the callers responsibility to free both the new CIB (output)
  *     and the new CIB (input)
  */
 xmlNode *
 createEmptyCib(void)
 {
     xmlNode *cib_root = NULL, *config = NULL;
 
     cib_root = create_xml_node(NULL, XML_TAG_CIB);
 
     config = create_xml_node(cib_root, XML_CIB_TAG_CONFIGURATION);
     create_xml_node(cib_root, XML_CIB_TAG_STATUS);
 
     create_xml_node(config, XML_CIB_TAG_CRMCONFIG);
     create_xml_node(config, XML_CIB_TAG_NODES);
     create_xml_node(config, XML_CIB_TAG_RESOURCES);
     create_xml_node(config, XML_CIB_TAG_CONSTRAINTS);
 
     return cib_root;
 }
 
 static bool
 cib_acl_enabled(xmlNode *xml, const char *user)
 {
     bool rc = FALSE;
 
 #if ENABLE_ACL
     if(pcmk_acl_required(user)) {
         const char *value = NULL;
         GHashTable *options = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
 
         cib_read_config(options, xml);
         value = cib_pref(options, "enable-acl");
         rc = crm_is_true(value);
         g_hash_table_destroy(options);
     }
 
     crm_debug("CIB ACL is %s", rc ? "enabled" : "disabled");
 #endif
     return rc;
 }
 
 int
 cib_perform_op(const char *op, int call_options, cib_op_t * fn, gboolean is_query,
                const char *section, xmlNode * req, xmlNode * input,
                gboolean manage_counters, gboolean * config_changed,
                xmlNode * current_cib, xmlNode ** result_cib, xmlNode ** diff, xmlNode ** output)
 {
     int rc = pcmk_ok;
     gboolean check_dtd = TRUE;
     xmlNode *top = NULL;
     xmlNode *scratch = NULL;
     xmlNode *local_diff = NULL;
 
     const char *new_version = NULL;
     static struct qb_log_callsite *diff_cs = NULL;
     const char *user = crm_element_value(req, F_CIB_USER);
 
     crm_trace("Begin %s%s op", is_query ? "read-only " : "", op);
 
     CRM_CHECK(output != NULL, return -ENOMSG);
     CRM_CHECK(result_cib != NULL, return -ENOMSG);
     CRM_CHECK(config_changed != NULL, return -ENOMSG);
 
     *output = NULL;
     *result_cib = NULL;
     *config_changed = FALSE;
 
     if (fn == NULL) {
         return -EINVAL;
     }
 
     if (is_query) {
         xmlNode *cib_ro = current_cib;
         xmlNode *cib_filtered = NULL;
 
         if(cib_acl_enabled(cib_ro, user)) {
             if(xml_acl_filtered_copy(user, cib_ro, &cib_filtered)) {
                 if (cib_filtered == NULL) {
                     crm_debug("Pre-filtered the entire cib");
                     return -EACCES;
                 }
                 cib_ro = cib_filtered;
                 crm_log_xml_trace(cib_ro, "filtered");
             }
         }
 
         rc = (*fn) (op, call_options, section, req, input, cib_ro, result_cib, output);
-        if(cib_filtered) {
-            if(*output == cib_filtered) {
-                cib_filtered = NULL;
 
-            } else if(*output) {
-                /* We're about to free the source XML */
-                *output = copy_xml(*output);
-            }
-            free_xml(cib_filtered);
+        if(output == NULL) {
+            /* nothing */
+
+        } else if(cib_filtered == *output) {
+            cib_filtered = NULL; /* Let them have this copy */
+
+        } else if(cib_filtered) {
+            /* We're about to free the document of which *output is a part */
+            *output = copy_xml(*output);
+
+        } else if(*output != current_cib) {
+            /* Give them a copy they can free */
+            *output = copy_xml(*output);
         }
+
+        free_xml(cib_filtered);
         return rc;
     }
 
 
     if (is_set(call_options, cib_zero_copy)) {
         /* Conditional on v2 patch style */
 
         scratch = current_cib;
 
         /* Create a shallow copy of current_cib for the version details */
         current_cib = create_xml_node(NULL, (const char *)scratch->name);
         copy_in_properties(current_cib, scratch);
         top = current_cib;
 
         xml_track_changes(scratch, user, cib_acl_enabled(scratch, user));
         rc = (*fn) (op, call_options, section, req, input, scratch, &scratch, output);
 
     } else {
         scratch = copy_xml(current_cib);
         xml_track_changes(scratch, user, cib_acl_enabled(scratch, user));
         rc = (*fn) (op, call_options, section, req, input, current_cib, &scratch, output);
 
         if(xml_tracking_changes(scratch) == FALSE) {
             crm_trace("Inferring changes after %s op", op);
             xml_track_changes(scratch, user, cib_acl_enabled(scratch, user));
             xml_calculate_changes(current_cib, scratch);
         }
         CRM_CHECK(current_cib != scratch, return -EINVAL);
     }
 
     xml_acl_disable(scratch); /* Allow the system to make any additional changes */
 
     if (rc == pcmk_ok && scratch == NULL) {
         rc = -EINVAL;
         goto done;
 
     } else if(rc == pcmk_ok && xml_acl_denied(scratch)) {
         crm_trace("ACL rejected part or all of the proposed changes");
         rc = -EACCES;
         goto done;
 
     } else if (rc != pcmk_ok) {
         goto done;
     }
 
     if (scratch) {
         new_version = crm_element_value(scratch, XML_ATTR_CRM_VERSION);
 
         if (new_version && compare_version(new_version, CRM_FEATURE_SET) > 0) {
             crm_err("Discarding update with feature set '%s' greater than our own '%s'",
                     new_version, CRM_FEATURE_SET);
             rc = -EPROTONOSUPPORT;
             goto done;
         }
     }
 
     if (current_cib) {
         int old = 0;
         int new = 0;
 
         crm_element_value_int(scratch, XML_ATTR_GENERATION_ADMIN, &new);
         crm_element_value_int(current_cib, XML_ATTR_GENERATION_ADMIN, &old);
 
         if (old > new) {
             crm_err("%s went backwards: %d -> %d (Opts: 0x%x)",
                     XML_ATTR_GENERATION_ADMIN, old, new, call_options);
             crm_log_xml_warn(req, "Bad Op");
             crm_log_xml_warn(input, "Bad Data");
             rc = -pcmk_err_old_data;
 
         } else if (old == new) {
             crm_element_value_int(scratch, XML_ATTR_GENERATION, &new);
             crm_element_value_int(current_cib, XML_ATTR_GENERATION, &old);
             if (old > new) {
                 crm_err("%s went backwards: %d -> %d (Opts: 0x%x)",
                         XML_ATTR_GENERATION, old, new, call_options);
                 crm_log_xml_warn(req, "Bad Op");
                 crm_log_xml_warn(input, "Bad Data");
                 rc = -pcmk_err_old_data;
             }
         }
     }
 
     crm_trace("Massaging CIB contents");
     strip_text_nodes(scratch);
     fix_plus_plus_recursive(scratch);
 
     if (is_set(call_options, cib_zero_copy)) {
         /* At this point, current_cib is just the 'cib' tag and its properties,
          *
          * The v1 format would barf on this, but we know the v2 patch
          * format only needs it for the top-level version fields
          */
         local_diff = xml_create_patchset(2, current_cib, scratch, (bool*)config_changed, manage_counters, FALSE);
 
     } else {
         static time_t expires = 0;
         time_t tm_now = time(NULL);
         bool with_digest = FALSE;
 
         if (expires < tm_now) {
             expires = tm_now + 60;  /* Validate clients are correctly applying v2-style diffs at most once a minute */
             with_digest = TRUE;
         }
 
         local_diff = xml_create_patchset(0, current_cib, scratch, (bool*)config_changed, manage_counters, with_digest);
     }
 
     if (diff_cs == NULL) {
         diff_cs = qb_log_callsite_get(__PRETTY_FUNCTION__, __FILE__, "diff-validation", LOG_DEBUG, __LINE__, crm_trace_nonlog);
     }
 
     xml_log_changes(LOG_INFO, __FUNCTION__, scratch);
     if (is_not_set(call_options, cib_zero_copy) /* The original to compare against doesn't exist */
         && local_diff
         && crm_is_callsite_active(diff_cs, LOG_TRACE, 0)) {
 
         /* Validate the calculated patch set */
         int test_rc, format = 1;
         xmlNode * c = copy_xml(current_cib);
 
         crm_element_value_int(local_diff, "format", &format);
         test_rc = xml_apply_patchset(c, local_diff, manage_counters);
 
         if(test_rc != pcmk_ok) {
             save_xml_to_file(c,           "PatchApply:calculated", NULL);
             save_xml_to_file(current_cib, "PatchApply:input", NULL);
             save_xml_to_file(scratch,     "PatchApply:actual", NULL);
             save_xml_to_file(local_diff,  "PatchApply:diff", NULL);
             crm_err("v%d patchset error, patch failed to apply: %s (%d)", format, pcmk_strerror(test_rc), test_rc);
         }
         free_xml(c);
     }
 
     xml_accept_changes(scratch);
 
     if (safe_str_eq(section, XML_CIB_TAG_STATUS)) {
         /* Throttle the amount of costly validation we perform due to status updates
          * a) we don't really care whats in the status section
          * b) we don't validate any of it's contents at the moment anyway
          */
         check_dtd = FALSE;
     }
 
     /* === scratch must not be modified after this point ===
      * Exceptions, anything in:
 
      static filter_t filter[] = {
      { 0, XML_ATTR_ORIGIN },
      { 0, XML_CIB_ATTR_WRITTEN },
      { 0, XML_ATTR_UPDATE_ORIG },
      { 0, XML_ATTR_UPDATE_CLIENT },
      { 0, XML_ATTR_UPDATE_USER },
      };
      */
 
     if (*config_changed && is_not_set(call_options, cib_no_mtime)) {
         char *now_str = NULL;
         time_t now = time(NULL);
         const char *schema = crm_element_value(scratch, XML_ATTR_VALIDATION);
 
         now_str = ctime(&now);
         now_str[24] = EOS;      /* replace the newline */
         crm_xml_replace(scratch, XML_CIB_ATTR_WRITTEN, now_str);
 
         if (schema) {
             static int minimum_schema = 0;
             int current_schema = get_schema_version(schema);
 
             if (minimum_schema == 0) {
                 minimum_schema = get_schema_version("pacemaker-1.1");
             }
 
             /* Does the CIB support the "update-*" attributes... */
             if (current_schema >= minimum_schema) {
                 const char *origin = crm_element_value(req, F_ORIG);
 
                 CRM_LOG_ASSERT(origin != NULL);
                 crm_xml_replace(scratch, XML_ATTR_UPDATE_ORIG, origin);
                 crm_xml_replace(scratch, XML_ATTR_UPDATE_CLIENT,
                                 crm_element_value(req, F_CIB_CLIENTNAME));
 #if ENABLE_ACL
                 crm_xml_replace(scratch, XML_ATTR_UPDATE_USER, crm_element_value(req, F_CIB_USER));
 #endif
             }
         }
     }
 
     crm_trace("Perform validation: %s", check_dtd ? "true" : "false");
     if (rc == pcmk_ok && check_dtd && validate_xml(scratch, NULL, TRUE) == FALSE) {
         const char *current_dtd = crm_element_value(scratch, XML_ATTR_VALIDATION);
 
         crm_warn("Updated CIB does not validate against %s schema/dtd", crm_str(current_dtd));
         rc = -pcmk_err_dtd_validation;
     }
 
   done:
 
     *result_cib = scratch;
 #if ENABLE_ACL
     if(rc != pcmk_ok && cib_acl_enabled(current_cib, user)) {
         if(xml_acl_filtered_copy(user, scratch, result_cib)) {
             if (*result_cib == NULL) {
                 crm_debug("Pre-filtered the entire cib result");
             }
             free_xml(scratch);
         }
     }
 #endif
 
     if(diff) {
         *diff = local_diff;
     } else {
         free_xml(local_diff);
     }
 
     free_xml(top);
     crm_trace("Done");
     return rc;
 }
 
 xmlNode *
 cib_create_op(int call_id, const char *token, const char *op, const char *host, const char *section,
               xmlNode * data, int call_options, const char *user_name)
 {
     xmlNode *op_msg = create_xml_node(NULL, "cib_command");
 
     CRM_CHECK(op_msg != NULL, return NULL);
     CRM_CHECK(token != NULL, return NULL);
 
     crm_xml_add(op_msg, F_XML_TAGNAME, "cib_command");
 
     crm_xml_add(op_msg, F_TYPE, T_CIB);
     crm_xml_add(op_msg, F_CIB_CALLBACK_TOKEN, token);
     crm_xml_add(op_msg, F_CIB_OPERATION, op);
     crm_xml_add(op_msg, F_CIB_HOST, host);
     crm_xml_add(op_msg, F_CIB_SECTION, section);
     crm_xml_add_int(op_msg, F_CIB_CALLID, call_id);
 #if ENABLE_ACL
     if (user_name) {
         crm_xml_add(op_msg, F_CIB_USER, user_name);
     }
 #endif
     crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options);
     crm_xml_add_int(op_msg, F_CIB_CALLOPTS, call_options);
 
     if (data != NULL) {
         add_message_xml(op_msg, F_CIB_CALLDATA, data);
     }
 
     if (call_options & cib_inhibit_bcast) {
         CRM_CHECK((call_options & cib_scope_local), return NULL);
     }
     return op_msg;
 }
 
 void
 cib_native_callback(cib_t * cib, xmlNode * msg, int call_id, int rc)
 {
     xmlNode *output = NULL;
     cib_callback_client_t *blob = NULL;
     cib_callback_client_t local_blob;
 
     local_blob.id = NULL;
     local_blob.callback = NULL;
     local_blob.user_data = NULL;
     local_blob.only_success = FALSE;
 
     if (msg != NULL) {
         crm_element_value_int(msg, F_CIB_RC, &rc);
         crm_element_value_int(msg, F_CIB_CALLID, &call_id);
         output = get_message_xml(msg, F_CIB_CALLDATA);
     }
 
     blob = g_hash_table_lookup(cib_op_callback_table, GINT_TO_POINTER(call_id));
 
     if (blob != NULL) {
         local_blob = *blob;
         blob = NULL;
 
         remove_cib_op_callback(call_id, FALSE);
 
     } else {
         crm_trace("No callback found for call %d", call_id);
         local_blob.callback = NULL;
     }
 
     if (cib == NULL) {
         crm_debug("No cib object supplied");
     }
 
     if (rc == -pcmk_err_diff_resync) {
         /* This is an internal value that clients do not and should not care about */
         rc = pcmk_ok;
     }
 
     if (local_blob.callback != NULL && (rc == pcmk_ok || local_blob.only_success == FALSE)) {
         crm_trace("Invoking callback %s for call %d", crm_str(local_blob.id), call_id);
         local_blob.callback(msg, call_id, rc, output, local_blob.user_data);
 
     } else if (cib && cib->op_callback == NULL && rc != pcmk_ok) {
         crm_warn("CIB command failed: %s", pcmk_strerror(rc));
         crm_log_xml_debug(msg, "Failed CIB Update");
     }
 
     if (cib && cib->op_callback != NULL) {
         crm_trace("Invoking global callback for call %d", call_id);
         cib->op_callback(msg, call_id, rc, output);
     }
     crm_trace("OP callback activated.");
 }
 
 void
 cib_native_notify(gpointer data, gpointer user_data)
 {
     xmlNode *msg = user_data;
     cib_notify_client_t *entry = data;
     const char *event = NULL;
 
     if (msg == NULL) {
         crm_warn("Skipping callback - NULL message");
         return;
     }
 
     event = crm_element_value(msg, F_SUBTYPE);
 
     if (entry == NULL) {
         crm_warn("Skipping callback - NULL callback client");
         return;
 
     } else if (entry->callback == NULL) {
         crm_warn("Skipping callback - NULL callback");
         return;
 
     } else if (safe_str_neq(entry->event, event)) {
         crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event);
         return;
     }
 
     crm_trace("Invoking callback for %p/%s event...", entry, event);
     entry->callback(event, msg);
     crm_trace("Callback invoked...");
 }
 
 pe_cluster_option cib_opts[] = {
     /* name, old-name, validate, default, description */
     {"enable-acl", NULL, "boolean", NULL, "false", &check_boolean,
      "Enable CIB ACL", NULL}
     ,
 };
 
 void
 cib_metadata(void)
 {
     config_metadata("Cluster Information Base", "1.0",
                     "Cluster Information Base Options",
                     "This is a fake resource that details the options that can be configured for the Cluster Information Base.",
                     cib_opts, DIMOF(cib_opts));
 }
 
 void
 verify_cib_options(GHashTable * options)
 {
     verify_all_options(options, cib_opts, DIMOF(cib_opts));
 }
 
 const char *
 cib_pref(GHashTable * options, const char *name)
 {
     return get_cluster_pref(options, cib_opts, DIMOF(cib_opts), name);
 }
 
 gboolean
 cib_read_config(GHashTable * options, xmlNode * current_cib)
 {
     xmlNode *config = NULL;
     crm_time_t *now = NULL;
 
     if (options == NULL || current_cib == NULL) {
         return FALSE;
     }
 
     now = crm_time_new(NULL);
 
     g_hash_table_remove_all(options);
 
     config = get_object_root(XML_CIB_TAG_CRMCONFIG, current_cib);
     if (config) {
         unpack_instance_attributes(current_cib, config, XML_CIB_TAG_PROPSET, NULL, options,
                                    CIB_OPTIONS_FIRST, FALSE, now);
     }
 
     verify_cib_options(options);
 
     crm_time_free(now);
 
     return TRUE;
 }
 
 int
 cib_apply_patch_event(xmlNode * event, xmlNode * input, xmlNode ** output, int level)
 {
     int rc = pcmk_err_generic;
 
     xmlNode *diff = NULL;
 
     CRM_ASSERT(event);
     CRM_ASSERT(input);
     CRM_ASSERT(output);
 
     crm_element_value_int(event, F_CIB_RC, &rc);
     diff = get_message_xml(event, F_CIB_UPDATE_RESULT);
 
     if (rc < pcmk_ok || diff == NULL) {
         return rc;
     }
 
     if (level > LOG_CRIT) {
         xml_log_patchset(level, "Config update", diff);
     }
 
     if (input != NULL) {
         rc = cib_process_diff(NULL, cib_none, NULL, event, diff, input, output, NULL);
 
         if (rc != pcmk_ok) {
             crm_debug("Update didn't apply: %s (%d) %p", pcmk_strerror(rc), rc, *output);
             free_xml(*output); *output = NULL;
             return rc;
         }
     }
 
     return rc;
 }
 
 gboolean
 cib_internal_config_changed(xmlNode * diff)
 {
     gboolean changed = FALSE;
     xmlXPathObject *xpathObj = NULL;
 
     if (diff == NULL) {
         return FALSE;
     }
 
     xpathObj = xpath_search(diff, "//" XML_CIB_TAG_CRMCONFIG);
     if (numXpathResults(xpathObj) > 0) {
         changed = TRUE;
     }
 
     freeXpathObject(xpathObj);
 
     return changed;
 }
 
 int
 cib_internal_op(cib_t * cib, const char *op, const char *host,
                 const char *section, xmlNode * data,
                 xmlNode ** output_data, int call_options, const char *user_name)
 {
     int (*delegate) (cib_t * cib, const char *op, const char *host,
                      const char *section, xmlNode * data,
                      xmlNode ** output_data, int call_options, const char *user_name) =
         cib->delegate_fn;
 
 #if ENABLE_ACL
     if(user_name == NULL) {
         user_name = getenv("CIB_user");
     }
 #endif
 
     return delegate(cib, op, host, section, data, output_data, call_options, user_name);
 }
diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index 1d110031a0..edf458125a 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1,2251 +1,2253 @@
 
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <libgen.h>
 
 #include <crm/msg_xml.h>
 #include <crm/services.h>
 #include <crm/common/xml.h>
 #include <crm/common/mainloop.h>
 
 #include <crm/cib.h>
 #include <crm/attrd.h>
 #include <crm/pengine/rules.h>
 #include <crm/pengine/status.h>
 #include <crm/pengine/internal.h>
 
 bool scope_master = FALSE;
 gboolean do_force = FALSE;
 gboolean BE_QUIET = FALSE;
 const char *attr_set_type = XML_TAG_ATTR_SETS;
 char *host_id = NULL;
 const char *rsc_id = NULL;
 const char *host_uname = NULL;
 const char *prop_name = NULL;
 const char *prop_value = NULL;
 const char *rsc_type = NULL;
 const char *prop_id = NULL;
 const char *prop_set = NULL;
 char *move_lifetime = NULL;
 char rsc_cmd = 'L';
 const char *rsc_long_cmd = NULL;
 char *our_pid = NULL;
 crm_ipc_t *crmd_channel = NULL;
 char *xml_file = NULL;
 int cib_options = cib_sync_call;
 int crmd_replies_needed = 1; /* The welcome message */
 GMainLoop *mainloop = NULL;
 gboolean print_pending = FALSE;
 
 extern void cleanup_alloc_calculations(pe_working_set_t * data_set);
 
 #define CMD_ERR(fmt, args...) do {		\
 	crm_warn(fmt, ##args);			\
 	fprintf(stderr, fmt, ##args);		\
     } while(0)
 
 #define message_timeout_ms 60*1000
 
 static gboolean
 resource_ipc_timeout(gpointer data)
 {
     fprintf(stderr, "No messages received in %d seconds.. aborting\n",
             (int)message_timeout_ms / 1000);
     crm_err("No messages received in %d seconds", (int)message_timeout_ms / 1000);
     return crm_exit(-1);
 }
 
 static void
 resource_ipc_connection_destroy(gpointer user_data)
 {
     crm_info("Connection to CRMd was terminated");
     crm_exit(1);
 }
 
 static void
 start_mainloop(void)
 {
     mainloop = g_main_new(FALSE);
     fprintf(stderr, "Waiting for %d replies from the CRMd", crmd_replies_needed);
     crm_debug("Waiting for %d replies from the CRMd", crmd_replies_needed);
 
     g_timeout_add(message_timeout_ms, resource_ipc_timeout, NULL);
     g_main_run(mainloop);
 }
 
 static int
 resource_ipc_callback(const char *buffer, ssize_t length, gpointer userdata)
 {
     xmlNode *msg = string2xml(buffer);
 
     fprintf(stderr, ".");
     crm_log_xml_trace(msg, "[inbound]");
 
     crmd_replies_needed--;
     if (crmd_replies_needed == 0) {
         fprintf(stderr, " OK\n");
         crm_debug("Got all the replies we expected");
         return crm_exit(pcmk_ok);
     }
 
     free_xml(msg);
     return 0;
 }
 
 struct ipc_client_callbacks crm_callbacks = {
     .dispatch = resource_ipc_callback,
     .destroy = resource_ipc_connection_destroy,
 };
 
 static int
 do_find_resource(const char *rsc, resource_t * the_rsc, pe_working_set_t * data_set)
 {
     int found = 0;
     GListPtr lpc = NULL;
 
     if (the_rsc == NULL) {
         the_rsc = pe_find_resource(data_set->resources, rsc);
     }
 
     if (the_rsc == NULL) {
         return -ENXIO;
     }
 
     if (the_rsc->variant >= pe_clone) {
         GListPtr gIter = the_rsc->children;
 
         for (; gIter != NULL; gIter = gIter->next) {
             found += do_find_resource(rsc, gIter->data, data_set);
         }
         return found;
     }
 
     for (lpc = the_rsc->running_on; lpc != NULL; lpc = lpc->next) {
         node_t *node = (node_t *) lpc->data;
 
         crm_trace("resource %s is running on: %s", rsc, node->details->uname);
         if (BE_QUIET) {
             fprintf(stdout, "%s\n", node->details->uname);
         } else {
             const char *state = "";
 
             if (the_rsc->variant == pe_native && the_rsc->role == RSC_ROLE_MASTER) {
                 state = "Master";
             }
             fprintf(stdout, "resource %s is running on: %s %s\n", rsc, node->details->uname, state);
         }
 
         found++;
     }
 
     if (BE_QUIET == FALSE && found == 0) {
         fprintf(stderr, "resource %s is NOT running\n", rsc);
     }
 
     return 0;
 }
 
 #define cons_string(x) x?x:"NA"
 static void
 print_cts_constraints(pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
     xmlNode *lifetime = NULL;
     xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);
 
     for (xml_obj = __xml_first_child(cib_constraints); xml_obj != NULL;
          xml_obj = __xml_next(xml_obj)) {
         const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
 
         if (id == NULL) {
             continue;
         }
 
         lifetime = first_named_child(xml_obj, "lifetime");
 
         if (test_ruleset(lifetime, NULL, data_set->now) == FALSE) {
             continue;
         }
 
         if (safe_str_eq(XML_CONS_TAG_RSC_DEPEND, crm_element_name(xml_obj))) {
             printf("Constraint %s %s %s %s %s %s %s\n",
                    crm_element_name(xml_obj),
                    cons_string(crm_element_value(xml_obj, XML_ATTR_ID)),
                    cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE)),
                    cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET)),
                    cons_string(crm_element_value(xml_obj, XML_RULE_ATTR_SCORE)),
                    cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE)),
                    cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE)));
 
         } else if (safe_str_eq(XML_CONS_TAG_RSC_LOCATION, crm_element_name(xml_obj))) {
             /* unpack_location(xml_obj, data_set); */
         }
     }
 }
 
 static void
 print_cts_rsc(resource_t * rsc)
 {
     GListPtr lpc = NULL;
     const char *host = NULL;
     gboolean needs_quorum = TRUE;
     const char *rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
     const char *rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
     const char *rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 
     if (safe_str_eq(rclass, "stonith")) {
         xmlNode *op = NULL;
 
         needs_quorum = FALSE;
 
         for (op = __xml_first_child(rsc->ops_xml); op != NULL; op = __xml_next(op)) {
             if (crm_str_eq((const char *)op->name, "op", TRUE)) {
                 const char *name = crm_element_value(op, "name");
 
                 if (safe_str_neq(name, CRMD_ACTION_START)) {
                     const char *value = crm_element_value(op, "requires");
 
                     if (safe_str_eq(value, "nothing")) {
                         needs_quorum = FALSE;
                     }
                     break;
                 }
             }
         }
     }
 
     if (rsc->running_on != NULL && g_list_length(rsc->running_on) == 1) {
         node_t *tmp = rsc->running_on->data;
 
         host = tmp->details->uname;
     }
 
     printf("Resource: %s %s %s %s %s %s %s %s %d %lld 0x%.16llx\n",
            crm_element_name(rsc->xml), rsc->id,
            rsc->clone_name ? rsc->clone_name : rsc->id, rsc->parent ? rsc->parent->id : "NA",
            rprov ? rprov : "NA", rclass, rtype, host ? host : "NA", needs_quorum, rsc->flags,
            rsc->flags);
 
     for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
         resource_t *child = (resource_t *) lpc->data;
 
         print_cts_rsc(child);
     }
 }
 
 static void
 print_raw_rsc(resource_t * rsc)
 {
     GListPtr lpc = NULL;
     GListPtr children = rsc->children;
 
     if (children == NULL) {
         printf("%s\n", rsc->id);
     }
 
     for (lpc = children; lpc != NULL; lpc = lpc->next) {
         resource_t *child = (resource_t *) lpc->data;
 
         print_raw_rsc(child);
     }
 }
 
 static int
 do_find_resource_list(pe_working_set_t * data_set, gboolean raw)
 {
     int found = 0;
 
     GListPtr lpc = NULL;
     int opts = pe_print_printf | pe_print_rsconly;
 
     if (print_pending) {
         opts |= pe_print_pending;
     }
 
     for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
         resource_t *rsc = (resource_t *) lpc->data;
 
         if (is_set(rsc->flags, pe_rsc_orphan)
             && rsc->fns->active(rsc, TRUE) == FALSE) {
             continue;
         }
         rsc->fns->print(rsc, NULL, opts, stdout);
         found++;
     }
 
     if (found == 0) {
         printf("NO resources configured\n");
         return -ENXIO;
     }
 
     return 0;
 }
 
 static resource_t *
 find_rsc_or_clone(const char *rsc, pe_working_set_t * data_set)
 {
     resource_t *the_rsc = pe_find_resource(data_set->resources, rsc);
 
     if (the_rsc == NULL) {
         char *as_clone = crm_concat(rsc, "0", ':');
 
         the_rsc = pe_find_resource(data_set->resources, as_clone);
         free(as_clone);
     }
     return the_rsc;
 }
 
 static int
 dump_resource(const char *rsc, pe_working_set_t * data_set, gboolean expanded)
 {
     char *rsc_xml = NULL;
     resource_t *the_rsc = find_rsc_or_clone(rsc, data_set);
     int opts = pe_print_printf;
 
     if (the_rsc == NULL) {
         return -ENXIO;
     }
 
     if (print_pending) {
         opts |= pe_print_pending;
     }
     the_rsc->fns->print(the_rsc, NULL, opts, stdout);
 
     if (expanded) {
         rsc_xml = dump_xml_formatted(the_rsc->xml);
     } else {
         if (the_rsc->orig_xml) {
             rsc_xml = dump_xml_formatted(the_rsc->orig_xml);
         } else {
             rsc_xml = dump_xml_formatted(the_rsc->xml);
         }
     }
 
     fprintf(stdout, "%sxml:\n%s\n", expanded ? "" : "raw ", rsc_xml);
 
     free(rsc_xml);
 
     return 0;
 }
 
 static int
 dump_resource_attr(const char *rsc, const char *attr, pe_working_set_t * data_set)
 {
     int rc = -ENXIO;
     node_t *current = NULL;
     GHashTable *params = NULL;
     resource_t *the_rsc = find_rsc_or_clone(rsc, data_set);
     const char *value = NULL;
 
     if (the_rsc == NULL) {
         return -ENXIO;
     }
 
     if (g_list_length(the_rsc->running_on) == 1) {
         current = the_rsc->running_on->data;
 
     } else if (g_list_length(the_rsc->running_on) > 1) {
         CMD_ERR("%s is active on more than one node,"
                 " returning the default value for %s\n", the_rsc->id, crm_str(value));
     }
 
     params = g_hash_table_new_full(crm_str_hash, g_str_equal,
                                    g_hash_destroy_str, g_hash_destroy_str);
 
     if (safe_str_eq(attr_set_type, XML_TAG_ATTR_SETS)) {
         get_rsc_attributes(params, the_rsc, current, data_set);
     } else if (safe_str_eq(attr_set_type, XML_TAG_META_SETS)) {
         get_meta_attributes(params, the_rsc, current, data_set);
     } else {
         unpack_instance_attributes(data_set->input, the_rsc->xml, XML_TAG_UTILIZATION, NULL,
                                    params, NULL, FALSE, data_set->now);
     }
 
     crm_debug("Looking up %s in %s", attr, the_rsc->id);
     value = g_hash_table_lookup(params, attr);
     if (value != NULL) {
         fprintf(stdout, "%s\n", value);
         rc = 0;
     }
 
     g_hash_table_destroy(params);
     return rc;
 }
 
 static int
 find_resource_attr(cib_t * the_cib, const char *attr, const char *rsc, const char *set_type,
                    const char *set_name, const char *attr_id, const char *attr_name, char **value)
 {
     int offset = 0;
     static int xpath_max = 1024;
     int rc = pcmk_ok;
     xmlNode *xml_search = NULL;
 
     char *xpath_string = NULL;
 
     CRM_ASSERT(value != NULL);
     *value = NULL;
 
     xpath_string = calloc(1, xpath_max);
     offset +=
         snprintf(xpath_string + offset, xpath_max - offset, "%s", get_object_path("resources"));
 
     offset += snprintf(xpath_string + offset, xpath_max - offset, "//*[@id=\"%s\"]", rsc);
 
     if (set_type) {
         offset += snprintf(xpath_string + offset, xpath_max - offset, "/%s", set_type);
         if (set_name) {
             offset += snprintf(xpath_string + offset, xpath_max - offset, "[@id=\"%s\"]", set_name);
         }
     }
 
     offset += snprintf(xpath_string + offset, xpath_max - offset, "//nvpair[");
     if (attr_id) {
         offset += snprintf(xpath_string + offset, xpath_max - offset, "@id=\"%s\"", attr_id);
     }
 
     if (attr_name) {
         if (attr_id) {
             offset += snprintf(xpath_string + offset, xpath_max - offset, " and ");
         }
         offset += snprintf(xpath_string + offset, xpath_max - offset, "@name=\"%s\"", attr_name);
     }
     offset += snprintf(xpath_string + offset, xpath_max - offset, "]");
 
     rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
                               cib_sync_call | cib_scope_local | cib_xpath);
 
     if (rc != pcmk_ok) {
         goto bail;
     }
 
     crm_log_xml_debug(xml_search, "Match");
     if (xml_has_children(xml_search)) {
         xmlNode *child = NULL;
 
         rc = -EINVAL;
         printf("Multiple attributes match name=%s\n", attr_name);
 
         for (child = __xml_first_child(xml_search); child != NULL; child = __xml_next(child)) {
             printf("  Value: %s \t(id=%s)\n",
                    crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
         }
 
     } else {
         const char *tmp = crm_element_value(xml_search, attr);
 
         if (tmp) {
             *value = strdup(tmp);
         }
     }
 
   bail:
     free(xpath_string);
     free_xml(xml_search);
     return rc;
 }
 
 #include "../pengine/pengine.h"
 
 
 static int
 set_resource_attr(const char *rsc_id, const char *attr_set, const char *attr_id,
                   const char *attr_name, const char *attr_value, bool recursive,
                   cib_t * cib, pe_working_set_t * data_set)
 {
     int rc = pcmk_ok;
     static bool need_init = TRUE;
 
     char *local_attr_id = NULL;
     char *local_attr_set = NULL;
 
     xmlNode *xml_top = NULL;
     xmlNode *xml_obj = NULL;
 
     gboolean use_attributes_tag = FALSE;
     resource_t *rsc = find_rsc_or_clone(rsc_id, data_set);
 
     if (rsc == NULL) {
         return -ENXIO;
     }
 
     if (safe_str_eq(attr_set_type, XML_TAG_ATTR_SETS)) {
         rc = find_resource_attr(cib, XML_ATTR_ID, rsc_id, XML_TAG_META_SETS, attr_set, attr_id,
                                 attr_name, &local_attr_id);
         if (rc == pcmk_ok) {
             printf("WARNING: There is already a meta attribute called %s (id=%s)\n", attr_name,
                    local_attr_id);
         }
     }
     rc = find_resource_attr(cib, XML_ATTR_ID, rsc_id, attr_set_type, attr_set, attr_id, attr_name,
                             &local_attr_id);
 
     if (rc == pcmk_ok) {
         crm_debug("Found a match for name=%s: id=%s", attr_name, local_attr_id);
         attr_id = local_attr_id;
 
     } else if (rc != -ENXIO) {
         free(local_attr_id);
         return rc;
 
     } else {
         const char *value = NULL;
         xmlNode *cib_top = NULL;
         const char *tag = crm_element_name(rsc->xml);
 
         rc = cib->cmds->query(cib, "/cib", &cib_top,
                               cib_sync_call | cib_scope_local | cib_xpath | cib_no_children);
         value = crm_element_value(cib_top, "ignore_dtd");
         if (value != NULL) {
             use_attributes_tag = TRUE;
 
         } else {
             value = crm_element_value(cib_top, XML_ATTR_VALIDATION);
             if (value && strstr(value, "-0.6")) {
                 use_attributes_tag = TRUE;
             }
         }
         free_xml(cib_top);
 
         if (attr_set == NULL) {
             local_attr_set = crm_concat(rsc_id, attr_set_type, '-');
             attr_set = local_attr_set;
         }
         if (attr_id == NULL) {
             local_attr_id = crm_concat(attr_set, attr_name, '-');
             attr_id = local_attr_id;
         }
 
         if (use_attributes_tag && safe_str_eq(tag, XML_CIB_TAG_MASTER)) {
             tag = "master_slave";       /* use the old name */
         }
 
         xml_top = create_xml_node(NULL, tag);
         crm_xml_add(xml_top, XML_ATTR_ID, rsc_id);
 
         xml_obj = create_xml_node(xml_top, attr_set_type);
         crm_xml_add(xml_obj, XML_ATTR_ID, attr_set);
 
         if (use_attributes_tag) {
             xml_obj = create_xml_node(xml_obj, XML_TAG_ATTRS);
         }
     }
 
     xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_NVPAIR);
     if (xml_top == NULL) {
         xml_top = xml_obj;
     }
 
     crm_xml_add(xml_obj, XML_ATTR_ID, attr_id);
     crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, attr_name);
     crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, attr_value);
 
     crm_log_xml_debug(xml_top, "Update");
 
     rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top, cib_options);
     free_xml(xml_top);
     free(local_attr_id);
     free(local_attr_set);
 
     if(recursive && safe_str_eq(attr_set_type, XML_TAG_META_SETS)) {
         GListPtr lpc = NULL;
 
         if(need_init) {
             xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);
 
             need_init = FALSE;
             unpack_constraints(cib_constraints, data_set);
 
             for (lpc = data_set->resources; lpc != NULL; lpc = lpc->next) {
                 resource_t *r = (resource_t *) lpc->data;
 
                 clear_bit(r->flags, pe_rsc_allocating);
             }
         }
 
         crm_debug("Looking for dependancies %p", rsc->rsc_cons_lhs);
         set_bit(rsc->flags, pe_rsc_allocating);
         for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
             rsc_colocation_t *cons = (rsc_colocation_t *) lpc->data;
             resource_t *peer = cons->rsc_lh;
 
             crm_debug("Checking %s %d", cons->id, cons->score);
             if (cons->score > 0 && is_not_set(peer->flags, pe_rsc_allocating)) {
                 /* Don't get into colocation loops */
                 crm_debug("Setting %s=%s for dependant resource %s", attr_name, attr_value, peer->id);
                 set_resource_attr(peer->id, NULL, NULL, attr_name, attr_value, recursive, cib, data_set);
             }
         }
     }
 
     return rc;
 }
 
 static int
 delete_resource_attr(const char *rsc_id, const char *attr_set, const char *attr_id,
                      const char *attr_name, cib_t * cib, pe_working_set_t * data_set)
 {
     xmlNode *xml_obj = NULL;
 
     int rc = pcmk_ok;
     char *local_attr_id = NULL;
     resource_t *rsc = find_rsc_or_clone(rsc_id, data_set);
 
     if (rsc == NULL) {
         return -ENXIO;
     }
 
     rc = find_resource_attr(cib, XML_ATTR_ID, rsc_id, attr_set_type, attr_set, attr_id, attr_name,
                             &local_attr_id);
 
     if (rc == -ENXIO) {
         return pcmk_ok;
 
     } else if (rc != pcmk_ok) {
         return rc;
     }
 
     if (attr_id == NULL) {
         attr_id = local_attr_id;
     }
 
     xml_obj = create_xml_node(NULL, XML_CIB_TAG_NVPAIR);
     crm_xml_add(xml_obj, XML_ATTR_ID, attr_id);
     crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, attr_name);
 
     crm_log_xml_debug(xml_obj, "Delete");
 
     rc = cib->cmds->delete(cib, XML_CIB_TAG_RESOURCES, xml_obj, cib_options);
 
     if (rc == pcmk_ok) {
         printf("Deleted %s option: id=%s%s%s%s%s\n", rsc_id, local_attr_id,
                attr_set ? " set=" : "", attr_set ? attr_set : "",
                attr_name ? " name=" : "", attr_name ? attr_name : "");
     }
 
     free_xml(xml_obj);
     free(local_attr_id);
     return rc;
 }
 
 static int
 dump_resource_prop(const char *rsc, const char *attr, pe_working_set_t * data_set)
 {
     const char *value = NULL;
     resource_t *the_rsc = pe_find_resource(data_set->resources, rsc);
 
     if (the_rsc == NULL) {
         return -ENXIO;
     }
 
     value = crm_element_value(the_rsc->xml, attr);
 
     if (value != NULL) {
         fprintf(stdout, "%s\n", value);
         return 0;
     }
     return -ENXIO;
 }
 
 static int
 send_lrm_rsc_op(crm_ipc_t * crmd_channel, const char *op,
                 const char *host_uname, const char *rsc_id,
                 gboolean only_failed, pe_working_set_t * data_set)
 {
     char *key = NULL;
     int rc = -ECOMM;
     xmlNode *cmd = NULL;
     xmlNode *xml_rsc = NULL;
     const char *value = NULL;
     const char *router_node = host_uname;
     xmlNode *params = NULL;
     xmlNode *msg_data = NULL;
     resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
 
     if (rsc == NULL) {
         CMD_ERR("Resource %s not found\n", rsc_id);
         return -ENXIO;
 
     } else if (rsc->variant != pe_native) {
         CMD_ERR("We can only process primitive resources, not %s\n", rsc_id);
         return -EINVAL;
 
     } else if (host_uname == NULL) {
         CMD_ERR("Please supply a hostname with -H\n");
         return -EINVAL;
     } else {
         node_t *node = pe_find_node(data_set->nodes, host_uname);
 
         if (node && is_remote_node(node)) {
             if (node->details->remote_rsc->running_on) {
                 node = node->details->remote_rsc->running_on->data;
                 router_node = node->details->uname;
             } else {
                 CMD_ERR("No lrmd connection detected to remote node %s", host_uname);
                 return -ENXIO;
             }
         }
     }
 
     key = generate_transition_key(0, getpid(), 0, "xxxxxxxx-xrsc-opxx-xcrm-resourcexxxx");
 
     msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
     crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
     free(key);
 
     crm_xml_add(msg_data, XML_LRM_ATTR_TARGET, host_uname);
     if (safe_str_neq(router_node, host_uname)) {
         crm_xml_add(msg_data, XML_LRM_ATTR_ROUTER_NODE, router_node);
     }
 
     xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
     if (rsc->clone_name) {
         crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->clone_name);
         crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc->id);
 
     } else {
         crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->id);
     }
 
     value = crm_element_value(rsc->xml, XML_ATTR_TYPE);
     crm_xml_add(xml_rsc, XML_ATTR_TYPE, value);
     if (value == NULL) {
         CMD_ERR("%s has no type!  Aborting...\n", rsc_id);
         return -ENXIO;
     }
 
     value = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
     crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, value);
     if (value == NULL) {
         CMD_ERR("%s has no class!  Aborting...\n", rsc_id);
         return -ENXIO;
     }
 
     value = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
     crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, value);
 
     params = create_xml_node(msg_data, XML_TAG_ATTRS);
     crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
 
     key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
     crm_xml_add(params, key, "60000");  /* 1 minute */
     free(key);
 
     cmd = create_request(op, msg_data, router_node, CRM_SYSTEM_CRMD, crm_system_name, our_pid);
 
 /* 	crm_log_xml_warn(cmd, "send_lrm_rsc_op"); */
     free_xml(msg_data);
 
     if (crm_ipc_send(crmd_channel, cmd, 0, 0, NULL) > 0) {
         rc = 0;
 
     } else {
         CMD_ERR("Could not send %s op to the crmd", op);
         rc = -ENOTCONN;
     }
 
     free_xml(cmd);
     return rc;
 }
 
 static int
 delete_lrm_rsc(cib_t *cib_conn, crm_ipc_t * crmd_channel, const char *host_uname,
                resource_t * rsc, pe_working_set_t * data_set)
 {
     int rc = pcmk_ok;
 
     if (rsc == NULL) {
         return -ENXIO;
 
     } else if (rsc->children) {
         GListPtr lpc = NULL;
 
         for (lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
             resource_t *child = (resource_t *) lpc->data;
 
             delete_lrm_rsc(cib_conn, crmd_channel, host_uname, child, data_set);
         }
         return pcmk_ok;
 
     } else if (host_uname == NULL) {
         GListPtr lpc = NULL;
 
         for (lpc = data_set->nodes; lpc != NULL; lpc = lpc->next) {
             node_t *node = (node_t *) lpc->data;
 
             if (node->details->online) {
                 delete_lrm_rsc(cib_conn, crmd_channel, node->details->uname, rsc, data_set);
             }
         }
 
         return pcmk_ok;
     }
 
     printf("Cleaning up %s on %s\n", rsc->id, host_uname);
     rc = send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_DELETE, host_uname, rsc->id, TRUE, data_set);
 
     if (rc == pcmk_ok) {
         char *attr_name = NULL;
         const char *id = rsc->id;
         node_t *node = pe_find_node(data_set->nodes, host_uname);
 
         if(node && node->details->remote_rsc == NULL) {
             crmd_replies_needed++;
         }
         if (rsc->clone_name) {
             id = rsc->clone_name;
         }
 
         attr_name = crm_concat("fail-count", id, '-');
         rc = attrd_update_delegate(NULL, 'D', host_uname, attr_name, NULL, XML_CIB_TAG_STATUS, NULL,
                               NULL, NULL, node ? is_remote_node(node) : FALSE);
         free(attr_name);
     }
     return rc;
 }
 
 static int
 fail_lrm_rsc(crm_ipc_t * crmd_channel, const char *host_uname,
              const char *rsc_id, pe_working_set_t * data_set)
 {
     crm_warn("Failing: %s", rsc_id);
     return send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_FAIL, host_uname, rsc_id, FALSE, data_set);
 }
 
 static char *
 parse_cli_lifetime(const char *input)
 {
     char *later_s = NULL;
     crm_time_t *now = NULL;
     crm_time_t *later = NULL;
     crm_time_t *duration = NULL;
 
     if (input == NULL) {
         return NULL;
     }
 
     duration = crm_time_parse_duration(move_lifetime);
     if (duration == NULL) {
         CMD_ERR("Invalid duration specified: %s\n", move_lifetime);
         CMD_ERR("Please refer to"
                 " http://en.wikipedia.org/wiki/ISO_8601#Duration"
                 " for examples of valid durations\n");
         return NULL;
     }
 
     now = crm_time_new(NULL);
     later = crm_time_add(now, duration);
     crm_time_log(LOG_INFO, "now     ", now,
                  crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone);
     crm_time_log(LOG_INFO, "later   ", later,
                  crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone);
     crm_time_log(LOG_INFO, "duration", duration, crm_time_log_date | crm_time_log_timeofday);
     later_s = crm_time_as_string(later, crm_time_log_date | crm_time_log_timeofday);
     printf("Migration will take effect until: %s\n", later_s);
 
     crm_time_free(duration);
     crm_time_free(later);
     crm_time_free(now);
     return later_s;
 }
 
 static int
 ban_resource(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn)
 {
     char *later_s = NULL;
     int rc = pcmk_ok;
     char *id = NULL;
     xmlNode *fragment = NULL;
     xmlNode *location = NULL;
 
     if(host == NULL) {
         GListPtr n = allnodes;
         for(; n && rc == pcmk_ok; n = n->next) {
             node_t *target = n->data;
 
             rc = ban_resource(rsc_id, target->details->uname, NULL, cib_conn);
         }
         return rc;
     }
 
     later_s = parse_cli_lifetime(move_lifetime);
     if(move_lifetime && later_s == NULL) {
         return -EINVAL;
     }
 
     fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS);
 
     id = g_strdup_printf("cli-ban-%s-on-%s", rsc_id, host);
     location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
     crm_xml_add(location, XML_ATTR_ID, id);
     free(id);
 
     if (BE_QUIET == FALSE) {
         CMD_ERR("WARNING: Creating rsc_location constraint '%s'"
                 " with a score of -INFINITY for resource %s"
                 " on %s.\n", ID(location), rsc_id, host);
         CMD_ERR("\tThis will prevent %s from %s"
                 " on %s until the constraint is removed using"
                 " the 'crm_resource --clear' command or manually"
                 " with cibadmin\n", rsc_id, scope_master?"being promoted":"running", host);
         CMD_ERR("\tThis will be the case even if %s is"
                 " the last node in the cluster\n", host);
         CMD_ERR("\tThis message can be disabled with --quiet\n");
     }
 
     crm_xml_add(location, XML_COLOC_ATTR_SOURCE, rsc_id);
     if(scope_master) {
         crm_xml_add(location, XML_RULE_ATTR_ROLE, RSC_ROLE_MASTER_S);
     } else {
         crm_xml_add(location, XML_RULE_ATTR_ROLE, RSC_ROLE_STARTED_S);
     }
 
     if (later_s == NULL) {
         /* Short form */
         crm_xml_add(location, XML_CIB_TAG_NODE, host);
         crm_xml_add(location, XML_RULE_ATTR_SCORE, MINUS_INFINITY_S);
 
     } else {
         xmlNode *rule = create_xml_node(location, XML_TAG_RULE);
         xmlNode *expr = create_xml_node(rule, XML_TAG_EXPRESSION);
 
         id = g_strdup_printf("cli-ban-%s-on-%s-rule", rsc_id, host);
         crm_xml_add(rule, XML_ATTR_ID, id);
         free(id);
 
         crm_xml_add(rule, XML_RULE_ATTR_SCORE, MINUS_INFINITY_S);
         crm_xml_add(rule, XML_RULE_ATTR_BOOLEAN_OP, "and");
 
         id = g_strdup_printf("cli-ban-%s-on-%s-expr", rsc_id, host);
         crm_xml_add(expr, XML_ATTR_ID, id);
         free(id);
 
         crm_xml_add(expr, XML_EXPR_ATTR_ATTRIBUTE, "#uname");
         crm_xml_add(expr, XML_EXPR_ATTR_OPERATION, "eq");
         crm_xml_add(expr, XML_EXPR_ATTR_VALUE, host);
         crm_xml_add(expr, XML_EXPR_ATTR_TYPE, "string");
 
         expr = create_xml_node(rule, "date_expression");
         id = g_strdup_printf("cli-ban-%s-on-%s-lifetime", rsc_id, host);
         crm_xml_add(expr, XML_ATTR_ID, id);
         free(id);
 
         crm_xml_add(expr, "operation", "lt");
         crm_xml_add(expr, "end", later_s);
     }
 
     crm_log_xml_notice(fragment, "Modify");
     rc = cib_conn->cmds->update(cib_conn, XML_CIB_TAG_CONSTRAINTS, fragment, cib_options);
 
     free_xml(fragment);
     free(later_s);
     return rc;
 }
 
 static int
 prefer_resource(const char *rsc_id, const char *host, cib_t * cib_conn)
 {
     char *later_s = parse_cli_lifetime(move_lifetime);
     int rc = pcmk_ok;
     char *id = NULL;
     xmlNode *location = NULL;
     xmlNode *fragment = NULL;
 
     if(move_lifetime && later_s == NULL) {
         return -EINVAL;
     }
 
     fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS);
 
     id = g_strdup_printf("cli-prefer-%s", rsc_id);
     location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
     crm_xml_add(location, XML_ATTR_ID, id);
     free(id);
 
     crm_xml_add(location, XML_COLOC_ATTR_SOURCE, rsc_id);
     if(scope_master) {
         crm_xml_add(location, XML_RULE_ATTR_ROLE, RSC_ROLE_MASTER_S);
     } else {
         crm_xml_add(location, XML_RULE_ATTR_ROLE, RSC_ROLE_STARTED_S);
     }
 
     if (later_s == NULL) {
         /* Short form */
         crm_xml_add(location, XML_CIB_TAG_NODE, host);
         crm_xml_add(location, XML_RULE_ATTR_SCORE, INFINITY_S);
 
     } else {
         xmlNode *rule = create_xml_node(location, XML_TAG_RULE);
         xmlNode *expr = create_xml_node(rule, XML_TAG_EXPRESSION);
 
         id = crm_concat("cli-prefer-rule", rsc_id, '-');
         crm_xml_add(rule, XML_ATTR_ID, id);
         free(id);
 
         crm_xml_add(rule, XML_RULE_ATTR_SCORE, INFINITY_S);
         crm_xml_add(rule, XML_RULE_ATTR_BOOLEAN_OP, "and");
 
         id = crm_concat("cli-prefer-expr", rsc_id, '-');
         crm_xml_add(expr, XML_ATTR_ID, id);
         free(id);
 
         crm_xml_add(expr, XML_EXPR_ATTR_ATTRIBUTE, "#uname");
         crm_xml_add(expr, XML_EXPR_ATTR_OPERATION, "eq");
         crm_xml_add(expr, XML_EXPR_ATTR_VALUE, host);
         crm_xml_add(expr, XML_EXPR_ATTR_TYPE, "string");
 
         expr = create_xml_node(rule, "date_expression");
         id = crm_concat("cli-prefer-lifetime-end", rsc_id, '-');
         crm_xml_add(expr, XML_ATTR_ID, id);
         free(id);
 
         crm_xml_add(expr, "operation", "lt");
         crm_xml_add(expr, "end", later_s);
     }
 
     crm_log_xml_info(fragment, "Modify");
     rc = cib_conn->cmds->update(cib_conn, XML_CIB_TAG_CONSTRAINTS, fragment, cib_options);
 
     free_xml(fragment);
     free(later_s);
     return rc;
 }
 
 static int
 clear_resource(const char *rsc_id, const char *host, GListPtr allnodes, cib_t * cib_conn)
 {
     char *id = NULL;
     int rc = pcmk_ok;
     xmlNode *fragment = NULL;
     xmlNode *location = NULL;
 
     fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS);
 
     if(host) {
         id = g_strdup_printf("cli-ban-%s-on-%s", rsc_id, host);
         location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
         crm_xml_add(location, XML_ATTR_ID, id);
         free(id);
 
     } else {
         GListPtr n = allnodes;
         for(; n; n = n->next) {
             node_t *target = n->data;
 
             id = g_strdup_printf("cli-ban-%s-on-%s", rsc_id, target->details->uname);
             location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
             crm_xml_add(location, XML_ATTR_ID, id);
             free(id);
         }
     }
 
     id = g_strdup_printf("cli-prefer-%s", rsc_id);
     location = create_xml_node(fragment, XML_CONS_TAG_RSC_LOCATION);
     crm_xml_add(location, XML_ATTR_ID, id);
     if(host && do_force == FALSE) {
         crm_xml_add(location, XML_CIB_TAG_NODE, host);
     }
     free(id);
 
     crm_log_xml_info(fragment, "Delete");
     rc = cib_conn->cmds->delete(cib_conn, XML_CIB_TAG_CONSTRAINTS, fragment, cib_options);
     if (rc == -ENXIO) {
         rc = pcmk_ok;
 
     } else if (rc != pcmk_ok) {
         goto bail;
     }
 
   bail:
     free_xml(fragment);
     return rc;
 }
 
 static int
 list_resource_operations(const char *rsc_id, const char *host_uname, gboolean active,
                          pe_working_set_t * data_set)
 {
     resource_t *rsc = NULL;
     int opts = pe_print_printf | pe_print_rsconly | pe_print_suppres_nl;
     GListPtr ops = find_operations(rsc_id, host_uname, active, data_set);
     GListPtr lpc = NULL;
 
     if (print_pending) {
         opts |= pe_print_pending;
     }
 
     for (lpc = ops; lpc != NULL; lpc = lpc->next) {
         xmlNode *xml_op = (xmlNode *) lpc->data;
 
         const char *op_rsc = crm_element_value(xml_op, "resource");
         const char *last = crm_element_value(xml_op, XML_RSC_OP_LAST_CHANGE);
         const char *status_s = crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS);
         const char *op_key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
         int status = crm_parse_int(status_s, "0");
 
         rsc = pe_find_resource(data_set->resources, op_rsc);
         if(rsc) {
             rsc->fns->print(rsc, "", opts, stdout);
         } else {
             fprintf(stdout, "Unknown resource %s", op_rsc);
         }
 
         fprintf(stdout, ": %s (node=%s, call=%s, rc=%s",
                 op_key ? op_key : ID(xml_op),
                 crm_element_value(xml_op, XML_ATTR_UNAME),
                 crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
                 crm_element_value(xml_op, XML_LRM_ATTR_RC));
         if (last) {
             time_t run_at = crm_parse_int(last, "0");
 
             fprintf(stdout, ", last-rc-change=%s, exec=%sms",
                     crm_strip_trailing_newline(ctime(&run_at)), crm_element_value(xml_op, XML_RSC_OP_T_EXEC));
         }
         fprintf(stdout, "): %s\n", services_lrm_status_str(status));
     }
     return pcmk_ok;
 }
 
 static void
 show_location(resource_t * rsc, const char *prefix)
 {
     GListPtr lpc = NULL;
     GListPtr list = rsc->rsc_location;
     int offset = 0;
 
     if (prefix) {
         offset = strlen(prefix) - 2;
     }
 
     for (lpc = list; lpc != NULL; lpc = lpc->next) {
         rsc_to_node_t *cons = (rsc_to_node_t *) lpc->data;
 
         GListPtr lpc2 = NULL;
 
         for (lpc2 = cons->node_list_rh; lpc2 != NULL; lpc2 = lpc2->next) {
             node_t *node = (node_t *) lpc2->data;
             char *score = score2char(node->weight);
 
             fprintf(stdout, "%s: Node %-*s (score=%s, id=%s)\n",
                     prefix ? prefix : "  ", 71 - offset, node->details->uname, score, cons->id);
             free(score);
         }
     }
 }
 
 static void
 show_colocation(resource_t * rsc, gboolean dependants, gboolean recursive, int offset)
 {
     char *prefix = NULL;
     GListPtr lpc = NULL;
     GListPtr list = rsc->rsc_cons;
 
     prefix = calloc(1, (offset * 4) + 1);
     memset(prefix, ' ', offset * 4);
 
     if (dependants) {
         list = rsc->rsc_cons_lhs;
     }
 
     if (is_set(rsc->flags, pe_rsc_allocating)) {
         /* Break colocation loops */
         printf("loop %s\n", rsc->id);
         free(prefix);
         return;
     }
 
     set_bit(rsc->flags, pe_rsc_allocating);
     for (lpc = list; lpc != NULL; lpc = lpc->next) {
         rsc_colocation_t *cons = (rsc_colocation_t *) lpc->data;
 
         char *score = NULL;
         resource_t *peer = cons->rsc_rh;
 
         if (dependants) {
             peer = cons->rsc_lh;
         }
 
         if (is_set(peer->flags, pe_rsc_allocating)) {
             if (dependants == FALSE) {
                 fprintf(stdout, "%s%-*s (id=%s - loop)\n", prefix, 80 - (4 * offset), peer->id,
                         cons->id);
             }
             continue;
         }
 
         if (dependants && recursive) {
             show_colocation(peer, dependants, recursive, offset + 1);
         }
 
         score = score2char(cons->score);
         if (cons->role_rh > RSC_ROLE_STARTED) {
             fprintf(stdout, "%s%-*s (score=%s, %s role=%s, id=%s)\n", prefix, 80 - (4 * offset),
                     peer->id, score, dependants ? "needs" : "with", role2text(cons->role_rh),
                     cons->id);
         } else {
             fprintf(stdout, "%s%-*s (score=%s, id=%s)\n", prefix, 80 - (4 * offset),
                     peer->id, score, cons->id);
         }
         show_location(peer, prefix);
         free(score);
 
         if (!dependants && recursive) {
             show_colocation(peer, dependants, recursive, offset + 1);
         }
     }
     free(prefix);
 }
 
 static GHashTable *
 generate_resource_params(resource_t * rsc, pe_working_set_t * data_set)
 {
     GHashTable *params = NULL;
     GHashTable *meta = NULL;
     GHashTable *combined = NULL;
     GHashTableIter iter;
 
     if (!rsc) {
         crm_err("Resource does not exist in config");
         return NULL;
     }
 
     params =
         g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
     meta = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
     combined =
         g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
 
     get_rsc_attributes(params, rsc, NULL /* TODO: Pass in local node */ , data_set);
     get_meta_attributes(meta, rsc, NULL /* TODO: Pass in local node */ , data_set);
 
     if (params) {
         char *key = NULL;
         char *value = NULL;
 
         g_hash_table_iter_init(&iter, params);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             g_hash_table_insert(combined, strdup(key), strdup(value));
         }
         g_hash_table_destroy(params);
     }
 
     if (meta) {
         char *key = NULL;
         char *value = NULL;
 
         g_hash_table_iter_init(&iter, meta);
         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
             char *crm_name = crm_meta_name(key);
 
             g_hash_table_insert(combined, crm_name, strdup(value));
         }
         g_hash_table_destroy(meta);
     }
 
     return combined;
 }
 
 
 /* *INDENT-OFF* */
 static struct crm_option long_options[] = {
     /* Top-level Options */
     {"help",    0, 0, '?', "\t\tThis text"},
     {"version", 0, 0, '$', "\t\tVersion information"  },
     {"verbose", 0, 0, 'V', "\t\tIncrease debug output"},
     {"quiet",   0, 0, 'Q', "\t\tPrint only the value on stdout\n"},
 
     {"resource",   1, 0, 'r', "\tResource ID" },
 
     {"-spacer-",1, 0, '-', "\nQueries:"},
     {"list",       0, 0, 'L', "\t\tList all cluster resources"},
     {"list-raw",   0, 0, 'l', "\tList the IDs of all instantiated resources (no groups/clones/...)"},
     {"list-cts",   0, 0, 'c', NULL, 1},
     {"list-operations", 0, 0, 'O', "\tList active resource operations.  Optionally filtered by resource (-r) and/or node (-N)"},
     {"list-all-operations", 0, 0, 'o', "List all resource operations.  Optionally filtered by resource (-r) and/or node (-N)"},
     {"pending",    0, 0, 'j', "\t\tDisplay pending state if 'record-pending' is enabled\n"},
 
     {"list-standards",        0, 0, 0, "\tList supported standards"},
     {"list-ocf-providers",    0, 0, 0, "List all available OCF providers"},
     {"list-agents",           1, 0, 0, "List all agents available for the named standard and/or provider."},
     {"list-ocf-alternatives", 1, 0, 0, "List all available providers for the named OCF agent\n"},
     {"show-metadata",         1, 0, 0, "Show the metadata for the named class:provider:agent"},
 
     {"query-xml",  0, 0, 'q', "\tQuery the definition of a resource (template expanded)"},
     {"query-xml-raw",  0, 0, 'w', "\tQuery the definition of a resource (raw xml)"},
     {"locate",     0, 0, 'W', "\t\tDisplay the current location(s) of a resource"},
     {"stack",      0, 0, 'A', "\t\tDisplay the prerequisites and dependents of a resource"},
     {"constraints",0, 0, 'a', "\tDisplay the (co)location constraints that apply to a resource"},
 
     {"-spacer-",	1, 0, '-', "\nCommands:"},
     {"cleanup",         0, 0, 'C', "\t\tDelete the resource history and re-check the current state. Optional: --resource"},
     {"set-parameter",   1, 0, 'p', "Set the named parameter for a resource. See also -m, --meta"},
     {"get-parameter",   1, 0, 'g', "Display the named parameter for a resource. See also -m, --meta"},
     {"delete-parameter",1, 0, 'd', "Delete the named parameter for a resource. See also -m, --meta"},
     {"get-property",    1, 0, 'G', "Display the 'class', 'type' or 'provider' of a resource", 1},
     {"set-property",    1, 0, 'S', "(Advanced) Set the class, type or provider of a resource", 1},
 
     {"-spacer-",	1, 0, '-', "\nResource location:"},
     {
         "move",    0, 0, 'M',
         "\t\tMove a resource from its current location to the named destination.\n  "
         "\t\t\t\tRequires: --host. Optional: --lifetime, --master\n\n"
         "\t\t\t\tNOTE: This may prevent the resource from running on the previous location node until the implicit constraints expire or are removed with --unban\n"
     },
     {
         "ban",    0, 0, 'B',
         "\t\tPrevent the named resource from running on the named --host.  \n"
         "\t\t\t\tRequires: --resource. Optional: --host, --lifetime, --master\n\n"
         "\t\t\t\tIf --host is not specified, it defaults to:\n"
         "\t\t\t\t * the curent location for primitives and groups, or\n\n"
         "\t\t\t\t * the curent location of the master for m/s resources with master-max=1\n\n"
         "\t\t\t\tAll other situations result in an error as there is no sane default.\n\n"
         "\t\t\t\tNOTE: This will prevent the resource from running on this node until the constraint expires or is removed with --clear\n"
     },
     {
         "clear", 0, 0, 'U', "\t\tRemove all constraints created by the --ban and/or --move commands.  \n"
         "\t\t\t\tRequires: --resource. Optional: --host, --master\n\n"
         "\t\t\t\tIf --host is not specified, all constraints created by --ban and --move will be removed for the named resource.\n"
     },
     {"lifetime",   1, 0, 'u', "\tLifespan of constraints created by the --ban and --move commands"},
     {
         "master",  0, 0,  0,
         "\t\tLimit the scope of the --ban, --move and --clear  commands to the Master role.\n"
         "\t\t\t\tFor --ban and --move, the previous master can still remain active in the Slave role."
     },
 
     {"-spacer-",   1, 0, '-', "\nAdvanced Commands:"},
     {"delete",     0, 0, 'D', "\t\t(Advanced) Delete a resource from the CIB"},
     {"fail",       0, 0, 'F', "\t\t(Advanced) Tell the cluster this resource has failed"},
     {"force-stop", 0, 0,  0,  "\t(Advanced) Bypass the cluster and stop a resource on the local node. Additional detail with -V"},
     {"force-start",0, 0,  0,  "\t(Advanced) Bypass the cluster and start a resource on the local node. Additional detail with -V"},
     {"force-check",0, 0,  0,  "\t(Advanced) Bypass the cluster and check the state of a resource on the local node. Additional detail with -V\n"},
 
     {"-spacer-",	1, 0, '-', "\nAdditional Options:"},
     {"node",		1, 0, 'N', "\tHost uname"},
     {"recursive",       0, 0,  0,  "\tFollow colocation chains when using --set-parameter"},
     {"resource-type",	1, 0, 't', "Resource type (primitive, clone, group, ...)"},
     {"parameter-value", 1, 0, 'v', "Value to use with -p or -S"},
     {"meta",		0, 0, 'm', "\t\tModify a resource's configuration option rather than one which is passed to the resource agent script. For use with -p, -g, -d"},
     {"utilization",	0, 0, 'z', "\tModify a resource's utilization attribute. For use with -p, -g, -d"},
     {"set-name",        1, 0, 's', "\t(Advanced) ID of the instance_attributes object to change"},
     {"nvpair",          1, 0, 'i', "\t(Advanced) ID of the nvpair object to change/delete"},
     {"force",		0, 0, 'f', "\n" /* Is this actually true anymore?
 					   "\t\tForce the resource to move by creating a rule for the current location and a score of -INFINITY"
 					   "\n\t\tThis should be used if the resource's stickiness and constraint scores total more than INFINITY (Currently 100,000)"
 					   "\n\t\tNOTE: This will prevent the resource from running on this node until the constraint is removed with -U or the --lifetime duration expires\n"*/ },
 
     {"xml-file", 1, 0, 'x', NULL, 1},\
 
     /* legacy options */
     {"host-uname", 1, 0, 'H', NULL, 1},
     {"migrate",    0, 0, 'M', NULL, 1},
     {"un-migrate", 0, 0, 'U', NULL, 1},
     {"un-move",    0, 0, 'U', NULL, 1},
 
     {"refresh",    0, 0, 'R', NULL, 1},
     {"reprobe",    0, 0, 'P', NULL, 1},
 
     {"-spacer-",	1, 0, '-', "\nExamples:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', "List the configured resources:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --list", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "List the available OCF agents:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --list-agents ocf", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "List the available OCF agents from the linux-ha project:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --list-agents ocf:heartbeat", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Display the current location of 'myResource':", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --locate", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Move 'myResource' to another machine:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --move", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Move 'myResource' to a specific machine:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --move --node altNode", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Allow (but not force) 'myResource' to move back to its original location:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --un-move", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Tell the cluster that 'myResource' failed:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --fail", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Stop a 'myResource' (and anything that depends on it):", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --set-parameter target-role --meta --parameter-value Stopped", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Tell the cluster not to manage 'myResource':", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', "The cluster will not attempt to start or stop the resource under any circumstances."},
     {"-spacer-",	1, 0, '-', "Useful when performing maintenance tasks on a resource.", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --set-parameter is-managed --meta --parameter-value false", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Erase the operation history of 'myResource' on 'aNode':", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', "The cluster will 'forget' the existing resource state (including any errors) and attempt to recover the resource."},
     {"-spacer-",	1, 0, '-', "Useful when a resource had failed permanently and has been repaired by an administrator.", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --cleanup --node aNode", pcmk_option_example},
 
     {0, 0, 0, 0}
 };
 /* *INDENT-ON* */
 
 int
 main(int argc, char **argv)
 {
     const char *longname = NULL;
     pe_working_set_t data_set;
     xmlNode *cib_xml_copy = NULL;
     cib_t *cib_conn = NULL;
     bool do_trace = FALSE;
     bool recursive = FALSE;
 
     int rc = pcmk_ok;
     int option_index = 0;
     int argerr = 0;
     int flag;
 
     crm_log_cli_init("crm_resource");
     crm_set_options(NULL, "(query|command) [options]", long_options,
                     "Perform tasks related to cluster resources.\nAllows resources to be queried (definition and location), modified, and moved around the cluster.\n");
 
     if (argc < 2) {
         crm_help('?', EX_USAGE);
     }
 
     while (1) {
         flag = crm_get_option_long(argc, argv, &option_index, &longname);
         if (flag == -1)
             break;
 
         switch (flag) {
             case 0:
                 if (safe_str_eq("master", longname)) {
                     scope_master = TRUE;
 
                 } else if(safe_str_eq(longname, "recursive")) {
                     recursive = TRUE;
 
                 } else if (safe_str_eq("force-stop", longname)
                     || safe_str_eq("force-start", longname)
                     || safe_str_eq("force-check", longname)) {
                     rsc_cmd = flag;
                     rsc_long_cmd = longname;
 
                 } else if (safe_str_eq("list-ocf-providers", longname)
                            || safe_str_eq("list-ocf-alternatives", longname)
                            || safe_str_eq("list-standards", longname)) {
                     const char *text = NULL;
                     lrmd_list_t *list = NULL;
                     lrmd_list_t *iter = NULL;
                     lrmd_t *lrmd_conn = lrmd_api_new();
 
                     if (safe_str_eq("list-ocf-providers", longname)
                         || safe_str_eq("list-ocf-alternatives", longname)) {
                         rc = lrmd_conn->cmds->list_ocf_providers(lrmd_conn, optarg, &list);
                         text = "OCF providers";
 
                     } else if (safe_str_eq("list-standards", longname)) {
                         rc = lrmd_conn->cmds->list_standards(lrmd_conn, &list);
                         text = "standards";
                     }
 
                     if (rc > 0) {
                         rc = 0;
                         for (iter = list; iter != NULL; iter = iter->next) {
                             rc++;
                             printf("%s\n", iter->val);
                         }
                         lrmd_list_freeall(list);
 
                     } else if (optarg) {
                         fprintf(stderr, "No %s found for %s\n", text, optarg);
                     } else {
                         fprintf(stderr, "No %s found\n", text);
                     }
 
                     lrmd_api_delete(lrmd_conn);
                     return crm_exit(rc);
 
                 } else if (safe_str_eq("show-metadata", longname)) {
                     char standard[512];
                     char provider[512];
                     char type[512];
                     char *metadata = NULL;
                     lrmd_t *lrmd_conn = lrmd_api_new();
 
                     rc = sscanf(optarg, "%[^:]:%[^:]:%s", standard, provider, type);
                     if (rc == 3) {
                         rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard, provider, type,
                                                            &metadata, 0);
 
                     } else if (rc == 2) {
                         rc = lrmd_conn->cmds->get_metadata(lrmd_conn, standard, NULL, provider,
                                                            &metadata, 0);
 
                     } else if (rc < 2) {
                         fprintf(stderr,
                                 "Please specify standard:type or standard:provider:type, not %s\n",
                                 optarg);
                         rc = -EINVAL;
                     }
 
                     if (metadata) {
                         printf("%s\n", metadata);
                     } else {
                         fprintf(stderr, "Metadata query for %s failed: %d\n", optarg, rc);
                     }
                     lrmd_api_delete(lrmd_conn);
                     return crm_exit(rc);
 
                 } else if (safe_str_eq("list-agents", longname)) {
                     lrmd_list_t *list = NULL;
                     lrmd_list_t *iter = NULL;
                     char standard[512];
                     char provider[512];
                     lrmd_t *lrmd_conn = lrmd_api_new();
 
                     rc = sscanf(optarg, "%[^:]:%s", standard, provider);
                     if (rc == 1) {
                         rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, optarg, NULL);
                         provider[0] = '*';
                         provider[1] = 0;
 
                     } else if (rc == 2) {
                         rc = lrmd_conn->cmds->list_agents(lrmd_conn, &list, standard, provider);
                     }
 
                     if (rc > 0) {
                         rc = 0;
                         for (iter = list; iter != NULL; iter = iter->next) {
                             printf("%s\n", iter->val);
                             rc++;
                         }
                         lrmd_list_freeall(list);
                         rc = 0;
                     } else {
                         fprintf(stderr, "No agents found for standard=%s, provider=%s\n", standard,
                                 provider);
                         rc = -1;
                     }
                     lrmd_api_delete(lrmd_conn);
                     return crm_exit(rc);
 
                 } else {
                     crm_err("Unhandled long option: %s", longname);
                 }
                 break;
             case 'V':
                 do_trace = TRUE;
                 crm_bump_log_level(argc, argv);
                 break;
             case '$':
             case '?':
                 crm_help(flag, EX_OK);
                 break;
             case 'x':
                 xml_file = strdup(optarg);
                 break;
             case 'Q':
                 BE_QUIET = TRUE;
                 break;
             case 'm':
                 attr_set_type = XML_TAG_META_SETS;
                 break;
             case 'z':
                 attr_set_type = XML_TAG_UTILIZATION;
                 break;
             case 'u':
                 move_lifetime = strdup(optarg);
                 break;
             case 'f':
                 do_force = TRUE;
                 break;
             case 'i':
                 prop_id = optarg;
                 break;
             case 's':
                 prop_set = optarg;
                 break;
             case 'r':
                 rsc_id = optarg;
                 break;
             case 'v':
                 prop_value = optarg;
                 break;
             case 't':
                 rsc_type = optarg;
                 break;
             case 'C':
             case 'R':
             case 'P':
                 rsc_cmd = 'C';
                 break;
             case 'L':
             case 'c':
             case 'l':
             case 'q':
             case 'w':
             case 'D':
             case 'F':
             case 'W':
             case 'M':
             case 'U':
             case 'B':
             case 'O':
             case 'o':
             case 'A':
             case 'a':
                 rsc_cmd = flag;
                 break;
             case 'j':
                 print_pending = TRUE;
                 break;
             case 'p':
             case 'g':
             case 'd':
             case 'S':
             case 'G':
                 prop_name = optarg;
                 rsc_cmd = flag;
                 break;
             case 'h':
             case 'H':
             case 'N':
                 crm_trace("Option %c => %s", flag, optarg);
                 host_uname = optarg;
                 break;
 
             default:
                 CMD_ERR("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
                 ++argerr;
                 break;
         }
     }
 
     if (optind < argc && argv[optind] != NULL) {
         CMD_ERR("non-option ARGV-elements: ");
         while (optind < argc && argv[optind] != NULL) {
             CMD_ERR("%s ", argv[optind++]);
             ++argerr;
         }
         CMD_ERR("\n");
     }
 
     if (optind > argc) {
         ++argerr;
     }
 
     if (argerr) {
         crm_help('?', EX_USAGE);
     }
 
     our_pid = calloc(1, 11);
     if (our_pid != NULL) {
         snprintf(our_pid, 10, "%d", getpid());
         our_pid[10] = '\0';
     }
 
     if (do_force) {
         crm_debug("Forcing...");
         cib_options |= cib_quorum_override;
     }
 
     set_working_set_defaults(&data_set);
     if (rsc_cmd != 'P' || rsc_id) {
         resource_t *rsc = NULL;
 
         cib_conn = cib_new();
         rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
         if (rc != pcmk_ok) {
             CMD_ERR("Error signing on to the CIB service: %s\n", pcmk_strerror(rc));
             return crm_exit(rc);
         }
 
         if (xml_file != NULL) {
             cib_xml_copy = filename2xml(xml_file);
 
         } else {
             cib_xml_copy = get_cib_copy(cib_conn);
         }
 
         if (cli_config_update(&cib_xml_copy, NULL, FALSE) == FALSE) {
             rc = -ENOKEY;
             goto bail;
         }
 
         data_set.input = cib_xml_copy;
         data_set.now = crm_time_new(NULL);
 
         cluster_status(&data_set);
         if (rsc_id) {
             rsc = find_rsc_or_clone(rsc_id, &data_set);
         }
         if (rsc == NULL && rsc_cmd != 'C') {
             rc = -ENXIO;
         }
     }
 
     if (rsc_cmd == 'R' || rsc_cmd == 'C' || rsc_cmd == 'F' || rsc_cmd == 'P') {
         xmlNode *xml = NULL;
         mainloop_io_t *source =
             mainloop_add_ipc_client(CRM_SYSTEM_CRMD, G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks);
         crmd_channel = mainloop_get_ipc_client(source);
 
         if (crmd_channel == NULL) {
             CMD_ERR("Error signing on to the CRMd service\n");
             rc = -ENOTCONN;
             goto bail;
         }
 
         xml = create_hello_message(our_pid, crm_system_name, "0", "1");
         crm_ipc_send(crmd_channel, xml, 0, 0, NULL);
         free_xml(xml);
     }
 
     if (rsc_cmd == 'L') {
         rc = pcmk_ok;
         do_find_resource_list(&data_set, FALSE);
 
     } else if (rsc_cmd == 'l') {
         int found = 0;
         GListPtr lpc = NULL;
 
         rc = pcmk_ok;
         for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
             resource_t *rsc = (resource_t *) lpc->data;
 
             found++;
             print_raw_rsc(rsc);
         }
 
         if (found == 0) {
             printf("NO resources configured\n");
             rc = -ENXIO;
             goto bail;
         }
 
     } else if (rsc_cmd == 0 && rsc_long_cmd) {
         svc_action_t *op = NULL;
         const char *rtype = NULL;
         const char *rprov = NULL;
         const char *rclass = NULL;
         const char *action = NULL;
         GHashTable *params = NULL;
         resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
 
         if (rsc == NULL) {
             CMD_ERR("Must supply a resource id with -r\n");
             rc = -ENXIO;
             goto bail;
         }
 
         if (safe_str_eq(rsc_long_cmd, "force-stop")) {
             action = "stop";
         } else if (safe_str_eq(rsc_long_cmd, "force-start")) {
             action = "start";
             if(rsc->variant >= pe_clone) {
                 rc = do_find_resource(rsc_id, NULL, &data_set);
                 if(rc > 0 && do_force == FALSE) {
                     CMD_ERR("It is not safe to start %s here: the cluster claims it is already active", rsc_id);
                     CMD_ERR("Try setting target-role=stopped first or specifying --force");
                     crm_exit(EPERM);
                 }
             }
 
         } else if (safe_str_eq(rsc_long_cmd, "force-check")) {
             action = "monitor";
         }
 
         rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
         rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
         rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
 
         if(safe_str_eq(rclass, "stonith")){
             CMD_ERR("Sorry, --%s doesn't support %s resources yet\n", rsc_long_cmd, rclass);
             crm_exit(EOPNOTSUPP);
         }
 
         params = generate_resource_params(rsc, &data_set);
         op = resources_action_create(rsc->id, rclass, rprov, rtype, action, 0, -1, params);
 
         if(do_trace) {
             setenv("OCF_TRACE_RA", "1", 1);
         }
 
         if(op == NULL) {
             /* Re-run but with stderr enabled so we can display a sane error message */
             crm_enable_stderr(TRUE);
             resources_action_create(rsc->id, rclass, rprov, rtype, action, 0, -1, params);
             return crm_exit(EINVAL);
 
         } else if (services_action_sync(op)) {
             int more, lpc, last;
             char *local_copy = NULL;
 
             if (op->status == PCMK_LRM_OP_DONE) {
                 printf("Operation %s for %s (%s:%s:%s) returned %d\n",
                        action, rsc->id, rclass, rprov ? rprov : "", rtype, op->rc);
             } else {
                 printf("Operation %s for %s (%s:%s:%s) failed: %d\n",
                        action, rsc->id, rclass, rprov ? rprov : "", rtype, op->status);
             }
 
             if (op->stdout_data) {
                 local_copy = strdup(op->stdout_data);
                 more = strlen(local_copy);
                 last = 0;
 
                 for (lpc = 0; lpc < more; lpc++) {
                     if (local_copy[lpc] == '\n' || local_copy[lpc] == 0) {
                         local_copy[lpc] = 0;
                         printf(" >  stdout: %s\n", local_copy + last);
                         last = lpc + 1;
                     }
                 }
                 free(local_copy);
             }
             if (op->stderr_data) {
                 local_copy = strdup(op->stderr_data);
                 more = strlen(local_copy);
                 last = 0;
 
                 for (lpc = 0; lpc < more; lpc++) {
                     if (local_copy[lpc] == '\n' || local_copy[lpc] == 0) {
                         local_copy[lpc] = 0;
                         printf(" >  stderr: %s\n", local_copy + last);
                         last = lpc + 1;
                     }
                 }
                 free(local_copy);
             }
         }
         rc = op->rc;
         services_action_free(op);
         return crm_exit(rc);
 
     } else if (rsc_cmd == 'A' || rsc_cmd == 'a') {
         GListPtr lpc = NULL;
         resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
         xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set.input);
 
         if (rsc == NULL) {
             CMD_ERR("Must supply a resource id with -r\n");
             rc = -ENXIO;
             goto bail;
         }
 
         unpack_constraints(cib_constraints, &data_set);
 
         for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
             resource_t *r = (resource_t *) lpc->data;
 
             clear_bit(r->flags, pe_rsc_allocating);
         }
 
         show_colocation(rsc, TRUE, rsc_cmd == 'A', 1);
 
         fprintf(stdout, "* %s\n", rsc->id);
         show_location(rsc, NULL);
 
         for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
             resource_t *r = (resource_t *) lpc->data;
 
             clear_bit(r->flags, pe_rsc_allocating);
         }
 
         show_colocation(rsc, FALSE, rsc_cmd == 'A', 1);
 
     } else if (rsc_cmd == 'c') {
         int found = 0;
         GListPtr lpc = NULL;
 
         rc = pcmk_ok;
         for (lpc = data_set.resources; lpc != NULL; lpc = lpc->next) {
             resource_t *rsc = (resource_t *) lpc->data;
 
             print_cts_rsc(rsc);
             found++;
         }
         print_cts_constraints(&data_set);
 
     } else if (rsc_cmd == 'F') {
         rc = fail_lrm_rsc(crmd_channel, host_uname, rsc_id, &data_set);
         if (rc == pcmk_ok) {
             start_mainloop();
         }
 
     } else if (rsc_cmd == 'O') {
         rc = list_resource_operations(rsc_id, host_uname, TRUE, &data_set);
 
     } else if (rsc_cmd == 'o') {
         rc = list_resource_operations(rsc_id, host_uname, FALSE, &data_set);
 
     } else if (rc == -ENXIO) {
         CMD_ERR("Resource '%s' not found: %s\n", crm_str(rsc_id), pcmk_strerror(rc));
 
     } else if (rsc_cmd == 'W') {
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r\n");
             rc = -ENXIO;
             goto bail;
         }
         rc = do_find_resource(rsc_id, NULL, &data_set);
 
     } else if (rsc_cmd == 'q') {
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r\n");
             rc = -ENXIO;
             goto bail;
         }
         rc = dump_resource(rsc_id, &data_set, TRUE);
 
     } else if (rsc_cmd == 'w') {
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r\n");
             rc = -ENXIO;
             goto bail;
         }
         rc = dump_resource(rsc_id, &data_set, FALSE);
 
     } else if (rsc_cmd == 'U') {
         node_t *dest = NULL;
 
         if (rsc_id == NULL) {
             CMD_ERR("No value specified for --resource\n");
             rc = -ENXIO;
             goto bail;
         }
 
         if (host_uname) {
             dest = pe_find_node(data_set.nodes, host_uname);
             if (dest == NULL) {
                 CMD_ERR("Unknown node: %s\n", host_uname);
                 rc = -ENXIO;
                 goto bail;
             }
             rc = clear_resource(rsc_id, dest->details->uname, NULL, cib_conn);
 
         } else {
             rc = clear_resource(rsc_id, NULL, data_set.nodes, cib_conn);
         }
 
     } else if (rsc_cmd == 'M' && host_uname) {
 
         int count = 0;
         node_t *current = NULL;
         node_t *dest = pe_find_node(data_set.nodes, host_uname);
         resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
 
         rc = -EINVAL;
 
         if (rsc == NULL) {
             CMD_ERR("Resource '%s' not moved: not found\n", rsc_id);
             rc = -ENXIO;
             goto bail;
 
         } else if (scope_master && rsc->variant < pe_master) {
             resource_t *p = uber_parent(rsc);
             if(p->variant == pe_master) {
                 CMD_ERR("Using parent '%s' for --move command instead of '%s'.\n", rsc->id, rsc_id);
                 rsc_id = p->id;
                 rsc = p;
 
             } else {
                 CMD_ERR("Ignoring '--master' option: not valid for %s resources.\n",
                         get_resource_typename(rsc->variant));
                 scope_master = FALSE;
             }
         }
 
         if(rsc->variant == pe_master) {
             GListPtr iter = NULL;
 
             for(iter = rsc->children; iter; iter = iter->next) {
                 resource_t *child = (resource_t *)iter->data;
                 if(child->role == RSC_ROLE_MASTER) {
                     rsc = child;
                     count++;
                 }
             }
 
             if(scope_master == FALSE && count == 0) {
                 count = g_list_length(rsc->running_on);
             }
 
         } else if (rsc->variant > pe_group) {
             count = g_list_length(rsc->running_on);
 
         } else if (g_list_length(rsc->running_on) > 1) {
             CMD_ERR("Resource '%s' not moved: active on multiple nodes\n", rsc_id);
             goto bail;
         }
 
         if(dest == NULL) {
             CMD_ERR("Error performing operation: node '%s' is unknown\n", host_uname);
             rc = -ENXIO;
             goto bail;
         }
 
         if(g_list_length(rsc->running_on) == 1) {
             current = rsc->running_on->data;
         }
 
         if(current == NULL) {
             /* Nothing to check */
 
         } else if(scope_master && rsc->role != RSC_ROLE_MASTER) {
             crm_trace("%s is already active on %s but not in correct state", rsc_id, dest->details->uname);
 
         } else if (safe_str_eq(current->details->uname, dest->details->uname)) {
             CMD_ERR("Error performing operation: %s is already %s on %s\n",
                     rsc_id, scope_master?"promoted":"active", dest->details->uname);
             goto bail;
         }
 
         /* Clear any previous constraints for 'dest' */
         clear_resource(rsc_id, dest->details->uname, data_set.nodes, cib_conn);
 
         /* Record an explicit preference for 'dest' */
         rc = prefer_resource(rsc_id, dest->details->uname, cib_conn);
 
         crm_trace("%s%s now prefers node %s%s",
                   rsc->id, scope_master?" (master)":"", dest->details->uname, do_force?"(forced)":"");
 
         if(do_force) {
             /* Ban the original location if possible */
             if(current) {
                 ban_resource(rsc_id, current->details->uname, NULL, cib_conn);
 
             } else if(count > 1) {
                 CMD_ERR("Resource '%s' is currently %s in %d locations.  One may now move one to %s\n",
                         rsc_id, scope_master?"promoted":"active", count, dest->details->uname);
                 CMD_ERR("You can prevent '%s' from being %s at a specific location with:"
                         " --ban %s--host <name>\n", rsc_id, scope_master?"promoted":"active", scope_master?"--master ":"");
 
             } else {
                 crm_trace("Not banning %s from it's current location: not active", rsc_id);
             }
         }
 
     } else if (rsc_cmd == 'B' && host_uname) {
         resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
         node_t *dest = pe_find_node(data_set.nodes, host_uname);
 
         rc = -ENXIO;
         if (rsc_id == NULL) {
             CMD_ERR("No value specified for --resource\n");
             goto bail;
         } else if(rsc == NULL) {
             CMD_ERR("Resource '%s' not moved: unknown\n", rsc_id);
 
         } else if (dest == NULL) {
             CMD_ERR("Error performing operation: node '%s' is unknown\n", host_uname);
             goto bail;
         }
         rc = ban_resource(rsc_id, dest->details->uname, NULL, cib_conn);
 
     } else if (rsc_cmd == 'B' || rsc_cmd == 'M') {
         resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
 
         rc = -ENXIO;
         if (rsc_id == NULL) {
             CMD_ERR("No value specified for --resource\n");
             goto bail;
         }
 
         rc = -EINVAL;
         if(rsc == NULL) {
             CMD_ERR("Resource '%s' not moved: unknown\n", rsc_id);
 
         } else if(g_list_length(rsc->running_on) == 1) {
             node_t *current = rsc->running_on->data;
             rc = ban_resource(rsc_id, current->details->uname, NULL, cib_conn);
 
         } else if(rsc->variant == pe_master) {
             int count = 0;
             GListPtr iter = NULL;
             node_t *current = NULL;
 
             for(iter = rsc->children; iter; iter = iter->next) {
                 resource_t *child = (resource_t *)iter->data;
                 if(child->role == RSC_ROLE_MASTER) {
                     count++;
                     current = child->running_on->data;
                 }
             }
 
             if(count == 1 && current) {
                 rc = ban_resource(rsc_id, current->details->uname, NULL, cib_conn);
 
             } else {
                 CMD_ERR("Resource '%s' not moved: active in %d locations (promoted in %d).\n", rsc_id, g_list_length(rsc->running_on), count);
                 CMD_ERR("You can prevent '%s' from running on a specific location with: --ban --host <name>\n", rsc_id);
                 CMD_ERR("You can prevent '%s' from being promoted at a specific location with:"
                         " --ban --master --host <name>\n", rsc_id);
             }
 
         } else {
             CMD_ERR("Resource '%s' not moved: active in %d locations.\n", rsc_id, g_list_length(rsc->running_on));
             CMD_ERR("You can prevent '%s' from running on a specific location with: --ban --host <name>\n", rsc_id);
         }
 
     } else if (rsc_cmd == 'G') {
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r\n");
             rc = -ENXIO;
             goto bail;
         }
         rc = dump_resource_prop(rsc_id, prop_name, &data_set);
 
     } else if (rsc_cmd == 'S') {
         xmlNode *msg_data = NULL;
 
         if (prop_value == NULL || strlen(prop_value) == 0) {
             CMD_ERR("You need to supply a value with the -v option\n");
             rc = -EINVAL;
             goto bail;
 
         } else if (cib_conn == NULL) {
             rc = -ENOTCONN;
             goto bail;
         }
 
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r\n");
             rc = -ENXIO;
             goto bail;
         }
         CRM_LOG_ASSERT(rsc_type != NULL);
         CRM_LOG_ASSERT(prop_name != NULL);
         CRM_LOG_ASSERT(prop_value != NULL);
 
         msg_data = create_xml_node(NULL, rsc_type);
         crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);
         crm_xml_add(msg_data, prop_name, prop_value);
 
         rc = cib_conn->cmds->modify(cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
         free_xml(msg_data);
 
     } else if (rsc_cmd == 'g') {
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r\n");
             rc = -ENXIO;
             goto bail;
         }
         rc = dump_resource_attr(rsc_id, prop_name, &data_set);
 
     } else if (rsc_cmd == 'p') {
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r\n");
             rc = -ENXIO;
             goto bail;
         }
         if (prop_value == NULL || strlen(prop_value) == 0) {
             CMD_ERR("You need to supply a value with the -v option\n");
             rc = -EINVAL;
             goto bail;
         }
 
         /* coverity[var_deref_model] False positive */
         rc = set_resource_attr(rsc_id, prop_set, prop_id, prop_name,
                                prop_value, recursive, cib_conn, &data_set);
 
     } else if (rsc_cmd == 'd') {
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r\n");
             rc = -ENXIO;
             goto bail;
         }
         /* coverity[var_deref_model] False positive */
         rc = delete_resource_attr(rsc_id, prop_set, prop_id, prop_name, cib_conn, &data_set);
 
     } else if (rsc_cmd == 'C' && rsc_id) {
         resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
 
         crm_debug("Re-checking the state of %s on %s", rsc_id, host_uname);
         if(rsc) {
             crmd_replies_needed = 0;
             rc = delete_lrm_rsc(cib_conn, crmd_channel, host_uname, rsc, &data_set);
         } else {
             rc = -ENODEV;
         }
 
         if (rc == pcmk_ok) {
             start_mainloop();
         }
 
     } else if (rsc_cmd == 'C') {
         xmlNode *cmd = create_request(CRM_OP_REPROBE, NULL, host_uname,
                                       CRM_SYSTEM_CRMD, crm_system_name, our_pid);
 
         crm_debug("Re-checking the state of all resources on %s", host_uname);
         if (crm_ipc_send(crmd_channel, cmd, 0, 0, NULL) > 0) {
             start_mainloop();
         }
 
         free_xml(cmd);
 
     } else if (rsc_cmd == 'D') {
         xmlNode *msg_data = NULL;
 
         if (rsc_id == NULL) {
             CMD_ERR("Must supply a resource id with -r\n");
             rc = -ENXIO;
             goto bail;
 
         }
         if (rsc_type == NULL) {
             CMD_ERR("You need to specify a resource type with -t");
             rc = -ENXIO;
             goto bail;
 
         } else if (cib_conn == NULL) {
             rc = -ENOTCONN;
             goto bail;
         }
 
         msg_data = create_xml_node(NULL, rsc_type);
         crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);
 
         rc = cib_conn->cmds->delete(cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
         free_xml(msg_data);
 
     } else {
         CMD_ERR("Unknown command: %c\n", rsc_cmd);
     }
 
   bail:
 
-    if (cib_conn != NULL) {
+    if (data_set.input != NULL) {
         cleanup_alloc_calculations(&data_set);
+    }
+    if (cib_conn != NULL) {
         cib_conn->cmds->signoff(cib_conn);
         cib_delete(cib_conn);
     }
 
     if (rc == -pcmk_err_no_quorum) {
         CMD_ERR("Error performing operation: %s\n", pcmk_strerror(rc));
         CMD_ERR("Try using -f\n");
 
     } else if (rc != pcmk_ok) {
         CMD_ERR("Error performing operation: %s\n", pcmk_strerror(rc));
     }
 
     return crm_exit(rc);
 }