diff --git a/crmd/callbacks.c b/crmd/callbacks.c
index 4c2b3a4912..24fccba788 100644
--- a/crmd/callbacks.c
+++ b/crmd/callbacks.c
@@ -1,309 +1,316 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <string.h>
 #include <crmd_fsa.h>
 
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crm/cluster.h>
 #include <crm/cib.h>
 
 #include <crmd.h>
 #include <crmd_messages.h>
 #include <crmd_callbacks.h>
 #include <crmd_lrm.h>
 #include <tengine.h>
 #include <membership.h>
 
 void crmd_ha_connection_destroy(gpointer user_data);
 
 /* From join_dc... */
 extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
 
 void
 crmd_ha_connection_destroy(gpointer user_data)
 {
     crm_trace("Invoked");
     if (is_set(fsa_input_register, R_HA_DISCONNECTED)) {
         /* we signed out, so this is expected */
         crm_info("Heartbeat disconnection complete");
         return;
     }
 
     crm_crit("Lost connection to heartbeat service!");
     register_fsa_input(C_HA_DISCONNECT, I_ERROR, NULL);
     trigger_fsa(fsa_source);
 }
 
 void
 crmd_ha_msg_filter(xmlNode * msg)
 {
     if (AM_I_DC) {
         const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
 
         if (safe_str_eq(sys_from, CRM_SYSTEM_DC)) {
             const char *from = crm_element_value(msg, F_ORIG);
 
             if (safe_str_neq(from, fsa_our_uname)) {
                 int level = LOG_INFO;
                 const char *op = crm_element_value(msg, F_CRM_TASK);
 
                 /* make sure the election happens NOW */
                 if (fsa_state != S_ELECTION) {
                     ha_msg_input_t new_input;
 
                     level = LOG_WARNING;
                     new_input.msg = msg;
                     register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, &new_input,
                                            __FUNCTION__);
                 }
 
                 do_crm_log(level, "Another DC detected: %s (op=%s)", from, op);
                 goto done;
             }
         }
 
     } else {
         const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
 
         if (safe_str_eq(sys_to, CRM_SYSTEM_DC)) {
             return;
         }
     }
 
     /* crm_log_xml_trace("HA[inbound]", msg); */
     route_message(C_HA_MESSAGE, msg);
 
   done:
     trigger_fsa(fsa_source);
 }
 
 #define state_text(state) ((state)? (const char *)(state) : "in unknown state")
 
 void
 peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
 {
     uint32_t old = 0;
     uint32_t changed = 0;
     bool appeared = FALSE;
     bool is_remote = is_set(node->flags, crm_remote_node);
     const char *status = NULL;
 
     /* Crmd waits to receive some information from the membership layer before
      * declaring itself operational. If this is being called for a cluster node,
      * indicate that we have it.
      */
     if (!is_remote) {
         set_bit(fsa_input_register, R_PEER_DATA);
     }
 
     if (node->uname == NULL) {
         return;
     }
 
     switch (type) {
         case crm_status_uname:
             /* If we've never seen the node, then it also won't be in the status section */
             crm_info("%s is now %s", node->uname, state_text(node->state));
             return;
 
         case crm_status_rstate:
         case crm_status_nstate:
             /* This callback should not be called unless the state actually
              * changed, but here's a failsafe just in case.
              */
             CRM_CHECK(safe_str_neq(data, node->state), return);
 
             crm_info("%s node %s is now %s (was %s)",
                      (is_remote? "Remote" : "Cluster"),
                      node->uname, state_text(node->state), state_text(data));
 
             if (safe_str_eq(CRM_NODE_MEMBER, node->state)) {
                 appeared = TRUE;
                 if (!is_remote) {
                     remove_stonith_cleanup(node->uname);
                 }
             }
 
             crmd_notify_node_event(node);
             break;
 
         case crm_status_processes:
             if (data) {
                 old = *(const uint32_t *)data;
                 changed = node->processes ^ old;
             }
 
             status = (node->processes & proc_flags) ? ONLINESTATUS : OFFLINESTATUS;
             crm_info("Client %s/%s now has status [%s] (DC=%s, changed=%6x)",
                      node->uname, peer2text(proc_flags), status,
                      AM_I_DC ? "true" : crm_str(fsa_our_dc), changed);
 
             if ((changed & proc_flags) == 0) {
                 /* Peer process did not change */
                 crm_trace("No change %6x %6x %6x", old, node->processes, proc_flags);
                 return;
             } else if (is_not_set(fsa_input_register, R_CIB_CONNECTED)) {
                 crm_trace("Not connected");
                 return;
             } else if (fsa_state == S_STOPPING) {
                 crm_trace("Stopping");
                 return;
             }
 
             appeared = (node->processes & proc_flags) != 0;
             if (safe_str_eq(node->uname, fsa_our_uname) && (node->processes & proc_flags) == 0) {
                 /* Did we get evicted? */
                 crm_notice("Our peer connection failed");
                 register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ERROR, NULL);
 
             } else if (safe_str_eq(node->uname, fsa_our_dc) && crm_is_peer_active(node) == FALSE) {
                 /* Did the DC leave us? */
                 crm_notice("Our peer on the DC (%s) is dead", fsa_our_dc);
                 register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL);
 
+                /* @COMPAT DC < 1.1.13: If a DC shuts down normally, we don't
+                 * want to fence it. Newer DCs will send their shutdown request
+                 * to all peers, who will update the DC's expected state to
+                 * down, thus avoiding fencing. We can safely erase the DC's
+                 * transient attributes when it leaves in that case. However,
+                 * the only way to avoid fencing older DCs is to leave the
+                 * transient attributes intact until it rejoins.
+                 */
                 if (compare_version(fsa_our_dc_version, "3.0.9") > 0) {
                     erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
                 }
 
             } else if(AM_I_DC && appeared == FALSE) {
                 crm_info("Peer %s left us", node->uname);
                 erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
-                /* crm_update_peer_join(__FUNCTION__, node, crm_join_none); */
             }
             break;
     }
 
     if (AM_I_DC) {
         xmlNode *update = NULL;
         int flags = node_update_peer;
         gboolean alive = is_remote? appeared : crm_is_peer_active(node);
         crm_action_t *down = match_down_event(node->uuid, appeared);
 
         crm_trace("Alive=%d, appeared=%d, down=%d",
                   alive, appeared, (down? down->id : -1));
 
         if (alive && type == crm_status_processes) {
             register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
         }
 
         if (down) {
             const char *task = crm_element_value(down->xml, XML_LRM_ATTR_TASK);
 
             if (safe_str_eq(task, CRM_OP_FENCE)) {
 
                 /* tengine_stonith_callback() confirms fence actions */
                 crm_trace("Updating CIB %s stonithd reported fencing of %s complete",
                           (down->confirmed? "after" : "before"), node->uname);
 
             } else if ((alive == FALSE) && safe_str_eq(task, CRM_OP_SHUTDOWN)) {
                 crm_notice("%s of peer %s is complete "CRM_XS" op=%d",
                            task, node->uname, down->id);
 
                 /* down->confirmed = TRUE; */
                 stop_te_timer(down->timer);
 
                 if (!is_remote) {
                     flags |= node_update_join | node_update_expected;
                     crmd_peer_down(node, FALSE);
                     check_join_state(fsa_state, __FUNCTION__);
                 }
 
                 update_graph(transition_graph, down);
                 trigger_graph();
 
             } else {
                 crm_trace("Node %s is %salive, was expected to %s (op %d)",
                           node->uname, (alive? "" : "not "), task, down->id);
             }
 
         } else if (appeared == FALSE) {
             crm_notice("Stonith/shutdown of %s not matched", node->uname);
 
             if (!is_remote) {
                 crm_update_peer_join(__FUNCTION__, node, crm_join_none);
                 check_join_state(fsa_state, __FUNCTION__);
             }
 
             abort_transition(INFINITY, tg_restart, "Node failure", NULL);
             fail_incompletable_actions(transition_graph, node->uuid);
 
         } else {
             crm_trace("Node %s came up, was not expected to be down",
                       node->uname);
         }
 
         if (is_remote) {
             /* A pacemaker_remote node won't have its cluster status updated
              * in the CIB by membership-layer callbacks, so do it here.
              */
             flags |= node_update_cluster;
 
             /* Trigger resource placement on newly integrated nodes */
             if (appeared) {
                 abort_transition(INFINITY, tg_restart,
                                  "pacemaker_remote node integrated", NULL);
             }
         }
 
         /* Update the CIB node state */
         update = create_node_state_update(node, flags, NULL, __FUNCTION__);
         fsa_cib_anon_update(XML_CIB_TAG_STATUS, update,
                             cib_scope_local | cib_quorum_override | cib_can_create);
         free_xml(update);
     }
 
     trigger_fsa(fsa_source);
 }
 
 void
 crmd_cib_connection_destroy(gpointer user_data)
 {
     CRM_CHECK(user_data == fsa_cib_conn,;);
 
     crm_trace("Invoked");
     trigger_fsa(fsa_source);
     fsa_cib_conn->state = cib_disconnected;
 
     if (is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) {
         crm_info("Connection to the CIB terminated...");
         return;
     }
 
     /* eventually this will trigger a reconnect, not a shutdown */
     crm_err("Connection to the CIB terminated...");
     register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
     clear_bit(fsa_input_register, R_CIB_CONNECTED);
 
     return;
 }
 
 gboolean
 crm_fsa_trigger(gpointer user_data)
 {
     crm_trace("Invoked (queue len: %d)", g_list_length(fsa_message_queue));
     s_crmd_fsa(C_FSA_INTERNAL);
     crm_trace("Exited  (queue len: %d)", g_list_length(fsa_message_queue));
     return TRUE;
 }
diff --git a/crmd/control.c b/crmd/control.c
index 5081dbd390..177ff5775e 100644
--- a/crmd/control.c
+++ b/crmd/control.c
@@ -1,1156 +1,1157 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 
 #include <crm/msg_xml.h>
 
 #include <crm/pengine/rules.h>
 #include <crm/cluster/internal.h>
 #include <crm/cluster/election.h>
 #include <crm/common/ipcs.h>
 
 #include <crmd.h>
 #include <crmd_fsa.h>
 #include <fsa_proto.h>
 #include <crmd_messages.h>
 #include <crmd_callbacks.h>
 #include <crmd_lrm.h>
 #include <tengine.h>
 #include <throttle.h>
 
 #include <sys/types.h>
 #include <sys/stat.h>
 
 qb_ipcs_service_t *ipcs = NULL;
 
 extern gboolean crm_connect_corosync(crm_cluster_t * cluster);
 extern void crmd_ha_connection_destroy(gpointer user_data);
 
 void crm_shutdown(int nsig);
 gboolean crm_read_options(gpointer user_data);
 
 gboolean fsa_has_quorum = FALSE;
 crm_trigger_t *fsa_source = NULL;
 crm_trigger_t *config_read = NULL;
 bool no_quorum_suicide_escalation = FALSE;
 
 static gboolean
 election_timeout_popped(gpointer data)
 {
     /* Not everyone voted */
     crm_info("Election failed: Declaring ourselves the winner");
     register_fsa_input(C_TIMER_POPPED, I_ELECTION_DC, NULL);
     return FALSE;
 }
 
 /*	 A_HA_CONNECT	*/
 void
 do_ha_control(long long action,
               enum crmd_fsa_cause cause,
               enum crmd_fsa_state cur_state,
               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     gboolean registered = FALSE;
     static crm_cluster_t *cluster = NULL;
 
     if (cluster == NULL) {
         cluster = calloc(1, sizeof(crm_cluster_t));
     }
 
     if (action & A_HA_DISCONNECT) {
         crm_cluster_disconnect(cluster);
         crm_info("Disconnected from the cluster");
 
         set_bit(fsa_input_register, R_HA_DISCONNECTED);
     }
 
     if (action & A_HA_CONNECT) {
         crm_set_status_callback(&peer_update_callback);
         crm_set_autoreap(FALSE);
 
         if (is_openais_cluster()) {
 #if SUPPORT_COROSYNC
             registered = crm_connect_corosync(cluster);
 #endif
         } else if (is_heartbeat_cluster()) {
 #if SUPPORT_HEARTBEAT
             cluster->destroy = crmd_ha_connection_destroy;
             cluster->hb_dispatch = crmd_ha_msg_callback;
 
             registered = crm_cluster_connect(cluster);
             fsa_cluster_conn = cluster->hb_conn;
 
             crm_trace("Be informed of Node Status changes");
             if (registered &&
                 fsa_cluster_conn->llc_ops->set_nstatus_callback(fsa_cluster_conn,
                                                                 crmd_ha_status_callback,
                                                                 fsa_cluster_conn) != HA_OK) {
 
                 crm_err("Cannot set nstatus callback: %s",
                         fsa_cluster_conn->llc_ops->errmsg(fsa_cluster_conn));
                 registered = FALSE;
             }
 
             crm_trace("Be informed of CRM Client Status changes");
             if (registered &&
                 fsa_cluster_conn->llc_ops->set_cstatus_callback(fsa_cluster_conn,
                                                                 crmd_client_status_callback,
                                                                 fsa_cluster_conn) != HA_OK) {
 
                 crm_err("Cannot set cstatus callback: %s",
                         fsa_cluster_conn->llc_ops->errmsg(fsa_cluster_conn));
                 registered = FALSE;
             }
 
             if (registered) {
                 crm_trace("Requesting an initial dump of CRMD client_status");
                 fsa_cluster_conn->llc_ops->client_status(fsa_cluster_conn, NULL, CRM_SYSTEM_CRMD,
                                                          -1);
             }
 #endif
         }
         fsa_election = election_init(NULL, cluster->uname, 60000/*60s*/, election_timeout_popped);
         fsa_our_uname = cluster->uname;
         fsa_our_uuid = cluster->uuid;
         if(cluster->uuid == NULL) {
             crm_err("Could not obtain local uuid");
             registered = FALSE;
         }
 
         if (registered == FALSE) {
             set_bit(fsa_input_register, R_HA_DISCONNECTED);
             register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
             return;
         }
 
         populate_cib_nodes(node_update_none, __FUNCTION__);
         clear_bit(fsa_input_register, R_HA_DISCONNECTED);
         crm_info("Connected to the cluster");
     }
 
     if (action & ~(A_HA_CONNECT | A_HA_DISCONNECT)) {
         crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__);
     }
 }
 
 static bool
 need_spawn_pengine_from_crmd(void)
 {
 	static int result = -1;
 
 	if (result != -1)
 		return result;
 	if (!is_heartbeat_cluster()) {
 		result = 0;
 		return result;
 	}
 
 	/* NULL, or "strange" value: rather spawn from here. */
 	result = TRUE;
 	crm_str_to_boolean(daemon_option("crmd_spawns_pengine"), &result);
 	return result;
 }
 
 /*	 A_SHUTDOWN	*/
 void
 do_shutdown(long long action,
             enum crmd_fsa_cause cause,
             enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     /* just in case */
     set_bit(fsa_input_register, R_SHUTDOWN);
 
     if (need_spawn_pengine_from_crmd()) {
         if (is_set(fsa_input_register, pe_subsystem->flag_connected)) {
             crm_info("Terminating the %s", pe_subsystem->name);
             if (stop_subsystem(pe_subsystem, TRUE) == FALSE) {
                 /* its gone... */
                 crm_err("Faking %s exit", pe_subsystem->name);
                 clear_bit(fsa_input_register, pe_subsystem->flag_connected);
             } else {
                 crm_info("Waiting for subsystems to exit");
                 crmd_fsa_stall(FALSE);
             }
         }
         crm_info("All subsystems stopped, continuing");
     }
 
     if (stonith_api) {
         /* Prevent it from coming up again */
         clear_bit(fsa_input_register, R_ST_REQUIRED);
 
         crm_info("Disconnecting STONITH...");
         stonith_api->cmds->disconnect(stonith_api);
     }
 }
 
 /*	 A_SHUTDOWN_REQ	*/
 void
 do_shutdown_req(long long action,
                 enum crmd_fsa_cause cause,
                 enum crmd_fsa_state cur_state,
                 enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     xmlNode *msg = NULL;
 
     set_bit(fsa_input_register, R_SHUTDOWN);
-    crm_info("Sending shutdown request to %s", crm_str(fsa_our_dc));
+    crm_info("Sending shutdown request to all peers (DC is %s)",
+             (fsa_our_dc? fsa_our_dc : "not set"));
     msg = create_request(CRM_OP_SHUTDOWN_REQ, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
 
 /* 	set_bit(fsa_input_register, R_STAYDOWN); */
     if (send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) {
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
     free_xml(msg);
 }
 
 extern crm_ipc_t *attrd_ipc;
 extern char *max_generation_from;
 extern xmlNode *max_generation_xml;
 extern GHashTable *resource_history;
 extern GHashTable *voted;
 extern GHashTable *metadata_hash;
 extern char *te_client_id;
 
 void log_connected_client(gpointer key, gpointer value, gpointer user_data);
 
 void
 log_connected_client(gpointer key, gpointer value, gpointer user_data)
 {
     crm_client_t *client = value;
 
     crm_err("%s is still connected at exit", crm_client_name(client));
 }
 
 int
 crmd_fast_exit(int rc) 
 {
     if (is_set(fsa_input_register, R_STAYDOWN)) {
         crm_warn("Inhibiting respawn "CRM_XS" remapping exit code %d to %d",
                  rc, DAEMON_RESPAWN_STOP);
         rc = DAEMON_RESPAWN_STOP;
     }
 
     if (rc == pcmk_ok && is_set(fsa_input_register, R_IN_RECOVERY)) {
         crm_err("Could not recover from internal error");
         rc = pcmk_err_generic;
     }
     return crm_exit(rc);
 }
 
 int
 crmd_exit(int rc)
 {
     GListPtr gIter = NULL;
     GMainLoop *mloop = crmd_mainloop;
 
     static bool in_progress = FALSE;
 
     if(in_progress && rc == 0) {
         crm_debug("Exit is already in progress");
         return rc;
 
     } else if(in_progress) {
         crm_notice("Error during shutdown process, terminating now with status %d: %s",
                    rc, pcmk_strerror(rc));
         crm_write_blackbox(SIGTRAP, NULL);
         crmd_fast_exit(rc);
     }
 
     in_progress = TRUE;
     crm_trace("Preparing to exit: %d", rc);
 
     /* Suppress secondary errors resulting from us disconnecting everything */
     set_bit(fsa_input_register, R_HA_DISCONNECTED);
 
 /* Close all IPC servers and clients to ensure any and all shared memory files are cleaned up */
 
     if(ipcs) {
         crm_trace("Closing IPC server");
         mainloop_del_ipc_server(ipcs);
         ipcs = NULL;
     }
 
     if (attrd_ipc) {
         crm_trace("Closing attrd connection");
         crm_ipc_close(attrd_ipc);
         crm_ipc_destroy(attrd_ipc);
         attrd_ipc = NULL;
     }
 
     if (pe_subsystem && pe_subsystem->client && pe_subsystem->client->ipcs) {
         crm_trace("Disconnecting Policy Engine");
         qb_ipcs_disconnect(pe_subsystem->client->ipcs);
     }
 
     if(stonith_api) {
         crm_trace("Disconnecting fencing API");
         clear_bit(fsa_input_register, R_ST_REQUIRED);
         stonith_api->cmds->free(stonith_api); stonith_api = NULL;
     }
 
     if (rc == pcmk_ok && crmd_mainloop == NULL) {
         crm_debug("No mainloop detected");
         rc = EPROTO;
     }
 
     /* On an error, just get out.
      *
      * Otherwise, make the effort to have mainloop exit gracefully so
      * that it (mostly) cleans up after itself and valgrind has less
      * to report on - allowing real errors stand out
      */
     if(rc != pcmk_ok) {
         crm_notice("Forcing immediate exit with status %d: %s",
                    rc, pcmk_strerror(rc));
         crm_write_blackbox(SIGTRAP, NULL);
         return crmd_fast_exit(rc);
     }
 
 /* Clean up as much memory as possible for valgrind */
 
     for (gIter = fsa_message_queue; gIter != NULL; gIter = gIter->next) {
         fsa_data_t *fsa_data = gIter->data;
 
         crm_info("Dropping %s: [ state=%s cause=%s origin=%s ]",
                  fsa_input2string(fsa_data->fsa_input),
                  fsa_state2string(fsa_state),
                  fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
         delete_fsa_input(fsa_data);
     }
 
     clear_bit(fsa_input_register, R_MEMBERSHIP);
     g_list_free(fsa_message_queue); fsa_message_queue = NULL;
 
     free(pe_subsystem); pe_subsystem = NULL;
     free(te_subsystem); te_subsystem = NULL;
     free(cib_subsystem); cib_subsystem = NULL;
 
     if (metadata_hash) {
         crm_trace("Destroying reload cache with %d members", g_hash_table_size(metadata_hash));
         g_hash_table_destroy(metadata_hash); metadata_hash = NULL;
     }
 
     election_fini(fsa_election);
     fsa_election = NULL;
 
     cib_delete(fsa_cib_conn);
     fsa_cib_conn = NULL;
 
     verify_stopped(fsa_state, LOG_WARNING);
     clear_bit(fsa_input_register, R_LRM_CONNECTED);
     lrm_state_destroy_all();
 
     /* This basically will not work, since mainloop has a reference to it */
     mainloop_destroy_trigger(fsa_source); fsa_source = NULL;
 
     mainloop_destroy_trigger(config_read); config_read = NULL;
     mainloop_destroy_trigger(stonith_reconnect); stonith_reconnect = NULL;
     mainloop_destroy_trigger(transition_trigger); transition_trigger = NULL;
 
     crm_client_cleanup();
     crm_peer_destroy();
 
     crm_timer_stop(transition_timer);
     crm_timer_stop(integration_timer);
     crm_timer_stop(finalization_timer);
     crm_timer_stop(election_trigger);
     election_timeout_stop(fsa_election);
     crm_timer_stop(shutdown_escalation_timer);
     crm_timer_stop(wait_timer);
     crm_timer_stop(recheck_timer);
 
     free(transition_timer); transition_timer = NULL;
     free(integration_timer); integration_timer = NULL;
     free(finalization_timer); finalization_timer = NULL;
     free(election_trigger); election_trigger = NULL;
     election_fini(fsa_election);
     free(shutdown_escalation_timer); shutdown_escalation_timer = NULL;
     free(wait_timer); wait_timer = NULL;
     free(recheck_timer); recheck_timer = NULL;
 
     free(fsa_our_dc_version); fsa_our_dc_version = NULL;
     free(fsa_our_uname); fsa_our_uname = NULL;
     free(fsa_our_uuid); fsa_our_uuid = NULL;
     free(fsa_our_dc); fsa_our_dc = NULL;
 
     free(fsa_cluster_name); fsa_cluster_name = NULL;
 
     free(te_uuid); te_uuid = NULL;
     free(te_client_id); te_client_id = NULL;
     free(fsa_pe_ref); fsa_pe_ref = NULL;
     free(failed_stop_offset); failed_stop_offset = NULL;
     free(failed_start_offset); failed_start_offset = NULL;
 
     free(max_generation_from); max_generation_from = NULL;
     free_xml(max_generation_xml); max_generation_xml = NULL;
 
     mainloop_destroy_signal(SIGPIPE);
     mainloop_destroy_signal(SIGUSR1);
     mainloop_destroy_signal(SIGTERM);
     mainloop_destroy_signal(SIGTRAP);
     /* leave SIGCHLD engaged as we might still want to drain some service-actions */
 
     if (mloop) {
         GMainContext *ctx = g_main_loop_get_context(crmd_mainloop);
 
         /* Don't re-enter this block */
         crmd_mainloop = NULL;
 
         crmd_drain_alerts(ctx);
 
         /* no signals on final draining anymore */
         mainloop_destroy_signal(SIGCHLD);
 
         crm_trace("Draining mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
 
         {
             int lpc = 0;
 
             while((g_main_context_pending(ctx) && lpc < 10)) {
                 lpc++;
                 crm_trace("Iteration %d", lpc);
                 g_main_context_dispatch(ctx);
             }
         }
 
         crm_trace("Closing mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
         g_main_loop_quit(mloop);
 
 #if SUPPORT_HEARTBEAT
         /* Do this only after g_main_loop_quit().
          *
          * This interface was broken (incomplete) since it was introduced.
          * ->delete() does cleanup and free most of it, but it does not
          * actually remove and destroy the corresponding GSource, so the next
          * prepare/check iteratioin would find a corrupt (because partially
          * freed) GSource, and segfault.
          *
          * Apparently one was supposed to store the GSource as returned by
          * G_main_add_ll_cluster(), and g_source_destroy() that "by hand".
          *
          * But no-one ever did this, not even in the old hb code when this was
          * introduced.
          *
          * Note that fsa_cluster_conn was set as an "alias" to cluster->hb_conn
          * in do_ha_control() right after crm_cluster_connect(), and only
          * happens to still point at that object, because do_ha_control() does
          * not reset it to NULL after crm_cluster_disconnect() above does
          * reset cluster->hb_conn to NULL.
          * Not sure if that's something to cleanup, too.
          *
          * I'll try to fix this up in heartbeat proper, so ->delete
          * will actually remove, and destroy, and unref, and free this thing.
          * Doing so after g_main_loop_quit() is valid with both old,
          * and eventually fixed heartbeat.
          *
          * If we introduce the "by hand" destroy/remove/unref,
          * this may break again once heartbeat is fixed :-(
          *
          *                                              -- Lars Ellenberg
          */
         if (fsa_cluster_conn) {
             crm_trace("Deleting heartbeat api object");
             fsa_cluster_conn->llc_ops->delete(fsa_cluster_conn);
             fsa_cluster_conn = NULL;
         }
 #endif
 
         /* Won't do anything yet, since we're inside it now */
         g_main_loop_unref(mloop);
 
         crm_trace("Done %d", rc);
     } else {
         mainloop_destroy_signal(SIGCHLD);
     }
 
     /* Graceful */
     return rc;
 }
 
 /*	 A_EXIT_0, A_EXIT_1	*/
 void
 do_exit(long long action,
         enum crmd_fsa_cause cause,
         enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     int exit_code = pcmk_ok;
     int log_level = LOG_INFO;
     const char *exit_type = "gracefully";
 
     if (action & A_EXIT_1) {
         /* exit_code = pcmk_err_generic; */
         log_level = LOG_ERR;
         exit_type = "forcefully";
         exit_code = pcmk_err_generic;
     }
 
     verify_stopped(cur_state, LOG_ERR);
     do_crm_log(log_level, "Performing %s - %s exiting the CRMd",
                fsa_action2string(action), exit_type);
 
     crm_info("[%s] stopped (%d)", crm_system_name, exit_code);
     crmd_exit(exit_code);
 }
 
 static void sigpipe_ignore(int nsig) { return; }
 
 /*	 A_STARTUP	*/
 void
 do_startup(long long action,
            enum crmd_fsa_cause cause,
            enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     int was_error = 0;
 
     crm_debug("Registering Signal Handlers");
     mainloop_add_signal(SIGTERM, crm_shutdown);
     mainloop_add_signal(SIGPIPE, sigpipe_ignore);
 
     fsa_source = mainloop_add_trigger(G_PRIORITY_HIGH, crm_fsa_trigger, NULL);
     config_read = mainloop_add_trigger(G_PRIORITY_HIGH, crm_read_options, NULL);
     transition_trigger = mainloop_add_trigger(G_PRIORITY_LOW, te_graph_trigger, NULL);
 
     crm_debug("Creating CIB and LRM objects");
     fsa_cib_conn = cib_new();
 
     lrm_state_init_local();
 
     /* set up the timers */
     transition_timer = calloc(1, sizeof(fsa_timer_t));
     integration_timer = calloc(1, sizeof(fsa_timer_t));
     finalization_timer = calloc(1, sizeof(fsa_timer_t));
     election_trigger = calloc(1, sizeof(fsa_timer_t));
     shutdown_escalation_timer = calloc(1, sizeof(fsa_timer_t));
     wait_timer = calloc(1, sizeof(fsa_timer_t));
     recheck_timer = calloc(1, sizeof(fsa_timer_t));
 
     if (election_trigger != NULL) {
         election_trigger->source_id = 0;
         election_trigger->period_ms = -1;
         election_trigger->fsa_input = I_DC_TIMEOUT;
         election_trigger->callback = crm_timer_popped;
         election_trigger->repeat = FALSE;
     } else {
         was_error = TRUE;
     }
 
     if (transition_timer != NULL) {
         transition_timer->source_id = 0;
         transition_timer->period_ms = -1;
         transition_timer->fsa_input = I_PE_CALC;
         transition_timer->callback = crm_timer_popped;
         transition_timer->repeat = FALSE;
     } else {
         was_error = TRUE;
     }
 
     if (integration_timer != NULL) {
         integration_timer->source_id = 0;
         integration_timer->period_ms = -1;
         integration_timer->fsa_input = I_INTEGRATED;
         integration_timer->callback = crm_timer_popped;
         integration_timer->repeat = FALSE;
     } else {
         was_error = TRUE;
     }
 
     if (finalization_timer != NULL) {
         finalization_timer->source_id = 0;
         finalization_timer->period_ms = -1;
         finalization_timer->fsa_input = I_FINALIZED;
         finalization_timer->callback = crm_timer_popped;
         finalization_timer->repeat = FALSE;
         /* for possible enabling... a bug in the join protocol left
          *    a slave in S_PENDING while we think its in S_NOT_DC
          *
          * raising I_FINALIZED put us into a transition loop which is
          *    never resolved.
          * in this loop we continually send probes which the node
          *    NACK's because its in S_PENDING
          *
          * if we have nodes where heartbeat is active but the
          *    CRM is not... then this will be handled in the
          *    integration phase
          */
         finalization_timer->fsa_input = I_ELECTION;
 
     } else {
         was_error = TRUE;
     }
 
     if (shutdown_escalation_timer != NULL) {
         shutdown_escalation_timer->source_id = 0;
         shutdown_escalation_timer->period_ms = -1;
         shutdown_escalation_timer->fsa_input = I_STOP;
         shutdown_escalation_timer->callback = crm_timer_popped;
         shutdown_escalation_timer->repeat = FALSE;
     } else {
         was_error = TRUE;
     }
 
     if (wait_timer != NULL) {
         wait_timer->source_id = 0;
         wait_timer->period_ms = 2000;
         wait_timer->fsa_input = I_NULL;
         wait_timer->callback = crm_timer_popped;
         wait_timer->repeat = FALSE;
     } else {
         was_error = TRUE;
     }
 
     if (recheck_timer != NULL) {
         recheck_timer->source_id = 0;
         recheck_timer->period_ms = -1;
         recheck_timer->fsa_input = I_PE_CALC;
         recheck_timer->callback = crm_timer_popped;
         recheck_timer->repeat = FALSE;
     } else {
         was_error = TRUE;
     }
 
     /* set up the sub systems */
     cib_subsystem = calloc(1, sizeof(struct crm_subsystem_s));
     te_subsystem = calloc(1, sizeof(struct crm_subsystem_s));
     pe_subsystem = calloc(1, sizeof(struct crm_subsystem_s));
 
     if (cib_subsystem != NULL) {
         cib_subsystem->pid = -1;
         cib_subsystem->name = CRM_SYSTEM_CIB;
         cib_subsystem->flag_connected = R_CIB_CONNECTED;
         cib_subsystem->flag_required = R_CIB_REQUIRED;
 
     } else {
         was_error = TRUE;
     }
 
     if (te_subsystem != NULL) {
         te_subsystem->pid = -1;
         te_subsystem->name = CRM_SYSTEM_TENGINE;
         te_subsystem->flag_connected = R_TE_CONNECTED;
         te_subsystem->flag_required = R_TE_REQUIRED;
 
     } else {
         was_error = TRUE;
     }
 
     if (pe_subsystem != NULL) {
         pe_subsystem->pid = -1;
         pe_subsystem->path = CRM_DAEMON_DIR;
         pe_subsystem->name = CRM_SYSTEM_PENGINE;
         pe_subsystem->command = CRM_DAEMON_DIR "/" CRM_SYSTEM_PENGINE;
         pe_subsystem->args = NULL;
         pe_subsystem->flag_connected = R_PE_CONNECTED;
         pe_subsystem->flag_required = R_PE_REQUIRED;
 
     } else {
         was_error = TRUE;
     }
 
     if (was_error == FALSE && need_spawn_pengine_from_crmd()) {
         if (start_subsystem(pe_subsystem) == FALSE) {
             was_error = TRUE;
         }
     }
 
     if (was_error) {
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
 
 }
 
 static int32_t
 crmd_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
 {
     crm_trace("Connection %p", c);
     if (crm_client_new(c, uid, gid) == NULL) {
         return -EIO;
     }
     return 0;
 }
 
 static void
 crmd_ipc_created(qb_ipcs_connection_t * c)
 {
     crm_trace("Connection %p", c);
 }
 
 static int32_t
 crmd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size)
 {
     uint32_t id = 0;
     uint32_t flags = 0;
     crm_client_t *client = crm_client_get(c);
 
     xmlNode *msg = crm_ipcs_recv(client, data, size, &id, &flags);
 
     crm_trace("Invoked: %s", crm_client_name(client));
     crm_ipcs_send_ack(client, id, flags, "ack", __FUNCTION__, __LINE__);
 
     if (msg == NULL) {
         return 0;
     }
 
 #if ENABLE_ACL
     CRM_ASSERT(client->user != NULL);
     crm_acl_get_set_user(msg, F_CRM_USER, client->user);
 #endif
 
     crm_trace("Processing msg from %s", crm_client_name(client));
     crm_log_xml_trace(msg, "CRMd[inbound]");
 
     crm_xml_add(msg, F_CRM_SYS_FROM, client->id);
     if (crmd_authorize_message(msg, client, NULL)) {
         route_message(C_IPC_MESSAGE, msg);
     }
 
     trigger_fsa(fsa_source);
     free_xml(msg);
     return 0;
 }
 
 static int32_t
 crmd_ipc_closed(qb_ipcs_connection_t * c)
 {
     crm_client_t *client = crm_client_get(c);
     struct crm_subsystem_s *the_subsystem = NULL;
 
     if (client == NULL) {
         return 0;
     }
 
     crm_trace("Connection %p", c);
 
     if (client->userdata == NULL) {
         crm_trace("Client hadn't registered with us yet");
 
     } else if (strcasecmp(CRM_SYSTEM_PENGINE, client->userdata) == 0) {
         the_subsystem = pe_subsystem;
 
     } else if (strcasecmp(CRM_SYSTEM_TENGINE, client->userdata) == 0) {
         the_subsystem = te_subsystem;
 
     } else if (strcasecmp(CRM_SYSTEM_CIB, client->userdata) == 0) {
         the_subsystem = cib_subsystem;
     }
 
     if (the_subsystem != NULL) {
         the_subsystem->source = NULL;
         the_subsystem->client = NULL;
         crm_info("Received HUP from %s:[%d]", the_subsystem->name, the_subsystem->pid);
 
     } else {
         /* else that was a transient client */
         crm_trace("Received HUP from transient client");
     }
 
     crm_trace("Disconnecting client %s (%p)", crm_client_name(client), client);
     free(client->userdata);
     crm_client_destroy(client);
 
     trigger_fsa(fsa_source);
     return 0;
 }
 
 static void
 crmd_ipc_destroy(qb_ipcs_connection_t * c)
 {
     crm_trace("Connection %p", c);
     crmd_ipc_closed(c);
 }
 
 /*	 A_STOP	*/
 void
 do_stop(long long action,
         enum crmd_fsa_cause cause,
         enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     crm_trace("Closing IPC server");
     mainloop_del_ipc_server(ipcs); ipcs = NULL;
     register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
 }
 
 /*	 A_STARTED	*/
 void
 do_started(long long action,
            enum crmd_fsa_cause cause,
            enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     static struct qb_ipcs_service_handlers crmd_callbacks = {
         .connection_accept = crmd_ipc_accept,
         .connection_created = crmd_ipc_created,
         .msg_process = crmd_ipc_dispatch,
         .connection_closed = crmd_ipc_closed,
         .connection_destroyed = crmd_ipc_destroy
     };
 
     if (cur_state != S_STARTING) {
         crm_err("Start cancelled... %s", fsa_state2string(cur_state));
         return;
 
     } else if (is_set(fsa_input_register, R_MEMBERSHIP) == FALSE) {
         crm_info("Delaying start, no membership data (%.16llx)", R_MEMBERSHIP);
 
         crmd_fsa_stall(TRUE);
         return;
 
     } else if (is_set(fsa_input_register, R_LRM_CONNECTED) == FALSE) {
         crm_info("Delaying start, LRM not connected (%.16llx)", R_LRM_CONNECTED);
 
         crmd_fsa_stall(TRUE);
         return;
 
     } else if (is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) {
         crm_info("Delaying start, CIB not connected (%.16llx)", R_CIB_CONNECTED);
 
         crmd_fsa_stall(TRUE);
         return;
 
     } else if (is_set(fsa_input_register, R_READ_CONFIG) == FALSE) {
         crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG);
 
         crmd_fsa_stall(TRUE);
         return;
 
     } else if (is_set(fsa_input_register, R_PEER_DATA) == FALSE) {
 
         /* try reading from HA */
         crm_info("Delaying start, No peer data (%.16llx)", R_PEER_DATA);
 
 #if SUPPORT_HEARTBEAT
         if (is_heartbeat_cluster()) {
             HA_Message *msg = NULL;
 
             crm_trace("Looking for a HA message");
             msg = fsa_cluster_conn->llc_ops->readmsg(fsa_cluster_conn, 0);
             if (msg != NULL) {
                 crm_trace("There was a HA message");
                 ha_msg_del(msg);
             }
         }
 #endif
         crmd_fsa_stall(TRUE);
         return;
     }
 
     crm_debug("Init server comms");
     ipcs = crmd_ipc_server_init(&crmd_callbacks);
     if (ipcs == NULL) {
         crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
     }
 
     if (stonith_reconnect == NULL) {
         int dummy;
 
         stonith_reconnect = mainloop_add_trigger(G_PRIORITY_LOW, te_connect_stonith, &dummy);
     }
     set_bit(fsa_input_register, R_ST_REQUIRED);
     mainloop_set_trigger(stonith_reconnect);
 
     crm_notice("The local CRM is operational");
     clear_bit(fsa_input_register, R_STARTING);
     register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL);
 }
 
 /*	 A_RECOVER	*/
 void
 do_recover(long long action,
            enum crmd_fsa_cause cause,
            enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     set_bit(fsa_input_register, R_IN_RECOVERY);
     crm_warn("Fast-tracking shutdown in response to errors");
 
     register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
 }
 
 /* *INDENT-OFF* */
 pe_cluster_option crmd_opts[] = {
 	/* name, old-name, validate, values, default, short description, long description */
 	{ "dc-version", NULL, "string", NULL, "none", NULL,
           "Version of Pacemaker on the cluster's DC.",
           "Includes the hash which identifies the exact changeset it was built from.  Used for diagnostic purposes."
         },
 	{ "cluster-infrastructure", NULL, "string", NULL, "heartbeat", NULL,
           "The messaging stack on which Pacemaker is currently running.",
           "Used for informational and diagnostic purposes." },
 	{ XML_CONFIG_ATTR_DC_DEADTIME, "dc_deadtime", "time", NULL, "20s", &check_time,
           "How long to wait for a response from other nodes during startup.",
           "The \"correct\" value will depend on the speed/load of your network and the type of switches used."
         },
 	{ XML_CONFIG_ATTR_RECHECK, "cluster_recheck_interval", "time",
 	  "Zero disables polling.  Positive values are an interval in seconds (unless other SI units are specified. eg. 5min)",
           "15min", &check_timer,
 	  "Polling interval for time based changes to options, resource parameters and constraints.",
 	  "The Cluster is primarily event driven, however the configuration can have elements that change based on time."
 	  "  To ensure these changes take effect, we can optionally poll the cluster's status for changes."
         },
 
 #ifdef RHEL7_COMPAT
     /* These options were superseded by the alerts feature and now are just an
      * alternate interface to it. It was never released upstream, but was
      * released in RHEL 7, so we allow it to be enabled at compile-time by
      * defining RHEL7_COMPAT.
      */
 	{ "notification-agent", NULL, "string", NULL, "/dev/null", &check_script,
           "Deprecated",
           "Use alert path in alerts section instead"
         },
 	{ "notification-recipient", NULL, "string", NULL, "", NULL,
           "Deprecated",
           "Use recipient value in alerts section instead"
         },
 #endif
 
 	{ "load-threshold", NULL, "percentage", NULL, "80%", &check_utilization,
 	  "The maximum amount of system resources that should be used by nodes in the cluster",
 	  "The cluster will slow down its recovery process when the amount of system resources used"
           " (currently CPU) approaches this limit",
         },
 	{ "node-action-limit", NULL, "integer", NULL, "0", &check_number,
           "The maximum number of jobs that can be scheduled per node. Defaults to 2x cores"},
 	{ XML_CONFIG_ATTR_ELECTION_FAIL, "election_timeout", "time", NULL, "2min", &check_timer,
           "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug."
         },
 	{ XML_CONFIG_ATTR_FORCE_QUIT, "shutdown_escalation", "time", NULL, "20min", &check_timer,
           "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug."
         },
 	{ "crmd-integration-timeout", NULL, "time", NULL, "3min", &check_timer,
           "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug."
         },
 	{ "crmd-finalization-timeout", NULL, "time", NULL, "30min", &check_timer,
           "*** Advanced Use Only ***.", "If you need to adjust this value, it probably indicates the presence of a bug."
         },
 	{ "crmd-transition-delay", NULL, "time", NULL, "0s", &check_timer,
           "*** Advanced Use Only ***\n"
           "Enabling this option will slow down cluster recovery under all conditions",
           "Delay cluster recovery for the configured interval to allow for additional/related events to occur.\n"
           "Useful if your configuration is sensitive to the order in which ping updates arrive."
         },
 	{ "stonith-watchdog-timeout", NULL, "time", NULL, NULL, &check_sbd_timeout,
 	  "How long to wait before we can assume nodes are safely down", NULL
         },
 	{ "no-quorum-policy", "no_quorum_policy", "enum", "stop, freeze, ignore, suicide", "stop", &check_quorum, NULL, NULL },
 
 #if SUPPORT_PLUGIN
 	{ XML_ATTR_EXPECTED_VOTES, NULL, "integer", NULL, "2", &check_number, "The number of nodes expected to be in the cluster", "Used to calculate quorum in openais based clusters." },
 #endif
 };
 /* *INDENT-ON* */
 
 void
 crmd_metadata(void)
 {
     config_metadata("CRM Daemon", "1.0",
                     "CRM Daemon Options",
                     "This is a fake resource that details the options that can be configured for the CRM Daemon.",
                     crmd_opts, DIMOF(crmd_opts));
 }
 
 static void
 verify_crmd_options(GHashTable * options)
 {
     verify_all_options(options, crmd_opts, DIMOF(crmd_opts));
 }
 
 static const char *
 crmd_pref(GHashTable * options, const char *name)
 {
     return get_cluster_pref(options, crmd_opts, DIMOF(crmd_opts), name);
 }
 
 static void
 config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
 #ifdef RHEL7_COMPAT
     const char *script = NULL;
 #endif
     const char *value = NULL;
     GHashTable *config_hash = NULL;
     crm_time_t *now = crm_time_new(NULL);
     xmlNode *crmconfig = NULL;
     xmlNode *alerts = NULL;
 
     if (rc != pcmk_ok) {
         fsa_data_t *msg_data = NULL;
 
         crm_err("Local CIB query resulted in an error: %s", pcmk_strerror(rc));
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
 
         if (rc == -EACCES || rc == -pcmk_err_schema_validation) {
             crm_err("The cluster is mis-configured - shutting down and staying down");
             set_bit(fsa_input_register, R_STAYDOWN);
         }
         goto bail;
     }
 
     crmconfig = output;
     if ((crmconfig) &&
         (crm_element_name(crmconfig)) &&
         (strcmp(crm_element_name(crmconfig), XML_CIB_TAG_CRMCONFIG) != 0)) {
         crmconfig = first_named_child(crmconfig, XML_CIB_TAG_CRMCONFIG);
     }
     if (!crmconfig) {
         fsa_data_t *msg_data = NULL;
 
         crm_err("Local CIB query for " XML_CIB_TAG_CRMCONFIG " section failed");
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
         goto bail;
     }
 
     crm_debug("Call %d : Parsing CIB options", call_id);
     config_hash =
         g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
 
     unpack_instance_attributes(crmconfig, crmconfig, XML_CIB_TAG_PROPSET, NULL, config_hash,
                                CIB_OPTIONS_FIRST, FALSE, now);
 
     verify_crmd_options(config_hash);
 
 #ifdef RHEL7_COMPAT
     script = crmd_pref(config_hash, "notification-agent");
     value  = crmd_pref(config_hash, "notification-recipient");
     crmd_enable_notifications(script, value);
 #endif
 
     value = crmd_pref(config_hash, XML_CONFIG_ATTR_DC_DEADTIME);
     election_trigger->period_ms = crm_get_msec(value);
 
     value = crmd_pref(config_hash, "node-action-limit"); /* Also checks migration-limit */
     throttle_update_job_max(value);
 
     value = crmd_pref(config_hash, "load-threshold");
     if(value) {
         throttle_load_target = strtof(value, NULL) / 100;
     }
 
     value = crmd_pref(config_hash, "no-quorum-policy");
     if (safe_str_eq(value, "suicide") && pcmk_locate_sbd()) {
         no_quorum_suicide_escalation = TRUE;
     }
 
     value = crmd_pref(config_hash, XML_CONFIG_ATTR_FORCE_QUIT);
     shutdown_escalation_timer->period_ms = crm_get_msec(value);
     /* How long to declare an election over - even if not everyone voted */
     crm_debug("Shutdown escalation occurs after: %dms", shutdown_escalation_timer->period_ms);
 
     value = crmd_pref(config_hash, XML_CONFIG_ATTR_ELECTION_FAIL);
     election_timeout_set_period(fsa_election, crm_get_msec(value));
 
     value = crmd_pref(config_hash, XML_CONFIG_ATTR_RECHECK);
     recheck_timer->period_ms = crm_get_msec(value);
     crm_debug("Checking for expired actions every %dms", recheck_timer->period_ms);
 
     value = crmd_pref(config_hash, "crmd-transition-delay");
     transition_timer->period_ms = crm_get_msec(value);
 
     value = crmd_pref(config_hash, "crmd-integration-timeout");
     integration_timer->period_ms = crm_get_msec(value);
 
     value = crmd_pref(config_hash, "crmd-finalization-timeout");
     finalization_timer->period_ms = crm_get_msec(value);
 
 #if SUPPORT_COROSYNC
     if (is_classic_ais_cluster()) {
         value = crmd_pref(config_hash, XML_ATTR_EXPECTED_VOTES);
         crm_debug("Sending expected-votes=%s to corosync", value);
         send_cluster_text(crm_class_quorum, value, TRUE, NULL, crm_msg_ais);
     }
 #endif
 
     free(fsa_cluster_name);
     fsa_cluster_name = NULL;
 
     value = g_hash_table_lookup(config_hash, "cluster-name");
     if (value) {
         fsa_cluster_name = strdup(value);
     }
 
     alerts = first_named_child(output, XML_CIB_TAG_ALERTS);
     parse_notifications(alerts);
 
     set_bit(fsa_input_register, R_READ_CONFIG);
     crm_trace("Triggering FSA: %s", __FUNCTION__);
     mainloop_set_trigger(fsa_source);
 
     g_hash_table_destroy(config_hash);
   bail:
     crm_time_free(now);
 }
 
 gboolean
 crm_read_options(gpointer user_data)
 {
     int call_id =
         fsa_cib_conn->cmds->query(fsa_cib_conn,
             "//" XML_CIB_TAG_CRMCONFIG " | //" XML_CIB_TAG_ALERTS,
             NULL, cib_xpath | cib_scope_local);
 
     fsa_register_cib_callback(call_id, FALSE, NULL, config_query_callback);
     crm_trace("Querying the CIB... call %d", call_id);
     return TRUE;
 }
 
 /*	 A_READCONFIG	*/
 void
 do_read_config(long long action,
                enum crmd_fsa_cause cause,
                enum crmd_fsa_state cur_state,
                enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     throttle_init();
     mainloop_set_trigger(config_read);
 }
 
 void
 crm_shutdown(int nsig)
 {
     if (crmd_mainloop != NULL && g_main_is_running(crmd_mainloop)) {
         if (is_set(fsa_input_register, R_SHUTDOWN)) {
             crm_err("Escalating the shutdown");
             register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL);
 
         } else {
             set_bit(fsa_input_register, R_SHUTDOWN);
             register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
 
             if (shutdown_escalation_timer->period_ms < 1) {
                 const char *value = crmd_pref(NULL, XML_CONFIG_ATTR_FORCE_QUIT);
                 int msec = crm_get_msec(value);
 
                 crm_debug("Using default shutdown escalation: %dms", msec);
                 shutdown_escalation_timer->period_ms = msec;
             }
 
             /* can't rely on this... */
             crm_notice("Shutting down cluster resource manager " CRM_XS
                        " limit=%dms", shutdown_escalation_timer->period_ms);
             crm_timer_start(shutdown_escalation_timer);
         }
 
     } else {
         crm_info("exit from shutdown");
         crmd_exit(pcmk_ok);
     }
 }
diff --git a/crmd/join_client.c b/crmd/join_client.c
index 155fae0e06..afd003ad33 100644
--- a/crmd/join_client.c
+++ b/crmd/join_client.c
@@ -1,283 +1,273 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 #include <crm_internal.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crmd_fsa.h>
 #include <crmd_messages.h>
 
 int reannounce_count = 0;
 void join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);
 
 extern ha_msg_input_t *copy_ha_msg_input(ha_msg_input_t * orig);
 
 /*	A_CL_JOIN_QUERY		*/
 /* is there a DC out there? */
 void
 do_cl_join_query(long long action,
                  enum crmd_fsa_cause cause,
                  enum crmd_fsa_state cur_state,
                  enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
                                   CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
 
     sleep(1);                   /* give the CCM time to propogate to the DC */
     update_dc(NULL);            /* Unset any existing value so that the result is not discarded */
     crm_debug("Querying for a DC");
     send_cluster_message(NULL, crm_msg_crmd, req, FALSE);
     free_xml(req);
 }
 
 /*	 A_CL_JOIN_ANNOUNCE	*/
 
 /* this is kind of a workaround for the fact that we may not be around or
  * are otherwise unable to reply when the DC sends out A_DC_JOIN_OFFER_ALL
  */
 void
 do_cl_join_announce(long long action,
                     enum crmd_fsa_cause cause,
                     enum crmd_fsa_state cur_state,
                     enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     /* Once we hear from the DC, we can stop the timer
      *
      * This timer was started either on startup or when a node
      * left the CCM list
      */
 
     /* don't announce if we're in one of these states */
     if (cur_state != S_PENDING) {
         crm_warn("Not announcing cluster join because in state %s",
                  fsa_state2string(cur_state));
         return;
     }
 
     if (AM_I_OPERATIONAL) {
         /* send as a broadcast */
         xmlNode *req = create_request(CRM_OP_JOIN_ANNOUNCE, NULL, NULL,
                                       CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
 
         crm_debug("Announcing availability");
         update_dc(NULL);
         send_cluster_message(NULL, crm_msg_crmd, req, FALSE);
         free_xml(req);
 
     } else {
         /* Delay announce until we have finished local startup */
         crm_warn("Delaying announce of cluster join until local startup is complete");
         return;
     }
 }
 
 static int query_call_id = 0;
 
 /*	 A_CL_JOIN_REQUEST	*/
 /* aka. accept the welcome offer */
 void
 do_cl_join_offer_respond(long long action,
                          enum crmd_fsa_cause cause,
                          enum crmd_fsa_state cur_state,
                          enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
     const char *welcome_from = crm_element_value(input->msg, F_CRM_HOST_FROM);
     const char *join_id = crm_element_value(input->msg, F_CRM_JOIN_ID);
 
 #if 0
     if (we are sick) {
         log error;
 
         /* save the request for later? */
         return;
     }
 #endif
 
     crm_trace("Accepting cluster join offer from node %s "CRM_XS" join-%s",
               welcome_from, crm_element_value(input->msg, F_CRM_JOIN_ID));
 
     /* we only ever want the last one */
     if (query_call_id > 0) {
         crm_trace("Cancelling previous join query: %d", query_call_id);
         remove_cib_op_callback(query_call_id, FALSE);
         query_call_id = 0;
     }
 
     if (update_dc(input->msg) == FALSE) {
         crm_warn("Discarding cluster join offer from node %s (expected %s)",
                  welcome_from, fsa_our_dc);
         return;
     }
 
     CRM_LOG_ASSERT(input != NULL);
     query_call_id =
         fsa_cib_conn->cmds->query(fsa_cib_conn, NULL, NULL, cib_scope_local | cib_no_children);
     fsa_register_cib_callback(query_call_id, FALSE, strdup(join_id), join_query_callback);
     crm_trace("Registered join query callback: %d", query_call_id);
 
     register_fsa_action(A_DC_TIMER_STOP);
 }
 
 void
 join_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     char *join_id = user_data;
     xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);
 
     CRM_LOG_ASSERT(join_id != NULL);
 
     if (query_call_id != call_id) {
         crm_trace("Query %d superseded", call_id);
         goto done;
     }
 
     query_call_id = 0;
     if(rc != pcmk_ok || output == NULL) {
         crm_err("Could not retrieve version details for join-%s: %s (%d)",
                 join_id, pcmk_strerror(rc), rc);
         register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, NULL, NULL, __FUNCTION__);
 
     } else if (fsa_our_dc == NULL) {
         crm_debug("Membership is in flux, not continuing join-%s", join_id);
 
     } else {
         xmlNode *reply = NULL;
 
         crm_debug("Respond to join offer join-%s from %s", join_id, fsa_our_dc);
         copy_in_properties(generation, output);
 
         reply = create_request(CRM_OP_JOIN_REQUEST, generation, fsa_our_dc,
                                CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
 
         crm_xml_add(reply, F_CRM_JOIN_ID, join_id);
         send_cluster_message(crm_get_peer(0, fsa_our_dc), crm_msg_crmd, reply, TRUE);
         free_xml(reply);
     }
 
   done:
     free_xml(generation);
 }
 
 /*	A_CL_JOIN_RESULT	*/
 /* aka. this is notification that we have (or have not) been accepted */
 void
 do_cl_join_finalize_respond(long long action,
                             enum crmd_fsa_cause cause,
                             enum crmd_fsa_state cur_state,
                             enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     xmlNode *tmp1 = NULL;
     gboolean was_nack = TRUE;
     static gboolean first_join = TRUE;
     ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
 
     int join_id = -1;
     const char *op = crm_element_value(input->msg, F_CRM_TASK);
     const char *ack_nack = crm_element_value(input->msg, CRM_OP_JOIN_ACKNAK);
     const char *welcome_from = crm_element_value(input->msg, F_CRM_HOST_FROM);
 
     if (safe_str_neq(op, CRM_OP_JOIN_ACKNAK)) {
         crm_trace("Ignoring op=%s message", op);
         return;
     }
 
     /* calculate if it was an ack or a nack */
     if (crm_is_true(ack_nack)) {
         was_nack = FALSE;
     }
 
     crm_element_value_int(input->msg, F_CRM_JOIN_ID, &join_id);
 
     if (was_nack) {
         crm_err("Shutting down because cluster join with leader %s failed "
                 CRM_XS" join-%d NACK'd", welcome_from, join_id);
         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
         return;
     }
 
     if (AM_I_DC == FALSE && safe_str_eq(welcome_from, fsa_our_uname)) {
         crm_warn("Discarding our own welcome - we're no longer the DC");
         return;
     }
 
     if (update_dc(input->msg) == FALSE) {
         crm_warn("Discarding %s from node %s (expected from %s)",
                  op, welcome_from, fsa_our_dc);
         return;
     }
 
     /* send our status section to the DC */
-    crm_debug("Confirming join join-%d: %s", join_id, crm_element_value(input->msg, F_CRM_TASK));
     tmp1 = do_lrm_query(TRUE, fsa_our_uname);
     if (tmp1 != NULL) {
         xmlNode *reply = create_request(CRM_OP_JOIN_CONFIRM, tmp1, fsa_our_dc,
                                         CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL);
 
         crm_xml_add_int(reply, F_CRM_JOIN_ID, join_id);
 
-        crm_debug("join-%d: Join complete."
-                  "  Sending local LRM status to %s", join_id, fsa_our_dc);
-
-        if (first_join) {
+        crm_debug("Confirming join-%d: sending local operation history to %s",
+                  join_id, fsa_our_dc);
+
+        /*
+         * If this is the node's first join since the crmd started on it, clear
+         * any previous transient node attributes, to handle the case where
+         * the node restarted so quickly that the cluster layer didn't notice.
+         *
+         * Do not remove the resources though, they'll be cleaned up in
+         * do_dc_join_ack(). Removing them here creates a race condition if the
+         * crmd is being recovered. Instead of a list of active resources from
+         * the lrmd, we may end up with a blank status section. If we are _NOT_
+         * lucky, we will probe for the "wrong" instance of anonymous clones and
+         * end up with multiple active instances on the machine.
+         */
+        if (first_join && is_not_set(fsa_input_register, R_SHUTDOWN)) {
             first_join = FALSE;
-
-            /*
-             * Clear any previous transient node attribute and lrm operations
-             *
-             * Corosync has a nasty habit of not being able to tell if a
-             *   node is returning or didn't leave in the first place.
-             * This confuses Pacemaker because it never gets a "node up"
-             *   event which is normally used to clean up the status section.
-             *
-             * Do not remove the resources though, they'll be cleaned up in
-             *   do_dc_join_ack().  Removing them here creates a race
-             *   condition if the crmd is being recovered.
-             * Instead of a list of active resources from the lrmd
-             *   we may end up with a blank status section.
-             * If we are _NOT_ lucky, we will probe for the "wrong" instance
-             *   of anonymous clones and end up with multiple active
-             *   instances on the machine.
-             */
             erase_status_tag(fsa_our_uname, XML_TAG_TRANSIENT_NODEATTRS, 0);
-
-            /* Just in case attrd was still around too */
-            if (is_not_set(fsa_input_register, R_SHUTDOWN)) {
-                update_attrd(fsa_our_uname, "terminate", NULL, NULL, FALSE);
-                update_attrd(fsa_our_uname, XML_CIB_ATTR_SHUTDOWN, "0", NULL, FALSE);
-            }
+            update_attrd(fsa_our_uname, "terminate", NULL, NULL, FALSE);
+            update_attrd(fsa_our_uname, XML_CIB_ATTR_SHUTDOWN, "0", NULL, FALSE);
         }
 
         send_cluster_message(crm_get_peer(0, fsa_our_dc), crm_msg_crmd, reply, TRUE);
         free_xml(reply);
 
         if (AM_I_DC == FALSE) {
             register_fsa_input_adv(cause, I_NOT_DC, NULL, A_NOTHING, TRUE, __FUNCTION__);
             update_attrd(NULL, NULL, NULL, NULL, FALSE);
         }
 
         free_xml(tmp1);
 
     } else {
-        crm_err("Could not send our LRM state to the DC");
+        crm_err("Could not confirm join-%d with %s: Local operation history failed",
+                join_id, fsa_our_dc);
         register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
     }
 }
diff --git a/crmd/utils.c b/crmd/utils.c
index 178ab979ee..bae30d1433 100644
--- a/crmd/utils.c
+++ b/crmd/utils.c
@@ -1,1135 +1,1160 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <signal.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/attrd.h>
 
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crm/cluster.h>
 
 #include <crmd_fsa.h>
 #include <crmd_utils.h>
 #include <crmd_messages.h>
 
 /*	A_DC_TIMER_STOP, A_DC_TIMER_START,
  *	A_FINALIZE_TIMER_STOP, A_FINALIZE_TIMER_START
  *	A_INTEGRATE_TIMER_STOP, A_INTEGRATE_TIMER_START
  */
 void
 do_timer_control(long long action,
                  enum crmd_fsa_cause cause,
                  enum crmd_fsa_state cur_state,
                  enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 {
     gboolean timer_op_ok = TRUE;
 
     if (action & A_DC_TIMER_STOP) {
         timer_op_ok = crm_timer_stop(election_trigger);
 
     } else if (action & A_FINALIZE_TIMER_STOP) {
         timer_op_ok = crm_timer_stop(finalization_timer);
 
     } else if (action & A_INTEGRATE_TIMER_STOP) {
         timer_op_ok = crm_timer_stop(integration_timer);
 
 /* 	} else if(action & A_ELECTION_TIMEOUT_STOP) { */
 /* 		timer_op_ok = crm_timer_stop(election_timeout); */
     }
 
     /* don't start a timer that wasn't already running */
     if (action & A_DC_TIMER_START && timer_op_ok) {
         crm_timer_start(election_trigger);
         if (AM_I_DC) {
             /* there can be only one */
             register_fsa_input(cause, I_ELECTION, NULL);
         }
 
     } else if (action & A_FINALIZE_TIMER_START) {
         crm_timer_start(finalization_timer);
 
     } else if (action & A_INTEGRATE_TIMER_START) {
         crm_timer_start(integration_timer);
 
 /* 	} else if(action & A_ELECTION_TIMEOUT_START) { */
 /* 		crm_timer_start(election_timeout); */
     }
 }
 
 const char *
 get_timer_desc(fsa_timer_t * timer)
 {
     if (timer == election_trigger) {
         return "Election Trigger";
 
     } else if (timer == shutdown_escalation_timer) {
         return "Shutdown Escalation";
 
     } else if (timer == integration_timer) {
         return "Integration Timer";
 
     } else if (timer == finalization_timer) {
         return "Finalization Timer";
 
     } else if (timer == transition_timer) {
         return "New Transition Timer";
 
     } else if (timer == wait_timer) {
         return "Wait Timer";
 
     } else if (timer == recheck_timer) {
         return "PEngine Recheck Timer";
 
     }
     return "Unknown Timer";
 }
 
 gboolean
 crm_timer_popped(gpointer data)
 {
     fsa_timer_t *timer = (fsa_timer_t *) data;
 
     if (timer == wait_timer
         || timer == recheck_timer
         || timer == transition_timer || timer == finalization_timer || timer == election_trigger) {
         crm_info("%s (%s) just popped (%dms)",
                  get_timer_desc(timer), fsa_input2string(timer->fsa_input), timer->period_ms);
         timer->counter++;
 
     } else {
         crm_err("%s (%s) just popped in state %s! (%dms)",
                 get_timer_desc(timer), fsa_input2string(timer->fsa_input),
                 fsa_state2string(fsa_state), timer->period_ms);
     }
 
     if (timer == election_trigger && election_trigger->counter > 5) {
         crm_notice("We appear to be in an election loop, something may be wrong");
         crm_write_blackbox(0, NULL);
         election_trigger->counter = 0;
     }
 
     if (timer->repeat == FALSE) {
         crm_timer_stop(timer);  /* make it _not_ go off again */
     }
 
     if (timer->fsa_input == I_INTEGRATED) {
         crm_info("Welcomed: %d, Integrated: %d",
                  crmd_join_phase_count(crm_join_welcomed),
                  crmd_join_phase_count(crm_join_integrated));
         if (crmd_join_phase_count(crm_join_welcomed) == 0) {
             /* If we don't even have ourself, start again */
             register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, NULL, __FUNCTION__);
 
         } else {
             register_fsa_input_before(C_TIMER_POPPED, timer->fsa_input, NULL);
         }
 
     } else if (timer == recheck_timer && fsa_state != S_IDLE) {
         crm_debug("Discarding %s event in state: %s",
                   fsa_input2string(timer->fsa_input), fsa_state2string(fsa_state));
 
     } else if (timer == finalization_timer && fsa_state != S_FINALIZE_JOIN) {
         crm_debug("Discarding %s event in state: %s",
                   fsa_input2string(timer->fsa_input), fsa_state2string(fsa_state));
 
     } else if (timer->fsa_input != I_NULL) {
         register_fsa_input(C_TIMER_POPPED, timer->fsa_input, NULL);
     }
 
     crm_trace("Triggering FSA: %s", __FUNCTION__);
     mainloop_set_trigger(fsa_source);
 
     return TRUE;
 }
 
 gboolean
 is_timer_started(fsa_timer_t * timer)
 {
     if (timer->period_ms > 0) {
         if (transition_timer->source_id == 0) {
             return FALSE;
         } else {
             return TRUE;
         }
     }
     return FALSE;
 }
 
 gboolean
 crm_timer_start(fsa_timer_t * timer)
 {
     const char *timer_desc = get_timer_desc(timer);
 
     if (timer->source_id == 0 && timer->period_ms > 0) {
         timer->source_id = g_timeout_add(timer->period_ms, timer->callback, (void *)timer);
         CRM_ASSERT(timer->source_id != 0);
         crm_debug("Started %s (%s:%dms), src=%d",
                   timer_desc, fsa_input2string(timer->fsa_input),
                   timer->period_ms, timer->source_id);
 
     } else if (timer->period_ms < 0) {
         crm_err("Tried to start %s (%s:%dms) with a -ve period",
                 timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms);
 
     } else {
         crm_debug("%s (%s:%dms) already running: src=%d",
                   timer_desc, fsa_input2string(timer->fsa_input),
                   timer->period_ms, timer->source_id);
         return FALSE;
     }
     return TRUE;
 }
 
 gboolean
 crm_timer_stop(fsa_timer_t * timer)
 {
     const char *timer_desc = get_timer_desc(timer);
 
     if (timer == NULL) {
         crm_err("Attempted to stop NULL timer");
         return FALSE;
 
     } else if (timer->source_id != 0) {
         crm_trace("Stopping %s (%s:%dms), src=%d",
                   timer_desc, fsa_input2string(timer->fsa_input),
                   timer->period_ms, timer->source_id);
         g_source_remove(timer->source_id);
         timer->source_id = 0;
 
     } else {
         crm_trace("%s (%s:%dms) already stopped",
                   timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms);
         return FALSE;
     }
     return TRUE;
 }
 
 const char *
 fsa_input2string(enum crmd_fsa_input input)
 {
     const char *inputAsText = NULL;
 
     switch (input) {
         case I_NULL:
             inputAsText = "I_NULL";
             break;
         case I_CIB_OP:
             inputAsText = "I_CIB_OP (unused)";
             break;
         case I_CIB_UPDATE:
             inputAsText = "I_CIB_UPDATE";
             break;
         case I_DC_TIMEOUT:
             inputAsText = "I_DC_TIMEOUT";
             break;
         case I_ELECTION:
             inputAsText = "I_ELECTION";
             break;
         case I_PE_CALC:
             inputAsText = "I_PE_CALC";
             break;
         case I_RELEASE_DC:
             inputAsText = "I_RELEASE_DC";
             break;
         case I_ELECTION_DC:
             inputAsText = "I_ELECTION_DC";
             break;
         case I_ERROR:
             inputAsText = "I_ERROR";
             break;
         case I_FAIL:
             inputAsText = "I_FAIL";
             break;
         case I_INTEGRATED:
             inputAsText = "I_INTEGRATED";
             break;
         case I_FINALIZED:
             inputAsText = "I_FINALIZED";
             break;
         case I_NODE_JOIN:
             inputAsText = "I_NODE_JOIN";
             break;
         case I_JOIN_OFFER:
             inputAsText = "I_JOIN_OFFER";
             break;
         case I_JOIN_REQUEST:
             inputAsText = "I_JOIN_REQUEST";
             break;
         case I_JOIN_RESULT:
             inputAsText = "I_JOIN_RESULT";
             break;
         case I_NOT_DC:
             inputAsText = "I_NOT_DC";
             break;
         case I_RECOVERED:
             inputAsText = "I_RECOVERED";
             break;
         case I_RELEASE_FAIL:
             inputAsText = "I_RELEASE_FAIL";
             break;
         case I_RELEASE_SUCCESS:
             inputAsText = "I_RELEASE_SUCCESS";
             break;
         case I_RESTART:
             inputAsText = "I_RESTART";
             break;
         case I_PE_SUCCESS:
             inputAsText = "I_PE_SUCCESS";
             break;
         case I_ROUTER:
             inputAsText = "I_ROUTER";
             break;
         case I_SHUTDOWN:
             inputAsText = "I_SHUTDOWN";
             break;
         case I_STARTUP:
             inputAsText = "I_STARTUP";
             break;
         case I_TE_SUCCESS:
             inputAsText = "I_TE_SUCCESS";
             break;
         case I_STOP:
             inputAsText = "I_STOP";
             break;
         case I_DC_HEARTBEAT:
             inputAsText = "I_DC_HEARTBEAT";
             break;
         case I_WAIT_FOR_EVENT:
             inputAsText = "I_WAIT_FOR_EVENT";
             break;
         case I_LRM_EVENT:
             inputAsText = "I_LRM_EVENT";
             break;
         case I_PENDING:
             inputAsText = "I_PENDING";
             break;
         case I_HALT:
             inputAsText = "I_HALT";
             break;
         case I_TERMINATE:
             inputAsText = "I_TERMINATE";
             break;
         case I_ILLEGAL:
             inputAsText = "I_ILLEGAL";
             break;
     }
 
     if (inputAsText == NULL) {
         crm_err("Input %d is unknown", input);
         inputAsText = "<UNKNOWN_INPUT>";
     }
 
     return inputAsText;
 }
 
 const char *
 fsa_state2string(enum crmd_fsa_state state)
 {
     const char *stateAsText = NULL;
 
     switch (state) {
         case S_IDLE:
             stateAsText = "S_IDLE";
             break;
         case S_ELECTION:
             stateAsText = "S_ELECTION";
             break;
         case S_INTEGRATION:
             stateAsText = "S_INTEGRATION";
             break;
         case S_FINALIZE_JOIN:
             stateAsText = "S_FINALIZE_JOIN";
             break;
         case S_NOT_DC:
             stateAsText = "S_NOT_DC";
             break;
         case S_POLICY_ENGINE:
             stateAsText = "S_POLICY_ENGINE";
             break;
         case S_RECOVERY:
             stateAsText = "S_RECOVERY";
             break;
         case S_RELEASE_DC:
             stateAsText = "S_RELEASE_DC";
             break;
         case S_PENDING:
             stateAsText = "S_PENDING";
             break;
         case S_STOPPING:
             stateAsText = "S_STOPPING";
             break;
         case S_TERMINATE:
             stateAsText = "S_TERMINATE";
             break;
         case S_TRANSITION_ENGINE:
             stateAsText = "S_TRANSITION_ENGINE";
             break;
         case S_STARTING:
             stateAsText = "S_STARTING";
             break;
         case S_HALT:
             stateAsText = "S_HALT";
             break;
         case S_ILLEGAL:
             stateAsText = "S_ILLEGAL";
             break;
     }
 
     if (stateAsText == NULL) {
         crm_err("State %d is unknown", state);
         stateAsText = "<UNKNOWN_STATE>";
     }
 
     return stateAsText;
 }
 
 const char *
 fsa_cause2string(enum crmd_fsa_cause cause)
 {
     const char *causeAsText = NULL;
 
     switch (cause) {
         case C_UNKNOWN:
             causeAsText = "C_UNKNOWN";
             break;
         case C_STARTUP:
             causeAsText = "C_STARTUP";
             break;
         case C_IPC_MESSAGE:
             causeAsText = "C_IPC_MESSAGE";
             break;
         case C_HA_MESSAGE:
             causeAsText = "C_HA_MESSAGE";
             break;
         case C_CCM_CALLBACK:
             causeAsText = "C_CCM_CALLBACK";
             break;
         case C_TIMER_POPPED:
             causeAsText = "C_TIMER_POPPED";
             break;
         case C_SHUTDOWN:
             causeAsText = "C_SHUTDOWN";
             break;
         case C_HEARTBEAT_FAILED:
             causeAsText = "C_HEARTBEAT_FAILED";
             break;
         case C_SUBSYSTEM_CONNECT:
             causeAsText = "C_SUBSYSTEM_CONNECT";
             break;
         case C_LRM_OP_CALLBACK:
             causeAsText = "C_LRM_OP_CALLBACK";
             break;
         case C_LRM_MONITOR_CALLBACK:
             causeAsText = "C_LRM_MONITOR_CALLBACK";
             break;
         case C_CRMD_STATUS_CALLBACK:
             causeAsText = "C_CRMD_STATUS_CALLBACK";
             break;
         case C_HA_DISCONNECT:
             causeAsText = "C_HA_DISCONNECT";
             break;
         case C_FSA_INTERNAL:
             causeAsText = "C_FSA_INTERNAL";
             break;
         case C_ILLEGAL:
             causeAsText = "C_ILLEGAL";
             break;
     }
 
     if (causeAsText == NULL) {
         crm_err("Cause %d is unknown", cause);
         causeAsText = "<UNKNOWN_CAUSE>";
     }
 
     return causeAsText;
 }
 
 const char *
 fsa_action2string(long long action)
 {
     const char *actionAsText = NULL;
 
     switch (action) {
 
         case A_NOTHING:
             actionAsText = "A_NOTHING";
             break;
         case A_ELECTION_START:
             actionAsText = "A_ELECTION_START";
             break;
         case A_DC_JOIN_FINAL:
             actionAsText = "A_DC_JOIN_FINAL";
             break;
         case A_READCONFIG:
             actionAsText = "A_READCONFIG";
             break;
         case O_RELEASE:
             actionAsText = "O_RELEASE";
             break;
         case A_STARTUP:
             actionAsText = "A_STARTUP";
             break;
         case A_STARTED:
             actionAsText = "A_STARTED";
             break;
         case A_HA_CONNECT:
             actionAsText = "A_HA_CONNECT";
             break;
         case A_HA_DISCONNECT:
             actionAsText = "A_HA_DISCONNECT";
             break;
         case A_LRM_CONNECT:
             actionAsText = "A_LRM_CONNECT";
             break;
         case A_LRM_EVENT:
             actionAsText = "A_LRM_EVENT";
             break;
         case A_LRM_INVOKE:
             actionAsText = "A_LRM_INVOKE";
             break;
         case A_LRM_DISCONNECT:
             actionAsText = "A_LRM_DISCONNECT";
             break;
         case O_LRM_RECONNECT:
             actionAsText = "O_LRM_RECONNECT";
             break;
         case A_CL_JOIN_QUERY:
             actionAsText = "A_CL_JOIN_QUERY";
             break;
         case A_DC_TIMER_STOP:
             actionAsText = "A_DC_TIMER_STOP";
             break;
         case A_DC_TIMER_START:
             actionAsText = "A_DC_TIMER_START";
             break;
         case A_INTEGRATE_TIMER_START:
             actionAsText = "A_INTEGRATE_TIMER_START";
             break;
         case A_INTEGRATE_TIMER_STOP:
             actionAsText = "A_INTEGRATE_TIMER_STOP";
             break;
         case A_FINALIZE_TIMER_START:
             actionAsText = "A_FINALIZE_TIMER_START";
             break;
         case A_FINALIZE_TIMER_STOP:
             actionAsText = "A_FINALIZE_TIMER_STOP";
             break;
         case A_ELECTION_COUNT:
             actionAsText = "A_ELECTION_COUNT";
             break;
         case A_ELECTION_VOTE:
             actionAsText = "A_ELECTION_VOTE";
             break;
         case A_ELECTION_CHECK:
             actionAsText = "A_ELECTION_CHECK";
             break;
         case A_CL_JOIN_ANNOUNCE:
             actionAsText = "A_CL_JOIN_ANNOUNCE";
             break;
         case A_CL_JOIN_REQUEST:
             actionAsText = "A_CL_JOIN_REQUEST";
             break;
         case A_CL_JOIN_RESULT:
             actionAsText = "A_CL_JOIN_RESULT";
             break;
         case A_DC_JOIN_OFFER_ALL:
             actionAsText = "A_DC_JOIN_OFFER_ALL";
             break;
         case A_DC_JOIN_OFFER_ONE:
             actionAsText = "A_DC_JOIN_OFFER_ONE";
             break;
         case A_DC_JOIN_PROCESS_REQ:
             actionAsText = "A_DC_JOIN_PROCESS_REQ";
             break;
         case A_DC_JOIN_PROCESS_ACK:
             actionAsText = "A_DC_JOIN_PROCESS_ACK";
             break;
         case A_DC_JOIN_FINALIZE:
             actionAsText = "A_DC_JOIN_FINALIZE";
             break;
         case A_MSG_PROCESS:
             actionAsText = "A_MSG_PROCESS";
             break;
         case A_MSG_ROUTE:
             actionAsText = "A_MSG_ROUTE";
             break;
         case A_RECOVER:
             actionAsText = "A_RECOVER";
             break;
         case A_DC_RELEASE:
             actionAsText = "A_DC_RELEASE";
             break;
         case A_DC_RELEASED:
             actionAsText = "A_DC_RELEASED";
             break;
         case A_DC_TAKEOVER:
             actionAsText = "A_DC_TAKEOVER";
             break;
         case A_SHUTDOWN:
             actionAsText = "A_SHUTDOWN";
             break;
         case A_SHUTDOWN_REQ:
             actionAsText = "A_SHUTDOWN_REQ";
             break;
         case A_STOP:
             actionAsText = "A_STOP  ";
             break;
         case A_EXIT_0:
             actionAsText = "A_EXIT_0";
             break;
         case A_EXIT_1:
             actionAsText = "A_EXIT_1";
             break;
         case A_CCM_CONNECT:
             actionAsText = "A_CCM_CONNECT";
             break;
         case A_CCM_DISCONNECT:
             actionAsText = "A_CCM_DISCONNECT";
             break;
         case O_CIB_RESTART:
             actionAsText = "O_CIB_RESTART";
             break;
         case A_CIB_START:
             actionAsText = "A_CIB_START";
             break;
         case A_CIB_STOP:
             actionAsText = "A_CIB_STOP";
             break;
         case A_TE_INVOKE:
             actionAsText = "A_TE_INVOKE";
             break;
         case O_TE_RESTART:
             actionAsText = "O_TE_RESTART";
             break;
         case A_TE_START:
             actionAsText = "A_TE_START";
             break;
         case A_TE_STOP:
             actionAsText = "A_TE_STOP";
             break;
         case A_TE_HALT:
             actionAsText = "A_TE_HALT";
             break;
         case A_TE_CANCEL:
             actionAsText = "A_TE_CANCEL";
             break;
         case A_PE_INVOKE:
             actionAsText = "A_PE_INVOKE";
             break;
         case O_PE_RESTART:
             actionAsText = "O_PE_RESTART";
             break;
         case A_PE_START:
             actionAsText = "A_PE_START";
             break;
         case A_PE_STOP:
             actionAsText = "A_PE_STOP";
             break;
         case A_NODE_BLOCK:
             actionAsText = "A_NODE_BLOCK";
             break;
         case A_UPDATE_NODESTATUS:
             actionAsText = "A_UPDATE_NODESTATUS";
             break;
         case A_LOG:
             actionAsText = "A_LOG   ";
             break;
         case A_ERROR:
             actionAsText = "A_ERROR ";
             break;
         case A_WARN:
             actionAsText = "A_WARN  ";
             break;
             /* Composite actions */
         case A_DC_TIMER_START | A_CL_JOIN_QUERY:
             actionAsText = "A_DC_TIMER_START|A_CL_JOIN_QUERY";
             break;
     }
 
     if (actionAsText == NULL) {
         crm_err("Action %.16llx is unknown", action);
         actionAsText = "<UNKNOWN_ACTION>";
     }
 
     return actionAsText;
 }
 
 void
 fsa_dump_inputs(int log_level, const char *text, long long input_register)
 {
     if (input_register == A_NOTHING) {
         return;
     }
     if (text == NULL) {
         text = "Input register contents:";
     }
 
     if (is_set(input_register, R_THE_DC)) {
         crm_trace("%s %.16llx (R_THE_DC)", text, R_THE_DC);
     }
     if (is_set(input_register, R_STARTING)) {
         crm_trace("%s %.16llx (R_STARTING)", text, R_STARTING);
     }
     if (is_set(input_register, R_SHUTDOWN)) {
         crm_trace("%s %.16llx (R_SHUTDOWN)", text, R_SHUTDOWN);
     }
     if (is_set(input_register, R_STAYDOWN)) {
         crm_trace("%s %.16llx (R_STAYDOWN)", text, R_STAYDOWN);
     }
     if (is_set(input_register, R_JOIN_OK)) {
         crm_trace("%s %.16llx (R_JOIN_OK)", text, R_JOIN_OK);
     }
     if (is_set(input_register, R_READ_CONFIG)) {
         crm_trace("%s %.16llx (R_READ_CONFIG)", text, R_READ_CONFIG);
     }
     if (is_set(input_register, R_INVOKE_PE)) {
         crm_trace("%s %.16llx (R_INVOKE_PE)", text, R_INVOKE_PE);
     }
     if (is_set(input_register, R_CIB_CONNECTED)) {
         crm_trace("%s %.16llx (R_CIB_CONNECTED)", text, R_CIB_CONNECTED);
     }
     if (is_set(input_register, R_PE_CONNECTED)) {
         crm_trace("%s %.16llx (R_PE_CONNECTED)", text, R_PE_CONNECTED);
     }
     if (is_set(input_register, R_TE_CONNECTED)) {
         crm_trace("%s %.16llx (R_TE_CONNECTED)", text, R_TE_CONNECTED);
     }
     if (is_set(input_register, R_LRM_CONNECTED)) {
         crm_trace("%s %.16llx (R_LRM_CONNECTED)", text, R_LRM_CONNECTED);
     }
     if (is_set(input_register, R_CIB_REQUIRED)) {
         crm_trace("%s %.16llx (R_CIB_REQUIRED)", text, R_CIB_REQUIRED);
     }
     if (is_set(input_register, R_PE_REQUIRED)) {
         crm_trace("%s %.16llx (R_PE_REQUIRED)", text, R_PE_REQUIRED);
     }
     if (is_set(input_register, R_TE_REQUIRED)) {
         crm_trace("%s %.16llx (R_TE_REQUIRED)", text, R_TE_REQUIRED);
     }
     if (is_set(input_register, R_REQ_PEND)) {
         crm_trace("%s %.16llx (R_REQ_PEND)", text, R_REQ_PEND);
     }
     if (is_set(input_register, R_PE_PEND)) {
         crm_trace("%s %.16llx (R_PE_PEND)", text, R_PE_PEND);
     }
     if (is_set(input_register, R_TE_PEND)) {
         crm_trace("%s %.16llx (R_TE_PEND)", text, R_TE_PEND);
     }
     if (is_set(input_register, R_RESP_PEND)) {
         crm_trace("%s %.16llx (R_RESP_PEND)", text, R_RESP_PEND);
     }
     if (is_set(input_register, R_CIB_DONE)) {
         crm_trace("%s %.16llx (R_CIB_DONE)", text, R_CIB_DONE);
     }
     if (is_set(input_register, R_HAVE_CIB)) {
         crm_trace("%s %.16llx (R_HAVE_CIB)", text, R_HAVE_CIB);
     }
     if (is_set(input_register, R_CIB_ASKED)) {
         crm_trace("%s %.16llx (R_CIB_ASKED)", text, R_CIB_ASKED);
     }
     if (is_set(input_register, R_MEMBERSHIP)) {
         crm_trace("%s %.16llx (R_MEMBERSHIP)", text, R_MEMBERSHIP);
     }
     if (is_set(input_register, R_PEER_DATA)) {
         crm_trace("%s %.16llx (R_PEER_DATA)", text, R_PEER_DATA);
     }
     if (is_set(input_register, R_IN_RECOVERY)) {
         crm_trace("%s %.16llx (R_IN_RECOVERY)", text, R_IN_RECOVERY);
     }
 }
 
 void
 fsa_dump_actions(long long action, const char *text)
 {
     if (is_set(action, A_READCONFIG)) {
         crm_trace("Action %.16llx (A_READCONFIG) %s", A_READCONFIG, text);
     }
     if (is_set(action, A_STARTUP)) {
         crm_trace("Action %.16llx (A_STARTUP) %s", A_STARTUP, text);
     }
     if (is_set(action, A_STARTED)) {
         crm_trace("Action %.16llx (A_STARTED) %s", A_STARTED, text);
     }
     if (is_set(action, A_HA_CONNECT)) {
         crm_trace("Action %.16llx (A_CONNECT) %s", A_HA_CONNECT, text);
     }
     if (is_set(action, A_HA_DISCONNECT)) {
         crm_trace("Action %.16llx (A_DISCONNECT) %s", A_HA_DISCONNECT, text);
     }
     if (is_set(action, A_LRM_CONNECT)) {
         crm_trace("Action %.16llx (A_LRM_CONNECT) %s", A_LRM_CONNECT, text);
     }
     if (is_set(action, A_LRM_EVENT)) {
         crm_trace("Action %.16llx (A_LRM_EVENT) %s", A_LRM_EVENT, text);
     }
     if (is_set(action, A_LRM_INVOKE)) {
         crm_trace("Action %.16llx (A_LRM_INVOKE) %s", A_LRM_INVOKE, text);
     }
     if (is_set(action, A_LRM_DISCONNECT)) {
         crm_trace("Action %.16llx (A_LRM_DISCONNECT) %s", A_LRM_DISCONNECT, text);
     }
     if (is_set(action, A_DC_TIMER_STOP)) {
         crm_trace("Action %.16llx (A_DC_TIMER_STOP) %s", A_DC_TIMER_STOP, text);
     }
     if (is_set(action, A_DC_TIMER_START)) {
         crm_trace("Action %.16llx (A_DC_TIMER_START) %s", A_DC_TIMER_START, text);
     }
     if (is_set(action, A_INTEGRATE_TIMER_START)) {
         crm_trace("Action %.16llx (A_INTEGRATE_TIMER_START) %s", A_INTEGRATE_TIMER_START, text);
     }
     if (is_set(action, A_INTEGRATE_TIMER_STOP)) {
         crm_trace("Action %.16llx (A_INTEGRATE_TIMER_STOP) %s", A_INTEGRATE_TIMER_STOP, text);
     }
     if (is_set(action, A_FINALIZE_TIMER_START)) {
         crm_trace("Action %.16llx (A_FINALIZE_TIMER_START) %s", A_FINALIZE_TIMER_START, text);
     }
     if (is_set(action, A_FINALIZE_TIMER_STOP)) {
         crm_trace("Action %.16llx (A_FINALIZE_TIMER_STOP) %s", A_FINALIZE_TIMER_STOP, text);
     }
     if (is_set(action, A_ELECTION_COUNT)) {
         crm_trace("Action %.16llx (A_ELECTION_COUNT) %s", A_ELECTION_COUNT, text);
     }
     if (is_set(action, A_ELECTION_VOTE)) {
         crm_trace("Action %.16llx (A_ELECTION_VOTE) %s", A_ELECTION_VOTE, text);
     }
     if (is_set(action, A_ELECTION_CHECK)) {
         crm_trace("Action %.16llx (A_ELECTION_CHECK) %s", A_ELECTION_CHECK, text);
     }
     if (is_set(action, A_CL_JOIN_ANNOUNCE)) {
         crm_trace("Action %.16llx (A_CL_JOIN_ANNOUNCE) %s", A_CL_JOIN_ANNOUNCE, text);
     }
     if (is_set(action, A_CL_JOIN_REQUEST)) {
         crm_trace("Action %.16llx (A_CL_JOIN_REQUEST) %s", A_CL_JOIN_REQUEST, text);
     }
     if (is_set(action, A_CL_JOIN_RESULT)) {
         crm_trace("Action %.16llx (A_CL_JOIN_RESULT) %s", A_CL_JOIN_RESULT, text);
     }
     if (is_set(action, A_DC_JOIN_OFFER_ALL)) {
         crm_trace("Action %.16llx (A_DC_JOIN_OFFER_ALL) %s", A_DC_JOIN_OFFER_ALL, text);
     }
     if (is_set(action, A_DC_JOIN_OFFER_ONE)) {
         crm_trace("Action %.16llx (A_DC_JOIN_OFFER_ONE) %s", A_DC_JOIN_OFFER_ONE, text);
     }
     if (is_set(action, A_DC_JOIN_PROCESS_REQ)) {
         crm_trace("Action %.16llx (A_DC_JOIN_PROCESS_REQ) %s", A_DC_JOIN_PROCESS_REQ, text);
     }
     if (is_set(action, A_DC_JOIN_PROCESS_ACK)) {
         crm_trace("Action %.16llx (A_DC_JOIN_PROCESS_ACK) %s", A_DC_JOIN_PROCESS_ACK, text);
     }
     if (is_set(action, A_DC_JOIN_FINALIZE)) {
         crm_trace("Action %.16llx (A_DC_JOIN_FINALIZE) %s", A_DC_JOIN_FINALIZE, text);
     }
     if (is_set(action, A_MSG_PROCESS)) {
         crm_trace("Action %.16llx (A_MSG_PROCESS) %s", A_MSG_PROCESS, text);
     }
     if (is_set(action, A_MSG_ROUTE)) {
         crm_trace("Action %.16llx (A_MSG_ROUTE) %s", A_MSG_ROUTE, text);
     }
     if (is_set(action, A_RECOVER)) {
         crm_trace("Action %.16llx (A_RECOVER) %s", A_RECOVER, text);
     }
     if (is_set(action, A_DC_RELEASE)) {
         crm_trace("Action %.16llx (A_DC_RELEASE) %s", A_DC_RELEASE, text);
     }
     if (is_set(action, A_DC_RELEASED)) {
         crm_trace("Action %.16llx (A_DC_RELEASED) %s", A_DC_RELEASED, text);
     }
     if (is_set(action, A_DC_TAKEOVER)) {
         crm_trace("Action %.16llx (A_DC_TAKEOVER) %s", A_DC_TAKEOVER, text);
     }
     if (is_set(action, A_SHUTDOWN)) {
         crm_trace("Action %.16llx (A_SHUTDOWN) %s", A_SHUTDOWN, text);
     }
     if (is_set(action, A_SHUTDOWN_REQ)) {
         crm_trace("Action %.16llx (A_SHUTDOWN_REQ) %s", A_SHUTDOWN_REQ, text);
     }
     if (is_set(action, A_STOP)) {
         crm_trace("Action %.16llx (A_STOP  ) %s", A_STOP, text);
     }
     if (is_set(action, A_EXIT_0)) {
         crm_trace("Action %.16llx (A_EXIT_0) %s", A_EXIT_0, text);
     }
     if (is_set(action, A_EXIT_1)) {
         crm_trace("Action %.16llx (A_EXIT_1) %s", A_EXIT_1, text);
     }
     if (is_set(action, A_CCM_CONNECT)) {
         crm_trace("Action %.16llx (A_CCM_CONNECT) %s", A_CCM_CONNECT, text);
     }
     if (is_set(action, A_CCM_DISCONNECT)) {
         crm_trace("Action %.16llx (A_CCM_DISCONNECT) %s", A_CCM_DISCONNECT, text);
     }
     if (is_set(action, A_CIB_START)) {
         crm_trace("Action %.16llx (A_CIB_START) %s", A_CIB_START, text);
     }
     if (is_set(action, A_CIB_STOP)) {
         crm_trace("Action %.16llx (A_CIB_STOP) %s", A_CIB_STOP, text);
     }
     if (is_set(action, A_TE_INVOKE)) {
         crm_trace("Action %.16llx (A_TE_INVOKE) %s", A_TE_INVOKE, text);
     }
     if (is_set(action, A_TE_START)) {
         crm_trace("Action %.16llx (A_TE_START) %s", A_TE_START, text);
     }
     if (is_set(action, A_TE_STOP)) {
         crm_trace("Action %.16llx (A_TE_STOP) %s", A_TE_STOP, text);
     }
     if (is_set(action, A_TE_CANCEL)) {
         crm_trace("Action %.16llx (A_TE_CANCEL) %s", A_TE_CANCEL, text);
     }
     if (is_set(action, A_PE_INVOKE)) {
         crm_trace("Action %.16llx (A_PE_INVOKE) %s", A_PE_INVOKE, text);
     }
     if (is_set(action, A_PE_START)) {
         crm_trace("Action %.16llx (A_PE_START) %s", A_PE_START, text);
     }
     if (is_set(action, A_PE_STOP)) {
         crm_trace("Action %.16llx (A_PE_STOP) %s", A_PE_STOP, text);
     }
     if (is_set(action, A_NODE_BLOCK)) {
         crm_trace("Action %.16llx (A_NODE_BLOCK) %s", A_NODE_BLOCK, text);
     }
     if (is_set(action, A_UPDATE_NODESTATUS)) {
         crm_trace("Action %.16llx (A_UPDATE_NODESTATUS) %s", A_UPDATE_NODESTATUS, text);
     }
     if (is_set(action, A_LOG)) {
         crm_trace("Action %.16llx (A_LOG   ) %s", A_LOG, text);
     }
     if (is_set(action, A_ERROR)) {
         crm_trace("Action %.16llx (A_ERROR ) %s", A_ERROR, text);
     }
     if (is_set(action, A_WARN)) {
         crm_trace("Action %.16llx (A_WARN  ) %s", A_WARN, text);
     }
 }
 
 gboolean
 update_dc(xmlNode * msg)
 {
     char *last_dc = fsa_our_dc;
     const char *dc_version = NULL;
     const char *welcome_from = NULL;
 
     if (msg != NULL) {
         gboolean invalid = FALSE;
 
         dc_version = crm_element_value(msg, F_CRM_VERSION);
         welcome_from = crm_element_value(msg, F_CRM_HOST_FROM);
 
         CRM_CHECK(dc_version != NULL, return FALSE);
         CRM_CHECK(welcome_from != NULL, return FALSE);
 
         if (AM_I_DC && safe_str_neq(welcome_from, fsa_our_uname)) {
             invalid = TRUE;
 
         } else if (fsa_our_dc && safe_str_neq(welcome_from, fsa_our_dc)) {
             invalid = TRUE;
         }
 
         if (invalid) {
             CRM_CHECK(fsa_our_dc != NULL, crm_err("We have no DC"));
             if (AM_I_DC) {
                 crm_err("Not updating DC to %s (%s): we are also a DC", welcome_from, dc_version);
             } else {
                 crm_warn("New DC %s is not %s", welcome_from, fsa_our_dc);
             }
 
             register_fsa_action(A_CL_JOIN_QUERY | A_DC_TIMER_START);
             return FALSE;
         }
     }
 
     free(fsa_our_dc_version);
     fsa_our_dc_version = NULL;
 
     fsa_our_dc = NULL;          /* Free'd as last_dc */
 
     if (welcome_from != NULL) {
         fsa_our_dc = strdup(welcome_from);
     }
     if (dc_version != NULL) {
         fsa_our_dc_version = strdup(dc_version);
     }
 
     if (safe_str_eq(fsa_our_dc, last_dc)) {
         /* do nothing */
 
     } else if (fsa_our_dc != NULL) {
         crm_node_t *dc_node = crm_get_peer(0, fsa_our_dc);
 
         crm_info("Set DC to %s (%s)", crm_str(fsa_our_dc), crm_str(fsa_our_dc_version));
         crm_update_peer_expected(__FUNCTION__, dc_node, CRMD_JOINSTATE_MEMBER);
 
     } else if (last_dc != NULL) {
         crm_info("Unset DC. Was %s", crm_str(last_dc));
     }
 
     free(last_dc);
     return TRUE;
 }
 
 #define STATUS_PATH_MAX 512
 static void
 erase_xpath_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
 {
     char *xpath = user_data;
 
     do_crm_log_unlikely(rc == 0 ? LOG_DEBUG : LOG_NOTICE,
                         "Deletion of \"%s\": %s (rc=%d)", xpath, pcmk_strerror(rc), rc);
 }
 
 void
 erase_status_tag(const char *uname, const char *tag, int options)
 {
     int rc = pcmk_ok;
     char xpath[STATUS_PATH_MAX];
     int cib_opts = cib_quorum_override | cib_xpath | options;
 
     if (fsa_cib_conn && uname) {
         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", uname, tag);
         crm_info("Deleting xpath: %s", xpath);
         rc = fsa_cib_conn->cmds->delete(fsa_cib_conn, xpath, NULL, cib_opts);
         fsa_register_cib_callback(rc, FALSE, strdup(xpath), erase_xpath_callback);
     }
 }
 
 crm_ipc_t *attrd_ipc = NULL;
 
 #if !HAVE_ATOMIC_ATTRD
 static int
 update_without_attrd(const char * host_uuid, const char * name, const char * value,
                      const char * user_name, gboolean is_remote_node, char command)
 {
     int call_opt = cib_none;
 
     if (fsa_cib_conn == NULL) {
         return -1;
     }
 
     call_opt = crmd_cib_smart_opt();
 
     if (command == 'C') {
         erase_status_tag(host_uuid, XML_TAG_TRANSIENT_NODEATTRS, call_opt);
         return pcmk_ok;
     }
 
     crm_trace("updating status for host_uuid %s, %s=%s", host_uuid, name ? name : "<null>", value ? value : "<null>");
     if (value) {
         return update_attr_delegate(fsa_cib_conn, call_opt, XML_CIB_TAG_STATUS, host_uuid, NULL, NULL,
                                     NULL, name, value, FALSE, user_name, is_remote_node ? "remote" : NULL);
     } else {
         return delete_attr_delegate(fsa_cib_conn, call_opt, XML_CIB_TAG_STATUS, host_uuid, NULL, NULL,
                                     NULL, name, NULL, FALSE, user_name);
     }
 }
 #endif
 
+static void
+log_attrd_error(const char *host, const char *name, const char *value,
+                gboolean is_remote, char command, int rc)
+{
+    const char *display_command; /* for commands without name/value */
+    const char *node_type = (is_remote? "Pacemaker Remote" : "cluster");
+    gboolean shutting_down = is_set(fsa_input_register, R_SHUTDOWN);
+    const char *when = (shutting_down? " at shutdown" : "");
+
+    switch (command) {
+        case 'R':
+            display_command = "refresh";
+            break;
+        case 'C':
+            display_command = "purge";
+            break;
+        default:
+            display_command = NULL;
+    }
+
+    if (display_command) {
+        crm_err("Could not request %s of %s node %s%s: %s (%d)",
+                display_command, node_type, host, when, pcmk_strerror(rc), rc);
+    } else {
+        crm_err("Could not request update of %s=%s for %s node %s%s: %s (%d)",
+                name, value, node_type, host, when, pcmk_strerror(rc), rc);
+    }
+
+    /* If we can't request shutdown via attribute, fast-track it */
+    if ((command == 'U') && shutting_down) {
+        register_fsa_input(C_FSA_INTERNAL, I_FAIL, NULL);
+    }
+}
+
 static void
 update_attrd_helper(const char *host, const char *name, const char *value, const char *user_name, gboolean is_remote_node, char command)
 {
     gboolean rc;
     int max = 5;
 
 #if !HAVE_ATOMIC_ATTRD
     /* Talk directly to cib for remote nodes if it's legacy attrd */
     if (is_remote_node) {
+        int rc;
+
         /* host is required for updating a remote node */
         CRM_CHECK(host != NULL, return;);
         /* remote node uname and uuid are equal */
-        if (update_without_attrd(host, name, value, user_name, is_remote_node, command) < pcmk_ok) {
-            crm_err("Could not update attribute %s for remote-node %s", name, host);
+        rc = update_without_attrd(host, name, value, user_name, is_remote_node,
+                                  command);
+        if (rc < pcmk_ok) {
+            log_attrd_error(host, name, value, is_remote_node, command, rc);
         }
         return;
     }
 #endif
 
     if (attrd_ipc == NULL) {
         attrd_ipc = crm_ipc_new(T_ATTRD, 0);
     }
 
     do {
         if (crm_ipc_connected(attrd_ipc) == FALSE) {
             crm_ipc_close(attrd_ipc);
             crm_info("Connecting to attribute manager ... %d retries remaining", max);
             if (crm_ipc_connect(attrd_ipc) == FALSE) {
                 crm_perror(LOG_INFO, "Connection to attribute manager failed");
             }
         }
 
         rc = attrd_update_delegate(attrd_ipc, command, host, name, value, XML_CIB_TAG_STATUS, NULL,
                                    NULL, user_name, is_remote_node?attrd_opt_remote:attrd_opt_none);
         if (rc == pcmk_ok) {
             break;
 
         } else if (rc != -EAGAIN && rc != -EALREADY) {
             crm_info("Disconnecting from attribute manager: %s (%d)", pcmk_strerror(rc), rc);
             crm_ipc_close(attrd_ipc);
         }
 
         sleep(5 - max);
 
     } while (max--);
 
     if (rc != pcmk_ok) {
-        if (name) {
-            crm_err("Could not send attrd %s update%s: %s (%d)",
-                    name, is_set(fsa_input_register, R_SHUTDOWN) ? " at shutdown" : "",
-                    pcmk_strerror(rc), rc);
-
-        } else {
-            crm_err("Could not send attrd refresh%s: %s (%d)",
-                    is_set(fsa_input_register, R_SHUTDOWN) ? " at shutdown" : "",
-                    pcmk_strerror(rc), rc);
-        }
-
-        if (is_set(fsa_input_register, R_SHUTDOWN)) {
-            register_fsa_input(C_FSA_INTERNAL, I_FAIL, NULL);
-        }
+        log_attrd_error(host, name, value, is_remote_node, command, rc);
     }
 }
 
 void
 update_attrd(const char *host, const char *name, const char *value, const char *user_name, gboolean is_remote_node)
 {
     update_attrd_helper(host, name, value, user_name, is_remote_node, 'U');
 }
 
 void
 update_attrd_remote_node_removed(const char *host, const char *user_name)
 {
-    crm_trace("telling attrd to clear attributes for remote host %s", host);
+    crm_trace("Asking attrd to purge Pacemaker Remote node %s", host);
     update_attrd_helper(host, NULL, NULL, user_name, TRUE, 'C');
 }
 
 void crmd_peer_down(crm_node_t *peer, bool full) 
 {
     if(full && peer->state == NULL) {
         crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_LOST, 0);
         crm_update_peer_proc(__FUNCTION__, peer, crm_proc_none, NULL);
     }
     crm_update_peer_join(__FUNCTION__, peer, crm_join_none);
     crm_update_peer_expected(__FUNCTION__, peer, CRMD_JOINSTATE_DOWN);
 }
diff --git a/include/crm/common/internal.h b/include/crm/common/internal.h
index c699596896..cef01851ba 100644
--- a/include/crm/common/internal.h
+++ b/include/crm/common/internal.h
@@ -1,73 +1,79 @@
 /*
  * Copyright (C) 2015
  *     Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU Lesser General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 /*!
  * \file
  * \brief   internal common utilities
  * \ingroup core
  * \note    Public APIs are declared in util.h
  */
 
 #ifndef CRM_COMMON_INTERNAL__H
 #define CRM_COMMON_INTERNAL__H
 
 #include <glib.h>       /* for gboolean */
 #include <dirent.h>     /* for struct dirent */
 #include <sys/types.h>  /* for uid_t and gid_t */
 
 /* internal I/O utilities (from io.c) */
 
 char *generate_series_filename(const char *directory, const char *series, int sequence,
                                gboolean bzip);
 int get_last_sequence(const char *directory, const char *series);
 void write_last_sequence(const char *directory, const char *series, int sequence, int max);
 int crm_chown_last_sequence(const char *directory, const char *series, uid_t uid, gid_t gid);
 
 gboolean crm_is_writable(const char *dir, const char *file, const char *user, const char *group,
                          gboolean need_both);
 
 void crm_sync_directory(const char *name);
 
 char *crm_read_contents(const char *filename);
 int crm_write_sync(int fd, const char *contents);
 
 
 /* internal procfs utilities (from procfs.c) */
 
 int crm_procfs_process_info(struct dirent *entry, char *name, int *pid);
 int crm_procfs_pid_of(const char *name);
 
 
+/* internal XML schema functions (from xml.c) */
+
+void crm_schema_init(void);
+void crm_schema_cleanup(void);
+
+
 /* internal generic string functions (from strings.c) */
 
 char *crm_concat(const char *prefix, const char *suffix, char join);
 void g_hash_destroy_str(gpointer data);
 long long crm_int_helper(const char *text, char **end_text);
 gboolean crm_ends_with(const char *s, const char *match);
 char *add_list_element(char *list, const char *value);
 bool crm_compress_string(const char *data, int length, int max, char **result,
                          unsigned int *result_len);
 
 static inline int
 crm_strlen_zero(const char *s)
 {
     return !s || *s == '\0';
 }
 
 #endif /* CRM_COMMON_INTERNAL__H */
diff --git a/include/crm/common/xml.h b/include/crm/common/xml.h
index 9e8bbb0428..3c697aace6 100644
--- a/include/crm/common/xml.h
+++ b/include/crm/common/xml.h
@@ -1,318 +1,318 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2 of the License, or (at your option) any later version.
  *
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 #ifndef CRM_COMMON_XML__H
 #  define CRM_COMMON_XML__H
 
 /**
  * \file
  * \brief Wrappers for and extensions to libxml2
  * \ingroup core
  */
 
 #  include <stdio.h>
 #  include <sys/types.h>
 #  include <unistd.h>
 
 #  include <stdlib.h>
 #  include <errno.h>
 #  include <fcntl.h>
 
 #  include <crm/crm.h>
 
 #  include <libxml/tree.h>
 #  include <libxml/xpath.h>
 
 /* Compression costs a LOT, don't do it unless we're hitting message limits
  *
  * For now, use 256k as the lower size, which means we can have 4 big data fields
  *  before we hit heartbeat's message limit
  *
  * The previous limit was 10k, compressing 184 of 1071 messages accounted for 23%
  *  of the total CPU used by the cib
  */
 #  define CRM_BZ2_BLOCKS		4
 #  define CRM_BZ2_WORK		20
 #  define CRM_BZ2_THRESHOLD	128 * 1024
 
 #  define XML_PARANOIA_CHECKS 0
 
 gboolean add_message_xml(xmlNode * msg, const char *field, xmlNode * xml);
 xmlNode *get_message_xml(xmlNode * msg, const char *field);
 GHashTable *xml2list(xmlNode * parent);
 
 void hash2nvpair(gpointer key, gpointer value, gpointer user_data);
 void hash2field(gpointer key, gpointer value, gpointer user_data);
 void hash2metafield(gpointer key, gpointer value, gpointer user_data);
 void hash2smartfield(gpointer key, gpointer value, gpointer user_data);
 
 xmlDoc *getDocPtr(xmlNode * node);
 
 /*
  * Replacement function for xmlCopyPropList which at the very least,
  * doesn't work the way *I* would expect it to.
  *
  * Copy all the attributes/properties from src into target.
  *
  * Not recursive, does not return anything.
  *
  */
 void copy_in_properties(xmlNode * target, xmlNode * src);
 void expand_plus_plus(xmlNode * target, const char *name, const char *value);
 void fix_plus_plus_recursive(xmlNode * target);
 
 /*
  * Create a node named "name" as a child of "parent"
  * If parent is NULL, creates an unconnected node.
  *
  * Returns the created node
  *
  */
 xmlNode *create_xml_node(xmlNode * parent, const char *name);
 
 /*
  * Make a copy of name and value and use the copied memory to create
  * an attribute for node.
  *
  * If node, name or value are NULL, nothing is done.
  *
  * If name or value are an empty string, nothing is done.
  *
  * Returns FALSE on failure and TRUE on success.
  *
  */
 const char *crm_xml_add(xmlNode * node, const char *name, const char *value);
 
 const char *crm_xml_replace(xmlNode * node, const char *name, const char *value);
 
 const char *crm_xml_add_int(xmlNode * node, const char *name, int value);
 
 /*!
  * \brief Add a boolean attribute to an XML object
  *
  * Add an attribute with the value XML_BOOLEAN_TRUE or XML_BOOLEAN_FALSE
  * as appropriate to an XML object.
  *
  * \param[in/out] node   XML object to add attribute to
  * \param[in]     name   Name of attribute to add
  * \param[in]     value  Boolean whose value will be tested
  *
  * \return Pointer to newly created XML attribute's content, or NULL on error
  */
 static inline const char *
 crm_xml_add_boolean(xmlNode *node, const char *name, gboolean value)
 {
     return crm_xml_add(node, name, (value? "true" : "false"));
 }
 
 /*
  * Unlink the node and set its doc pointer to NULL so free_xml()
  * will act appropriately
  */
 void unlink_xml_node(xmlNode * node);
 
 /*
  *
  */
 void purge_diff_markers(xmlNode * a_node);
 
 /*
  * Returns a deep copy of src_node
  *
  */
 xmlNode *copy_xml(xmlNode * src_node);
 
 /*
  * Add a copy of xml_node to new_parent
  */
 xmlNode *add_node_copy(xmlNode * new_parent, xmlNode * xml_node);
 
 int add_node_nocopy(xmlNode * parent, const char *name, xmlNode * child);
 
 /*
  * XML I/O Functions
  *
  * Whitespace between tags is discarded.
  */
 xmlNode *filename2xml(const char *filename);
 
 xmlNode *stdin2xml(void);
 
 xmlNode *string2xml(const char *input);
 
 int write_xml_fd(xmlNode * xml_node, const char *filename, int fd, gboolean compress);
 int write_xml_file(xmlNode * xml_node, const char *filename, gboolean compress);
 
 char *dump_xml_formatted(xmlNode * msg);
 /* Also dump the text node with xml_log_option_text enabled */ 
 char *dump_xml_formatted_with_text(xmlNode * msg);
 
 char *dump_xml_unformatted(xmlNode * msg);
 
 /*
  * Diff related Functions
  */
 xmlNode *diff_xml_object(xmlNode * left, xmlNode * right, gboolean suppress);
 
 xmlNode *subtract_xml_object(xmlNode * parent, xmlNode * left, xmlNode * right,
                              gboolean full, gboolean * changed, const char *marker);
 
 gboolean can_prune_leaf(xmlNode * xml_node);
 
 void print_xml_diff(FILE * where, xmlNode * diff);
 
 gboolean apply_xml_diff(xmlNode * old, xmlNode * diff, xmlNode ** new);
 
 /*
  * Searching & Modifying
  */
 xmlNode *find_xml_node(xmlNode * cib, const char *node_path, gboolean must_find);
 
 xmlNode *find_entity(xmlNode * parent, const char *node_name, const char *id);
 
 void xml_remove_prop(xmlNode * obj, const char *name);
 
 gboolean replace_xml_child(xmlNode * parent, xmlNode * child, xmlNode * update,
                            gboolean delete_only);
 
 gboolean update_xml_child(xmlNode * child, xmlNode * to_update);
 
 int find_xml_children(xmlNode ** children, xmlNode * root,
                       const char *tag, const char *field, const char *value,
                       gboolean search_matches);
 
 int crm_element_value_int(xmlNode * data, const char *name, int *dest);
 char *crm_element_value_copy(xmlNode * data, const char *name);
 int crm_element_value_const_int(const xmlNode * data, const char *name, int *dest);
 const char *crm_element_value_const(const xmlNode * data, const char *name);
 xmlNode *get_xpath_object(const char *xpath, xmlNode * xml_obj, int error_level);
 xmlNode *get_xpath_object_relative(const char *xpath, xmlNode * xml_obj, int error_level);
 
 static inline const char *
 crm_element_name(xmlNode *xml)
 {
     return xml? (const char *)(xml->name) : NULL;
 }
 
 const char *crm_element_value(xmlNode * data, const char *name);
 
 void xml_validate(const xmlNode * root);
 
 gboolean xml_has_children(const xmlNode * root);
 
 char *calculate_on_disk_digest(xmlNode * local_cib);
 char *calculate_operation_digest(xmlNode * local_cib, const char *version);
 char *calculate_xml_versioned_digest(xmlNode * input, gboolean sort, gboolean do_filter,
                                      const char *version);
 
+/* schema-related functions (from schemas.c) */
 gboolean validate_xml(xmlNode * xml_blob, const char *validation, gboolean to_logs);
 gboolean validate_xml_verbose(xmlNode * xml_blob);
 int update_validation(xmlNode ** xml_blob, int *best, int max, gboolean transform, gboolean to_logs);
 int get_schema_version(const char *name);
 const char *get_schema_name(int version);
+const char *xml_latest_schema(void);
+gboolean cli_config_update(xmlNode ** xml, int *best_version, gboolean to_logs);
 
 void crm_xml_init(void);
 void crm_xml_cleanup(void);
 
 static inline xmlNode *
 __xml_first_child(xmlNode * parent)
 {
     xmlNode *child = NULL;
 
     if (parent) {
         child = parent->children;
         while (child && child->type == XML_TEXT_NODE) {
             child = child->next;
         }
     }
     return child;
 }
 
 static inline xmlNode *
 __xml_next(xmlNode * child)
 {
     if (child) {
         child = child->next;
         while (child && child->type == XML_TEXT_NODE) {
             child = child->next;
         }
     }
     return child;
 }
 
 static inline xmlNode *
 __xml_next_element(xmlNode * child)
 {
     if (child) {
         child = child->next;
         while (child && child->type != XML_ELEMENT_NODE) {
             child = child->next;
         }
     }
     return child;
 }
 
 void free_xml(xmlNode * child);
 
 xmlNode *first_named_child(xmlNode * parent, const char *name);
 
 xmlNode *sorted_xml(xmlNode * input, xmlNode * parent, gboolean recursive);
 xmlXPathObjectPtr xpath_search(xmlNode * xml_top, const char *path);
 void crm_foreach_xpath_result(xmlNode *xml, const char *xpath,
                               void (*helper)(xmlNode*, void*), void *user_data);
-gboolean cli_config_update(xmlNode ** xml, int *best_version, gboolean to_logs);
 xmlNode *expand_idref(xmlNode * input, xmlNode * top);
 
 void freeXpathObject(xmlXPathObjectPtr xpathObj);
 xmlNode *getXpathResult(xmlXPathObjectPtr xpathObj, int index);
 void dedupXpathResults(xmlXPathObjectPtr xpathObj);
 
 static inline int numXpathResults(xmlXPathObjectPtr xpathObj)
 {
     if(xpathObj == NULL || xpathObj->nodesetval == NULL) {
         return 0;
     }
     return xpathObj->nodesetval->nodeNr;
 }
 
-const char *xml_latest_schema(void);
-
 bool xml_acl_enabled(xmlNode *xml);
 void xml_acl_disable(xmlNode *xml);
 bool xml_acl_denied(xmlNode *xml); /* Part or all of a change was rejected */
 bool xml_acl_filtered_copy(const char *user, xmlNode* acl_source, xmlNode *xml, xmlNode ** result);
 
 bool xml_tracking_changes(xmlNode * xml);
 bool xml_document_dirty(xmlNode *xml);
 void xml_track_changes(xmlNode * xml, const char *user, xmlNode *acl_source, bool enforce_acls);
 void xml_calculate_changes(xmlNode * old, xmlNode * new); /* For comparing two documents after the fact */
 void xml_accept_changes(xmlNode * xml);
 void xml_log_changes(uint8_t level, const char *function, xmlNode *xml);
 void xml_log_patchset(uint8_t level, const char *function, xmlNode *xml);
 bool xml_patch_versions(xmlNode *patchset, int add[3], int del[3]);
 
 xmlNode *xml_create_patchset(
     int format, xmlNode *source, xmlNode *target, bool *config, bool manage_version);
 int xml_apply_patchset(xmlNode *xml, xmlNode *patchset, bool check_version);
 
 void patchset_process_digest(xmlNode *patch, xmlNode *source, xmlNode *target, bool with_digest);
 
 void save_xml_to_file(xmlNode * xml, const char *desc, const char *filename);
 char *xml_get_path(xmlNode *xml);
 
 char * crm_xml_escape(const char *text);
 #endif
diff --git a/lib/common/Makefile.am b/lib/common/Makefile.am
index efcbb977c4..03526b7ff1 100644
--- a/lib/common/Makefile.am
+++ b/lib/common/Makefile.am
@@ -1,50 +1,50 @@
 #
 # Copyright (C) 2004 Andrew Beekhof
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 include $(top_srcdir)/Makefile.common
 
 AM_CPPFLAGS		+= -I$(top_builddir)/lib/gnu -I$(top_srcdir)/lib/gnu \
 			-DSBINDIR=\"$(sbindir)\"
 
 ## libraries
 lib_LTLIBRARIES	= libcrmcommon.la
 
 # Can't use -Wcast-qual here because glib insists on pretending things are const  
 # when they're not and thus we need the crm_element_value_const() hack
 
 # s390 needs -fPIC 
 # s390-suse-linux/bin/ld: .libs/ipc.o: relocation R_390_PC32DBL against `__stack_chk_fail@@GLIBC_2.4' can not be used when making a shared object; recompile with -fPIC
 
 CFLAGS		= $(CFLAGS_COPY:-Wcast-qual=) -fPIC
 
 libcrmcommon_la_LDFLAGS	= -version-info 9:0:6
 
 libcrmcommon_la_CFLAGS	= $(CFLAGS_HARDENED_LIB)
 libcrmcommon_la_LDFLAGS	+= $(LDFLAGS_HARDENED_LIB)
 
 libcrmcommon_la_LIBADD	= @LIBADD_DL@ $(GNUTLSLIBS) -lm
 
 libcrmcommon_la_SOURCES	= compat.c digest.c ipc.c io.c procfs.c utils.c xml.c \
 			  iso8601.c remote.c mainloop.c logging.c watchdog.c \
-			  strings.c xpath.c
+			  schemas.c strings.c xpath.c
 if BUILD_CIBSECRETS
 libcrmcommon_la_SOURCES	+= cib_secrets.c
 endif
 libcrmcommon_la_SOURCES	+= $(top_builddir)/lib/gnu/md5.c
 
 clean-generic:
 	rm -f *.log *.debug *.xml *~
diff --git a/lib/common/schemas.c b/lib/common/schemas.c
new file mode 100644
index 0000000000..a27aa92224
--- /dev/null
+++ b/lib/common/schemas.c
@@ -0,0 +1,927 @@
+/*
+ * Copyright (C) 2004-2016 Andrew Beekhof <andrew@beekhof.net>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#include <crm_internal.h>
+
+#include <stdio.h>
+#include <string.h>
+#include <dirent.h>
+#include <errno.h>
+#include <math.h>
+#include <sys/stat.h>
+
+#if HAVE_LIBXML2
+#  include <libxml/relaxng.h>
+#endif
+
+#if HAVE_LIBXSLT
+#  include <libxslt/xslt.h>
+#  include <libxslt/transform.h>
+#endif
+
+#include <crm/msg_xml.h>
+#include <crm/common/xml.h>
+
+typedef struct {
+    xmlRelaxNGPtr rng;
+    xmlRelaxNGValidCtxtPtr valid;
+    xmlRelaxNGParserCtxtPtr parser;
+} relaxng_ctx_cache_t;
+
+struct schema_s {
+    int type;
+    float version;
+    char *name;
+    char *location;
+    char *transform;
+    int after_transform;
+    void *cache;
+};
+
+static struct schema_s *known_schemas = NULL;
+static int xml_schema_max = 0;
+
+void
+xml_log(int priority, const char *fmt, ...)
+G_GNUC_PRINTF(2, 3);
+
+void
+xml_log(int priority, const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    qb_log_from_external_source_va(__FUNCTION__, __FILE__, fmt, priority,
+                                   __LINE__, 0, ap);
+    va_end(ap);
+}
+
+static int
+xml_latest_schema_index(void)
+{
+    return xml_schema_max - 4;
+}
+
+static int
+xml_minimum_schema_index(void)
+{
+    static int best = 0;
+    if (best == 0) {
+        int lpc = 0;
+        float target = 0.0;
+
+        best = xml_latest_schema_index();
+        target = floor(known_schemas[best].version);
+
+        for (lpc = best; lpc > 0; lpc--) {
+            if (known_schemas[lpc].version < target) {
+                return best;
+            } else {
+                best = lpc;
+            }
+        }
+        best = xml_latest_schema_index();
+    }
+    return best;
+}
+
+const char *
+xml_latest_schema(void)
+{
+    return get_schema_name(xml_latest_schema_index());
+}
+
+static const char *
+get_schema_root(void)
+{
+    static const char *base = NULL;
+
+    if (base == NULL) {
+        base = getenv("PCMK_schema_directory");
+    }
+    if (base == NULL || strlen(base) == 0) {
+        base = CRM_DTD_DIRECTORY;
+    }
+    return base;
+}
+
+static char *
+get_schema_path(const char *name, const char *file)
+{
+    const char *base = get_schema_root();
+
+    if (file) {
+        return crm_strdup_printf("%s/%s", base, file);
+    }
+    return crm_strdup_printf("%s/%s.rng", base, name);
+}
+
+static int
+schema_filter(const struct dirent *a)
+{
+    int rc = 0;
+    float version = 0;
+
+    if (strstr(a->d_name, "pacemaker-") != a->d_name) {
+        /* crm_trace("%s - wrong prefix", a->d_name); */
+
+    } else if (!crm_ends_with(a->d_name, ".rng")) {
+        /* crm_trace("%s - wrong suffix", a->d_name); */
+
+    } else if (sscanf(a->d_name, "pacemaker-%f.rng", &version) == 0) {
+        /* crm_trace("%s - wrong format", a->d_name); */
+
+    } else if (strcmp(a->d_name, "pacemaker-1.1.rng") == 0) {
+        /* "-1.1" was used for what later became "-next" */
+        /* crm_trace("%s - hack", a->d_name); */
+
+    } else {
+        /* crm_debug("%s - candidate", a->d_name); */
+        rc = 1;
+    }
+
+    return rc;
+}
+
+static int
+schema_sort(const struct dirent **a, const struct dirent **b)
+{
+    int rc = 0;
+    float a_version = 0.0;
+    float b_version = 0.0;
+
+    sscanf(a[0]->d_name, "pacemaker-%f.rng", &a_version);
+    sscanf(b[0]->d_name, "pacemaker-%f.rng", &b_version);
+
+    if (a_version > b_version) {
+        rc = 1;
+    } else if(a_version < b_version) {
+        rc = -1;
+    }
+
+    /* crm_trace("%s (%f) vs. %s (%f) : %d", a[0]->d_name, a_version, b[0]->d_name, b_version, rc); */
+    return rc;
+}
+
+static void
+__xml_schema_add(int type, float version, const char *name,
+                 const char *location, const char *transform,
+                 int after_transform)
+{
+    int last = xml_schema_max;
+
+    xml_schema_max++;
+    known_schemas = realloc_safe(known_schemas,
+                                 xml_schema_max * sizeof(struct schema_s));
+    CRM_ASSERT(known_schemas != NULL);
+    memset(known_schemas+last, 0, sizeof(struct schema_s));
+    known_schemas[last].type = type;
+    known_schemas[last].after_transform = after_transform;
+
+    if (version > 0.0) {
+        known_schemas[last].version = version;
+        known_schemas[last].name = crm_strdup_printf("pacemaker-%.1f", version);
+        known_schemas[last].location = crm_strdup_printf("%s.rng", known_schemas[last].name);
+
+    } else {
+        char dummy[1024];
+        CRM_ASSERT(name);
+        CRM_ASSERT(location);
+        sscanf(name, "%[^-]-%f", dummy, &version);
+        known_schemas[last].version = version;
+        known_schemas[last].name = strdup(name);
+        known_schemas[last].location = strdup(location);
+    }
+
+    if (transform) {
+        known_schemas[last].transform = strdup(transform);
+    }
+    if (after_transform == 0) {
+        after_transform = xml_schema_max;  /* upgrade is a one-way */
+    }
+    known_schemas[last].after_transform = after_transform;
+
+    if (known_schemas[last].after_transform < 0) {
+        crm_debug("Added supported schema %d: %s (%s)",
+                  last, known_schemas[last].name, known_schemas[last].location);
+
+    } else if (known_schemas[last].transform) {
+        crm_debug("Added supported schema %d: %s (%s upgrades to %d with %s)",
+                  last, known_schemas[last].name, known_schemas[last].location,
+                  known_schemas[last].after_transform,
+                  known_schemas[last].transform);
+
+    } else {
+        crm_debug("Added supported schema %d: %s (%s upgrades to %d)",
+                  last, known_schemas[last].name, known_schemas[last].location,
+                  known_schemas[last].after_transform);
+    }
+}
+
+/*!
+ * \internal
+ * \brief Load pacemaker schemas into cache
+ */
+void
+crm_schema_init(void)
+{
+    int lpc, max;
+    const char *base = get_schema_root();
+    struct dirent **namelist = NULL;
+
+    max = scandir(base, &namelist, schema_filter, schema_sort);
+    __xml_schema_add(1, 0.0, "pacemaker-0.6", "crm.dtd", "upgrade06.xsl", 3);
+    __xml_schema_add(1, 0.0, "transitional-0.6", "crm-transitional.dtd",
+                     "upgrade06.xsl", 3);
+    __xml_schema_add(2, 0.0, "pacemaker-0.7", "pacemaker-1.0.rng", NULL, 0);
+
+    if (max < 0) {
+        crm_notice("scandir(%s) failed: %s (%d)", base, strerror(errno), errno);
+
+    } else {
+        for (lpc = 0; lpc < max; lpc++) {
+            int next = 0;
+            float version = 0.0;
+            char *transform = NULL;
+
+            sscanf(namelist[lpc]->d_name, "pacemaker-%f.rng", &version);
+            if ((lpc + 1) < max) {
+                float next_version = 0.0;
+
+                sscanf(namelist[lpc+1]->d_name, "pacemaker-%f.rng",
+                       &next_version);
+
+                if (floor(version) < floor(next_version)) {
+                    struct stat s;
+                    char *xslt = NULL;
+
+                    transform = crm_strdup_printf("upgrade-%.1f.xsl", version);
+                    xslt = get_schema_path(NULL, transform);
+                    if (stat(xslt, &s) != 0) {
+                        crm_err("Transform %s not found", xslt);
+                        free(xslt);
+                        __xml_schema_add(2, version, NULL, NULL, NULL, -1);
+                        break;
+                    } else {
+                        free(xslt);
+                    }
+                }
+
+            } else {
+                next = -1;
+            }
+            __xml_schema_add(2, version, NULL, NULL, transform, next);
+            free(namelist[lpc]);
+            free(transform);
+        }
+    }
+
+    /* 1.1 was the old name for -next */
+    __xml_schema_add(2, 0.0, "pacemaker-1.1", "pacemaker-next.rng", NULL, 0);
+    __xml_schema_add(2, 0.0, "pacemaker-next", "pacemaker-next.rng", NULL, -1);
+    __xml_schema_add(0, 0.0, "none", "N/A", NULL, -1);
+    free(namelist);
+}
+
+static gboolean
+validate_with_dtd(xmlDocPtr doc, gboolean to_logs, const char *dtd_file)
+{
+    gboolean valid = TRUE;
+
+    xmlDtdPtr dtd = NULL;
+    xmlValidCtxtPtr cvp = NULL;
+
+    CRM_CHECK(doc != NULL, return FALSE);
+    CRM_CHECK(dtd_file != NULL, return FALSE);
+
+    dtd = xmlParseDTD(NULL, (const xmlChar *)dtd_file);
+    if (dtd == NULL) {
+        crm_err("Could not locate/parse DTD: %s", dtd_file);
+        return TRUE;
+    }
+
+    cvp = xmlNewValidCtxt();
+    if (cvp) {
+        if (to_logs) {
+            cvp->userData = (void *)LOG_ERR;
+            cvp->error = (xmlValidityErrorFunc) xml_log;
+            cvp->warning = (xmlValidityWarningFunc) xml_log;
+        } else {
+            cvp->userData = (void *)stderr;
+            cvp->error = (xmlValidityErrorFunc) fprintf;
+            cvp->warning = (xmlValidityWarningFunc) fprintf;
+        }
+
+        if (!xmlValidateDtd(cvp, doc, dtd)) {
+            valid = FALSE;
+        }
+        xmlFreeValidCtxt(cvp);
+
+    } else {
+        crm_err("Internal error: No valid context");
+    }
+
+    xmlFreeDtd(dtd);
+    return valid;
+}
+
+#if 0
+static void
+relaxng_invalid_stderr(void *userData, xmlErrorPtr error)
+{
+    /*
+       Structure xmlError
+       struct _xmlError {
+       int      domain  : What part of the library raised this er
+       int      code    : The error code, e.g. an xmlParserError
+       char *   message : human-readable informative error messag
+       xmlErrorLevel    level   : how consequent is the error
+       char *   file    : the filename
+       int      line    : the line number if available
+       char *   str1    : extra string information
+       char *   str2    : extra string information
+       char *   str3    : extra string information
+       int      int1    : extra number information
+       int      int2    : column number of the error or 0 if N/A
+       void *   ctxt    : the parser context if available
+       void *   node    : the node in the tree
+       }
+     */
+    crm_err("Structured error: line=%d, level=%d %s", error->line, error->level, error->message);
+}
+#endif
+
+static gboolean
+validate_with_relaxng(xmlDocPtr doc, gboolean to_logs, const char *relaxng_file,
+                      relaxng_ctx_cache_t **cached_ctx)
+{
+    int rc = 0;
+    gboolean valid = TRUE;
+    relaxng_ctx_cache_t *ctx = NULL;
+
+    CRM_CHECK(doc != NULL, return FALSE);
+    CRM_CHECK(relaxng_file != NULL, return FALSE);
+
+    if (cached_ctx && *cached_ctx) {
+        ctx = *cached_ctx;
+
+    } else {
+        crm_info("Creating RNG parser context");
+        ctx = calloc(1, sizeof(relaxng_ctx_cache_t));
+
+        xmlLoadExtDtdDefaultValue = 1;
+        ctx->parser = xmlRelaxNGNewParserCtxt(relaxng_file);
+        CRM_CHECK(ctx->parser != NULL, goto cleanup);
+
+        if (to_logs) {
+            xmlRelaxNGSetParserErrors(ctx->parser,
+                                      (xmlRelaxNGValidityErrorFunc) xml_log,
+                                      (xmlRelaxNGValidityWarningFunc) xml_log,
+                                      GUINT_TO_POINTER(LOG_ERR));
+        } else {
+            xmlRelaxNGSetParserErrors(ctx->parser,
+                                      (xmlRelaxNGValidityErrorFunc) fprintf,
+                                      (xmlRelaxNGValidityWarningFunc) fprintf,
+                                      stderr);
+        }
+
+        ctx->rng = xmlRelaxNGParse(ctx->parser);
+        CRM_CHECK(ctx->rng != NULL,
+                  crm_err("Could not find/parse %s", relaxng_file);
+                  goto cleanup);
+
+        ctx->valid = xmlRelaxNGNewValidCtxt(ctx->rng);
+        CRM_CHECK(ctx->valid != NULL, goto cleanup);
+
+        if (to_logs) {
+            xmlRelaxNGSetValidErrors(ctx->valid,
+                                     (xmlRelaxNGValidityErrorFunc) xml_log,
+                                     (xmlRelaxNGValidityWarningFunc) xml_log,
+                                     GUINT_TO_POINTER(LOG_ERR));
+        } else {
+            xmlRelaxNGSetValidErrors(ctx->valid,
+                                     (xmlRelaxNGValidityErrorFunc) fprintf,
+                                     (xmlRelaxNGValidityWarningFunc) fprintf,
+                                     stderr);
+        }
+    }
+
+    /* xmlRelaxNGSetValidStructuredErrors( */
+    /*  valid, relaxng_invalid_stderr, valid); */
+
+    xmlLineNumbersDefault(1);
+    rc = xmlRelaxNGValidateDoc(ctx->valid, doc);
+    if (rc > 0) {
+        valid = FALSE;
+
+    } else if (rc < 0) {
+        crm_err("Internal libxml error during validation\n");
+    }
+
+  cleanup:
+
+    if (cached_ctx) {
+        *cached_ctx = ctx;
+
+    } else {
+        if (ctx->parser != NULL) {
+            xmlRelaxNGFreeParserCtxt(ctx->parser);
+        }
+        if (ctx->valid != NULL) {
+            xmlRelaxNGFreeValidCtxt(ctx->valid);
+        }
+        if (ctx->rng != NULL) {
+            xmlRelaxNGFree(ctx->rng);
+        }
+        free(ctx);
+    }
+
+    return valid;
+}
+
+/*!
+ * \internal
+ * \brief Clean up global memory associated with XML schemas
+ */
+void
+crm_schema_cleanup(void)
+{
+    int lpc;
+    relaxng_ctx_cache_t *ctx = NULL;
+
+    for (lpc = 0; lpc < xml_schema_max; lpc++) {
+
+        switch (known_schemas[lpc].type) {
+            case 0:
+                /* None */
+                break;
+            case 1:
+                /* DTD - Not cached */
+                break;
+            case 2:
+                /* RNG - Cached */
+                ctx = (relaxng_ctx_cache_t *) known_schemas[lpc].cache;
+                if (ctx == NULL) {
+                    break;
+                }
+                if (ctx->parser != NULL) {
+                    xmlRelaxNGFreeParserCtxt(ctx->parser);
+                }
+                if (ctx->valid != NULL) {
+                    xmlRelaxNGFreeValidCtxt(ctx->valid);
+                }
+                if (ctx->rng != NULL) {
+                    xmlRelaxNGFree(ctx->rng);
+                }
+                free(ctx);
+                known_schemas[lpc].cache = NULL;
+                break;
+            default:
+                break;
+        }
+        free(known_schemas[lpc].name);
+        free(known_schemas[lpc].location);
+        free(known_schemas[lpc].transform);
+    }
+    free(known_schemas);
+    known_schemas = NULL;
+}
+
+static gboolean
+validate_with(xmlNode *xml, int method, gboolean to_logs)
+{
+    xmlDocPtr doc = NULL;
+    gboolean valid = FALSE;
+    int type = 0;
+    char *file = NULL;
+
+    if (method < 0) {
+        return FALSE;
+    }
+
+    type = known_schemas[method].type;
+    if(type == 0) {
+        return TRUE;
+    }
+
+    CRM_CHECK(xml != NULL, return FALSE);
+    doc = getDocPtr(xml);
+    file = get_schema_path(known_schemas[method].name,
+                           known_schemas[method].location);
+
+    crm_trace("Validating with: %s (type=%d)", crm_str(file), type);
+    switch (type) {
+        case 1:
+            valid = validate_with_dtd(doc, to_logs, file);
+            break;
+        case 2:
+            valid =
+                validate_with_relaxng(doc, to_logs, file,
+                                      (relaxng_ctx_cache_t **) & (known_schemas[method].cache));
+            break;
+        default:
+            crm_err("Unknown validator type: %d", type);
+            break;
+    }
+
+    free(file);
+    return valid;
+}
+
+static void
+dump_file(const char *filename)
+{
+
+    FILE *fp = NULL;
+    int ch, line = 0;
+
+    CRM_CHECK(filename != NULL, return);
+
+    fp = fopen(filename, "r");
+    CRM_CHECK(fp != NULL, return);
+
+    fprintf(stderr, "%4d ", ++line);
+    do {
+        ch = getc(fp);
+        if (ch == EOF) {
+            putc('\n', stderr);
+            break;
+        } else if (ch == '\n') {
+            fprintf(stderr, "\n%4d ", ++line);
+        } else {
+            putc(ch, stderr);
+        }
+    } while (1);
+
+    fclose(fp);
+}
+
+gboolean
+validate_xml_verbose(xmlNode *xml_blob)
+{
+    int fd = 0;
+    xmlDoc *doc = NULL;
+    xmlNode *xml = NULL;
+    gboolean rc = FALSE;
+    char *filename = strdup(CRM_STATE_DIR "/cib-invalid.XXXXXX");
+
+    umask(S_IWGRP | S_IWOTH | S_IROTH);
+    fd = mkstemp(filename);
+    write_xml_fd(xml_blob, filename, fd, FALSE);
+
+    dump_file(filename);
+
+    doc = xmlParseFile(filename);
+    xml = xmlDocGetRootElement(doc);
+    rc = validate_xml(xml, NULL, FALSE);
+    free_xml(xml);
+
+    unlink(filename);
+    free(filename);
+
+    return rc;
+}
+
+gboolean
+validate_xml(xmlNode *xml_blob, const char *validation, gboolean to_logs)
+{
+    int version = 0;
+
+    if (validation == NULL) {
+        validation = crm_element_value(xml_blob, XML_ATTR_VALIDATION);
+    }
+
+    if (validation == NULL) {
+        int lpc = 0;
+        bool valid = FALSE;
+
+        validation = crm_element_value(xml_blob, "ignore-dtd");
+        if (crm_is_true(validation)) {
+            /* Legacy compatibilty */
+            crm_xml_add(xml_blob, XML_ATTR_VALIDATION, "none");
+            return TRUE;
+        }
+
+        /* Work it out */
+        for (lpc = 0; lpc < xml_schema_max; lpc++) {
+            if (validate_with(xml_blob, lpc, FALSE)) {
+                valid = TRUE;
+                crm_xml_add(xml_blob, XML_ATTR_VALIDATION,
+                            known_schemas[lpc].name);
+                crm_info("XML validated against %s", known_schemas[lpc].name);
+                if(known_schemas[lpc].after_transform == 0) {
+                    break;
+                }
+            }
+        }
+
+        return valid;
+    }
+
+    version = get_schema_version(validation);
+    if (strcmp(validation, "none") == 0) {
+        return TRUE;
+    } else if (version < xml_schema_max) {
+        return validate_with(xml_blob, version, to_logs);
+    }
+
+    crm_err("Unknown validator: %s", validation);
+    return FALSE;
+}
+
+#if HAVE_LIBXSLT
+static xmlNode *
+apply_transformation(xmlNode *xml, const char *transform)
+{
+    char *xform = NULL;
+    xmlNode *out = NULL;
+    xmlDocPtr res = NULL;
+    xmlDocPtr doc = NULL;
+    xsltStylesheet *xslt = NULL;
+
+    CRM_CHECK(xml != NULL, return FALSE);
+    doc = getDocPtr(xml);
+    xform = get_schema_path(NULL, transform);
+
+    xmlLoadExtDtdDefaultValue = 1;
+    xmlSubstituteEntitiesDefault(1);
+
+    xslt = xsltParseStylesheetFile((const xmlChar *)xform);
+    CRM_CHECK(xslt != NULL, goto cleanup);
+
+    res = xsltApplyStylesheet(xslt, doc, NULL);
+    CRM_CHECK(res != NULL, goto cleanup);
+
+    out = xmlDocGetRootElement(res);
+
+  cleanup:
+    if (xslt) {
+        xsltFreeStylesheet(xslt);
+    }
+
+    free(xform);
+
+    return out;
+}
+#endif
+
+const char *
+get_schema_name(int version)
+{
+    if (version < 0 || version >= xml_schema_max) {
+        return "unknown";
+    }
+    return known_schemas[version].name;
+}
+
+int
+get_schema_version(const char *name)
+{
+    int lpc = 0;
+
+    if (name == NULL) {
+        name = "none";
+    }
+    for (; lpc < xml_schema_max; lpc++) {
+        if (safe_str_eq(name, known_schemas[lpc].name)) {
+            return lpc;
+        }
+    }
+    return -1;
+}
+
+/* set which validation to use */
+int
+update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform,
+                  gboolean to_logs)
+{
+    xmlNode *xml = NULL;
+    char *value = NULL;
+    int max_stable_schemas = xml_latest_schema_index();
+    int lpc = 0, match = -1, rc = pcmk_ok;
+    int next = -1;  /* -1 denotes "inactive" value */
+
+    CRM_CHECK(best != NULL, return -EINVAL);
+    *best = 0;
+
+    CRM_CHECK(xml_blob != NULL, return -EINVAL);
+    CRM_CHECK(*xml_blob != NULL, return -EINVAL);
+
+    xml = *xml_blob;
+    value = crm_element_value_copy(xml, XML_ATTR_VALIDATION);
+
+    if (value != NULL) {
+        match = get_schema_version(value);
+
+        lpc = match;
+        if (lpc >= 0 && transform == FALSE) {
+            lpc++;
+
+        } else if (lpc < 0) {
+            crm_debug("Unknown validation type");
+            lpc = 0;
+        }
+    }
+
+    if (match >= max_stable_schemas) {
+        /* nothing to do */
+        free(value);
+        *best = match;
+        return pcmk_ok;
+    }
+
+    while (lpc <= max_stable_schemas) {
+        crm_debug("Testing '%s' validation (%d of %d)",
+                  known_schemas[lpc].name ? known_schemas[lpc].name : "<unset>",
+                  lpc, max_stable_schemas);
+
+        if (validate_with(xml, lpc, to_logs) == FALSE) {
+            if (next != -1) {
+                crm_info("Configuration not valid for schema: %s",
+                         known_schemas[lpc].name);
+                next = -1;
+            } else {
+                crm_trace("%s validation failed",
+                          known_schemas[lpc].name ? known_schemas[lpc].name : "<unset>");
+            }
+            if (*best) {
+                /* we've satisfied the validation, no need to check further */
+                break;
+            }
+            rc = -pcmk_err_schema_validation;
+
+        } else {
+            if (next != -1) {
+                crm_debug("Configuration valid for schema: %s",
+                          known_schemas[next].name);
+                next = -1;
+            }
+            rc = pcmk_ok;
+        }
+
+        if (rc == pcmk_ok) {
+            *best = lpc;
+        }
+
+        if (rc == pcmk_ok && transform) {
+            xmlNode *upgrade = NULL;
+            next = known_schemas[lpc].after_transform;
+
+            if (next <= lpc) {
+                /* There is no next version, or next would regress */
+                crm_trace("Stopping at %s", known_schemas[lpc].name);
+                break;
+
+            } else if (max > 0 && (lpc == max || next > max)) {
+                crm_trace("Upgrade limit reached at %s (lpc=%d, next=%d, max=%d)",
+                          known_schemas[lpc].name, lpc, next, max);
+                break;
+
+            } else if (known_schemas[lpc].transform == NULL) {
+                crm_debug("%s-style configuration is also valid for %s",
+                           known_schemas[lpc].name, known_schemas[next].name);
+
+                lpc = next;
+
+            } else {
+                crm_debug("Upgrading %s-style configuration to %s with %s",
+                           known_schemas[lpc].name, known_schemas[next].name,
+                           known_schemas[lpc].transform ? known_schemas[lpc].transform : "no-op");
+
+#if HAVE_LIBXSLT
+                upgrade = apply_transformation(xml, known_schemas[lpc].transform);
+#endif
+                if (upgrade == NULL) {
+                    crm_err("Transformation %s failed",
+                            known_schemas[lpc].transform);
+                    rc = -pcmk_err_transform_failed;
+
+                } else if (validate_with(upgrade, next, to_logs)) {
+                    crm_info("Transformation %s successful",
+                             known_schemas[lpc].transform);
+                    lpc = next;
+                    *best = next;
+                    free_xml(xml);
+                    xml = upgrade;
+                    rc = pcmk_ok;
+
+                } else {
+                    crm_err("Transformation %s did not produce a valid configuration",
+                            known_schemas[lpc].transform);
+                    crm_log_xml_info(upgrade, "transform:bad");
+                    free_xml(upgrade);
+                    rc = -pcmk_err_schema_validation;
+                }
+                next = -1;
+            }
+        }
+
+        if (transform == FALSE || rc != pcmk_ok) {
+            /* we need some progress! */
+            lpc++;
+        }
+    }
+
+    if (*best > match) {
+        crm_info("%s the configuration from %s to %s",
+                   transform?"Transformed":"Upgraded",
+                   value ? value : "<none>", known_schemas[*best].name);
+        crm_xml_add(xml, XML_ATTR_VALIDATION, known_schemas[*best].name);
+    }
+
+    *xml_blob = xml;
+    free(value);
+    return rc;
+}
+
+gboolean
+cli_config_update(xmlNode **xml, int *best_version, gboolean to_logs)
+{
+    gboolean rc = TRUE;
+    const char *value = crm_element_value(*xml, XML_ATTR_VALIDATION);
+
+    int version = get_schema_version(value);
+    int orig_version = version;
+    int min_version = xml_minimum_schema_index();
+
+    if (version < min_version) {
+        xmlNode *converted = NULL;
+
+        converted = copy_xml(*xml);
+        update_validation(&converted, &version, 0, TRUE, to_logs);
+
+        value = crm_element_value(converted, XML_ATTR_VALIDATION);
+        if (version < min_version) {
+            if (version < orig_version) {
+                if (to_logs) {
+                    crm_config_err("Your current configuration could not validate"
+                                   " with any schema in range [%s, %s]\n",
+                                   get_schema_name(orig_version),
+                                   xml_latest_schema());
+                } else {
+                    fprintf(stderr, "Your current configuration could not validate"
+                                    " with any schema in range [%s, %s]\n",
+                                    get_schema_name(orig_version),
+                                    xml_latest_schema());
+                }
+            } else if (to_logs) {
+                crm_config_err("Your current configuration could only be upgraded to %s... "
+                               "the minimum requirement is %s.\n", crm_str(value),
+                               get_schema_name(min_version));
+            } else {
+                fprintf(stderr, "Your current configuration could only be upgraded to %s... "
+                        "the minimum requirement is %s.\n",
+                        crm_str(value), get_schema_name(min_version));
+            }
+
+            free_xml(converted);
+            converted = NULL;
+            rc = FALSE;
+
+        } else {
+            free_xml(*xml);
+            *xml = converted;
+
+            if (version < xml_latest_schema_index()) {
+                crm_config_warn("Your configuration was internally updated to %s... "
+                                "which is acceptable but not the most recent",
+                                get_schema_name(version));
+
+            } else if (to_logs) {
+                crm_info("Your configuration was internally updated to the latest version (%s)",
+                         get_schema_name(version));
+            }
+        }
+
+    } else if (version >= get_schema_version("none")) {
+        if (to_logs) {
+            crm_config_warn("Configuration validation is currently disabled."
+                            " It is highly encouraged and prevents many common cluster issues.");
+
+        } else {
+            fprintf(stderr, "Configuration validation is currently disabled."
+                    " It is highly encouraged and prevents many common cluster issues.\n");
+        }
+    }
+
+    if (best_version) {
+        *best_version = version;
+    }
+
+    return rc;
+}
diff --git a/lib/common/utils.c b/lib/common/utils.c
index 685ad545a3..78c6195748 100644
--- a/lib/common/utils.c
+++ b/lib/common/utils.c
@@ -1,2147 +1,2154 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
 #include <dlfcn.h>
 
 #ifndef _GNU_SOURCE
 #  define _GNU_SOURCE
 #endif
 
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <sys/stat.h>
 #include <sys/utsname.h>
 
 #include <stdio.h>
 #include <unistd.h>
 #include <string.h>
 #include <stdlib.h>
 #include <limits.h>
 #include <ctype.h>
 #include <pwd.h>
 #include <time.h>
 #include <libgen.h>
 #include <signal.h>
 
 #include <qb/qbdefs.h>
 
 #include <crm/crm.h>
 #include <crm/lrmd.h>
 #include <crm/services.h>
 #include <crm/msg_xml.h>
 #include <crm/cib/internal.h>
 #include <crm/common/xml.h>
 #include <crm/common/util.h>
 #include <crm/common/ipc.h>
 #include <crm/common/iso8601.h>
 #include <crm/common/mainloop.h>
 #include <crm/attrd.h>
 #include <libxml2/libxml/relaxng.h>
 
 #ifndef MAXLINE
 #  define MAXLINE 512
 #endif
 
 #ifdef HAVE_GETOPT_H
 #  include <getopt.h>
 #endif
 
 #ifndef PW_BUFFER_LEN
 #  define PW_BUFFER_LEN		500
 #endif
 
 CRM_TRACE_INIT_DATA(common);
 
 gboolean crm_config_error = FALSE;
 gboolean crm_config_warning = FALSE;
 char *crm_system_name = NULL;
 
 int node_score_red = 0;
 int node_score_green = 0;
 int node_score_yellow = 0;
 int node_score_infinity = INFINITY;
 
 static struct crm_option *crm_long_options = NULL;
 static const char *crm_app_description = NULL;
 static char *crm_short_options = NULL;
 static const char *crm_app_usage = NULL;
 
 int
 crm_exit(int rc)
 {
     mainloop_cleanup();
 
 #if HAVE_LIBXML2
     crm_trace("cleaning up libxml");
     crm_xml_cleanup();
 #endif
 
     crm_trace("exit %d", rc);
     qb_log_fini();
 
     free(crm_short_options);
     free(crm_system_name);
 
     exit(ABS(rc)); /* Always exit with a positive value so that it can be passed to crm_error
                     *
                     * Otherwise the system wraps it around and people
                     * have to jump through hoops figuring out what the
                     * error was
                     */
     return rc;     /* Can never happen, but allows return crm_exit(rc)
                     * where "return rc" was used previously - which
                     * keeps compilers happy.
                     */
 }
 
 gboolean
 check_time(const char *value)
 {
     if (crm_get_msec(value) < 5000) {
         return FALSE;
     }
     return TRUE;
 }
 
 gboolean
 check_timer(const char *value)
 {
     if (crm_get_msec(value) < 0) {
         return FALSE;
     }
     return TRUE;
 }
 
 gboolean
 check_sbd_timeout(const char *value)
 {
     const char *env_value = getenv("SBD_WATCHDOG_TIMEOUT");
 
     long sbd_timeout = crm_get_msec(env_value);
     long st_timeout = crm_get_msec(value);
 
     if(value == NULL || st_timeout <= 0) {
         crm_notice("Watchdog may be enabled but stonith-watchdog-timeout is disabled: %s", value);
 
     } else if(pcmk_locate_sbd() == 0) {
         do_crm_log_always(LOG_EMERG, "Shutting down: stonith-watchdog-timeout is configured (%ldms) but SBD is not active", st_timeout);
         crm_exit(DAEMON_RESPAWN_STOP);
         return FALSE;
 
     } else if(st_timeout < sbd_timeout) {
         do_crm_log_always(LOG_EMERG, "Shutting down: stonith-watchdog-timeout (%ldms) is too short (must be greater than %ldms)",
                           st_timeout, sbd_timeout);
         crm_exit(DAEMON_RESPAWN_STOP);
         return FALSE;
     }
 
     crm_info("Watchdog functionality is consistent: %s delay exceeds timeout of %s", value, env_value);
     return TRUE;
 }
 
 
 gboolean
 check_boolean(const char *value)
 {
     int tmp = FALSE;
 
     if (crm_str_to_boolean(value, &tmp) != 1) {
         return FALSE;
     }
     return TRUE;
 }
 
 gboolean
 check_number(const char *value)
 {
     errno = 0;
     if (value == NULL) {
         return FALSE;
 
     } else if (safe_str_eq(value, MINUS_INFINITY_S)) {
 
     } else if (safe_str_eq(value, INFINITY_S)) {
 
     } else {
         crm_int_helper(value, NULL);
     }
 
     if (errno != 0) {
         return FALSE;
     }
     return TRUE;
 }
 
 gboolean
 check_quorum(const char *value)
 {
     if (safe_str_eq(value, "stop")) {
         return TRUE;
 
     } else if (safe_str_eq(value, "freeze")) {
         return TRUE;
 
     } else if (safe_str_eq(value, "ignore")) {
         return TRUE;
 
     } else if (safe_str_eq(value, "suicide")) {
         return TRUE;
     }
     return FALSE;
 }
 
 gboolean
 check_script(const char *value)
 {
     struct stat st;
 
     if(safe_str_eq(value, "/dev/null")) {
         return TRUE;
     }
 
     if(stat(value, &st) != 0) {
         crm_err("Script %s does not exist", value);
         return FALSE;
     }
 
     if(S_ISREG(st.st_mode) == 0) {
         crm_err("Script %s is not a regular file", value);
         return FALSE;
     }
 
     if( (st.st_mode & (S_IXUSR | S_IXGRP )) == 0) {
         crm_err("Script %s is not executable", value);
         return FALSE;
     }
 
     return TRUE;
 }
 
 gboolean
 check_utilization(const char *value)
 {
     char *end = NULL;
     long number = strtol(value, &end, 10);
 
     if(end && end[0] != '%') {
         return FALSE;
     } else if(number < 0) {
         return FALSE;
     }
 
     return TRUE;
 }
 
 int
 char2score(const char *score)
 {
     int score_f = 0;
 
     if (score == NULL) {
 
     } else if (safe_str_eq(score, MINUS_INFINITY_S)) {
         score_f = -node_score_infinity;
 
     } else if (safe_str_eq(score, INFINITY_S)) {
         score_f = node_score_infinity;
 
     } else if (safe_str_eq(score, "+" INFINITY_S)) {
         score_f = node_score_infinity;
 
     } else if (safe_str_eq(score, "red")) {
         score_f = node_score_red;
 
     } else if (safe_str_eq(score, "yellow")) {
         score_f = node_score_yellow;
 
     } else if (safe_str_eq(score, "green")) {
         score_f = node_score_green;
 
     } else {
         score_f = crm_parse_int(score, NULL);
         if (score_f > 0 && score_f > node_score_infinity) {
             score_f = node_score_infinity;
 
         } else if (score_f < 0 && score_f < -node_score_infinity) {
             score_f = -node_score_infinity;
         }
     }
 
     return score_f;
 }
 
 char *
 score2char_stack(int score, char *buf, size_t len)
 {
     if (score >= node_score_infinity) {
         strncpy(buf, INFINITY_S, 9);
     } else if (score <= -node_score_infinity) {
         strncpy(buf, MINUS_INFINITY_S , 10);
     } else {
         return crm_itoa_stack(score, buf, len);
     }
 
     return buf;
 }
 
 char *
 score2char(int score)
 {
     if (score >= node_score_infinity) {
         return strdup(INFINITY_S);
 
     } else if (score <= -node_score_infinity) {
         return strdup("-" INFINITY_S);
     }
     return crm_itoa(score);
 }
 
 const char *
 cluster_option(GHashTable * options, gboolean(*validate) (const char *),
                const char *name, const char *old_name, const char *def_value)
 {
     const char *value = NULL;
 
     CRM_ASSERT(name != NULL);
 
     if (options != NULL) {
         value = g_hash_table_lookup(options, name);
     }
 
     if (value == NULL && old_name && options != NULL) {
         value = g_hash_table_lookup(options, old_name);
         if (value != NULL) {
             crm_config_warn("Using deprecated name '%s' for"
                             " cluster option '%s'", old_name, name);
             g_hash_table_insert(options, strdup(name), strdup(value));
             value = g_hash_table_lookup(options, old_name);
         }
     }
 
     if (value == NULL) {
         crm_trace("Using default value '%s' for cluster option '%s'", def_value, name);
 
         if (options == NULL) {
             return def_value;
 
         } else if(def_value == NULL) {
             return def_value;
         }
 
         g_hash_table_insert(options, strdup(name), strdup(def_value));
         value = g_hash_table_lookup(options, name);
     }
 
     if (validate && validate(value) == FALSE) {
         crm_config_err("Value '%s' for cluster option '%s' is invalid."
                        "  Defaulting to %s", value, name, def_value);
         g_hash_table_replace(options, strdup(name), strdup(def_value));
         value = g_hash_table_lookup(options, name);
     }
 
     return value;
 }
 
 const char *
 get_cluster_pref(GHashTable * options, pe_cluster_option * option_list, int len, const char *name)
 {
     int lpc = 0;
     const char *value = NULL;
     gboolean found = FALSE;
 
     for (lpc = 0; lpc < len; lpc++) {
         if (safe_str_eq(name, option_list[lpc].name)) {
             found = TRUE;
             value = cluster_option(options,
                                    option_list[lpc].is_valid,
                                    option_list[lpc].name,
                                    option_list[lpc].alt_name, option_list[lpc].default_value);
         }
     }
     CRM_CHECK(found, crm_err("No option named: %s", name));
     return value;
 }
 
 void
 config_metadata(const char *name, const char *version, const char *desc_short,
                 const char *desc_long, pe_cluster_option * option_list, int len)
 {
     int lpc = 0;
 
     fprintf(stdout, "<?xml version=\"1.0\"?>"
             "<!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n"
             "<resource-agent name=\"%s\">\n"
             "  <version>%s</version>\n"
             "  <longdesc lang=\"en\">%s</longdesc>\n"
             "  <shortdesc lang=\"en\">%s</shortdesc>\n"
             "  <parameters>\n", name, version, desc_long, desc_short);
 
     for (lpc = 0; lpc < len; lpc++) {
         if (option_list[lpc].description_long == NULL && option_list[lpc].description_short == NULL) {
             continue;
         }
         fprintf(stdout, "    <parameter name=\"%s\" unique=\"0\">\n"
                 "      <shortdesc lang=\"en\">%s</shortdesc>\n"
                 "      <content type=\"%s\" default=\"%s\"/>\n"
                 "      <longdesc lang=\"en\">%s%s%s</longdesc>\n"
                 "    </parameter>\n",
                 option_list[lpc].name,
                 option_list[lpc].description_short,
                 option_list[lpc].type,
                 option_list[lpc].default_value,
                 option_list[lpc].description_long ? option_list[lpc].
                 description_long : option_list[lpc].description_short,
                 option_list[lpc].values ? "  Allowed values: " : "",
                 option_list[lpc].values ? option_list[lpc].values : "");
     }
     fprintf(stdout, "  </parameters>\n</resource-agent>\n");
 }
 
 void
 verify_all_options(GHashTable * options, pe_cluster_option * option_list, int len)
 {
     int lpc = 0;
 
     for (lpc = 0; lpc < len; lpc++) {
         cluster_option(options,
                        option_list[lpc].is_valid,
                        option_list[lpc].name,
                        option_list[lpc].alt_name, option_list[lpc].default_value);
     }
 }
 
 char *
 generate_hash_key(const char *crm_msg_reference, const char *sys)
 {
     char *hash_key = crm_concat(sys ? sys : "none", crm_msg_reference, '_');
 
     crm_trace("created hash key: (%s)", hash_key);
     return hash_key;
 }
 
 
 int
 crm_user_lookup(const char *name, uid_t * uid, gid_t * gid)
 {
     int rc = -1;
     char *buffer = NULL;
     struct passwd pwd;
     struct passwd *pwentry = NULL;
 
     buffer = calloc(1, PW_BUFFER_LEN);
     getpwnam_r(name, &pwd, buffer, PW_BUFFER_LEN, &pwentry);
     if (pwentry) {
         rc = 0;
         if (uid) {
             *uid = pwentry->pw_uid;
         }
         if (gid) {
             *gid = pwentry->pw_gid;
         }
         crm_trace("Cluster user %s has uid=%d gid=%d", name, pwentry->pw_uid, pwentry->pw_gid);
 
     } else {
         crm_err("Cluster user %s does not exist", name);
     }
 
     free(buffer);
     return rc;
 }
 
 static int
 crm_version_helper(const char *text, char **end_text)
 {
     int atoi_result = -1;
 
     CRM_ASSERT(end_text != NULL);
 
     errno = 0;
 
     if (text != NULL && text[0] != 0) {
         atoi_result = (int)strtol(text, end_text, 10);
 
         if (errno == EINVAL) {
             crm_err("Conversion of '%s' %c failed", text, text[0]);
             atoi_result = -1;
         }
     }
     return atoi_result;
 }
 
 /*
  * version1 < version2 : -1
  * version1 = version2 :  0
  * version1 > version2 :  1
  */
 int
 compare_version(const char *version1, const char *version2)
 {
     int rc = 0;
     int lpc = 0;
     char *ver1_copy = NULL, *ver2_copy = NULL;
     char *rest1 = NULL, *rest2 = NULL;
 
     if (version1 == NULL && version2 == NULL) {
         return 0;
     } else if (version1 == NULL) {
         return -1;
     } else if (version2 == NULL) {
         return 1;
     }
 
     ver1_copy = strdup(version1);
     ver2_copy = strdup(version2);
     rest1 = ver1_copy;
     rest2 = ver2_copy;
 
     while (1) {
         int digit1 = 0;
         int digit2 = 0;
 
         lpc++;
 
         if (rest1 == rest2) {
             break;
         }
 
         if (rest1 != NULL) {
             digit1 = crm_version_helper(rest1, &rest1);
         }
 
         if (rest2 != NULL) {
             digit2 = crm_version_helper(rest2, &rest2);
         }
 
         if (digit1 < digit2) {
             rc = -1;
             break;
 
         } else if (digit1 > digit2) {
             rc = 1;
             break;
         }
 
         if (rest1 != NULL && rest1[0] == '.') {
             rest1++;
         }
         if (rest1 != NULL && rest1[0] == 0) {
             rest1 = NULL;
         }
 
         if (rest2 != NULL && rest2[0] == '.') {
             rest2++;
         }
         if (rest2 != NULL && rest2[0] == 0) {
             rest2 = NULL;
         }
     }
 
     free(ver1_copy);
     free(ver2_copy);
 
     if (rc == 0) {
         crm_trace("%s == %s (%d)", version1, version2, lpc);
     } else if (rc < 0) {
         crm_trace("%s < %s (%d)", version1, version2, lpc);
     } else if (rc > 0) {
         crm_trace("%s > %s (%d)", version1, version2, lpc);
     }
 
     return rc;
 }
 
 gboolean do_stderr = FALSE;
 
 #ifndef NUMCHARS
 #  define	NUMCHARS	"0123456789."
 #endif
 
 #ifndef WHITESPACE
 #  define	WHITESPACE	" \t\n\r\f"
 #endif
 
 unsigned long long
 crm_get_interval(const char *input)
 {
     unsigned long long msec = 0;
 
     if (input == NULL) {
         return msec;
 
     } else if (input[0] != 'P') {
         long long tmp = crm_get_msec(input);
 
         if(tmp > 0) {
             msec = tmp;
         }
 
     } else {
         crm_time_t *interval = crm_time_parse_duration(input);
 
         msec = 1000 * crm_time_get_seconds(interval);
         crm_time_free(interval);
     }
 
     return msec;
 }
 
 long long
 crm_get_msec(const char *input)
 {
     const char *cp = input;
     const char *units;
     long long multiplier = 1000;
     long long divisor = 1;
     long long msec = -1;
     char *end_text = NULL;
 
     /* double dret; */
 
     if (input == NULL) {
         return msec;
     }
 
     cp += strspn(cp, WHITESPACE);
     units = cp + strspn(cp, NUMCHARS);
     units += strspn(units, WHITESPACE);
 
     if (strchr(NUMCHARS, *cp) == NULL) {
         return msec;
     }
 
     if (strncasecmp(units, "ms", 2) == 0 || strncasecmp(units, "msec", 4) == 0) {
         multiplier = 1;
         divisor = 1;
     } else if (strncasecmp(units, "us", 2) == 0 || strncasecmp(units, "usec", 4) == 0) {
         multiplier = 1;
         divisor = 1000;
     } else if (strncasecmp(units, "s", 1) == 0 || strncasecmp(units, "sec", 3) == 0) {
         multiplier = 1000;
         divisor = 1;
     } else if (strncasecmp(units, "m", 1) == 0 || strncasecmp(units, "min", 3) == 0) {
         multiplier = 60 * 1000;
         divisor = 1;
     } else if (strncasecmp(units, "h", 1) == 0 || strncasecmp(units, "hr", 2) == 0) {
         multiplier = 60 * 60 * 1000;
         divisor = 1;
     } else if (*units != EOS && *units != '\n' && *units != '\r') {
         return msec;
     }
 
     msec = crm_int_helper(cp, &end_text);
     if (msec > LLONG_MAX/multiplier) {
         /* arithmetics overflow while multiplier/divisor mutually exclusive */
         return LLONG_MAX;
     }
     msec *= multiplier;
     msec /= divisor;
     /* dret += 0.5; */
     /* msec = (long long)dret; */
     return msec;
 }
 
 char *
 generate_op_key(const char *rsc_id, const char *op_type, int interval)
 {
     int len = 35;
     char *op_id = NULL;
 
     CRM_CHECK(rsc_id != NULL, return NULL);
     CRM_CHECK(op_type != NULL, return NULL);
 
     len += strlen(op_type);
     len += strlen(rsc_id);
     op_id = malloc(len);
     CRM_CHECK(op_id != NULL, return NULL);
     sprintf(op_id, "%s_%s_%d", rsc_id, op_type, interval);
     return op_id;
 }
 
 gboolean
 parse_op_key(const char *key, char **rsc_id, char **op_type, int *interval)
 {
     char *notify = NULL;
     char *mutable_key = NULL;
     char *mutable_key_ptr = NULL;
     int len = 0, offset = 0, ch = 0;
 
     CRM_CHECK(key != NULL, return FALSE);
 
     *interval = 0;
     len = strlen(key);
     offset = len - 1;
 
     crm_trace("Source: %s", key);
 
     while (offset > 0 && isdigit(key[offset])) {
         int digits = len - offset;
 
         ch = key[offset] - '0';
         CRM_CHECK(ch < 10, return FALSE);
         CRM_CHECK(ch >= 0, return FALSE);
         while (digits > 1) {
             digits--;
             ch = ch * 10;
         }
         *interval += ch;
         offset--;
     }
 
     crm_trace("  Interval: %d", *interval);
     CRM_CHECK(key[offset] == '_', return FALSE);
 
     mutable_key = strdup(key);
     mutable_key[offset] = 0;
     offset--;
 
     while (offset > 0 && key[offset] != '_') {
         offset--;
     }
 
     CRM_CHECK(key[offset] == '_', free(mutable_key);
               return FALSE);
 
     mutable_key_ptr = mutable_key + offset + 1;
 
     crm_trace("  Action: %s", mutable_key_ptr);
 
     *op_type = strdup(mutable_key_ptr);
 
     mutable_key[offset] = 0;
     offset--;
 
     CRM_CHECK(mutable_key != mutable_key_ptr, free(mutable_key);
               return FALSE);
 
     notify = strstr(mutable_key, "_post_notify");
     if (notify && safe_str_eq(notify, "_post_notify")) {
         notify[0] = 0;
     }
 
     notify = strstr(mutable_key, "_pre_notify");
     if (notify && safe_str_eq(notify, "_pre_notify")) {
         notify[0] = 0;
     }
 
     crm_trace("  Resource: %s", mutable_key);
     *rsc_id = mutable_key;
 
     return TRUE;
 }
 
 char *
 generate_notify_key(const char *rsc_id, const char *notify_type, const char *op_type)
 {
     int len = 12;
     char *op_id = NULL;
 
     CRM_CHECK(rsc_id != NULL, return NULL);
     CRM_CHECK(op_type != NULL, return NULL);
     CRM_CHECK(notify_type != NULL, return NULL);
 
     len += strlen(op_type);
     len += strlen(rsc_id);
     len += strlen(notify_type);
     if(len > 0) {
         op_id = malloc(len);
     }
     if (op_id != NULL) {
         sprintf(op_id, "%s_%s_notify_%s_0", rsc_id, notify_type, op_type);
     }
     return op_id;
 }
 
 char *
 generate_transition_magic_v202(const char *transition_key, int op_status)
 {
     int len = 80;
     char *fail_state = NULL;
 
     CRM_CHECK(transition_key != NULL, return NULL);
 
     len += strlen(transition_key);
 
     fail_state = malloc(len);
     if (fail_state != NULL) {
         snprintf(fail_state, len, "%d:%s", op_status, transition_key);
     }
     return fail_state;
 }
 
 char *
 generate_transition_magic(const char *transition_key, int op_status, int op_rc)
 {
     int len = 80;
     char *fail_state = NULL;
 
     CRM_CHECK(transition_key != NULL, return NULL);
 
     len += strlen(transition_key);
 
     fail_state = malloc(len);
     if (fail_state != NULL) {
         snprintf(fail_state, len, "%d:%d;%s", op_status, op_rc, transition_key);
     }
     return fail_state;
 }
 
 gboolean
 decode_transition_magic(const char *magic, char **uuid, int *transition_id, int *action_id,
                         int *op_status, int *op_rc, int *target_rc)
 {
     int res = 0;
     char *key = NULL;
     gboolean result = TRUE;
 
     CRM_CHECK(magic != NULL, return FALSE);
     CRM_CHECK(op_rc != NULL, return FALSE);
     CRM_CHECK(op_status != NULL, return FALSE);
 
     key = calloc(1, strlen(magic) + 1);
     res = sscanf(magic, "%d:%d;%s", op_status, op_rc, key);
     if (res != 3) {
         crm_warn("Only found %d items in: '%s'", res, magic);
         free(key);
         return FALSE;
     }
 
     CRM_CHECK(decode_transition_key(key, uuid, transition_id, action_id, target_rc), result = FALSE);
 
     free(key);
     return result;
 }
 
 char *
 generate_transition_key(int transition_id, int action_id, int target_rc, const char *node)
 {
     int len = 40;
     char *fail_state = NULL;
 
     CRM_CHECK(node != NULL, return NULL);
 
     len += strlen(node);
 
     fail_state = malloc(len);
     if (fail_state != NULL) {
         snprintf(fail_state, len, "%d:%d:%d:%-*s", action_id, transition_id, target_rc, 36, node);
     }
     return fail_state;
 }
 
 gboolean
 decode_transition_key(const char *key, char **uuid, int *transition_id, int *action_id,
                       int *target_rc)
 {
     int res = 0;
     gboolean done = FALSE;
 
     CRM_CHECK(uuid != NULL, return FALSE);
     CRM_CHECK(target_rc != NULL, return FALSE);
     CRM_CHECK(action_id != NULL, return FALSE);
     CRM_CHECK(transition_id != NULL, return FALSE);
 
     *uuid = calloc(1, 37);
     res = sscanf(key, "%d:%d:%d:%36s", action_id, transition_id, target_rc, *uuid);
     switch (res) {
         case 4:
             /* Post Pacemaker 0.6 */
             done = TRUE;
             break;
         case 3:
         case 2:
             /* this can be tricky - the UUID might start with an integer */
 
             /* Until Pacemaker 0.6 */
             done = TRUE;
             *target_rc = -1;
             res = sscanf(key, "%d:%d:%36s", action_id, transition_id, *uuid);
             if (res == 2) {
                 *action_id = -1;
                 res = sscanf(key, "%d:%36s", transition_id, *uuid);
                 CRM_CHECK(res == 2, done = FALSE);
 
             } else if (res != 3) {
                 CRM_CHECK(res == 3, done = FALSE);
             }
             break;
 
         case 1:
             /* Prior to Heartbeat 2.0.8 */
             done = TRUE;
             *action_id = -1;
             *target_rc = -1;
             res = sscanf(key, "%d:%36s", transition_id, *uuid);
             CRM_CHECK(res == 2, done = FALSE);
             break;
         default:
             crm_crit("Unhandled sscanf result (%d) for %s", res, key);
     }
 
     if (strlen(*uuid) != 36) {
         crm_warn("Bad UUID (%s) in sscanf result (%d) for %s", *uuid, res, key);
     }
 
     if (done == FALSE) {
         crm_err("Cannot decode '%s' rc=%d", key, res);
 
         free(*uuid);
         *uuid = NULL;
         *target_rc = -1;
         *action_id = -1;
         *transition_id = -1;
     }
 
     return done;
 }
 
 void
 filter_action_parameters(xmlNode * param_set, const char *version)
 {
     char *key = NULL;
     char *timeout = NULL;
     char *interval = NULL;
 
     const char *attr_filter[] = {
         XML_ATTR_ID,
         XML_ATTR_CRM_VERSION,
         XML_LRM_ATTR_OP_DIGEST,
     };
 
     gboolean do_delete = FALSE;
     int lpc = 0;
     static int meta_len = 0;
 
     if (meta_len == 0) {
         meta_len = strlen(CRM_META);
     }
 
     if (param_set == NULL) {
         return;
     }
 
     for (lpc = 0; lpc < DIMOF(attr_filter); lpc++) {
         xml_remove_prop(param_set, attr_filter[lpc]);
     }
 
     key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
     interval = crm_element_value_copy(param_set, key);
     free(key);
 
     key = crm_meta_name(XML_ATTR_TIMEOUT);
     timeout = crm_element_value_copy(param_set, key);
 
     if (param_set) {
         xmlAttrPtr xIter = param_set->properties;
 
         while (xIter) {
             const char *prop_name = (const char *)xIter->name;
 
             xIter = xIter->next;
             do_delete = FALSE;
             if (strncasecmp(prop_name, CRM_META, meta_len) == 0) {
                 do_delete = TRUE;
             }
 
             if (do_delete) {
                 xml_remove_prop(param_set, prop_name);
             }
         }
     }
 
     if (crm_get_msec(interval) > 0 && compare_version(version, "1.0.8") > 0) {
         /* Re-instate the operation's timeout value */
         if (timeout != NULL) {
             crm_xml_add(param_set, key, timeout);
         }
     }
 
     free(interval);
     free(timeout);
     free(key);
 }
 
 extern bool crm_is_daemon;
 
 /* coverity[+kill] */
 void
 crm_abort(const char *file, const char *function, int line,
           const char *assert_condition, gboolean do_core, gboolean do_fork)
 {
     int rc = 0;
     int pid = 0;
     int status = 0;
 
     /* Implied by the parent's error logging below */
     /* crm_write_blackbox(0); */
 
     if(crm_is_daemon == FALSE) {
         /* This is a command line tool - do not fork */
 
         /* crm_add_logfile(NULL);   * Record it to a file? */
         crm_enable_stderr(TRUE); /* Make sure stderr is enabled so we can tell the caller */
         do_fork = FALSE;         /* Just crash if needed */
     }
 
     if (do_core == FALSE) {
         crm_err("%s: Triggered assert at %s:%d : %s", function, file, line, assert_condition);
         return;
 
     } else if (do_fork) {
         pid = fork();
 
     } else {
         crm_err("%s: Triggered fatal assert at %s:%d : %s", function, file, line, assert_condition);
     }
 
     if (pid == -1) {
         crm_crit("%s: Cannot create core for non-fatal assert at %s:%d : %s",
                  function, file, line, assert_condition);
         return;
 
     } else if(pid == 0) {
         /* Child process */
         abort();
         return;
     }
 
     /* Parent process */
     crm_err("%s: Forked child %d to record non-fatal assert at %s:%d : %s",
             function, pid, file, line, assert_condition);
     crm_write_blackbox(SIGTRAP, NULL);
 
     do {
         rc = waitpid(pid, &status, 0);
         if(rc == pid) {
             return; /* Job done */
         }
 
     } while(errno == EINTR);
 
     if (errno == ECHILD) {
         /* crm_mon does this */
         crm_trace("Cannot wait on forked child %d - SIGCHLD is probably set to SIG_IGN", pid);
         return;
     }
     crm_perror(LOG_ERR, "Cannot wait on forked child %d", pid);
 }
 
 int
 crm_pid_active(long pid, const char *daemon)
 {
     static int have_proc_pid = 0;
 
     if(have_proc_pid == 0) {
         char proc_path[PATH_MAX], exe_path[PATH_MAX];
 
         /* check to make sure pid hasn't been reused by another process */
         snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", (long unsigned int)getpid());
 
         have_proc_pid = 1;
         if(readlink(proc_path, exe_path, PATH_MAX - 1) < 0) {
             have_proc_pid = -1;
         }
     }
 
     if (pid <= 0) {
         return -1;
 
     } else if (kill(pid, 0) < 0 && errno == ESRCH) {
         return 0;
 
     } else if(daemon == NULL || have_proc_pid == -1) {
         return 1;
 
     } else {
         int rc = 0;
         char proc_path[PATH_MAX], exe_path[PATH_MAX], myexe_path[PATH_MAX];
 
         /* check to make sure pid hasn't been reused by another process */
         snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", pid);
 
         rc = readlink(proc_path, exe_path, PATH_MAX - 1);
         if (rc < 0 && errno == EACCES) {
             crm_perror(LOG_INFO, "Could not read from %s", proc_path);
             return 1;
         } else if (rc < 0) {
             crm_perror(LOG_ERR, "Could not read from %s", proc_path);
             return 0;
         }
         
 
         exe_path[rc] = 0;
 
         if(daemon[0] != '/') {
             rc = snprintf(myexe_path, sizeof(proc_path), CRM_DAEMON_DIR"/%s", daemon);
             myexe_path[rc] = 0;
         } else {
             rc = snprintf(myexe_path, sizeof(proc_path), "%s", daemon);
             myexe_path[rc] = 0;
         }
         
         if (strcmp(exe_path, myexe_path) == 0) {
             return 1;
         }
     }
 
     return 0;
 }
 
 #define	LOCKSTRLEN	11
 
 long
 crm_read_pidfile(const char *filename)
 {
     int fd;
     struct stat sbuf;
     long pid = -ENOENT;
     char buf[LOCKSTRLEN + 1];
 
     if ((fd = open(filename, O_RDONLY)) < 0) {
         goto bail;
     }
 
     if (fstat(fd, &sbuf) >= 0 && sbuf.st_size < LOCKSTRLEN) {
         sleep(2);           /* if someone was about to create one,
                              * give'm a sec to do so
                              */
     }
 
     if (read(fd, buf, sizeof(buf)) < 1) {
         goto bail;
     }
 
     if (sscanf(buf, "%lu", &pid) > 0) {
         if (pid <= 0) {
             pid = -ESRCH;
         } else {
             crm_trace("Got pid %lu from %s\n", pid, filename);
         }
     }
 
   bail:
     if (fd >= 0) {
         close(fd);
     }
     return pid;
 }
 
 long
 crm_pidfile_inuse(const char *filename, long mypid, const char *daemon)
 {
     long pid = crm_read_pidfile(filename);
 
     if (pid < 2) {
         /* Invalid pid */
         pid = -ENOENT;
         unlink(filename);
 
     } else if (mypid && pid == mypid) {
         /* In use by us */
         pid = pcmk_ok;
 
     } else if (crm_pid_active(pid, daemon) == FALSE) {
         /* Contains a stale value */
         unlink(filename);
         pid = -ENOENT;
 
     } else if (mypid && pid != mypid) {
         /* locked by existing process - give up */
         pid = -EEXIST;
     }
 
     return pid;
 }
 
 static int
 crm_lock_pidfile(const char *filename, const char *name)
 {
     long mypid = 0;
     int fd = 0, rc = 0;
     char buf[LOCKSTRLEN + 1];
 
     mypid = (unsigned long)getpid();
 
     rc = crm_pidfile_inuse(filename, 0, name);
     if (rc == -ENOENT) {
         /* exists but the process is not active */
 
     } else if (rc != pcmk_ok) {
         /* locked by existing process - give up */
         return rc;
     }
 
     if ((fd = open(filename, O_CREAT | O_WRONLY | O_EXCL, 0644)) < 0) {
         /* Hmmh, why did we fail? Anyway, nothing we can do about it */
         return -errno;
     }
 
     snprintf(buf, sizeof(buf), "%*lu\n", LOCKSTRLEN - 1, mypid);
     rc = write(fd, buf, LOCKSTRLEN);
     close(fd);
 
     if (rc != LOCKSTRLEN) {
         crm_perror(LOG_ERR, "Incomplete write to %s", filename);
         return -errno;
     }
 
     return crm_pidfile_inuse(filename, mypid, name);
 }
 
 void
 crm_make_daemon(const char *name, gboolean daemonize, const char *pidfile)
 {
     int rc;
     long pid;
     const char *devnull = "/dev/null";
 
     if (daemonize == FALSE) {
         return;
     }
 
     /* Check before we even try... */
     rc = crm_pidfile_inuse(pidfile, 1, name);
     if(rc < pcmk_ok && rc != -ENOENT) {
         pid = crm_read_pidfile(pidfile);
         crm_err("%s: already running [pid %ld in %s]", name, pid, pidfile);
         printf("%s: already running [pid %ld in %s]\n", name, pid, pidfile);
         crm_exit(rc);
     }
 
     pid = fork();
     if (pid < 0) {
         fprintf(stderr, "%s: could not start daemon\n", name);
         crm_perror(LOG_ERR, "fork");
         crm_exit(EINVAL);
 
     } else if (pid > 0) {
         crm_exit(pcmk_ok);
     }
 
     rc = crm_lock_pidfile(pidfile, name);
     if(rc < pcmk_ok) {
         crm_err("Could not lock '%s' for %s: %s (%d)", pidfile, name, pcmk_strerror(rc), rc);
         printf("Could not lock '%s' for %s: %s (%d)\n", pidfile, name, pcmk_strerror(rc), rc);
         crm_exit(rc);
     }
 
     umask(S_IWGRP | S_IWOTH | S_IROTH);
 
     close(STDIN_FILENO);
     (void)open(devnull, O_RDONLY);      /* Stdin:  fd 0 */
     close(STDOUT_FILENO);
     (void)open(devnull, O_WRONLY);      /* Stdout: fd 1 */
     close(STDERR_FILENO);
     (void)open(devnull, O_WRONLY);      /* Stderr: fd 2 */
 }
 
 char *
 crm_meta_name(const char *field)
 {
     int lpc = 0;
     int max = 0;
     char *crm_name = NULL;
 
     CRM_CHECK(field != NULL, return NULL);
     crm_name = crm_concat(CRM_META, field, '_');
 
     /* Massage the names so they can be used as shell variables */
     max = strlen(crm_name);
     for (; lpc < max; lpc++) {
         switch (crm_name[lpc]) {
             case '-':
                 crm_name[lpc] = '_';
                 break;
         }
     }
     return crm_name;
 }
 
 const char *
 crm_meta_value(GHashTable * hash, const char *field)
 {
     char *key = NULL;
     const char *value = NULL;
 
     key = crm_meta_name(field);
     if (key) {
         value = g_hash_table_lookup(hash, key);
         free(key);
     }
 
     return value;
 }
 
 static struct option *
 crm_create_long_opts(struct crm_option *long_options)
 {
     struct option *long_opts = NULL;
 
 #ifdef HAVE_GETOPT_H
     int index = 0, lpc = 0;
 
     /*
      * A previous, possibly poor, choice of '?' as the short form of --help
      * means that getopt_long() returns '?' for both --help and for "unknown option"
      *
      * This dummy entry allows us to differentiate between the two in crm_get_option()
      * and exit with the correct error code
      */
     long_opts = realloc_safe(long_opts, (index + 1) * sizeof(struct option));
     long_opts[index].name = "__dummmy__";
     long_opts[index].has_arg = 0;
     long_opts[index].flag = 0;
     long_opts[index].val = '_';
     index++;
 
     for (lpc = 0; long_options[lpc].name != NULL; lpc++) {
         if (long_options[lpc].name[0] == '-') {
             continue;
         }
 
         long_opts = realloc_safe(long_opts, (index + 1) * sizeof(struct option));
         /*fprintf(stderr, "Creating %d %s = %c\n", index,
          * long_options[lpc].name, long_options[lpc].val);      */
         long_opts[index].name = long_options[lpc].name;
         long_opts[index].has_arg = long_options[lpc].has_arg;
         long_opts[index].flag = long_options[lpc].flag;
         long_opts[index].val = long_options[lpc].val;
         index++;
     }
 
     /* Now create the list terminator */
     long_opts = realloc_safe(long_opts, (index + 1) * sizeof(struct option));
     long_opts[index].name = NULL;
     long_opts[index].has_arg = 0;
     long_opts[index].flag = 0;
     long_opts[index].val = 0;
 #endif
 
     return long_opts;
 }
 
 void
 crm_set_options(const char *short_options, const char *app_usage, struct crm_option *long_options,
                 const char *app_desc)
 {
     if (short_options) {
         crm_short_options = strdup(short_options);
 
     } else if (long_options) {
         int lpc = 0;
         int opt_string_len = 0;
         char *local_short_options = NULL;
 
         for (lpc = 0; long_options[lpc].name != NULL; lpc++) {
             if (long_options[lpc].val && long_options[lpc].val != '-' && long_options[lpc].val < UCHAR_MAX) {
                 local_short_options = realloc_safe(local_short_options, opt_string_len + 4);
                 local_short_options[opt_string_len++] = long_options[lpc].val;
                 /* getopt(3) says: Two colons mean an option takes an optional arg; */
                 if (long_options[lpc].has_arg == optional_argument) {
                     local_short_options[opt_string_len++] = ':';
                 }
                 if (long_options[lpc].has_arg >= required_argument) {
                     local_short_options[opt_string_len++] = ':';
                 }
                 local_short_options[opt_string_len] = 0;
             }
         }
         crm_short_options = local_short_options;
         crm_trace("Generated short option string: '%s'", local_short_options);
     }
 
     if (long_options) {
         crm_long_options = long_options;
     }
     if (app_desc) {
         crm_app_description = app_desc;
     }
     if (app_usage) {
         crm_app_usage = app_usage;
     }
 }
 
 int
 crm_get_option(int argc, char **argv, int *index)
 {
     return crm_get_option_long(argc, argv, index, NULL);
 }
 
 int
 crm_get_option_long(int argc, char **argv, int *index, const char **longname)
 {
 #ifdef HAVE_GETOPT_H
     static struct option *long_opts = NULL;
 
     if (long_opts == NULL && crm_long_options) {
         long_opts = crm_create_long_opts(crm_long_options);
     }
 
     *index = 0;
     if (long_opts) {
         int flag = getopt_long(argc, argv, crm_short_options, long_opts, index);
 
         switch (flag) {
             case 0:
                 if (long_opts[*index].val) {
                     return long_opts[*index].val;
                 } else if (longname) {
                     *longname = long_opts[*index].name;
                 } else {
                     crm_notice("Unhandled option --%s", long_opts[*index].name);
                     return flag;
                 }
             case -1:           /* End of option processing */
                 break;
             case ':':
                 crm_trace("Missing argument");
                 crm_help('?', 1);
                 break;
             case '?':
                 crm_help('?', *index ? 0 : 1);
                 break;
         }
         return flag;
     }
 #endif
 
     if (crm_short_options) {
         return getopt(argc, argv, crm_short_options);
     }
 
     return -1;
 }
 
 int
 crm_help(char cmd, int exit_code)
 {
     int i = 0;
     FILE *stream = (exit_code ? stderr : stdout);
 
     if (cmd == 'v' || cmd == '$') {
         fprintf(stream, "Pacemaker %s\n", PACEMAKER_VERSION);
         fprintf(stream, "Written by Andrew Beekhof\n");
         goto out;
     }
 
     if (cmd == '!') {
         fprintf(stream, "Pacemaker %s (Build: %s): %s\n", PACEMAKER_VERSION, BUILD_VERSION, CRM_FEATURES);
         goto out;
     }
 
     fprintf(stream, "%s - %s\n", crm_system_name, crm_app_description);
 
     if (crm_app_usage) {
         fprintf(stream, "Usage: %s %s\n", crm_system_name, crm_app_usage);
     }
 
     if (crm_long_options) {
         fprintf(stream, "Options:\n");
         for (i = 0; crm_long_options[i].name != NULL; i++) {
             if (crm_long_options[i].flags & pcmk_option_hidden) {
 
             } else if (crm_long_options[i].flags & pcmk_option_paragraph) {
                 fprintf(stream, "%s\n\n", crm_long_options[i].desc);
 
             } else if (crm_long_options[i].flags & pcmk_option_example) {
                 fprintf(stream, "\t#%s\n\n", crm_long_options[i].desc);
 
             } else if (crm_long_options[i].val == '-' && crm_long_options[i].desc) {
                 fprintf(stream, "%s\n", crm_long_options[i].desc);
 
             } else {
                 /* is val printable as char ? */
                 if (crm_long_options[i].val && crm_long_options[i].val <= UCHAR_MAX) {
                     fprintf(stream, " -%c,", crm_long_options[i].val);
                 } else {
                     fputs("    ", stream);
                 }
                 fprintf(stream, " --%s%s\t%s\n", crm_long_options[i].name,
                         crm_long_options[i].has_arg == optional_argument ? "[=value]" :
                         crm_long_options[i].has_arg == required_argument ? "=value" : "",
                         crm_long_options[i].desc ? crm_long_options[i].desc : "");
             }
         }
 
     } else if (crm_short_options) {
         fprintf(stream, "Usage: %s - %s\n", crm_system_name, crm_app_description);
         for (i = 0; crm_short_options[i] != 0; i++) {
             int has_arg = no_argument /* 0 */;
 
             if (crm_short_options[i + 1] == ':') {
                 if (crm_short_options[i + 2] == ':')
                     has_arg = optional_argument /* 2 */;
                 else
                     has_arg = required_argument /* 1 */;
             }
 
             fprintf(stream, " -%c %s\n", crm_short_options[i],
                     has_arg == optional_argument ? "[value]" :
                     has_arg == required_argument ? "{value}" : "");
             i += has_arg;
         }
     }
 
     fprintf(stream, "\nReport bugs to %s\n", PACKAGE_BUGREPORT);
 
   out:
     return crm_exit(exit_code);
 }
 
 void cib_ipc_servers_init(qb_ipcs_service_t **ipcs_ro,
         qb_ipcs_service_t **ipcs_rw,
         qb_ipcs_service_t **ipcs_shm,
         struct qb_ipcs_service_handlers *ro_cb,
         struct qb_ipcs_service_handlers *rw_cb)
 {
     *ipcs_ro = mainloop_add_ipc_server(cib_channel_ro, QB_IPC_NATIVE, ro_cb);
     *ipcs_rw = mainloop_add_ipc_server(cib_channel_rw, QB_IPC_NATIVE, rw_cb);
     *ipcs_shm = mainloop_add_ipc_server(cib_channel_shm, QB_IPC_SHM, rw_cb);
 
     if (*ipcs_ro == NULL || *ipcs_rw == NULL || *ipcs_shm == NULL) {
         crm_err("Failed to create cib servers: exiting and inhibiting respawn.");
         crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
         crm_exit(DAEMON_RESPAWN_STOP);
     }
 }
 
 void cib_ipc_servers_destroy(qb_ipcs_service_t *ipcs_ro,
         qb_ipcs_service_t *ipcs_rw,
         qb_ipcs_service_t *ipcs_shm)
 {
     qb_ipcs_destroy(ipcs_ro);
     qb_ipcs_destroy(ipcs_rw);
     qb_ipcs_destroy(ipcs_shm);
 }
 
 qb_ipcs_service_t *
 crmd_ipc_server_init(struct qb_ipcs_service_handlers *cb)
 {
     return mainloop_add_ipc_server(CRM_SYSTEM_CRMD, QB_IPC_NATIVE, cb);
 }
 
 void
 attrd_ipc_server_init(qb_ipcs_service_t **ipcs, struct qb_ipcs_service_handlers *cb)
 {
     *ipcs = mainloop_add_ipc_server(T_ATTRD, QB_IPC_NATIVE, cb);
 
     if (*ipcs == NULL) {
         crm_err("Failed to create attrd servers: exiting and inhibiting respawn.");
         crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
         crm_exit(DAEMON_RESPAWN_STOP);
     }
 }
 
 void
 stonith_ipc_server_init(qb_ipcs_service_t **ipcs, struct qb_ipcs_service_handlers *cb)
 {
     *ipcs = mainloop_add_ipc_server("stonith-ng", QB_IPC_NATIVE, cb);
 
     if (*ipcs == NULL) {
         crm_err("Failed to create stonith-ng servers: exiting and inhibiting respawn.");
         crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
         crm_exit(DAEMON_RESPAWN_STOP);
     }
 }
 
 int
 attrd_update_delegate(crm_ipc_t * ipc, char command, const char *host, const char *name,
                       const char *value, const char *section, const char *set, const char *dampen,
                       const char *user_name, int options)
 {
     int rc = -ENOTCONN;
     int max = 5;
     const char *task = NULL;
     const char *name_as = NULL;
+    const char *display_host = (host ? host : "localhost");
+    const char *display_command = NULL; /* for commands without name/value */
     xmlNode *update = create_xml_node(NULL, __FUNCTION__);
 
     static gboolean connected = TRUE;
     static crm_ipc_t *local_ipc = NULL;
     static enum crm_ipc_flags flags = crm_ipc_flags_none;
 
     if (ipc == NULL && local_ipc == NULL) {
         local_ipc = crm_ipc_new(T_ATTRD, 0);
         flags |= crm_ipc_client_response;
         connected = FALSE;
     }
 
     if (ipc == NULL) {
         ipc = local_ipc;
     }
 
     /* remap common aliases */
     if (safe_str_eq(section, "reboot")) {
         section = XML_CIB_TAG_STATUS;
 
     } else if (safe_str_eq(section, "forever")) {
         section = XML_CIB_TAG_NODES;
     }
 
     crm_xml_add(update, F_TYPE, T_ATTRD);
     crm_xml_add(update, F_ORIG, crm_system_name?crm_system_name:"unknown");
 
     if (name == NULL && command == 'U') {
         command = 'R';
     }
 
     switch (command) {
         case 'u':
             task = ATTRD_OP_UPDATE;
             name_as = F_ATTRD_REGEX;
             break;
         case 'D':
         case 'U':
         case 'v':
             task = ATTRD_OP_UPDATE;
             name_as = F_ATTRD_ATTRIBUTE;
             break;
         case 'R':
             task = ATTRD_OP_REFRESH;
+            display_command = "refresh";
             break;
         case 'B':
             task = ATTRD_OP_UPDATE_BOTH;
             name_as = F_ATTRD_ATTRIBUTE;
             break;
         case 'Y':
             task = ATTRD_OP_UPDATE_DELAY;
             name_as = F_ATTRD_ATTRIBUTE;
             break;
         case 'Q':
             task = ATTRD_OP_QUERY;
             name_as = F_ATTRD_ATTRIBUTE;
             break;
         case 'C':
             task = ATTRD_OP_PEER_REMOVE;
+            display_command = "purge";
             break;
     }
 
     if (name_as != NULL) {
         if (name == NULL) {
             rc = -EINVAL;
             goto done;
         }
         crm_xml_add(update, name_as, name);
     }
 
     crm_xml_add(update, F_ATTRD_TASK, task);
     crm_xml_add(update, F_ATTRD_VALUE, value);
     crm_xml_add(update, F_ATTRD_DAMPEN, dampen);
     crm_xml_add(update, F_ATTRD_SECTION, section);
     crm_xml_add(update, F_ATTRD_HOST, host);
     crm_xml_add(update, F_ATTRD_SET, set);
     crm_xml_add_int(update, F_ATTRD_IS_REMOTE, is_set(options, attrd_opt_remote));
     crm_xml_add_int(update, F_ATTRD_IS_PRIVATE, is_set(options, attrd_opt_private));
 #if ENABLE_ACL
     if (user_name) {
         crm_xml_add(update, F_ATTRD_USER, user_name);
     }
 #endif
 
     while (max > 0) {
         if (connected == FALSE) {
             crm_info("Connecting to cluster... %d retries remaining", max);
             connected = crm_ipc_connect(ipc);
         }
 
         if (connected) {
             rc = crm_ipc_send(ipc, update, flags, 0, NULL);
         } else {
             crm_perror(LOG_INFO, "Connection to cluster attribute manager failed");
         }
 
         if (ipc != local_ipc) {
             break;
 
         } else if (rc > 0) {
             break;
 
         } else if (rc == -EAGAIN || rc == -EALREADY) {
             sleep(5 - max);
             max--;
 
         } else {
             crm_ipc_close(ipc);
             connected = FALSE;
             sleep(5 - max);
             max--;
         }
     }
 
 done:
     free_xml(update);
     if (rc > 0) {
-        crm_debug("Sent update: %s=%s for %s", name, value, host ? host : "localhost");
         rc = pcmk_ok;
+    }
 
+    if (display_command) {
+        crm_debug("Asked attrd to %s %s: %s (%d)",
+                  display_command, display_host, pcmk_strerror(rc), rc);
     } else {
-        crm_debug("Could not send update %s=%s for %s: %s (%d)", name, value,
-                  host ? host : "localhost", pcmk_strerror(rc), rc);
+        crm_debug("Asked attrd to update %s=%s for %s: %s (%d)",
+                  name, value, display_host, pcmk_strerror(rc), rc);
     }
     return rc;
 }
 
 #define FAKE_TE_ID	"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
 static void
 append_digest(lrmd_event_data_t * op, xmlNode * update, const char *version, const char *magic,
               int level)
 {
     /* this will enable us to later determine that the
      *   resource's parameters have changed and we should force
      *   a restart
      */
     char *digest = NULL;
     xmlNode *args_xml = NULL;
 
     if (op->params == NULL) {
         return;
     }
 
     args_xml = create_xml_node(NULL, XML_TAG_PARAMS);
     g_hash_table_foreach(op->params, hash2field, args_xml);
     filter_action_parameters(args_xml, version);
     crm_summarize_versioned_params(args_xml, op->versioned_params);
     digest = calculate_operation_digest(args_xml, version);
 
 #if 0
     if (level < get_crm_log_level()
         && op->interval == 0 && crm_str_eq(op->op_type, CRMD_ACTION_START, TRUE)) {
         char *digest_source = dump_xml_unformatted(args_xml);
 
         do_crm_log(level, "Calculated digest %s for %s (%s). Source: %s\n",
                    digest, ID(update), magic, digest_source);
         free(digest_source);
     }
 #endif
     crm_xml_add(update, XML_LRM_ATTR_OP_DIGEST, digest);
 
     free_xml(args_xml);
     free(digest);
 }
 
 int
 rsc_op_expected_rc(lrmd_event_data_t * op)
 {
     int rc = 0;
 
     if (op && op->user_data) {
         int dummy = 0;
         char *uuid = NULL;
 
         decode_transition_key(op->user_data, &uuid, &dummy, &dummy, &rc);
         free(uuid);
     }
     return rc;
 }
 
 gboolean
 did_rsc_op_fail(lrmd_event_data_t * op, int target_rc)
 {
     switch (op->op_status) {
         case PCMK_LRM_OP_CANCELLED:
         case PCMK_LRM_OP_PENDING:
             return FALSE;
             break;
 
         case PCMK_LRM_OP_NOTSUPPORTED:
         case PCMK_LRM_OP_TIMEOUT:
         case PCMK_LRM_OP_ERROR:
             return TRUE;
             break;
 
         default:
             if (target_rc != op->rc) {
                 return TRUE;
             }
     }
 
     return FALSE;
 }
 
 xmlNode *
 create_operation_update(xmlNode * parent, lrmd_event_data_t * op, const char * caller_version,
                         int target_rc, const char * node, const char * origin, int level)
 {
     char *key = NULL;
     char *magic = NULL;
     char *op_id = NULL;
     char *op_id_additional = NULL;
     char *local_user_data = NULL;
     const char *exit_reason = NULL;
 
     xmlNode *xml_op = NULL;
     const char *task = NULL;
     gboolean dc_munges_migrate_ops = (compare_version(caller_version, "3.0.3") < 0);
     gboolean dc_needs_unique_ops = (compare_version(caller_version, "3.0.6") < 0);
 
     CRM_CHECK(op != NULL, return NULL);
     do_crm_log(level, "%s: Updating resource %s after %s op %s (interval=%d)",
                origin, op->rsc_id, op->op_type, services_lrm_status_str(op->op_status),
                op->interval);
 
     crm_trace("DC version: %s", caller_version);
 
     task = op->op_type;
     /* remap the task name under various scenarios
      * this makes life easier for the PE when trying determine the current state
      */
     if (crm_str_eq(task, "reload", TRUE)) {
         if (op->op_status == PCMK_LRM_OP_DONE) {
             task = CRMD_ACTION_START;
         } else {
             task = CRMD_ACTION_STATUS;
         }
 
     } else if (dc_munges_migrate_ops && crm_str_eq(task, CRMD_ACTION_MIGRATE, TRUE)) {
         /* if the migrate_from fails it will have enough info to do the right thing */
         if (op->op_status == PCMK_LRM_OP_DONE) {
             task = CRMD_ACTION_STOP;
         } else {
             task = CRMD_ACTION_STATUS;
         }
 
     } else if (dc_munges_migrate_ops
                && op->op_status == PCMK_LRM_OP_DONE
                && crm_str_eq(task, CRMD_ACTION_MIGRATED, TRUE)) {
         task = CRMD_ACTION_START;
     }
 
     key = generate_op_key(op->rsc_id, task, op->interval);
     if (dc_needs_unique_ops && op->interval > 0) {
         op_id = strdup(key);
 
     } else if (crm_str_eq(task, CRMD_ACTION_NOTIFY, TRUE)) {
         const char *n_type = crm_meta_value(op->params, "notify_type");
         const char *n_task = crm_meta_value(op->params, "notify_operation");
 
         CRM_LOG_ASSERT(n_type != NULL);
         CRM_LOG_ASSERT(n_task != NULL);
         op_id = generate_notify_key(op->rsc_id, n_type, n_task);
 
         /* these are not yet allowed to fail */
         op->op_status = PCMK_LRM_OP_DONE;
         op->rc = 0;
 
     } else if (did_rsc_op_fail(op, target_rc)) {
         op_id = generate_op_key(op->rsc_id, "last_failure", 0);
         if (op->interval == 0) {
             /* Ensure 'last' gets updated too in case recording-pending="true" */
             op_id_additional = generate_op_key(op->rsc_id, "last", 0);
         }
         exit_reason = op->exit_reason;
 
     } else if (op->interval > 0) {
         op_id = strdup(key);
 
     } else {
         op_id = generate_op_key(op->rsc_id, "last", 0);
     }
 
   again:
     xml_op = find_entity(parent, XML_LRM_TAG_RSC_OP, op_id);
     if (xml_op == NULL) {
         xml_op = create_xml_node(parent, XML_LRM_TAG_RSC_OP);
     }
 
     if (op->user_data == NULL) {
         crm_debug("Generating fake transition key for:"
                   " %s_%s_%d %d from %s",
                   op->rsc_id, op->op_type, op->interval, op->call_id, origin);
         local_user_data = generate_transition_key(-1, op->call_id, target_rc, FAKE_TE_ID);
         op->user_data = local_user_data;
     }
 
     if(magic == NULL) {
         magic = generate_transition_magic(op->user_data, op->op_status, op->rc);
     }
 
     crm_xml_add(xml_op, XML_ATTR_ID, op_id);
     crm_xml_add(xml_op, XML_LRM_ATTR_TASK_KEY, key);
     crm_xml_add(xml_op, XML_LRM_ATTR_TASK, task);
     crm_xml_add(xml_op, XML_ATTR_ORIGIN, origin);
     crm_xml_add(xml_op, XML_ATTR_CRM_VERSION, caller_version);
     crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY, op->user_data);
     crm_xml_add(xml_op, XML_ATTR_TRANSITION_MAGIC, magic);
     crm_xml_add(xml_op, XML_LRM_ATTR_EXIT_REASON, exit_reason);
     crm_xml_add(xml_op, XML_LRM_ATTR_TARGET, node); /* For context during triage */
 
     crm_xml_add_int(xml_op, XML_LRM_ATTR_CALLID, op->call_id);
     crm_xml_add_int(xml_op, XML_LRM_ATTR_RC, op->rc);
     crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS, op->op_status);
     crm_xml_add_int(xml_op, XML_LRM_ATTR_INTERVAL, op->interval);
 
     if (compare_version("2.1", caller_version) <= 0) {
         if (op->t_run || op->t_rcchange || op->exec_time || op->queue_time) {
             crm_trace("Timing data (%s_%s_%d): last=%u change=%u exec=%u queue=%u",
                       op->rsc_id, op->op_type, op->interval,
                       op->t_run, op->t_rcchange, op->exec_time, op->queue_time);
 
             if (op->interval == 0) {
                 /* The values are the same for non-recurring ops */
                 crm_xml_add_int(xml_op, XML_RSC_OP_LAST_RUN, op->t_run);
                 crm_xml_add_int(xml_op, XML_RSC_OP_LAST_CHANGE, op->t_run);
 
             } else if(op->t_rcchange) {
                 /* last-run is not accurate for recurring ops */
                 crm_xml_add_int(xml_op, XML_RSC_OP_LAST_CHANGE, op->t_rcchange);
 
             } else {
                 /* ...but is better than nothing otherwise */
                 crm_xml_add_int(xml_op, XML_RSC_OP_LAST_CHANGE, op->t_run);
             }
 
             crm_xml_add_int(xml_op, XML_RSC_OP_T_EXEC, op->exec_time);
             crm_xml_add_int(xml_op, XML_RSC_OP_T_QUEUE, op->queue_time);
         }
     }
 
     if (crm_str_eq(op->op_type, CRMD_ACTION_MIGRATE, TRUE)
         || crm_str_eq(op->op_type, CRMD_ACTION_MIGRATED, TRUE)) {
         /*
          * Record migrate_source and migrate_target always for migrate ops.
          */
         const char *name = XML_LRM_ATTR_MIGRATE_SOURCE;
 
         crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
 
         name = XML_LRM_ATTR_MIGRATE_TARGET;
         crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
     }
 
     append_digest(op, xml_op, caller_version, magic, LOG_DEBUG);
 
     if (op_id_additional) {
         free(op_id);
         op_id = op_id_additional;
         op_id_additional = NULL;
         goto again;
     }
 
     if (local_user_data) {
         free(local_user_data);
         op->user_data = NULL;
     }
     free(magic);
     free(op_id);
     free(key);
     return xml_op;
 }
 
 bool
 pcmk_acl_required(const char *user) 
 {
 #if ENABLE_ACL
     if(user == NULL || strlen(user) == 0) {
         crm_trace("no user set");
         return FALSE;
 
     } else if (strcmp(user, CRM_DAEMON_USER) == 0) {
         return FALSE;
 
     } else if (strcmp(user, "root") == 0) {
         return FALSE;
     }
     crm_trace("acls required for %s", user);
     return TRUE;
 #else
     crm_trace("acls not supported");
     return FALSE;
 #endif
 }
 
 #if ENABLE_ACL
 char *
 uid2username(uid_t uid)
 {
     struct passwd *pwent = getpwuid(uid);
 
     if (pwent == NULL) {
         crm_perror(LOG_ERR, "Cannot get password entry of uid: %d", uid);
         return NULL;
 
     } else {
         return strdup(pwent->pw_name);
     }
 }
 
 const char *
 crm_acl_get_set_user(xmlNode * request, const char *field, const char *peer_user)
 {
     /* field is only checked for backwards compatibility */
     static const char *effective_user = NULL;
     const char *requested_user = NULL;
     const char *user = NULL;
 
     if(effective_user == NULL) {
         effective_user = uid2username(geteuid());
     }
 
     requested_user = crm_element_value(request, XML_ACL_TAG_USER);
     if(requested_user == NULL) {
         requested_user = crm_element_value(request, field);
     }
 
     if (is_privileged(effective_user) == FALSE) {
         /* We're not running as a privileged user, set or overwrite any existing value for $XML_ACL_TAG_USER */
         user = effective_user;
 
     } else if(peer_user == NULL && requested_user == NULL) {
         /* No user known or requested, use 'effective_user' and make sure one is set for the request */
         user = effective_user;
 
     } else if(peer_user == NULL) {
         /* No user known, trusting 'requested_user' */
         user = requested_user;
 
     } else if (is_privileged(peer_user) == FALSE) {
         /* The peer is not a privileged user, set or overwrite any existing value for $XML_ACL_TAG_USER */
         user = peer_user;
 
     } else if (requested_user == NULL) {
         /* Even if we're privileged, make sure there is always a value set */
         user = peer_user;
 
     } else {
         /* Legal delegation to 'requested_user' */
         user = requested_user;
     }
 
     /* Yes, pointer comparision */
     if(user != crm_element_value(request, XML_ACL_TAG_USER)) {
         crm_xml_add(request, XML_ACL_TAG_USER, user);
     }
 
     if(field != NULL && user != crm_element_value(request, field)) {
         crm_xml_add(request, field, user);
     }
 
     return requested_user;
 }
 
 void
 determine_request_user(const char *user, xmlNode * request, const char *field)
 {
     /* Get our internal validation out of the way first */
     CRM_CHECK(user != NULL && request != NULL && field != NULL, return);
 
     /* If our peer is a privileged user, we might be doing something on behalf of someone else */
     if (is_privileged(user) == FALSE) {
         /* We're not a privileged user, set or overwrite any existing value for $field */
         crm_xml_replace(request, field, user);
 
     } else if (crm_element_value(request, field) == NULL) {
         /* Even if we're privileged, make sure there is always a value set */
         crm_xml_replace(request, field, user);
 
 /*  } else { Legal delegation */
     }
 
     crm_trace("Processing msg as user '%s'", crm_element_value(request, field));
 }
 #endif
 
 void *
 find_library_function(void **handle, const char *lib, const char *fn, gboolean fatal)
 {
     char *error;
     void *a_function;
 
     if (*handle == NULL) {
         *handle = dlopen(lib, RTLD_LAZY);
     }
 
     if (!(*handle)) {
         crm_err("%sCould not open %s: %s", fatal ? "Fatal: " : "", lib, dlerror());
         if (fatal) {
             crm_exit(DAEMON_RESPAWN_STOP);
         }
         return NULL;
     }
 
     a_function = dlsym(*handle, fn);
     if (a_function == NULL) {
         error = dlerror();
         crm_err("%sCould not find %s in %s: %s", fatal ? "Fatal: " : "", fn, lib, error);
         if (fatal) {
             crm_exit(DAEMON_RESPAWN_STOP);
         }
     }
 
     return a_function;
 }
 
 void *
 convert_const_pointer(const void *ptr)
 {
     /* Worst function ever */
     return (void *)ptr;
 }
 
 #ifdef HAVE_UUID_UUID_H
 #  include <uuid/uuid.h>
 #endif
 
 char *
 crm_generate_uuid(void)
 {
     unsigned char uuid[16];
     char *buffer = malloc(37);  /* Including NUL byte */
 
     uuid_generate(uuid);
     uuid_unparse(uuid, buffer);
     return buffer;
 }
 
 #include <md5.h>
 
 char *
 crm_md5sum(const char *buffer)
 {
     int lpc = 0, len = 0;
     char *digest = NULL;
     unsigned char raw_digest[MD5_DIGEST_SIZE];
 
     if (buffer == NULL) {
         buffer = "";
     }
     len = strlen(buffer);
 
     crm_trace("Beginning digest of %d bytes", len);
     digest = malloc(2 * MD5_DIGEST_SIZE + 1);
     if(digest) {
         md5_buffer(buffer, len, raw_digest);
         for (lpc = 0; lpc < MD5_DIGEST_SIZE; lpc++) {
             sprintf(digest + (2 * lpc), "%02x", raw_digest[lpc]);
         }
         digest[(2 * MD5_DIGEST_SIZE)] = 0;
         crm_trace("Digest %s.", digest);
 
     } else {
         crm_err("Could not create digest");
     }
     return digest;
 }
 
 #ifdef HAVE_GNUTLS_GNUTLS_H
 void
 crm_gnutls_global_init(void)
 {
     signal(SIGPIPE, SIG_IGN);
     gnutls_global_init();
 }
 #endif
diff --git a/lib/common/xml.c b/lib/common/xml.c
index 1d2afe6ea3..a6a6d0a45e 100644
--- a/lib/common/xml.c
+++ b/lib/common/xml.c
@@ -1,5876 +1,5005 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  *
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  */
 
 #include <crm_internal.h>
+
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
-#include <sys/stat.h>
 #include <unistd.h>
 #include <time.h>
 #include <string.h>
-#include <dirent.h>
-
 #include <stdlib.h>
-#include <errno.h>
-#include <fcntl.h>
 #include <ctype.h>
-#include <math.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
-#include <libxml/xmlreader.h>
 
 #if HAVE_BZLIB_H
 #  include <bzlib.h>
 #endif
 
 #if HAVE_LIBXML2
 #  include <libxml/parser.h>
 #  include <libxml/tree.h>
-#  include <libxml/relaxng.h>
-#endif
-
-#if HAVE_LIBXSLT
-#  include <libxslt/xslt.h>
-#  include <libxslt/transform.h>
 #endif
 
 #define XML_BUFFER_SIZE	4096
 #define XML_PARSER_DEBUG 0
 
-void
-xml_log(int priority, const char *fmt, ...)
-G_GNUC_PRINTF(2, 3);
 static inline int
 __get_prefix(const char *prefix, xmlNode *xml, char *buffer, int offset);
 
-void
-xml_log(int priority, const char *fmt, ...)
-{
-    va_list ap;
-
-    va_start(ap, fmt);
-    qb_log_from_external_source_va(__FUNCTION__, __FILE__, fmt, priority, __LINE__, 0, ap);
-    va_end(ap);
-}
-
-typedef struct {
-    xmlRelaxNGPtr rng;
-    xmlRelaxNGValidCtxtPtr valid;
-    xmlRelaxNGParserCtxtPtr parser;
-} relaxng_ctx_cache_t;
-
-struct schema_s {
-    int type;
-    float version;
-    char *name;
-    char *location;
-    char *transform;
-    int after_transform;
-    void *cache;
-};
-
 typedef struct {
     int found;
     const char *string;
 } filter_t;
 
 enum xml_private_flags {
      xpf_none        = 0x0000,
      xpf_dirty       = 0x0001,
      xpf_deleted     = 0x0002,
      xpf_created     = 0x0004,
      xpf_modified    = 0x0008,
 
      xpf_tracking    = 0x0010,
      xpf_processed   = 0x0020,
      xpf_skip        = 0x0040,
      xpf_moved       = 0x0080,
 
      xpf_acl_enabled = 0x0100,
      xpf_acl_read    = 0x0200,
      xpf_acl_write   = 0x0400,
      xpf_acl_deny    = 0x0800,
 
      xpf_acl_create  = 0x1000,
      xpf_acl_denied  = 0x2000,
 };
 
 typedef struct xml_private_s 
 {
         long check;
         uint32_t flags;
         char *user;
         GListPtr acls;
         GListPtr deleted_paths;
 } xml_private_t;
 
 typedef struct xml_acl_s {
         enum xml_private_flags mode;
         char *xpath;
 } xml_acl_t;
 
 /* *INDENT-OFF* */
 
 static filter_t filter[] = {
     { 0, XML_ATTR_ORIGIN },
     { 0, XML_CIB_ATTR_WRITTEN },
     { 0, XML_ATTR_UPDATE_ORIG },
     { 0, XML_ATTR_UPDATE_CLIENT },
     { 0, XML_ATTR_UPDATE_USER },
 };
 /* *INDENT-ON* */
 
-static struct schema_s *known_schemas = NULL;
-static int xml_schema_max = 0;
-
 static xmlNode *subtract_xml_comment(xmlNode * parent, xmlNode * left, xmlNode * right, gboolean * changed);
 static xmlNode *find_xml_comment(xmlNode * root, xmlNode * search_comment);
 static int add_xml_comment(xmlNode * parent, xmlNode * target, xmlNode * update);
 static bool __xml_acl_check(xmlNode *xml, const char *name, enum xml_private_flags mode);
 const char *__xml_acl_to_text(enum xml_private_flags flags);
 
-static int
-xml_latest_schema_index(void)
-{
-    return xml_schema_max - 4;
-}
-
-static int
-xml_minimum_schema_index(void)
-{
-    static int best = 0;
-    if(best == 0) {
-        int lpc = 0;
-        float target = 0.0;
-
-        best = xml_latest_schema_index();
-        target = floor(known_schemas[best].version);
-
-        for(lpc = best; lpc > 0; lpc--) {
-            if(known_schemas[lpc].version < target) {
-                return best;
-            } else {
-                best = lpc;
-            }
-        }
-        best = xml_latest_schema_index();
-    }
-    return best;
-}
-
-const char *
-xml_latest_schema(void)
-{
-    return get_schema_name(xml_latest_schema_index());
-}
-
 #define CHUNK_SIZE 1024
 static inline bool TRACKING_CHANGES(xmlNode *xml)
 {
     if(xml == NULL || xml->doc == NULL || xml->doc->_private == NULL) {
         return FALSE;
     } else if(is_set(((xml_private_t *)xml->doc->_private)->flags, xpf_tracking)) {
         return TRUE;
     }
     return FALSE;
 }
 
 #define buffer_print(buffer, max, offset, fmt, args...) do {            \
         int rc = (max);                                                 \
         if(buffer) {                                                    \
             rc = snprintf((buffer) + (offset), (max) - (offset), fmt, ##args); \
         }                                                               \
         if(buffer && rc < 0) {                                          \
             crm_perror(LOG_ERR, "snprintf failed at offset %d", offset); \
             (buffer)[(offset)] = 0;                                     \
         } else if(rc >= ((max) - (offset))) {                           \
             char *tmp = NULL;                                           \
             (max) = QB_MAX(CHUNK_SIZE, (max) * 2);                      \
             tmp = realloc_safe((buffer), (max) + 1);                         \
             CRM_ASSERT(tmp);                                            \
             (buffer) = tmp;                                             \
         } else {                                                        \
             offset += rc;                                               \
             break;                                                      \
         }                                                               \
     } while(1);
 
 static void
 insert_prefix(int options, char **buffer, int *offset, int *max, int depth)
 {
     if (options & xml_log_option_formatted) {
         size_t spaces = 2 * depth;
 
         if ((*buffer) == NULL || spaces >= ((*max) - (*offset))) {
             (*max) = QB_MAX(CHUNK_SIZE, (*max) * 2);
             (*buffer) = realloc_safe((*buffer), (*max) + 1);
         }
         memset((*buffer) + (*offset), ' ', spaces);
         (*offset) += spaces;
     }
 }
 
-static const char *
-get_schema_root(void)
-{
-    static const char *base = NULL;
-
-    if (base == NULL) {
-        base = getenv("PCMK_schema_directory");
-    }
-    if (base == NULL || strlen(base) == 0) {
-        base = CRM_DTD_DIRECTORY;
-    }
-    return base;
-}
-
-static char *
-get_schema_path(const char *name, const char *file)
-{
-    const char *base = get_schema_root();
-
-    if(file) {
-        return crm_strdup_printf("%s/%s", base, file);
-    }
-    return crm_strdup_printf("%s/%s.rng", base, name);
-}
-
-static int schema_filter(const struct dirent * a)
-{
-    int rc = 0;
-    float version = 0;
-
-    if(strstr(a->d_name, "pacemaker-") != a->d_name) {
-        /* crm_trace("%s - wrong prefix", a->d_name); */
-
-    } else if (!crm_ends_with(a->d_name, ".rng")) {
-        /* crm_trace("%s - wrong suffix", a->d_name); */
-
-    } else if(sscanf(a->d_name, "pacemaker-%f.rng", &version) == 0) {
-        /* crm_trace("%s - wrong format", a->d_name); */
-
-    } else if(strcmp(a->d_name, "pacemaker-1.1.rng") == 0) {
-        /* "-1.1" was used for what later became "-next" */
-        /* crm_trace("%s - hack", a->d_name); */
-
-    } else {
-        /* crm_debug("%s - candidate", a->d_name); */
-        rc = 1;
-    }
-
-    return rc;
-}
-
-static int schema_sort(const struct dirent ** a, const struct dirent **b)
-{
-    int rc = 0;
-    float a_version = 0.0;
-    float b_version = 0.0;
-
-    sscanf(a[0]->d_name, "pacemaker-%f.rng", &a_version);
-    sscanf(b[0]->d_name, "pacemaker-%f.rng", &b_version);
-
-    if(a_version > b_version) {
-        rc = 1;
-    } else if(a_version < b_version) {
-        rc = -1;
-    }
-
-    /* crm_trace("%s (%f) vs. %s (%f) : %d", a[0]->d_name, a_version, b[0]->d_name, b_version, rc); */
-    return rc;
-}
-
-static void __xml_schema_add(
-    int type, float version, const char *name, const char *location, const char *transform, int after_transform)
-{
-    int last = xml_schema_max;
-
-    xml_schema_max++;
-    known_schemas = realloc_safe(known_schemas, xml_schema_max*sizeof(struct schema_s));
-    CRM_ASSERT(known_schemas != NULL);
-    memset(known_schemas+last, 0, sizeof(struct schema_s));
-    known_schemas[last].type = type;
-    known_schemas[last].after_transform = after_transform;
-
-    if(version > 0.0) {
-        known_schemas[last].version = version;
-        known_schemas[last].name = crm_strdup_printf("pacemaker-%.1f", version);
-        known_schemas[last].location = crm_strdup_printf("%s.rng", known_schemas[last].name);
-
-    } else {
-        char dummy[1024];
-        CRM_ASSERT(name);
-        CRM_ASSERT(location);
-        sscanf(name, "%[^-]-%f", dummy, &version);
-        known_schemas[last].version = version;
-        known_schemas[last].name = strdup(name);
-        known_schemas[last].location = strdup(location);
-    }
-
-    if(transform) {
-        known_schemas[last].transform = strdup(transform);
-    }
-    if(after_transform == 0) {
-        after_transform = xml_schema_max;  /* upgrade is a one-way */
-    }
-    known_schemas[last].after_transform = after_transform;
-
-    if(known_schemas[last].after_transform < 0) {
-        crm_debug("Added supported schema %d: %s (%s)",
-                  last, known_schemas[last].name, known_schemas[last].location);
-
-    } else if(known_schemas[last].transform) {
-        crm_debug("Added supported schema %d: %s (%s upgrades to %d with %s)",
-                  last, known_schemas[last].name, known_schemas[last].location,
-                  known_schemas[last].after_transform,
-                  known_schemas[last].transform);
-
-    } else {
-        crm_debug("Added supported schema %d: %s (%s upgrades to %d)",
-                  last, known_schemas[last].name, known_schemas[last].location,
-                  known_schemas[last].after_transform);
-    }
-}
-
-
-static int __xml_build_schema_list(void) 
-{
-    int lpc, max;
-    const char *base = get_schema_root();
-    struct dirent **namelist = NULL;
-
-    max = scandir(base, &namelist, schema_filter, schema_sort);
-    __xml_schema_add(1, 0.0, "pacemaker-0.6", "crm.dtd", "upgrade06.xsl", 3);
-    __xml_schema_add(1, 0.0, "transitional-0.6", "crm-transitional.dtd", "upgrade06.xsl", 3);
-    __xml_schema_add(2, 0.0, "pacemaker-0.7", "pacemaker-1.0.rng", NULL, 0);
-
-    if (max < 0) {
-        crm_notice("scandir(%s) failed: %s (%d)", base, strerror(errno), errno);
-
-    } else {
-        for (lpc = 0; lpc < max; lpc++) {
-            int next = 0;
-            float version = 0.0;
-            char *transform = NULL;
-
-            sscanf(namelist[lpc]->d_name, "pacemaker-%f.rng", &version);
-            if((lpc + 1) < max) {
-                float next_version = 0.0;
-
-                sscanf(namelist[lpc+1]->d_name, "pacemaker-%f.rng", &next_version);
-
-                if(floor(version) < floor(next_version)) {
-                    struct stat s;
-                    char *xslt = NULL;
-
-                    transform = crm_strdup_printf("upgrade-%.1f.xsl", version);
-                    xslt = get_schema_path(NULL, transform);
-                    if(stat(xslt, &s) != 0) {
-                        crm_err("Transform %s not found", xslt);
-                        free(xslt);
-                        __xml_schema_add(2, version, NULL, NULL, NULL, -1);
-                        break;
-                    } else {
-                        free(xslt);
-                    }
-                }
-
-            } else {
-                next = -1;
-            }
-            __xml_schema_add(2, version, NULL, NULL, transform, next);
-            free(namelist[lpc]);
-            free(transform);
-        }
-    }
-
-    /* 1.1 was the old name for -next */
-    __xml_schema_add(2, 0.0, "pacemaker-1.1", "pacemaker-next.rng", NULL, 0);
-    __xml_schema_add(2, 0.0, "pacemaker-next", "pacemaker-next.rng", NULL, -1);
-    __xml_schema_add(0, 0.0, "none", "N/A", NULL, -1);
-    free(namelist);
-    return TRUE;
-}
-
 static void
 set_parent_flag(xmlNode *xml, long flag) 
 {
 
     for(; xml; xml = xml->parent) {
         xml_private_t *p = xml->_private;
 
         if(p == NULL) {
             /* During calls to xmlDocCopyNode(), _private will be unset for parent nodes */
         } else {
             p->flags |= flag;
             /* crm_trace("Setting flag %x due to %s[@id=%s]", flag, xml->name, ID(xml)); */
         }
     }
 }
 
 static void
 set_doc_flag(xmlNode *xml, long flag) 
 {
 
     if(xml && xml->doc && xml->doc->_private){
         /* During calls to xmlDocCopyNode(), xml->doc may be unset */
         xml_private_t *p = xml->doc->_private;
 
         p->flags |= flag;
         /* crm_trace("Setting flag %x due to %s[@id=%s]", flag, xml->name, ID(xml)); */
     }
 }
 
 static void
 __xml_node_dirty(xmlNode *xml) 
 {
     set_doc_flag(xml, xpf_dirty);
     set_parent_flag(xml, xpf_dirty);
 }
 
 static void
 __xml_node_clean(xmlNode *xml) 
 {
     xmlNode *cIter = NULL;
     xml_private_t *p = xml->_private;
 
     if(p) {
         p->flags = 0;
     }
 
     for (cIter = __xml_first_child(xml); cIter != NULL; cIter = __xml_next(cIter)) {
         __xml_node_clean(cIter);
     }
 }
 
 static void
 crm_node_created(xmlNode *xml) 
 {
     xmlNode *cIter = NULL;
     xml_private_t *p = xml->_private;
 
     if(p && TRACKING_CHANGES(xml)) {
         if(is_not_set(p->flags, xpf_created)) {
             p->flags |= xpf_created;
             __xml_node_dirty(xml);
         }
 
         for (cIter = __xml_first_child(xml); cIter != NULL; cIter = __xml_next(cIter)) {
            crm_node_created(cIter);
         }
     }
 }
 
 static void
 crm_attr_dirty(xmlAttr *a) 
 {
     xmlNode *parent = a->parent;
     xml_private_t *p = NULL;
 
     p = a->_private;
     p->flags |= (xpf_dirty|xpf_modified);
     p->flags = (p->flags & ~xpf_deleted);
     /* crm_trace("Setting flag %x due to %s[@id=%s, @%s=%s]", */
     /*           xpf_dirty, parent?parent->name:NULL, ID(parent), a->name, a->children->content); */
 
     __xml_node_dirty(parent);
 }
 
 int get_tag_name(const char *input, size_t offset, size_t max);
 int get_attr_name(const char *input, size_t offset, size_t max);
 int get_attr_value(const char *input, size_t offset, size_t max);
 gboolean can_prune_leaf(xmlNode * xml_node);
 
 void diff_filter_context(int context, int upper_bound, int lower_bound,
                          xmlNode * xml_node, xmlNode * parent);
 int in_upper_context(int depth, int context, xmlNode * xml_node);
 int add_xml_object(xmlNode * parent, xmlNode * target, xmlNode * update, gboolean as_diff);
 
 static inline const char *
 crm_attr_value(xmlAttr * attr)
 {
     if (attr == NULL || attr->children == NULL) {
         return NULL;
     }
     return (const char *)attr->children->content;
 }
 
 static inline xmlAttr *
 crm_first_attr(xmlNode * xml)
 {
     if (xml == NULL) {
         return NULL;
     }
     return xml->properties;
 }
 
 #define XML_PRIVATE_MAGIC (long) 0x81726354
 
 static void
 __xml_acl_free(void *data)
 {
     if(data) {
         xml_acl_t *acl = data;
 
         free(acl->xpath);
         free(acl);
     }
 }
 
 static void
 __xml_private_clean(xml_private_t *p)
 {
     if(p) {
         CRM_ASSERT(p->check == XML_PRIVATE_MAGIC);
 
         free(p->user);
         p->user = NULL;
 
         if(p->acls) {
             g_list_free_full(p->acls, __xml_acl_free);
             p->acls = NULL;
         }
 
         if(p->deleted_paths) {
             g_list_free_full(p->deleted_paths, free);
             p->deleted_paths = NULL;
         }
     }
 }
 
 
 static void
 __xml_private_free(xml_private_t *p)
 {
     __xml_private_clean(p);
     free(p);
 }
 
 static void
 pcmkDeregisterNode(xmlNodePtr node)
 {
     __xml_private_free(node->_private);
 }
 
 static void
 pcmkRegisterNode(xmlNodePtr node)
 {
     xml_private_t *p = NULL;
 
     switch(node->type) {
         case XML_ELEMENT_NODE:
         case XML_DOCUMENT_NODE:
         case XML_ATTRIBUTE_NODE:
         case XML_COMMENT_NODE:
             p = calloc(1, sizeof(xml_private_t));
             p->check = XML_PRIVATE_MAGIC;
             /* Flags will be reset if necessary when tracking is enabled */
             p->flags |= (xpf_dirty|xpf_created);
             node->_private = p;
             break;
         case XML_TEXT_NODE:
         case XML_DTD_NODE:
         case XML_CDATA_SECTION_NODE:
             break;
         default:
             /* Ignore */
             crm_trace("Ignoring %p %d", node, node->type);
             CRM_LOG_ASSERT(node->type == XML_ELEMENT_NODE);
             break;
     }
 
     if(p && TRACKING_CHANGES(node)) {
         /* XML_ELEMENT_NODE doesn't get picked up here, node->doc is
          * not hooked up at the point we are called
          */
         set_doc_flag(node, xpf_dirty);
         __xml_node_dirty(node);
     }
 }
 
 static xml_acl_t *
 __xml_acl_create(xmlNode * xml, xmlNode *target, enum xml_private_flags mode)
 {
     xml_acl_t *acl = NULL;
 
     xml_private_t *p = NULL;
     const char *tag = crm_element_value(xml, XML_ACL_ATTR_TAG);
     const char *ref = crm_element_value(xml, XML_ACL_ATTR_REF);
     const char *xpath = crm_element_value(xml, XML_ACL_ATTR_XPATH);
 
     if(tag == NULL) {
         /* Compatibility handling for pacemaker < 1.1.12 */
         tag = crm_element_value(xml, XML_ACL_ATTR_TAGv1);
     }
     if(ref == NULL) {
         /* Compatibility handling for pacemaker < 1.1.12 */
         ref = crm_element_value(xml, XML_ACL_ATTR_REFv1);
     }
 
     if(target == NULL || target->doc == NULL || target->doc->_private == NULL){
         CRM_ASSERT(target);
         CRM_ASSERT(target->doc);
         CRM_ASSERT(target->doc->_private);
         return NULL;
 
     } else if (tag == NULL && ref == NULL && xpath == NULL) {
         crm_trace("No criteria %p", xml);
         return NULL;
     }
 
     p = target->doc->_private;
     acl = calloc(1, sizeof(xml_acl_t));
     if (acl) {
         const char *attr = crm_element_value(xml, XML_ACL_ATTR_ATTRIBUTE);
 
         acl->mode = mode;
         if(xpath) {
             acl->xpath = strdup(xpath);
             crm_trace("Using xpath: %s", acl->xpath);
 
         } else {
             int offset = 0;
             char buffer[XML_BUFFER_SIZE];
 
             if(tag) {
                 offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "//%s", tag);
             } else {
                 offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "//*");
             }
 
             if(ref || attr) {
                 offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "[");
             }
 
             if(ref) {
                 offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "@id='%s'", ref);
             }
 
             if(ref && attr) {
                 offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, " and ");
             }
 
             if(attr) {
                 offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "@%s", attr);
             }
 
             if(ref || attr) {
                 offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "]");
             }
 
             CRM_LOG_ASSERT(offset > 0);
             acl->xpath = strdup(buffer);
             crm_trace("Built xpath: %s", acl->xpath);
         }
 
         p->acls = g_list_append(p->acls, acl);
     }
     return acl;
 }
 
 static gboolean
 __xml_acl_parse_entry(xmlNode * acl_top, xmlNode * acl_entry, xmlNode *target)
 {
     xmlNode *child = NULL;
 
     for (child = __xml_first_child(acl_entry); child; child = __xml_next(child)) {
         const char *tag = crm_element_name(child);
         const char *kind = crm_element_value(child, XML_ACL_ATTR_KIND);
 
         if (strcmp(XML_ACL_TAG_PERMISSION, tag) == 0){
             tag = kind;
         }
 
         crm_trace("Processing %s %p", tag, child);
         if(tag == NULL) {
             CRM_ASSERT(tag != NULL);
 
         } else if (strcmp(XML_ACL_TAG_ROLE_REF, tag) == 0
                    || strcmp(XML_ACL_TAG_ROLE_REFv1, tag) == 0) {
             const char *ref_role = crm_element_value(child, XML_ATTR_ID);
 
             if (ref_role) {
                 xmlNode *role = NULL;
 
                 for (role = __xml_first_child(acl_top); role; role = __xml_next(role)) {
                     if (strcmp(XML_ACL_TAG_ROLE, (const char *)role->name) == 0) {
                         const char *role_id = crm_element_value(role, XML_ATTR_ID);
 
                         if (role_id && strcmp(ref_role, role_id) == 0) {
                             crm_debug("Unpacking referenced role: %s", role_id);
                             __xml_acl_parse_entry(acl_top, role, target);
                             break;
                         }
                     }
                 }
             }
 
         } else if (strcmp(XML_ACL_TAG_READ, tag) == 0) {
             __xml_acl_create(child, target, xpf_acl_read);
 
         } else if (strcmp(XML_ACL_TAG_WRITE, tag) == 0) {
             __xml_acl_create(child, target, xpf_acl_write);
 
         } else if (strcmp(XML_ACL_TAG_DENY, tag) == 0) {
             __xml_acl_create(child, target, xpf_acl_deny);
 
         } else {
             crm_warn("Unknown ACL entry: %s/%s", tag, kind);
         }
     }
 
     return TRUE;
 }
 
 /*
     <acls>
       <acl_target id="l33t-haxor"><role id="auto-l33t-haxor"/></acl_target>
       <acl_role id="auto-l33t-haxor">
         <acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
       </acl_role>
       <acl_target id="niceguy">
         <role id="observer"/>
       </acl_target>
       <acl_role id="observer">
         <acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
         <acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
         <acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
       </acl_role>
       <acl_target id="badidea"><role id="auto-badidea"/></acl_target>
       <acl_role id="auto-badidea">
         <acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
         <acl_permission id="badidea-resources-2" kind="deny" reference="dummy-meta_attributes"/>
       </acl_role>
     </acls>
 */
 
 const char *
 __xml_acl_to_text(enum xml_private_flags flags)
 {
     if(is_set(flags, xpf_acl_deny)) {
         return "deny";
     }
     if(is_set(flags, xpf_acl_write)) {
         return "read/write";
     }
     if(is_set(flags, xpf_acl_read)) {
         return "read";
     }
 
     return "none";
 }
 
 static void
 __xml_acl_apply(xmlNode *xml) 
 {
     GListPtr aIter = NULL;
     xml_private_t *p = NULL;
     xmlXPathObjectPtr xpathObj = NULL;
 
     if(xml_acl_enabled(xml) == FALSE) {
         p = xml->doc->_private;
         crm_trace("Not applying ACLs for %s", p->user);
         return;
     }
 
     p = xml->doc->_private;
     for(aIter = p->acls; aIter != NULL; aIter = aIter->next) {
         int max = 0, lpc = 0;
         xml_acl_t *acl = aIter->data;
 
         xpathObj = xpath_search(xml, acl->xpath);
         max = numXpathResults(xpathObj);
 
         for(lpc = 0; lpc < max; lpc++) {
             xmlNode *match = getXpathResult(xpathObj, lpc);
             char *path = xml_get_path(match);
 
             p = match->_private;
             crm_trace("Applying %x to %s for %s", acl->mode, path, acl->xpath);
 
 #ifdef SUSE_ACL_COMPAT
             if(is_not_set(p->flags, acl->mode)) {
                 if(is_set(p->flags, xpf_acl_read)
                    || is_set(p->flags, xpf_acl_write)
                    || is_set(p->flags, xpf_acl_deny)) {
                     crm_config_warn("Configuration element %s is matched by multiple ACL rules, only the first applies ('%s' wins over '%s')",
                                     path, __xml_acl_to_text(p->flags), __xml_acl_to_text(acl->mode));
                     free(path);
                     continue;
                 }
             }
 #endif
 
             p->flags |= acl->mode;
             free(path);
         }
         crm_trace("Now enforcing ACL: %s (%d matches)", acl->xpath, max);
         freeXpathObject(xpathObj);
     }
 
     p = xml->_private;
     if(is_not_set(p->flags, xpf_acl_read) && is_not_set(p->flags, xpf_acl_write)) {
         p->flags |= xpf_acl_deny;
         p = xml->doc->_private;
         crm_info("Enforcing default ACL for %s to %s", p->user, crm_element_name(xml));
     }
 
 }
 
 static void
 __xml_acl_unpack(xmlNode *source, xmlNode *target, const char *user)
 {
 #if ENABLE_ACL
     xml_private_t *p = NULL;
 
     if(target == NULL || target->doc == NULL || target->doc->_private == NULL) {
         return;
     }
 
     p = target->doc->_private;
     if(pcmk_acl_required(user) == FALSE) {
         crm_trace("no acls needed for '%s'", user);
 
     } else if(p->acls == NULL) {
         xmlNode *acls = get_xpath_object("//"XML_CIB_TAG_ACLS, source, LOG_TRACE);
 
         free(p->user);
         p->user = strdup(user);
 
         if(acls) {
             xmlNode *child = NULL;
 
             for (child = __xml_first_child(acls); child; child = __xml_next(child)) {
                 const char *tag = crm_element_name(child);
 
                 if (strcmp(tag, XML_ACL_TAG_USER) == 0 || strcmp(tag, XML_ACL_TAG_USERv1) == 0) {
                     const char *id = crm_element_value(child, XML_ATTR_ID);
 
                     if(id && strcmp(id, user) == 0) {
                         crm_debug("Unpacking ACLs for %s", id);
                         __xml_acl_parse_entry(acls, child, target);
                     }
                 }
             }
         }
     }
 #endif
 }
 
 static inline bool
 __xml_acl_mode_test(enum xml_private_flags allowed, enum xml_private_flags requested)
 {
     if(is_set(allowed, xpf_acl_deny)) {
         return FALSE;
 
     } else if(is_set(allowed, requested)) {
         return TRUE;
 
     } else if(is_set(requested, xpf_acl_read) && is_set(allowed, xpf_acl_write)) {
         return TRUE;
 
     } else if(is_set(requested, xpf_acl_create) && is_set(allowed, xpf_acl_write)) {
         return TRUE;
 
     } else if(is_set(requested, xpf_acl_create) && is_set(allowed, xpf_created)) {
         return TRUE;
     }
     return FALSE;
 }
 
 /* rc = TRUE if orig_cib has been filtered
  * That means '*result' rather than 'xml' should be exploited afterwards
  */
 static bool
 __xml_purge_attributes(xmlNode *xml)
 {
     xmlNode *child = NULL;
     xmlAttr *xIter = NULL;
     bool readable_children = FALSE;
     xml_private_t *p = xml->_private;
 
     if(__xml_acl_mode_test(p->flags, xpf_acl_read)) {
         crm_trace("%s[@id=%s] is readable", crm_element_name(xml), ID(xml));
         return TRUE;
     }
 
     xIter = crm_first_attr(xml);
     while(xIter != NULL) {
         xmlAttr *tmp = xIter;
         const char *prop_name = (const char *)xIter->name;
 
         xIter = xIter->next;
         if (strcmp(prop_name, XML_ATTR_ID) == 0) {
             continue;
         }
 
         xmlUnsetProp(xml, tmp->name);
     }
 
     child = __xml_first_child(xml);
     while ( child != NULL ) {
         xmlNode *tmp = child;
 
         child = __xml_next(child);
         readable_children |= __xml_purge_attributes(tmp);
     }
 
     if(readable_children == FALSE) {
         free_xml(xml); /* Nothing readable under here, purge completely */
     }
     return readable_children;
 }
 
 bool
 xml_acl_filtered_copy(const char *user, xmlNode* acl_source, xmlNode *xml, xmlNode ** result)
 {
     GListPtr aIter = NULL;
     xmlNode *target = NULL;
     xml_private_t *p = NULL;
     xml_private_t *doc = NULL;
 
     *result = NULL;
     if(xml == NULL || pcmk_acl_required(user) == FALSE) {
         crm_trace("no acls needed for '%s'", user);
         return FALSE;
     }
 
     crm_trace("filtering copy of %p for '%s'", xml, user);
     target = copy_xml(xml);
     if(target == NULL) {
         return TRUE;
     }
 
     __xml_acl_unpack(acl_source, target, user);
     set_doc_flag(target, xpf_acl_enabled);
     __xml_acl_apply(target);
 
     doc = target->doc->_private;
     for(aIter = doc->acls; aIter != NULL && target; aIter = aIter->next) {
         int max = 0;
         xml_acl_t *acl = aIter->data;
 
         if(acl->mode != xpf_acl_deny) {
             /* Nothing to do */
 
         } else if(acl->xpath) {
             int lpc = 0;
             xmlXPathObjectPtr xpathObj = xpath_search(target, acl->xpath);
 
             max = numXpathResults(xpathObj);
             for(lpc = 0; lpc < max; lpc++) {
                 xmlNode *match = getXpathResult(xpathObj, lpc);
 
                 crm_trace("Purging attributes from %s", acl->xpath);
                 if(__xml_purge_attributes(match) == FALSE && match == target) {
                     crm_trace("No access to the entire document for %s", user);
                     freeXpathObject(xpathObj);
                     return TRUE;
                 }
             }
             crm_trace("Enforced ACL %s (%d matches)", acl->xpath, max);
             freeXpathObject(xpathObj);
         }
     }
 
     p = target->_private;
     if(is_set(p->flags, xpf_acl_deny) && __xml_purge_attributes(target) == FALSE) {
         crm_trace("No access to the entire document for %s", user);
         return TRUE;
     }
 
     if(doc->acls) {
         g_list_free_full(doc->acls, __xml_acl_free);
         doc->acls = NULL;
 
     } else {
         crm_trace("Ordinary user '%s' cannot access the CIB without any defined ACLs", doc->user);
         free_xml(target);
         target = NULL;
     }
 
     if(target) {
         *result = target;
     }
 
     return TRUE;
 }
 
 static void
 __xml_acl_post_process(xmlNode * xml)
 {
     xmlNode *cIter = __xml_first_child(xml);
     xml_private_t *p = xml->_private;
 
     if(is_set(p->flags, xpf_created)) {
         xmlAttr *xIter = NULL;
         char *path = xml_get_path(xml);
 
         /* Always allow new scaffolding, ie. node with no attributes or only an 'id'
          * Except in the ACLs section
          */
 
         for (xIter = crm_first_attr(xml); xIter != NULL; xIter = xIter->next) {
             const char *prop_name = (const char *)xIter->name;
 
             if (strcmp(prop_name, XML_ATTR_ID) == 0 && strstr(path, "/"XML_CIB_TAG_ACLS"/") == NULL) {
                 /* Delay the acl check */
                 continue;
 
             } else if(__xml_acl_check(xml, NULL, xpf_acl_write)) {
                 crm_trace("Creation of %s=%s is allowed", crm_element_name(xml), ID(xml));
                 break;
 
             } else {
                 crm_trace("Cannot add new node %s at %s", crm_element_name(xml), path);
 
                 if(xml != xmlDocGetRootElement(xml->doc)) {
                     xmlUnlinkNode(xml);
                     xmlFreeNode(xml);
                 }
                 free(path);
                 return;
             }
         }
         free(path);
     }
 
     while (cIter != NULL) {
         xmlNode *child = cIter;
         cIter = __xml_next(cIter); /* In case it is free'd */
         __xml_acl_post_process(child);
     }
 }
 
 bool
 xml_acl_denied(xmlNode *xml) 
 {
     if(xml && xml->doc && xml->doc->_private){
         xml_private_t *p = xml->doc->_private;
 
         return is_set(p->flags, xpf_acl_denied);
     }
     return FALSE;
 }
 
 void
 xml_acl_disable(xmlNode *xml)
 {
     if(xml_acl_enabled(xml)) {
         xml_private_t *p = xml->doc->_private;
 
         /* Catch anything that was created but shouldn't have been */
         __xml_acl_apply(xml);
         __xml_acl_post_process(xml);
         clear_bit(p->flags, xpf_acl_enabled);
     }
 }
 
 bool
 xml_acl_enabled(xmlNode *xml)
 {
     if(xml && xml->doc && xml->doc->_private){
         xml_private_t *p = xml->doc->_private;
 
         return is_set(p->flags, xpf_acl_enabled);
     }
     return FALSE;
 }
 
 void
 xml_track_changes(xmlNode * xml, const char *user, xmlNode *acl_source, bool enforce_acls) 
 {
     xml_accept_changes(xml);
     crm_trace("Tracking changes%s to %p", enforce_acls?" with ACLs":"", xml);
     set_doc_flag(xml, xpf_tracking);
     if(enforce_acls) {
         if(acl_source == NULL) {
             acl_source = xml;
         }
         set_doc_flag(xml, xpf_acl_enabled);
         __xml_acl_unpack(acl_source, xml, user);
         __xml_acl_apply(xml);
     }
 }
 
 bool xml_tracking_changes(xmlNode * xml)
 {
     if(xml == NULL) {
         return FALSE;
 
     } else if(is_set(((xml_private_t *)xml->doc->_private)->flags, xpf_tracking)) {
         return TRUE;
     }
     return FALSE;
 }
 
 bool xml_document_dirty(xmlNode *xml) 
 {
     if(xml != NULL && xml->doc && xml->doc->_private) {
         xml_private_t *doc = xml->doc->_private;
 
         return is_set(doc->flags, xpf_dirty);
     }
     return FALSE;
 }
 
 /*
 <diff format="2.0">
   <version>
     <source admin_epoch="1" epoch="2" num_updates="3"/>
     <target admin_epoch="1" epoch="3" num_updates="0"/>
   </version>
   <change operation="add" xpath="/cib/configuration/nodes">
     <node id="node2" uname="node2" description="foo"/>
   </change>
   <change operation="add" xpath="/cib/configuration/nodes/node[node2]">
     <instance_attributes id="nodes-node"><!-- NOTE: can be a full tree -->
       <nvpair id="nodes-node2-ram" name="ram" value="1024M"/>
     </instance_attributes>
   </change>
   <change operation="update" xpath="/cib/configuration/nodes[@id='node2']">
     <change-list>
       <change-attr operation="set" name="type" value="member"/>
       <change-attr operation="unset" name="description"/>
     </change-list>
     <change-result>
       <node id="node2" uname="node2" type="member"/><!-- NOTE: not recursive -->
     </change-result>
   </change>
   <change operation="delete" xpath="/cib/configuration/nodes/node[@id='node3'] /">
   <change operation="update" xpath="/cib/configuration/resources/group[@id='g1']">
     <change-list>
       <change-attr operation="set" name="description" value="some grabage here"/>
     </change-list>
     <change-result>
       <group id="g1" description="some grabage here"/><!-- NOTE: not recursive -->
     </change-result>
   </change>
   <change operation="update" xpath="/cib/status/node_state[@id='node2]/lrm[@id='node2']/lrm_resources/lrm_resource[@id='Fence']">
     <change-list>
       <change-attr operation="set" name="oper" value="member"/>
       <change-attr operation="set" name="operation_key" value="Fence_start_0"/>
       <change-attr operation="set" name="operation" value="start"/>
       <change-attr operation="set" name="transition-key" value="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"/>
       <change-attr operation="set" name="transition-magic" value="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"/>
       <change-attr operation="set" name="call-id" value="2"/>
       <change-attr operation="set" name="rc-code" value="0"/>
     </change-list>
     <change-result>
       <lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate"  transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
     </change-result>
   </change>
 </diff>
  */
 static int __xml_offset(xmlNode *xml) 
 {
     int position = 0;
     xmlNode *cIter = NULL;
 
     for(cIter = xml; cIter->prev; cIter = cIter->prev) {
         xml_private_t *p = ((xmlNode*)cIter->prev)->_private;
 
         if(is_not_set(p->flags, xpf_skip)) {
             position++;
         }
     }
 
     return position;
 }
 
 static int __xml_offset_no_deletions(xmlNode *xml) 
 {
     int position = 0;
     xmlNode *cIter = NULL;
 
     for(cIter = xml; cIter->prev; cIter = cIter->prev) {
         xml_private_t *p = ((xmlNode*)cIter->prev)->_private;
 
         if(is_not_set(p->flags, xpf_deleted)) {
             position++;
         }
     }
 
     return position;
 }
 
 static void
 __xml_build_changes(xmlNode * xml, xmlNode *patchset)
 {
     xmlNode *cIter = NULL;
     xmlAttr *pIter = NULL;
     xmlNode *change = NULL;
     xml_private_t *p = xml->_private;
 
     if(patchset && is_set(p->flags, xpf_created)) {
         int offset = 0;
         char buffer[XML_BUFFER_SIZE];
 
         if(__get_prefix(NULL, xml->parent, buffer, offset) > 0) {
             int position = __xml_offset_no_deletions(xml);
 
             change = create_xml_node(patchset, XML_DIFF_CHANGE);
 
             crm_xml_add(change, XML_DIFF_OP, "create");
             crm_xml_add(change, XML_DIFF_PATH, buffer);
             crm_xml_add_int(change, XML_DIFF_POSITION, position);
             add_node_copy(change, xml);
         }
 
         return;
     }
 
     for (pIter = crm_first_attr(xml); pIter != NULL; pIter = pIter->next) {
         xmlNode *attr = NULL;
 
         p = pIter->_private;
         if(is_not_set(p->flags, xpf_deleted) && is_not_set(p->flags, xpf_dirty)) {
             continue;
         }
 
         if(change == NULL) {
             int offset = 0;
             char buffer[XML_BUFFER_SIZE];
 
             if(__get_prefix(NULL, xml, buffer, offset) > 0) {
                 change = create_xml_node(patchset, XML_DIFF_CHANGE);
 
                 crm_xml_add(change, XML_DIFF_OP, "modify");
                 crm_xml_add(change, XML_DIFF_PATH, buffer);
 
                 change = create_xml_node(change, XML_DIFF_LIST);
             }
         }
 
         attr = create_xml_node(change, XML_DIFF_ATTR);
 
         crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, (const char *)pIter->name);
         if(p->flags & xpf_deleted) {
             crm_xml_add(attr, XML_DIFF_OP, "unset");
 
         } else {
             const char *value = crm_element_value(xml, (const char *)pIter->name);
 
             crm_xml_add(attr, XML_DIFF_OP, "set");
             crm_xml_add(attr, XML_NVPAIR_ATTR_VALUE, value);
         }
     }
 
     if(change) {
         xmlNode *result = NULL;
 
         change = create_xml_node(change->parent, XML_DIFF_RESULT);
         result = create_xml_node(change, (const char *)xml->name);
 
         for (pIter = crm_first_attr(xml); pIter != NULL; pIter = pIter->next) {
             const char *value = crm_element_value(xml, (const char *)pIter->name);
 
             p = pIter->_private;
             if (is_not_set(p->flags, xpf_deleted)) {
                 crm_xml_add(result, (const char *)pIter->name, value);
             }
         }
     }
 
     for (cIter = __xml_first_child(xml); cIter != NULL; cIter = __xml_next(cIter)) {
         __xml_build_changes(cIter, patchset);
     }
 
     p = xml->_private;
     if(patchset && is_set(p->flags, xpf_moved)) {
         int offset = 0;
         char buffer[XML_BUFFER_SIZE];
 
         crm_trace("%s.%s moved to position %d", xml->name, ID(xml), __xml_offset(xml));
         if(__get_prefix(NULL, xml, buffer, offset) > 0) {
             change = create_xml_node(patchset, XML_DIFF_CHANGE);
 
             crm_xml_add(change, XML_DIFF_OP, "move");
             crm_xml_add(change, XML_DIFF_PATH, buffer);
             crm_xml_add_int(change, XML_DIFF_POSITION, __xml_offset_no_deletions(xml));
         }
     }
 }
 
 static void
 __xml_accept_changes(xmlNode * xml)
 {
     xmlNode *cIter = NULL;
     xmlAttr *pIter = NULL;
     xml_private_t *p = xml->_private;
 
     p->flags = xpf_none;
     pIter = crm_first_attr(xml);
 
     while (pIter != NULL) {
         const xmlChar *name = pIter->name;
 
         p = pIter->_private;
         pIter = pIter->next;
 
         if(p->flags & xpf_deleted) {
             xml_remove_prop(xml, (const char *)name);
 
         } else {
             p->flags = xpf_none;
         }
     }
 
     for (cIter = __xml_first_child(xml); cIter != NULL; cIter = __xml_next(cIter)) {
         __xml_accept_changes(cIter);
     }
 }
 
 static bool
 is_config_change(xmlNode *xml)
 {
     GListPtr gIter = NULL;
     xml_private_t *p = NULL;
     xmlNode *config = first_named_child(xml, XML_CIB_TAG_CONFIGURATION);
 
     if(config) {
         p = config->_private;
     }
     if(p && is_set(p->flags, xpf_dirty)) {
         return TRUE;
     }
 
     if(xml->doc && xml->doc->_private) {
         p = xml->doc->_private;
         for(gIter = p->deleted_paths; gIter; gIter = gIter->next) {
             char *path = gIter->data;
 
             if(strstr(path, "/"XML_TAG_CIB"/"XML_CIB_TAG_CONFIGURATION) != NULL) {
                 return TRUE;
             }
         }
     }
 
     return FALSE;
 }
 
 static void
 xml_repair_v1_diff(xmlNode * last, xmlNode * next, xmlNode * local_diff, gboolean changed)
 {
     int lpc = 0;
     xmlNode *cib = NULL;
     xmlNode *diff_child = NULL;
 
     const char *tag = NULL;
 
     const char *vfields[] = {
         XML_ATTR_GENERATION_ADMIN,
         XML_ATTR_GENERATION,
         XML_ATTR_NUMUPDATES,
     };
 
     if (local_diff == NULL) {
         crm_trace("Nothing to do");
         return;
     }
 
     tag = "diff-removed";
     diff_child = find_xml_node(local_diff, tag, FALSE);
     if (diff_child == NULL) {
         diff_child = create_xml_node(local_diff, tag);
     }
 
     tag = XML_TAG_CIB;
     cib = find_xml_node(diff_child, tag, FALSE);
     if (cib == NULL) {
         cib = create_xml_node(diff_child, tag);
     }
 
     for(lpc = 0; last && lpc < DIMOF(vfields); lpc++){
         const char *value = crm_element_value(last, vfields[lpc]);
 
         crm_xml_add(diff_child, vfields[lpc], value);
         if(changed || lpc == 2) {
             crm_xml_add(cib, vfields[lpc], value);
         }
     }
 
     tag = "diff-added";
     diff_child = find_xml_node(local_diff, tag, FALSE);
     if (diff_child == NULL) {
         diff_child = create_xml_node(local_diff, tag);
     }
 
     tag = XML_TAG_CIB;
     cib = find_xml_node(diff_child, tag, FALSE);
     if (cib == NULL) {
         cib = create_xml_node(diff_child, tag);
     }
 
     for(lpc = 0; next && lpc < DIMOF(vfields); lpc++){
         const char *value = crm_element_value(next, vfields[lpc]);
 
         crm_xml_add(diff_child, vfields[lpc], value);
     }
 
     if (next) {
         xmlAttrPtr xIter = NULL;
 
         for (xIter = next->properties; xIter; xIter = xIter->next) {
             const char *p_name = (const char *)xIter->name;
             const char *p_value = crm_element_value(next, p_name);
 
             xmlSetProp(cib, (const xmlChar *)p_name, (const xmlChar *)p_value);
         }
     }
 
     crm_log_xml_explicit(local_diff, "Repaired-diff");
 }
 
 static xmlNode *
 xml_create_patchset_v1(xmlNode *source, xmlNode *target, bool config, bool suppress)
 {
     xmlNode *patchset = diff_xml_object(source, target, suppress);
 
     if(patchset) {
         CRM_LOG_ASSERT(xml_document_dirty(target));
         xml_repair_v1_diff(source, target, patchset, config);
         crm_xml_add(patchset, "format", "1");
     }
     return patchset;
 }
 
 static xmlNode *
 xml_create_patchset_v2(xmlNode *source, xmlNode *target)
 {
     int lpc = 0;
     GListPtr gIter = NULL;
     xml_private_t *doc = NULL;
 
     xmlNode *v = NULL;
     xmlNode *version = NULL;
     xmlNode *patchset = NULL;
     const char *vfields[] = {
         XML_ATTR_GENERATION_ADMIN,
         XML_ATTR_GENERATION,
         XML_ATTR_NUMUPDATES,
     };
 
     CRM_ASSERT(target);
     if(xml_document_dirty(target) == FALSE) {
         return NULL;
     }
 
     CRM_ASSERT(target->doc);
     doc = target->doc->_private;
 
     patchset = create_xml_node(NULL, XML_TAG_DIFF);
     crm_xml_add_int(patchset, "format", 2);
 
     version = create_xml_node(patchset, XML_DIFF_VERSION);
 
     v = create_xml_node(version, XML_DIFF_VSOURCE);
     for(lpc = 0; lpc < DIMOF(vfields); lpc++){
         const char *value = crm_element_value(source, vfields[lpc]);
 
         if(value == NULL) {
             value = "1";
         }
         crm_xml_add(v, vfields[lpc], value);
     }
 
     v = create_xml_node(version, XML_DIFF_VTARGET);
     for(lpc = 0; lpc < DIMOF(vfields); lpc++){
         const char *value = crm_element_value(target, vfields[lpc]);
 
         if(value == NULL) {
             value = "1";
         }
         crm_xml_add(v, vfields[lpc], value);
     }
 
     for(gIter = doc->deleted_paths; gIter; gIter = gIter->next) {
         xmlNode *change = create_xml_node(patchset, XML_DIFF_CHANGE);
 
         crm_xml_add(change, XML_DIFF_OP, "delete");
         crm_xml_add(change, XML_DIFF_PATH, gIter->data);
     }
 
     __xml_build_changes(target, patchset);
     return patchset;
 }
 
 static gboolean patch_legacy_mode(void)
 {
     static gboolean init = TRUE;
     static gboolean legacy = FALSE;
 
     if(init) {
         init = FALSE;
         legacy = daemon_option_enabled("cib", "legacy");
         if(legacy) {
             crm_notice("Enabled legacy mode");
         }
     }
     return legacy;
 }
 
 xmlNode *
 xml_create_patchset(int format, xmlNode *source, xmlNode *target, bool *config_changed, bool manage_version)
 {
     int counter = 0;
     bool config = FALSE;
     xmlNode *patch = NULL;
     const char *version = crm_element_value(source, XML_ATTR_CRM_VERSION);
 
     xml_acl_disable(target);
     if(xml_document_dirty(target) == FALSE) {
         crm_trace("No change %d", format);
         return NULL; /* No change */
     }
 
     config = is_config_change(target);
     if(config_changed) {
         *config_changed = config;
     }
 
     if(manage_version && config) {
         crm_trace("Config changed %d", format);
         crm_xml_add(target, XML_ATTR_NUMUPDATES, "0");
 
         crm_element_value_int(target, XML_ATTR_GENERATION, &counter);
         crm_xml_add_int(target, XML_ATTR_GENERATION, counter+1);
 
     } else if(manage_version) {
         crm_element_value_int(target, XML_ATTR_NUMUPDATES, &counter);
         crm_trace("Status changed %d - %d %s", format, counter, crm_element_value(source, XML_ATTR_NUMUPDATES));
         crm_xml_add_int(target, XML_ATTR_NUMUPDATES, counter+1);
     }
 
     if(format == 0) {
         if(patch_legacy_mode()) {
             format = 1;
 
         } else if(compare_version("3.0.8", version) < 0) {
             format = 2;
 
         } else {
             format = 1;
         }
         crm_trace("Using patch format %d for version: %s", format, version);
     }
 
     switch(format) {
         case 1:
             patch = xml_create_patchset_v1(source, target, config, FALSE);
             break;
         case 2:
             patch = xml_create_patchset_v2(source, target);
             break;
         default:
             crm_err("Unknown patch format: %d", format);
             return NULL;
     }
 
     return patch;
 }
 
 void
 patchset_process_digest(xmlNode *patch, xmlNode *source, xmlNode *target, bool with_digest)
 {
     int format = 1;
     const char *version = NULL;
     char *digest = NULL;
 
     if (patch == NULL || source == NULL || target == NULL) {
         return;
     }
 
     /* NOTE: We should always call xml_accept_changes() before calculating digest. */
     /* Otherwise, with an on-tracking dirty target, we could get a wrong digest. */
     CRM_LOG_ASSERT(xml_document_dirty(target) == FALSE);
 
     crm_element_value_int(patch, "format", &format);
     if (format > 1 && with_digest == FALSE) {
         return;
     }
 
     version = crm_element_value(source, XML_ATTR_CRM_VERSION);
     digest = calculate_xml_versioned_digest(target, FALSE, TRUE, version);
 
     crm_xml_add(patch, XML_ATTR_DIGEST, digest);
     free(digest);
 
     return;
 }
 
 static void
 __xml_log_element(int log_level, const char *file, const char *function, int line,
                   const char *prefix, xmlNode * data, int depth, int options);
 
 void
 xml_log_patchset(uint8_t log_level, const char *function, xmlNode * patchset)
 {
     int format = 1;
     xmlNode *child = NULL;
     xmlNode *added = NULL;
     xmlNode *removed = NULL;
     gboolean is_first = TRUE;
 
     int add[] = { 0, 0, 0 };
     int del[] = { 0, 0, 0 };
 
     const char *fmt = NULL;
     const char *digest = NULL;
     int options = xml_log_option_formatted;
 
     static struct qb_log_callsite *patchset_cs = NULL;
 
     if (patchset_cs == NULL) {
         patchset_cs = qb_log_callsite_get(function, __FILE__, "xml-patchset", log_level, __LINE__, 0);
     }
 
     if (patchset == NULL) {
         crm_trace("Empty patch");
         return;
 
     } else if (log_level == 0) {
         /* Log to stdout */
     } else if (crm_is_callsite_active(patchset_cs, log_level, 0) == FALSE) {
         return;
     }
 
     xml_patch_versions(patchset, add, del);
     fmt = crm_element_value(patchset, "format");
     digest = crm_element_value(patchset, XML_ATTR_DIGEST);
 
     if (add[2] != del[2] || add[1] != del[1] || add[0] != del[0]) {
         do_crm_log_alias(log_level, __FILE__, function, __LINE__,
                          "Diff: --- %d.%d.%d %s", del[0], del[1], del[2], fmt);
         do_crm_log_alias(log_level, __FILE__, function, __LINE__,
                          "Diff: +++ %d.%d.%d %s", add[0], add[1], add[2], digest);
 
     } else if (patchset != NULL && (add[0] || add[1] || add[2])) {
         do_crm_log_alias(log_level, __FILE__, function, __LINE__, 
                          "%s: Local-only Change: %d.%d.%d", function ? function : "",
                          add[0], add[1], add[2]);
     }
 
     crm_element_value_int(patchset, "format", &format);
     if(format == 2) {
         xmlNode *change = NULL;
 
         for (change = __xml_first_child(patchset); change != NULL; change = __xml_next(change)) {
             const char *op = crm_element_value(change, XML_DIFF_OP);
             const char *xpath = crm_element_value(change, XML_DIFF_PATH);
 
             if(op == NULL) {
             } else if(strcmp(op, "create") == 0) {
                 int lpc = 0, max = 0;
                 char *prefix = crm_strdup_printf("++ %s: ", xpath);
 
                 max = strlen(prefix);
                 __xml_log_element(log_level, __FILE__, function, __LINE__, prefix, change->children,
                                   0, xml_log_option_formatted|xml_log_option_open);
 
                 for(lpc = 2; lpc < max; lpc++) {
                     prefix[lpc] = ' ';
                 }
 
                 __xml_log_element(log_level, __FILE__, function, __LINE__, prefix, change->children,
                                   0, xml_log_option_formatted|xml_log_option_close|xml_log_option_children);
                 free(prefix);
 
             } else if(strcmp(op, "move") == 0) {
                 do_crm_log_alias(log_level, __FILE__, function, __LINE__, "+~ %s moved to offset %s", xpath, crm_element_value(change, XML_DIFF_POSITION));
 
             } else if(strcmp(op, "modify") == 0) {
                 xmlNode *clist = first_named_child(change, XML_DIFF_LIST);
                 char buffer_set[XML_BUFFER_SIZE];
                 char buffer_unset[XML_BUFFER_SIZE];
                 int o_set = 0;
                 int o_unset = 0;
 
                 buffer_set[0] = 0;
                 buffer_unset[0] = 0;
                 for (child = __xml_first_child(clist); child != NULL; child = __xml_next(child)) {
                     const char *name = crm_element_value(child, "name");
 
                     op = crm_element_value(child, XML_DIFF_OP);
                     if(op == NULL) {
                     } else if(strcmp(op, "set") == 0) {
                         const char *value = crm_element_value(child, "value");
 
                         if(o_set > 0) {
                             o_set += snprintf(buffer_set + o_set, XML_BUFFER_SIZE - o_set, ", ");
                         }
                         o_set += snprintf(buffer_set + o_set, XML_BUFFER_SIZE - o_set, "@%s=%s", name, value);
 
                     } else if(strcmp(op, "unset") == 0) {
                         if(o_unset > 0) {
                             o_unset += snprintf(buffer_unset + o_unset, XML_BUFFER_SIZE - o_unset, ", ");
                         }
                         o_unset += snprintf(buffer_unset + o_unset, XML_BUFFER_SIZE - o_unset, "@%s", name);
                     }
                 }
                 if(o_set) {
                     do_crm_log_alias(log_level, __FILE__, function, __LINE__, "+  %s:  %s", xpath, buffer_set);
                 }
                 if(o_unset) {
                     do_crm_log_alias(log_level, __FILE__, function, __LINE__, "-- %s:  %s", xpath, buffer_unset);
                 }
 
             } else if(strcmp(op, "delete") == 0) {
                 do_crm_log_alias(log_level, __FILE__, function, __LINE__, "-- %s", xpath);
             }
         }
         return;
     }
 
     if (log_level < LOG_DEBUG || function == NULL) {
         options |= xml_log_option_diff_short;
     }
 
     removed = find_xml_node(patchset, "diff-removed", FALSE);
     for (child = __xml_first_child(removed); child != NULL; child = __xml_next(child)) {
         log_data_element(log_level, __FILE__, function, __LINE__, "- ", child, 0,
                          options | xml_log_option_diff_minus);
         if (is_first) {
             is_first = FALSE;
         } else {
             do_crm_log_alias(log_level, __FILE__, function, __LINE__, " --- ");
         }
     }
 
     is_first = TRUE;
     added = find_xml_node(patchset, "diff-added", FALSE);
     for (child = __xml_first_child(added); child != NULL; child = __xml_next(child)) {
         log_data_element(log_level, __FILE__, function, __LINE__, "+ ", child, 0,
                          options | xml_log_option_diff_plus);
         if (is_first) {
             is_first = FALSE;
         } else {
             do_crm_log_alias(log_level, __FILE__, function, __LINE__, " +++ ");
         }
     }
 }
 
 void
 xml_log_changes(uint8_t log_level, const char *function, xmlNode * xml)
 {
     GListPtr gIter = NULL;
     xml_private_t *doc = NULL;
 
     CRM_ASSERT(xml);
     CRM_ASSERT(xml->doc);
 
     doc = xml->doc->_private;
     if(is_not_set(doc->flags, xpf_dirty)) {
         return;
     }
 
     for(gIter = doc->deleted_paths; gIter; gIter = gIter->next) {
         do_crm_log_alias(log_level, __FILE__, function, __LINE__, "-- %s", (char*)gIter->data);
     }
 
     log_data_element(log_level, __FILE__, function, __LINE__, "+ ", xml, 0,
                      xml_log_option_formatted|xml_log_option_dirty_add);
 }
 
 void
 xml_accept_changes(xmlNode * xml)
 {
     xmlNode *top = NULL;
     xml_private_t *doc = NULL;
 
     if(xml == NULL) {
         return;
     }
 
     crm_trace("Accepting changes to %p", xml);
     doc = xml->doc->_private;
     top = xmlDocGetRootElement(xml->doc);
 
     __xml_private_clean(xml->doc->_private);
 
     if(is_not_set(doc->flags, xpf_dirty)) {
         doc->flags = xpf_none;
         return;
     }
 
     doc->flags = xpf_none;
     __xml_accept_changes(top);
 }
 
 static xmlNode *
 find_element(xmlNode *haystack, xmlNode *needle)
 {
     CRM_CHECK(needle != NULL, return NULL);
     return (needle->type == XML_COMMENT_NODE)?
            find_xml_comment(haystack, needle)
            : find_entity(haystack, crm_element_name(needle), ID(needle));
 }
 
 /* Simplified version for applying v1-style XML patches */
 static void
 __subtract_xml_object(xmlNode * target, xmlNode * patch)
 {
     xmlNode *patch_child = NULL;
     xmlNode *cIter = NULL;
     xmlAttrPtr xIter = NULL;
 
     char *id = NULL;
     const char *name = NULL;
     const char *value = NULL;
 
     if (target == NULL || patch == NULL) {
         return;
     }
 
     if (target->type == XML_COMMENT_NODE) {
         gboolean dummy;
 
         subtract_xml_comment(target->parent, target, patch, &dummy);
     }
 
     name = crm_element_name(target);
     CRM_CHECK(name != NULL, return);
     CRM_CHECK(safe_str_eq(crm_element_name(target), crm_element_name(patch)), return);
     CRM_CHECK(safe_str_eq(ID(target), ID(patch)), return);
 
     /* check for XML_DIFF_MARKER in a child */
     id = crm_element_value_copy(target, XML_ATTR_ID);
     value = crm_element_value(patch, XML_DIFF_MARKER);
     if (value != NULL && strcmp(value, "removed:top") == 0) {
         crm_trace("We are the root of the deletion: %s.id=%s", name, id);
         free_xml(target);
         free(id);
         return;
     }
 
     for (xIter = crm_first_attr(patch); xIter != NULL; xIter = xIter->next) {
         const char *p_name = (const char *)xIter->name;
 
         /* Removing and then restoring the id field would change the ordering of properties */
         if (safe_str_neq(p_name, XML_ATTR_ID)) {
             xml_remove_prop(target, p_name);
         }
     }
 
     /* changes to child objects */
     cIter = __xml_first_child(target);
     while (cIter) {
         xmlNode *target_child = cIter;
 
         cIter = __xml_next(cIter);
         patch_child = find_element(patch, target_child);
         __subtract_xml_object(target_child, patch_child);
     }
     free(id);
 }
 
 static void
 __add_xml_object(xmlNode * parent, xmlNode * target, xmlNode * patch)
 {
     xmlNode *patch_child = NULL;
     xmlNode *target_child = NULL;
     xmlAttrPtr xIter = NULL;
 
     const char *id = NULL;
     const char *name = NULL;
     const char *value = NULL;
 
     if (patch == NULL) {
         return;
     } else if (parent == NULL && target == NULL) {
         return;
     }
 
     /* check for XML_DIFF_MARKER in a child */
     value = crm_element_value(patch, XML_DIFF_MARKER);
     if (target == NULL
         && value != NULL
         && strcmp(value, "added:top") == 0) {
         id = ID(patch);
         name = crm_element_name(patch);
         crm_trace("We are the root of the addition: %s.id=%s", name, id);
         add_node_copy(parent, patch);
         return;
 
     } else if(target == NULL) {
         id = ID(patch);
         name = crm_element_name(patch);
         crm_err("Could not locate: %s.id=%s", name, id);
         return;
     }
 
     if (target->type == XML_COMMENT_NODE) {
         add_xml_comment(parent, target, patch);
     }
 
     name = crm_element_name(target);
     CRM_CHECK(name != NULL, return);
     CRM_CHECK(safe_str_eq(crm_element_name(target), crm_element_name(patch)), return);
     CRM_CHECK(safe_str_eq(ID(target), ID(patch)), return);
 
     for (xIter = crm_first_attr(patch); xIter != NULL; xIter = xIter->next) {
         const char *p_name = (const char *)xIter->name;
         const char *p_value = crm_element_value(patch, p_name);
 
         xml_remove_prop(target, p_name); /* Preserve the patch order */
         crm_xml_add(target, p_name, p_value);
     }
 
     /* changes to child objects */
     for (patch_child = __xml_first_child(patch); patch_child != NULL;
          patch_child = __xml_next(patch_child)) {
 
         target_child = find_element(target, patch_child);
         __add_xml_object(target, target_child, patch_child);
     }
 }
 
 /*!
  * \internal
  * \brief Find additions or removals in a patch set
  *
  * \param[in]     patchset   XML of patch
  * \param[in]     format     Patch version
  * \param[in]     added      TRUE if looking for additions, FALSE if removals
  * \param[in/out] patch_node Will be set to node if found
  *
  * \return TRUE if format is valid, FALSE if invalid
  */
 static bool
 find_patch_xml_node(xmlNode *patchset, int format, bool added,
                     xmlNode **patch_node)
 {
     xmlNode *cib_node;
     const char *label;
 
     switch(format) {
         case 1:
             label = added? "diff-added" : "diff-removed";
             *patch_node = find_xml_node(patchset, label, FALSE);
             cib_node = find_xml_node(*patch_node, "cib", FALSE);
             if (cib_node != NULL) {
                 *patch_node = cib_node;
             }
             break;
         case 2:
             label = added? "target" : "source";
             *patch_node = find_xml_node(patchset, "version", FALSE);
             *patch_node = find_xml_node(*patch_node, label, FALSE);
             break;
         default:
             crm_warn("Unknown patch format: %d", format);
             *patch_node = NULL;
             return FALSE;
     }
     return TRUE;
 }
 
 bool xml_patch_versions(xmlNode *patchset, int add[3], int del[3])
 {
     int lpc = 0;
     int format = 1;
     xmlNode *tmp = NULL;
 
     const char *vfields[] = {
         XML_ATTR_GENERATION_ADMIN,
         XML_ATTR_GENERATION,
         XML_ATTR_NUMUPDATES,
     };
 
 
     crm_element_value_int(patchset, "format", &format);
 
     /* Process removals */
     if (!find_patch_xml_node(patchset, format, FALSE, &tmp)) {
         return -EINVAL;
     }
     if (tmp) {
         for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
             crm_element_value_int(tmp, vfields[lpc], &(del[lpc]));
             crm_trace("Got %d for del[%s]", del[lpc], vfields[lpc]);
         }
     }
 
     /* Process additions */
     if (!find_patch_xml_node(patchset, format, TRUE, &tmp)) {
         return -EINVAL;
     }
     if (tmp) {
         for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
             crm_element_value_int(tmp, vfields[lpc], &(add[lpc]));
             crm_trace("Got %d for add[%s]", add[lpc], vfields[lpc]);
         }
     }
 
     return pcmk_ok;
 }
 
 static int
 xml_patch_version_check(xmlNode *xml, xmlNode *patchset, int format) 
 {
     int lpc = 0;
     bool changed = FALSE;
 
     int this[] = { 0, 0, 0 };
     int add[] = { 0, 0, 0 };
     int del[] = { 0, 0, 0 };
 
     const char *vfields[] = {
         XML_ATTR_GENERATION_ADMIN,
         XML_ATTR_GENERATION,
         XML_ATTR_NUMUPDATES,
     };
 
     for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
         crm_element_value_int(xml, vfields[lpc], &(this[lpc]));
         crm_trace("Got %d for this[%s]", this[lpc], vfields[lpc]);
         if (this[lpc] < 0) {
             this[lpc] = 0;
         }
     }
 
     /* Set some defaults in case nothing is present */
     add[0] = this[0];
     add[1] = this[1];
     add[2] = this[2] + 1;
     for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
         del[lpc] = this[lpc];
     }
 
     xml_patch_versions(patchset, add, del);
 
     for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
         if(this[lpc] < del[lpc]) {
             crm_debug("Current %s is too low (%d.%d.%d < %d.%d.%d --> %d.%d.%d)", vfields[lpc],
                       this[0], this[1], this[2], del[0], del[1], del[2], add[0], add[1], add[2]);
             return -pcmk_err_diff_resync;
 
         } else if(this[lpc] > del[lpc]) {
             crm_info("Current %s is too high (%d.%d.%d > %d.%d.%d --> %d.%d.%d) %p", vfields[lpc],
                      this[0], this[1], this[2], del[0], del[1], del[2], add[0], add[1], add[2], patchset);
             crm_log_xml_info(patchset, "OldPatch");
             return -pcmk_err_old_data;
         }
     }
 
     for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
         if(add[lpc] > del[lpc]) {
             changed = TRUE;
         }
     }
 
     if(changed == FALSE) {
         crm_notice("Versions did not change in patch %d.%d.%d", add[0], add[1], add[2]);
         return -pcmk_err_old_data;
     }
 
     crm_debug("Can apply patch %d.%d.%d to %d.%d.%d",
              add[0], add[1], add[2], this[0], this[1], this[2]);
     return pcmk_ok;
 }
 
 static int
 xml_apply_patchset_v1(xmlNode *xml, xmlNode *patchset, bool check_version) 
 {
     int rc = pcmk_ok;
     int root_nodes_seen = 0;
     char *version = crm_element_value_copy(xml, XML_ATTR_CRM_VERSION);
 
     xmlNode *child_diff = NULL;
     xmlNode *added = find_xml_node(patchset, "diff-added", FALSE);
     xmlNode *removed = find_xml_node(patchset, "diff-removed", FALSE);
     xmlNode *old = copy_xml(xml);
 
     crm_trace("Subtraction Phase");
     for (child_diff = __xml_first_child(removed); child_diff != NULL;
          child_diff = __xml_next(child_diff)) {
         CRM_CHECK(root_nodes_seen == 0, rc = FALSE);
         if (root_nodes_seen == 0) {
             __subtract_xml_object(xml, child_diff);
         }
         root_nodes_seen++;
     }
 
     if (root_nodes_seen > 1) {
         crm_err("(-) Diffs cannot contain more than one change set... saw %d", root_nodes_seen);
         rc = -ENOTUNIQ;
     }
 
     root_nodes_seen = 0;
     crm_trace("Addition Phase");
     if (rc == pcmk_ok) {
         xmlNode *child_diff = NULL;
 
         for (child_diff = __xml_first_child(added); child_diff != NULL;
              child_diff = __xml_next(child_diff)) {
             CRM_CHECK(root_nodes_seen == 0, rc = FALSE);
             if (root_nodes_seen == 0) {
                 __add_xml_object(NULL, xml, child_diff);
             }
             root_nodes_seen++;
         }
     }
 
     if (root_nodes_seen > 1) {
         crm_err("(+) Diffs cannot contain more than one change set... saw %d", root_nodes_seen);
         rc = -ENOTUNIQ;
     }
 
     purge_diff_markers(xml);       /* Purge prior to checking the digest */
 
     free_xml(old);
     free(version);
     return rc;
 }
 
 static xmlNode *
 __first_xml_child_match(xmlNode *parent, const char *name, const char *id)
 {
     xmlNode *cIter = NULL;
 
     for (cIter = __xml_first_child(parent); cIter != NULL; cIter = __xml_next(cIter)) {
         if(strcmp((const char *)cIter->name, name) != 0) {
             continue;
         } else if(id) {
             const char *cid = ID(cIter);
             if(cid == NULL || strcmp(cid, id) != 0) {
                 continue;
             }
         }
         return cIter;
     }
     return NULL;
 }
 
 static xmlNode *
 __xml_find_path(xmlNode *top, const char *key)
 {
     xmlNode *target = (xmlNode*)top->doc;
     char *id = malloc(XML_BUFFER_SIZE);
     char *tag = malloc(XML_BUFFER_SIZE);
     char *section = malloc(XML_BUFFER_SIZE);
     char *current = strdup(key);
     char *remainder = malloc(XML_BUFFER_SIZE);
     int rc = 0;
 
     while(current) {
         rc = sscanf (current, "/%[^/]%s", section, remainder);
         if(rc <= 0) {
             crm_trace("Done");
             break;
 
         } else if(rc > 2) {
             crm_trace("Aborting on %s", current);
             target = NULL;
             break;
 
         } else if(tag && section) {
             int f = sscanf (section, "%[^[][@id='%[^']", tag, id);
 
             switch(f) {
                 case 1:
                     target = __first_xml_child_match(target, tag, NULL);
                     break;
                 case 2:
                     target = __first_xml_child_match(target, tag, id);
                     break;
                 default:
                     crm_trace("Aborting on %s", section);
                     target = NULL;
                     break;
             }
 
             if(rc == 1 || target == NULL) {
                 crm_trace("Done");
                 break;
 
             } else {
                 char *tmp = current;
                 current = remainder;
                 remainder = tmp;
             }
         }
     }
 
     if(target) {
         char *path = (char *)xmlGetNodePath(target);
 
         crm_trace("Found %s for %s", path, key);
         free(path);
     } else {
         crm_debug("No match for %s", key);
     }
 
     free(remainder);
     free(current);
     free(section);
     free(tag);
     free(id);
     return target;
 }
 
 static int
 xml_apply_patchset_v2(xmlNode *xml, xmlNode *patchset, bool check_version) 
 {
     int rc = pcmk_ok;
     xmlNode *change = NULL;
     for (change = __xml_first_child(patchset); change != NULL; change = __xml_next(change)) {
         xmlNode *match = NULL;
         const char *op = crm_element_value(change, XML_DIFF_OP);
         const char *xpath = crm_element_value(change, XML_DIFF_PATH);
 
         crm_trace("Processing %s %s", change->name, op);
         if(op == NULL) {
             continue;
         }
 
 #if 0
         match = get_xpath_object(xpath, xml, LOG_TRACE);
 #else
         match = __xml_find_path(xml, xpath);
 #endif
         crm_trace("Performing %s on %s with %p", op, xpath, match);
 
         if(match == NULL && strcmp(op, "delete") == 0) {
             crm_debug("No %s match for %s in %p", op, xpath, xml->doc);
             continue;
 
         } else if(match == NULL) {
             crm_err("No %s match for %s in %p", op, xpath, xml->doc);
             rc = -pcmk_err_diff_failed;
             continue;
 
         } else if(strcmp(op, "create") == 0) {
             int position = 0;
             xmlNode *child = NULL;
             xmlNode *match_child = NULL;
 
             match_child = match->children;
             crm_element_value_int(change, XML_DIFF_POSITION, &position);
 
             while(match_child && position != __xml_offset(match_child)) {
                 match_child = match_child->next;
             }
 
             child = xmlDocCopyNode(change->children, match->doc, 1);
             if(match_child) {
                 crm_trace("Adding %s at position %d", child->name, position);
                 xmlAddPrevSibling(match_child, child);
 
             } else if(match->last) { /* Add to the end */
                 crm_trace("Adding %s at position %d (end)", child->name, position);
                 xmlAddNextSibling(match->last, child);
 
             } else {
                 crm_trace("Adding %s at position %d (first)", child->name, position);
                 CRM_LOG_ASSERT(position == 0);
                 xmlAddChild(match, child);
             }
             crm_node_created(child);
 
         } else if(strcmp(op, "move") == 0) {
             int position = 0;
 
             crm_element_value_int(change, XML_DIFF_POSITION, &position);
             if(position != __xml_offset(match)) {
                 xmlNode *match_child = NULL;
                 int p = position;
 
                 if(p > __xml_offset(match)) {
                     p++; /* Skip ourselves */
                 }
 
                 CRM_ASSERT(match->parent != NULL);
                 match_child = match->parent->children;
 
                 while(match_child && p != __xml_offset(match_child)) {
                     match_child = match_child->next;
                 }
 
                 crm_trace("Moving %s to position %d (was %d, prev %p, %s %p)",
                          match->name, position, __xml_offset(match), match->prev,
                          match_child?"next":"last", match_child?match_child:match->parent->last);
 
                 if(match_child) {
                     xmlAddPrevSibling(match_child, match);
 
                 } else {
                     CRM_ASSERT(match->parent->last != NULL);
                     xmlAddNextSibling(match->parent->last, match);
                 }
 
             } else {
                 crm_trace("%s is already in position %d", match->name, position);
             }
 
             if(position != __xml_offset(match)) {
                 crm_err("Moved %s.%d to position %d instead of %d (%p)",
                         match->name, ID(match), __xml_offset(match), position, match->prev);
                 rc = -pcmk_err_diff_failed;
             }
 
         } else if(strcmp(op, "delete") == 0) {
             free_xml(match);
 
         } else if(strcmp(op, "modify") == 0) {
             xmlAttr *pIter = crm_first_attr(match);
             xmlNode *attrs = __xml_first_child(first_named_child(change, XML_DIFF_RESULT));
 
             if(attrs == NULL) {
                 rc = -ENOMSG;
                 continue;
             }
             while(pIter != NULL) {
                 const char *name = (const char *)pIter->name;
 
                 pIter = pIter->next;
                 xml_remove_prop(match, name);
             }
 
             for (pIter = crm_first_attr(attrs); pIter != NULL; pIter = pIter->next) {
                 const char *name = (const char *)pIter->name;
                 const char *value = crm_element_value(attrs, name);
 
                 crm_xml_add(match, name, value);
             }
 
         } else {
             crm_err("Unknown operation: %s", op);
         }
     }
     return rc;
 }
 
 int
 xml_apply_patchset(xmlNode *xml, xmlNode *patchset, bool check_version) 
 {
     int format = 1;
     int rc = pcmk_ok;
     xmlNode *old = NULL;
     const char *digest = crm_element_value(patchset, XML_ATTR_DIGEST);
 
     if(patchset == NULL) {
         return rc;
     }
 
     xml_log_patchset(LOG_TRACE, __FUNCTION__, patchset);
 
     crm_element_value_int(patchset, "format", &format);
     if(check_version) {
         rc = xml_patch_version_check(xml, patchset, format);
         if(rc != pcmk_ok) {
             return rc;
         }
     }
 
     if(digest) {
         /* Make it available for logging if the result doesn't have the expected digest */
         old = copy_xml(xml);
     }
 
     if(rc == pcmk_ok) {
         switch(format) {
             case 1:
                 rc = xml_apply_patchset_v1(xml, patchset, check_version);
                 break;
             case 2:
                 rc = xml_apply_patchset_v2(xml, patchset, check_version);
                 break;
             default:
                 crm_err("Unknown patch format: %d", format);
                 rc = -EINVAL;
         }
     }
 
     if(rc == pcmk_ok && digest) {
         static struct qb_log_callsite *digest_cs = NULL;
 
         char *new_digest = NULL;
         char *version = crm_element_value_copy(xml, XML_ATTR_CRM_VERSION);
 
         if (digest_cs == NULL) {
             digest_cs =
                 qb_log_callsite_get(__func__, __FILE__, "diff-digest", LOG_TRACE, __LINE__,
                                     crm_trace_nonlog);
         }
 
         new_digest = calculate_xml_versioned_digest(xml, FALSE, TRUE, version);
         if (safe_str_neq(new_digest, digest)) {
             crm_info("v%d digest mis-match: expected %s, calculated %s", format, digest, new_digest);
             rc = -pcmk_err_diff_failed;
 
             if (digest_cs && digest_cs->targets) {
                 save_xml_to_file(old,     "PatchDigest:input", NULL);
                 save_xml_to_file(xml,     "PatchDigest:result", NULL);
                 save_xml_to_file(patchset,"PatchDigest:diff", NULL);
 
             } else {
                 crm_trace("%p %.6x", digest_cs, digest_cs ? digest_cs->targets : 0);
             }
 
         } else {
             crm_trace("v%d digest matched: expected %s, calculated %s", format, digest, new_digest);
         }
         free(new_digest);
         free(version);
     }
     free_xml(old);
     return rc;
 }
 
 xmlNode *
 find_xml_node(xmlNode * root, const char *search_path, gboolean must_find)
 {
     xmlNode *a_child = NULL;
     const char *name = "NULL";
 
     if (root != NULL) {
         name = crm_element_name(root);
     }
 
     if (search_path == NULL) {
         crm_warn("Will never find <NULL>");
         return NULL;
     }
 
     for (a_child = __xml_first_child(root); a_child != NULL; a_child = __xml_next(a_child)) {
         if (strcmp((const char *)a_child->name, search_path) == 0) {
 /* 		crm_trace("returning node (%s).", crm_element_name(a_child)); */
             return a_child;
         }
     }
 
     if (must_find) {
         crm_warn("Could not find %s in %s.", search_path, name);
     } else if (root != NULL) {
         crm_trace("Could not find %s in %s.", search_path, name);
     } else {
         crm_trace("Could not find %s in <NULL>.", search_path);
     }
 
     return NULL;
 }
 
 xmlNode *
 find_entity(xmlNode * parent, const char *node_name, const char *id)
 {
     xmlNode *a_child = NULL;
 
     for (a_child = __xml_first_child(parent); a_child != NULL; a_child = __xml_next(a_child)) {
         /* Uncertain if node_name == NULL check is strictly necessary here */
         if (node_name == NULL || strcmp((const char *)a_child->name, node_name) == 0) {
             const char *cid = ID(a_child);
             if (id == NULL || (cid != NULL && strcmp(id, cid) == 0)) {
                 return a_child;
             }
         }
     }
 
     crm_trace("node <%s id=%s> not found in %s.", node_name, id, crm_element_name(parent));
     return NULL;
 }
 
 void
 copy_in_properties(xmlNode * target, xmlNode * src)
 {
     if (src == NULL) {
         crm_warn("No node to copy properties from");
 
     } else if (target == NULL) {
         crm_err("No node to copy properties into");
 
     } else {
         xmlAttrPtr pIter = NULL;
 
         for (pIter = crm_first_attr(src); pIter != NULL; pIter = pIter->next) {
             const char *p_name = (const char *)pIter->name;
             const char *p_value = crm_attr_value(pIter);
 
             expand_plus_plus(target, p_name, p_value);
         }
     }
 
     return;
 }
 
 void
 fix_plus_plus_recursive(xmlNode * target)
 {
     /* TODO: Remove recursion and use xpath searches for value++ */
     xmlNode *child = NULL;
     xmlAttrPtr pIter = NULL;
 
     for (pIter = crm_first_attr(target); pIter != NULL; pIter = pIter->next) {
         const char *p_name = (const char *)pIter->name;
         const char *p_value = crm_attr_value(pIter);
 
         expand_plus_plus(target, p_name, p_value);
     }
     for (child = __xml_first_child(target); child != NULL; child = __xml_next(child)) {
         fix_plus_plus_recursive(child);
     }
 }
 
 void
 expand_plus_plus(xmlNode * target, const char *name, const char *value)
 {
     int offset = 1;
     int name_len = 0;
     int int_value = 0;
     int value_len = 0;
 
     const char *old_value = NULL;
 
     if (value == NULL || name == NULL) {
         return;
     }
 
     old_value = crm_element_value(target, name);
 
     if (old_value == NULL) {
         /* if no previous value, set unexpanded */
         goto set_unexpanded;
 
     } else if (strstr(value, name) != value) {
         goto set_unexpanded;
     }
 
     name_len = strlen(name);
     value_len = strlen(value);
     if (value_len < (name_len + 2)
         || value[name_len] != '+' || (value[name_len + 1] != '+' && value[name_len + 1] != '=')) {
         goto set_unexpanded;
     }
 
     /* if we are expanding ourselves,
      * then no previous value was set and leave int_value as 0
      */
     if (old_value != value) {
         int_value = char2score(old_value);
     }
 
     if (value[name_len + 1] != '+') {
         const char *offset_s = value + (name_len + 2);
 
         offset = char2score(offset_s);
     }
     int_value += offset;
 
     if (int_value > INFINITY) {
         int_value = (int)INFINITY;
     }
 
     crm_xml_add_int(target, name, int_value);
     return;
 
   set_unexpanded:
     if (old_value == value) {
         /* the old value is already set, nothing to do */
         return;
     }
     crm_xml_add(target, name, value);
     return;
 }
 
 xmlDoc *
 getDocPtr(xmlNode * node)
 {
     xmlDoc *doc = NULL;
 
     CRM_CHECK(node != NULL, return NULL);
 
     doc = node->doc;
     if (doc == NULL) {
         doc = xmlNewDoc((const xmlChar *)"1.0");
         xmlDocSetRootElement(doc, node);
         xmlSetTreeDoc(node, doc);
     }
     return doc;
 }
 
 xmlNode *
 add_node_copy(xmlNode * parent, xmlNode * src_node)
 {
     xmlNode *child = NULL;
     xmlDoc *doc = getDocPtr(parent);
 
     CRM_CHECK(src_node != NULL, return NULL);
 
     child = xmlDocCopyNode(src_node, doc, 1);
     xmlAddChild(parent, child);
     crm_node_created(child);
     return child;
 }
 
 int
 add_node_nocopy(xmlNode * parent, const char *name, xmlNode * child)
 {
     add_node_copy(parent, child);
     free_xml(child);
     return 1;
 }
 
 static bool
 __xml_acl_check(xmlNode *xml, const char *name, enum xml_private_flags mode)
 {
     CRM_ASSERT(xml);
     CRM_ASSERT(xml->doc);
     CRM_ASSERT(xml->doc->_private);
 
 #if ENABLE_ACL
     {
         if(TRACKING_CHANGES(xml) && xml_acl_enabled(xml)) {
             int offset = 0;
             xmlNode *parent = xml;
             char buffer[XML_BUFFER_SIZE];
             xml_private_t *docp = xml->doc->_private;
 
             if(docp->acls == NULL) {
                 crm_trace("Ordinary user %s cannot access the CIB without any defined ACLs", docp->user);
                 set_doc_flag(xml, xpf_acl_denied);
                 return FALSE;
             }
 
             offset = __get_prefix(NULL, xml, buffer, offset);
             if(name) {
                 offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "[@%s]", name);
             }
             CRM_LOG_ASSERT(offset > 0);
 
             /* Walk the tree upwards looking for xml_acl_* flags
              * - Creating an attribute requires write permissions for the node
              * - Creating a child requires write permissions for the parent
              */
 
             if(name) {
                 xmlAttr *attr = xmlHasProp(xml, (const xmlChar *)name);
 
                 if(attr && mode == xpf_acl_create) {
                     mode = xpf_acl_write;
                 }
             }
 
             while(parent && parent->_private) {
                 xml_private_t *p = parent->_private;
                 if(__xml_acl_mode_test(p->flags, mode)) {
                     return TRUE;
 
                 } else if(is_set(p->flags, xpf_acl_deny)) {
                     crm_trace("%x access denied to %s: parent", mode, buffer);
                     set_doc_flag(xml, xpf_acl_denied);
                     return FALSE;
                 }
                 parent = parent->parent;
             }
 
             crm_trace("%x access denied to %s: default", mode, buffer);
             set_doc_flag(xml, xpf_acl_denied);
             return FALSE;
         }
     }
 #endif
 
     return TRUE;
 }
 
 const char *
 crm_xml_add(xmlNode * node, const char *name, const char *value)
 {
     bool dirty = FALSE;
     xmlAttr *attr = NULL;
 
     CRM_CHECK(node != NULL, return NULL);
     CRM_CHECK(name != NULL, return NULL);
 
     if (value == NULL) {
         return NULL;
     }
 #if XML_PARANOIA_CHECKS
     {
         const char *old_value = NULL;
 
         old_value = crm_element_value(node, name);
 
         /* Could be re-setting the same value */
         CRM_CHECK(old_value != value, crm_err("Cannot reset %s with crm_xml_add(%s)", name, value);
                   return value);
     }
 #endif
 
     if(TRACKING_CHANGES(node)) {
         const char *old = crm_element_value(node, name);
 
         if(old == NULL || value == NULL || strcmp(old, value) != 0) {
             dirty = TRUE;
         }
     }
 
     if(dirty && __xml_acl_check(node, name, xpf_acl_create) == FALSE) {
         crm_trace("Cannot add %s=%s to %s", name, value, node->name);
         return NULL;
     }
 
     attr = xmlSetProp(node, (const xmlChar *)name, (const xmlChar *)value);
     if(dirty) {
         crm_attr_dirty(attr);
     }
 
     CRM_CHECK(attr && attr->children && attr->children->content, return NULL);
     return (char *)attr->children->content;
 }
 
 const char *
 crm_xml_replace(xmlNode * node, const char *name, const char *value)
 {
     bool dirty = FALSE;
     xmlAttr *attr = NULL;
     const char *old_value = NULL;
 
     CRM_CHECK(node != NULL, return NULL);
     CRM_CHECK(name != NULL && name[0] != 0, return NULL);
 
     old_value = crm_element_value(node, name);
 
     /* Could be re-setting the same value */
     CRM_CHECK(old_value != value, return value);
 
     if(__xml_acl_check(node, name, xpf_acl_write) == FALSE) {
         /* Create a fake object linked to doc->_private instead? */
         crm_trace("Cannot replace %s=%s to %s", name, value, node->name);
         return NULL;
 
     } else if (old_value != NULL && value == NULL) {
         xml_remove_prop(node, name);
         return NULL;
 
     } else if (value == NULL) {
         return NULL;
     }
 
     if(TRACKING_CHANGES(node)) {
         if(old_value == NULL || value == NULL || strcmp(old_value, value) != 0) {
             dirty = TRUE;
         }
     }
 
     attr = xmlSetProp(node, (const xmlChar *)name, (const xmlChar *)value);
     if(dirty) {
         crm_attr_dirty(attr);
     }
     CRM_CHECK(attr && attr->children && attr->children->content, return NULL);
     return (char *)attr->children->content;
 }
 
 const char *
 crm_xml_add_int(xmlNode * node, const char *name, int value)
 {
     char *number = crm_itoa(value);
     const char *added = crm_xml_add(node, name, number);
 
     free(number);
     return added;
 }
 
 xmlNode *
 create_xml_node(xmlNode * parent, const char *name)
 {
     xmlDoc *doc = NULL;
     xmlNode *node = NULL;
 
     if (name == NULL || name[0] == 0) {
         CRM_CHECK(name != NULL && name[0] == 0, return NULL);
         return NULL;
     }
 
     if (parent == NULL) {
         doc = xmlNewDoc((const xmlChar *)"1.0");
         node = xmlNewDocRawNode(doc, NULL, (const xmlChar *)name, NULL);
         xmlDocSetRootElement(doc, node);
 
     } else {
         doc = getDocPtr(parent);
         node = xmlNewDocRawNode(doc, NULL, (const xmlChar *)name, NULL);
         xmlAddChild(parent, node);
     }
     crm_node_created(node);
     return node;
 }
 
 static inline int
 __get_prefix(const char *prefix, xmlNode *xml, char *buffer, int offset)
 {
     const char *id = ID(xml);
 
     if(offset == 0 && prefix == NULL && xml->parent) {
         offset = __get_prefix(NULL, xml->parent, buffer, offset);
     }
 
     if(id) {
         offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "/%s[@id='%s']", (const char *)xml->name, id);
     } else if(xml->name) {
         offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "/%s", (const char *)xml->name);
     }
 
     return offset;
 }
 
 char *
 xml_get_path(xmlNode *xml)
 {
     int offset = 0;
     char buffer[XML_BUFFER_SIZE];
 
     if(__get_prefix(NULL, xml, buffer, offset) > 0) {
         return strdup(buffer);
     }
     return NULL;
 }
 
 void
 free_xml(xmlNode * child)
 {
     if (child != NULL) {
         xmlNode *top = NULL;
         xmlDoc *doc = child->doc;
         xml_private_t *p = child->_private;
 
         if (doc != NULL) {
             top = xmlDocGetRootElement(doc);
         }
 
         if (doc != NULL && top == child) {
             /* Free everything */
             xmlFreeDoc(doc);
 
         } else if(__xml_acl_check(child, NULL, xpf_acl_write) == FALSE) {
             int offset = 0;
             char buffer[XML_BUFFER_SIZE];
 
             __get_prefix(NULL, child, buffer, offset);
             crm_trace("Cannot remove %s %x", buffer, p->flags);
             return;
 
         } else {
             if(doc && TRACKING_CHANGES(child) && is_not_set(p->flags, xpf_created)) {
                 int offset = 0;
                 char buffer[XML_BUFFER_SIZE];
 
                 if(__get_prefix(NULL, child, buffer, offset) > 0) {
                     crm_trace("Deleting %s %p from %p", buffer, child, doc);
                     p = doc->_private;
                     p->deleted_paths = g_list_append(p->deleted_paths, strdup(buffer));
                     set_doc_flag(child, xpf_dirty);
                 }
             }
 
             /* Free this particular subtree
              * Make sure to unlink it from the parent first
              */
             xmlUnlinkNode(child);
             xmlFreeNode(child);
         }
     }
 }
 
 xmlNode *
 copy_xml(xmlNode * src)
 {
     xmlDoc *doc = xmlNewDoc((const xmlChar *)"1.0");
     xmlNode *copy = xmlDocCopyNode(src, doc, 1);
 
     xmlDocSetRootElement(doc, copy);
     xmlSetTreeDoc(copy, doc);
     return copy;
 }
 
 static void
 crm_xml_err(void *ctx, const char *msg, ...)
 G_GNUC_PRINTF(2, 3);
 
 static void
 crm_xml_err(void *ctx, const char *msg, ...)
 {
     int len = 0;
     va_list args;
     char *buf = NULL;
     static int buffer_len = 0;
     static char *buffer = NULL;
     static struct qb_log_callsite *xml_error_cs = NULL;
 
     va_start(args, msg);
     len = vasprintf(&buf, msg, args);
 
     if(xml_error_cs == NULL) {
         xml_error_cs = qb_log_callsite_get(
             __func__, __FILE__, "xml library error", LOG_TRACE, __LINE__, crm_trace_nonlog);
     }
 
     if (strchr(buf, '\n')) {
         buf[len - 1] = 0;
         if (buffer) {
             crm_err("XML Error: %s%s", buffer, buf);
             free(buffer);
         } else {
             crm_err("XML Error: %s", buf);
         }
         if (xml_error_cs && xml_error_cs->targets) {
             crm_abort(__FILE__, __PRETTY_FUNCTION__, __LINE__, "xml library error", TRUE, TRUE);
         }
         buffer = NULL;
         buffer_len = 0;
 
     } else if (buffer == NULL) {
         buffer_len = len;
         buffer = buf;
         buf = NULL;
 
     } else {
         buffer = realloc_safe(buffer, 1 + buffer_len + len);
         memcpy(buffer + buffer_len, buf, len);
         buffer_len += len;
         buffer[buffer_len] = 0;
     }
 
     va_end(args);
     free(buf);
 }
 
 xmlNode *
 string2xml(const char *input)
 {
     xmlNode *xml = NULL;
     xmlDocPtr output = NULL;
     xmlParserCtxtPtr ctxt = NULL;
     xmlErrorPtr last_error = NULL;
 
     if (input == NULL) {
         crm_err("Can't parse NULL input");
         return NULL;
     }
 
     /* create a parser context */
     ctxt = xmlNewParserCtxt();
     CRM_CHECK(ctxt != NULL, return NULL);
 
     /* xmlCtxtUseOptions(ctxt, XML_PARSE_NOBLANKS|XML_PARSE_RECOVER); */
 
     xmlCtxtResetLastError(ctxt);
     xmlSetGenericErrorFunc(ctxt, crm_xml_err);
     /* initGenericErrorDefaultFunc(crm_xml_err); */
     output =
         xmlCtxtReadDoc(ctxt, (const xmlChar *)input, NULL, NULL,
                        XML_PARSE_NOBLANKS | XML_PARSE_RECOVER);
     if (output) {
         xml = xmlDocGetRootElement(output);
     }
     last_error = xmlCtxtGetLastError(ctxt);
     if (last_error && last_error->code != XML_ERR_OK) {
         /* crm_abort(__FILE__,__FUNCTION__,__LINE__, "last_error->code != XML_ERR_OK", TRUE, TRUE); */
         /*
          * http://xmlsoft.org/html/libxml-xmlerror.html#xmlErrorLevel
          * http://xmlsoft.org/html/libxml-xmlerror.html#xmlParserErrors
          */
         crm_warn("Parsing failed (domain=%d, level=%d, code=%d): %s",
                  last_error->domain, last_error->level, last_error->code, last_error->message);
 
         if (last_error->code == XML_ERR_DOCUMENT_EMPTY) {
             CRM_LOG_ASSERT("Cannot parse an empty string");
 
         } else if (last_error->code != XML_ERR_DOCUMENT_END) {
             crm_err("Couldn't%s parse %d chars: %s", xml ? " fully" : "", (int)strlen(input),
                     input);
             if (xml != NULL) {
                 crm_log_xml_err(xml, "Partial");
             }
 
         } else {
             int len = strlen(input);
             int lpc = 0;
 
             while(lpc < len) {
                 crm_warn("Parse error[+%.3d]: %.80s", lpc, input+lpc);
                 lpc += 80;
             }
 
             CRM_LOG_ASSERT("String parsing error");
         }
     }
 
     xmlFreeParserCtxt(ctxt);
     return xml;
 }
 
 xmlNode *
 stdin2xml(void)
 {
     size_t data_length = 0;
     size_t read_chars = 0;
 
     char *xml_buffer = NULL;
     xmlNode *xml_obj = NULL;
 
     do {
         size_t next = XML_BUFFER_SIZE + data_length + 1;
 
         if(next <= 0) {
             crm_err("Buffer size exceeded at: %l + %d", data_length, XML_BUFFER_SIZE);
             break;
         }
 
         xml_buffer = realloc_safe(xml_buffer, next);
         read_chars = fread(xml_buffer + data_length, 1, XML_BUFFER_SIZE, stdin);
         data_length += read_chars;
     } while (read_chars > 0);
 
     if (data_length == 0) {
         crm_warn("No XML supplied on stdin");
         free(xml_buffer);
         return NULL;
     }
 
     xml_buffer[data_length] = '\0';
 
     xml_obj = string2xml(xml_buffer);
     free(xml_buffer);
 
     crm_log_xml_trace(xml_obj, "Created fragment");
     return xml_obj;
 }
 
 static char *
 decompress_file(const char *filename)
 {
     char *buffer = NULL;
 
 #if HAVE_BZLIB_H
     int rc = 0;
     size_t length = 0, read_len = 0;
 
     BZFILE *bz_file = NULL;
     FILE *input = fopen(filename, "r");
 
     if (input == NULL) {
         crm_perror(LOG_ERR, "Could not open %s for reading", filename);
         return NULL;
     }
 
     bz_file = BZ2_bzReadOpen(&rc, input, 0, 0, NULL, 0);
 
     if (rc != BZ_OK) {
         BZ2_bzReadClose(&rc, bz_file);
         return NULL;
     }
 
     rc = BZ_OK;
     while (rc == BZ_OK) {
         buffer = realloc_safe(buffer, XML_BUFFER_SIZE + length + 1);
         read_len = BZ2_bzRead(&rc, bz_file, buffer + length, XML_BUFFER_SIZE);
 
         crm_trace("Read %ld bytes from file: %d", (long)read_len, rc);
 
         if (rc == BZ_OK || rc == BZ_STREAM_END) {
             length += read_len;
         }
     }
 
     buffer[length] = '\0';
 
     if (rc != BZ_STREAM_END) {
         crm_err("Couldn't read compressed xml from file");
         free(buffer);
         buffer = NULL;
     }
 
     BZ2_bzReadClose(&rc, bz_file);
     fclose(input);
 
 #else
     crm_err("Cannot read compressed files:" " bzlib was not available at compile time");
 #endif
     return buffer;
 }
 
 void
 strip_text_nodes(xmlNode * xml)
 {
     xmlNode *iter = xml->children;
 
     while (iter) {
         xmlNode *next = iter->next;
 
         switch (iter->type) {
             case XML_TEXT_NODE:
                 /* Remove it */
                 xmlUnlinkNode(iter);
                 xmlFreeNode(iter);
                 break;
 
             case XML_ELEMENT_NODE:
                 /* Search it */
                 strip_text_nodes(iter);
                 break;
 
             default:
                 /* Leave it */
                 break;
         }
 
         iter = next;
     }
 }
 
 xmlNode *
 filename2xml(const char *filename)
 {
     xmlNode *xml = NULL;
     xmlDocPtr output = NULL;
     gboolean uncompressed = TRUE;
     xmlParserCtxtPtr ctxt = NULL;
     xmlErrorPtr last_error = NULL;
     static int xml_options = XML_PARSE_NOBLANKS | XML_PARSE_RECOVER;
 
     /* create a parser context */
     ctxt = xmlNewParserCtxt();
     CRM_CHECK(ctxt != NULL, return NULL);
 
     /* xmlCtxtUseOptions(ctxt, XML_PARSE_NOBLANKS|XML_PARSE_RECOVER); */
 
     xmlCtxtResetLastError(ctxt);
     xmlSetGenericErrorFunc(ctxt, crm_xml_err);
     /* initGenericErrorDefaultFunc(crm_xml_err); */
 
     if (filename) {
         uncompressed = !crm_ends_with(filename, ".bz2");
     }
 
     if (filename == NULL) {
         /* STDIN_FILENO == fileno(stdin) */
         output = xmlCtxtReadFd(ctxt, STDIN_FILENO, "unknown.xml", NULL, xml_options);
 
     } else if (uncompressed) {
         output = xmlCtxtReadFile(ctxt, filename, NULL, xml_options);
 
     } else {
         char *input = decompress_file(filename);
 
         output = xmlCtxtReadDoc(ctxt, (const xmlChar *)input, NULL, NULL, xml_options);
         free(input);
     }
 
     if (output && (xml = xmlDocGetRootElement(output))) {
         strip_text_nodes(xml);
     }
 
     last_error = xmlCtxtGetLastError(ctxt);
     if (last_error && last_error->code != XML_ERR_OK) {
         /* crm_abort(__FILE__,__FUNCTION__,__LINE__, "last_error->code != XML_ERR_OK", TRUE, TRUE); */
         /*
          * http://xmlsoft.org/html/libxml-xmlerror.html#xmlErrorLevel
          * http://xmlsoft.org/html/libxml-xmlerror.html#xmlParserErrors
          */
         crm_err("Parsing failed (domain=%d, level=%d, code=%d): %s",
                 last_error->domain, last_error->level, last_error->code, last_error->message);
 
         if (last_error && last_error->code != XML_ERR_OK) {
             crm_err("Couldn't%s parse %s", xml ? " fully" : "", filename);
             if (xml != NULL) {
                 crm_log_xml_err(xml, "Partial");
             }
         }
     }
 
     xmlFreeParserCtxt(ctxt);
     return xml;
 }
 
 /*!
  * \internal
  * \brief Add a "last written" attribute to an XML node, set to current time
  *
  * \param[in] xml_node XML node to get attribute
  *
  * \return Value that was set, or NULL on error
  */
 const char *
 crm_xml_add_last_written(xmlNode *xml_node)
 {
     time_t now = time(NULL);
     char *now_str = ctime(&now);
 
     now_str[24] = EOS; /* replace the newline */
     return crm_xml_add(xml_node, XML_CIB_ATTR_WRITTEN, now_str);
 }
 
 static int
 write_xml_stream(xmlNode * xml_node, const char *filename, FILE * stream, gboolean compress)
 {
     int res = 0;
     char *buffer = NULL;
     unsigned int out = 0;
 
     CRM_CHECK(stream != NULL, return -1);
 
     crm_trace("Writing XML out to %s", filename);
     if (xml_node == NULL) {
         crm_err("Cannot write NULL to %s", filename);
         fclose(stream);
         return -1;
     }
 
 
     crm_log_xml_trace(xml_node, "Writing out");
 
     buffer = dump_xml_formatted(xml_node);
     CRM_CHECK(buffer != NULL && strlen(buffer) > 0, crm_log_xml_warn(xml_node, "dump:failed");
               goto bail);
 
     if (compress) {
 #if HAVE_BZLIB_H
         int rc = BZ_OK;
         unsigned int in = 0;
         BZFILE *bz_file = NULL;
 
         bz_file = BZ2_bzWriteOpen(&rc, stream, 5, 0, 30);
         if (rc != BZ_OK) {
             crm_err("bzWriteOpen failed: %d", rc);
         } else {
             BZ2_bzWrite(&rc, bz_file, buffer, strlen(buffer));
             if (rc != BZ_OK) {
                 crm_err("bzWrite() failed: %d", rc);
             }
         }
 
         if (rc == BZ_OK) {
             BZ2_bzWriteClose(&rc, bz_file, 0, &in, &out);
             if (rc != BZ_OK) {
                 crm_err("bzWriteClose() failed: %d", rc);
                 out = -1;
             } else {
                 crm_trace("%s: In: %d, out: %d", filename, in, out);
             }
         }
 #else
         crm_err("Cannot write compressed files:" " bzlib was not available at compile time");
 #endif
     }
 
     if (out <= 0) {
         res = fprintf(stream, "%s", buffer);
         if (res < 0) {
             crm_perror(LOG_ERR, "Cannot write output to %s", filename);
             goto bail;
         }
     }
 
   bail:
 
     if (fflush(stream) != 0) {
         crm_perror(LOG_ERR, "fflush for %s failed:", filename);
         res = -1;
     }
 
     if (fsync(fileno(stream)) < 0) {
         crm_perror(LOG_ERR, "fsync for %s failed:", filename);
         res = -1;
     }
 
     fclose(stream);
 
     crm_trace("Saved %d bytes to the Cib as XML", res);
     free(buffer);
 
     return res;
 }
 
 int
 write_xml_fd(xmlNode * xml_node, const char *filename, int fd, gboolean compress)
 {
     FILE *stream = NULL;
 
     CRM_CHECK(fd > 0, return -1);
     stream = fdopen(fd, "w");
     return write_xml_stream(xml_node, filename, stream, compress);
 }
 
 int
 write_xml_file(xmlNode * xml_node, const char *filename, gboolean compress)
 {
     FILE *stream = NULL;
 
     stream = fopen(filename, "w");
 
     return write_xml_stream(xml_node, filename, stream, compress);
 }
 
 xmlNode *
 get_message_xml(xmlNode * msg, const char *field)
 {
     xmlNode *tmp = first_named_child(msg, field);
 
     return __xml_first_child(tmp);
 }
 
 gboolean
 add_message_xml(xmlNode * msg, const char *field, xmlNode * xml)
 {
     xmlNode *holder = create_xml_node(msg, field);
 
     add_node_copy(holder, xml);
     return TRUE;
 }
 
 static char *
 crm_xml_escape_shuffle(char *text, int start, int *length, const char *replace)
 {
     int lpc;
     int offset = strlen(replace) - 1;   /* We have space for 1 char already */
 
     *length += offset;
     text = realloc_safe(text, *length);
 
     for (lpc = (*length) - 1; lpc > (start + offset); lpc--) {
         text[lpc] = text[lpc - offset];
     }
 
     memcpy(text + start, replace, offset + 1);
     return text;
 }
 
 char *
 crm_xml_escape(const char *text)
 {
     int index;
     int changes = 0;
     int length = 1 + strlen(text);
     char *copy = strdup(text);
 
     /*
      * When xmlCtxtReadDoc() parses &lt; and friends in a
      * value, it converts them to their human readable
      * form.
      *
      * If one uses xmlNodeDump() to convert it back to a
      * string, all is well, because special characters are
      * converted back to their escape sequences.
      *
      * However xmlNodeDump() is randomly dog slow, even with the same
      * input. So we need to replicate the escapeing in our custom
      * version so that the result can be re-parsed by xmlCtxtReadDoc()
      * when necessary.
      */
 
     for (index = 0; index < length; index++) {
         switch (copy[index]) {
             case 0:
                 break;
             case '<':
                 copy = crm_xml_escape_shuffle(copy, index, &length, "&lt;");
                 changes++;
                 break;
             case '>':
                 copy = crm_xml_escape_shuffle(copy, index, &length, "&gt;");
                 changes++;
                 break;
             case '"':
                 copy = crm_xml_escape_shuffle(copy, index, &length, "&quot;");
                 changes++;
                 break;
             case '\'':
                 copy = crm_xml_escape_shuffle(copy, index, &length, "&apos;");
                 changes++;
                 break;
             case '&':
                 copy = crm_xml_escape_shuffle(copy, index, &length, "&amp;");
                 changes++;
                 break;
             case '\t':
                 /* Might as well just expand to a few spaces... */
                 copy = crm_xml_escape_shuffle(copy, index, &length, "    ");
                 changes++;
                 break;
             case '\n':
                 /* crm_trace("Convert: \\%.3o", copy[index]); */
                 copy = crm_xml_escape_shuffle(copy, index, &length, "\\n");
                 changes++;
                 break;
             case '\r':
                 copy = crm_xml_escape_shuffle(copy, index, &length, "\\r");
                 changes++;
                 break;
                 /* For debugging...
             case '\\':
                 crm_trace("Passthrough: \\%c", copy[index+1]);
                 break;
                 */
             default:
                 /* Check for and replace non-printing characters with their octal equivalent */
                 if(copy[index] < ' ' || copy[index] > '~') {
                     char *replace = crm_strdup_printf("\\%.3o", copy[index]);
 
                     /* crm_trace("Convert to octal: \\%.3o", copy[index]); */
                     copy = crm_xml_escape_shuffle(copy, index, &length, replace);
                     free(replace);
                     changes++;
                 }
         }
     }
 
     if (changes) {
         crm_trace("Dumped '%s'", copy);
     }
     return copy;
 }
 
 static inline void
 dump_xml_attr(xmlAttrPtr attr, int options, char **buffer, int *offset, int *max)
 {
     char *p_value = NULL;
     const char *p_name = NULL;
     xml_private_t *p = NULL;
 
     CRM_ASSERT(buffer != NULL);
     if (attr == NULL || attr->children == NULL) {
         return;
     }
 
     p = attr->_private;
     if (p && is_set(p->flags, xpf_deleted)) {
         return;
     }
 
     p_name = (const char *)attr->name;
     p_value = crm_xml_escape((const char *)attr->children->content);
     buffer_print(*buffer, *max, *offset, " %s=\"%s\"", p_name, p_value);
     free(p_value);
 }
 
 static void
 __xml_log_element(int log_level, const char *file, const char *function, int line,
                   const char *prefix, xmlNode * data, int depth, int options)
 {
     int max = 0;
     int offset = 0;
     const char *name = NULL;
     const char *hidden = NULL;
 
     xmlNode *child = NULL;
     xmlAttrPtr pIter = NULL;
 
     if(data == NULL) {
         return;
     }
 
     name = crm_element_name(data);
 
     if(is_set(options, xml_log_option_open)) {
         char *buffer = NULL;
 
         insert_prefix(options, &buffer, &offset, &max, depth);
 
         if (data->type == XML_COMMENT_NODE) {
             buffer_print(buffer, max, offset, "<!--%s-->", data->content);
 
         } else {
             buffer_print(buffer, max, offset, "<%s", name);
 
             hidden = crm_element_value(data, "hidden");
             for (pIter = crm_first_attr(data); pIter != NULL; pIter = pIter->next) {
                 xml_private_t *p = pIter->_private;
                 const char *p_name = (const char *)pIter->name;
                 const char *p_value = crm_attr_value(pIter);
                 char *p_copy = NULL;
 
                 if(is_set(p->flags, xpf_deleted)) {
                     continue;
                 } else if ((is_set(options, xml_log_option_diff_plus)
                      || is_set(options, xml_log_option_diff_minus))
                     && strcmp(XML_DIFF_MARKER, p_name) == 0) {
                     continue;
 
                 } else if (hidden != NULL && p_name[0] != 0 && strstr(hidden, p_name) != NULL) {
                     p_copy = strdup("*****");
 
                 } else {
                     p_copy = crm_xml_escape(p_value);
                 }
 
                 buffer_print(buffer, max, offset, " %s=\"%s\"", p_name, p_copy);
                 free(p_copy);
             }
 
             if(xml_has_children(data) == FALSE) {
                 buffer_print(buffer, max, offset, "/>");
 
             } else if(is_set(options, xml_log_option_children)) {
                 buffer_print(buffer, max, offset, ">");
 
             } else {
                 buffer_print(buffer, max, offset, "/>");
             }
         }
 
         do_crm_log_alias(log_level, file, function, line, "%s %s", prefix, buffer);
         free(buffer);
     }
 
     if(data->type == XML_COMMENT_NODE) {
         return;
 
     } else if(xml_has_children(data) == FALSE) {
         return;
 
     } else if(is_set(options, xml_log_option_children)) {
         offset = 0;
         max = 0;
 
         for (child = __xml_first_child(data); child != NULL; child = __xml_next(child)) {
             __xml_log_element(log_level, file, function, line, prefix, child, depth + 1, options|xml_log_option_open|xml_log_option_close);
         }
     }
 
     if(is_set(options, xml_log_option_close)) {
         char *buffer = NULL;
 
         insert_prefix(options, &buffer, &offset, &max, depth);
         buffer_print(buffer, max, offset, "</%s>", name);
 
         do_crm_log_alias(log_level, file, function, line, "%s %s", prefix, buffer);
         free(buffer);
     }
 }
 
 static void
 __xml_log_change_element(int log_level, const char *file, const char *function, int line,
                          const char *prefix, xmlNode * data, int depth, int options)
 {
     xml_private_t *p;
     char *prefix_m = NULL;
     xmlNode *child = NULL;
     xmlAttrPtr pIter = NULL;
 
     if(data == NULL) {
         return;
     }
 
     p = data->_private;
 
     prefix_m = strdup(prefix);
     prefix_m[1] = '+';
 
     if(is_set(p->flags, xpf_dirty) && is_set(p->flags, xpf_created)) {
         /* Continue and log full subtree */
         __xml_log_element(log_level, file, function, line,
                           prefix_m, data, depth, options|xml_log_option_open|xml_log_option_close|xml_log_option_children);
 
     } else if(is_set(p->flags, xpf_dirty)) {
         char *spaces = calloc(80, 1);
         int s_count = 0, s_max = 80;
         char *prefix_del = NULL;
         char *prefix_moved = NULL;
         const char *flags = prefix;
 
         insert_prefix(options, &spaces, &s_count, &s_max, depth);
         prefix_del = strdup(prefix);
         prefix_del[0] = '-';
         prefix_del[1] = '-';
         prefix_moved = strdup(prefix);
         prefix_moved[1] = '~';
 
         if(is_set(p->flags, xpf_moved)) {
             flags = prefix_moved;
         } else {
             flags = prefix;
         }
 
         __xml_log_element(log_level, file, function, line,
                           flags, data, depth, options|xml_log_option_open);
 
         for (pIter = crm_first_attr(data); pIter != NULL; pIter = pIter->next) {
             const char *aname = (const char*)pIter->name;
 
             p = pIter->_private;
             if(is_set(p->flags, xpf_deleted)) {
                 const char *value = crm_element_value(data, aname);
                 flags = prefix_del;
                 do_crm_log_alias(log_level, file, function, line,
                                  "%s %s @%s=%s", flags, spaces, aname, value);
 
             } else if(is_set(p->flags, xpf_dirty)) {
                 const char *value = crm_element_value(data, aname);
 
                 if(is_set(p->flags, xpf_created)) {
                     flags = prefix_m;
 
                 } else if(is_set(p->flags, xpf_modified)) {
                     flags = prefix;
 
                 } else if(is_set(p->flags, xpf_moved)) {
                     flags = prefix_moved;
 
                 } else {
                     flags = prefix;
                 }
                 do_crm_log_alias(log_level, file, function, line,
                                  "%s %s @%s=%s", flags, spaces, aname, value);
             }
         }
         free(prefix_moved);
         free(prefix_del);
         free(spaces);
 
         for (child = __xml_first_child(data); child != NULL; child = __xml_next(child)) {
             __xml_log_change_element(log_level, file, function, line, prefix, child, depth + 1, options);
         }
 
         __xml_log_element(log_level, file, function, line,
                           prefix, data, depth, options|xml_log_option_close);
 
     } else {
         for (child = __xml_first_child(data); child != NULL; child = __xml_next(child)) {
             __xml_log_change_element(log_level, file, function, line, prefix, child, depth + 1, options);
         }
     }
 
     free(prefix_m);
 
 }
 
 void
 log_data_element(int log_level, const char *file, const char *function, int line,
                  const char *prefix, xmlNode * data, int depth, int options)
 {
     xmlNode *a_child = NULL;
 
     char *prefix_m = NULL;
 
     if (prefix == NULL) {
         prefix = "";
     }
 
     /* Since we use the same file and line, to avoid confusing libqb, we need to use the same format strings */
     if (data == NULL) {
         do_crm_log_alias(log_level, file, function, line, "%s: %s", prefix,
                          "No data to dump as XML");
         return;
     }
 
     if(is_set(options, xml_log_option_dirty_add) || is_set(options, xml_log_option_dirty_add)) {
         __xml_log_change_element(log_level, file, function, line, prefix, data, depth, options);
         return;
     }
 
     if (is_set(options, xml_log_option_formatted)) {
         if (is_set(options, xml_log_option_diff_plus)
             && (data->children == NULL || crm_element_value(data, XML_DIFF_MARKER))) {
             options |= xml_log_option_diff_all;
             prefix_m = strdup(prefix);
             prefix_m[1] = '+';
             prefix = prefix_m;
 
         } else if (is_set(options, xml_log_option_diff_minus)
                    && (data->children == NULL || crm_element_value(data, XML_DIFF_MARKER))) {
             options |= xml_log_option_diff_all;
             prefix_m = strdup(prefix);
             prefix_m[1] = '-';
             prefix = prefix_m;
         }
     }
 
     if (is_set(options, xml_log_option_diff_short)
                && is_not_set(options, xml_log_option_diff_all)) {
         /* Still searching for the actual change */
         for (a_child = __xml_first_child(data); a_child != NULL; a_child = __xml_next(a_child)) {
             log_data_element(log_level, file, function, line, prefix, a_child, depth + 1, options);
         }
     } else {
         __xml_log_element(log_level, file, function, line, prefix, data, depth,
                           options|xml_log_option_open|xml_log_option_close|xml_log_option_children);
     }
     free(prefix_m);
 }
 
 static void
 dump_filtered_xml(xmlNode * data, int options, char **buffer, int *offset, int *max)
 {
     int lpc;
     xmlAttrPtr xIter = NULL;
     static int filter_len = DIMOF(filter);
 
     for (lpc = 0; options && lpc < filter_len; lpc++) {
         filter[lpc].found = FALSE;
     }
 
     for (xIter = crm_first_attr(data); xIter != NULL; xIter = xIter->next) {
         bool skip = FALSE;
         const char *p_name = (const char *)xIter->name;
 
         for (lpc = 0; skip == FALSE && lpc < filter_len; lpc++) {
             if (filter[lpc].found == FALSE && strcmp(p_name, filter[lpc].string) == 0) {
                 filter[lpc].found = TRUE;
                 skip = TRUE;
                 break;
             }
         }
 
         if (skip == FALSE) {
             dump_xml_attr(xIter, options, buffer, offset, max);
         }
     }
 }
 
 static void
 dump_xml_element(xmlNode * data, int options, char **buffer, int *offset, int *max, int depth)
 {
     const char *name = NULL;
 
     CRM_ASSERT(max != NULL);
     CRM_ASSERT(offset != NULL);
     CRM_ASSERT(buffer != NULL);
 
     if (data == NULL) {
         crm_trace("Nothing to dump");
         return;
     }
 
     if (*buffer == NULL) {
         *offset = 0;
         *max = 0;
     }
 
     name = crm_element_name(data);
     CRM_ASSERT(name != NULL);
 
     insert_prefix(options, buffer, offset, max, depth);
     buffer_print(*buffer, *max, *offset, "<%s", name);
 
     if (options & xml_log_option_filtered) {
         dump_filtered_xml(data, options, buffer, offset, max);
 
     } else {
         xmlAttrPtr xIter = NULL;
 
         for (xIter = crm_first_attr(data); xIter != NULL; xIter = xIter->next) {
             dump_xml_attr(xIter, options, buffer, offset, max);
         }
     }
 
     if (data->children == NULL) {
         buffer_print(*buffer, *max, *offset, "/>");
 
     } else {
         buffer_print(*buffer, *max, *offset, ">");
     }
 
     if (options & xml_log_option_formatted) {
         buffer_print(*buffer, *max, *offset, "\n");
     }
 
     if (data->children) {
         xmlNode *xChild = NULL;
         for(xChild = data->children; xChild != NULL; xChild = xChild->next) {
             crm_xml_dump(xChild, options, buffer, offset, max, depth + 1);
         }
 
         insert_prefix(options, buffer, offset, max, depth);
         buffer_print(*buffer, *max, *offset, "</%s>", name);
 
         if (options & xml_log_option_formatted) {
             buffer_print(*buffer, *max, *offset, "\n");
         }
     }
 }
 
 static void
 dump_xml_text(xmlNode * data, int options, char **buffer, int *offset, int *max, int depth)
 {
     CRM_ASSERT(max != NULL);
     CRM_ASSERT(offset != NULL);
     CRM_ASSERT(buffer != NULL);
 
     if (data == NULL) {
         crm_trace("Nothing to dump");
         return;
     }
 
     if (*buffer == NULL) {
         *offset = 0;
         *max = 0;
     }
 
     insert_prefix(options, buffer, offset, max, depth);
 
     buffer_print(*buffer, *max, *offset, "%s", data->content);
 
     if (options & xml_log_option_formatted) {
         buffer_print(*buffer, *max, *offset, "\n");
     }
 }
 
 
 static void
 dump_xml_comment(xmlNode * data, int options, char **buffer, int *offset, int *max, int depth)
 {
     CRM_ASSERT(max != NULL);
     CRM_ASSERT(offset != NULL);
     CRM_ASSERT(buffer != NULL);
 
     if (data == NULL) {
         crm_trace("Nothing to dump");
         return;
     }
 
     if (*buffer == NULL) {
         *offset = 0;
         *max = 0;
     }
 
     insert_prefix(options, buffer, offset, max, depth);
 
     buffer_print(*buffer, *max, *offset, "<!--");
     buffer_print(*buffer, *max, *offset, "%s", data->content);
     buffer_print(*buffer, *max, *offset, "-->");
 
     if (options & xml_log_option_formatted) {
         buffer_print(*buffer, *max, *offset, "\n");
     }
 }
 
 void
 crm_xml_dump(xmlNode * data, int options, char **buffer, int *offset, int *max, int depth)
 {
     if(data == NULL) {
         *offset = 0;
         *max = 0;
         return;
     }
 #if 0
     if (is_not_set(options, xml_log_option_filtered)) {
         /* Turning this code on also changes the PE tests for some reason
          * (not just newlines).  Figure out why before considering to
          * enable this permanently.
          *
          * It exists to help debug slowness in xmlNodeDump() and
          * potentially if we ever want to go back to it.
          *
          * In theory it's a good idea (reuse) but our custom version does
          * better for the filtered case and avoids the final strdup() for
          * everything
          */
 
         time_t now, next;
         xmlDoc *doc = NULL;
         xmlBuffer *xml_buffer = NULL;
 
         *buffer = NULL;
         doc = getDocPtr(data);
         /* doc will only be NULL if data is */
         CRM_CHECK(doc != NULL, return);
 
         now = time(NULL);
         xml_buffer = xmlBufferCreate();
         CRM_ASSERT(xml_buffer != NULL);
 
         /* The default allocator XML_BUFFER_ALLOC_EXACT does far too many
          * realloc()s and it can take upwards of 18 seconds (yes, seconds)
          * to dump a 28kb tree which XML_BUFFER_ALLOC_DOUBLEIT can do in
          * less than 1 second.
          *
          * We could also use xmlBufferCreateSize() to start with a
          * sane-ish initial size and avoid the first few doubles.
          */
         xmlBufferSetAllocationScheme(xml_buffer, XML_BUFFER_ALLOC_DOUBLEIT);
 
         *max = xmlNodeDump(xml_buffer, doc, data, 0, (options & xml_log_option_formatted));
         if (*max > 0) {
             *buffer = strdup((char *)xml_buffer->content);
         }
 
         next = time(NULL);
         if ((now + 1) < next) {
             crm_log_xml_trace(data, "Long time");
             crm_err("xmlNodeDump() -> %dbytes took %ds", *max, next - now);
         }
 
         xmlBufferFree(xml_buffer);
         return;
     }
 #endif
 
     switch(data->type) {
         case XML_ELEMENT_NODE:
             /* Handle below */
             dump_xml_element(data, options, buffer, offset, max, depth);
             break;
         case XML_TEXT_NODE:
             /* if option xml_log_option_text is enabled, then dump XML_TEXT_NODE */
             if (options & xml_log_option_text) {
                 dump_xml_text(data, options, buffer, offset, max, depth);
             }
             return;
         case XML_COMMENT_NODE:
             dump_xml_comment(data, options, buffer, offset, max, depth);
             break;
         default:
             crm_warn("Unhandled type: %d", data->type);
             return;
 
             /*
             XML_ATTRIBUTE_NODE = 2
             XML_CDATA_SECTION_NODE = 4
             XML_ENTITY_REF_NODE = 5
             XML_ENTITY_NODE = 6
             XML_PI_NODE = 7
             XML_DOCUMENT_NODE = 9
             XML_DOCUMENT_TYPE_NODE = 10
             XML_DOCUMENT_FRAG_NODE = 11
             XML_NOTATION_NODE = 12
             XML_HTML_DOCUMENT_NODE = 13
             XML_DTD_NODE = 14
             XML_ELEMENT_DECL = 15
             XML_ATTRIBUTE_DECL = 16
             XML_ENTITY_DECL = 17
             XML_NAMESPACE_DECL = 18
             XML_XINCLUDE_START = 19
             XML_XINCLUDE_END = 20
             XML_DOCB_DOCUMENT_NODE = 21
             */
     }
 
 }
 
 void
 crm_buffer_add_char(char **buffer, int *offset, int *max, char c)
 {
     buffer_print(*buffer, *max, *offset, "%c", c);
 }
 
 char *
 dump_xml_formatted_with_text(xmlNode * an_xml_node)
 {
     char *buffer = NULL;
     int offset = 0, max = 0;
 
     crm_xml_dump(an_xml_node, xml_log_option_formatted|xml_log_option_text, &buffer, &offset, &max, 0);
     return buffer;
 }
 
 char *
 dump_xml_formatted(xmlNode * an_xml_node)
 {
     char *buffer = NULL;
     int offset = 0, max = 0;
 
     crm_xml_dump(an_xml_node, xml_log_option_formatted, &buffer, &offset, &max, 0);
     return buffer;
 }
 
 char *
 dump_xml_unformatted(xmlNode * an_xml_node)
 {
     char *buffer = NULL;
     int offset = 0, max = 0;
 
     crm_xml_dump(an_xml_node, 0, &buffer, &offset, &max, 0);
     return buffer;
 }
 
 gboolean
 xml_has_children(const xmlNode * xml_root)
 {
     if (xml_root != NULL && xml_root->children != NULL) {
         return TRUE;
     }
     return FALSE;
 }
 
 int
 crm_element_value_int(xmlNode * data, const char *name, int *dest)
 {
     const char *value = crm_element_value(data, name);
 
     CRM_CHECK(dest != NULL, return -1);
     if (value) {
         *dest = crm_int_helper(value, NULL);
         return 0;
     }
     return -1;
 }
 
 int
 crm_element_value_const_int(const xmlNode * data, const char *name, int *dest)
 {
     return crm_element_value_int((xmlNode *) data, name, dest);
 }
 
 const char *
 crm_element_value_const(const xmlNode * data, const char *name)
 {
     return crm_element_value((xmlNode *) data, name);
 }
 
 char *
 crm_element_value_copy(xmlNode * data, const char *name)
 {
     char *value_copy = NULL;
     const char *value = crm_element_value(data, name);
 
     if (value != NULL) {
         value_copy = strdup(value);
     }
     return value_copy;
 }
 
 void
 xml_remove_prop(xmlNode * obj, const char *name)
 {
     if(__xml_acl_check(obj, NULL, xpf_acl_write) == FALSE) {
         crm_trace("Cannot remove %s from %s", name, obj->name);
 
     } else if(TRACKING_CHANGES(obj)) {
         /* Leave in place (marked for removal) until after the diff is calculated */
         xml_private_t *p = NULL;
         xmlAttr *attr = xmlHasProp(obj, (const xmlChar *)name);
 
         p = attr->_private;
         set_parent_flag(obj, xpf_dirty);
         p->flags |= xpf_deleted;
         /* crm_trace("Setting flag %x due to %s[@id=%s].%s", xpf_dirty, obj->name, ID(obj), name); */
 
     } else {
         xmlUnsetProp(obj, (const xmlChar *)name);
     }
 }
 
 void
 purge_diff_markers(xmlNode * a_node)
 {
     xmlNode *child = NULL;
 
     CRM_CHECK(a_node != NULL, return);
 
     xml_remove_prop(a_node, XML_DIFF_MARKER);
     for (child = __xml_first_child(a_node); child != NULL; child = __xml_next(child)) {
         purge_diff_markers(child);
     }
 }
 
 void
 save_xml_to_file(xmlNode * xml, const char *desc, const char *filename)
 {
     char *f = NULL;
 
     if (filename == NULL) {
         char *uuid = crm_generate_uuid();
 
         f = crm_strdup_printf("/tmp/%s", uuid);
         filename = f;
         free(uuid);
     }
 
     crm_info("Saving %s to %s", desc, filename);
     write_xml_file(xml, filename, FALSE);
     free(f);
 }
 
 gboolean
 apply_xml_diff(xmlNode * old, xmlNode * diff, xmlNode ** new)
 {
     gboolean result = TRUE;
     int root_nodes_seen = 0;
     static struct qb_log_callsite *digest_cs = NULL;
     const char *digest = crm_element_value(diff, XML_ATTR_DIGEST);
     const char *version = crm_element_value(diff, XML_ATTR_CRM_VERSION);
 
     xmlNode *child_diff = NULL;
     xmlNode *added = find_xml_node(diff, "diff-added", FALSE);
     xmlNode *removed = find_xml_node(diff, "diff-removed", FALSE);
 
     CRM_CHECK(new != NULL, return FALSE);
     if (digest_cs == NULL) {
         digest_cs =
             qb_log_callsite_get(__func__, __FILE__, "diff-digest", LOG_TRACE, __LINE__,
                                 crm_trace_nonlog);
     }
 
     crm_trace("Subtraction Phase");
     for (child_diff = __xml_first_child(removed); child_diff != NULL;
          child_diff = __xml_next(child_diff)) {
         CRM_CHECK(root_nodes_seen == 0, result = FALSE);
         if (root_nodes_seen == 0) {
             *new = subtract_xml_object(NULL, old, child_diff, FALSE, NULL, NULL);
         }
         root_nodes_seen++;
     }
 
     if (root_nodes_seen == 0) {
         *new = copy_xml(old);
 
     } else if (root_nodes_seen > 1) {
         crm_err("(-) Diffs cannot contain more than one change set..." " saw %d", root_nodes_seen);
         result = FALSE;
     }
 
     root_nodes_seen = 0;
     crm_trace("Addition Phase");
     if (result) {
         xmlNode *child_diff = NULL;
 
         for (child_diff = __xml_first_child(added); child_diff != NULL;
              child_diff = __xml_next(child_diff)) {
             CRM_CHECK(root_nodes_seen == 0, result = FALSE);
             if (root_nodes_seen == 0) {
                 add_xml_object(NULL, *new, child_diff, TRUE);
             }
             root_nodes_seen++;
         }
     }
 
     if (root_nodes_seen > 1) {
         crm_err("(+) Diffs cannot contain more than one change set..." " saw %d", root_nodes_seen);
         result = FALSE;
 
     } else if (result && digest) {
         char *new_digest = NULL;
 
         purge_diff_markers(*new);       /* Purge now so the diff is ok */
         new_digest = calculate_xml_versioned_digest(*new, FALSE, TRUE, version);
         if (safe_str_neq(new_digest, digest)) {
             crm_info("Digest mis-match: expected %s, calculated %s", digest, new_digest);
             result = FALSE;
 
             crm_trace("%p %.6x", digest_cs, digest_cs ? digest_cs->targets : 0);
             if (digest_cs && digest_cs->targets) {
                 save_xml_to_file(old, "diff:original", NULL);
                 save_xml_to_file(diff, "diff:input", NULL);
                 save_xml_to_file(*new, "diff:new", NULL);
             }
 
         } else {
             crm_trace("Digest matched: expected %s, calculated %s", digest, new_digest);
         }
         free(new_digest);
 
     } else if (result) {
         purge_diff_markers(*new);       /* Purge now so the diff is ok */
     }
 
     return result;
 }
 
 static void
 __xml_diff_object(xmlNode * old, xmlNode * new)
 {
     xmlNode *cIter = NULL;
     xmlAttr *pIter = NULL;
 
     CRM_CHECK(new != NULL, return);
     if(old == NULL) {
         crm_node_created(new);
         __xml_acl_post_process(new); /* Check creation is allowed */
         return;
 
     } else {
         xml_private_t *p = new->_private;
 
         if(p->flags & xpf_processed) {
             /* Avoid re-comparing nodes */
             return;
         }
         p->flags |= xpf_processed;
     }
 
     for (pIter = crm_first_attr(new); pIter != NULL; pIter = pIter->next) {
         xml_private_t *p = pIter->_private;
 
         /* Assume everything was just created and take it from there */
         p->flags |= xpf_created;
     }
 
     for (pIter = crm_first_attr(old); pIter != NULL; ) {
         xmlAttr *prop = pIter;
         xml_private_t *p = NULL;
         const char *name = (const char *)pIter->name;
         const char *old_value = crm_element_value(old, name);
         xmlAttr *exists = xmlHasProp(new, pIter->name);
 
         pIter = pIter->next;
         if(exists == NULL) {
             p = new->doc->_private;
 
             /* Prevent the dirty flag being set recursively upwards */
             clear_bit(p->flags, xpf_tracking);
             exists = xmlSetProp(new, (const xmlChar *)name, (const xmlChar *)old_value);
             set_bit(p->flags, xpf_tracking);
 
             p = exists->_private;
             p->flags = 0;
 
             crm_trace("Lost %s@%s=%s", old->name, name, old_value);
             xml_remove_prop(new, name);
 
         } else {
             int p_new = __xml_offset((xmlNode*)exists);
             int p_old = __xml_offset((xmlNode*)prop);
             const char *value = crm_element_value(new, name);
 
             p = exists->_private;
             p->flags = (p->flags & ~xpf_created);
 
             if(strcmp(value, old_value) != 0) {
                 /* Restore the original value, so we can call crm_xml_add(),
                  * which checks ACLs
                  */
                 char *vcopy = crm_element_value_copy(new, name);
 
                 crm_trace("Modified %s@%s %s->%s", old->name, name, old_value, vcopy);
                 xmlSetProp(new, prop->name, (const xmlChar *)old_value);
                 crm_xml_add(new, name, vcopy);
                 free(vcopy);
 
             } else if(p_old != p_new) {
                 crm_info("Moved %s@%s (%d -> %d)", old->name, name, p_old, p_new);
                 __xml_node_dirty(new);
                 p->flags |= xpf_dirty|xpf_moved;
 
                 if(p_old > p_new) {
                     p = prop->_private;
                     p->flags |= xpf_skip;
 
                 } else {
                     p = exists->_private;
                     p->flags |= xpf_skip;
                 }
             }
         }
     }
 
     for (pIter = crm_first_attr(new); pIter != NULL; ) {
         xmlAttr *prop = pIter;
         xml_private_t *p = pIter->_private;
 
         pIter = pIter->next;
         if(is_set(p->flags, xpf_created)) {
             char *name = strdup((const char *)prop->name);
             char *value = crm_element_value_copy(new, name);
 
             crm_trace("Created %s@%s=%s", new->name, name, value);
             /* Remove plus create won't work as it will modify the relative attribute ordering */
             if(__xml_acl_check(new, name, xpf_acl_write)) {
                 crm_attr_dirty(prop);
             } else {
                 xmlUnsetProp(new, prop->name); /* Remove - change not allowed */
             }
 
             free(value);
             free(name);
         }
     }
 
     for (cIter = __xml_first_child(old); cIter != NULL; ) {
         xmlNode *old_child = cIter;
         xmlNode *new_child = find_element(new, cIter);
 
         cIter = __xml_next(cIter);
         if(new_child) {
             __xml_diff_object(old_child, new_child);
 
         } else {
             /* Create then free (which will check the acls if necessary) */
             xmlNode *candidate = add_node_copy(new, old_child);
             xmlNode *top = xmlDocGetRootElement(candidate->doc);
 
             __xml_node_clean(candidate);
             __xml_acl_apply(top); /* Make sure any ACLs are applied to 'candidate' */
             free_xml(candidate);
 
             if (find_element(new, old_child) == NULL) {
                 xml_private_t *p = old_child->_private;
 
                 p->flags |= xpf_skip;
             }
         }
     }
 
     for (cIter = __xml_first_child(new); cIter != NULL; ) {
         xmlNode *new_child = cIter;
         xmlNode *old_child = find_element(old, cIter);
 
         cIter = __xml_next(cIter);
         if(old_child == NULL) {
             xml_private_t *p = new_child->_private;
             p->flags |= xpf_skip;
             __xml_diff_object(old_child, new_child);
 
         } else {
             /* Check for movement, we already checked for differences */
             int p_new = __xml_offset(new_child);
             int p_old = __xml_offset(old_child);
 
             if(p_old != p_new) {
                 xml_private_t *p = new_child->_private;
 
                 crm_info("%s.%s moved from %d to %d",
                          new_child->name, ID(new_child), p_old, p_new);
                 __xml_node_dirty(new);
                 p->flags |= xpf_moved;
 
                 if(p_old > p_new) {
                     p = old_child->_private;
                 } else {
                     p = new_child->_private;
                 }
                 p->flags |= xpf_skip;
             }
         }
     }
 }
 
 void
 xml_calculate_changes(xmlNode * old, xmlNode * new)
 {
     CRM_CHECK(safe_str_eq(crm_element_name(old), crm_element_name(new)), return);
     CRM_CHECK(safe_str_eq(ID(old), ID(new)), return);
 
     if(xml_tracking_changes(new) == FALSE) {
         xml_track_changes(new, NULL, NULL, FALSE);
     }
 
     __xml_diff_object(old, new);
 }
 
 xmlNode *
 diff_xml_object(xmlNode * old, xmlNode * new, gboolean suppress)
 {
     xmlNode *tmp1 = NULL;
     xmlNode *diff = create_xml_node(NULL, "diff");
     xmlNode *removed = create_xml_node(diff, "diff-removed");
     xmlNode *added = create_xml_node(diff, "diff-added");
 
     crm_xml_add(diff, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
 
     tmp1 = subtract_xml_object(removed, old, new, FALSE, NULL, "removed:top");
     if (suppress && tmp1 != NULL && can_prune_leaf(tmp1)) {
         free_xml(tmp1);
     }
 
     tmp1 = subtract_xml_object(added, new, old, TRUE, NULL, "added:top");
     if (suppress && tmp1 != NULL && can_prune_leaf(tmp1)) {
         free_xml(tmp1);
     }
 
     if (added->children == NULL && removed->children == NULL) {
         free_xml(diff);
         diff = NULL;
     }
 
     return diff;
 }
 
 gboolean
 can_prune_leaf(xmlNode * xml_node)
 {
     xmlNode *cIter = NULL;
     xmlAttrPtr pIter = NULL;
     gboolean can_prune = TRUE;
     const char *name = crm_element_name(xml_node);
 
     if (safe_str_eq(name, XML_TAG_RESOURCE_REF)
         || safe_str_eq(name, XML_CIB_TAG_OBJ_REF)
         || safe_str_eq(name, XML_ACL_TAG_ROLE_REF)
         || safe_str_eq(name, XML_ACL_TAG_ROLE_REFv1)) {
         return FALSE;
     }
 
     for (pIter = crm_first_attr(xml_node); pIter != NULL; pIter = pIter->next) {
         const char *p_name = (const char *)pIter->name;
 
         if (strcmp(p_name, XML_ATTR_ID) == 0) {
             continue;
         }
         can_prune = FALSE;
     }
 
     cIter = __xml_first_child(xml_node);
     while (cIter) {
         xmlNode *child = cIter;
 
         cIter = __xml_next(cIter);
         if (can_prune_leaf(child)) {
             free_xml(child);
         } else {
             can_prune = FALSE;
         }
     }
     return can_prune;
 }
 
 void
 diff_filter_context(int context, int upper_bound, int lower_bound,
                     xmlNode * xml_node, xmlNode * parent)
 {
     xmlNode *us = NULL;
     xmlNode *child = NULL;
     xmlAttrPtr pIter = NULL;
     xmlNode *new_parent = parent;
     const char *name = crm_element_name(xml_node);
 
     CRM_CHECK(xml_node != NULL && name != NULL, return);
 
     us = create_xml_node(parent, name);
     for (pIter = crm_first_attr(xml_node); pIter != NULL; pIter = pIter->next) {
         const char *p_name = (const char *)pIter->name;
         const char *p_value = crm_attr_value(pIter);
 
         lower_bound = context;
         crm_xml_add(us, p_name, p_value);
     }
 
     if (lower_bound >= 0 || upper_bound >= 0) {
         crm_xml_add(us, XML_ATTR_ID, ID(xml_node));
         new_parent = us;
 
     } else {
         upper_bound = in_upper_context(0, context, xml_node);
         if (upper_bound >= 0) {
             crm_xml_add(us, XML_ATTR_ID, ID(xml_node));
             new_parent = us;
         } else {
             free_xml(us);
             us = NULL;
         }
     }
 
     for (child = __xml_first_child(us); child != NULL; child = __xml_next(child)) {
         diff_filter_context(context, upper_bound - 1, lower_bound - 1, child, new_parent);
     }
 }
 
 int
 in_upper_context(int depth, int context, xmlNode * xml_node)
 {
     if (context == 0) {
         return 0;
     }
 
     if (xml_node->properties) {
         return depth;
 
     } else if (depth < context) {
         xmlNode *child = NULL;
 
         for (child = __xml_first_child(xml_node); child != NULL; child = __xml_next(child)) {
             if (in_upper_context(depth + 1, context, child)) {
                 return depth;
             }
         }
     }
     return 0;
 }
 
 static xmlNode *
 find_xml_comment(xmlNode * root, xmlNode * search_comment)
 {
     xmlNode *a_child = NULL;
 
     CRM_CHECK(search_comment->type == XML_COMMENT_NODE, return NULL);
 
     for (a_child = __xml_first_child(root); a_child != NULL; a_child = __xml_next(a_child)) {
         if (a_child->type != XML_COMMENT_NODE) {
             continue;
         }
         if (safe_str_eq((const char *)a_child->content, (const char *)search_comment->content)) {
             return a_child;
         }
     }
 
     return NULL;
 }
 
 static xmlNode *
 subtract_xml_comment(xmlNode * parent, xmlNode * left, xmlNode * right,
                      gboolean * changed)
 {
     CRM_CHECK(left != NULL, return NULL);
     CRM_CHECK(left->type == XML_COMMENT_NODE, return NULL);
 
     if (right == NULL
         || safe_str_neq((const char *)left->content, (const char *)right->content)) {
         xmlNode *deleted = NULL;
 
         deleted = add_node_copy(parent, left);
         *changed = TRUE;
 
         return deleted;
     }
 
     return NULL;
 }
 
 xmlNode *
 subtract_xml_object(xmlNode * parent, xmlNode * left, xmlNode * right,
                     gboolean full, gboolean * changed, const char *marker)
 {
     gboolean dummy = FALSE;
     gboolean skip = FALSE;
     xmlNode *diff = NULL;
     xmlNode *right_child = NULL;
     xmlNode *left_child = NULL;
     xmlAttrPtr xIter = NULL;
 
     const char *id = NULL;
     const char *name = NULL;
     const char *value = NULL;
     const char *right_val = NULL;
 
     int lpc = 0;
     static int filter_len = DIMOF(filter);
 
     if (changed == NULL) {
         changed = &dummy;
     }
 
     if (left == NULL) {
         return NULL;
     }
 
     if (left->type == XML_COMMENT_NODE) {
         return subtract_xml_comment(parent, left, right, changed);
     }
 
     id = ID(left);
     if (right == NULL) {
         xmlNode *deleted = NULL;
 
         crm_trace("Processing <%s id=%s> (complete copy)", crm_element_name(left), id);
         deleted = add_node_copy(parent, left);
         crm_xml_add(deleted, XML_DIFF_MARKER, marker);
 
         *changed = TRUE;
         return deleted;
     }
 
     name = crm_element_name(left);
     CRM_CHECK(name != NULL, return NULL);
     CRM_CHECK(safe_str_eq(crm_element_name(left), crm_element_name(right)), return NULL);
 
     /* check for XML_DIFF_MARKER in a child */
     value = crm_element_value(right, XML_DIFF_MARKER);
     if (value != NULL && strcmp(value, "removed:top") == 0) {
         crm_trace("We are the root of the deletion: %s.id=%s", name, id);
         *changed = TRUE;
         return NULL;
     }
 
     /* Avoiding creating the full heirarchy would save even more work here */
     diff = create_xml_node(parent, name);
 
     /* Reset filter */
     for (lpc = 0; lpc < filter_len; lpc++) {
         filter[lpc].found = FALSE;
     }
 
     /* changes to child objects */
     for (left_child = __xml_first_child(left); left_child != NULL;
          left_child = __xml_next(left_child)) {
         gboolean child_changed = FALSE;
 
         right_child = find_element(right, left_child);
         subtract_xml_object(diff, left_child, right_child, full, &child_changed, marker);
         if (child_changed) {
             *changed = TRUE;
         }
     }
 
     if (*changed == FALSE) {
         /* Nothing to do */
 
     } else if (full) {
         xmlAttrPtr pIter = NULL;
 
         for (pIter = crm_first_attr(left); pIter != NULL; pIter = pIter->next) {
             const char *p_name = (const char *)pIter->name;
             const char *p_value = crm_attr_value(pIter);
 
             xmlSetProp(diff, (const xmlChar *)p_name, (const xmlChar *)p_value);
         }
 
         /* We already have everything we need... */
         goto done;
 
     } else if (id) {
         xmlSetProp(diff, (const xmlChar *)XML_ATTR_ID, (const xmlChar *)id);
     }
 
     /* changes to name/value pairs */
     for (xIter = crm_first_attr(left); xIter != NULL; xIter = xIter->next) {
         const char *prop_name = (const char *)xIter->name;
         xmlAttrPtr right_attr = NULL;
         xml_private_t *p = NULL;
 
         if (strcmp(prop_name, XML_ATTR_ID) == 0) {
             continue;
         }
 
         skip = FALSE;
         for (lpc = 0; skip == FALSE && lpc < filter_len; lpc++) {
             if (filter[lpc].found == FALSE && strcmp(prop_name, filter[lpc].string) == 0) {
                 filter[lpc].found = TRUE;
                 skip = TRUE;
                 break;
             }
         }
 
         if (skip) {
             continue;
         }
 
         right_attr = xmlHasProp(right, (const xmlChar *)prop_name);
         if (right_attr) {
             p = right_attr->_private;
         }
 
         right_val = crm_element_value(right, prop_name);
         if (right_val == NULL || (p && is_set(p->flags, xpf_deleted))) {
             /* new */
             *changed = TRUE;
             if (full) {
                 xmlAttrPtr pIter = NULL;
 
                 for (pIter = crm_first_attr(left); pIter != NULL; pIter = pIter->next) {
                     const char *p_name = (const char *)pIter->name;
                     const char *p_value = crm_attr_value(pIter);
 
                     xmlSetProp(diff, (const xmlChar *)p_name, (const xmlChar *)p_value);
                 }
                 break;
 
             } else {
                 const char *left_value = crm_element_value(left, prop_name);
 
                 xmlSetProp(diff, (const xmlChar *)prop_name, (const xmlChar *)value);
                 crm_xml_add(diff, prop_name, left_value);
             }
 
         } else {
             /* Only now do we need the left value */
             const char *left_value = crm_element_value(left, prop_name);
 
             if (strcmp(left_value, right_val) == 0) {
                 /* unchanged */
 
             } else {
                 *changed = TRUE;
                 if (full) {
                     xmlAttrPtr pIter = NULL;
 
                     crm_trace("Changes detected to %s in <%s id=%s>", prop_name,
                               crm_element_name(left), id);
                     for (pIter = crm_first_attr(left); pIter != NULL; pIter = pIter->next) {
                         const char *p_name = (const char *)pIter->name;
                         const char *p_value = crm_attr_value(pIter);
 
                         xmlSetProp(diff, (const xmlChar *)p_name, (const xmlChar *)p_value);
                     }
                     break;
 
                 } else {
                     crm_trace("Changes detected to %s (%s -> %s) in <%s id=%s>",
                               prop_name, left_value, right_val, crm_element_name(left), id);
                     crm_xml_add(diff, prop_name, left_value);
                 }
             }
         }
     }
 
     if (*changed == FALSE) {
         free_xml(diff);
         return NULL;
 
     } else if (full == FALSE && id) {
         crm_xml_add(diff, XML_ATTR_ID, id);
     }
   done:
     return diff;
 }
 
 static int
 add_xml_comment(xmlNode * parent, xmlNode * target, xmlNode * update)
 {
     CRM_CHECK(update != NULL, return 0);
     CRM_CHECK(update->type == XML_COMMENT_NODE, return 0);
 
     if (target == NULL) {
         target = find_xml_comment(parent, update);
     }
 
     if (target == NULL) {
         add_node_copy(parent, update);
 
     /* We won't reach here currently */
     } else if (safe_str_neq((const char *)target->content, (const char *)update->content)) {
         xmlFree(target->content);
         target->content = xmlStrdup(update->content);
     }
 
     return 0;
 }
 
 int
 add_xml_object(xmlNode * parent, xmlNode * target, xmlNode * update, gboolean as_diff)
 {
     xmlNode *a_child = NULL;
     const char *object_id = NULL;
     const char *object_name = NULL;
 
 #if XML_PARSE_DEBUG
     crm_log_xml_trace("update:", update);
     crm_log_xml_trace("target:", target);
 #endif
 
     CRM_CHECK(update != NULL, return 0);
 
     if (update->type == XML_COMMENT_NODE) {
         return add_xml_comment(parent, target, update);
     }
 
     object_name = crm_element_name(update);
     object_id = ID(update);
 
     CRM_CHECK(object_name != NULL, return 0);
 
     if (target == NULL && object_id == NULL) {
         /*  placeholder object */
         target = find_xml_node(parent, object_name, FALSE);
 
     } else if (target == NULL) {
         target = find_entity(parent, object_name, object_id);
     }
 
     if (target == NULL) {
         target = create_xml_node(parent, object_name);
         CRM_CHECK(target != NULL, return 0);
 #if XML_PARSER_DEBUG
         crm_trace("Added  <%s%s%s/>", crm_str(object_name),
                   object_id ? " id=" : "", object_id ? object_id : "");
 
     } else {
         crm_trace("Found node <%s%s%s/> to update",
                   crm_str(object_name), object_id ? " id=" : "", object_id ? object_id : "");
 #endif
     }
 
     CRM_CHECK(safe_str_eq(crm_element_name(target), crm_element_name(update)), return 0);
 
     if (as_diff == FALSE) {
         /* So that expand_plus_plus() gets called */
         copy_in_properties(target, update);
 
     } else {
         /* No need for expand_plus_plus(), just raw speed */
         xmlAttrPtr pIter = NULL;
 
         for (pIter = crm_first_attr(update); pIter != NULL; pIter = pIter->next) {
             const char *p_name = (const char *)pIter->name;
             const char *p_value = crm_attr_value(pIter);
 
             /* Remove it first so the ordering of the update is preserved */
             xmlUnsetProp(target, (const xmlChar *)p_name);
             xmlSetProp(target, (const xmlChar *)p_name, (const xmlChar *)p_value);
         }
     }
 
     for (a_child = __xml_first_child(update); a_child != NULL; a_child = __xml_next(a_child)) {
 #if XML_PARSER_DEBUG
         crm_trace("Updating child <%s id=%s>", crm_element_name(a_child), ID(a_child));
 #endif
         add_xml_object(target, NULL, a_child, as_diff);
     }
 
 #if XML_PARSER_DEBUG
     crm_trace("Finished with <%s id=%s>", crm_str(object_name), crm_str(object_id));
 #endif
     return 0;
 }
 
 gboolean
 update_xml_child(xmlNode * child, xmlNode * to_update)
 {
     gboolean can_update = TRUE;
     xmlNode *child_of_child = NULL;
 
     CRM_CHECK(child != NULL, return FALSE);
     CRM_CHECK(to_update != NULL, return FALSE);
 
     if (safe_str_neq(crm_element_name(to_update), crm_element_name(child))) {
         can_update = FALSE;
 
     } else if (safe_str_neq(ID(to_update), ID(child))) {
         can_update = FALSE;
 
     } else if (can_update) {
 #if XML_PARSER_DEBUG
         crm_log_xml_trace(child, "Update match found...");
 #endif
         add_xml_object(NULL, child, to_update, FALSE);
     }
 
     for (child_of_child = __xml_first_child(child); child_of_child != NULL;
          child_of_child = __xml_next(child_of_child)) {
         /* only update the first one */
         if (can_update) {
             break;
         }
         can_update = update_xml_child(child_of_child, to_update);
     }
 
     return can_update;
 }
 
 int
 find_xml_children(xmlNode ** children, xmlNode * root,
                   const char *tag, const char *field, const char *value, gboolean search_matches)
 {
     int match_found = 0;
 
     CRM_CHECK(root != NULL, return FALSE);
     CRM_CHECK(children != NULL, return FALSE);
 
     if (tag != NULL && safe_str_neq(tag, crm_element_name(root))) {
 
     } else if (value != NULL && safe_str_neq(value, crm_element_value(root, field))) {
 
     } else {
         if (*children == NULL) {
             *children = create_xml_node(NULL, __FUNCTION__);
         }
         add_node_copy(*children, root);
         match_found = 1;
     }
 
     if (search_matches || match_found == 0) {
         xmlNode *child = NULL;
 
         for (child = __xml_first_child(root); child != NULL; child = __xml_next(child)) {
             match_found += find_xml_children(children, child, tag, field, value, search_matches);
         }
     }
 
     return match_found;
 }
 
 gboolean
 replace_xml_child(xmlNode * parent, xmlNode * child, xmlNode * update, gboolean delete_only)
 {
     gboolean can_delete = FALSE;
     xmlNode *child_of_child = NULL;
 
     const char *up_id = NULL;
     const char *child_id = NULL;
     const char *right_val = NULL;
 
     CRM_CHECK(child != NULL, return FALSE);
     CRM_CHECK(update != NULL, return FALSE);
 
     up_id = ID(update);
     child_id = ID(child);
 
     if (up_id == NULL || (child_id && strcmp(child_id, up_id) == 0)) {
         can_delete = TRUE;
     }
     if (safe_str_neq(crm_element_name(update), crm_element_name(child))) {
         can_delete = FALSE;
     }
     if (can_delete && delete_only) {
         xmlAttrPtr pIter = NULL;
 
         for (pIter = crm_first_attr(update); pIter != NULL; pIter = pIter->next) {
             const char *p_name = (const char *)pIter->name;
             const char *p_value = crm_attr_value(pIter);
 
             right_val = crm_element_value(child, p_name);
             if (safe_str_neq(p_value, right_val)) {
                 can_delete = FALSE;
             }
         }
     }
 
     if (can_delete && parent != NULL) {
         crm_log_xml_trace(child, "Delete match found...");
         if (delete_only || update == NULL) {
             free_xml(child);
 
         } else {
             xmlNode *tmp = copy_xml(update);
             xmlDoc *doc = tmp->doc;
             xmlNode *old = NULL;
 
             xml_accept_changes(tmp);
             old = xmlReplaceNode(child, tmp);
 
             if(xml_tracking_changes(tmp)) {
                 /* Replaced sections may have included relevant ACLs */
                 __xml_acl_apply(tmp);
             }
 
             xml_calculate_changes(old, tmp);
             xmlDocSetRootElement(doc, old);
             free_xml(old);
         }
         child = NULL;
         return TRUE;
 
     } else if (can_delete) {
         crm_log_xml_debug(child, "Cannot delete the search root");
         can_delete = FALSE;
     }
 
     child_of_child = __xml_first_child(child);
     while (child_of_child) {
         xmlNode *next = __xml_next(child_of_child);
 
         can_delete = replace_xml_child(child, child_of_child, update, delete_only);
 
         /* only delete the first one */
         if (can_delete) {
             child_of_child = NULL;
         } else {
             child_of_child = next;
         }
     }
 
     return can_delete;
 }
 
 void
 hash2nvpair(gpointer key, gpointer value, gpointer user_data)
 {
     const char *name = key;
     const char *s_value = value;
 
     xmlNode *xml_node = user_data;
     xmlNode *xml_child = create_xml_node(xml_node, XML_CIB_TAG_NVPAIR);
 
     crm_xml_add(xml_child, XML_ATTR_ID, name);
     crm_xml_add(xml_child, XML_NVPAIR_ATTR_NAME, name);
     crm_xml_add(xml_child, XML_NVPAIR_ATTR_VALUE, s_value);
 
     crm_trace("dumped: name=%s value=%s", name, s_value);
 }
 
 void
 hash2smartfield(gpointer key, gpointer value, gpointer user_data)
 {
     const char *name = key;
     const char *s_value = value;
 
     xmlNode *xml_node = user_data;
 
     if (isdigit(name[0])) {
         xmlNode *tmp = create_xml_node(xml_node, XML_TAG_PARAM);
 
         crm_xml_add(tmp, XML_NVPAIR_ATTR_NAME, name);
         crm_xml_add(tmp, XML_NVPAIR_ATTR_VALUE, s_value);
 
     } else if (crm_element_value(xml_node, name) == NULL) {
         crm_xml_add(xml_node, name, s_value);
         crm_trace("dumped: %s=%s", name, s_value);
 
     } else {
         crm_trace("duplicate: %s=%s", name, s_value);
     }
 }
 
 void
 hash2field(gpointer key, gpointer value, gpointer user_data)
 {
     const char *name = key;
     const char *s_value = value;
 
     xmlNode *xml_node = user_data;
 
     if (crm_element_value(xml_node, name) == NULL) {
         crm_xml_add(xml_node, name, s_value);
 
     } else {
         crm_trace("duplicate: %s=%s", name, s_value);
     }
 }
 
 void
 hash2metafield(gpointer key, gpointer value, gpointer user_data)
 {
     char *crm_name = NULL;
 
     if (key == NULL || value == NULL) {
         return;
     } else if (((char *)key)[0] == '#') {
         return;
     } else if (strstr(key, ":")) {
         return;
     }
 
     crm_name = crm_meta_name(key);
     hash2field(crm_name, value, user_data);
     free(crm_name);
 }
 
 GHashTable *
 xml2list(xmlNode * parent)
 {
     xmlNode *child = NULL;
     xmlAttrPtr pIter = NULL;
     xmlNode *nvpair_list = NULL;
     GHashTable *nvpair_hash = g_hash_table_new_full(crm_str_hash, g_str_equal,
                                                     g_hash_destroy_str, g_hash_destroy_str);
 
     CRM_CHECK(parent != NULL, return nvpair_hash);
 
     nvpair_list = find_xml_node(parent, XML_TAG_ATTRS, FALSE);
     if (nvpair_list == NULL) {
         crm_trace("No attributes in %s", crm_element_name(parent));
         crm_log_xml_trace(parent, "No attributes for resource op");
     }
 
     crm_log_xml_trace(nvpair_list, "Unpacking");
 
     for (pIter = crm_first_attr(nvpair_list); pIter != NULL; pIter = pIter->next) {
         const char *p_name = (const char *)pIter->name;
         const char *p_value = crm_attr_value(pIter);
 
         crm_trace("Added %s=%s", p_name, p_value);
 
         g_hash_table_insert(nvpair_hash, strdup(p_name), strdup(p_value));
     }
 
     for (child = __xml_first_child(nvpair_list); child != NULL; child = __xml_next(child)) {
         if (strcmp((const char *)child->name, XML_TAG_PARAM) == 0) {
             const char *key = crm_element_value(child, XML_NVPAIR_ATTR_NAME);
             const char *value = crm_element_value(child, XML_NVPAIR_ATTR_VALUE);
 
             crm_trace("Added %s=%s", key, value);
             if (key != NULL && value != NULL) {
                 g_hash_table_insert(nvpair_hash, strdup(key), strdup(value));
             }
         }
     }
 
     return nvpair_hash;
 }
 
 typedef struct name_value_s {
     const char *name;
     const void *value;
 } name_value_t;
 
 static gint
 sort_pairs(gconstpointer a, gconstpointer b)
 {
     int rc = 0;
     const name_value_t *pair_a = a;
     const name_value_t *pair_b = b;
 
     CRM_ASSERT(a != NULL);
     CRM_ASSERT(pair_a->name != NULL);
 
     CRM_ASSERT(b != NULL);
     CRM_ASSERT(pair_b->name != NULL);
 
     rc = strcmp(pair_a->name, pair_b->name);
     if (rc < 0) {
         return -1;
     } else if (rc > 0) {
         return 1;
     }
     return 0;
 }
 
 static void
 dump_pair(gpointer data, gpointer user_data)
 {
     name_value_t *pair = data;
     xmlNode *parent = user_data;
 
     crm_xml_add(parent, pair->name, pair->value);
 }
 
 xmlNode *
 sorted_xml(xmlNode * input, xmlNode * parent, gboolean recursive)
 {
     xmlNode *child = NULL;
     GListPtr sorted = NULL;
     GListPtr unsorted = NULL;
     name_value_t *pair = NULL;
     xmlNode *result = NULL;
     const char *name = NULL;
     xmlAttrPtr pIter = NULL;
 
     CRM_CHECK(input != NULL, return NULL);
 
     name = crm_element_name(input);
     CRM_CHECK(name != NULL, return NULL);
 
     result = create_xml_node(parent, name);
 
     for (pIter = crm_first_attr(input); pIter != NULL; pIter = pIter->next) {
         const char *p_name = (const char *)pIter->name;
         const char *p_value = crm_attr_value(pIter);
 
         pair = calloc(1, sizeof(name_value_t));
         pair->name = p_name;
         pair->value = p_value;
         unsorted = g_list_prepend(unsorted, pair);
         pair = NULL;
     }
 
     sorted = g_list_sort(unsorted, sort_pairs);
     g_list_foreach(sorted, dump_pair, result);
     g_list_free_full(sorted, free);
 
     for (child = __xml_first_child(input); child != NULL; child = __xml_next(child)) {
         if (recursive) {
             sorted_xml(child, result, recursive);
         } else {
             add_node_copy(result, child);
         }
     }
 
     return result;
 }
 
-static gboolean
-validate_with_dtd(xmlDocPtr doc, gboolean to_logs, const char *dtd_file)
-{
-    gboolean valid = TRUE;
-
-    xmlDtdPtr dtd = NULL;
-    xmlValidCtxtPtr cvp = NULL;
-
-    CRM_CHECK(doc != NULL, return FALSE);
-    CRM_CHECK(dtd_file != NULL, return FALSE);
-
-    dtd = xmlParseDTD(NULL, (const xmlChar *)dtd_file);
-    if(dtd == NULL) {
-        crm_err("Could not locate/parse DTD: %s", dtd_file);
-        return TRUE;
-    }
-
-    cvp = xmlNewValidCtxt();
-    if(cvp) {
-        if (to_logs) {
-            cvp->userData = (void *)LOG_ERR;
-            cvp->error = (xmlValidityErrorFunc) xml_log;
-            cvp->warning = (xmlValidityWarningFunc) xml_log;
-        } else {
-            cvp->userData = (void *)stderr;
-            cvp->error = (xmlValidityErrorFunc) fprintf;
-            cvp->warning = (xmlValidityWarningFunc) fprintf;
-        }
-
-        if (!xmlValidateDtd(cvp, doc, dtd)) {
-            valid = FALSE;
-        }
-        xmlFreeValidCtxt(cvp);
-
-    } else {
-        crm_err("Internal error: No valid context");
-    }
-
-    xmlFreeDtd(dtd);
-    return valid;
-}
-
 xmlNode *
 first_named_child(xmlNode * parent, const char *name)
 {
     xmlNode *match = NULL;
 
     for (match = __xml_first_child(parent); match != NULL; match = __xml_next(match)) {
         /*
          * name == NULL gives first child regardless of name; this is
          * semantically incorrect in this function, but may be necessary
          * due to prior use of xml_child_iter_filter
          */
         if (name == NULL || strcmp((const char *)match->name, name) == 0) {
             return match;
         }
     }
     return NULL;
 }
 
-#if 0
-static void
-relaxng_invalid_stderr(void *userData, xmlErrorPtr error)
-{
-    /*
-       Structure xmlError
-       struct _xmlError {
-       int      domain  : What part of the library raised this er
-       int      code    : The error code, e.g. an xmlParserError
-       char *   message : human-readable informative error messag
-       xmlErrorLevel    level   : how consequent is the error
-       char *   file    : the filename
-       int      line    : the line number if available
-       char *   str1    : extra string information
-       char *   str2    : extra string information
-       char *   str3    : extra string information
-       int      int1    : extra number information
-       int      int2    : column number of the error or 0 if N/A
-       void *   ctxt    : the parser context if available
-       void *   node    : the node in the tree
-       }
-     */
-    crm_err("Structured error: line=%d, level=%d %s", error->line, error->level, error->message);
-}
-#endif
-
-static gboolean
-validate_with_relaxng(xmlDocPtr doc, gboolean to_logs, const char *relaxng_file,
-                      relaxng_ctx_cache_t ** cached_ctx)
-{
-    int rc = 0;
-    gboolean valid = TRUE;
-    relaxng_ctx_cache_t *ctx = NULL;
-
-    CRM_CHECK(doc != NULL, return FALSE);
-    CRM_CHECK(relaxng_file != NULL, return FALSE);
-
-    if (cached_ctx && *cached_ctx) {
-        ctx = *cached_ctx;
-
-    } else {
-        crm_info("Creating RNG parser context");
-        ctx = calloc(1, sizeof(relaxng_ctx_cache_t));
-
-        xmlLoadExtDtdDefaultValue = 1;
-        ctx->parser = xmlRelaxNGNewParserCtxt(relaxng_file);
-        CRM_CHECK(ctx->parser != NULL, goto cleanup);
-
-        if (to_logs) {
-            xmlRelaxNGSetParserErrors(ctx->parser,
-                                      (xmlRelaxNGValidityErrorFunc) xml_log,
-                                      (xmlRelaxNGValidityWarningFunc) xml_log,
-                                      GUINT_TO_POINTER(LOG_ERR));
-        } else {
-            xmlRelaxNGSetParserErrors(ctx->parser,
-                                      (xmlRelaxNGValidityErrorFunc) fprintf,
-                                      (xmlRelaxNGValidityWarningFunc) fprintf, stderr);
-        }
-
-        ctx->rng = xmlRelaxNGParse(ctx->parser);
-        CRM_CHECK(ctx->rng != NULL, crm_err("Could not find/parse %s", relaxng_file);
-                  goto cleanup);
-
-        ctx->valid = xmlRelaxNGNewValidCtxt(ctx->rng);
-        CRM_CHECK(ctx->valid != NULL, goto cleanup);
-
-        if (to_logs) {
-            xmlRelaxNGSetValidErrors(ctx->valid,
-                                     (xmlRelaxNGValidityErrorFunc) xml_log,
-                                     (xmlRelaxNGValidityWarningFunc) xml_log,
-                                     GUINT_TO_POINTER(LOG_ERR));
-        } else {
-            xmlRelaxNGSetValidErrors(ctx->valid,
-                                     (xmlRelaxNGValidityErrorFunc) fprintf,
-                                     (xmlRelaxNGValidityWarningFunc) fprintf, stderr);
-        }
-    }
-
-    /* xmlRelaxNGSetValidStructuredErrors( */
-    /*  valid, relaxng_invalid_stderr, valid); */
-
-    xmlLineNumbersDefault(1);
-    rc = xmlRelaxNGValidateDoc(ctx->valid, doc);
-    if (rc > 0) {
-        valid = FALSE;
-
-    } else if (rc < 0) {
-        crm_err("Internal libxml error during validation\n");
-    }
-
-  cleanup:
-
-    if (cached_ctx) {
-        *cached_ctx = ctx;
-
-    } else {
-        if (ctx->parser != NULL) {
-            xmlRelaxNGFreeParserCtxt(ctx->parser);
-        }
-        if (ctx->valid != NULL) {
-            xmlRelaxNGFreeValidCtxt(ctx->valid);
-        }
-        if (ctx->rng != NULL) {
-            xmlRelaxNGFree(ctx->rng);
-        }
-        free(ctx);
-    }
-
-    return valid;
-}
-
 void
 crm_xml_init(void)
 {
     static bool init = TRUE;
 
     if(init) {
         init = FALSE;
         /* The default allocator XML_BUFFER_ALLOC_EXACT does far too many
          * realloc_safe()s and it can take upwards of 18 seconds (yes, seconds)
          * to dump a 28kb tree which XML_BUFFER_ALLOC_DOUBLEIT can do in
          * less than 1 second.
          */
         xmlSetBufferAllocationScheme(XML_BUFFER_ALLOC_DOUBLEIT);
 
         /* Populate and free the _private field when nodes are created and destroyed */
         xmlDeregisterNodeDefault(pcmkDeregisterNode);
         xmlRegisterNodeDefault(pcmkRegisterNode);
 
-        __xml_build_schema_list();
+        crm_schema_init();
     }
 }
 
 void
 crm_xml_cleanup(void)
 {
-    int lpc = 0;
-    relaxng_ctx_cache_t *ctx = NULL;
-
     crm_info("Cleaning up memory from libxml2");
-    for (; lpc < xml_schema_max; lpc++) {
-
-        switch (known_schemas[lpc].type) {
-            case 0:
-                /* None */
-                break;
-            case 1:
-                /* DTD - Not cached */
-                break;
-            case 2:
-                /* RNG - Cached */
-                ctx = (relaxng_ctx_cache_t *) known_schemas[lpc].cache;
-                if (ctx == NULL) {
-                    break;
-                }
-                if (ctx->parser != NULL) {
-                    xmlRelaxNGFreeParserCtxt(ctx->parser);
-                }
-                if (ctx->valid != NULL) {
-                    xmlRelaxNGFreeValidCtxt(ctx->valid);
-                }
-                if (ctx->rng != NULL) {
-                    xmlRelaxNGFree(ctx->rng);
-                }
-                free(ctx);
-                known_schemas[lpc].cache = NULL;
-                break;
-            default:
-                break;
-        }
-        free(known_schemas[lpc].name);
-        free(known_schemas[lpc].location);
-        free(known_schemas[lpc].transform);
-    }
-    free(known_schemas);
-    xsltCleanupGlobals();
+    crm_schema_cleanup();
     xmlCleanupParser();
 }
 
-static gboolean
-validate_with(xmlNode * xml, int method, gboolean to_logs)
-{
-    xmlDocPtr doc = NULL;
-    gboolean valid = FALSE;
-    int type = 0;
-    char *file = NULL;
-
-    if(method < 0) {
-        return FALSE;
-    }
-
-    type = known_schemas[method].type;
-    if(type == 0) {
-        return TRUE;
-    }
-
-    CRM_CHECK(xml != NULL, return FALSE);
-    doc = getDocPtr(xml);
-    file = get_schema_path(known_schemas[method].name, known_schemas[method].location);
-
-    crm_trace("Validating with: %s (type=%d)", crm_str(file), type);
-    switch (type) {
-        case 1:
-            valid = validate_with_dtd(doc, to_logs, file);
-            break;
-        case 2:
-            valid =
-                validate_with_relaxng(doc, to_logs, file,
-                                      (relaxng_ctx_cache_t **) & (known_schemas[method].cache));
-            break;
-        default:
-            crm_err("Unknown validator type: %d", type);
-            break;
-    }
-
-    free(file);
-    return valid;
-}
-
-#include <stdio.h>
-static void
-dump_file(const char *filename)
-{
-
-    FILE *fp = NULL;
-    int ch, line = 0;
-
-    CRM_CHECK(filename != NULL, return);
-
-    fp = fopen(filename, "r");
-    CRM_CHECK(fp != NULL, return);
-
-    fprintf(stderr, "%4d ", ++line);
-    do {
-        ch = getc(fp);
-        if (ch == EOF) {
-            putc('\n', stderr);
-            break;
-        } else if (ch == '\n') {
-            fprintf(stderr, "\n%4d ", ++line);
-        } else {
-            putc(ch, stderr);
-        }
-    } while (1);
-
-    fclose(fp);
-}
-
-gboolean
-validate_xml_verbose(xmlNode * xml_blob)
-{
-    int fd = 0;
-    xmlDoc *doc = NULL;
-    xmlNode *xml = NULL;
-    gboolean rc = FALSE;
-    char *filename = strdup(CRM_STATE_DIR "/cib-invalid.XXXXXX");
-
-    umask(S_IWGRP | S_IWOTH | S_IROTH);
-    fd = mkstemp(filename);
-    write_xml_fd(xml_blob, filename, fd, FALSE);
-
-    dump_file(filename);
-
-    doc = xmlParseFile(filename);
-    xml = xmlDocGetRootElement(doc);
-    rc = validate_xml(xml, NULL, FALSE);
-    free_xml(xml);
-
-    unlink(filename);
-    free(filename);
-
-    return rc;
-}
-
-gboolean
-validate_xml(xmlNode * xml_blob, const char *validation, gboolean to_logs)
-{
-    int version = 0;
-
-    if (validation == NULL) {
-        validation = crm_element_value(xml_blob, XML_ATTR_VALIDATION);
-    }
-
-    if (validation == NULL) {
-        int lpc = 0;
-        bool valid = FALSE;
-
-        validation = crm_element_value(xml_blob, "ignore-dtd");
-        if (crm_is_true(validation)) {
-            /* Legacy compatibilty */
-            crm_xml_add(xml_blob, XML_ATTR_VALIDATION, "none");
-            return TRUE;
-        }
-
-        /* Work it out */
-        for (lpc = 0; lpc < xml_schema_max; lpc++) {
-            if(validate_with(xml_blob, lpc, FALSE)) {
-                valid = TRUE;
-                crm_xml_add(xml_blob, XML_ATTR_VALIDATION, known_schemas[lpc].name);
-                crm_info("XML validated against %s", known_schemas[lpc].name);
-                if(known_schemas[lpc].after_transform == 0) {
-                    break;
-                }
-            }
-        }
-
-        return valid;
-    }
-
-    version = get_schema_version(validation);
-    if (strcmp(validation, "none") == 0) {
-        return TRUE;
-
-    } else if(version < xml_schema_max) {
-        return validate_with(xml_blob, version, to_logs);
-
-    }
-
-    crm_err("Unknown validator: %s", validation);
-    return FALSE;
-}
-
-#if HAVE_LIBXSLT
-static xmlNode *
-apply_transformation(xmlNode * xml, const char *transform)
-{
-    char *xform = NULL;
-    xmlNode *out = NULL;
-    xmlDocPtr res = NULL;
-    xmlDocPtr doc = NULL;
-    xsltStylesheet *xslt = NULL;
-
-    CRM_CHECK(xml != NULL, return FALSE);
-    doc = getDocPtr(xml);
-    xform = get_schema_path(NULL, transform);
-
-    xmlLoadExtDtdDefaultValue = 1;
-    xmlSubstituteEntitiesDefault(1);
-
-    xslt = xsltParseStylesheetFile((const xmlChar *)xform);
-    CRM_CHECK(xslt != NULL, goto cleanup);
-
-    res = xsltApplyStylesheet(xslt, doc, NULL);
-    CRM_CHECK(res != NULL, goto cleanup);
-
-    out = xmlDocGetRootElement(res);
-
-  cleanup:
-    if (xslt) {
-        xsltFreeStylesheet(xslt);
-    }
-
-    free(xform);
-
-    return out;
-}
-#endif
-
-const char *
-get_schema_name(int version)
-{
-    if (version < 0 || version >= xml_schema_max) {
-        return "unknown";
-    }
-    return known_schemas[version].name;
-}
-
-int
-get_schema_version(const char *name)
-{
-    int lpc = 0;
-
-    if(name == NULL) {
-        name = "none";
-    }
-    for (; lpc < xml_schema_max; lpc++) {
-        if (safe_str_eq(name, known_schemas[lpc].name)) {
-            return lpc;
-        }
-    }
-    return -1;
-}
-
-/* set which validation to use */
-#include <crm/cib.h>
-int
-update_validation(xmlNode ** xml_blob, int *best, int max, gboolean transform, gboolean to_logs)
-{
-    xmlNode *xml = NULL;
-    char *value = NULL;
-    int max_stable_schemas = xml_latest_schema_index();
-    int lpc = 0, match = -1, rc = pcmk_ok;
-    int next = -1;  /* -1 denotes "inactive" value */
-
-    CRM_CHECK(best != NULL, return -EINVAL);
-    CRM_CHECK(xml_blob != NULL, return -EINVAL);
-    CRM_CHECK(*xml_blob != NULL, return -EINVAL);
-
-    *best = 0;
-    xml = *xml_blob;
-    value = crm_element_value_copy(xml, XML_ATTR_VALIDATION);
-
-    if (value != NULL) {
-        match = get_schema_version(value);
-
-        lpc = match;
-        if (lpc >= 0 && transform == FALSE) {
-            lpc++;
-
-        } else if (lpc < 0) {
-            crm_debug("Unknown validation type");
-            lpc = 0;
-        }
-    }
-
-    if (match >= max_stable_schemas) {
-        /* nothing to do */
-        free(value);
-        *best = match;
-        return pcmk_ok;
-    }
-
-    while(lpc <= max_stable_schemas) {
-        crm_debug("Testing '%s' validation (%d of %d)",
-                  known_schemas[lpc].name ? known_schemas[lpc].name : "<unset>",
-                  lpc, max_stable_schemas);
-
-        if (validate_with(xml, lpc, to_logs) == FALSE) {
-            if (next != -1) {
-                crm_info("Configuration not valid for schema: %s", known_schemas[lpc].name);
-                next = -1;
-            } else {
-                crm_trace("%s validation failed", known_schemas[lpc].name ? known_schemas[lpc].name : "<unset>");
-            }
-            if (*best) {
-                /* we've satisfied the validation, no need to check further */
-                break;
-            }
-            rc = -pcmk_err_schema_validation;
-
-        } else {
-            if (next != -1) {
-                crm_debug("Configuration valid for schema: %s", known_schemas[next].name);
-                next = -1;
-            }
-            rc = pcmk_ok;
-        }
-
-        if (rc == pcmk_ok) {
-            *best = lpc;
-        }
-
-        if (rc == pcmk_ok && transform) {
-            xmlNode *upgrade = NULL;
-            next = known_schemas[lpc].after_transform;
-
-            if (next < 0 || next <= lpc) {
-                crm_trace("Stopping at %s", known_schemas[lpc].name);
-                break;
-
-            } else if (max > 0 && (lpc == max || next > max)) {
-                crm_trace("Upgrade limit reached at %s (lpc=%d, next=%d, max=%d)",
-                          known_schemas[lpc].name, lpc, next, max);
-                break;
-
-            } else if (known_schemas[lpc].transform == NULL) {
-                crm_debug("%s-style configuration is also valid for %s",
-                           known_schemas[lpc].name, known_schemas[next].name);
-
-                lpc = next;
-
-            } else {
-                crm_debug("Upgrading %s-style configuration to %s with %s",
-                           known_schemas[lpc].name, known_schemas[next].name,
-                           known_schemas[lpc].transform ? known_schemas[lpc].transform : "no-op");
-
-#if HAVE_LIBXSLT
-                upgrade = apply_transformation(xml, known_schemas[lpc].transform);
-#endif
-                if (upgrade == NULL) {
-                    crm_err("Transformation %s failed", known_schemas[lpc].transform);
-                    rc = -pcmk_err_transform_failed;
-
-                } else if (validate_with(upgrade, next, to_logs)) {
-                    crm_info("Transformation %s successful", known_schemas[lpc].transform);
-                    lpc = next;
-                    *best = next;
-                    free_xml(xml);
-                    xml = upgrade;
-                    rc = pcmk_ok;
-
-                } else {
-                    crm_err("Transformation %s did not produce a valid configuration",
-                            known_schemas[lpc].transform);
-                    crm_log_xml_info(upgrade, "transform:bad");
-                    free_xml(upgrade);
-                    rc = -pcmk_err_schema_validation;
-                }
-                next = -1;
-            }
-        }
-
-        if (transform == FALSE || rc != pcmk_ok) {
-            /* we need some progress! */
-            lpc++;
-        }
-    }
-
-    if (*best > match) {
-        crm_info("%s the configuration from %s to %s",
-                   transform?"Transformed":"Upgraded",
-                   value ? value : "<none>", known_schemas[*best].name);
-        crm_xml_add(xml, XML_ATTR_VALIDATION, known_schemas[*best].name);
-    }
-
-    *xml_blob = xml;
-    free(value);
-    return rc;
-}
-
-gboolean
-cli_config_update(xmlNode ** xml, int *best_version, gboolean to_logs)
-{
-    gboolean rc = TRUE;
-    const char *value = crm_element_value(*xml, XML_ATTR_VALIDATION);
-
-    int version = get_schema_version(value);
-    int orig_version = version;
-    int min_version = xml_minimum_schema_index();
-
-    if (version < min_version) {
-        xmlNode *converted = NULL;
-
-        converted = copy_xml(*xml);
-        update_validation(&converted, &version, 0, TRUE, to_logs);
-
-        value = crm_element_value(converted, XML_ATTR_VALIDATION);
-        if (version < min_version) {
-            if (version < orig_version) {
-                if (to_logs) {
-                    crm_config_err("Your current configuration could not validate"
-                                   " with any schema in range [%s, %s]\n",
-                                   get_schema_name(orig_version),
-                                   xml_latest_schema());
-                } else {
-                    fprintf(stderr, "Your current configuration could not validate"
-                                    " with any schema in range [%s, %s]\n",
-                                    get_schema_name(orig_version),
-                                    xml_latest_schema());
-                }
-            } else if (to_logs) {
-                crm_config_err("Your current configuration could only be upgraded to %s... "
-                               "the minimum requirement is %s.\n", crm_str(value),
-                               get_schema_name(min_version));
-            } else {
-                fprintf(stderr, "Your current configuration could only be upgraded to %s... "
-                        "the minimum requirement is %s.\n",
-                        crm_str(value), get_schema_name(min_version));
-            }
-
-            free_xml(converted);
-            converted = NULL;
-            rc = FALSE;
-
-        } else {
-            free_xml(*xml);
-            *xml = converted;
-
-            if (version < xml_latest_schema_index()) {
-                crm_config_warn("Your configuration was internally updated to %s... "
-                                "which is acceptable but not the most recent",
-                                get_schema_name(version));
-
-            } else if (to_logs) {
-                crm_info("Your configuration was internally updated to the latest version (%s)",
-                         get_schema_name(version));
-            }
-        }
-
-    } else if (version >= get_schema_version("none")) {
-        if (to_logs) {
-            crm_config_warn("Configuration validation is currently disabled."
-                            " It is highly encouraged and prevents many common cluster issues.");
-
-        } else {
-            fprintf(stderr, "Configuration validation is currently disabled."
-                    " It is highly encouraged and prevents many common cluster issues.\n");
-        }
-    }
-
-    if (best_version) {
-        *best_version = version;
-    }
-
-    return rc;
-}
-
 xmlNode *
 expand_idref(xmlNode * input, xmlNode * top)
 {
     const char *tag = NULL;
     const char *ref = NULL;
     xmlNode *result = input;
     char *xpath_string = NULL;
 
     if (result == NULL) {
         return NULL;
 
     } else if (top == NULL) {
         top = input;
     }
 
     tag = crm_element_name(result);
     ref = crm_element_value(result, XML_ATTR_IDREF);
 
     if (ref != NULL) {
         int xpath_max = 512, offset = 0;
 
         xpath_string = calloc(1, xpath_max);
 
         offset += snprintf(xpath_string + offset, xpath_max - offset, "//%s[@id='%s']", tag, ref);
         CRM_LOG_ASSERT(offset > 0);
 
         result = get_xpath_object(xpath_string, top, LOG_ERR);
         if (result == NULL) {
             char *nodePath = (char *)xmlGetNodePath(top);
 
             crm_err("No match for %s found in %s: Invalid configuration", xpath_string,
                     crm_str(nodePath));
             free(nodePath);
         }
     }
 
     free(xpath_string);
     return result;
 }
 
 const char *
 crm_element_value(xmlNode * data, const char *name)
 {
     xmlAttr *attr = NULL;
 
     if (data == NULL) {
         crm_err("Couldn't find %s in NULL", name ? name : "<null>");
         CRM_LOG_ASSERT(data != NULL);
         return NULL;
 
     } else if (name == NULL) {
         crm_err("Couldn't find NULL in %s", crm_element_name(data));
         return NULL;
     }
 
     attr = xmlHasProp(data, (const xmlChar *)name);
     if (attr == NULL || attr->children == NULL) {
         return NULL;
     }
     return (const char *)attr->children->content;
 }
-