diff --git a/cib/Makefile.am b/cib/Makefile.am
index 66afc33ac1..820a5a812f 100644
--- a/cib/Makefile.am
+++ b/cib/Makefile.am
@@ -1,64 +1,63 @@
 #
 # Copyright (C) 2004 Andrew Beekhof
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 MAINTAINERCLEANFILES    = Makefile.in
 
 INCLUDES        = -I$(top_builddir)/include  -I$(top_srcdir)/include  \
 		  -I$(top_builddir)/libltdl  -I$(top_srcdir)/libltdl  \
 		  -I$(AISPREFIX)/include/openais
 
 
 EXTRA_DIST	= cib.pam
 
 hadir		= $(sysconfdir)/ha.d
 halibdir	= $(libdir)/@HB_PKG@
 commmoddir	= $(halibdir)/modules/comm
 
 COMMONLIBS	= $(top_builddir)/lib/common/libcrmcommon.la	\
 		  $(top_builddir)/lib/cib/libcib.la			\
 		  $(GLIBLIB) $(LIBRT)
 
 ## binary progs
 halib_PROGRAMS	= cib cibmon
 sbin_PROGRAMS   = cibpipe
 
 ## SOURCES
 #noinst_HEADERS		= config.h control.h crmd.h
-noinst_HEADERS		= cibio.h cibmessages.h cibprimatives.h notify.h \
-			callbacks.h
+noinst_HEADERS		= cibio.h cibmessages.h notify.h callbacks.h
 
-cib_SOURCES		= io.c primatives.c messages.c cib.c notify.c	\
+cib_SOURCES		= io.c messages.c cib.c notify.c	\
 			callbacks.c main.c remote.c common.c
 
 cib_LDADD		= $(COMMONLIBS) $(CRYPTOLIB) $(CLUSTERLIBS)	\
 			$(top_builddir)/lib/common/libcrmcluster.la
 
 cibmon_SOURCES		= cibmon.c
 cibmon_LDADD		= $(COMMONLIBS)
 
-cibpipe_SOURCES		= primatives.c cibpipe.c
+cibpipe_SOURCES		= cibpipe.c
 
 cibpipe_LDADD		= $(COMMONLIBS) $(CRYPTOLIB)
 cibpipe_CFLAGS		= -DCIBPIPE=1
 
 clean-generic:
 	rm -f *.log *.debug *.xml *~
 
 install-exec-local:
 #	cp -f $(top_srcdir)/crm/cib/cib.pam $(DESTDIR)/etc/pam.d/cib
 
 uninstall-local:
diff --git a/cib/callbacks.c b/cib/callbacks.c
index 2f0447922c..149c6321ac 100644
--- a/cib/callbacks.c
+++ b/cib/callbacks.c
@@ -1,1281 +1,1280 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 
 #include <clplumbing/uids.h>
 #include <clplumbing/cl_uuid.h>
 #include <clplumbing/Gmain_timeout.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/cluster.h>
 #include <crm/common/ctrl.h>
 #include <crm/common/xml.h>
 #include <crm/common/msg.h>
 
 #include <cibio.h>
 #include <callbacks.h>
 #include <cibmessages.h>
-#include <cibprimatives.h>
 #include <notify.h>
 #include <heartbeat.h>
 #include "common.h"
 
 extern GMainLoop*  mainloop;
 extern gboolean cib_shutdown_flag;
 extern gboolean stand_alone;
 extern const char* cib_root;
 #if SUPPORT_HEARTBEAT
 extern ll_cluster_t *hb_conn;
 #endif
 
 extern void cib_ha_connection_destroy(gpointer user_data);
 
 extern enum cib_errors cib_update_counter(
 	xmlNode *xml_obj, const char *field, gboolean reset);
 
 extern void GHFunc_count_peers(
 	gpointer key, gpointer value, gpointer user_data);
 
 void initiate_exit(void);
 void terminate_cib(const char *caller);
 gint cib_GCompareFunc(gconstpointer a, gconstpointer b);
 gboolean can_write(int flags);
 void send_cib_replace(const xmlNode *sync_request, const char *host);
 void cib_process_request(
 	xmlNode *request, gboolean privileged, gboolean force_synchronous,
 	gboolean from_peer, cib_client_t *cib_client);
 void cib_common_callback_worker(xmlNode *op_request, cib_client_t *cib_client,
 				gboolean force_synchronous, gboolean privileged);
 
 extern GHashTable *client_list;
 
 int        next_client_id  = 0;
 extern const char *cib_our_uname;
 extern unsigned long cib_num_ops, cib_num_local, cib_num_updates, cib_num_fail;
 extern unsigned long cib_bad_connects, cib_num_timeouts;
 extern longclock_t cib_call_time;
 extern enum cib_errors cib_status;
 
 
 int send_via_callback_channel(xmlNode *msg, const char *token);
 
 enum cib_errors cib_process_command(
 	xmlNode *request, xmlNode **reply,
 	xmlNode **cib_diff, gboolean privileged);
 
 gboolean cib_common_callback(IPC_Channel *channel, cib_client_t *cib_client,
 			     gboolean force_synchronous, gboolean privileged);
 
 gboolean cib_process_disconnect(IPC_Channel *channel, cib_client_t *cib_client);
 int num_clients = 0;
 
 static void
 cib_ipc_connection_destroy(gpointer user_data)
 {
 	cib_client_t *cib_client = user_data;
 	
 	/* cib_process_disconnect */
 
 	if(cib_client == NULL) {
 		crm_debug_4("Destroying %p", user_data);
 		return;
 	}
 
 	if(cib_client->source != NULL) {
 		crm_debug_4("Deleting %s (%p) from mainloop",
 			    cib_client->name, cib_client->source);
 		G_main_del_IPC_Channel(cib_client->source); 
 		cib_client->source = NULL;
 	}
 	
 	crm_debug_3("Destroying %s (%p)", cib_client->name, user_data);
 	num_clients--;
 	crm_debug_2("Num unfree'd clients: %d", num_clients);
 	crm_free(cib_client->name);
 	crm_free(cib_client->callback_id);
 	crm_free(cib_client->id);
 	crm_free(cib_client);
 	crm_debug_4("Freed the cib client");
 
 	return;
 }
 
 gboolean
 cib_client_connect(IPC_Channel *channel, gpointer user_data)
 {
 	cl_uuid_t client_id;
 	xmlNode *reg_msg = NULL;
 	cib_client_t *new_client = NULL;
 	char uuid_str[UU_UNPARSE_SIZEOF];
 	const char *channel_name = user_data;
 	gboolean (*callback)(IPC_Channel *channel, gpointer user_data);
 	
 	crm_debug_3("Connecting channel");
 
 	if (channel == NULL) {
 		crm_err("Channel was NULL");
 		cib_bad_connects++;
 		return FALSE;
 
 	} else if (channel->ch_status != IPC_CONNECT) {
 		crm_err("Channel was disconnected");
 		cib_bad_connects++;
 		return FALSE;
 		
 	} else if(channel_name == NULL) {
 		crm_err("user_data must contain channel name");
 		cib_bad_connects++;
 		return FALSE;
 		
 	} else if(cib_shutdown_flag) {
 		crm_info("Ignoring new client [%d] during shutdown",
 			channel->farside_pid);
 		return FALSE;		
 	}
 
 	callback = cib_ro_callback;
 	if(safe_str_eq(channel_name, cib_channel_rw)) {
 		callback = cib_rw_callback;
 	}
 	
 	crm_malloc0(new_client, sizeof(cib_client_t));
 	num_clients++;
 	new_client->channel = channel;
 	new_client->channel_name = channel_name;
 	
 	crm_debug_3("Created channel %p for channel %s",
 		    new_client, new_client->channel_name);
 	
 	channel->ops->set_recv_qlen(channel, 1024);
 	channel->ops->set_send_qlen(channel, 1024);
 	
 	new_client->source = G_main_add_IPC_Channel(
 	    G_PRIORITY_DEFAULT, channel, FALSE, callback,
 	    new_client, cib_ipc_connection_destroy);
 	
 	crm_debug_3("Channel %s connected for client %s",
 		    new_client->channel_name, new_client->id);
 	
 	cl_uuid_generate(&client_id);
 	cl_uuid_unparse(&client_id, uuid_str);
 
 	CRM_CHECK(new_client->id == NULL, crm_free(new_client->id));
 	new_client->id = crm_strdup(uuid_str);
 	
 	/* make sure we can find ourselves later for sync calls
 	 * redirected to the master instance
 	 */
 	g_hash_table_insert(client_list, new_client->id, new_client);
 	
 	reg_msg = create_xml_node(NULL, "callback");
 	crm_xml_add(reg_msg, F_CIB_OPERATION, CRM_OP_REGISTER);
 	crm_xml_add(reg_msg, F_CIB_CLIENTID,  new_client->id);
 	
 	send_ipc_message(channel, reg_msg);		
 	free_xml(reg_msg);
 	
 	return TRUE;
 }
 
 gboolean
 cib_rw_callback(IPC_Channel *channel, gpointer user_data)
 {
 	gboolean result = FALSE;
 	result = cib_common_callback(channel, user_data, FALSE, TRUE);
 	return result;
 }
 
 gboolean
 cib_ro_callback(IPC_Channel *channel, gpointer user_data)
 {
 	gboolean result = FALSE;
 	result = cib_common_callback(channel, user_data, FALSE, FALSE);
 	return result;
 }
 
 void
 cib_common_callback_worker(xmlNode *op_request, cib_client_t *cib_client,
 			   gboolean force_synchronous, gboolean privileged)
 {
 	longclock_t call_stop = 0;
 	longclock_t call_start = 0;
 	
 	const char *op = crm_element_value(op_request, F_CIB_OPERATION);
 
 	if(crm_str_eq(op, CRM_OP_REGISTER, TRUE)) {
 	    return;
 	    
 	} else if(crm_str_eq(op, T_CIB_NOTIFY, TRUE)) {
 	    /* Update the notify filters for this client */
 	    int on_off = 0;
 	    const char *type = crm_element_value(op_request, F_CIB_NOTIFY_TYPE);;
 	    crm_element_value_int(op_request, F_CIB_NOTIFY_ACTIVATE, &on_off);
 	    
 	    crm_info("Setting %s callbacks for %s (%s): %s",
 		     type, cib_client->name, cib_client->id, on_off?"on":"off");
 	    
 	    if(safe_str_eq(type, T_CIB_POST_NOTIFY)) {
 		cib_client->post_notify = on_off;
 		
 	    } else if(safe_str_eq(type, T_CIB_PRE_NOTIFY)) {
 		cib_client->pre_notify = on_off;
 		
 	    } else if(safe_str_eq(type, T_CIB_UPDATE_CONFIRM)) {
 		cib_client->confirmations = on_off;
 		
 	    } else if(safe_str_eq(type, T_CIB_DIFF_NOTIFY)) {
 		cib_client->diffs = on_off;
 		
 	    } else if(safe_str_eq(type, T_CIB_REPLACE_NOTIFY)) {
 		cib_client->replace = on_off;
 	    }
 	    return;
 	}
 	
 	cib_client->num_calls++;
 	call_start = time_longclock();
 	cib_process_request(
 	    op_request, force_synchronous, privileged, FALSE, cib_client);
 
 	call_stop = time_longclock();
 	cib_call_time += (call_stop - call_start);
 }
 
 gboolean
 cib_common_callback(IPC_Channel *channel, cib_client_t *cib_client,
 		    gboolean force_synchronous, gboolean privileged)
 {
 	int lpc = 0;
 	const char *value = NULL;
 	xmlNode *op_request = NULL;
 	gboolean keep_channel = TRUE;
 
 	CRM_CHECK(cib_client != NULL, crm_err("Invalid client"); return FALSE);
 	CRM_CHECK(cib_client->id != NULL, crm_err("Invalid client: %p", cib_client); return FALSE);
 
 	/*
 	 * Do enough work to make entering worthwhile
 	 * But don't allow a single client to monopolize the CIB
 	 */
 	while(lpc < 5
 	      && IPC_ISRCONN(channel)
 	      && channel->ops->is_message_pending(channel)) {
 
 		lpc++;
 		op_request = xmlfromIPC(channel, 0);
 		if (op_request == NULL) {
 			break;
 		}
 
 		if(cib_client->name == NULL) {
 		    value = crm_element_value(op_request, F_CIB_CLIENTNAME);
 		    if(value == NULL) {
 			cib_client->name = crm_itoa(channel->farside_pid);
 		    } else {
 			cib_client->name = crm_strdup(value);
 		    }
 		}
 
 		crm_xml_add(op_request, F_CIB_CLIENTID, cib_client->id);
 		crm_xml_add(op_request, F_CIB_CLIENTNAME, cib_client->name);
 		/* crm_log_xml(LOG_MSG, "Client[inbound]", op_request); */
 
 		if(cib_client->callback_id == NULL) {
 		    value = crm_element_value(op_request, F_CIB_CALLBACK_TOKEN);
 		    if(value != NULL) {
 			cib_client->callback_id = crm_strdup(value);
 
 		    } else {
 			cib_client->callback_id = crm_strdup(cib_client->id);			
 		    }
 		}
 		
 		cib_common_callback_worker(
 			op_request, cib_client, force_synchronous, privileged);
 
 		free_xml(op_request);
 	}
 
 	if(channel->ch_status != IPC_CONNECT) {
 		crm_debug_2("Client disconnected");
 		keep_channel = cib_process_disconnect(channel, cib_client);	
 	}
 
 	return keep_channel;
 }
 
 extern void cib_send_remote_msg(void *session, xmlNode *msg);
 
 static void
 do_local_notify(xmlNode *notify_src, const char *client_id,
 		gboolean sync_reply, gboolean from_peer) 
 {
 	/* send callback to originating child */
 	cib_client_t *client_obj = NULL;
 	enum cib_errors local_rc = cib_ok;
 
 	crm_debug_2("Performing notification");
 
 	if(client_id != NULL) {
 		client_obj = g_hash_table_lookup(client_list, client_id);
 	} else {
 		crm_debug_2("No client to sent the response to."
 			    "  F_CIB_CLIENTID not set.");
 	}
 	
 	crm_debug_3("Sending callback to request originator");
 	if(client_obj == NULL) {
 		local_rc = cib_reply_failed;
 		
 	} else {
 		const char *client_id = client_obj->callback_id;
 		crm_debug_2("Sending %ssync response to %s %s",
 			    sync_reply?"":"an a-",
 			    client_obj->name,
 			    from_peer?"(originator of delegated request)":"");
 		
 		if(sync_reply) {
 			client_id = client_obj->id;
 		}
 		local_rc = send_via_callback_channel(notify_src, client_id);
 	} 
 	
 	if(local_rc != cib_ok && client_obj != NULL) {
 		crm_warn("%sSync reply to %s failed: %s",
 			 sync_reply?"":"A-",
 			 client_obj?client_obj->name:"<unknown>", cib_error2string(local_rc));
 	}
 }
 
 static void
 parse_local_options(
 	cib_client_t *cib_client, int call_type, int call_options, const char *host, const char *op, 
 	gboolean *local_notify, gboolean *needs_reply, gboolean *process, gboolean *needs_forward) 
 {
 	if(cib_op_modifies(call_type)
 	   && !(call_options & cib_inhibit_bcast)) {
 		/* we need to send an update anyway */
 		*needs_reply = TRUE;
 	} else {
 		*needs_reply = FALSE;
 	}
 	
 	if(host == NULL && (call_options & cib_scope_local)) {
 		crm_debug_2("Processing locally scoped %s op from %s",
 			    op, cib_client->name);
 		*local_notify = TRUE;
 		
 	} else if(host == NULL && cib_is_master) {
 		crm_debug_2("Processing master %s op locally from %s",
 			    op, cib_client->name);
 		*local_notify = TRUE;
 		
 	} else if(safe_str_eq(host, cib_our_uname)) {
 		crm_debug_2("Processing locally addressed %s op from %s",
 			    op, cib_client->name);
 		*local_notify = TRUE;
 
 	} else if(stand_alone) {
 		*needs_forward = FALSE;
 		*local_notify = TRUE;
 		*process = TRUE;
 		
 	} else {
 		crm_debug_2("%s op from %s needs to be forwarded to %s",
 			    op, cib_client->name,
 			    host?host:"the master instance");
 		*needs_forward = TRUE;
 		*process = FALSE;
 	}		
 }
 
 static gboolean
 parse_peer_options(
 	int call_type, xmlNode *request, 
 	gboolean *local_notify, gboolean *needs_reply, gboolean *process, gboolean *needs_forward) 
 {
 	const char *op         = crm_element_value(request, F_CIB_OPERATION);
 	const char *originator = crm_element_value(request, F_ORIG);
 	const char *host       = crm_element_value(request, F_CIB_HOST);
 	const char *reply_to   = crm_element_value(request, F_CIB_ISREPLY);
 	const char *update     = crm_element_value(request, F_CIB_GLOBAL_UPDATE);
 	const char *delegated  = crm_element_value(request, F_CIB_DELEGATED);
 
 	if(safe_str_eq(op, "cib_shutdown_req")) {
 		if(reply_to != NULL) {
 			crm_debug("Processing %s from %s", op, host);
 			*needs_reply = FALSE;
 			
 		} else {
 			crm_debug("Processing %s reply from %s", op, host);
 		}
 		return TRUE;
 		
 	} else if(crm_is_true(update) && safe_str_eq(reply_to, cib_our_uname)) {
 		crm_debug_2("Processing global/peer update from %s"
 			    " that originated from us", originator);
 		*needs_reply = FALSE;
 		if(crm_element_value(request, F_CIB_CLIENTID) != NULL) {
 			*local_notify = TRUE;
 		}
 		return TRUE;
 		
 	} else if(crm_is_true(update)) {
 		crm_debug_2("Processing global/peer update from %s", originator);
 		*needs_reply = FALSE;
 		return TRUE;
 
 	} else if(host != NULL && safe_str_eq(host, cib_our_uname)) {
 		crm_debug_2("Processing request sent to us from %s", originator);
 		return TRUE;
 
 	} else if(delegated != NULL && cib_is_master == TRUE) {
 		crm_debug_2("Processing request sent to master instance from %s",
 			originator);
 		return TRUE;
 
 	} else if(reply_to != NULL && safe_str_eq(reply_to, cib_our_uname)) {
 		crm_debug_2("Forward reply sent from %s to local clients",
 			  originator);
 		*process = FALSE;
 		*needs_reply = FALSE;
 		*local_notify = TRUE;
 		return TRUE;
 
 	} else if(delegated != NULL) {
 		crm_debug_2("Ignoring msg for master instance");
 
 	} else if(host != NULL) {
 		/* this is for a specific instance and we're not it */
 		crm_debug_2("Ignoring msg for instance on %s", crm_str(host));
 		
 	} else if(reply_to == NULL && cib_is_master == FALSE) {
 		/* this is for the master instance and we're not it */
 		crm_debug_2("Ignoring reply to %s", crm_str(reply_to));
 		
 	} else {
 		crm_err("Nothing for us to do?");
 		crm_log_xml(LOG_ERR, "Peer[inbound]", request);
 	}
 
 	return FALSE;
 }
 
 		
 static void
 forward_request(xmlNode *request, cib_client_t *cib_client, int call_options)
 {
 	xmlNode *forward_msg = NULL;
 	const char *op         = crm_element_value(request, F_CIB_OPERATION);
 	const char *host       = crm_element_value(request, F_CIB_HOST);
 
 	forward_msg = cib_msg_copy(request, TRUE);
 	crm_xml_add(forward_msg, F_CIB_DELEGATED, cib_our_uname);
 	
 	if(host != NULL) {
 		crm_debug_2("Forwarding %s op to %s", op, host);
 		send_cluster_message(host, crm_msg_cib, forward_msg, FALSE);
 		
 	} else {
 		crm_debug_2("Forwarding %s op to master instance", op);
 		send_cluster_message(NULL, crm_msg_cib, forward_msg, FALSE);
 	}
 	
 	if(call_options & cib_discard_reply) {
 		crm_debug_2("Client not interested in reply");
 	} 
 	free_xml(forward_msg);
 }
 
 static void
 send_peer_reply(
 	xmlNode *msg, xmlNode *result_diff, const char *originator, gboolean broadcast)
 {
 	xmlNode *reply_copy = NULL;
 
 	CRM_ASSERT(msg != NULL);
 
  	reply_copy = cib_msg_copy(msg, TRUE);
 	
 	if(broadcast) {
 		/* this (successful) call modified the CIB _and_ the
 		 * change needs to be broadcast...
 		 *   send via HA to other nodes
 		 */
 		int diff_add_updates = 0;
 		int diff_add_epoch   = 0;
 		int diff_add_admin_epoch = 0;
 		
 		int diff_del_updates = 0;
 		int diff_del_epoch   = 0;
 		int diff_del_admin_epoch = 0;
 
 		char *digest = NULL;
 		
 		cib_diff_version_details(
 			result_diff,
 			&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates, 
 			&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
 
 		crm_debug_2("Sending update diff %d.%d.%d -> %d.%d.%d",
 			    diff_del_admin_epoch,diff_del_epoch,diff_del_updates,
 			    diff_add_admin_epoch,diff_add_epoch,diff_add_updates);
 
 		crm_xml_add(reply_copy, F_CIB_ISREPLY, originator);
 		crm_xml_add(reply_copy, F_CIB_GLOBAL_UPDATE, XML_BOOLEAN_TRUE);
 		crm_xml_add(reply_copy, F_CIB_OPERATION, CIB_OP_APPLY_DIFF);
 
 		digest = calculate_xml_digest(the_cib, FALSE, TRUE);
 		crm_xml_add(result_diff, XML_ATTR_DIGEST, digest);
 /* 		crm_log_xml_debug(the_cib, digest); */
 		crm_free(digest);
 		
  		add_message_xml(reply_copy, F_CIB_UPDATE_DIFF, result_diff);
 		crm_log_xml(LOG_DEBUG_3, "copy", reply_copy);
 		send_cluster_message(NULL, crm_msg_cib, reply_copy, TRUE);
 		
 	} else if(originator != NULL) {
 		/* send reply via HA to originating node */
 		crm_debug_2("Sending request result to originator only");
 		crm_xml_add(reply_copy, F_CIB_ISREPLY, originator);
 		send_cluster_message(originator, crm_msg_cib, reply_copy, FALSE);
 	}
 	
 	free_xml(reply_copy);
 }
 	
 void
 cib_process_request(
 	xmlNode *request, gboolean force_synchronous, gboolean privileged,
 	gboolean from_peer, cib_client_t *cib_client) 
 {
 	int call_type    = 0;
 	int call_options = 0;
 
 	gboolean process = TRUE;		
 	gboolean is_update = TRUE;
 	gboolean needs_reply = TRUE;
 	gboolean local_notify = FALSE;
 	gboolean needs_forward = FALSE;
 	gboolean global_update = crm_is_true(crm_element_value(request, F_CIB_GLOBAL_UPDATE));
 	
 	xmlNode *op_reply = NULL;
 	xmlNode *result_diff = NULL;
 
 	enum cib_errors rc = cib_ok;
 	const char *op         = crm_element_value(request, F_CIB_OPERATION);
 	const char *originator = crm_element_value(request, F_ORIG);
 	const char *host       = crm_element_value(request, F_CIB_HOST);
 
 	crm_debug_4("%s Processing msg %s",
 		  cib_our_uname, crm_element_value(request, F_SEQ));
 
 	cib_num_ops++;
 	if(cib_num_ops == 0) {
 		cib_num_fail = 0;
 		cib_num_local = 0;
 		cib_num_updates = 0;
 		crm_info("Stats wrapped around");
 	}
 	
 	if(host != NULL && strlen(host) == 0) {
 		host = NULL;
 	}	
 
 	crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
 	if(force_synchronous) {
 		call_options |= cib_sync_call;
 	}
 	
 	crm_debug_2("Processing %s message (%s) for %s...",
 		    from_peer?"peer":"local",
 		    from_peer?originator:cib_our_uname, host?host:"master");
 
 	rc = cib_get_operation_id(op, &call_type);
 	if(rc != cib_ok) {
 		/* TODO: construct error reply? */
 		crm_err("Pre-processing of command failed: %s", cib_error2string(rc));
 		return;
 	}
 	
 	is_update = cib_op_modifies(call_type);
 	if(is_update) {
 		cib_num_updates++;
 	}
 	
 	if(from_peer == FALSE) {
 		parse_local_options(cib_client, call_type, call_options, host, op,
 				    &local_notify, &needs_reply, &process, &needs_forward);
 		
 	} else if(parse_peer_options(call_type, request, &local_notify,
 				     &needs_reply, &process, &needs_forward) == FALSE) {
 		return;
 	}
 	crm_debug_3("Finished determining processing actions");
 
 	if(call_options & cib_discard_reply) {
 		needs_reply = is_update;
 		local_notify = FALSE;
 	}
 	
 	if(needs_forward) {
 		forward_request(request, cib_client, call_options);
 		return;
 	}
 
 	if(cib_status != cib_ok) {
 	    rc = cib_status;
 	    crm_err("Operation ignored, cluster configuration is invalid."
 		    " Please repair and restart: %s",
 		    cib_error2string(cib_status));
 	    op_reply = cib_construct_reply(request, the_cib, cib_status);
 
 	} else if(process) {
 		int level = LOG_INFO;
 		const char *section = crm_element_value(request, F_CIB_SECTION);
 		
 		cib_num_local++;
 		rc = cib_process_command(
 			request, &op_reply, &result_diff, privileged);
 
 		if(global_update) {
 		    switch(rc) {
 			case cib_ok:
 			case cib_old_data:
 			case cib_diff_resync:
 			case cib_diff_failed:
 			    level = LOG_DEBUG_2;
 			    break;
 			default:
 			    level = LOG_ERR;
 		    }
 
 		} else if(safe_str_eq(op, CIB_OP_QUERY)) {
 		    level = LOG_DEBUG_2;
 
 		} else if(rc != cib_ok) {
 		    cib_num_fail++;
 		    level = LOG_ERR;
 
 		} else if(safe_str_eq(op, CIB_OP_QUERY)) {
 		    level = LOG_DEBUG_2;
 
 		} else if(safe_str_eq(section, XML_CIB_TAG_STATUS)) {
 		    level = LOG_DEBUG_2;
 		}
 		
 		do_crm_log(level, "Operation complete: op %s for section %s (origin=%s/%s/%s): %s (rc=%d)",
 			   op, section?section:"'all'", originator?originator:"local",
 			   crm_element_value(request, F_CIB_CLIENTID),
 			   crm_element_value(request, F_CIB_CALLID),
 			   cib_error2string(rc), rc);
 
 		if(op_reply == NULL && (needs_reply || local_notify)) {
 			crm_err("Unexpected NULL reply to message");
 			crm_log_xml(LOG_ERR, "null reply", request);
 			needs_reply = FALSE;
 			local_notify = FALSE;
 		}		
 	}
 	crm_debug_3("processing response cases");
 	
 	if(local_notify) {
 		const char *client_id = crm_element_value(request, F_CIB_CLIENTID);
 		if(process == FALSE) {
 			do_local_notify(request, client_id, call_options & cib_sync_call, from_peer);
 		} else {
 			do_local_notify(op_reply, client_id, call_options & cib_sync_call, from_peer);
 		}
 	}
 
 	/* from now on we are the server */ 
 	if(needs_reply == FALSE || stand_alone) {
 		/* nothing more to do...
 		 * this was a non-originating slave update
 		 */
 		crm_debug_2("Completed slave update");
 
 	} else if(rc == cib_ok
 		  && result_diff != NULL
 		  && !(call_options & cib_inhibit_bcast)) {
 		send_peer_reply(request, result_diff, originator, TRUE);
 		
 	} else if(call_options & cib_discard_reply) {
 		crm_debug_4("Caller isn't interested in reply");
 		
 	} else if (from_peer) {		
 		if(is_update == FALSE || result_diff == NULL) {
 			crm_debug_3("Request not broadcast: R/O call");
 
 		} else if(call_options & cib_inhibit_bcast) {
 			crm_debug_3("Request not broadcast: inhibited");
 
 		} else if(rc != cib_ok) {
 			crm_debug_3("Request not broadcast: call failed: %s",
 				    cib_error2string(rc));
 		} else {
 		    crm_debug_2("Directing reply to %s", originator);
 		}
 
 		send_peer_reply(op_reply, result_diff, originator, FALSE);
 	}
 	
 	free_xml(op_reply);
 	free_xml(result_diff);
 
 	return;	
 }
 
 xmlNode *
 cib_construct_reply(xmlNode *request, xmlNode *output, int rc) 
 {
 	int lpc = 0;
 	xmlNode *reply = NULL;
 	
 	const char *name = NULL;
 	const char *value = NULL;
 	const char *names[] = {
 		F_CIB_OPERATION,
 		F_CIB_CALLID,
 		F_CIB_CLIENTID,
 		F_CIB_CALLOPTS
 	};
 
 	crm_debug_4("Creating a basic reply");
 	reply = create_xml_node(NULL, "cib-reply");
 	crm_xml_add(reply, F_TYPE, T_CIB);
 
 	for(lpc = 0; lpc < DIMOF(names); lpc++) {
 		name = names[lpc];
 		value = crm_element_value(request, name);
 		crm_xml_add(reply, name, value);
 	}
 
 	crm_xml_add_int(reply, F_CIB_RC, rc);
 
 	if(output != NULL) {
 		crm_debug_4("Attaching reply output");
 		add_message_xml(reply, F_CIB_CALLDATA, output);
 	}
 	return reply;
 }
 
 enum cib_errors
 cib_process_command(xmlNode *request, xmlNode **reply,
 		    xmlNode **cib_diff, gboolean privileged)
 {
     xmlNode *input       = NULL;
     xmlNode *output      = NULL;
     xmlNode *result_cib  = NULL;
     xmlNode *current_cib = NULL;
 	
     int call_type    = 0;
     int call_options = 0;
     int log_level    = LOG_DEBUG_4;
 
     const char *op = NULL;
     const char *section = NULL;
 
     enum cib_errors rc = cib_ok;
     enum cib_errors rc2 = cib_ok;
 	
     gboolean send_r_notify = FALSE;
     gboolean global_update = FALSE;
     gboolean config_changed = FALSE;
     gboolean manage_counters = TRUE;
 	
     CRM_ASSERT(cib_status == cib_ok);
 
     *reply = NULL;
     *cib_diff = NULL;
     current_cib = the_cib;
 	
     /* Start processing the request... */
     op = crm_element_value(request, F_CIB_OPERATION);
     crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
     rc = cib_get_operation_id(op, &call_type);
 	
     if(rc == cib_ok) {
 	rc = cib_op_can_run(call_type, call_options, privileged, global_update);
     }
 	
     rc2 = cib_op_prepare(call_type, request, &input, &section);
     if(rc == cib_ok) {
 	rc = rc2;
     }
 	
     if(rc != cib_ok) {
 	crm_debug_2("Call setup failed: %s", cib_error2string(rc));
 	goto done;
 		
     } else if(cib_op_modifies(call_type) == FALSE) {
 	rc = cib_perform_op(op, call_options, cib_op_func(call_type), TRUE,
 			    section, request, input, FALSE, &config_changed,
 			    current_cib, &result_cib, NULL, &output);
 
 	CRM_CHECK(result_cib == NULL, free_xml(result_cib));
 	goto done;
     }	
 
     /* Handle a valid write action */
     global_update = crm_is_true(crm_element_value(request, F_CIB_GLOBAL_UPDATE));
     if(global_update) {
 	manage_counters = FALSE;
 	call_options |= cib_force_diff;		
 
 	CRM_CHECK(call_type == 3 || call_type == 4,
 		  crm_err("Call type: %d", call_type);
 		  crm_log_xml(LOG_ERR, "bad op", request));
     }
 #ifdef SUPPORT_PRENOTIFY
     if((call_options & cib_inhibit_notify) == 0) {
 	cib_pre_notify(call_options, op, the_cib, input);
     }
 #endif
     
     if(rc == cib_ok) {
 	if(call_options & cib_inhibit_bcast) {
 	    /* skip */
 	    crm_debug_2("Skipping update: inhibit broadcast");
 	    manage_counters = FALSE;
 	}	
 	    
 	rc = cib_perform_op(op, call_options, cib_op_func(call_type), FALSE,
 			    section, request, input, manage_counters, &config_changed,
 			    current_cib, &result_cib, cib_diff, &output);
 
 	if(manage_counters == FALSE) {
 	    *cib_diff = diff_cib_object(current_cib, result_cib, FALSE);
 	}
     }    
     
     if(rc == cib_ok) {
 	rc = activateCibXml(result_cib, config_changed, op);
 	
 	if(crm_str_eq(CIB_OP_REPLACE, op, TRUE)) {
 	    if(section == NULL) {
 		send_r_notify = TRUE;
 		
 	    } else if(safe_str_eq(section, XML_TAG_CIB)) {
 		send_r_notify = TRUE;
 		
 	    } else if(safe_str_eq(section, XML_CIB_TAG_NODES)) {
 		send_r_notify = TRUE;
 		
 	    } else if(safe_str_eq(section, XML_CIB_TAG_STATUS)) {
 		send_r_notify = TRUE;
 	    }
 
 	} else if(crm_str_eq(CIB_OP_ERASE, op, TRUE)) {
 	    send_r_notify = TRUE;
 	}
 
     } else {
 	free_xml(result_cib);    
     }
     
     if((call_options & cib_inhibit_notify) == 0) {
 	const char *call_id = crm_element_value(request, F_CIB_CALLID);
 	const char *client = crm_element_value(request, F_CIB_CLIENTNAME);
 
 #ifdef SUPPORT_POSTNOTIFY
 	cib_post_notify(call_options, op, input, rc, the_cib);
 #endif
 	cib_diff_notify(call_options, client, call_id, op, input, rc, *cib_diff);
     }
 
     if(send_r_notify) {
 	cib_replace_notify(the_cib, rc, *cib_diff);
     }	
     
     if(rc != cib_ok) {
 	log_level = LOG_DEBUG_4;
 	if(rc == cib_dtd_validation && global_update) {
 	    log_level = LOG_WARNING;
 	    crm_log_xml_info(input, "cib:global_update");
 	}
 	
     } else if(config_changed) {
 	log_level = LOG_DEBUG_3;
 	if(cib_is_master) {
 	    log_level = LOG_INFO;
 	}
 	
     } else if(cib_is_master) {
 	log_level = LOG_DEBUG_2;
     }
 	
     log_xml_diff(log_level, *cib_diff, "cib:diff");
 
   done:
     if((call_options & cib_discard_reply) == 0) {
 	*reply = cib_construct_reply(request, output, rc);
 	/* crm_log_xml_info(*reply, "cib:reply"); */
     }
 
     if(call_type >= 0) {
 	cib_op_cleanup(call_type, op, &input, &output);
     }
     return rc;
 }
 
 int
 send_via_callback_channel(xmlNode *msg, const char *token) 
 {
 	cib_client_t *hash_client = NULL;
 	enum cib_errors rc = cib_ok;
 	
 	crm_debug_3("Delivering msg %p to client %s", msg, token);
 
 	if(token == NULL) {
 		crm_err("No client id token, cant send message");
 		if(rc == cib_ok) {
 			rc = cib_missing;
 		}
 
 	} else if(msg == NULL) {
 		crm_err("No message to send");
 		rc = cib_reply_failed;
 	    
 	} else {
 		/* A client that left before we could reply is not really
 		 * _our_ error.  Warn instead.
 		 */
 		hash_client = g_hash_table_lookup(client_list, token);
 		if(hash_client == NULL) {
 			crm_warn("Cannot find client for token %s", token);
 			rc = cib_client_gone;
 			
 		} else if (crm_str_eq(hash_client->channel_name, "remote", FALSE)) {
 		    /* just hope it's alive */
 		    
 		} else if(hash_client->channel == NULL) {
 			crm_err("Cannot find channel for client %s", token);
 			rc = cib_client_corrupt;
 		}
 	}
 
 	if(rc == cib_ok) {
 	    crm_debug_3("Delivering reply to client %s (%s)",
 			token, hash_client->channel_name);
 	    if (crm_str_eq(hash_client->channel_name, "remote", FALSE)) {
 		cib_send_remote_msg(hash_client->channel, msg);
 		
 	    } else if(send_ipc_message(hash_client->channel, msg) == FALSE) {
 		crm_warn("Delivery of reply to client %s/%s failed",
 			 hash_client->name, token);
 		rc = cib_reply_failed;
 	    }
 	}
 	
 	return rc;
 }
 
 gint cib_GCompareFunc(gconstpointer a, gconstpointer b)
 {
 	const xmlNode *a_msg = a;
 	const xmlNode *b_msg = b;
 
 	int msg_a_id = 0;
 	int msg_b_id = 0;
 	const char *value = NULL;
 	
 	value = crm_element_value_const(a_msg, F_CIB_CALLID);
 	msg_a_id = crm_parse_int(value, NULL);
 
 	value = crm_element_value_const(b_msg, F_CIB_CALLID);
 	msg_b_id = crm_parse_int(value, NULL);
 	
 	if(msg_a_id == msg_b_id) {
 		return 0;
 	} else if(msg_a_id < msg_b_id) {
 		return -1;
 	}
 	return 1;
 }
 
 gboolean
 cib_process_disconnect(IPC_Channel *channel, cib_client_t *cib_client)
 {
 	if (channel == NULL) {
 		CRM_DEV_ASSERT(cib_client == NULL);
 		
 	} else if (cib_client == NULL) {
 		crm_err("No client");
 		
 	} else {
 		CRM_DEV_ASSERT(channel->ch_status != IPC_CONNECT);
 		crm_debug_2("Cleaning up after client disconnect: %s/%s/%s",
 			    crm_str(cib_client->name),
 			    cib_client->channel_name,
 			    cib_client->id);
 		
 		if(cib_client->id != NULL) {
 			if(!g_hash_table_remove(client_list, cib_client->id)) {
 				crm_err("Client %s not found in the hashtable",
 					cib_client->name);
 			}
 		}		
 	}
 
 	if(cib_shutdown_flag && g_hash_table_size(client_list) == 0) {
 		crm_info("All clients disconnected...");
 		initiate_exit();
 	}
 	
 	return FALSE;
 }
 
 void
 cib_ha_peer_callback(HA_Message * msg, void* private_data)
 {
     xmlNode *xml = convert_ha_message(NULL, msg, __FUNCTION__);
     cib_peer_callback(xml, private_data);
 }
 
 void
 cib_peer_callback(xmlNode * msg, void* private_data)
 {
     crm_node_t *node = NULL;
     const char *reason = NULL;
     const char *originator = crm_element_value(msg, F_ORIG);
     
     if(originator == NULL || crm_str_eq(originator, cib_our_uname, TRUE)) {
 	/* message is from ourselves */
 	return;
 	
     } else if(crm_peer_cache == NULL) {
 	reason = "membership not established";
 	goto bail;
     }
 
     node = crm_get_peer(0, originator);
     if(node == NULL || crm_is_member_active(node) == FALSE) {
 	reason = "not in our membership";
 	goto bail;
     }
     
     if(crm_element_value(msg, F_CIB_CLIENTNAME) == NULL) {
 	crm_xml_add(msg, F_CIB_CLIENTNAME, originator);
     }
     
     /* crm_log_xml(LOG_MSG, "Peer[inbound]", msg); */
     cib_process_request(msg, FALSE, TRUE, TRUE, NULL);
     return;
     
   bail:
     if(reason) {
 	const char *seq = crm_element_value(msg, F_SEQ);
 	const char *op  = crm_element_value(msg, F_CIB_OPERATION);
 	crm_warn("Discarding %s message (%s) from %s: %s", op, seq, originator, reason);
     }
 }
 
 void
 cib_client_status_callback(const char * node, const char * client,
 			   const char * status, void * private)
 {
     crm_node_t *member = NULL;
     if(safe_str_eq(client, CRM_SYSTEM_CIB)) {
 	crm_info("Status update: Client %s/%s now has status [%s]",
 		 node, client, status);
 	
 	if(safe_str_eq(status, JOINSTATUS)){
 		    status = ONLINESTATUS;
 		    
 	} else if(safe_str_eq(status, LEAVESTATUS)){
 	    status = OFFLINESTATUS;
 	}
 
 	member = crm_get_peer(0, node);
 	if(member == NULL) {
 	    /* Make sure it gets created */
 	    const char *uuid = get_uuid(node);
 	    member = crm_update_peer(0, 0, 0, -1, 0, uuid, node, NULL, NULL);
 	}
 	
 	crm_update_peer_proc(node, crm_proc_cib, status);
     }
     return;
 }
 
 #if SUPPORT_HEARTBEAT
 extern oc_ev_t *cib_ev_token;
 
 gboolean cib_ccm_dispatch(int fd, gpointer user_data)
 {
 	int rc = 0;
 	oc_ev_t *ccm_token = (oc_ev_t*)user_data;
 	crm_debug_2("received callback");	
 	rc = oc_ev_handle_event(ccm_token);
 	if(0 == rc) {
 		return TRUE;
 
 	}
 
 	crm_err("CCM connection appears to have failed: rc=%d.", rc);
 
 	/* eventually it might be nice to recover and reconnect... but until then... */
 	crm_err("Exiting to recover from CCM connection failure");
 	exit(2);
 	
 	return FALSE;
 }
 
 int current_instance = 0;
 void 
 cib_ccm_msg_callback(
 	oc_ed_t event, void *cookie, size_t size, const void *data)
 {
 	gboolean update_id = FALSE;
 	const oc_ev_membership_t *membership = data;
 
 	CRM_ASSERT(membership != NULL);
 
 	crm_info("Processing CCM event=%s (id=%d)",
 		 ccm_event_name(event), membership->m_instance);
 
 	if(current_instance > membership->m_instance) {
 		crm_err("Membership instance ID went backwards! %d->%d",
 			current_instance, membership->m_instance);
 		CRM_ASSERT(current_instance <= membership->m_instance);
 	}
 	
 	switch(event) {
 		case OC_EV_MS_NEW_MEMBERSHIP:
 		case OC_EV_MS_INVALID:
 			update_id = TRUE;
 			break;
 		case OC_EV_MS_PRIMARY_RESTORED:
 			update_id = TRUE;
 			break;
 		case OC_EV_MS_NOT_PRIMARY:
 			crm_debug_2("Ignoring transitional CCM event: %s",
 				    ccm_event_name(event));
 			break;
 		case OC_EV_MS_EVICTED:
 			crm_err("Evicted from CCM: %s", ccm_event_name(event));
 			break;
 		default:
 			crm_err("Unknown CCM event: %d", event);
 	}
 	
 	if(update_id) {
 		unsigned int lpc = 0;
 		CRM_CHECK(membership != NULL, return);
 	
 		current_instance = membership->m_instance;
 
 		for(lpc=0; lpc < membership->m_n_out; lpc++) {
 		    crm_update_ccm_node(
 			membership, lpc+membership->m_out_idx, CRM_NODE_LOST, current_instance);
 		}
 		
 		for(lpc=0; lpc < membership->m_n_member; lpc++) {
 		    crm_update_ccm_node(
 			membership, lpc+membership->m_memb_idx,CRM_NODE_ACTIVE, current_instance);
 		}
 	}
 	
 	oc_ev_callback_done(cookie);
 	return;
 }
 #endif
 
 gboolean
 can_write(int flags)
 {
 	return TRUE;
 }
 
 static gboolean
 cib_force_exit(gpointer data)
 {
 	crm_notice("Forcing exit!");
 	terminate_cib(__FUNCTION__);
 	return FALSE;
 }
 
 void
 initiate_exit(void)
 {
 	int active = 0;
 	xmlNode *leaving = NULL;
 
 	active = crm_active_peers(crm_proc_cib);
 	if(active < 2) {
 		terminate_cib(__FUNCTION__);
 		return;
 	} 
 
 	crm_info("Sending disconnect notification to %d peers...", active);
 
 	leaving = create_xml_node(NULL, "exit-notification");	
 	crm_xml_add(leaving, F_TYPE, "cib");
 	crm_xml_add(leaving, F_CIB_OPERATION, "cib_shutdown_req");
 	
 	send_cluster_message(NULL, crm_msg_cib, leaving, TRUE);
 	free_xml(leaving);
 	
 	Gmain_timeout_add(crm_get_msec("5s"), cib_force_exit, NULL);
 }
 
 void
 terminate_cib(const char *caller) 
 {
 #if SUPPORT_AIS
     if(is_openais_cluster()) {
 	cib_ha_connection_destroy(NULL);
 	return;
     } 
 #endif
 #if SUPPORT_HEARTBEAT
     if(hb_conn != NULL) {
 	crm_info("%s: Disconnecting heartbeat", caller);
 	hb_conn->llc_ops->signoff(hb_conn, FALSE);
 
     } else {
 	crm_err("%s: No heartbeat connection", caller);
     }
 #endif
 		
     uninitializeCib();
      
     crm_info("Exiting...");
     
     if (mainloop != NULL && g_main_is_running(mainloop)) {
 	g_main_quit(mainloop);
 	
     } else {
 	exit(LSB_EXIT_OK);
     }
 
 }
diff --git a/cib/cibmessages.h b/cib/cibmessages.h
index 54d763c817..88dca66b17 100644
--- a/cib/cibmessages.h
+++ b/cib/cibmessages.h
@@ -1,71 +1,67 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 #ifndef CIB_MESSAGES__H
 #define CIB_MESSAGES__H
 
 #include <crm/cib_ops.h>
 extern xmlNode *createCibRequest(
 	gboolean isLocal, const char *operation, const char *section,
 	const char *verbose, xmlNode *data);
 
 extern enum cib_errors 
 cib_process_shutdown_req(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 extern enum cib_errors cib_process_default(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 extern enum cib_errors cib_process_quit(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 extern enum cib_errors cib_process_ping(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 extern enum cib_errors cib_process_readwrite(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 extern enum cib_errors cib_process_replace_svr(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 extern enum cib_errors cib_server_process_diff(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 extern enum cib_errors cib_process_sync(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 extern enum cib_errors cib_process_sync_one(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
-extern enum cib_errors cib_process_create(
-	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
-	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
-
 extern enum cib_errors cib_process_delete_absolute(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 #endif
diff --git a/cib/cibpipe.c b/cib/cibpipe.c
index 4f2d7e4276..5d4d7abebb 100644
--- a/cib/cibpipe.c
+++ b/cib/cibpipe.c
@@ -1,372 +1,371 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/ctrl.h>
 #include <crm/common/xml.h>
 
 #include "common.h"
 
 #ifdef HAVE_GETOPT_H
 #  include <getopt.h>
 #endif
 
 void usage(const char* cmd, int exit_status);
 
 struct cib_func_entry 
 {
 	const char *op;
 	gboolean    read_only;
 	cib_op_t    fn;
 };
 
 static struct cib_func_entry cib_pipe_ops[] = {
     {CIB_OP_QUERY,      TRUE,  cib_process_query},
     {CIB_OP_MODIFY,     FALSE, cib_process_modify},
-    /* {CIB_OP_UPDATE,     FALSE, cib_process_change}, */
     {CIB_OP_APPLY_DIFF, FALSE, cib_process_diff},
     {CIB_OP_BUMP,       FALSE, cib_process_bump},
     {CIB_OP_REPLACE,    FALSE, cib_process_replace},
-    /* {CIB_OP_CREATE,     FALSE, cib_process_change}, */
+    {CIB_OP_CREATE,     FALSE, cib_process_create},
     {CIB_OP_DELETE,     FALSE, cib_process_delete},
     {CIB_OP_ERASE,      FALSE, cib_process_erase},
 };
 
 #define OPTARGS	"V?o:QDUCEX:t:MBfRx:P5S"
 
 int
 main(int argc, char ** argv)
 {
     int lpc;
     int flag;
     int rc = 0;
     int argerr = 0;
     int max_msg_types = DIMOF(cib_pipe_ops);
 
     int command_options = 0;
     gboolean changed = FALSE;
     gboolean force_flag = FALSE;
     gboolean dangerous_cmd = FALSE;
 	
     char *buffer = NULL;
     const char *section = NULL;
     const char *input_xml = NULL;
     const char *input_file = NULL;
     const char *output_file = NULL;
     const char *cib_action = NULL;
 	
     xmlNode *input = NULL;
     xmlNode *output = NULL;
     xmlNode *result_cib = NULL;
     xmlNode *current_cib = NULL;
 
     gboolean query = FALSE;
     cib_op_t *fn = NULL;
     
 #ifdef HAVE_GETOPT_H
     int option_index = 0;
     static struct option long_options[] = {
 	{CIB_OP_ERASE,   0, 0, 'E'},
 	{CIB_OP_QUERY,   0, 0, 'Q'},
 	{CIB_OP_CREATE,  0, 0, 'C'},
 	{CIB_OP_REPLACE, 0, 0, 'R'},
 	{CIB_OP_UPDATE,  0, 0, 'U'},
 	{CIB_OP_MODIFY,  0, 0, 'M'},
 	{"patch",	 0, 0, 'P'},
 	{CIB_OP_DELETE,  0, 0, 'D'},
 	{CIB_OP_BUMP,    0, 0, 'B'},
 	{"md5-sum",	 0, 0, '5'},
 
 	{"force",	0, 0, 'f'},
 	{"xml-file",    1, 0, 'x'},
 	{"xml-text",    1, 0, 'X'},
 	{"xml-save",    1, 0, 'S'},
 	{"obj_type",    1, 0, 'o'},
 
 	{"verbose",     0, 0, 'V'},
 	{"help",        0, 0, '?'},
 
 	{0, 0, 0, 0}
     };
 #endif
 	
     crm_log_init("cibpipe", LOG_ERR, FALSE, FALSE, argc, argv);
 
     while (1) {
 #ifdef HAVE_GETOPT_H
 	flag = getopt_long(argc, argv, OPTARGS,
 			   long_options, &option_index);
 #else
 	flag = getopt(argc, argv, OPTARGS);
 #endif
 	if (flag == -1)
 	    break;
 		
 	switch(flag) {
 	    case 'E':
 		cib_action = CIB_OP_ERASE;
 		dangerous_cmd = TRUE;
 		break;
 	    case 'Q':
 		cib_action = CIB_OP_QUERY;
 		break;
 	    case 'P':
 		cib_action = CIB_OP_APPLY_DIFF;
 		break;
 	    case 'S':
 		cib_action = CIB_OP_SYNC;
 		break;
 	    case 'U':
 	    case 'M':
 		cib_action = CIB_OP_MODIFY;
 		break;
 	    case 'R':
 		cib_action = CIB_OP_REPLACE;
 		break;
 	    case 'C':
 		cib_action = CIB_OP_CREATE;
 		break;
 	    case 'D':
 		cib_action = CIB_OP_DELETE;
 		break;
 	    case '5':
 		cib_action = "md5-sum";
 		break;
 	    case 'd':
 		cib_action = CIB_OP_DELETE_ALT;
 		break;
 	    case 'm':
 		cib_action = CIB_OP_ISMASTER;
 		command_options |= cib_scope_local;
 		break;
 	    case 'B':
 		cib_action = CIB_OP_BUMP;
 		break;
 	    case 'o':
 		crm_debug_2("Option %c => %s", flag, optarg);
 		section = crm_strdup(optarg);
 		break;
 	    case 'x':
 		crm_debug_2("Option %c => %s", flag, optarg);
 		input_file = crm_strdup(optarg);
 		break;
 	    case 'X':
 		crm_debug_2("Option %c => %s", flag, optarg);
 		input_xml = crm_strdup(optarg);
 		break;
 	    case 'f':
 		force_flag = TRUE;
 		command_options |= cib_quorum_override;
 		break;		    
 	    case 'V':
 		alter_debug(DEBUG_INC);
 		cl_log_enable_stderr(1);
 		break;
 	    case '?':		/* Help message */
 		usage(crm_system_name, LSB_EXIT_OK);
 		break;
 	    default:
 		++argerr;
 		break;
 	}
     }
 
     if (cib_action == NULL) {
 	++argerr;
     }
     
     if (optind > argc) {
 	++argerr;
     }
     
     if (argerr) {
 	usage(crm_system_name, LSB_EXIT_GENERIC);
     }
 	
     if(dangerous_cmd && force_flag == FALSE) {
 	fprintf(stderr, "The supplied command is considered dangerous."
 		"  To prevent accidental destruction of the cluster,"
 		" the --force flag is required in order to proceed.\n");
 	fflush(stderr);
 	usage(crm_system_name, LSB_EXIT_GENERIC);	    
     }
 
     if(input_file != NULL) {
 	input = filename2xml(input_file);
 	if(input == NULL) {
 	    fprintf(stderr, "Couldn't parse input file: %s\n", input_file);
 	    return 1;
 	}
 	    
     } else if(input_xml != NULL) {
 	input = string2xml(input_xml);
 	if(input == NULL) {
 	    fprintf(stderr, "Couldn't parse input string: %s\n", input_xml);
 	    return 1;
 	}
     }
 
     if(input && safe_str_eq(cib_action, CIB_OP_QUERY)) {
 	current_cib = copy_xml(input);
 
     } else {
 	current_cib = stdin2xml();
 	if(current_cib == NULL && safe_str_neq(cib_action, CIB_OP_ERASE)) {
 	    fprintf(stderr, "Couldn't parse existing CIB from STDIN.\n");
 	    return 1;
 	}
     }
 	
 	
     if(current_cib == NULL) {
 	current_cib = createEmptyCib();
     }
     result_cib = copy_xml(current_cib);
 
     if(safe_str_eq(cib_action, "md5-sum")) {
 	char *digest = NULL;
 	digest = calculate_xml_digest(current_cib, FALSE, FALSE);
 	fprintf(stdout, "%s\n", crm_str(digest));
 	crm_free(digest);
 	return 0;
     }
 
     
     /* read local config file */
     if(cib_action == NULL) {
 	crm_err("No operation specified");
 	return cib_operation;
     }
 
     for (lpc = 0; lpc < max_msg_types; lpc++) {
 	if (safe_str_eq(cib_action, cib_pipe_ops[lpc].op)) {
 	    fn = &(cib_pipe_ops[lpc].fn);
 	    query = cib_pipe_ops[lpc].read_only;
 	    break;
 	}
     }
     
     if(fn == NULL) {
 	rc = cib_NOTSUPPORTED;
     } else {
 	rc = cib_perform_op(cib_action, command_options, fn, query,
 			    section, NULL, input, TRUE, &changed,
 			    current_cib, &result_cib, NULL, &output);
     }
 
     if(rc != cib_ok) {
 	fprintf(stderr, "Call failed: %s\n", cib_error2string(rc));
 	fprintf(stdout, "%c", 0);
 	return -rc;    
     }
 
     cl_log_args(argc, argv);
     
     if(output) {
 	buffer = dump_xml_formatted(output);
     } else {
 	buffer = dump_xml_formatted(result_cib);
     }
 
     fprintf(stdout, "%s\n", buffer);
     fflush(stdout);
 
     if(output_file != NULL) {
 	FILE *output_strm = fopen(output_file, "w");
 	if(output_strm == NULL) {
 	    cl_perror("Could not open %s for writing", output_file);
 	} else {
 	    if(fprintf(output_strm, "%s\n", buffer) < 0) {
 		cl_perror("Write to %s failed", output_file);
 	    }
 	    fflush(output_strm);
 	    fclose(output_strm);
 	}
     }
     
     crm_info("Done");
     return 0;
 }
 
 
 
 void
 usage(const char* cmd, int exit_status)
 {
     FILE* stream;
 
     stream = exit_status ? stderr : stdout;
 
     fprintf(stream, "usage: %s -Q -(x|X)\n", cmd);
     fprintf(stream, "usage: %s -Q -(x|X) | %s [-%s] | %s [-%s] | ...\n",
 	    cmd, cmd, OPTARGS, cmd, OPTARGS);
     fprintf(stream, "usage: cibadmin -Q  | %s [-%s] | %s [-%s] | ...\n",
 	    cmd, OPTARGS, cmd, OPTARGS);
 
     fprintf(stream, "\nOptions\n");
     fprintf(stream, "\t--%s (-%c) <type>\tobject type being operated on\n",
 	    "obj_type", 'o');
     fprintf(stream, "\t\tValid values are:"
 	    " nodes, resources, constraints, crm_config, status\n");
     fprintf(stream, "\t--%s (-%c)\tturn on debug info."
 	    "  additional instance increase verbosity\n", "verbose", 'V');
     fprintf(stream, "\t--%s (-%c)\tthis help message\n", "help", '?');
     
     fprintf(stream, "\nCommands\n");
     fprintf(stream, "\t--%s (-%c)\tErase the contents of the whole CIB\n",
 	    CIB_OP_ERASE,  'E');
     fprintf(stream, "\t--%s (-%c)\t\n", CIB_OP_QUERY,  'Q');
     fprintf(stream, "\t--%s (-%c)\tCreate an object that does not yet exist\n", CIB_OP_CREATE, 'C');
     fprintf(stream, "\t--%s (-%c)\tRecursivly update an object in the CIB\n",
 	    CIB_OP_UPDATE, 'U');
     fprintf(stream, "\t--%s (-%c)\tFind the object somewhere in the CIB's XML tree and update it as --"CIB_OP_UPDATE" would\n", CIB_OP_MODIFY, 'M');
     fprintf(stream, "\t--%s (-%c)\tRecursivly replace an object in the CIB\n",
 	    CIB_OP_REPLACE,'R');
     fprintf(stream, "\t--%s (-%c)\t\n", CIB_OP_DELETE, 'D');
     fprintf(stream, "\t\t\tDelete the first object matching the supplied criteria\n");
     fprintf(stream, "\t\t\tEg. <op id=\"rsc1_op1\" name=\"monitor\"/>\n");
     fprintf(stream, "\t\t\tThe tagname and all attributes must match in order for the element to be deleted\n");
     
     fprintf(stream, "\t--%s (-%c)\t\n", CIB_OP_BUMP,   'B');
     fprintf(stream, "\t--%s (-%c)\t\tCalculate the configuration's digest.\n",
 	    "md5-sum", '5');
     fprintf(stream, "\nXML data\n");
     fprintf(stream, "\t--%s (-%c) <filename>\tRetrieve XML from the named file\n",
 	    "xml-file", 'x');
     fprintf(stream, "\t--%s (-%c) <string>\tRetrieve XML from the supplied string\n",
 	    "xml-text", 'X');
     fprintf(stream, "\t--%s (-%c) <filename>\tSave the XML output to the named file\n",
 	    "xml-save", 'S');
     fprintf(stream, "\nNOTE: The current CIB is assumed to be passed in via stdin,"
 	    " unless -Q is used in which case -x or -X are also acceptable\n");
     fflush(stream);
     
     exit(exit_status);
 }
diff --git a/cib/common.c b/cib/common.c
index 17a0658b93..13b18979c1 100644
--- a/cib/common.c
+++ b/cib/common.c
@@ -1,336 +1,335 @@
 /* 
  * Copyright (C) 2008 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 
 #include <clplumbing/uids.h>
 #include <clplumbing/cl_uuid.h>
 #include <clplumbing/Gmain_timeout.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/cluster.h>
 #include <crm/common/ctrl.h>
 #include <crm/common/xml.h>
 #include <crm/common/msg.h>
 
 #include <cibio.h>
 #include <callbacks.h>
 #include <cibmessages.h>
-#include <cibprimatives.h>
 #include "common.h"
 
 extern gboolean cib_is_master;
 extern const char* cib_root;
 gboolean stand_alone = FALSE;
 extern enum cib_errors cib_status;
 extern gboolean can_write(int flags);
 extern enum cib_errors cib_perform_command(
     xmlNode *request, xmlNode **reply, xmlNode **cib_diff, gboolean privileged);
 
 
 static xmlNode *
 cib_prepare_common(xmlNode *root, const char *section)
 {
     xmlNode *data = NULL;
 	
     /* extract the CIB from the fragment */
     if(root == NULL) {
 	return NULL;
 
     } else if(safe_str_eq(crm_element_name(root), XML_TAG_FRAGMENT)
 	      || safe_str_eq(crm_element_name(root), F_CIB_CALLDATA)) {
 	data = first_named_child(root, XML_TAG_CIB);
 
     } else {
 	data = root;
     }
 
     /* grab the section specified for the command */
     if(section != NULL
        && data != NULL
        && crm_str_eq(crm_element_name(data), XML_TAG_CIB, TRUE)){
 	data = get_object_root(section, data);
     }
 
     /* crm_log_xml_debug_4(root, "cib:input"); */
     return data;
 }
 
 static enum cib_errors
 cib_prepare_none(xmlNode *request, xmlNode **data, const char **section)
 {
     *data = NULL;
     *section = crm_element_value(request, F_CIB_SECTION);
     return cib_ok;
 }
 
 static enum cib_errors
 cib_prepare_data(xmlNode *request, xmlNode **data, const char **section)
 {
     xmlNode *input_fragment = get_message_xml(request, F_CIB_CALLDATA);
     *section = crm_element_value(request, F_CIB_SECTION);
     *data = cib_prepare_common(input_fragment, *section);
     /* crm_log_xml_debug(*data, "data"); */
     return cib_ok;
 }
 
 static enum cib_errors
 cib_prepare_sync(xmlNode *request, xmlNode **data, const char **section)
 {
     *data = NULL;
     *section = crm_element_value(request, F_CIB_SECTION);
     return cib_ok;
 }
 
 static enum cib_errors
 cib_prepare_diff(xmlNode *request, xmlNode **data, const char **section)
 {
     xmlNode *input_fragment = NULL;
     const char *update = crm_element_value(request, F_CIB_GLOBAL_UPDATE);
 
     *data = NULL;
     *section = NULL;
 
     if(crm_is_true(update)) {
 	input_fragment = get_message_xml(request,F_CIB_UPDATE_DIFF);
 		
     } else {
 	input_fragment = get_message_xml(request, F_CIB_CALLDATA);
     }
 
     CRM_CHECK(input_fragment != NULL,crm_log_xml(LOG_WARNING, "no input", request));
     *data = cib_prepare_common(input_fragment, NULL);
     return cib_ok;
 }
 
 static enum cib_errors
 cib_cleanup_query(const char *op, xmlNode **data, xmlNode **output) 
 {
     CRM_DEV_ASSERT(*data == NULL);
     return cib_ok;
 }
 
 static enum cib_errors
 cib_cleanup_data(const char *op, xmlNode **data, xmlNode **output) 
 {
     free_xml(*output);
     *data = NULL;
     return cib_ok;
 }
 
 static enum cib_errors
 cib_cleanup_output(const char *op, xmlNode **data, xmlNode **output) 
 {
     free_xml(*output);
     return cib_ok;
 }
 
 static enum cib_errors
 cib_cleanup_none(const char *op, xmlNode **data, xmlNode **output) 
 {
     CRM_DEV_ASSERT(*data == NULL);
     CRM_DEV_ASSERT(*output == NULL);
     return cib_ok;
 }
 
 static enum cib_errors
 cib_cleanup_sync(const char *op, xmlNode **data, xmlNode **output) 
 {
     /* data is non-NULL but doesnt need to be free'd */
     CRM_DEV_ASSERT(*data == NULL);
     CRM_DEV_ASSERT(*output == NULL);
     return cib_ok;
 }
 
 /*
   typedef struct cib_operation_s
   {
   const char* 	operation;
   gboolean	modifies_cib;
   gboolean	needs_privileges;
   gboolean	needs_quorum;
   enum cib_errors (*prepare)(xmlNode *, xmlNode**, const char **);
   enum cib_errors (*cleanup)(xmlNode**, xmlNode**);
   enum cib_errors (*fn)(
   const char *, int, const char *,
   xmlNode*, xmlNode*, xmlNode**, xmlNode**);
   } cib_operation_t;
 */
 /* technically bump does modify the cib...
  * but we want to split the "bump" from the "sync"
  */
 static cib_operation_t cib_server_ops[] = {
     {NULL,             FALSE, FALSE, FALSE, cib_prepare_none, cib_cleanup_none,   cib_process_default},
     {CIB_OP_QUERY,     FALSE, FALSE, FALSE, cib_prepare_none, cib_cleanup_query,  cib_process_query},
     {CIB_OP_MODIFY,    TRUE,  TRUE,  TRUE,  cib_prepare_data, cib_cleanup_data,   cib_process_modify},
     {CIB_OP_APPLY_DIFF,TRUE,  TRUE,  TRUE,  cib_prepare_diff, cib_cleanup_data,   cib_server_process_diff},
     {CIB_OP_REPLACE,   TRUE,  TRUE,  TRUE,  cib_prepare_data, cib_cleanup_data,   cib_process_replace_svr},
     {CIB_OP_CREATE,    TRUE,  TRUE,  TRUE,  cib_prepare_data, cib_cleanup_data,   cib_process_create},
     {CIB_OP_DELETE,    TRUE,  TRUE,  TRUE,  cib_prepare_data, cib_cleanup_data,   cib_process_delete},
     {CIB_OP_SYNC,      FALSE, TRUE,  FALSE, cib_prepare_sync, cib_cleanup_sync,   cib_process_sync},
     {CIB_OP_BUMP,      TRUE,  TRUE,  TRUE,  cib_prepare_none, cib_cleanup_output, cib_process_bump},
     {CIB_OP_ERASE,     TRUE,  TRUE,  TRUE,  cib_prepare_none, cib_cleanup_output, cib_process_erase},
     {CRM_OP_NOOP,      FALSE, FALSE, FALSE, cib_prepare_none, cib_cleanup_none,   cib_process_default},
     {CIB_OP_DELETE_ALT,TRUE,  TRUE,  TRUE,  cib_prepare_data, cib_cleanup_data,   cib_process_delete_absolute},
     {CIB_OP_UPGRADE,   TRUE,  TRUE,  TRUE,  cib_prepare_none, cib_cleanup_output, cib_process_upgrade},
     {CIB_OP_SLAVE,     FALSE, TRUE,  FALSE, cib_prepare_none, cib_cleanup_none,   cib_process_readwrite},
     {CIB_OP_SLAVEALL,  FALSE, TRUE,  FALSE, cib_prepare_none, cib_cleanup_none,   cib_process_readwrite},
     {CIB_OP_SYNC_ONE,  FALSE, TRUE,  FALSE, cib_prepare_sync, cib_cleanup_sync,   cib_process_sync_one},
     {CIB_OP_MASTER,    TRUE,  TRUE,  FALSE, cib_prepare_data, cib_cleanup_data,   cib_process_readwrite},
     {CIB_OP_ISMASTER,  FALSE, TRUE,  FALSE, cib_prepare_none, cib_cleanup_none,   cib_process_readwrite},
     {"cib_shutdown_req",FALSE, TRUE, FALSE, cib_prepare_sync, cib_cleanup_sync,   cib_process_shutdown_req},
     {CRM_OP_QUIT,      FALSE, TRUE,  FALSE, cib_prepare_none, cib_cleanup_none,   cib_process_quit},
     {CRM_OP_PING,      FALSE, FALSE, FALSE, cib_prepare_none, cib_cleanup_output, cib_process_ping},
 };
 
 enum cib_errors
 cib_get_operation_id(const char *op, int *operation) 
 {
     int lpc = 0;
     static int max_msg_types = DIMOF(cib_server_ops);
 
     if(op != NULL) {
 	for (lpc = 1; lpc < max_msg_types; lpc++) {
 	    if (strcmp(op, cib_server_ops[lpc].operation) == 0) {
 		*operation = lpc;
 		return cib_ok;
 	    }
 	}
     }
     crm_err("Operation %s is not valid", op);
     *operation = -1;
     return cib_operation;
 }
 
 xmlNode *
 cib_msg_copy(xmlNode *msg, gboolean with_data) 
 {
 	int lpc = 0;
 	const char *field = NULL;
 	const char *value = NULL;
 	xmlNode *value_struct = NULL;
 
 	static const char *field_list[] = {
 		F_XML_TAGNAME	,
 		F_TYPE		,
 		F_CIB_CLIENTID  ,
 		F_CIB_CALLOPTS  ,
 		F_CIB_CALLID    ,
 		F_CIB_OPERATION ,
 		F_CIB_ISREPLY   ,
 		F_CIB_SECTION   ,
 		F_CIB_HOST	,
 		F_CIB_RC	,
 		F_CIB_DELEGATED	,
 		F_CIB_OBJID	,
 		F_CIB_OBJTYPE	,
 		F_CIB_EXISTING	,
 		F_CIB_SEENCOUNT	,
 		F_CIB_TIMEOUT	,
 		F_CIB_CALLBACK_TOKEN	,
 		F_CIB_GLOBAL_UPDATE	,
 		F_CIB_CLIENTNAME	,
 		F_CIB_NOTIFY_TYPE	,
 		F_CIB_NOTIFY_ACTIVATE
 	};
 	
 	static const char *data_list[] = {
 		F_CIB_CALLDATA  ,
 		F_CIB_UPDATE	,
 		F_CIB_UPDATE_RESULT
 	};
 
 	xmlNode *copy = create_xml_node(NULL, "copy");
 	CRM_ASSERT(copy != NULL);
 	
 	for(lpc = 0; lpc < DIMOF(field_list); lpc++) {
 		field = field_list[lpc];
 		value = crm_element_value(msg, field);
 		if(value != NULL) {
 			crm_xml_add(copy, field, value);
 		}
 	}
 	for(lpc = 0; with_data && lpc < DIMOF(data_list); lpc++) {
 		field = data_list[lpc];
 		value_struct = get_message_xml(msg, field);
 		if(value_struct != NULL) {
 			add_message_xml(copy, field, value_struct);
 		}
 	}
 
 	return copy;
 }
 
 cib_op_t *cib_op_func(int call_type) 
 {
     return &(cib_server_ops[call_type].fn);
 }
 
 gboolean cib_op_modifies(int call_type) 
 {
     return cib_server_ops[call_type].modifies_cib;
 }
 
 int cib_op_can_run(
     int call_type, int call_options, gboolean privileged, gboolean global_update)
 {
     int rc = cib_ok;
     
     if(rc == cib_ok &&
        cib_server_ops[call_type].needs_privileges
        && privileged == FALSE) {
 	/* abort */
 	return cib_not_authorized;
     }
 #if 0
     if(rc == cib_ok
        && stand_alone == FALSE
        && global_update == FALSE
        && (call_options & cib_quorum_override) == 0
        && cib_server_ops[call_type].needs_quorum) {
 	return cib_no_quorum;
     }
 #endif
     return cib_ok;
 }
 
 
 int cib_op_prepare(
     int call_type, xmlNode *request, xmlNode **input, const char **section) 
 {
     return cib_server_ops[call_type].prepare(request, input, section);
 }
 
 int cib_op_cleanup(
     int call_type, const char *op, xmlNode **input, xmlNode **output) 
 {
     return cib_server_ops[call_type].cleanup(op, input, output);
 }
 
diff --git a/cib/io.c b/cib/io.c
index e6070d7ea2..c89c16660c 100644
--- a/cib/io.c
+++ b/cib/io.c
@@ -1,729 +1,727 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <string.h>
 #include <stdlib.h>
 
 #include <errno.h>
 #include <fcntl.h>
 
 #include <heartbeat.h>
 #include <crm/crm.h>
 
 #include <cibio.h>
 #include <crm/cib.h>
 #include <crm/common/util.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/util.h>
 #include <crm/common/cluster.h>
 #include <clplumbing/cl_misc.h>
 #include <clplumbing/lsb_exitcodes.h>
 
-#include <cibprimatives.h>
-
 #define CIB_WRITE_PARANOIA	0
 
 int archive_file(const char *oldname, const char *newname, const char *ext, gboolean preserve);
 
 const char * local_resource_path[] =
 {
 	XML_CIB_TAG_STATUS,
 };
 
 const char * resource_path[] =
 {
 	XML_CIB_TAG_RESOURCES,
 };
 
 const char * node_path[] =
 {
 	XML_CIB_TAG_NODES,
 };
 
 const char * constraint_path[] =
 {
 	XML_CIB_TAG_CONSTRAINTS,
 };
 
 gboolean initialized = FALSE;
 xmlNode *node_search = NULL;
 xmlNode *resource_search = NULL;
 xmlNode *constraint_search = NULL;
 xmlNode *status_search = NULL;
 
 extern gboolean cib_writes_enabled;
 extern GTRIGSource *cib_writer;
 extern enum cib_errors cib_status;
 
 int set_connected_peers(xmlNode *xml_obj);
 void GHFunc_count_peers(gpointer key, gpointer value, gpointer user_data);
 int write_cib_contents(gpointer p);
 extern void cib_cleanup(void);
 
 
 static gboolean
 validate_cib_digest(xmlNode *local_cib, const char *sigfile)
 {
 	int s_res = -1;
 	struct stat buf;
 	char *digest = NULL;
 	char *expected = NULL;
 	gboolean passed = FALSE;
 	FILE *expected_strm = NULL;
 	int start = 0, length = 0, read_len = 0;
 	
 	CRM_ASSERT(sigfile != NULL);
 	s_res = stat(sigfile, &buf);
 	
 	if (s_res != 0) {
 		crm_warn("No on-disk digest present");
 		return TRUE;
 	}
 
 	if(local_cib != NULL) {
 	    digest = calculate_xml_digest(local_cib, FALSE, FALSE);
 	}
 	
 	expected_strm = fopen(sigfile, "r");
 	if(expected_strm == NULL) {
 		cl_perror("Could not open signature file %s for reading", sigfile);
 		goto bail;
 	}
 
 	start  = ftell(expected_strm);
 	fseek(expected_strm, 0L, SEEK_END);
 	length = ftell(expected_strm);
 	fseek(expected_strm, 0L, start);
 	
 	CRM_ASSERT(start == ftell(expected_strm));
 
 	crm_debug_3("Reading %d bytes from file", length);
 	crm_malloc0(expected, (length+1));
 	read_len = fread(expected, 1, length, expected_strm);
 	CRM_ASSERT(read_len == length);
 	fclose(expected_strm);
 
   bail:
 	if(expected == NULL) {
 		crm_err("On-disk digest is empty");
 		
 	} else if(safe_str_eq(expected, digest)) {
 		crm_debug_2("Digest comparision passed: %s", digest);
 		passed = TRUE;
 
 	} else {
 		crm_err("Digest comparision failed: expected %s (%s), calculated %s",
 			expected, sigfile, digest);
 	}
 
  	crm_free(digest);
  	crm_free(expected);
 	return passed;
 }
 
 static int
 write_cib_digest(xmlNode *local_cib, char *digest)
 {
 	int rc = 0;
 	char *local_digest = NULL;
 	FILE *digest_strm = fopen(CIB_FILENAME ".sig", "w");
 	if(digest_strm == NULL) {
 		cl_perror("Cannot open signature file "CIB_FILENAME ".sig for writing");
 		return -1;
 	}
 
 	if(digest == NULL) {
 		local_digest = calculate_xml_digest(local_cib, FALSE, FALSE);
 		CRM_ASSERT(digest != NULL);
 		digest = local_digest;
 	}
 	
 	rc = fprintf(digest_strm, "%s", digest);
 	if(rc < 0) {
 		cl_perror("Cannot write to signature file "CIB_FILENAME ".sig");
 	}
 
 	CRM_ASSERT(digest_strm != NULL);
 	if(fflush(digest_strm) != 0) {
 	    cl_perror("fflush for %s failed:", digest);
 	    rc = -1;
 	}
 	
 	if(fsync(fileno(digest_strm)) < 0) {
 	    cl_perror("fsync for %s failed:", digest);
 	    rc = -1;
 	}
 	
 	fclose(digest_strm);
 	crm_free(local_digest);
 	return rc;
 }
 
 static gboolean
 validate_on_disk_cib(const char *filename, xmlNode **on_disk_cib)
 {
 	int s_res = -1;
 	struct stat buf;
 	gboolean passed = TRUE;
 	xmlNode *root = NULL;
 
 	CRM_ASSERT(filename != NULL);
 	
 	s_res = stat(filename, &buf);
 	if (s_res == 0) {
 		char *sigfile = NULL;
 		size_t		fnsize;
 		crm_debug_2("Reading cluster configuration from: %s", filename);
 		root = filename2xml(filename);
 		
 		fnsize =  strlen(filename) + 5;
 		crm_malloc0(sigfile, fnsize);
 		snprintf(sigfile, fnsize, "%s.sig", filename);
 		if(validate_cib_digest(root, sigfile) == FALSE) {
 			passed = FALSE;
 		}
 		crm_free(sigfile);
 	}
 	
 	if(on_disk_cib != NULL) {
 		*on_disk_cib = root;
 	} else {
 		free_xml(root);
 	}
 	
 	return passed;
 }
 
 static int
 cib_unlink(const char *file)
 {
     int rc = unlink(file);
     if (rc < 0) {
 	cl_perror("Could not unlink %s - Disabling disk writes and continuing", file);
 	cib_writes_enabled = FALSE;
     }
     return rc;
 }
 
 /*
  * It is the callers responsibility to free the output of this function
  */
 
 static xmlNode*
 retrieveCib(const char *filename, const char *sigfile, gboolean archive_invalid)
 {
     struct stat buf;
     xmlNode *root = NULL;
     crm_info("Reading cluster configuration from: %s (digest: %s)",
 	     filename, sigfile);
 
     if(stat(filename, &buf) != 0) {
 	crm_warn("Cluster configuration not found: %s", filename);
 	return NULL;
     }
 
     root = filename2xml(filename);
     
     if(root == NULL) {
 	crm_err("%s exists but does NOT contain valid XML. ", filename);
 	crm_warn("Continuing but %s will NOT used.", filename);
 	
     } else if(validate_cib_digest(root, sigfile) == FALSE) {
 	crm_err("Checksum of %s failed!  Configuration contents ignored!", filename);
 	crm_err("Usually this is caused by manual changes, "
 		"please refer to http://linux-ha.org/v2/faq/cib_changes_detected");
 	crm_warn("Continuing but %s will NOT used.", filename);
 	free_xml(root);
 	root = NULL;
 
 	if(archive_invalid) {
 	    int rc = 0;
 	    char *suffix = crm_itoa(getpid());
 	    
 	    /* Archive the original files so the contents are not lost */
 	    crm_err("Archiving corrupt or unusable configuration to %s.%s", filename, suffix);
 	    rc = archive_file(filename, NULL, suffix, TRUE);
 	    if(rc < 0) {
 		crm_err("Archival of %s failed - Disabling disk writes and continuing", filename);
 		cib_writes_enabled = FALSE;
 	    }
 
 	    rc = archive_file(sigfile, NULL, suffix, TRUE);
 	    if(rc < 0) {
 		crm_err("Archival of %s failed - Disabling disk writes and continuing", sigfile);
 		cib_writes_enabled = FALSE;
 	    }
 	    
 	    /* Unlink the original files so they dont get in the way later */
 	    cib_unlink(filename);
 	    cib_unlink(sigfile);
 	    crm_free(suffix);
 	}
     }
     return root;
 }
 
 xmlNode*
 readCibXmlFile(const char *dir, const char *file, gboolean discard_status)
 {
 	char *filename = NULL, *sigfile = NULL;
 	const char *name = NULL;
 	const char *value = NULL;
 	const char *validation = NULL;
 	const char *use_valgrind = getenv("HA_VALGRIND_ENABLED");
 	
 	xmlNode *root = NULL;
 	xmlNode *status = NULL;
 
 	if(!crm_is_writable(dir, file, HA_CCMUSER, NULL, FALSE)) {
 		cib_status = cib_bad_permissions;
 		return NULL;
 	}
 	
 	filename = crm_concat(dir, file, '/');
 	sigfile  = crm_concat(filename, "sig", '.');
 
 	cib_status = cib_ok;
 	root = retrieveCib(filename, sigfile, TRUE);
 	if(root == NULL) {
 	    char *tmp = NULL;
 	    
 	    /* Try the backups */
 	    tmp = filename;
 	    filename = crm_concat(tmp, "last", '.');
 	    crm_free(tmp);
 	    
 	    tmp = sigfile;
 	    sigfile = crm_concat(tmp, "last", '.');
 	    crm_free(tmp);
 	    
 	    crm_warn("Primary configuration corrupt or unusable, trying backup...");
 	    root = retrieveCib(filename, sigfile, FALSE);
 	}
 
 	if(root == NULL) {
 	    root = createEmptyCib();
 	    crm_xml_add(root, XML_ATTR_GENERATION, "0");
 	    crm_xml_add(root, XML_ATTR_NUMUPDATES, "0");
 	    crm_xml_add(root, XML_ATTR_GENERATION_ADMIN, "0");
 	    crm_xml_add(root, XML_ATTR_VALIDATION, LATEST_SCHEMA_VERSION);
 	    crm_warn("Continuing with an empty configuration.");
 	}	
 
 	if(cib_writes_enabled && use_valgrind) {
 	    if(crm_is_true(use_valgrind) || strstr(use_valgrind, "cib")) {
 		cib_writes_enabled = FALSE;
 		crm_err("HA_VALGRIND_ENABLED: %s",
 			getenv("HA_VALGRIND_ENABLED"));
 		crm_err("*********************************************************");
 		crm_err("*** Disabling disk writes to avoid confusing Valgrind ***");
 		crm_err("*********************************************************");
 	    }
 	}
 	
 	status = find_xml_node(root, XML_CIB_TAG_STATUS, FALSE);
 	if(discard_status && status != NULL) {
 		/* strip out the status section if there is one */
 		free_xml_from_parent(root, status);
 		status = NULL;
 	}
 	if(status == NULL) {
 		create_xml_node(root, XML_CIB_TAG_STATUS);		
 	}
 	
 	/* Do this before DTD validation happens */
 
 	/* fill in some defaults */
 	name = XML_ATTR_GENERATION_ADMIN;
 	value = crm_element_value(root, name);
 	if(value == NULL) {
 		crm_warn("No value for %s was specified in the configuration.",
 			 name);
 		crm_warn("The reccomended course of action is to shutdown,"
 			 " run crm_verify and fix any errors it reports.");
 		crm_warn("We will default to zero and continue but may get"
 			 " confused about which configuration to use if"
 			 " multiple nodes are powered up at the same time.");
 		crm_xml_add_int(root, name, 0);
 	}
 	
 	name = XML_ATTR_GENERATION;
 	value = crm_element_value(root, name);
 	if(value == NULL) {
 		crm_xml_add_int(root, name, 0);
 	}
 	
 	name = XML_ATTR_NUMUPDATES;
 	value = crm_element_value(root, name);
 	if(value == NULL) {
 		crm_xml_add_int(root, name, 0);
 	}
 	
 	/* unset these and require the DC/CCM to update as needed */
 	xml_remove_prop(root, XML_ATTR_DC_UUID);
 
 	if(discard_status) {
 		crm_log_xml_debug(root, "[on-disk]");
 	}
 
 	validation = crm_element_value(root, XML_ATTR_VALIDATION);
 	if(validate_xml(root, NULL, TRUE) == FALSE) {
 	    crm_err("CIB does not validate with %s", crm_str(validation));
 	    cib_status = cib_dtd_validation;
 		
 	} else if(validation == NULL) {
 	    int version = 0;
 	    update_validation(&root, &version, FALSE, FALSE);
 	    if(version > 0) {
 		crm_notice("Enabling %s validation on"
 			   " the existing (sane) configuration",
 			   get_schema_name(version));
 	    } else {
 		crm_err("CIB does not validate with any known DTD or schema");
 		cib_status = cib_dtd_validation;
 	    }
 	}
 
 	crm_free(filename);
 	crm_free(sigfile);
 	return root;
 }
 
 /*
  * The caller should never free the return value
  */
 xmlNode*
 get_the_CIB(void)
 {
 	return the_cib;
 }
 
 gboolean
 uninitializeCib(void)
 {
 	xmlNode *tmp_cib = the_cib;
 	
 	
 	if(tmp_cib == NULL) {
 		crm_debug("The CIB has already been deallocated.");
 		return FALSE;
 	}
 	
 	initialized = FALSE;
 	the_cib = NULL;
 	node_search = NULL;
 	resource_search = NULL;
 	constraint_search = NULL;
 	status_search = NULL;
 
 	crm_debug("Deallocating the CIB.");
 	
 	free_xml(tmp_cib);
 
 	crm_debug("The CIB has been deallocated.");
 	
 	return TRUE;
 }
 
 
 
 
 /*
  * This method will not free the old CIB pointer or the new one.
  * We rely on the caller to have saved a pointer to the old CIB
  *   and to free the old/bad one depending on what is appropriate.
  */
 gboolean
 initializeCib(xmlNode *new_cib)
 {
 	if(new_cib == NULL) {
 		return FALSE;
 	}
 	
 	the_cib = new_cib;
 	initialized = TRUE;
 	return TRUE;
 }
 
 static void
 sync_file(const char *file) 
 {
     FILE *syncme = fopen(file, "a");
     if(syncme == NULL) {
 	cl_perror("Cannot open file %s for syncing", file);
 	return;
     }
     
     if(fsync(fileno(syncme)) < 0) {
 	cl_perror("fsync for %s failed:", file);
     }
     fclose(syncme);
 }
 
 int
 archive_file(const char *oldname, const char *newname, const char *ext, gboolean preserve)
 {
 	/* move 'oldname' to 'newname' by creating a hard link to it
 	 *  and then removing the original hard link
 	 */
 	int rc = 0;
 	int res = 0;
 	struct stat tmp;
 	int s_res = 0;
 	char *backup_file = NULL;
 	static const char *back_ext = "bak";
 
 	/* calculate the backup name if required */
 	if(newname != NULL) {
 		backup_file = crm_strdup(newname);
 
 	} else {
 		int max_name_len = 1024;
 		crm_malloc0(backup_file, max_name_len);
 		if (ext == NULL) {
 			ext = back_ext;
 		}
 		snprintf(backup_file, max_name_len - 1, "%s.%s", oldname, ext);
 	}
 
 	if(backup_file == NULL || strlen(backup_file) == 0) {
 		crm_err("%s backup filename was %s",
 			newname == NULL?"calculated":"supplied",
 			backup_file == NULL?"null":"empty");
 		rc = -4;		
 	}
 	
 	s_res = stat(backup_file, &tmp);
 	
 	/* move the old backup */
 	if (rc == 0 && s_res >= 0) {
 		if(preserve == FALSE) {
 			res = unlink(backup_file);
 			if (res < 0) {
 				cl_perror("Could not unlink %s", backup_file);
 				rc = -1;
 			}
 		} else {
 			crm_info("Archive file %s exists... backing it up first", backup_file);
 			res = archive_file(backup_file, NULL, NULL, preserve);
 			if (res < 0) {
 				return res;
 			}
 		}
 	}
     
 	s_res = stat(oldname, &tmp);
 
 	/* copy */
 	if (rc == 0 && s_res >= 0) {
 		res = link(oldname, backup_file);
 		if (res < 0) {
 			cl_perror("Could not create backup %s from %s",
 				  backup_file, oldname);
 			rc = -2;
 
 		} else if(preserve) {
 			crm_info("%s archived as %s", oldname, backup_file);
 
 		} else {
 			crm_debug("%s archived as %s", oldname, backup_file);
 		}
 		sync_file(backup_file);
 	}
 	crm_free(backup_file);
 	return rc;
     
 }
 
 /*
  * This method will free the old CIB pointer on success and the new one
  * on failure.
  */
 int
 activateCibXml(xmlNode *new_cib, gboolean to_disk, const char *op)
 {
 	xmlNode *saved_cib = the_cib;
 
 	CRM_ASSERT(new_cib != saved_cib);
 	if(initializeCib(new_cib) == FALSE) {
 		free_xml(new_cib);
 		crm_err("Ignoring invalid or NULL CIB");
 
 		if(saved_cib != NULL) {
 			crm_warn("Reverting to last known CIB");
 			if (initializeCib(saved_cib) == FALSE) {
 				/* oh we are so dead  */
 				crm_crit("Couldn't re-initialize the old CIB!");
 				cl_flush_logs();
 				exit(1);
 			}
 			
 		} else {
 			crm_crit("Could not write out new CIB and no saved"
 				 " version to revert to");
 		}
 		return cib_ACTIVATION;		
 	} 
 
 	free_xml(saved_cib);
 	if(cib_writes_enabled && cib_status == cib_ok && to_disk) {
 	    crm_debug("Triggering CIB write for %s op", op);
 	    G_main_set_trigger(cib_writer);
 	}
 	
 	return cib_ok;
     
 }
 
 int
 write_cib_contents(gpointer p) 
 {
 	int rc = 0;
 	gboolean need_archive = FALSE;
 	struct stat buf;
 	char *digest = NULL;
 	int exit_rc = LSB_EXIT_OK;
 	xmlNode *cib_status_root = NULL;
 	
 	/* we can scribble on "the_cib" here and not affect the parent */
 	const char *epoch = crm_element_value(the_cib, XML_ATTR_GENERATION);
 	const char *updates = crm_element_value(the_cib, XML_ATTR_NUMUPDATES);
 	const char *admin_epoch = crm_element_value(
 		the_cib, XML_ATTR_GENERATION_ADMIN);
 
 	if(crm_log_level > LOG_INFO) {
 	    crm_log_level--;
 	}
 	
 	need_archive = (stat(CIB_FILENAME, &buf) == 0);
 	if (need_archive) {
 	    crm_debug("Archiving current version");	    
 
 	    /* check the admin didnt modify it underneath us */
 	    if(validate_on_disk_cib(CIB_FILENAME, NULL) == FALSE) {
 		crm_err("%s was manually modified while Heartbeat was active!",
 			CIB_FILENAME);
 		exit_rc = LSB_EXIT_GENERIC;
 		goto cleanup;
 	    }
 
 #if CIB_WRITE_PARANOIA
 	    /* These calls leak, but we're in a separate process that will exit
 	     * when the function does... so it's of no consequence
 	     */
 	    CRM_ASSERT(retrieveCib(CIB_FILENAME, CIB_FILENAME".sig", FALSE) != NULL);
 #endif	    
 	    rc = archive_file(CIB_FILENAME, NULL, "last", FALSE);
 	    if(rc != 0) {
 		crm_err("Could not make backup of the existing CIB: %d", rc);
 		exit_rc = LSB_EXIT_GENERIC;
 		goto cleanup;
 	    }
 	
 	    rc = archive_file(CIB_FILENAME".sig", NULL, "last", FALSE);
 	    if(rc != 0) {
 		crm_warn("Could not make backup of the existing CIB digest: %d",
 			 rc);
 	    }
 
 #if CIB_WRITE_PARANOIA
 	    CRM_ASSERT(retrieveCib(CIB_FILENAME, CIB_FILENAME".sig", FALSE) != NULL);
 	    CRM_ASSERT(retrieveCib(CIB_FILENAME".last", CIB_FILENAME".sig.last", FALSE) != NULL);
 	    crm_debug("Verified CIB archive");	    
 #endif    
 	}
 	
 	/* Given that we discard the status section on startup
 	 *   there is no point writing it out in the first place
 	 *   since users just get confused by it
 	 *
 	 * Although, it does help me once in a while
 	 *
 	 * So delete the status section before we write it out
 	 */
 	crm_debug("Writing CIB to disk");	    
 	if(p == NULL) {
 	    cib_status_root = find_xml_node(the_cib, XML_CIB_TAG_STATUS, TRUE);
 	    CRM_DEV_ASSERT(cib_status_root != NULL);
 	    
 	    if(cib_status_root != NULL) {
 		free_xml_from_parent(the_cib, cib_status_root);
 	    }
 	}
 	
 	rc = write_xml_file(the_cib, CIB_FILENAME, FALSE);
 	crm_debug("Wrote CIB to disk");
 	if(rc <= 0) {
 		crm_err("Changes couldn't be written to disk");
 		exit_rc = LSB_EXIT_GENERIC;
 		goto cleanup;
 	}
 
 	digest = calculate_xml_digest(the_cib, FALSE, FALSE);
 	crm_info("Wrote version %s.%s.%s of the CIB to disk (digest: %s)",
 		 admin_epoch?admin_epoch:"0",
 		 epoch?epoch:"0", updates?updates:"0", digest);	
 	
 	rc = write_cib_digest(the_cib, digest);
 	crm_debug("Wrote digest to disk");
 
 	if(rc <= 0) {
 		crm_err("Digest couldn't be written to disk");
 		exit_rc = LSB_EXIT_GENERIC;
 		goto cleanup;
 	}
 
 	CRM_ASSERT(retrieveCib(CIB_FILENAME, CIB_FILENAME".sig", FALSE) != NULL);
 #if CIB_WRITE_PARANOIA
 	if(need_archive) {
 	    CRM_ASSERT(retrieveCib(CIB_FILENAME".last", CIB_FILENAME".sig.last", FALSE) != NULL);
 	}
 #endif
 	crm_debug("Wrote and verified CIB");
 
   cleanup:
 	crm_free(digest);
 
 	if(p == NULL) {
 		/* fork-and-write mode */
 		exit(exit_rc);
 	}
 
 	/* stand-alone mode */
 	return exit_rc;
 }
 
 void GHFunc_count_peers(gpointer key, gpointer value, gpointer user_data)
 {
 	int *active = user_data;
 	if(safe_str_eq(value, ONLINESTATUS)) {
 		(*active)++;
 		
 	} else if(safe_str_eq(value, JOINSTATUS)) {
 		(*active)++;
 	}
 }
 
diff --git a/cib/messages.c b/cib/messages.c
index 22ddbb29f9..c050097d08 100644
--- a/cib/messages.c
+++ b/cib/messages.c
@@ -1,500 +1,466 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 
 #include <heartbeat.h>
 #include <clplumbing/cl_log.h>
 
 #include <time.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/msg.h>
 #include <crm/common/xml.h>
 #include <crm/common/cluster.h>
 
 #include <cibio.h>
 #include <cibmessages.h>
-#include <cibprimatives.h>
 #include <callbacks.h>
 
 #define MAX_DIFF_RETRY 5
 
 #ifdef CIBPIPE
 gboolean cib_is_master = TRUE;
 #else
 gboolean cib_is_master = FALSE;
 #endif
 
 xmlNode *the_cib = NULL;
 gboolean syncd_once = FALSE;
 extern const char *cib_our_uname;
 enum cib_errors revision_check(xmlNode *cib_update, xmlNode *cib_copy, int flags);
 int get_revision(xmlNode *xml_obj, int cur_revision);
 
 enum cib_errors updateList(
 	xmlNode *local_cib, xmlNode *update_command, xmlNode *failed,
 	int operation, const char *section);
 
 gboolean check_generation(xmlNode *newCib, xmlNode *oldCib);
 
 gboolean update_results(
 	xmlNode *failed, xmlNode *target, const char *operation, int return_code);
 
 enum cib_errors cib_update_counter(
 	xmlNode *xml_obj, const char *field, gboolean reset);
 
 enum cib_errors sync_our_cib(xmlNode *request, gboolean all);
 
 extern xmlNode *cib_msg_copy(const xmlNode *msg, gboolean with_data);
 extern gboolean cib_shutdown_flag;
 extern void terminate_cib(const char *caller);
 
 enum cib_errors 
 cib_process_shutdown_req(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 #ifdef CIBPIPE
     return cib_invalid_argument;
 #else
 	enum cib_errors result = cib_ok;
 	const char *host = crm_element_value(req, F_ORIG);
 	
 	*answer = NULL;
 
 	if(crm_element_value(req, F_CIB_ISREPLY) == NULL) {
 		crm_info("Shutdown REQ from %s", host);
 		return cib_ok;
 
 	} else if(cib_shutdown_flag) {
 		crm_info("Shutdown ACK from %s", host);
 		terminate_cib(__FUNCTION__);
 		return cib_ok;
 
 	} else {
 		crm_err("Shutdown ACK from %s - not shutting down",host);
 		result = cib_unknown;
 	}
 	
 	return result;
 #endif
 }
 
 enum cib_errors 
 cib_process_default(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	enum cib_errors result = cib_ok;
 	crm_debug_2("Processing \"%s\" event", op);
 	*answer = NULL;
 
 	if(op == NULL) {
 		result = cib_operation;
 		crm_err("No operation specified");
 		
 	} else if(strcasecmp(CRM_OP_NOOP, op) == 0) {
 		;
 
 	} else {
 		result = cib_NOTSUPPORTED;
 		crm_err("Action [%s] is not supported by the CIB", op);
 	}
 	return result;
 }
 
 enum cib_errors 
 cib_process_quit(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	enum cib_errors result = cib_ok;
 	crm_debug_2("Processing \"%s\" event", op);
 
 	crm_warn("The CRMd has asked us to exit... complying");
 	exit(0);
 	return result;
 }
 
 enum cib_errors 
 cib_process_readwrite(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 #ifdef CIBPIPE
     return cib_invalid_argument;
 #else
 	enum cib_errors result = cib_ok;
 	crm_debug_2("Processing \"%s\" event", op);
 
 	if(safe_str_eq(op, CIB_OP_ISMASTER)) {
 		if(cib_is_master == TRUE) {
 			result = cib_ok;
 		} else {
 			result = cib_not_master;
 		}
 		return result;
 	}
 
 	if(safe_str_eq(op, CIB_OP_MASTER)) {
 		if(cib_is_master == FALSE) {
 			crm_info("We are now in R/W mode");
 			cib_is_master = TRUE;
 			syncd_once = TRUE;
 			
 		} else {
 			crm_debug("We are still in R/W mode");
 		}
 		
 	} else if(cib_is_master) {
 		crm_info("We are now in R/O mode");
 		cib_is_master = FALSE;
 	}
 
 	return result;
 #endif
 }
 
 enum cib_errors 
 cib_process_ping(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 #ifdef CIBPIPE
     return cib_invalid_argument;
 #else
 	enum cib_errors result = cib_ok;
 	crm_debug_2("Processing \"%s\" event", op);
 	*answer = createPingAnswerFragment(CRM_SYSTEM_CIB, "ok");
 	return result;
 #endif
 }
 
 
 enum cib_errors 
 cib_process_sync(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 #ifdef CIBPIPE
     return cib_invalid_argument;
 #else
     return sync_our_cib(req, TRUE);
 #endif
 }
 
 enum cib_errors 
 cib_process_sync_one(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 #ifdef CIBPIPE
     return cib_invalid_argument;
 #else
     return sync_our_cib(req, FALSE);
 #endif
 }
 
 int sync_in_progress = 0;
 
 enum cib_errors 
 cib_server_process_diff(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	int rc = cib_ok;
 
 	if(cib_is_master) {
 		/* the master is never waiting for a resync */
 		sync_in_progress = 0;
 	}
 	
 	if(sync_in_progress > MAX_DIFF_RETRY) {
 		/* request another full-sync,
 		 * the last request may have been lost
 		 */
 		sync_in_progress = 0;
 	} 
 
 	if(sync_in_progress) {
 	    int diff_add_updates = 0;
 	    int diff_add_epoch  = 0;
 	    int diff_add_admin_epoch = 0;
 	    
 	    int diff_del_updates = 0;
 	    int diff_del_epoch  = 0;
 	    int diff_del_admin_epoch = 0;
 	    
 	    cib_diff_version_details(
 		input,
 		&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates, 
 		&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
 	    
 	    sync_in_progress++;
 	    crm_warn("Not applying diff %d.%d.%d -> %d.%d.%d (sync in progress)",
 		     diff_del_admin_epoch,diff_del_epoch,diff_del_updates,
 		     diff_add_admin_epoch,diff_add_epoch,diff_add_updates);
 	    return cib_diff_resync;
 	}
 
     
 	rc = cib_process_diff(op, options, section, req, input, existing_cib, result_cib, answer);
 	
 	if(rc == cib_diff_resync && cib_is_master == FALSE) {
 		xmlNode *sync_me = create_xml_node(NULL, "sync-me");
 		free_xml(*result_cib);
 		*result_cib = NULL;
 		crm_info("Requesting re-sync from peer");
 		sync_in_progress++;
 		
 		crm_xml_add(sync_me, F_TYPE, "cib");
 		crm_xml_add(sync_me, F_CIB_OPERATION, CIB_OP_SYNC_ONE);
 		crm_xml_add(sync_me, F_CIB_DELEGATED, cib_our_uname);
 
 		if(send_cluster_message(NULL, crm_msg_cib, sync_me, FALSE) == FALSE) {
 			rc = cib_not_connected;
 		}
 		free_xml(sync_me);
 		
 	} else if(rc == cib_diff_resync) {
 	    rc = cib_diff_failed;
 	    if(options & cib_force_diff) {
 		crm_warn("Not requesting full refresh in slave mode.");
 	    }
 	}
 	
 	return rc;
 }
 
 enum cib_errors 
 cib_process_replace_svr(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
     const char *tag = crm_element_name(input);
     enum cib_errors rc = cib_process_replace(
 	op, options, section, req, input, existing_cib, result_cib, answer);
     if(rc == cib_ok && safe_str_eq(tag, XML_TAG_CIB)) {
 	sync_in_progress = 0;
     }
     return rc;
 }
 
-enum cib_errors 
-cib_process_create(
-    const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
-    xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
+static int
+delete_cib_object(xmlNode *parent, xmlNode *delete_spec)
 {
-    xmlNode *failed = NULL;
-    enum cib_errors result = cib_ok;
-    xmlNode *update_section = NULL;
+	const char *object_name = NULL;
+	const char *object_id = NULL;
+	xmlNode *equiv_node = NULL;
+	int result = cib_ok;
 	
-    crm_debug_2("Processing \"%s\" event for section=%s", op, crm_str(section));
-    if(safe_str_eq(XML_CIB_TAG_SECTION_ALL, section)) {
-	section = NULL;
+	if(delete_spec != NULL) {
+		object_name = crm_element_name(delete_spec);
+	}
+	object_id = crm_element_value(delete_spec, XML_ATTR_ID);
 
-    } else if(safe_str_eq(XML_TAG_CIB, section)) {
-	section = NULL;
+	crm_debug_3("Processing: <%s id=%s>",
+		    crm_str(object_name), crm_str(object_id));
+	
+	if(delete_spec == NULL) {
+		result = cib_NOOBJECT;
 
-    } else if(safe_str_eq(crm_element_name(input), XML_TAG_CIB)) {
-	section = NULL;
-    }
+	} else if(parent == NULL) {
+		result = cib_NOPARENT;
 
-    CRM_CHECK(strcasecmp(CIB_OP_CREATE, op) == 0, return cib_operation);
-	
-    if(input == NULL) {
-	crm_err("Cannot perform modification with no data");
-	return cib_NOOBJECT;
-    }
-	
-    if(section == NULL) {
-	return cib_process_modify(op, options, section, req, input, existing_cib, result_cib, answer);
-    }
+	} else if(object_id == NULL) {
+		/*  placeholder object */
+		equiv_node = find_xml_node(parent, object_name, FALSE);
+		
+	} else {
+		equiv_node = find_entity(parent, object_name, object_id);
+	}
 
-    failed = create_xml_node(NULL, XML_TAG_FAILED);
+	if(result != cib_ok) {
+		; /* nothing */
+		
+	} else if(equiv_node == NULL) {
+		result = cib_ok;
 
-    update_section = get_object_root(section, *result_cib);
-    if(safe_str_eq(crm_element_name(input), section)) {
-	xml_child_iter(input, a_child, 
-		       result = add_cib_object(update_section, a_child);
-		       if(update_results(failed, a_child, op, result)) {
-			   break;
-		       }
-	    );
+	} else if(xml_has_children(delete_spec) == FALSE) {
+		/*  only leaves are deleted */
+		crm_debug("Removing leaf: <%s id=%s>",
+			  crm_str(object_name), crm_str(object_id));
+		zap_xml_from_parent(parent, equiv_node);
 
-    } else {
-	result = add_cib_object(update_section, input);
-	update_results(failed, input, op, result);
-    }
-	
-    if(xml_has_children(failed)) {
-	CRM_CHECK(result != cib_ok, result = cib_unknown);
-    }
+	} else {
 
-    if (result != cib_ok) {
-	crm_log_xml_err(failed, "CIB Update failures");
-	*answer = failed;
+		xml_child_iter(
+			delete_spec, child, 
 
-    } else {
-	free_xml(failed);
-    }
+			int tmp_result = delete_cib_object(equiv_node, child);
+			
+			/*  only the first error is likely to be interesting */
+			if(tmp_result != cib_ok && result == cib_ok) {
+				result = tmp_result;
+			}
+			);
+	}
 
-    return result;
+	return result;
 }
 
 enum cib_errors 
 cib_process_delete_absolute(
     const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
     xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
     xmlNode *failed = NULL;
     enum cib_errors result = cib_ok;
     xmlNode *update_section = NULL;
 	
     crm_debug_2("Processing \"%s\" event for section=%s", op, crm_str(section));
     if(safe_str_eq(XML_CIB_TAG_SECTION_ALL, section)) {
 	section = NULL;
 
     } else if(safe_str_eq(XML_TAG_CIB, section)) {
 	section = NULL;
 
     } else if(safe_str_eq(crm_element_name(input), XML_TAG_CIB)) {
 	section = NULL;
     }
 
     CRM_CHECK(strcasecmp(CIB_OP_DELETE, op) == 0, return cib_operation);
 	
     if(input == NULL) {
 	crm_err("Cannot perform modification with no data");
 	return cib_NOOBJECT;
     }
 	
     failed = create_xml_node(NULL, XML_TAG_FAILED);
 
     update_section = get_object_root(section, *result_cib);
     result = delete_cib_object(update_section, input);
     update_results(failed, input, op, result);
 	
     if(xml_has_children(failed)) {
 	CRM_CHECK(result != cib_ok, result = cib_unknown);
     }
 
     if (result != cib_ok) {
 	crm_log_xml_err(failed, "CIB Update failures");
 	*answer = failed;
 	    
     } else {
 	free_xml(failed);
     }
 	
     return result;
 }
 
 gboolean
 check_generation(xmlNode *newCib, xmlNode *oldCib)
 {
 	if(cib_compare_generation(newCib, oldCib) >= 0) {
 		return TRUE;
 	}
 
 	crm_warn("Generation from update is older than the existing one");
 	return FALSE;
 }
 
-gboolean
-update_results(
-	xmlNode *failed, xmlNode *target, const char* operation, int return_code)
-{
-	xmlNode *xml_node = NULL;
-	gboolean was_error = FALSE;
-	const char *error_msg = NULL;
-	
-    
-	if (return_code != cib_ok) {
-		error_msg = cib_error2string(return_code);
-
-		xml_node = create_xml_node(failed, XML_FAIL_TAG_CIB);
-
-		was_error = TRUE;
-
-		add_node_copy(xml_node, target);
-		
-		crm_xml_add(xml_node, XML_FAILCIB_ATTR_ID,      ID(target));
-		crm_xml_add(xml_node, XML_FAILCIB_ATTR_OBJTYPE, TYPE(target));
-		crm_xml_add(xml_node, XML_FAILCIB_ATTR_OP,      operation);
-		crm_xml_add(xml_node, XML_FAILCIB_ATTR_REASON,  error_msg);
-
-		crm_warn("Action %s failed: %s (cde=%d)",
-			  operation, error_msg, return_code);
-	}
-
-	return was_error;
-}
-
 #ifndef CIBPIPE
 enum cib_errors
 sync_our_cib(xmlNode *request, gboolean all) 
 {
 	enum cib_errors result      = cib_ok;
 	const char *host            = crm_element_value(request, F_ORIG);
 	const char *op              = crm_element_value(request, F_CIB_OPERATION);
 
 	xmlNode *replace_request = cib_msg_copy(request, FALSE);
 	
 	CRM_CHECK(the_cib != NULL, ;);
 	CRM_CHECK(replace_request != NULL, ;);
 	
 	crm_info("Syncing CIB to %s", all?"all peers":host);
 	if(all == FALSE && host == NULL) {
 	    crm_log_xml(LOG_ERR, "bad sync", request);
 	}
 	
 	/* remove the "all == FALSE" condition
 	 *
 	 * sync_from was failing, the local client wasnt being notified
 	 *    because it didnt know it was a reply
 	 * setting this does not prevent the other nodes from applying it
 	 *    if all == TRUE
 	 */
 	if(host != NULL) {
 		crm_xml_add(replace_request, F_CIB_ISREPLY, host);
 	}
 	crm_xml_add(replace_request, F_CIB_OPERATION, CIB_OP_REPLACE);
 	crm_xml_add(replace_request, "original_"F_CIB_OPERATION, op);
 	crm_xml_add(replace_request, F_CIB_GLOBAL_UPDATE, XML_BOOLEAN_TRUE);
 	add_message_xml(replace_request, F_CIB_CALLDATA, the_cib);
 	
 	if(send_cluster_message(all?NULL:host, crm_msg_cib, replace_request, FALSE) == FALSE) {
 		result = cib_not_connected;
 	}
 	free_xml(replace_request);
 	return result;
 }
 #endif
diff --git a/include/crm/cib_ops.h b/include/crm/cib_ops.h
index bce261d1cd..85ce444aae 100644
--- a/include/crm/cib_ops.h
+++ b/include/crm/cib_ops.h
@@ -1,88 +1,95 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 #ifndef CIB_OPS__H
 #define CIB_OPS__H
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 
 #include <time.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/msg.h>
 #include <crm/common/xml.h>
 
 enum cib_errors 
 cib_process_query(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 enum cib_errors 
 cib_process_erase(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 enum cib_errors 
 cib_process_bump(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 enum cib_errors 
 cib_process_replace(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
+enum cib_errors 
+cib_process_create(
+    const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
+    xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
+
 enum cib_errors 
 cib_process_modify(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 enum cib_errors 
 cib_process_delete(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 enum cib_errors 
 cib_process_diff(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 enum cib_errors 
 cib_process_upgrade(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 enum cib_errors 
 cib_process_xpath(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer);
 
 enum cib_errors cib_update_counter(xmlNode *xml_obj, const char *field, gboolean reset);
 xmlNode *diff_cib_object(xmlNode *old_cib, xmlNode *new_cib, gboolean suppress);
 gboolean apply_cib_diff(xmlNode *old, xmlNode *diff, xmlNode **new);
 gboolean cib_config_changed(xmlNode *old_cib, xmlNode *new_cib, xmlNode **result);
+gboolean update_results(
+    xmlNode *failed, xmlNode *target, const char* operation, int return_code);
 
 #endif
diff --git a/lib/cib/cib_file.c b/lib/cib/cib_file.c
index 303207910c..3298ef0d0d 100644
--- a/lib/cib/cib_file.c
+++ b/lib/cib/cib_file.c
@@ -1,305 +1,305 @@
 /*
  * Copyright (c) 2004 International Business Machines
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  * 
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  *
  */
 #include <crm_internal.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdarg.h>
 #include <string.h>
 
 #include <glib.h>
 #include <heartbeat.h>
 #include <clplumbing/ipc.h>
 #include <ha_msg.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/ipc.h>
 #include <cib_private.h>
 #include <clplumbing/Gmain_timeout.h>
 
 typedef struct cib_file_opaque_s 
 {
 	int flags;
 	char *filename;
 	
 } cib_file_opaque_t;
 
 int cib_file_perform_op(
     cib_t *cib, const char *op, const char *host, const char *section,
     xmlNode *data, xmlNode **output_data, int call_options);
 
 int cib_file_signon(cib_t* cib, const char *name, enum cib_conn_type type);
 int cib_file_signoff(cib_t* cib);
 int cib_file_free(cib_t* cib);
 
 static int cib_file_inputfd(cib_t* cib) { return cib_NOTSUPPORTED; }
 
 static int cib_file_set_connection_dnotify(
     cib_t *cib, void (*dnotify)(gpointer user_data))
 {
     return cib_NOTSUPPORTED;
 }
 
 
 static int cib_file_register_notification(cib_t* cib, const char *callback, int enabled) 
 {
     return cib_NOTSUPPORTED;
 }
 
 cib_t*
 cib_file_new (const char *cib_location)
 {
     cib_file_opaque_t *private = NULL;
     cib_t *cib = cib_new_variant();
 
     crm_malloc0(private, sizeof(cib_file_opaque_t));
 
     cib->variant = cib_file;
     cib->variant_opaque = private;
 
     if(cib_location == NULL) {
 	cib_location = getenv("CIB_file");
     }
     private->filename = crm_strdup(cib_location);
 
     /* assign variant specific ops*/
     cib->cmds->variant_op = cib_file_perform_op;
     cib->cmds->signon     = cib_file_signon;
     cib->cmds->signoff    = cib_file_signoff;
     cib->cmds->free       = cib_file_free;
     cib->cmds->inputfd    = cib_file_inputfd;
 
     cib->cmds->register_notification = cib_file_register_notification;
     cib->cmds->set_connection_dnotify = cib_file_set_connection_dnotify;
 
     return cib;
 }
 
 static xmlNode *in_mem_cib = NULL;
 static int load_file_cib(const char *filename) 
 {
     int rc = cib_ok;
     struct stat buf;
     xmlNode *root = NULL;
     gboolean dtd_ok = TRUE;
     const char *ignore_dtd = NULL;
     xmlNode *status = NULL;
     
     rc = stat(filename, &buf);
     if (rc == 0) {
 	root = filename2xml(filename);
 	if(root == NULL) {
 	    return cib_dtd_validation;
 	}
 
     } else {
 	return cib_NOTEXISTS;
     }
 
     rc = 0;
     
     status = find_xml_node(root, XML_CIB_TAG_STATUS, FALSE);
     if(status == NULL) {
 	create_xml_node(root, XML_CIB_TAG_STATUS);		
     }
 
     ignore_dtd = crm_element_value(root, XML_ATTR_VALIDATION);
     dtd_ok = validate_xml(root, NULL, TRUE);
     if(dtd_ok == FALSE) {
 	crm_err("CIB does not validate against %s", ignore_dtd);
 	rc = cib_dtd_validation;
 	goto bail;
     }	
     
     in_mem_cib = root;
     return rc;
 
   bail:
     free_xml(root);
     root = NULL;
     return rc;
 }
 
 int
 cib_file_signon(cib_t* cib, const char *name, enum cib_conn_type type)
 {
     int rc = cib_ok;
     cib_file_opaque_t *private = cib->variant_opaque;
 
     if(private->filename == FALSE) {
 	rc = cib_missing;
     } else {
 	rc = load_file_cib(private->filename);
     }
     
     if(rc == cib_ok) {
 	crm_debug("%s: Opened connection to local file '%s'", name, private->filename);
 	cib->state = cib_connected_command;
 	cib->type  = cib_command;
 
     } else {
 	fprintf(stderr, "%s: Connection to local file '%s' failed: %s\n",
 		name, private->filename, cib_error2string(rc));
     }
     
     return rc;
 }
 	
 int
 cib_file_signoff(cib_t* cib)
 {
     int rc = cib_ok;
     cib_file_opaque_t *private = cib->variant_opaque;
 
     crm_debug("Signing out of the CIB Service");
     
     if(strstr(private->filename, ".bz2") != NULL) {
 	rc = write_xml_file(in_mem_cib, private->filename, TRUE);
 	
     } else {
 	rc = write_xml_file(in_mem_cib, private->filename, FALSE);
     }
 
     if(rc > 0) {
 	crm_info("Wrote CIB to %s", private->filename);
     } else {
 	crm_err("Could not write CIB to %s", private->filename);
     }
     crm_free(in_mem_cib);
     
     cib->state = cib_disconnected;
     cib->type  = cib_none;
     
     return rc;
 }
 
 int
 cib_file_free (cib_t* cib)
 {
     int rc = cib_ok;
 
     crm_warn("Freeing CIB");
     if(cib->state != cib_disconnected) {
 	rc = cib_file_signoff(cib);
 	if(rc == cib_ok) {
 	    cib_file_opaque_t *private = cib->variant_opaque;
 	    crm_free(private->filename);
 	    crm_free(cib->cmds);
 	    crm_free(private);
 	    crm_free(cib);
 	}
     }
 	
     return rc;
 }
 
 struct cib_func_entry 
 {
 	const char *op;
 	gboolean    read_only;
 	cib_op_t    fn;
 };
 
 static struct cib_func_entry cib_file_ops[] = {
     {CIB_OP_QUERY,      TRUE,  cib_process_query},
     {CIB_OP_MODIFY,     FALSE, cib_process_modify},
     {CIB_OP_APPLY_DIFF, FALSE, cib_process_diff},
     {CIB_OP_BUMP,       FALSE, cib_process_bump},
     {CIB_OP_REPLACE,    FALSE, cib_process_replace},
-    /* {CIB_OP_CREATE,     FALSE, cib_process_create}, */
+    {CIB_OP_CREATE,     FALSE, cib_process_create},
     {CIB_OP_DELETE,     FALSE, cib_process_delete},
     {CIB_OP_ERASE,      FALSE, cib_process_erase},
     {CIB_OP_UPGRADE,    FALSE, cib_process_upgrade},
 };
 
 
 int
 cib_file_perform_op(
     cib_t *cib, const char *op, const char *host, const char *section,
     xmlNode *data, xmlNode **output_data, int call_options) 
 {
     int rc = cib_ok;
     gboolean query = FALSE;
     gboolean changed = FALSE;
     xmlNode *output = NULL;
     xmlNode *cib_diff = NULL;
     xmlNode *result_cib = NULL;
     cib_op_t *fn = NULL;
     int lpc = 0;
     static int max_msg_types = DIMOF(cib_file_ops);
 
     crm_info("%s on %s", op, section);
     
     if(cib->state == cib_disconnected) {
 	return cib_not_connected;
     }
 
     if(output_data != NULL) {
 	*output_data = NULL;
     }
 	
     if(op == NULL) {
 	return cib_operation;
     }
 
     for (lpc = 0; lpc < max_msg_types; lpc++) {
 	if (safe_str_eq(op, cib_file_ops[lpc].op)) {
 	    fn = &(cib_file_ops[lpc].fn);
 	    query = cib_file_ops[lpc].read_only;
 	    break;
 	}
     }
     
     if(fn == NULL) {
 	return cib_NOTSUPPORTED;
     }
 
     cib->call_id++;
     rc = cib_perform_op(op, call_options, fn, query,
     			section, NULL, data, TRUE, &changed, in_mem_cib, &result_cib, &cib_diff, &output);
     
     if(rc != cib_ok) {
 	free_xml(result_cib);
 	    
     } else if(query == FALSE) {
 	log_xml_diff(LOG_INFO, cib_diff, "cib:diff");	
 	free_xml(in_mem_cib);
 	in_mem_cib = result_cib;
     }
 
     free_xml(cib_diff);
 
     if(cib->op_callback != NULL) {
 	cib->op_callback(NULL, cib->call_id, rc, output);
     }
 	
     if(output_data && output) {
 	*output_data = copy_xml(output);
     }
 
     if(query == FALSE) {
 	free_xml(output);
     }
     
     return rc;
 }
 
 
diff --git a/lib/cib/cib_ops.c b/lib/cib/cib_ops.c
index ec05d89b37..014ac3d43e 100644
--- a/lib/cib/cib_ops.c
+++ b/lib/cib/cib_ops.c
@@ -1,795 +1,1031 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 
 #include <heartbeat.h>
 #include <clplumbing/cl_log.h>
 
 #include <time.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/msg.h>
 #include <crm/common/xml.h>
 
 enum cib_errors 
 cib_process_query(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	xmlNode *obj_root = NULL;
 	enum cib_errors result = cib_ok;
 	
 	crm_debug_2("Processing \"%s\" event for section=%s",
 		  op, crm_str(section));
 
 	if(options & cib_xpath) {
 	    return cib_process_xpath(op, options, section, req, input,
 				     existing_cib, result_cib, answer);
 	}
 
 	CRM_CHECK(*answer == NULL, free_xml(*answer));
 	*answer = NULL;
 	
 	if (safe_str_eq(XML_CIB_TAG_SECTION_ALL, section)) {
 		section = NULL;
 	}
 
 	obj_root = get_object_root(section, existing_cib);
 	
 	if(obj_root == NULL) {
 		result = cib_NOTEXISTS;
 
 	} else {
 		*answer = obj_root;
 	}
 
 	if(result == cib_ok && *answer == NULL) {
 		crm_err("Error creating query response");
 		result = cib_output_data;
 	}
 	
 	return result;
 }
 
 enum cib_errors 
 cib_process_erase(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	enum cib_errors result = cib_ok;
 
 	crm_debug_2("Processing \"%s\" event", op);
 	*answer = NULL;
 	free_xml(*result_cib);
 	*result_cib = createEmptyCib();
 
 	copy_in_properties(*result_cib, existing_cib);	
 	cib_update_counter(*result_cib, XML_ATTR_GENERATION, FALSE);
 	
 	return result;
 }
 
 enum cib_errors 
 cib_process_upgrade(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
     int rc = 0;
     int new_version = 0;
     int current_version = 0;
 
     const char *value = NULL;
 
     *answer = NULL;
     crm_debug_2("Processing \"%s\" event", op);
     
     value = crm_element_value_copy(existing_cib, XML_ATTR_VALIDATION);
     if(value != NULL) {
 	current_version = get_schema_version(value);
     }
 
     rc = update_validation(result_cib, &new_version, TRUE, TRUE);
     if(new_version > current_version) {
 	return cib_ok;
     }
     
     return rc;
 }
 
 enum cib_errors 
 cib_process_bump(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	enum cib_errors result = cib_ok;
 
 	crm_debug_2("Processing \"%s\" event for epoch=%s",
 		  op, crm_str(crm_element_value(existing_cib, XML_ATTR_GENERATION)));
 	
 	*answer = NULL;
 	cib_update_counter(*result_cib, XML_ATTR_GENERATION, FALSE);
 	
 	return result;
 }
 
 
 enum cib_errors 
 cib_update_counter(xmlNode *xml_obj, const char *field, gboolean reset)
 {
 	char *new_value = NULL;
 	char *old_value = NULL;
 	int  int_value  = -1;
 	
 	if(reset == FALSE && crm_element_value(xml_obj, field) != NULL) {
 		old_value = crm_element_value_copy(xml_obj, field);
 	}
 	if(old_value != NULL) {
 		crm_malloc0(new_value, 128);
 		int_value = atoi(old_value);
 		sprintf(new_value, "%d", ++int_value);
 	} else {
 		new_value = crm_strdup("1");
 	}
 
 	crm_debug_4("%s %d(%s)->%s",
 		  field, int_value, crm_str(old_value), crm_str(new_value));
 	crm_xml_add(xml_obj, field, new_value);
 
 	crm_free(new_value);
 	crm_free(old_value);
 
 	return cib_ok;
 }
 
 enum cib_errors 
 cib_process_replace(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	const char *tag = NULL;
 	gboolean verbose       = FALSE;
 	enum cib_errors result = cib_ok;
 	
 	crm_debug_2("Processing \"%s\" event for section=%s",
 		    op, crm_str(section));
 
 	if(options & cib_xpath) {
 	    return cib_process_xpath(op, options, section, req, input,
 				     existing_cib, result_cib, answer);
 	}
 
 	*answer = NULL;
 
 	if (input == NULL) {
 		return cib_NOOBJECT;
 	}
 
 	tag = crm_element_name(input);
 
 	if (options & cib_verbose) {
 		verbose = TRUE;
 	}
 	if(safe_str_eq(XML_CIB_TAG_SECTION_ALL, section)) {
 		section = NULL;
 
 	} else if(safe_str_eq(tag, section)) {
 		section = NULL;
 	}
 	
 	if(safe_str_eq(tag, XML_TAG_CIB)) {
 		int updates = 0;
 		int epoch  = 0;
 		int admin_epoch = 0;
 		
 		int replace_updates = 0;
 		int replace_epoch  = 0;
 		int replace_admin_epoch = 0;
 		const char *reason = NULL;
 		
 		cib_version_details(
 			existing_cib, &admin_epoch, &epoch, &updates);
 		cib_version_details(input, &replace_admin_epoch,
 				    &replace_epoch, &replace_updates);
 
 		if(replace_admin_epoch < admin_epoch) {
 			reason = XML_ATTR_GENERATION_ADMIN;
 
 		} else if(replace_admin_epoch > admin_epoch) {
 			/* no more checks */
 
 		} else if(replace_epoch < epoch) {
 			reason = XML_ATTR_GENERATION;
 
 		} else if(replace_epoch > epoch) {
 			/* no more checks */
 
 		} else if(replace_updates < updates) {
 			reason = XML_ATTR_NUMUPDATES;
 		}
 
 		if(reason != NULL) {
 			crm_warn("Replacement %d.%d.%d not applied to %d.%d.%d:"
 				 " current %s is greater than the replacement",
 				 replace_admin_epoch, replace_epoch,
 				 replace_updates, admin_epoch, epoch, updates,
 				 reason);
 			result = cib_old_data;
 		}
 
 		free_xml(*result_cib);
 		*result_cib = copy_xml(input);
 		
 	} else {
 		xmlNode *obj_root = NULL;
 		gboolean ok = TRUE;
 		obj_root = get_object_root(section, *result_cib);
 		ok = replace_xml_child(NULL, obj_root, input, FALSE);
 		if(ok == FALSE) {
 			crm_debug_2("No matching object to replace");
 			result = cib_NOTEXISTS;
 		}
 	}
 
 	return result;
 }
 
 enum cib_errors 
 cib_process_delete(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	xmlNode *obj_root = NULL;
 	crm_debug_2("Processing \"%s\" event", op);
 
 	if(options & cib_xpath) {
 	    return cib_process_xpath(op, options, section, req, input,
 				     existing_cib, result_cib, answer);
 	}
 
 	if(input == NULL) {
 		crm_err("Cannot perform modification with no data");
 		return cib_NOOBJECT;
 	}
 	
 	obj_root = get_object_root(section, *result_cib);
 	
 	crm_validate_data(input);
 	crm_validate_data(*result_cib);
 
 	if(replace_xml_child(NULL, obj_root, input, TRUE) == FALSE) {
 		crm_debug_2("No matching object to delete");
 	}
 	
 	return cib_ok;
 }
 
 enum cib_errors 
 cib_process_modify(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	xmlNode *obj_root = NULL;
 	crm_debug_2("Processing \"%s\" event", op);
 
 	if(options & cib_xpath) {
 	    return cib_process_xpath(op, options, section, req, input,
 				     existing_cib, result_cib, answer);
 	}
 
 	if(input == NULL) {
 		crm_err("Cannot perform modification with no data");
 		return cib_NOOBJECT;
 	}
 	
 	obj_root = get_object_root(section, *result_cib);
 	
 	crm_validate_data(input);
 	crm_validate_data(*result_cib);
 
 	if(obj_root == NULL) {
 	    xmlNode *tmp_section = NULL;
 	    const char *path = get_object_parent(section);
 	    if(path == NULL) {
 		return cib_bad_section;		
 	    }
 
 	    tmp_section = create_xml_node(NULL, section);
 	    cib_process_xpath(
 		CIB_OP_CREATE, 0, path, NULL, tmp_section, NULL, result_cib, answer);
 	    free_xml(tmp_section);
 	    
 	    obj_root = get_object_root(section, *result_cib);
 	}
 
 	CRM_CHECK(obj_root != NULL, return cib_unknown);
 	
 	if(update_xml_child(obj_root, input) == FALSE) {
 	    if(options & cib_can_create) {
 		add_node_copy(obj_root, input);
 	    } else {
 		return cib_NOTEXISTS;		
 	    }
 	}
 	
 	return cib_ok;
 }
 
+static int
+update_cib_object(xmlNode *parent, xmlNode *update)
+{
+	const char *replace = NULL;
+	const char *object_name = NULL;
+	const char *object_id = NULL;
+	xmlNode *target = NULL;
+	int result = cib_ok;
+
+	CRM_CHECK(update != NULL, return cib_NOOBJECT);
+	CRM_CHECK(parent != NULL, return cib_NOPARENT);
+
+	object_name = crm_element_name(update);
+	CRM_CHECK(object_name != NULL, return cib_NOOBJECT);
+
+	object_id = ID(update);
+	crm_debug_3("Processing: <%s id=%s>",
+		    crm_str(object_name), crm_str(object_id));
+	
+	if(object_id == NULL) {
+		/*  placeholder object */
+		target = find_xml_node(parent, object_name, FALSE);
+
+	} else {
+		target = find_entity(parent, object_name, object_id);
+	}
+
+	if(target == NULL) {
+		target = create_xml_node(parent, object_name);
+	} 
+
+	crm_debug_2("Found node <%s id=%s> to update",
+		    crm_str(object_name), crm_str(object_id));
+	
+	replace = crm_element_value(update, XML_CIB_ATTR_REPLACE);
+	if(replace != NULL) {
+	    xmlNode *remove = NULL;
+	    int last = 0, lpc = 0, len = 0;
+
+	    len = strlen(replace);
+	    while(lpc <= len) {
+		if(replace[lpc] == ',' || replace[lpc] == 0) {
+		    char *replace_item = NULL;
+		    if ( last == lpc ) {
+			/* nothing to do */
+			last = lpc+1;
+			goto incr;
+		    }
+
+		    crm_malloc0(replace_item, lpc - last + 1);
+		    strncpy(replace_item, replace+last, lpc-last);
+		    
+		    remove = find_xml_node(target, replace_item, FALSE);
+		    if(remove != NULL) {
+			crm_debug_3("Replacing node <%s> in <%s>",
+				    replace_item, crm_element_name(target));
+			zap_xml_from_parent(target, remove);
+		    }
+		    crm_free(replace_item);
+		    last = lpc+1;
+		}
+	      incr:
+		lpc++;
+	    }
+	    xml_remove_prop(update, XML_CIB_ATTR_REPLACE);
+	    xml_remove_prop(target, XML_CIB_ATTR_REPLACE);
+	}
+	
+	copy_in_properties(target, update);
+
+	crm_debug_3("Processing children of <%s id=%s>",
+		    crm_str(object_name), crm_str(object_id));
+	
+	xml_child_iter(
+		update, a_child,  
+		int tmp_result = 0;
+		crm_debug_3("Updating child <%s id=%s>",
+			    crm_element_name(a_child), ID(a_child));
+		
+		tmp_result = update_cib_object(target, a_child);
+		
+		/*  only the first error is likely to be interesting */
+		if(tmp_result != cib_ok) {
+			crm_err("Error updating child <%s id=%s>",
+				crm_element_name(a_child), ID(a_child));
+			
+			if(result == cib_ok) {
+				result = tmp_result;
+			}
+		}
+		);
+	
+	crm_debug_3("Finished with <%s id=%s>",
+		  crm_str(object_name), crm_str(object_id));
+
+	return result;
+}
+
+static int
+add_cib_object(xmlNode *parent, xmlNode *new_obj)
+{
+	enum cib_errors result = cib_ok;
+	const char *object_name = NULL;
+	const char *object_id = NULL;
+	xmlNode *equiv_node = NULL;
+	
+	if(new_obj != NULL) {
+		object_name = crm_element_name(new_obj);
+	}
+	object_id = crm_element_value(new_obj, XML_ATTR_ID);
+
+	crm_debug_3("Processing: <%s id=%s>",
+		    crm_str(object_name), crm_str(object_id));
+	
+	if(new_obj == NULL || object_name == NULL) {
+		result = cib_NOOBJECT;
+
+	} else if(parent == NULL) {
+		result = cib_NOPARENT;
+
+	} else if(object_id == NULL) {
+		/*  placeholder object */
+		equiv_node = find_xml_node(parent, object_name, FALSE);
+		
+	} else {
+		equiv_node = find_entity(parent, object_name, object_id);
+	}
+
+	if(result != cib_ok) {
+		; /* do nothing */
+		
+	} else if(equiv_node != NULL) {
+		result = cib_EXISTS;
+
+	} else {
+		result = update_cib_object(parent, new_obj);
+	}
+
+	return result;
+}
+
+
+
+enum cib_errors 
+cib_process_create(
+    const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
+    xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
+{
+    xmlNode *failed = NULL;
+    enum cib_errors result = cib_ok;
+    xmlNode *update_section = NULL;
+	
+    crm_debug_2("Processing \"%s\" event for section=%s", op, crm_str(section));
+    if(safe_str_eq(XML_CIB_TAG_SECTION_ALL, section)) {
+	section = NULL;
+
+    } else if(safe_str_eq(XML_TAG_CIB, section)) {
+	section = NULL;
+
+    } else if(safe_str_eq(crm_element_name(input), XML_TAG_CIB)) {
+	section = NULL;
+    }
+
+    CRM_CHECK(strcasecmp(CIB_OP_CREATE, op) == 0, return cib_operation);
+	
+    if(input == NULL) {
+	crm_err("Cannot perform modification with no data");
+	return cib_NOOBJECT;
+    }
+	
+    if(section == NULL) {
+	return cib_process_modify(op, options, section, req, input, existing_cib, result_cib, answer);
+    }
+
+    failed = create_xml_node(NULL, XML_TAG_FAILED);
+
+    update_section = get_object_root(section, *result_cib);
+    if(safe_str_eq(crm_element_name(input), section)) {
+	xml_child_iter(input, a_child, 
+		       result = add_cib_object(update_section, a_child);
+		       if(update_results(failed, a_child, op, result)) {
+			   break;
+		       }
+	    );
+
+    } else {
+	result = add_cib_object(update_section, input);
+	update_results(failed, input, op, result);
+    }
+	
+    if(xml_has_children(failed)) {
+	CRM_CHECK(result != cib_ok, result = cib_unknown);
+    }
+
+    if (result != cib_ok) {
+	crm_log_xml_err(failed, "CIB Update failures");
+	*answer = failed;
+
+    } else {
+	free_xml(failed);
+    }
+
+    return result;
+}
+
 enum cib_errors 
 cib_process_diff(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	unsigned int log_level = LOG_DEBUG;
 	const char *reason = NULL;
 	gboolean apply_diff = TRUE;
 	enum cib_errors result = cib_ok;
 
 	int this_updates = 0;
 	int this_epoch  = 0;
 	int this_admin_epoch = 0;
 
 	int diff_add_updates = 0;
 	int diff_add_epoch  = 0;
 	int diff_add_admin_epoch = 0;
 
 	int diff_del_updates = 0;
 	int diff_del_epoch  = 0;
 	int diff_del_admin_epoch = 0;
 
 	crm_debug_2("Processing \"%s\" event", op);
 
 	cib_diff_version_details(
 		input,
 		&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates, 
 		&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
 
 	crm_element_value_int(existing_cib, XML_ATTR_GENERATION, &this_epoch);
 	crm_element_value_int(existing_cib, XML_ATTR_NUMUPDATES, &this_updates);
 	crm_element_value_int(existing_cib, XML_ATTR_GENERATION_ADMIN, &this_admin_epoch);
 
 	if(this_epoch < 0) { this_epoch = 0; }
 	if(this_updates < 0) { this_updates = 0; }
 	if(this_admin_epoch < 0) { this_admin_epoch = 0; }
 
 	if(diff_del_admin_epoch == diff_add_admin_epoch
 	   && diff_del_epoch == diff_add_epoch
 	   && diff_del_updates == diff_add_updates) {
 		if(options & cib_force_diff) {
 			apply_diff = FALSE;
 			log_level = LOG_ERR;
 			reason = "+ and - versions in the diff did not change in global update";
 			crm_log_xml_warn(input, "Bad global update");
 			
 		} else if(diff_add_admin_epoch == -1 && diff_add_epoch == -1 && diff_add_updates == -1) {
 			crm_err("Massaging diff versions");
 			diff_add_epoch = this_epoch;
 			diff_add_updates = this_updates + 1;
 			diff_add_admin_epoch = this_admin_epoch;
 			diff_del_epoch = this_epoch;
 			diff_del_updates = this_updates;
 			diff_del_admin_epoch = this_admin_epoch;
 			crm_log_xml_err(input, __FUNCTION__);
 
 		} else {
 			apply_diff = FALSE;
 			log_level = LOG_ERR;
 			reason = "+ and - versions in the diff did not change";
 			log_cib_diff(LOG_ERR, input, __FUNCTION__);
 		}
 	}
 
 	if(apply_diff && diff_del_admin_epoch > this_admin_epoch) {
 		result = cib_diff_resync;
 		apply_diff = FALSE;
 		log_level = LOG_INFO;
 		reason = "current \""XML_ATTR_GENERATION_ADMIN"\" is less than required";
 		
 	} else if(apply_diff && diff_del_admin_epoch < this_admin_epoch) {
 		apply_diff = FALSE;
 		log_level = LOG_WARNING;
 		reason = "current \""XML_ATTR_GENERATION_ADMIN"\" is greater than required";
 
 	} else if(apply_diff && diff_del_epoch > this_epoch) {
 		result = cib_diff_resync;
 		apply_diff = FALSE;
 		log_level = LOG_INFO;
 		reason = "current \""XML_ATTR_GENERATION"\" is less than required";
 		
 	} else if(apply_diff && diff_del_epoch < this_epoch) {
 		apply_diff = FALSE;
 		log_level = LOG_WARNING;
 		reason = "current \""XML_ATTR_GENERATION"\" is greater than required";
 
 	} else if(apply_diff && diff_del_updates > this_updates) {
 		result = cib_diff_resync;
 		apply_diff = FALSE;
 		log_level = LOG_INFO;
 		reason = "current \""XML_ATTR_NUMUPDATES"\" is less than required";
 		
 	} else if(apply_diff && diff_del_updates < this_updates) {
 		apply_diff = FALSE;
 		log_level = LOG_WARNING;
 		reason = "current \""XML_ATTR_NUMUPDATES"\" is greater than required";
 	}
 
 	if(apply_diff) {
 		free_xml(*result_cib);
 		*result_cib = NULL;
 		if(apply_xml_diff(existing_cib, input, result_cib) == FALSE) {
 		    log_level = LOG_NOTICE;
 		    reason = "Failed application of an update diff";
 		    
 		    if(options & cib_force_diff) {
 			result = cib_diff_resync;
 		    }
 		}
 	}
 	
 	if(reason != NULL) {
 		do_crm_log(
 			log_level,
 			"Diff %d.%d.%d -> %d.%d.%d not applied to %d.%d.%d: %s",
 			diff_del_admin_epoch,diff_del_epoch,diff_del_updates,
 			diff_add_admin_epoch,diff_add_epoch,diff_add_updates,
 			this_admin_epoch,this_epoch,this_updates, reason);
 
 		if(result == cib_ok) {
 		    result = cib_diff_failed;
 		}
 		
 	} else if(apply_diff) {
 		crm_debug_2("Diff %d.%d.%d -> %d.%d.%d was applied to %d.%d.%d",
 			    diff_del_admin_epoch,diff_del_epoch,diff_del_updates,
 			    diff_add_admin_epoch,diff_add_epoch,diff_add_updates,
 			    this_admin_epoch,this_epoch,this_updates);
 	    
 	}
 	return result;
 }
 
 gboolean
 apply_cib_diff(xmlNode *old, xmlNode *diff, xmlNode **new)
 {
 	gboolean result = TRUE;
 	const char *value = NULL;
 
 	int this_updates = 0;
 	int this_epoch  = 0;
 	int this_admin_epoch = 0;
 
 	int diff_add_updates = 0;
 	int diff_add_epoch  = 0;
 	int diff_add_admin_epoch = 0;
 
 	int diff_del_updates = 0;
 	int diff_del_epoch  = 0;
 	int diff_del_admin_epoch = 0;
 
 	CRM_CHECK(diff != NULL, return FALSE);
 	CRM_CHECK(old != NULL, return FALSE);
 	
 	value = crm_element_value(old, XML_ATTR_GENERATION_ADMIN);
 	this_admin_epoch = crm_parse_int(value, "0");
 	crm_debug_3("%s=%d (%s)", XML_ATTR_GENERATION_ADMIN,
 		  this_admin_epoch, value);
 	
 	value = crm_element_value(old, XML_ATTR_GENERATION);
 	this_epoch = crm_parse_int(value, "0");
 	crm_debug_3("%s=%d (%s)", XML_ATTR_GENERATION, this_epoch, value);
 	
 	value = crm_element_value(old, XML_ATTR_NUMUPDATES);
 	this_updates = crm_parse_int(value, "0");
 	crm_debug_3("%s=%d (%s)", XML_ATTR_NUMUPDATES, this_updates, value);
 	
 	cib_diff_version_details(
 		diff,
 		&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates, 
 		&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
 
 	value = NULL;
 	if(result && diff_del_admin_epoch != this_admin_epoch) {
 		value = XML_ATTR_GENERATION_ADMIN;
 		result = FALSE;
 		crm_debug_3("%s=%d", value, diff_del_admin_epoch);
 
 	} else if(result && diff_del_epoch != this_epoch) {
 		value = XML_ATTR_GENERATION;
 		result = FALSE;
 		crm_debug_3("%s=%d", value, diff_del_epoch);
 
 	} else if(result && diff_del_updates != this_updates) {
 		value = XML_ATTR_NUMUPDATES;
 		result = FALSE;
 		crm_debug_3("%s=%d", value, diff_del_updates);
 	}
 
 	if(result) {
 		xmlNode *tmp = NULL;
 		xmlNode *diff_copy = copy_xml(diff);
 		
 		tmp = find_xml_node(diff_copy, "diff-removed", TRUE);
 		if(tmp != NULL) {
 			xml_remove_prop(tmp, XML_ATTR_GENERATION_ADMIN);
 			xml_remove_prop(tmp, XML_ATTR_GENERATION);
 			xml_remove_prop(tmp, XML_ATTR_NUMUPDATES);
 		}
 		
 		tmp = find_xml_node(diff_copy, "diff-added", TRUE);
 		if(tmp != NULL) {
 			xml_remove_prop(tmp, XML_ATTR_GENERATION_ADMIN);
 			xml_remove_prop(tmp, XML_ATTR_GENERATION);
 			xml_remove_prop(tmp, XML_ATTR_NUMUPDATES);
 		}
 		
 		result = apply_xml_diff(old, diff_copy, new);
 		free_xml(diff_copy);
 		
 	} else {
 		crm_err("target and diff %s values didnt match", value);
 	}
 	
 	
 	return result;
 }
 
 gboolean
 cib_config_changed(xmlNode *old_cib, xmlNode *new_cib, xmlNode **result)
 {
 	gboolean config_changes = FALSE;
 	const char *tag = NULL;
 	xmlNode *diff = NULL;
 	xmlNode *dest = NULL;
 
 	if(result) {
 		*result = NULL;
 	}
 
 	diff = diff_xml_object(old_cib, new_cib, FALSE);
 	if(diff == NULL) {
 		return FALSE;
 	}
 
 	tag = "diff-removed";
 	dest = find_xml_node(diff, tag, FALSE);
 	if(dest) {
 		dest = find_xml_node(dest, XML_TAG_CIB, FALSE);
 	}
 
 	if(dest) {
 		xml_child_iter(dest, child,
 			       const char *tag = crm_element_name(child);
 			       if(crm_str_eq(tag, XML_CIB_TAG_STATUS, TRUE)) {
 				   continue;
 			       }
 			       config_changes = TRUE;
 			       goto done;
 		    );
 	}
 
 	tag = "diff-added";
 	dest = find_xml_node(diff, tag, FALSE);
 	if(dest) {
 		dest = find_xml_node(dest, XML_TAG_CIB, FALSE);
 	}
 
 	if(dest) {
 		xml_child_iter(dest, child,
 			       const char *tag = crm_element_name(child);
 			       if(crm_str_eq(tag, XML_CIB_TAG_STATUS, TRUE)) {
 				   continue;
 			       }
 			       config_changes = TRUE;
 			       goto done;
 		    );
 
 		xml_prop_iter(dest, name, value,
 			      if(crm_str_eq(name, XML_ATTR_NUMUPDATES, TRUE)) {
 				  continue;
 			      }
 			      config_changes = TRUE;
 			      goto done;
 		    );
 
 	}
 
   done:
 	if(result) {
 		*result = diff;
 	} else {
 		free_xml(diff);
 	}
 	
 	return config_changes;
 }
 
 xmlNode *
 diff_cib_object(xmlNode *old_cib, xmlNode *new_cib, gboolean suppress)
 {
 	xmlNode *dest = NULL;
 	xmlNode *src = NULL;
 	const char *name = NULL;
 	const char *value = NULL;
 
 	xmlNode *diff = diff_xml_object(old_cib, new_cib, suppress);
 	
 	/* add complete version information */
 	src = old_cib;
 	dest = find_xml_node(diff, "diff-removed", FALSE);
 	if(src != NULL && dest != NULL) {
 		name = XML_ATTR_GENERATION_ADMIN;
 		value = crm_element_value(src, name);
 		if(value == NULL) {
 			value = "0";
 		}
 		crm_xml_add(dest, name, value);
 
 		name = XML_ATTR_GENERATION;
 		value = crm_element_value(src, name);
 		if(value == NULL) {
 			value = "0";
 		}
 		crm_xml_add(dest, name, value);
 
 		name = XML_ATTR_NUMUPDATES;
 		value = crm_element_value(src, name);
 		if(value == NULL) {
 			value = "0";
 		}
 		crm_xml_add(dest, name, value);
 	}
 	
 	src = new_cib;
 	dest = find_xml_node(diff, "diff-added", FALSE);
 	if(src != NULL && dest != NULL) {
 		name = XML_ATTR_GENERATION_ADMIN;
 		value = crm_element_value(src, name);
 		if(value == NULL) {
 			value = "0";
 		}
 		crm_xml_add(dest, name, value);
 
 		name = XML_ATTR_GENERATION;
 		value = crm_element_value(src, name);
 		if(value == NULL) {
 			value = "0";
 		}
 		crm_xml_add(dest, name, value);
 
 		name = XML_ATTR_NUMUPDATES;
 		value = crm_element_value(src, name);
 		if(value == NULL) {
 			value = "0";
 		}
 		crm_xml_add(dest, name, value);
 	}
 	return diff;
 }
 
 enum cib_errors 
 cib_process_xpath(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
     int lpc = 0;
     int max = 0;
     int rc = cib_ok;
     xmlXPathObjectPtr xpathObj = NULL;
     crm_debug_2("Processing \"%s\" event", op);
 
     if(safe_str_eq(op, CIB_OP_QUERY)) {
 	xpathObj = xpath_search(existing_cib, section);
     } else {
 	xpathObj = xpath_search(*result_cib, section);
     }
     
     if(xpathObj != NULL && xpathObj->nodesetval != NULL) {
 	max = xpathObj->nodesetval->nodeNr;
     }
 
     if(max < 1 && safe_str_eq(op, CIB_OP_DELETE)) {
 	crm_debug("%s was already removed", section);
 
     } else if(max < 1) {
 	crm_debug("%s: %s does not exist", op, section);
 	rc = cib_NOTEXISTS;
 
     } else if(safe_str_eq(op, CIB_OP_QUERY)) {
 	if(max > 1) {
 	    *answer = create_xml_node(NULL, "xpath-query");
 	}
     }
 
     for(lpc = 0; lpc < max; lpc++) {
 	xmlNode *match = xpathObj->nodesetval->nodeTab[lpc];
 	CRM_CHECK(match != NULL, goto out);
 
 	if(match->type == XML_DOCUMENT_NODE) {
 	    /* Will happen if section = '/' */
 	    match = match->children;
 	}
 
 	crm_info("Processing %s op for %s (%s)", op, section, xmlGetNodePath(match));
 	CRM_CHECK(match->type == XML_ELEMENT_NODE,
 		  crm_info("Wrong node type: %d", match->type);
 		  continue);
 
 	if(safe_str_eq(op, CIB_OP_DELETE)) {
 	    free_xml_from_parent(NULL, match);
 	    break;
 
 	} else if(safe_str_eq(op, CIB_OP_MODIFY)) {
 	    if(update_xml_child(match, input) == FALSE) {
 		rc = cib_NOTEXISTS;		
 	    } else {
 		rc = cib_ok;
 		if((options & cib_multiple) == 0) {
 		    break;
 		}
 	    }
 	    
 	} else if(safe_str_eq(op, CIB_OP_CREATE)) {
 	    add_node_copy(match, input);
 	    break;
 
 	} else if(safe_str_eq(op, CIB_OP_QUERY)) {
 
 	    if(options & cib_no_children) {
 		const char *tag = TYPE(match);
 		*answer = create_xml_node(NULL, tag);
 		copy_in_properties(*answer, match);
 		break;
 		
 	    } else if(*answer) {
 		add_node_copy(*answer, match);
 
 	    } else {
 		*answer = match;
 	    }
 	    
 	} else if(safe_str_eq(op, CIB_OP_REPLACE)) {
 	    xmlNode *parent = match->parent;
 	    free_xml_from_parent(NULL, match);
 	    if(input != NULL) {
 		add_node_copy(parent, input);
 	    }
 	    
 	    if((options & cib_multiple) == 0) {
 		break;
 	    }
 	}
     }
 	
   out:
     if(xpathObj) {
 	xmlXPathFreeObject(xpathObj);
     }
 	    
     return rc;
 }
+
+/* remove this function */
+gboolean
+update_results(
+	xmlNode *failed, xmlNode *target, const char* operation, int return_code)
+{
+	xmlNode *xml_node = NULL;
+	gboolean was_error = FALSE;
+	const char *error_msg = NULL;
+	
+    
+	if (return_code != cib_ok) {
+		error_msg = cib_error2string(return_code);
+
+		xml_node = create_xml_node(failed, XML_FAIL_TAG_CIB);
+
+		was_error = TRUE;
+
+		add_node_copy(xml_node, target);
+		
+		crm_xml_add(xml_node, XML_FAILCIB_ATTR_ID,      ID(target));
+		crm_xml_add(xml_node, XML_FAILCIB_ATTR_OBJTYPE, TYPE(target));
+		crm_xml_add(xml_node, XML_FAILCIB_ATTR_OP,      operation);
+		crm_xml_add(xml_node, XML_FAILCIB_ATTR_REASON,  error_msg);
+
+		crm_warn("Action %s failed: %s (cde=%d)",
+			  operation, error_msg, return_code);
+	}
+
+	return was_error;
+}