diff --git a/crm/admin/cibadmin.c b/crm/admin/cibadmin.c
index 420cbe4278..62856aebd6 100644
--- a/crm/admin/cibadmin.c
+++ b/crm/admin/cibadmin.c
@@ -1,570 +1,570 @@
 
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <lha_internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 
 #include <hb_api.h>
 #include <clplumbing/uids.h>
 #include <clplumbing/Gmain_timeout.h>
 
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/ctrl.h>
 #include <crm/common/ipc.h>
 
 #include <crm/cib.h>
 
 #ifdef HAVE_GETOPT_H
 #  include <getopt.h>
 #endif
 #include <ha_msg.h> /* someone complaining about _ha_msg_mod not being found */
 
 int exit_code = cib_ok;
 int message_timer_id = -1;
 int message_timeout_ms = 30*1000;
 
 GMainLoop *mainloop = NULL;
 const char *crm_system_name = "cibadmin";
 IPC_Channel *crmd_channel = NULL;
 
 const char *host = NULL;
 void usage(const char *cmd, int exit_status);
 enum cib_errors do_init(void);
 int do_work(crm_data_t *input, int command_options, crm_data_t **output);
 
 gboolean admin_msg_callback(IPC_Channel * source_data, void *private_data);
 gboolean admin_message_timeout(gpointer data);
 void cib_connection_destroy(gpointer user_data);
 void cibadmin_op_callback(const HA_Message *msg, int call_id, int rc,
 			  crm_data_t *output, void *user_data);
 
 int command_options = 0;
 const char *cib_action = NULL;
 
 typedef struct str_list_s
 {
 		int num_items;
 		char *value;
 		struct str_list_s *next;
 } str_list_t;
 
 char *this_msg_reference = NULL;
 char *obj_type = NULL;
 char *status = NULL;
 char *migrate_from = NULL;
 char *migrate_res = NULL;
 char *subtype = NULL;
 char *reset = NULL;
 
 int request_id = 0;
 int operation_status = 0;
 cib_t *the_cib = NULL;
 
 #define OPTARGS	"V?o:QDUCEX:t:Srwlsh:MmBfbdRx:pP"
 
 int
 main(int argc, char **argv)
 {
 	int argerr = 0;
 	int flag;
 	char *admin_input_xml = NULL;
 	char *admin_input_file = NULL;
 	gboolean admin_input_stdin = FALSE;
 	crm_data_t *output = NULL;
 	crm_data_t *input = NULL;
 	
 #ifdef HAVE_GETOPT_H
 	int option_index = 0;
 	static struct option long_options[] = {
 		/* Top-level Options */
 		{CIB_OP_ERASE,   0, 0, 'E'},
 		{CIB_OP_QUERY,   0, 0, 'Q'},
 		{CIB_OP_CREATE,  0, 0, 'C'},
 		{CIB_OP_REPLACE, 0, 0, 'R'},
 		{CIB_OP_UPDATE,  0, 0, 'U'},
 		{CIB_OP_MODIFY,  0, 0, 'M'},
 		{"patch",	 0, 0, 'P'},
 		{CIB_OP_DELETE,  0, 0, 'D'},
 		{CIB_OP_DELETE_ALT,  0, 0, 'd'},
 		{CIB_OP_BUMP,    0, 0, 'B'},
 		{CIB_OP_SYNC,    0, 0, 'S'},
 		{CIB_OP_SLAVE,   0, 0, 'r'},
 		{CIB_OP_MASTER,  0, 0, 'w'},
 		{CIB_OP_ISMASTER,0, 0, 'm'},
 		
 		{"force-quorum",0, 0, 'f'},
 		{"local",	0, 0, 'l'},
 		{"sync-call",	0, 0, 's'},
 		{"no-bcast",	0, 0, 'b'},
 		{"host",	0, 0, 'h'},
 		{F_CRM_DATA,    1, 0, 'X'},
 		{"xml-file",    1, 0, 'x'},
 		{"xml-pipe",    0, 0, 'p'},
 		{"verbose",     0, 0, 'V'},
 		{"help",        0, 0, '?'},
 		{"reference",   1, 0, 0},
 		{"timeout",	1, 0, 't'},
 
 		/* common options */
 		{"obj_type", 1, 0, 'o'},
 
 		{0, 0, 0, 0}
 	};
 #endif
 
 	cl_log_set_entity("cibadmin");
 	cl_log_set_facility(LOG_USER);
-	set_crm_log_level(LOG_CRIT-1);
+	set_crm_log_level(LOG_CRIT);
 	
 	if(argc < 2) {
 		usage(crm_system_name, LSB_EXIT_EINVAL);
 	}
 
 	while (1) {
 #ifdef HAVE_GETOPT_H
 		flag = getopt_long(argc, argv, OPTARGS,
 				   long_options, &option_index);
 #else
 		flag = getopt(argc, argv, OPTARGS);
 #endif
 		if (flag == -1)
 			break;
 
 		switch(flag) {
 #ifdef HAVE_GETOPT_H
 			case 0:
 				printf("option %s",
 				       long_options[option_index].name);
 				if (optarg)
 					printf(" with arg %s", optarg);
 				printf("\n");
 	if (safe_str_eq("reference", long_options[option_index].name)) {
 		this_msg_reference = crm_strdup(optarg);
 
 	} else {
 		printf("Long option (--%s) is not (yet?) properly supported\n",
 		       long_options[option_index].name);
 		++argerr;
 	}
 	break;
 #endif
 			case 't':
 				message_timeout_ms = atoi(optarg);
 				if(message_timeout_ms < 1) {
 					message_timeout_ms = 30*1000;
 				}
 				break;
 				
 			case 'E':
 				cib_action = CIB_OP_ERASE;
 				break;
 			case 'Q':
 				cib_action = CIB_OP_QUERY;
 				break;
 			case 'P':
 				cib_action = CIB_OP_APPLY_DIFF;
 				break;
 			case 'S':
 				cib_action = CIB_OP_SYNC;
 				break;
 			case 'U':
 			case 'M':
 				cib_action = CIB_OP_MODIFY;
 				break;
 			case 'R':
 				cib_action = CIB_OP_REPLACE;
 				break;
 			case 'C':
 				cib_action = CIB_OP_CREATE;
 				break;
 			case 'D':
 				cib_action = CIB_OP_DELETE;
 				break;
 			case 'd':
 				cib_action = CIB_OP_DELETE_ALT;
 				break;
 			case 'm':
 				cib_action = CIB_OP_ISMASTER;
 				command_options |= cib_scope_local;
 				break;
 			case 'B':
 				cib_action = CIB_OP_BUMP;
 				break;
 			case 'r':
 				cib_action = CIB_OP_SLAVE;
 				break;
 			case 'w':
 				cib_action = CIB_OP_MASTER;
 				command_options |= cib_scope_local;
 				break;
 			case 'V':
 				command_options = command_options | cib_verbose;
 				cl_log_enable_stderr(TRUE);
 				alter_debug(DEBUG_INC);
 				break;
 			case '?':
 				usage(crm_system_name, LSB_EXIT_OK);
 				break;
 			case 'o':
 				crm_debug_2("Option %c => %s", flag, optarg);
 				obj_type = crm_strdup(optarg);
 				break;
 			case 'X':
 				crm_debug_2("Option %c => %s", flag, optarg);
 				admin_input_xml = crm_strdup(optarg);
 				break;
 			case 'x':
 				crm_debug_2("Option %c => %s", flag, optarg);
 				admin_input_file = crm_strdup(optarg);
 				break;
 			case 'p':
 				admin_input_stdin = TRUE;
 				break;
 			case 'h':
 				host = crm_strdup(optarg);
 				break;
 			case 'l':
 				command_options |= cib_scope_local;
 				break;
 			case 'b':
 				command_options |= cib_inhibit_bcast;
 				command_options |= cib_scope_local;
 				break;
 			case 's':
 				command_options |= cib_sync_call;
 				break;
 			case 'f':
 				command_options |= cib_quorum_override;
 				break;
 			default:
 				printf("Argument code 0%o (%c)"
 				       " is not (?yet?) supported\n",
 				       flag, flag);
 				++argerr;
 				break;
 		}
 	}
 
 	if (optind < argc) {
 		printf("non-option ARGV-elements: ");
 		while (optind < argc)
 			printf("%s ", argv[optind++]);
 		printf("\n");
 	}
 
 	if (optind > argc) {
 		++argerr;
 	}
 
 	if(cib_action == NULL) {
 		usage(crm_system_name, cib_operation);
 	}
 	
 	if (argerr) {
 		usage(crm_system_name, LSB_EXIT_GENERIC);
 	}
 	
 	if(admin_input_file != NULL) {
 		FILE *xml_strm = fopen(admin_input_file, "r");
 		input = file2xml(xml_strm, FALSE);
 		if(input == NULL) {
 			fprintf(stderr, "Couldn't parse input file: %s\n", admin_input_file);
 			return 1;
 		}
 		fclose(xml_strm);
 		
 	} else if(admin_input_xml != NULL) {
 		input = string2xml(admin_input_xml);
 		if(input == NULL) {
 			fprintf(stderr, "Couldn't parse input string: %s\n", admin_input_xml);
 			return 1;
 		}
 
 	} else if(admin_input_stdin) {
 		input = stdin2xml();
 		if(input == NULL) {
 			fprintf(stderr, "Couldn't parse input from STDIN.\n");
 			return 1;
 		}
 	}
 	
 	if(input != NULL) {
 		crm_log_xml_debug(input, "[admin input]");
 	}
 	
 	exit_code = do_init();
 	if(exit_code != cib_ok) {
 		crm_err("Init failed, could not perform requested operations");
 		fprintf(stderr, "Init failed, could not perform requested operations\n");
 		return -exit_code;
 	}	
 
 	cl_log_args(argc, argv);
 	
 	exit_code = do_work(input, command_options, &output);
 	if (exit_code > 0) {
 		/* wait for the reply by creating a mainloop and running it until
 		 * the callbacks are invoked...
 		 */
 		request_id = exit_code;
 
 		add_cib_op_callback(
 			request_id, FALSE, NULL, cibadmin_op_callback);
 
 		mainloop = g_main_new(FALSE);
 
 		crm_debug("Setting operation timeout to %dms for call %d",
 			  message_timeout_ms, request_id);
 
 		message_timer_id = Gmain_timeout_add(
 			message_timeout_ms, admin_message_timeout, NULL);
 
 		crm_debug_3("%s waiting for reply from the local CIB",
 			 crm_system_name);
 		
 		crm_info("Starting mainloop");
 		g_main_run(mainloop);
 		
 	} else if(exit_code < 0) {
 		crm_err("Call failed: %s", cib_error2string(exit_code));
 		fprintf(stderr, "Call failed: %s\n",
 			cib_error2string(exit_code));
 		operation_status = exit_code;
 	}
 
 
 	if(output != NULL) {
 		char *buffer = dump_xml_formatted(output);
 		fprintf(stdout, "%s", crm_str(buffer));
 		crm_free(buffer);
 	}
 	
 	crm_debug_3("%s exiting normally", crm_system_name);
 	return -exit_code;
 }
 
 int
 do_work(crm_data_t *input, int call_options, crm_data_t **output) 
 {
 	/* construct the request */
 	if (strcasecmp(CIB_OP_SYNC, cib_action) == 0) {
 		crm_debug_4("Performing %s op...", cib_action);
 		return the_cib->cmds->sync_from(
 			the_cib, host, obj_type, call_options);
 
 	} else if (strcasecmp(CIB_OP_SLAVE, cib_action) == 0
 		   && (call_options ^ cib_scope_local) ) {
 		crm_debug_4("Performing %s op on all nodes...", cib_action);
 		return the_cib->cmds->set_slave_all(the_cib, call_options);
 
 	} else if (strcasecmp(CIB_OP_MASTER, cib_action) == 0) {
 		crm_debug_4("Performing %s op on all nodes...", cib_action);
 		return the_cib->cmds->set_master(the_cib, call_options);
 
 
 	} else if(cib_action != NULL) {
 		crm_debug_4("Passing \"%s\" to variant_op...", cib_action);
 		if(strcasecmp(CIB_OP_APPLY_DIFF, cib_action) != 0 
 		   && input != NULL
 		   && do_id_check(input, NULL, TRUE, FALSE)) {
 			crm_err("ID Check failed.");
 			return cib_id_check;
 		}
 			
 		return the_cib->cmds->variant_op(
 			the_cib, cib_action, host, obj_type,
 			input, output, call_options);
 		
 	} else {
 		crm_err("You must specify an operation");
 	}
 	return cib_operation;
 }
 
 enum cib_errors
 do_init(void)
 {
 	enum cib_errors rc = cib_ok;
 	
 	the_cib = cib_new();
 	rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command);
 	if(rc != cib_ok) {
 		crm_err("Signon to CIB failed: %s",
 			cib_error2string(rc));
 		fprintf(stderr, "Signon to CIB failed: %s\n",
 			cib_error2string(rc));
 	}
 	
 	return rc;
 }
 
 
 void
 usage(const char *cmd, int exit_status)
 {
 	FILE *stream;
 
 	stream = exit_status != 0 ? stderr : stdout;
 
 	fprintf(stream, "usage: %s [%s] command\n"
 		"\twhere necessary, XML data will be obtained using -X,"
 		" -x, or -p options\n", cmd, OPTARGS);
 
 	fprintf(stream, "Options\n");
 	fprintf(stream, "\t--%s (-%c) <type>\tobject type being operated on\n",
 		"obj_type", 'o');
 	fprintf(stream, "\t\tValid values are: nodes, resources, constraints, crm_config, status\n");
 	fprintf(stream, "\t--%s (-%c)\tturn on debug info."
 		"  additional instance increase verbosity\n", "verbose", 'V');
 	fprintf(stream, "\t--%s (-%c)\tthis help message\n", "help", '?');
 	fprintf(stream, "\nCommands\n");
 	fprintf(stream, "\t--%s (-%c)\tErase the contents of the whole CIB\n", CIB_OP_ERASE,  'E');
 	fprintf(stream, "\t--%s (-%c)\t\n", CIB_OP_QUERY,  'Q');
 	fprintf(stream, "\t--%s (-%c)\t\n", CIB_OP_CREATE, 'C');
 	fprintf(stream, "\t--%s (-%c)\tRecursivly replace an object in the CIB\n", CIB_OP_REPLACE,'R');
 	fprintf(stream, "\t--%s (-%c)\tRecursivly update an object in the CIB\n", CIB_OP_UPDATE, 'U');
 	fprintf(stream, "\t--%s (-%c)\tFind the object somewhere in the CIB's XML tree and update is as --"CIB_OP_UPDATE" would\n", CIB_OP_MODIFY, 'M');
 	fprintf(stream, "\t--%s (-%c)\t\n", CIB_OP_DELETE, 'D');
 	fprintf(stream, "\t\t\tDelete the first object matching the supplied criteria\n");
 	fprintf(stream, "\t\t\tEg. <op id=\"rsc1_op1\" name=\"monitor\"/>\n");
 	fprintf(stream, "\t\t\tThe tagname and all attributes must match in order for the element to be deleted\n");
 	
 	fprintf(stream, "\t--%s (-%c)\t\n", CIB_OP_DELETE_ALT, 'd');
 	fprintf(stream, "\t\t\tDelete the object at specified fully qualified location\n");
 	fprintf(stream, "\t\t\tEg. <resource id=\"rsc1\"><operations><op id=\"rsc1_op1\"/>...\n");
 	fprintf(stream, "\t\t\tRequires -o\n");
 
 	fprintf(stream, "\t--%s (-%c)\t\n", CIB_OP_BUMP,   'B');
 	fprintf(stream, "\t--%s (-%c)\t\n", CIB_OP_ISMASTER,'m');
 	fprintf(stream, "\t--%s (-%c)\t\n", CIB_OP_SYNC,   'S');
 	fprintf(stream, "\nXML data\n");
 	fprintf(stream, "\t--%s (-%c) <string>\t\tRetrieve XML from the supplied string\n", F_CRM_DATA, 'X');
 	fprintf(stream, "\t--%s (-%c) <filename>\tRetrieve XML from the named file\n", "xml-file", 'x');
 	fprintf(stream, "\t--%s (-%c)\t\t\tRetrieve XML from STDIN\n", "xml-pipe", 'p');
 	fprintf(stream, "\nAdvanced Options\n");
 	fprintf(stream, "\t--%s (-%c)\tsend command to specified host."
 		" Applies to %s and %s commands only\n", "host", 'h',
 		CIB_OP_QUERY, CIB_OP_SYNC);
 	fprintf(stream, "\t--%s (-%c)\tcommand takes effect locally"
 		" on the specified host\n", "local", 'l');
 	fprintf(stream, "\t--%s (-%c)\tcommand will not be broadcast even if"
 		" it altered the CIB\n", "no-bcast", 'b');
 	fprintf(stream, "\t--%s (-%c)\twait for call to complete before"
 		" returning\n", "sync-call", 's');
 
 	fflush(stream);
 
 	exit(exit_status);
 }
 
 
 gboolean
 admin_message_timeout(gpointer data)
 {
 	if(safe_str_eq(cib_action, CIB_OP_SLAVE)) {
 		exit_code = cib_ok;
 		fprintf(stdout, "CIB service(s) are in slave mode.\n");
 		
 	} else {
 		exit_code = cib_reply_failed;
 		fprintf(stderr,
 			"No messages received in %d seconds.. aborting\n",
 			(int)message_timeout_ms/1000);
 		crm_err("No messages received in %d seconds",
 			(int)message_timeout_ms/1000);
 	}
 	
 	g_main_quit(mainloop);
 	return FALSE;
 }
 
 
 void
 cib_connection_destroy(gpointer user_data)
 {
 	crm_err("Connection to the CIB terminated... exiting");
 	g_main_quit(mainloop);
 	return;
 }
 
 void
 cibadmin_op_callback(const HA_Message *msg, int call_id, int rc,
 		     crm_data_t *output, void *user_data)
 {
 	char *admin_input_xml = NULL;
 	
 	crm_info("our callback was invoked");
 	crm_log_message(LOG_MSG, msg);
 	exit_code = rc;
 
 	if(output != NULL) {
 		admin_input_xml = dump_xml_formatted(output);
 	}
 	
 	if(safe_str_eq(cib_action, CIB_OP_ISMASTER) && rc != cib_ok) {
 		crm_info("CIB on %s is _not_ the master instance",
 			 host?host:"localhost");
 		fprintf(stderr, "CIB on %s is _not_ the master instance\n",
 			 host?host:"localhost");
 		
 	} else if(safe_str_eq(cib_action, CIB_OP_ISMASTER)) {
 		crm_info("CIB on %s _is_ the master instance",
 			 host?host:"localhost");
 		fprintf(stderr, "CIB on %s _is_ the master instance\n",
 			 host?host:"localhost");
 		
 	} else if(rc != 0) {
 		crm_warn("Call %s failed (%d): %s",
 			cib_action, rc, cib_error2string(rc));
 		fprintf(stderr, "Call %s failed (%d): %s\n",
 			cib_action, rc, cib_error2string(rc));
 		fprintf(stdout, "%s\n",	crm_str(admin_input_xml));
 
 	} else if(safe_str_eq(cib_action, CIB_OP_QUERY) && output==NULL) {
 		crm_err("Output expected in query response");
 		crm_log_message(LOG_ERR, msg);
 
 	} else if(output == NULL) {
 		crm_info("Call passed");
 
 	} else {
 		crm_info("Call passed");
 		fprintf(stdout, "%s\n", crm_str(admin_input_xml));
 	}
 	crm_free(admin_input_xml);
 
 	if(call_id == request_id) {
 		g_main_quit(mainloop);
 
 	} else {
 		crm_info("Message was not the response we were looking for (%d vs. %d", call_id, request_id);
 	}
 }
diff --git a/crm/crm-1.0.dtd b/crm/crm-1.0.dtd
index bf3b7d92ba..52d27b4a5f 100644
--- a/crm/crm-1.0.dtd
+++ b/crm/crm-1.0.dtd
@@ -1,840 +1,845 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <!--
 GLOBAL TODOs:
 
 Versionize DTD so we can validate against a specific version
 
 Background
  The CIB is described quite well in section 5 of the crm.txt (checked into CVS in the crm directory) so it is not repeated here.
  Suffice to say that it stores the configuration and runtime data required for cluster-wide resource management in XML format.
 
 CIB: Information Structure
  The CIB is divided into two main sections: The "static" configuration part and the "dynamic" status.
 
  The configuration contains - surprisingly - the configuration of the cluster, namely node attributes, resource instance configuration, and the constraints which describe the dependencies between all these.
  To identify the most recent configuration available in the cluster, this section is time-stamped with the unique timestamp of the last update.
 
  The status part is dynamically generated / updated by the CRM system and represents the current status of the cluster; which nodes are up, down or crashed, which resources are running where etc.
 
  Every information carrying object has an "id" tag, which is basically the UUID of it, should we ever need to access it directly.
  Unless otherwise stated, the id field is a short name consisting simple ascii characters [a-zA-Z0-9_\-]
  The exception is for resources because the LRM can support only id's of up to 64 characters.
 
 Other Notes
  The description field in all elements is opaque to the CRM and is for administrative comments.
 
 TODO
  * Figure out a sane way to version the DTD
  * Do we need to know about ping nodes...?
  * The integer comparison type really should be number
 -->
 <!ELEMENT cib (configuration, status)>
 <!ATTLIST cib
           cib-last-written CDATA        #IMPLIED
 
           admin_epoch  CDATA        #REQUIRED
           epoch        CDATA        #REQUIRED
           num_updates  CDATA        #REQUIRED
           num_peers    CDATA        #IMPLIED        
 
           cib_feature_revision  CDATA   #IMPLIED
           crm_feature_set       CDATA   #IMPLIED
           remote_access_port    CDATA   #IMPLIED        
 
           dc_uuid        CDATA             #IMPLIED
           ccm_transition CDATA             #IMPLIED
           have_quorum    (true|1|false|0)  'false'
           ignore_dtd     (true|1|false|0)  #IMPLIED
 
           generated    CDATA    #IMPLIED        
           crm-debug-origin CDATA    #IMPLIED>
 
 <!--
 The CIB's version is a tuple of admin_epoch, epoch and num_updates (in that order).
 
 This is used when applying updates from the master CIB instance.
 
 Additionally, num_peers and have_quorum are used during the election process to determin who has the latest configuration.
  * num_updates is incremented every time the CIB changes.
  * epoch is incremented after every DC election.
  * admin_epoch is exclusivly for the admin to change.
  * num_peers is the number of CIB instances that we can talk to
  * have_quorum is derived from the ConsensusClusterMembership layer
  * dc_uuid stored the UUID of the current DesignatedController
  * ccm_transition stores the membership instance from the ConsensusClusterMembership layer.
  * cib_feature_revision is the feature set that this configuration requires
 -->
 <!ELEMENT configuration (crm_config, nodes, resources, constraints)>
 
 <!--
 crm_config
 
 Used to specify cluster-wide options.
 
 The use of multiple cluster_property_set sections and time-based rule expressions allows the the cluster to behave differently (for example) during buisness hours than it does overnight.
 -->
 <!ELEMENT crm_config (cluster_property_set)*>
 
 <!--
 Current crm_config options:
 
  * transition_idle_timeout (interval, default=60s):
    If no activity is recorded in this time, the transition is deemed failed as are all sent actions that have not yet been confirmed complete.
    If any operation initiated has an explicit higher timeout, the higher value applies.
 
  * symmetric_cluster (boolean, default=TRUE):
    If true, resources are permitted to run anywhere by default.
    Otherwise, explicit constraints must be created to specify where they can run.
 
  * stonith_enabled (boolean, default=FALSE):
    If true, failed nodes will be fenced.
 
  * no_quorum_policy (enum, default=stop)
    * ignore - Pretend we have quorum
    * freeze - Do not start any resources not currently in our partition.
      Resources in our partition may be moved to another node within the partition
      Fencing is disabled
    * stop - Stop all running resources in our partition
      Fencing is disabled
 
  * default_resource_stickiness
    Do we prefer to run on the existing node or be moved to a "better" one?
    * 0 : resources will be placed optimally in the system.
      This may mean they are moved when a "better" or less loaded node becomes available.
      This option is almost equivalent to auto_failback on except that the resource may be moved to other nodes than the one it was previously active on.
    * value > 0 : resources will prefer to remain in their current location but may be moved if a more suitable node is available.
      Higher values indicate a stronger preference for resources to stay where they are.
    * value < 0 : resources prefer to move away from their current location.
      Higher absolute values indicate a stronger preference for resources to be moved.
    * INFINITY : resources will always remain in their current locations until forced off because the node is no longer eligible to run the resource (node shutdown, node standby or configuration change).
      This option is almost equivalent to auto_failback off except that the resource may be moved to other nodes than the one it was previously active on.
    * -INFINITY : resources will always move away from their current location.
 
  * is_managed_default (boolean, default=TRUE)
    Unless the resource's definition says otherwise,
    * TRUE : resources will be started, stopped, monitored and moved as necessary/required
    * FALSE : resources will not started if stopped, stopped if started nor have any recurring actions scheduled.
 
  * stop_orphan_resources (boolean, default=TRUE (as of release 2.0.6))
    If a resource is found for which we have no definition for;
    * TRUE : Stop the resource
    * FALSE : Ignore the resource
    This mostly effects the CRM's behavior when a resource is deleted by an admin without it first being stopped.
 
  * stop_orphan_actions (boolean, default=TRUE)
    If a recurring action is found for which we have no definition for;
    * TRUE : Stop the action
    * FALSE : Ignore the action
    This mostly effects the CRM's behavior when the interval for a recurring action is changed.
 -->
 <!ELEMENT cluster_property_set (rule*, attributes)>
 <!ATTLIST cluster_property_set
           id                CDATA        #REQUIRED
           score             CDATA        #IMPLIED>
 
 <!ELEMENT nodes       (node*)>
 
 <!--
  * id    : the node's UUID.
  * uname : the result of uname -n
  * type  : should either be "normal" or "member" for nodes you with to run resources 
    "normal" is preferred as of version 2.0.4
 
 Each node can also have additional "instance" attributes.
 These attributes are completely arbitrary and can be used later in constraints.
 In this way it is possible to define groups of nodes to which a constraint can apply.
 
 It is also theoretically possible to have a process on each node which updates these values automatically.
 This would make it possible to have an attribute that represents "connected to SAN subsystem" or perhaps "system_load (low|medium|high)".
 
 Ideally it would be possible to have the CRMd on each node gather some of this information and automatically populate things like architecture and OS/kernel version.
 -->
 <!ELEMENT node (instance_attributes*)>
 <!ATTLIST node
           id            CDATA         #REQUIRED
           uname         CDATA         #REQUIRED
           description   CDATA         #IMPLIED
           type          (normal|member|ping) #REQUIRED>
 
 <!ELEMENT resources   (primitive|group|clone|master_slave)*>
 
 <!--
  * class
    Specifies the location and standard the resource script conforms to
    * ocf
      Most OCF RAs started out life as v1 Heartbeat resource agents.
      These have all been ported to meet the OCF specifications.
      As an added advantage, in accordance with the OCF spec, they also describe the parameters they take and what their defaults are.
      It is also easier to configure them as each part of the configuration is passed as its own parameter.
      In accordance with the OCF spec, each parameter is passed to the RA with an OCF_RESKEY_ prefix.
      So ip=192.168.1.1 in the CIB would be passed as OCF_RESKEY_ip=192.168.1.1.
      Located under /usr/lib/ocf/resource.d/heartbeat/.
    * lsb
      Most Linux init scripts conform to the LSB specification.
      The class allows you to use those that do as resource agents controlled by Heartbeat.
      Located in /etc/init.d/.
    * heartbeat
      This class gives you access to the v1 Heartbeat resource agents and allows you to reuse any custom agents you may have written.
      Located at /etc/heartbeat/resource.d/ or /etc/ha.d/resource.d.
 
  * type : The name of the ResourceAgent you wish to use.
 
  * provider
    The OCF spec allows multiple vendors to supply the same ResourceAgent.
    To use the OCF resource agents supplied with Heartbeat, you should specify heartbeat here
 
  * is_managed : Is the ClusterResourceManager in control of this resource.
    * true : (default) the resource will be started, stopped, monitored and moved as necessary/required
    * false : the resource will not started if stopped, stopped if started nor have any recurring actions scheduled.
      The resource may still be referenced in colocation constraints and ordering constraints (though obviously if no actions are performed on it then it will prevent the action on the other resource too)
 
  * restart_type
    Used when the other side of an ordering dependency is restarted/moved.
    * ignore : the default.
      Don't do anything extra.
    * restart
      Use this for example to have a restart of your database also trigger a restart of your web-server.
    * multiple_active
      Used when a resource is detected as being active on more than one machine.
      The default value, stop_start, will stop all instances and start only 1
    * block : don't do anything, wait for the administrator
    * stop_only : stop all the active instances
    * stop_start : start the resource on one node after having stopped all the active instances
 
  * resource_stickiness
    See the description of the default_resource_stickiness cluster attribute.
    resource_stickiness allows you to override the cluster's default for the individual resource.
 
 NOTE: primitive resources may contain at most one "operations" object.
       The CRM will complain about your configuration if this criteria is not met.
       Please use crm_verify to ensure your configuration is valid.
       The DTD is written this way to be order in-sensitive.
 -->
 <!ELEMENT primitive (operations|meta_attributes|instance_attributes)*>
 <!ATTLIST primitive
           id                CDATA        #REQUIRED
           description       CDATA        #IMPLIED
           class             (ocf|lsb|heartbeat|stonith) #REQUIRED
           type              CDATA        #REQUIRED
           provider          CDATA        #IMPLIED
 
           is_managed            CDATA                            #IMPLIED
           restart_type          (ignore|restart)                 'ignore'
           multiple_active       (stop_start|stop_only|block)     'stop_start'
           resource_stickiness   CDATA                             #IMPLIED>
 <!--
 This allows us to specify how long an action can take
 
  * name : the name of the operation.
    Supported operations are start, stop, & monitor
 
  * start_delay : delay the operation after starting the resource
    By default this value is in milliseconds, however you can also specify a value in seconds like so timeout="5s". Used for the monitor operation.
 
  * timeout : the maximum period of time before considering the action failed.
    By default this value is in milliseconds, however you can also specify a value in seconds like so timeout="5s".
 
  * interval : This currently only applies to monitor operations and specifies how often the LRM should check the resource is active.
    The same notation for timeout applies.
 
  * prereq : What conditions need to be met before this action can be run
    * nothing : This action can be performed at any time
    * quorum : This action requires the partition to have quorum
    * fencing : This action requires the partition to have quorum and any fencing operations to have completed before it can be executed
 
  * on_fail : The action to take if this action ever fails.
    * nothing : Pretend the action didnt actually fail
    * block : Take no further action on the resource - wait for the administrator to resolve the issue
    * restart : Stop the resource and re-allocate it elsewhere
    * stop : Stop the resource and DO NOT re-allocate it elsewhere
    * fence : Currently this means fence the node on which the resource is running.
      Any other resources currently active on the machine will be migrated away before fencing occurs.
 
 Only one entry per supported action+interval is currently permitted.
 Parameters specific to each operation can be passed using the instance_attributes section.
 -->
 <!ELEMENT operations (op*)>
 <!ELEMENT op (meta_attributes|instance_attributes)*>
 <!ATTLIST op
           id            CDATA         #REQUIRED
           name          CDATA         #REQUIRED
           description   CDATA         #IMPLIED
           interval      CDATA         #IMPLIED
           timeout       CDATA         #IMPLIED
           start_delay   CDATA         '0'
           disabled      (true|1|false|0)               'false'
           role          (Master|Slave|Started|Stopped) 'Started'
           prereq        (nothing|quorum|fencing)       #IMPLIED
           on_fail       (ignore|block|stop|restart|fence)     #IMPLIED>
 <!--
 Use this to emulate v1 type Heartbeat groups.
 Defining a resource group is a quick way to make sure that the resources:
  * are all started on the same node, and
  * are started and stopped in the correct (sequential) order
 though either or both of these properties can be disabled.
 
 NOTE: Do not create empty groups.  
       They are temporarily supported because the GUI requires it but will be removed as soon as possible.
       The DTD is written this way to be order in-sensitive.
 -->
 <!ELEMENT group (meta_attributes|instance_attributes|primitive)*>
 <!ATTLIST group
           id            CDATA               #REQUIRED
           description   CDATA               #IMPLIED
 
           is_managed            CDATA                         #IMPLIED
           restart_type          (ignore|restart)              'ignore'
           multiple_active       (stop_start|stop_only|block)  'stop_start'
           resource_stickiness   CDATA			      #IMPLIED
 
           ordered               (true|1|false|0)              'true'
           collocated            (true|1|false|0)              'true'>
 <!--
 Clones are intended as a mechanism for easily starting a number of resources (such as a web-server) with the same configuration.
 As an added benefit, the number that should be started is an instance parameter and when combined with time-based constraints, allows the administrator to run more instances during peak times and save on resources during idle periods.
 
  * ordered
    Start (or stop) each clone only after the operation on the previous clone completed.
 
  * interleaved
    If a colocation constraint is created between two clone resources and interleaved is true, then clone N from one resource will be assigned the same location as clone N from the other resource.
    If the number of runnable clones differs, then the leftovers can be located anywhere.
 Using a cloned group is a much better way of achieving the same result.
 
  * notify
    If true, inform peers before and after any clone is stopped or started.
    If an action failed, you will (currently) not recieve a post-notification.
    Instead you can next expect to see a pre-notification for a stop.
    If a stop fails, and you have fencing you will get a post-notification for the stop after the fencing operation has completed.
    In order to use the notification service ALL decendants of the clone MUST support the notify action.
    Currently this action is not permitted to fail, though depending on your configuration, can block almost indefinitly.
    Behaviour in response to a failed action or notificaiton is likely to be improved in future releases.
 
    See http://linux-ha.org/v2/Concepts/Clones for more information on notify actions
 
 
 NOTE: Clones must contain exactly one primitive or one group resource. 
       The CRM will complain about your configuration if this criteria is not met.
       Please use crm_verify to ensure your configuration is valid.
       The DTD is written this way to be order in-sensitive.
 -->
 
 <!ELEMENT clone (meta_attributes|instance_attributes|primitive|group)*>
 <!ATTLIST clone
           id            CDATA               #REQUIRED
           description   CDATA               #IMPLIED
 
           is_managed            CDATA                         #IMPLIED
           restart_type          (ignore|restart)              'ignore'
           multiple_active       (stop_start|stop_only|block)  'stop_start'
           resource_stickiness   CDATA                         #IMPLIED
 
           notify                (true|1|false|0)              'false'
           globally_unique       (true|1|false|0)              'true'
           ordered               (true|1|false|0)              'false'
           interleave            (true|1|false|0)              'false'>
 <!--
 Master/Slave resources are a superset of Clones in that instances can also be in one of two states.
 The meaning of the states is specific to the resource.
 
 NOTE: master_slave must contain exactly one primitive resource OR one group resource.
       It may not contain both, nor may it contain neither.
       The CRM will complain about your configuration if this criteria is not met.
       Please use crm_verify to ensure your configuration is valid.
       The DTD is written this way to be order in-sensitive.
 -->
 <!ELEMENT master_slave (meta_attributes|instance_attributes|primitive|group)*>
 <!ATTLIST master_slave
           id            CDATA       #REQUIRED
           description   CDATA       #IMPLIED
 
           is_managed            CDATA                         #IMPLIED
           restart_type          (ignore|restart)              'ignore'
           multiple_active       (stop_start|stop_only|block)  'stop_start'
           resource_stickiness   CDATA                         #IMPLIED
 
           notify                (true|1|false|0)              'false'
           globally_unique       (true|1|false|0)              'true'
           ordered               (true|1|false|0)              'false'
           interleave            (true|1|false|0)              'false'>
 
 <!--
 Most resource options are configured as instance attributes.
 Some of the built-in options can be configured directly on the resource or as an instance attribute.
 The advantage of using instance attributes is the added flexibility that can be achieved through conditional ?<rule/>s (see below).
 
 You can have multiple sets of 'instance attributes', they are first sorted by score and then processed.
 The first to have its ?<rule/> satisfied and define an attribute wins.
 Subsequent values for the attribute will be ignored.
 
 Note that:
  * instance_attributes sets with id equal to cib-bootstrap-options are treated as if they have a score of INFINITY.
  * instance_attributes sets with no score implicitly have a score of zero.
  * instance_attributes sets with no rule implicitly have a rule that evaluates to true.
 
 The addition of conditional <rule/>s to the instance_attributes object allows for an infinite variety of configurations.
 Just some of the possibilities are:
  * Specify different resource parameters
    * depending on the node it is allocated to (a resource may need to use eth1 on host1 but eth0 on host2)
    * depending on the time of day (run 10 web-servers at night an 100 during the day)
  * Allow nodes to have different attributes depending on the time-of-day
    * Set resource_stickiness to avoid failback during business hours but allow resources to be moved to a more preferred node on the weekend
    * Switch a node between a "front-end" processing group during the day to a "back-end" group at night.
 
 Common instance attributes for all resource types:
  * priority (integer, default=0):
    dictates the order in which resources will be processed.
    If there is an insufficient number of nodes to run all resources, the lower priority resources will be stopped to make sure the higher priority resources remain active.
 
  * is_managed: See previous description.
 
  * resource_stickiness: See previous description.
 
  * target_role: (Started|Stopped|Master|Slave|default, default=#default)
    * #default : Let the cluster decide what to do with the resource
    * Started : Ignore any specified value of is_managed or is_managed_default and attempt to start the resource
    * Stopped : Ignore any specified value of is_managed or is_managed_default and attempt to stop the resource
    * Master : Ignore any specified value of is_managed, is_managed_default or promotion preferences and attempt to put all instances of a cloned resource into Master mode.
    * Slave : Ignore any specified value of is_managed, is_managed_default or promotion preferences and attempt to put all instances of a cloned resource into Slave mode.
 
 Common instance attributes for clones:
  * clone_max (integer, default=1):
    the number of clones to be run
 
 * clone_node_max (integer, default=1):
   the maximum number of clones to be run on a single node
 
 Common instance attributes for nodes:
  * standby (boolean, default=FALSE)
    if TRUE, indicates that resources can not be run on the node
 -->
 <!ELEMENT instance_attributes (rule*, attributes)>
 <!ATTLIST instance_attributes
           id                CDATA        #REQUIRED
           score             CDATA        #IMPLIED>
 
 <!ELEMENT meta_attributes (rule*, attributes)>
 <!ATTLIST meta_attributes
           id                CDATA        #REQUIRED
           score             CDATA        #IMPLIED>
 
 <!--
 Every constraint entry also has a 'lifetime' attribute, which expresses when this constraint is applicable.
 For example, a constraint may only be valid during certain times of the day, or days of the week.
 Eventually, we would like to be able to support constraints that only last until events such as the next reboot or the next transition.
 -->
 <!ELEMENT constraints (rsc_order|rsc_colocation|rsc_location)*>
 
 <!--
 rsc_ordering constraints express dependencies between the actions on two resources.
  * from : A resource id
  * action : What action does this constraint apply to.
  * type : Should the action on from occur before or after action on to
  * to : A resource id
  * symmetrical : If TRUE, create the reverse constraint for the other action also.
 -->
 <!ELEMENT rsc_order (lifetime?)>
 <!ATTLIST rsc_order
           id        CDATA #REQUIRED
           from      CDATA #REQUIRED
           to        CDATA #REQUIRED
           action    CDATA		'start'
           to_action CDATA		'start'
           type      (before|after)	'after'
           score     CDATA		'0'
           symmetrical (true|1|false|0)	'true'>
 
 <!--
+
 Specify where a resource should run relative to another resource
 
-NOTE: Currently, only values of + and - INFINITY are permitted for the score.
-This may change in the future.
+Make rsc 'from' run on the same machine as rsc 'to'
+
+If rsc 'to' cannot run anywhere and 'score' is INFINITY, 
+  then rsc 'from' wont be allowed to run anywhere either
+If rsc 'from' cannot run anywhere, then 'to' wont be affected
+
 -->
 <!ELEMENT rsc_colocation (lifetime?)>
 <!ATTLIST rsc_colocation
           id             CDATA #REQUIRED
           from           CDATA #REQUIRED
           from_role      CDATA #IMPLIED
           to             CDATA #REQUIRED
           to_role        CDATA #IMPLIED
 	  symmetrical    CDATA #IMPLIED
           node_attribute CDATA #IMPLIED
           score          CDATA #REQUIRED>
 
 <!--
 Specify which nodes are eligible for running a given resource.
 
 During processing, all rsc_location for a given rsc are evaluated.
 
 All nodes start out with their base weight (which defaults to zero).
 This can then be modified (up or down) using any number of rsc_location constraints.
 
 Then the highest non-zero available node is determined to place the resource.
 If multiple nodes have the same weighting, the node with the fewest running resources is chosen.
 
 The rsc field is, surprisingly, a resource id.
 -->
 <!ELEMENT rsc_location (lifetime?,rule*)>
 <!ATTLIST rsc_location
           id          CDATA #REQUIRED
           description CDATA #IMPLIED
           rsc         CDATA #REQUIRED>
 <!ELEMENT lifetime (rule+)>
 <!ATTLIST lifetime id  CDATA     #REQUIRED>
 
 <!--
  * boolean_op
    determines how the results of multiple expressions are combined.
 
  * role
    limits this rule to applying to Multi State resources with the named role.
    Roles include Started, Stopped, Slave, Master though only the last two are considered useful.
    NOTE: A rule with role="Master" can not determin the initial location of a clone instance.
    It will only affect which of the active instances will be promoted.
 
  * score
    adjusts the preference for running on the matched nodes.
    NOTE: Nodes that end up with a negative score will never run the resource.
    Two special values of "score" exist: INFINITY and -INFINITY.
    Processing of these special values is as follows:
 
       INFINITY +/- -INFINITY : -INFINITY
       INFINITY +/-  int      :  INFINITY
      -INFINITY +/-  int      : -INFINITY
   
  * score_attribute 
    an alternative to the score attribute that provides extra flexibility.
   Each node matched by the rule has its score adjusted differently, according to its value for the named node attribute.
   Thus in the example below, if score_attribute="installed_ram" and nodeA would have its preference to run "the resource" increased by 1024 whereas nodeB would have its preference increased only by half as much.
 
     <nodes>
       <node id="uuid1" uname="nodeA" type="normal">
         <instance_attributes id="uuid1:custom_attrs">
           <attributes>
             <nvpair id="uuid1:installed_ram" name="installed_ram" value="1024"/>
             <nvpair id="uuid1:my_other_attr" name="my_other_attr" value="bob"/>
           </attributes>
         </instance_attributes>
       </node>
       <node id="uuid2" uname="nodeB" type="normal">
         <instance_attributes id="uuid2:custom_attrs">
           <attributes>
             <nvpair id="uuid2:installed_ram" name="installed_ram" value="512"/>
           </attributes>
         </instance_attributes>
       </node>
     </nodes>
 -->
 <!ELEMENT rule (expression|date_expression|rule)*>
 <!ATTLIST rule
           id                  CDATA          #REQUIRED
           role                CDATA          #IMPLIED
           score               CDATA          #IMPLIED
           score_attribute     CDATA          #IMPLIED
           boolean_op          (or|and)      'and'>
 
 <!--
 Returns TRUE or FALSE depending on the properties of the object being tested.
 
  * type determines how the values being tested.
    * integer Values are converted to floats before being compared.
    * version The "version" type is intended to solve the problem of comparing 1.2 and 1.10
    * string Uses strcmp
 
 Two built-in attributes are node id #id and node uname #uname so that:
       attribute=#id value=8C05CA5C-C9E3-11D8-BEE6-000A95B71D78 operation=eq, and
       attribute=#uname value=test1 operation=eq
 would both be valid tests.
 
 An extra built-in attribute called #is_dc will be set to true or false depending on whether the node is operating as the DC for the cluster.
 Valid tests using this test would be of the form:
 
         attribute=#is_dc operation=eq value=true,  and
         attribute=#is_dc operation=eq value=false, and
         attribute=#is_dc operation=ne value=false
                         (for those liking double negatives :))
 -->
 <!ELEMENT expression EMPTY>
 <!ATTLIST expression
           id         CDATA                    #REQUIRED
           attribute  CDATA                    #REQUIRED
           operation  (lt|gt|lte|gte|eq|ne|defined|not_defined) #REQUIRED
           value      CDATA                    #IMPLIED
           type       (number|string|version) 'string'>
 
 <!--
  * start : A date-time conforming to the ISO8601 specification.
  * end : A date-time conforming to the ISO8601 specification.
    A value for end may, for any usage, be omitted and instead inferred using start and duration.
  * operation
    * gt : Compares the current date-time with start date.
      Checks now > start.
    * lt : Compares the current date-time with end date.
      Checks end > now
    * in_range : Compares the current date-time with start and end.
      Checks now > start and end > now.
      If either start or end is omitted, then that part of the comparision is not performed.
    * date_spec : Performs a cron-like comparision between the contents of date_spec and now.
      If values for start and/or end are included, now must also be within that range.
      Or in other words, the date_spec operation can also be made to perform an extra in_range check.
 
 NOTE: Because the comparisions (except for date_spec) include the time, the eq, neq, gte and lte operators have not been implemented.
 -->
 <!ELEMENT date_expression (date_spec?,duration?)>
 <!ATTLIST date_expression
         id         CDATA  #REQUIRED
         operation  (in_range|date_spec|gt|lt) 'in_range'
         start      CDATA  #IMPLIED
         end        CDATA  #IMPLIED>
 
 <!--
 date_spec is used for (surprisingly  ) date_spec operations.
 
 Fields that are not supplied are ignored.
 
 Fields can contain a single number or a single range.
 Eg.
 monthdays="1" (Matches the first day of every month) and hours="09-17" (Matches hours between 9am and 5pm inclusive) are both valid values.
 weekdays="1,2" and weekdays="1-2,5-6" are NOT valid ranges.
 This may change in a future release.
 
  * seconds : Value range 0-59
  * minutes : Value range 0-59
  * hours : Value range 0-23
  * monthdays : Value range 0-31 (depending on current month and year)
  * weekdays : Value range 1-7 (1=Monday, 7=Sunday)
  * yeardays : Value range 1-366 (depending on current year)
  * months : Value range 1-12
  * weeks : Value range 1-53 (depending on weekyear)
  * weekyears : Value range 0...
   (NOTE: weekyears may differ from Gregorian years.
   Eg. 2005-001 Ordinal == 2005-01-01 Gregorian == 2004-W53-6 Weekly )
  * years : Value range 0...
  * moon : Value range 0..7 - 0 is new, 4 is full moon.
    Because we can(tm)
 -->
 
 <!ELEMENT date_spec EMPTY>
 <!ATTLIST date_spec
         id         CDATA  #REQUIRED
         hours      CDATA  #IMPLIED
         monthdays  CDATA  #IMPLIED
         weekdays   CDATA  #IMPLIED
         yeardays   CDATA  #IMPLIED
         months     CDATA  #IMPLIED
         weeks      CDATA  #IMPLIED
         weekyears  CDATA  #IMPLIED
         years      CDATA  #IMPLIED
         moon       CDATA  #IMPLIED>
 
 <!--
 duration is optionally used for calculating a value for end.
 Any field not supplied is assumed to be zero and ignored.
 Negative values might work.
 Eg. months=11 should be equivalent to writing years=1, months=-1 but is not encouraged.
 -->
 <!ELEMENT duration EMPTY>
 <!ATTLIST duration
         id         CDATA  #REQUIRED
         hours      CDATA  #IMPLIED
         monthdays  CDATA  #IMPLIED
         weekdays   CDATA  #IMPLIED
         yeardays   CDATA  #IMPLIED
         months     CDATA  #IMPLIED
         weeks      CDATA  #IMPLIED
         years      CDATA  #IMPLIED>
 <!--
 Example 1: True if now is any time in the year 2005.
 
 <rule id="rule1">
   <date_expression id="date_expr1" start="2005-001" operation="in_range">
     <duration years="1"/>
   </date_expression>
 </rule>
 Example 2: Equivalent expression.
 
 <rule id="rule2">
   <date_expression id="date_expr2" operation="date_spec">
     <date_spec years="2005"/>
   </date_expression>
 </rule>
 Example 3: 9am-5pm, Mon-Friday
 
 <rule id="rule3">
   <date_expression id="date_expr3" operation="date_spec">
     <date_spec hours="9-16" days="1-5"/>
   </date_expression>
 </rule>
 Example 4: 9am-5pm, Mon-Friday, or all day saturday
 
 <rule id="rule4" boolean_op="or">
   <date_expression id="date_expr4-1" operation="date_spec">
     <date_spec hours="9-16" days="1-5"/>
   </date_expression>
   <date_expression id="date_expr4-2" operation="date_spec">
     <date_spec days="6"/>
   </date_expression>
 </rule>
 Example 5: 9am-5pm or 9pm-12pm, Mon-Friday
 
 <rule id="rule5" boolean_op="and">
   <rule id="rule5-nested1" boolean_op="or">
     <date_expression id="date_expr5-1" operation="date_spec">
       <date_spec hours="9-16"/>
     </date_expression>
     <date_expression id="date_expr5-2" operation="date_spec">
       <date_spec hours="21-23"/>
     </date_expression>
   </rule>
   <date_expression id="date_expr5-3" operation="date_spec">
     <date_spec days="1-5"/>
   </date_expression>
 </rule>
 Example 6: Mondays in March 2005
 
 <rule id="rule6" boolean_op="and">
   <date_expression id="date_expr6" operation="date_spec" start="2005-03-01" end="2005-04-01">
     <date_spec weekdays="1"/>
   </date_expression>
 </rule>
 NOTE: Because no time is specified, 00:00:00 is implied.
 This means that the range includes all of 2005-03-01 but none of 2005-04-01.
 You may wish to write end="2005-03-31T23:59:59" to avoid confusion.
 
 Example 7: Friday the 13th if it is a full moon
 
 <rule id="rule7" boolean_op="and">
   <date_expression id="date_expr7" operation="date_spec">
     <date_spec weekdays="5" monthdays="13" moon="4"/>
   </date_expression>
 </rule>
 -->
 
 <!--
 You don't have to give a value.
 There's a difference between a key not being present and a key not having a value.
 -->
 <!ELEMENT nvpair EMPTY>
 <!ATTLIST nvpair
           id     CDATA  #REQUIRED
           name   CDATA  #REQUIRED
           value  CDATA  #IMPLIED>
 
 <!ELEMENT attributes (nvpair*)>
 
 <!--
 These attributes take effect only if no value has previously been applied as part of the node's definition.
 Additionally, when the node reboots all settings made here are erased.
 
 id must be the UUID of the node.
 -->
 <!ELEMENT transient_attributes (instance_attributes*)>
 <!ATTLIST transient_attributes id CDATA #IMPLIED>
 
 <!--=========== Status - Advanced Use Only ===========-->
 
 <!--
 Details about the status of each node configured.
 
 HERE BE DRAGONS
 
 Never, ever edit this section directly or using cibadmin.
 The consequences of doing so are many and varied but rarely ever good or what you anticipated.
 To discourage this, the status section is no longer even written to disk, and is always discarded at startup.
 
 To avoid duplication of data, state entries only carry references to nodes and resources.
 -->
 <!ELEMENT status (node_state*)>
 
 <!--
 The state of a given node.
 
 This information is updated by the DC based on inputs from sources such as the CCM, status messages from remote LRMs and requests from other nodes.
  * id       -  is the node's UUID.
  * uname    - is the result of uname -n for the node.
  * crmd     - records whether the crmd process is running on the node
  * in_ccm   - records whether the node is part of our membership partition
  * join     - is the node's membership status with the current DC.
  * expected - is the DC's expectation of whether the node is up or not.
  * shutdown - is set to the time at which the node last asked to be shut down
 
 Ideally, there should be a node_state entry for every entry in the <nodes> list.
 
 -->
 <!ELEMENT node_state (transient_attributes|lrm)*>
 <!ATTLIST node_state
         id              CDATA                   #REQUIRED
         uname           CDATA                   #REQUIRED
         ha              (active|dead)           #IMPLIED
         crmd            (online|offline)        'offline'
         join            (pending|member|down)   'down'
         expected        (pending|member|down)   'down'
         in_ccm          (true|1|false|0)        'false'
         crm-debug-origin CDATA                  #IMPLIED
         shutdown        CDATA                   #IMPLIED
         clear_shutdown  CDATA                   #IMPLIED>
 
 <!--
 Information from the Local Resource Manager of the node.
 It contains a list of all resource's added (but not necessarily still active) on the node.
 -->
 <!ELEMENT lrm (lrm_resources)>
 <!ATTLIST lrm id CDATA #REQUIRED>
 
 <!ELEMENT lrm_resources (lrm_resource*)>
 <!ELEMENT lrm_resource (lrm_rsc_op*)>
 <!ATTLIST lrm_resource
           id            CDATA #REQUIRED
           class             (lsb|ocf|heartbeat|stonith) #REQUIRED
           type              CDATA        #REQUIRED
           provider          CDATA        #IMPLIED>
 <!--
 lrm_rsc_op (Resource Status)
 
 id: Set to [operation] +"_"+ [operation] +"_"+ [an_interval_in_milliseconds]
 
 operation typically start, stop, or monitor
 
 call_id: Supplied by the LRM, determins the order of in which lrm_rsc_op objects should be processed in order to determin the resource's true state
 
 rc_code is the last return code from the resource
 
 rsc_state is the state of the resource after the action completed and should be used as a guide only.
 
 transition_key contains an identifier and seqence number for the transition.
 
 At startup, the TEngine registers the identifier and starts the sequence at zero.
 It is used to identify the source of resource actions.
 
 transition_magic contains an identifier containing call_id, rc_code, and {{transition_key}}}.
 
 As the name suggests, it is a piece of magic that allows the TE to always identify the action from the stream of xml-diffs it subscribes to from the CIB.
 
 op_status is supplied by the LRM and conforms to this enum:
 
 typedef enum {
         LRM_OP_PENDING = -1,
         LRM_OP_DONE,
         LRM_OP_CANCELLED,
         LRM_OP_TIMEOUT,
         LRM_OP_NOTSUPPORTED,
         LRM_OP_ERROR,
 } op_status_t;
 The parameters section allows us to detect when a resource's definition has changed and the needs to be restarted (so the changes take effect).
 -->
 <!ELEMENT lrm_rsc_op EMPTY>
 <!ATTLIST lrm_rsc_op
           id                    CDATA #REQUIRED
           operation             CDATA #REQUIRED
           op_status             CDATA #REQUIRED
           rc_code               CDATA #REQUIRED
           call_id               CDATA #REQUIRED
           crm_feature_set       CDATA #REQUIRED
           crm-debug-origin      CDATA #IMPLIED
           migrate_from          CDATA #IMPLIED
           transition_key        CDATA #IMPLIED
           op_digest             CDATA #IMPLIED
           op_restart_digest     CDATA #IMPLIED
           op_force_restart      CDATA #IMPLIED
           interval              CDATA #REQUIRED
           transition_magic      CDATA #REQUIRED>
diff --git a/crm/crmd/lrm.c b/crm/crmd/lrm.c
index f84b15fa61..f74f9f94b7 100644
--- a/crm/crmd/lrm.c
+++ b/crm/crmd/lrm.c
@@ -1,1794 +1,1792 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <lha_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <crmd_fsa.h>
 
 #include <sys/types.h>
 #include <sys/wait.h>
 
 #include <unistd.h>			/* for access */
 #include <heartbeat.h>
 #include <clplumbing/cl_signal.h>
 
 #include <errno.h>
 
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crmd.h>
 #include <crmd_messages.h>
 #include <crmd_callbacks.h>
 #include <crmd_lrm.h>
 
 #include <lrm/raexec.h>
 
 
 struct recurring_op_s 
 {
 		char *rsc_id;
 		char *op_key;
 		int   call_id;
 		int   interval;
 };
 
 char *make_stop_id(const char *rsc, int call_id);
 gboolean verify_stopped(gboolean force, int log_level);
 
 gboolean build_operation_update(
 	crm_data_t *rsc_list, lrm_op_t *op, const char *src, int lpc);
 
 gboolean build_active_RAs(crm_data_t *rsc_list);
 gboolean is_rsc_active(const char *rsc_id);
 
 void do_update_resource(lrm_op_t *op);
 gboolean process_lrm_event(lrm_op_t *op);
 
 enum crmd_fsa_input do_lrm_rsc_op(
 	lrm_rsc_t *rsc, const char *operation,
 	crm_data_t *msg, HA_Message *request);
 
 lrm_op_t *construct_op(
 	crm_data_t *rsc_op, const char *rsc_id, const char *operation);
 
 void send_direct_ack(const char *to_host, const char *to_sys,
 		     lrm_op_t* op, const char *rsc_id);
 
 void free_recurring_op(gpointer value);
 
 GHashTable *meta_hash = NULL;
 
 GHashTable *resources = NULL;
 GHashTable *pending_ops = NULL;
 GCHSource *lrm_source = NULL;
 
 int num_lrm_register_fails = 0;
 int max_lrm_register_fails = 30;
 
 /*	 A_LRM_CONNECT	*/
 enum crmd_fsa_input
 do_lrm_control(long long action,
 	       enum crmd_fsa_cause cause,
 	       enum crmd_fsa_state cur_state,
 	       enum crmd_fsa_input current_input,
 	       fsa_data_t *msg_data)
 {
 	int ret = HA_OK;
 
 	if(action & A_LRM_DISCONNECT) {
 		verify_stopped(TRUE, LOG_ERR);
 		if(lrm_source) {
 			crm_debug("Removing LRM connection from MainLoop");
 			if(G_main_del_IPC_Channel(lrm_source) == FALSE) {
 				crm_err("Could not remove LRM connection"
 					" from MainLoop");
 			}
 			lrm_source = NULL;			
 		}
 		if(fsa_lrm_conn) {
 			fsa_lrm_conn->lrm_ops->signoff(fsa_lrm_conn);
 			crm_info("Disconnected from the LRM");
 			clear_bit_inplace(fsa_input_register, R_LRM_CONNECTED);
 		}
 		/* TODO: Clean up the hashtable */
 	}
 
 	if(action & A_LRM_CONNECT) {
 	
 		ret = HA_OK;
 		
 		pending_ops = g_hash_table_new_full(
 			g_str_hash, g_str_equal,
 			g_hash_destroy_str, free_recurring_op);
 
 		resources = g_hash_table_new_full(
 			g_str_hash, g_str_equal,
 			g_hash_destroy_str, g_hash_destroy_str);
 		
 		if(NULL == fsa_lrm_conn) {
 			register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
 			ret = HA_FAIL;
 		}
 
 		if(ret == HA_OK) {
 			crm_debug("Connecting to the LRM");
 			ret = fsa_lrm_conn->lrm_ops->signon(
 				fsa_lrm_conn, CRM_SYSTEM_CRMD);
 		}
 		
 		if(ret != HA_OK) {
 			if(++num_lrm_register_fails < max_lrm_register_fails) {
 				crm_warn("Failed to sign on to the LRM %d"
 					 " (%d max) times",
 					 num_lrm_register_fails,
 					 max_lrm_register_fails);
 				
 				crm_timer_start(wait_timer);
 				crmd_fsa_stall(NULL);
 				return I_NULL;
 			}
 		}
 
 		if(ret == HA_OK) {
 			crm_debug_4("LRM: set_lrm_callback...");
 			ret = fsa_lrm_conn->lrm_ops->set_lrm_callback(
 				fsa_lrm_conn, lrm_op_callback);
 			if(ret != HA_OK) {
 				crm_err("Failed to set LRM callbacks");
 			}
 		}
 		
 		if(ret != HA_OK) {
 			crm_err("Failed to sign on to the LRM %d"
 				" (max) times", num_lrm_register_fails);
 			register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
 			return I_NULL;
 		}
 
 		/* TODO: create a destroy handler that causes
 		 * some recovery to happen
 		 */
 		lrm_source = G_main_add_IPC_Channel(
 			G_PRIORITY_LOW,
 			fsa_lrm_conn->lrm_ops->ipcchan(fsa_lrm_conn),
 			FALSE, lrm_dispatch, fsa_lrm_conn,
 			default_ipc_connection_destroy);
 
 		set_bit_inplace(fsa_input_register, R_LRM_CONNECTED);
 		crm_debug("LRM connection established");
 		
 	}	
 
 	if(action & ~(A_LRM_CONNECT|A_LRM_DISCONNECT)) {
 		crm_err("Unexpected action %s in %s",
 		       fsa_action2string(action), __FUNCTION__);
 	}
 		
 	
 	return I_NULL;
 }
 
 static void
 ghash_print_pending(gpointer key, gpointer value, gpointer user_data) 
 {
 	const char *action = key;
 	int *log_level = user_data;
 	do_crm_log(*log_level, "Pending action: %s", action);
 }
 
 gboolean
 verify_stopped(gboolean force, int log_level)
 {
 	GListPtr lrm_list = NULL;
 	IPC_Channel *lrm_channel = NULL;
 
 	crm_info("Checking for active resources before exit");
 	if(fsa_lrm_conn != NULL) {
 		lrm_channel = fsa_lrm_conn->lrm_ops->ipcchan(fsa_lrm_conn);
 	}
 	
 	if(fsa_lrm_conn == NULL
 	   || lrm_channel == NULL
 	   || lrm_channel->ch_status != IPC_CONNECT) {
 		crm_err("Exiting with no LRM connection..."
 			" resources may be active!");
 		force = TRUE;
 
 	} else {
 		lrm_list = fsa_lrm_conn->lrm_ops->get_all_rscs(fsa_lrm_conn);
 	}
 
 	if(g_hash_table_size(pending_ops) > 0) {
 		do_crm_log(log_level,
 			      "%d pending LRM operations at shutdown%s",
 			      g_hash_table_size(pending_ops),
 			      force?"":"... waiting");
 
 		if(force || !is_set(fsa_input_register, R_SENT_RSC_STOP)) {
 			g_hash_table_foreach(
 				pending_ops, ghash_print_pending, &log_level);
 		}
 
 		if(force == FALSE) {
 			return FALSE;
 		}
 	}
 	
 	slist_iter(
 		rsc_id, char, lrm_list, lpc,
 		if(is_rsc_active(rsc_id) == FALSE) {
 			continue;
 		}
 		crm_err("Resource %s was active at shutdown."
 			"  You may ignore this error if it is unmanaged.",
 			rsc_id);
 		);
 	
 	set_bit_inplace(fsa_input_register, R_SENT_RSC_STOP);
 	register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
 
 	return TRUE;
 }
 
 static const char *
 get_rsc_metadata(const char *type, const char *class, const char *provider)
 {
 	int len = 0;
 	char *key = NULL;
 	char *metadata = NULL;
 	if(meta_hash == NULL) {
 		meta_hash = g_hash_table_new_full(
 			g_str_hash, g_str_equal,
 			g_hash_destroy_str, g_hash_destroy_str);
 	}
 
 	CRM_CHECK(type != NULL, return NULL);
 	CRM_CHECK(class != NULL, return NULL);
 	if(provider == NULL) {
 		provider = "heartbeat";
 	}
 
 	len = strlen(type) + strlen(class) + strlen(provider) + 4;
 
 	crm_malloc0(key, len);
 	sprintf(key, "%s::%s:%s", type, class, provider);
 	key[len-1] = 0;
 
 	metadata = g_hash_table_lookup(meta_hash, key);
 	if(metadata) {
 		crm_debug_2("Returning cached metadata for %s", key);
 		goto out;
 	}
 
 	crm_debug("Retreiving metadata for %s", key);
 	metadata = fsa_lrm_conn->lrm_ops->get_rsc_type_metadata(
 		fsa_lrm_conn, class, type, provider);
 
 	if(metadata) {
 		/* copy the metadata because the LRM likes using
 		 *   g_alloc instead of cl_malloc
 		 */
 		char *m_copy = crm_strdup(metadata);
  		g_hash_table_insert(meta_hash, key, m_copy);
 		key = NULL; /* prevent it from being free'd */
 		g_free(metadata);
 		metadata = m_copy;
 		
 	} else {
 		crm_warn("No metadata found for %s", key);
 	}		
 
   out:
 	crm_free(key);
 	return metadata;
 }
 
 static GListPtr
 get_rsc_restart_list(lrm_rsc_t *rsc, lrm_op_t *op) 
 {
 	gboolean supported = FALSE;
 	GListPtr restart_list = NULL;
 
 	const char *value = NULL;
 	const char *metadata_str = get_rsc_metadata(
 		rsc->type, rsc->class, rsc->provider);
 
 	crm_data_t *params = NULL;
 	crm_data_t *actions = NULL;
 	crm_data_t *metadata = NULL;
 
 	if(metadata_str == NULL) {
 		return NULL;
 	}
 	
 	metadata = string2xml(metadata_str);
 	if(metadata == NULL) {
 		crm_err("Metadata for %s::%s:%s is not valid XML",
 			rsc->provider, rsc->class, rsc->type);
 		return NULL;
 	}
 
 	actions = find_xml_node(metadata, "actions", TRUE);
 	
 	xml_child_iter_filter(
 		actions, action, "action",
 		value = crm_element_value(action, "name");
 		if(safe_str_eq("reload", value)) {
 			supported = TRUE;
 			break;
 		}
 		);
 
 	if(supported == FALSE) {
 		goto cleanup;
 	}
 	
 	params = find_xml_node(metadata, "parameters", TRUE);
 	xml_child_iter_filter(
 		params, param, "parameter",
 		value = crm_element_value(param, "unique");
 		if(crm_is_true(value)) {
 			value = crm_element_value(param, "name");
 			crm_debug("Attr %s is not reloadable", value);
 			restart_list = g_list_append(
 				restart_list, crm_strdup(value));
 		}
 		);
   cleanup:
 	free_xml(metadata);
 	return restart_list;
 }
 
 static void
 append_restart_list(crm_data_t *update, lrm_op_t *op, const char *version) 
 {
 	int len = 0;
 	char *list = NULL;
 	char *digest = NULL;
 	lrm_rsc_t *rsc = NULL;
 	const char *value = NULL;
 	gboolean non_empty = FALSE;
 	crm_data_t *restart = NULL;
 	GListPtr restart_list = NULL;
 
 	if(op->interval > 0) {
 		/* monitors are not reloadable */
 		return;
 
 	} else if(safe_str_neq(CRMD_ACTION_START, op->op_type)) {
 		/* only starts are potentially reloadable */
 		return;
 		
 	} else if(compare_version("1.0.8", version) > 0) {
 		crm_debug("Caller version %s does not support reloads", version);
 		return;
 	}
 
 	rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, op->rsc_id);
 	if(rsc == NULL) {
 		crm_info("Resource %s no longer in the LRM", op->rsc_id);
 		return;
 	}
 
 	restart_list = get_rsc_restart_list(rsc, op);
 	if(restart_list == NULL) {
 		crm_debug("Resource %s does not support reloads", op->rsc_id);
 		return;
 	}
 
 	restart = create_xml_node(NULL, "parameters");
 	slist_iter(param, const char, restart_list, lpc,
 		   int start = len;
 		   value = g_hash_table_lookup(op->params, param);
 		   if(value != NULL) {
 			   non_empty = TRUE;
 			   crm_xml_add(restart, param, value);
 		   }
 		   len += strlen(param) + 2;
 		   crm_realloc(list, len+1);
 		   sprintf(list+start, " %s ", param);
 		);
 	
 	digest = calculate_xml_digest(restart, TRUE);
 	crm_xml_add(update, XML_LRM_ATTR_OP_RESTART, list);
 	crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest);
 
 	crm_debug("%s : %s", digest, list);
 	if(non_empty) {
 		crm_log_xml_debug(restart, "restart digest source");
 	}
 	slist_destroy(char, child, restart_list,
 		      crm_free(child));
 	free_xml(restart);
 	crm_free(digest);
 	crm_free(list);
 }
 
 gboolean
 build_operation_update(
 	crm_data_t *xml_rsc, lrm_op_t *op, const char *src, int lpc)
 {
 	char *magic = NULL;
 	const char *task = NULL;
 	crm_data_t *xml_op = NULL;
 	char *op_id = NULL;
 	char *local_user_data = NULL;
 	const char *caller_version = NULL;	
 	char *digest = NULL;
 	crm_data_t *args_xml = NULL;
 	crm_data_t *args_parent = NULL;
 
 	CRM_DEV_ASSERT(op != NULL);
 	if(crm_assert_failed) {
 		return FALSE;
 	}
 
 	crm_debug_2("%s: Updating resouce %s after %s %s op",
 		  src, op->rsc_id, op_status2text(op->op_status), op->op_type);
 
 	if(op->op_status == LRM_OP_CANCELLED) {
 		crm_debug_3("Ignoring cancelled op");
 		return TRUE;
 	}
 
 	if(AM_I_DC) {
 		caller_version = CRM_FEATURE_SET;
 
 	} else if(fsa_our_dc_version != NULL) {
 		caller_version = fsa_our_dc_version;
 
 	} else {
 		/* there is a small risk in formerly mixed clusters that
 		 *   it will be sub-optimal.
 		 * however with our upgrade policy, the update we send
 		 *   should still be completely supported anyway
 		 */
 		caller_version = g_hash_table_lookup(
 			op->params, XML_ATTR_CRM_VERSION);
 		crm_warn("Falling back to operation originator version: %s",
 			 caller_version);
 	}
 	crm_debug_3("DC version: %s", caller_version);
 
 	task = op->op_type;
 	/* remap the task name under various scenarios
 	 * this makes life easier for the PE when its trying determin the current state 
 	 */
 	if(crm_str_eq(task, "reload", TRUE)) {
 		if(op->op_status == LRM_OP_DONE) {
 			task = CRMD_ACTION_START;
 		} else {
 			task = CRMD_ACTION_STATUS;
 		}
 
 	} else if(crm_str_eq(task, CRMD_ACTION_MIGRATE, TRUE)) {
 		/* if the migrate_from fails it will have enough info to do the right thing */
 		if(op->op_status == LRM_OP_DONE) {
 			task = CRMD_ACTION_STOP;
 		} else {
 			task = CRMD_ACTION_STATUS;
 		}
 
 	} else if(op->op_status == LRM_OP_DONE
 		  && crm_str_eq(task, CRMD_ACTION_MIGRATED, TRUE)) {
 		task = CRMD_ACTION_START;
 	}
 
 	if(safe_str_eq(task, CRMD_ACTION_NOTIFY)) {
 		const char *n_type = g_hash_table_lookup(
 			op->params, crm_meta_name("notify_type"));
 		const char *n_task = g_hash_table_lookup(
 			op->params, crm_meta_name("notify_operation"));
 #if CRM_DEPRECATED_SINCE_2_0_5
 		if(n_type == NULL) {
 			n_type = g_hash_table_lookup(op->params, "notify_type");
 		}
 		if(n_task == NULL) {
 			n_task = g_hash_table_lookup(op->params, "notify_operation");
 		}
 #endif
 		CRM_DEV_ASSERT(n_type != NULL);
 		CRM_DEV_ASSERT(n_task != NULL);
 		op_id = generate_notify_key(op->rsc_id, n_type, n_task);
 
 		/* these are not yet allowed to fail */
 		op->op_status = LRM_OP_DONE;
 		op->rc = 0;
 		
 	} else {
 		op_id = generate_op_key(op->rsc_id, task, op->interval);
 	}
 
 	/* Handle recurring ops - infer last op_status */
 	if(op->op_status == LRM_OP_PENDING && op->interval > 0) {
 		if(op->rc == 0) {
 			crm_debug("Mapping pending operation to DONE");
 			op->op_status = LRM_OP_DONE;
 		} else {
 			crm_debug("Mapping pending operation to ERROR");
 			op->op_status = LRM_OP_ERROR;
 		}
 	}
 
 	xml_op = find_entity(xml_rsc, XML_LRM_TAG_RSC_OP, op_id);
 	if(xml_op != NULL) {
 		crm_log_xml(LOG_DEBUG, "Replacing existing entry", xml_op);
 		
 	} else {
 		xml_op = create_xml_node(xml_rsc, XML_LRM_TAG_RSC_OP);
 	}
 	crm_xml_add(xml_op, XML_ATTR_ID, op_id);
 	crm_free(op_id);
 
 	crm_xml_add(xml_op, XML_LRM_ATTR_TASK, task);
 	crm_xml_add(xml_op, XML_ATTR_ORIGIN,   src);
 	
 	if(op->user_data == NULL) {
 		char *id = crm_itoa(op->call_id);
 
 		crm_debug("Generating fake transition key for:"
 			  " %s_%s_%d %d from %s",
 			  op->rsc_id, op->op_type, op->interval, op->call_id,
 			  op->app_name);
 		local_user_data = generate_transition_key(-1, 0, id);
 		op->user_data = local_user_data;
 		crm_free(id);
 	}
 	
 	if(compare_version("1.0.3", caller_version) > 0) {
 		magic = generate_transition_magic_v202(
 			op->user_data, op->op_status);
 
 	} else {
 		magic = generate_transition_magic(
 			op->user_data, op->op_status, op->rc);
 	}
 	
 	crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY,   op->user_data);
 	crm_xml_add(xml_op, XML_ATTR_TRANSITION_MAGIC, magic);
 	crm_free(magic);	
 	
 	switch(op->op_status) {
 		case LRM_OP_PENDING:
 			break;
 		case LRM_OP_CANCELLED:
 			crm_err("What to do here");
 			break;
 		case LRM_OP_ERROR:
 		case LRM_OP_TIMEOUT:
 		case LRM_OP_NOTSUPPORTED:
 			crm_debug_2("Resource action %s/%s %s: %d",
 				    op->rsc_id, task,
 				    op_status2text(op->op_status), op->rc);
 			break;
 		case LRM_OP_DONE:
 			break;
 	}
 	
 	crm_xml_add_int(xml_op,  XML_LRM_ATTR_CALLID, op->call_id);
 
 	/* set these on 'xml_rsc' too to make life easy for the PE */
 	crm_xml_add(xml_op, XML_ATTR_CRM_VERSION, caller_version);
 	crm_xml_add_int(xml_op, XML_LRM_ATTR_RC, op->rc);
 	crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS, op->op_status);
 	crm_xml_add_int(xml_op, XML_LRM_ATTR_INTERVAL, op->interval);
 	
 	/* this will enable us to later determin that the
 	 *   resource's parameters have changed and we should force
 	 *   a restart
 	 */
 	args_parent = NULL;
 #if CRM_DEPRECATED_SINCE_2_0_4
 	if(compare_version("1.0.4", caller_version) > 0) {
 		args_parent = xml_op;
 	}
 #endif
 	args_xml = create_xml_node(args_parent, XML_TAG_PARAMS);
 	g_hash_table_foreach(op->params, hash2field, args_xml);
 	filter_action_parameters(args_xml, caller_version);
 	digest = calculate_xml_digest(args_xml, TRUE);
 	if(safe_str_eq(task, CRMD_ACTION_START)) {
 		/* info for now as this area has been problematic to debug */
 		crm_info("Digest for %s (%s) was %s\n", 
 			  crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC),
 			  ID(xml_op), digest);
 		crm_log_xml(LOG_INFO,  "digest:source", args_xml);
 	}
 	
 	crm_xml_add(xml_op, XML_LRM_ATTR_OP_DIGEST, digest);
 	crm_free(digest);
 	if(args_parent == NULL) {
 		free_xml(args_xml);
 	}
 
 	append_restart_list(xml_op, op, caller_version);
 
 	if(op->op_status != LRM_OP_DONE
 	   && crm_str_eq(op->op_type, CRMD_ACTION_MIGRATED, TRUE)) {
 		const char *host = g_hash_table_lookup(
 			op->params, crm_meta_name("migrate_source_uuid"));
 		crm_xml_add(xml_op, CRMD_ACTION_MIGRATED, host);
 	}	
 	
 	if(local_user_data) {
 		crm_free(local_user_data);
 		op->user_data = NULL;
 	}
 	return TRUE;
 }
 
 gboolean
 is_rsc_active(const char *rsc_id) 
 {
 	GList *op_list  = NULL;
 	gboolean active = FALSE;
 	lrm_rsc_t *the_rsc = NULL;
 	state_flag_t cur_state = 0;
 	int max_call_id = -1;
 	
 	if(fsa_lrm_conn == NULL) {
 		return FALSE;
 	}
 
 	the_rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rsc_id);
 
 	crm_debug_3("Processing lrm_rsc_t entry %s", rsc_id);
 	
 	if(the_rsc == NULL) {
 		crm_err("NULL resource returned from the LRM");
 		return FALSE;
 	}
 	
 	op_list = the_rsc->ops->get_cur_state(the_rsc, &cur_state);
 	
 	crm_debug_3("\tcurrent state:%s",cur_state==LRM_RSC_IDLE?"Idle":"Busy");
 	
 	slist_iter(
 		op, lrm_op_t, op_list, llpc,
 		
 		crm_debug_2("Processing op %s_%d (%d) for %s (status=%d, rc=%d)", 
 			    op->op_type, op->interval, op->call_id, the_rsc->id,
 			    op->op_status, op->rc);
 		
 		CRM_ASSERT(max_call_id <= op->call_id);			
 		if(op->rc == EXECRA_OK
 		   && safe_str_eq(op->op_type, CRMD_ACTION_STOP)) {
 			active = FALSE;
 			
 		} else if(op->rc == EXECRA_OK
 			  && safe_str_eq(op->op_type, CRMD_ACTION_MIGRATE)) {
 			/* a stricter check is too complex...
 			 * leave that to the PE
 			 */
 			active = FALSE;
 			
 		} else if(op->rc == EXECRA_NOT_RUNNING) {
 			active = FALSE;
 
 		} else {
 			active = TRUE;
 		}
 		
 		max_call_id = op->call_id;
 		lrm_free_op(op);
 		);
 
 	g_list_free(op_list);
 	lrm_free_rsc(the_rsc);
 
 	return active;
 }
 
 
 gboolean
 build_active_RAs(crm_data_t *rsc_list)
 {
 	GList *op_list  = NULL;
 	GList *lrm_list = NULL;
 	gboolean found_op = FALSE;
 	state_flag_t cur_state = 0;
 	
 	if(fsa_lrm_conn == NULL) {
 		return FALSE;
 	}
 
 	lrm_list = fsa_lrm_conn->lrm_ops->get_all_rscs(fsa_lrm_conn);
 
 	slist_iter(
 		rid, char, lrm_list, lpc,
 
 		lrm_rsc_t *the_rsc =
 			fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rid);
 
 		crm_data_t *xml_rsc = create_xml_node(
 			rsc_list, XML_LRM_TAG_RESOURCE);
 
 		int max_call_id = -1;
 		
 		crm_debug_2("Processing lrm_rsc_t entry %s", rid);
 		
 		if(the_rsc == NULL) {
 			crm_err("NULL resource returned from the LRM");
 			continue;
 		}
 
 		crm_xml_add(xml_rsc, XML_ATTR_ID, the_rsc->id);
 		crm_xml_add(xml_rsc, XML_ATTR_TYPE, the_rsc->type);
 		crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, the_rsc->class);
 		crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER,the_rsc->provider);
 
 		op_list = the_rsc->ops->get_cur_state(the_rsc, &cur_state);
 
 		crm_debug_2("\tcurrent state:%s",
 			    cur_state==LRM_RSC_IDLE?"Idle":"Busy");
 
 		slist_iter(
 			op, lrm_op_t, op_list, llpc,
 
 			crm_debug_2("Processing op %s for %s (status=%d, rc=%d)", 
 				    op->op_type, the_rsc->id, op->op_status, op->rc);
 
 			if(max_call_id < op->call_id) {
 				build_operation_update(
 					xml_rsc, op, __FUNCTION__, llpc);
 
 			} else if(max_call_id > op->call_id) {
 				crm_err("Bad call_id in list=%d. Previous call_id=%d",
 					op->call_id, max_call_id);
 
 			} else {
 				crm_warn("lrm->get_cur_state() returned"
 					 " duplicate entries for call_id=%d",
 					 op->call_id);
 			}
 			max_call_id = op->call_id;
 			found_op = TRUE;
 			lrm_free_op(op);
 			);
 		if(found_op == FALSE && g_list_length(op_list) != 0) {
 			crm_err("Could not properly determin last op"
 				" for %s from %d entries", the_rsc->id,
 				g_list_length(op_list));
 		}
 
 		g_list_free(op_list);
 		lrm_free_rsc(the_rsc);
 		);
 
 	g_list_free(lrm_list);
 
 	return TRUE;
 }
 
 crm_data_t*
 do_lrm_query(gboolean is_replace)
 {
 	gboolean shut_down = FALSE;
 	crm_data_t *xml_result= NULL;
 	crm_data_t *xml_state = NULL;
 	crm_data_t *xml_data  = NULL;
 	crm_data_t *rsc_list  = NULL;
 	const char *exp_state = CRMD_JOINSTATE_MEMBER;
 
 	if(is_set(fsa_input_register, R_SHUTDOWN)) {
 		exp_state = CRMD_STATE_INACTIVE;
 		shut_down = TRUE;
 	}
 	
 	xml_state = create_node_state(
 		fsa_our_uname, ACTIVESTATUS, XML_BOOLEAN_TRUE,
 		ONLINESTATUS, CRMD_JOINSTATE_MEMBER, exp_state,
 		!shut_down, __FUNCTION__);
 
 	xml_data  = create_xml_node(xml_state, XML_CIB_TAG_LRM);
 	crm_xml_add(xml_data, XML_ATTR_ID, fsa_our_uuid);
 	rsc_list  = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES);
 
 	/* Build a list of active (not always running) resources */
 	build_active_RAs(rsc_list);
 
 	if(is_replace) {
 		crm_xml_add(xml_state, XML_CIB_ATTR_REPLACE, XML_CIB_TAG_LRM);
 	}
 
 	xml_result = create_cib_fragment(xml_state, XML_CIB_TAG_STATUS);
 	free_xml(xml_state);
 	
 	crm_log_xml_debug_3(xml_state, "Current state of the LRM");
 	
 	return xml_result;
 }
 
 
 static void
 delete_rsc_entry(const char *rsc_id) 
 {
 	crm_data_t *xml_top = NULL;
 	crm_data_t *xml_tmp = NULL;
 
 	/*
 	 * Remove the rsc from the CIB
 	 *
 	 * Avoids refreshing the entire LRM section of this host
 	 */
 	CRM_CHECK(rsc_id != NULL, return);
 	
 	xml_top = create_xml_node(NULL, XML_CIB_TAG_STATE);
 	crm_xml_add(xml_top, XML_ATTR_ID, fsa_our_uuid);
 	
 	xml_tmp = create_xml_node(xml_top, XML_CIB_TAG_LRM);
 	crm_xml_add(xml_tmp, XML_ATTR_ID, fsa_our_uuid);
 	
 	xml_tmp = create_xml_node(xml_tmp, XML_LRM_TAG_RESOURCES);
 	
 	xml_tmp = create_xml_node(xml_tmp, XML_LRM_TAG_RESOURCE);
 	crm_xml_add(xml_tmp, XML_ATTR_ID, rsc_id);
 
 	crm_debug("sync: Sending delete op for %s", rsc_id);
 	fsa_cib_conn->cmds->delete_absolute(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top,
 					    NULL, cib_quorum_override);
 	
 
 /* 	crm_log_xml_err(xml_top, "op:cancel"); */
 
  	free_xml(xml_top);
 }
 
 static void
 delete_op_entry(lrm_op_t *op, const char *rsc_id, const char *key, int call_id) 
 {
 	crm_data_t *xml_top = NULL;
 	/*
 	 * Remove the op from the CIB
 	 *
 	 * Avoids refreshing the entire LRM section of this host
 	 */
 	if(op != NULL) {
-		char *magic = NULL;
 		xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP);
-		magic = generate_transition_magic(op->user_data, op->op_status, op->rc);
-		crm_xml_add(xml_top, XML_ATTR_TRANSITION_MAGIC, magic);
-		crm_free(magic);
+		crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id);
+		crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data);
 		
 		crm_debug("async: Sending delete op for %s_%s_%d (call=%d)",
 			  op->rsc_id, op->op_type, op->interval, op->call_id);
 
 		fsa_cib_conn->cmds->delete(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top,
 					   NULL, cib_quorum_override);		
 
 	} else if (rsc_id != NULL && key != NULL) {
 		crm_data_t *xml_tmp = NULL;
 
 		xml_top = create_xml_node(NULL, XML_CIB_TAG_STATE);
 		crm_xml_add(xml_top, XML_ATTR_ID, fsa_our_uuid);
 
 		xml_tmp = create_xml_node(xml_top, XML_CIB_TAG_LRM);
 		crm_xml_add(xml_tmp, XML_ATTR_ID, fsa_our_uuid);
 
 		xml_tmp = create_xml_node(xml_tmp, XML_LRM_TAG_RESOURCES);
 
 		xml_tmp = create_xml_node(xml_tmp, XML_LRM_TAG_RESOURCE);
 		crm_xml_add(xml_tmp, XML_ATTR_ID, rsc_id);
 
 		xml_tmp = create_xml_node(xml_tmp, XML_LRM_TAG_RSC_OP);
 		crm_xml_add(xml_tmp, XML_ATTR_ID, key);
 		if(call_id > 0) {
 			crm_xml_add_int(xml_tmp, XML_LRM_ATTR_CALLID, call_id);
 		}
 		
 		crm_debug("sync: Sending delete op for %s (call=%d)", key, call_id);
 		fsa_cib_conn->cmds->delete_absolute(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top,
 						    NULL, cib_quorum_override);
 		
 	} else {
 		crm_err("Not enough information to delete op entry: rsc=%p key=%p", rsc_id, key);
 		return;
 	}
 
-/* 	crm_log_xml_err(xml_top, "op:cancel"); */
+ 	crm_log_xml_debug_2(xml_top, "op:cancel");
 
  	free_xml(xml_top);
 }
 
 static gboolean
 cancel_op(lrm_rsc_t *rsc, const char *key, int op, gboolean remove)
 {
 	int rc = HA_OK;
 
 	CRM_CHECK(op != 0, return FALSE);
 	CRM_CHECK(rsc != NULL, return FALSE);
 	if(key == NULL) {
 		key = "unknown";
 	}
 	
 	crm_debug("Cancelling op %d for %s (%s)", op, rsc->id, key);
 
 	rc = rsc->ops->cancel_op(rsc, op);
 	if(rc != HA_OK) {
 		crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc->id, key);
 		if(key && remove) {
 			delete_op_entry(NULL, rsc->id, key, op);
 		}
 	}
 	
 	return TRUE;
 }
 
 const char *cancel_key = NULL;
 gboolean cancel_done = FALSE;
 lrm_rsc_t *cancel_rsc = NULL;
 
 static void
 cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
 {
 	struct recurring_op_s *op = (struct recurring_op_s*)value;
 	
 	if(safe_str_eq(op->op_key, cancel_key)) {
 		cancel_done = TRUE;
 		cancel_op(cancel_rsc, key, op->call_id, TRUE);
 	}
 }
 
 static gboolean
 cancel_op_key(lrm_rsc_t *rsc, const char *key, gboolean remove)
 {
 	CRM_CHECK(rsc != NULL, return FALSE);
 	
 	cancel_key = key;
 	cancel_rsc = rsc;
 	cancel_done = FALSE;
 
 	CRM_CHECK(key != NULL, return FALSE);
 	
 	g_hash_table_foreach(pending_ops, cancel_action_by_key, NULL);
 
 	if(cancel_done == FALSE && remove) {
 		crm_err("No known %s operation to cancel", key);
 		delete_op_entry(NULL, rsc->id, key, 0);
 	}
 	
 	return cancel_done;
 }
 
 static lrm_rsc_t *
 get_lrm_resource(crm_data_t *resource, crm_data_t *op_msg, gboolean do_create)
 {
 	char rid[64];
 	lrm_rsc_t *rsc = NULL;
 	const char *short_id = ID(resource);
 	const char *long_id = crm_element_value(resource, XML_ATTR_ID_LONG);
 		
 	crm_debug_2("Retrieving %s from the LRM.", short_id);
 	CRM_CHECK(short_id != NULL, return NULL);
 	
 	if(rsc == NULL) {
 		/* check if its already there (short name) */
 		strncpy(rid, short_id, 64);
 		rid[63] = 0;
 		rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rid);
 	}
 	if(rsc == NULL && long_id != NULL) {
 		/* try the long name instead */
 		strncpy(rid, long_id, 64);
 		rid[63] = 0;
 		rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rid);
 	}
 
 	if(rsc == NULL && do_create) {
 		/* add it to the LRM */
 		const char *type = crm_element_value(resource, XML_ATTR_TYPE);
 		const char *class = crm_element_value(resource, XML_AGENT_ATTR_CLASS);
 		const char *provider = crm_element_value(resource, XML_AGENT_ATTR_PROVIDER);
 		GHashTable *params = xml2list(op_msg);
 
 		CRM_CHECK(class != NULL, return NULL);
 		CRM_CHECK(type != NULL, return NULL);
 
 		crm_debug("Adding rsc %s before operation", short_id);
 		strncpy(rid, short_id, 64);
 		rid[63] = 0;
 
 #if CRM_DEPRECATED_SINCE_2_0_3
 		if(op_msg != NULL) {
 			if(g_hash_table_lookup(
 				   params, XML_ATTR_CRM_VERSION) == NULL) {
 				g_hash_table_destroy(params);
 				params = xml2list_202(op_msg);
 			}
 		}
 #endif
 		if(g_hash_table_size(params) == 0) {
 			crm_log_xml_warn(op_msg, "EmptyParams");
 		}
 		
 		fsa_lrm_conn->lrm_ops->add_rsc(
 			fsa_lrm_conn, rid, class, type, provider, params);
 		
 		rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rid);
 		g_hash_table_destroy(params);
 
 		if(rsc == NULL) {
 			fsa_data_t *msg_data = NULL;
 			crm_err("Could not add resource %s to LRM", rid);
 			register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
 		}
 	}
 	return rsc;
 }
 
 /*	 A_LRM_INVOKE	*/
 enum crmd_fsa_input
 do_lrm_invoke(long long action,
 	      enum crmd_fsa_cause cause,
 	      enum crmd_fsa_state cur_state,
 	      enum crmd_fsa_input current_input,
 	      fsa_data_t *msg_data)
 {
 	gboolean create_rsc = TRUE;
 	const char *crm_op = NULL;
 	const char *from_sys = NULL;
 	const char *from_host = NULL;
 	const char *operation = NULL;
 	enum crmd_fsa_input next_input = I_NULL;
 	ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
 
 	crm_op    = cl_get_string(input->msg, F_CRM_TASK);
 	from_sys  = cl_get_string(input->msg, F_CRM_SYS_FROM);
 	if(safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
 		from_host = cl_get_string(input->msg, F_CRM_HOST_FROM);
 	}
 	
 	crm_debug_2("LRM command from: %s", from_sys);
 	
 	if(safe_str_eq(crm_op, CRM_OP_LRM_DELETE)) {
 		operation = CRMD_ACTION_DELETE;
 
 	} else if(safe_str_eq(operation, CRM_OP_LRM_REFRESH)) {
 		crm_op = CRM_OP_LRM_REFRESH;
 
 	} else if(safe_str_eq(crm_op, CRM_OP_LRM_FAIL)) {
 		crm_info("Failing resource...");
 		operation = "fail";
 
 	} else if(input->xml != NULL) {
 		operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK);
 	}
 
 	if(safe_str_eq(crm_op, CRM_OP_LRM_REFRESH)) {
 		enum cib_errors rc = cib_ok;
 		crm_data_t *fragment = do_lrm_query(TRUE);
 		crm_info("Forcing a local LRM refresh");
 
 		fsa_cib_update(XML_CIB_TAG_STATUS, fragment,
 			       cib_quorum_override, rc);
 		free_xml(fragment);
 		
 	} else if(safe_str_eq(crm_op, CRM_OP_LRM_QUERY)) {
 		crm_data_t *data = do_lrm_query(FALSE);
 		HA_Message *reply = create_reply(input->msg, data);
 
 		if(relay_message(reply, TRUE) == FALSE) {
 			crm_err("Unable to route reply");
 			crm_log_message(LOG_ERR, reply);
 			crm_msg_del(reply);
 		}
 		free_xml(data);
 
 	} else if(safe_str_eq(operation, CRM_OP_PROBED)
 		  || safe_str_eq(crm_op, CRM_OP_REPROBE)) {
 		const char *probed = XML_BOOLEAN_TRUE;
 		if(safe_str_eq(crm_op, CRM_OP_REPROBE)) {
 			probed = XML_BOOLEAN_FALSE;
 		}
 		
 		update_attr(fsa_cib_conn, cib_none, XML_CIB_TAG_STATUS,
 			    fsa_our_uuid, NULL, NULL, CRM_OP_PROBED, probed);
 
 	} else if(operation != NULL) {
 		lrm_rsc_t *rsc = NULL;
 		crm_data_t *params = NULL;
 		crm_data_t *xml_rsc = find_xml_node(
 			input->xml, XML_CIB_TAG_RESOURCE, TRUE);
 
 		CRM_CHECK(xml_rsc != NULL, return I_NULL);
 		
 		/* only the first 16 chars are used by the LRM */
 		params  = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE);
 
 		if(safe_str_eq(operation, CRMD_ACTION_DELETE)) {
 			create_rsc = FALSE;
 		}
 		
 		rsc = get_lrm_resource(xml_rsc, input->xml, create_rsc);
 
 		if(rsc == NULL && create_rsc) {
 			crm_err("Invalid resource definition");
 			crm_log_xml_warn(input->msg, "Bad command");
 
 		} else if(rsc == NULL) {
 			lrm_op_t* op = NULL;
 			crm_err("Not creating resource for a %s event: %s",
 				operation, ID(input->xml));
 			crm_log_xml_warn(input->msg, "Bad command");
 
 			op = construct_op(input->xml, ID(xml_rsc), operation);
 			op->op_status = LRM_OP_DONE;
 			op->rc = 0;
 			CRM_ASSERT(op != NULL);
 			send_direct_ack(from_host, from_sys, op, ID(xml_rsc));
 			free_lrm_op(op);			
 			
 		} else if(safe_str_eq(operation, CRMD_ACTION_CANCEL)) {
 			lrm_op_t* op = NULL;
 			char *op_key = NULL;
 			int call = 0;
 			const char *call_id = NULL;
 			const char *op_task = NULL;
 			const char *op_interval = NULL;
 
 			CRM_CHECK(params != NULL,
 				  crm_log_xml_warn(input->xml, "Bad command");
 				  return I_NULL);
 
 			op_interval = crm_element_value(params, crm_meta_name("interval"));
 			op_task = crm_element_value(params, crm_meta_name(XML_LRM_ATTR_TASK));
 			call_id = crm_element_value(params, crm_meta_name(XML_LRM_ATTR_CALLID));
 #if CRM_DEPRECATED_SINCE_2_0_5
 			if(op_interval == NULL) {
 				op_interval = crm_element_value(params, "interval");
 			}
 			if(op_task == NULL) {
 				op_task = crm_element_value(params, XML_LRM_ATTR_TASK);
 				if(op_task == NULL) {
 					op_task = crm_element_value(params, "task");
 				}
 			}
 #endif
 			CRM_CHECK(op_task != NULL,
 				  crm_log_xml_warn(input->xml, "Bad command");
 				  return I_NULL);
 			CRM_CHECK(op_interval != NULL,
 				  crm_log_xml_warn(input->xml, "Bad command");
 				  return I_NULL);
 
 			op = construct_op(input->xml, rsc->id, op_task);
 			CRM_ASSERT(op != NULL);
 			op_key = generate_op_key(
 				rsc->id,op_task,crm_parse_int(op_interval,"0"));
 
 			call = crm_parse_int(call_id, "0");
 			if(call == 0) {
 				cancel_op_key(rsc, op_key, TRUE);
 			} else {
 				cancel_op(rsc, op_key, call, TRUE);
 			}
 			
 			op->op_status = LRM_OP_DONE;
 			op->rc = EXECRA_OK;
 			send_direct_ack(from_host, from_sys, op, rsc->id);
 			crm_free(op_key);
 			free_lrm_op(op);			
 			
 		} else if(safe_str_eq(operation, CRMD_ACTION_DELETE)) {
 			int rc = HA_OK;
 			lrm_op_t* op = NULL;
 
 			op = construct_op(input->xml, rsc->id, operation);
 			CRM_ASSERT(op != NULL);
 			op->op_status = LRM_OP_DONE;
 			op->rc = EXECRA_OK;
 
 			if(rsc == NULL) {
 				crm_debug("Resource %s was already removed", rsc->id);
 
 			} else {
 				crm_info("Removing resource %s from the LRM", rsc->id);
 				rc = fsa_lrm_conn->lrm_ops->delete_rsc(fsa_lrm_conn, rsc->id);
 
 				if(rc != HA_OK) {
 					crm_err("Failed to remove resource %s", rsc->id);
 					op->op_status = LRM_OP_ERROR;
 					op->rc = EXECRA_UNKNOWN_ERROR;
 				}
 			}
 
 			delete_rsc_entry(rsc->id);
 			send_direct_ack(from_host, from_sys, op, rsc->id);
 			free_lrm_op(op);			
 
 			if(safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) {
 				/* this isn't expected - trigger a new transition */
 				time_t now = time(NULL);
 				char *now_s = crm_itoa(now);
 
 				crm_debug("Triggering a refresh after %s deleted %s from the LRM",
 					  from_sys, rsc->id);
 
 				update_attr(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG,
 					    NULL, NULL, NULL, "last-lrm-refresh", now_s);
 				crm_free(now_s);
 			}
 			
 			
 		} else if(rsc != NULL) {
 			next_input = do_lrm_rsc_op(
 				rsc, operation, input->xml, input->msg);
 		}
 		
 		lrm_free_rsc(rsc);
 
 	} else {
 		crm_err("Operation was neither a lrm_query, nor a rsc op.  %s",
 			crm_str(crm_op));
 		register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
 	}
 
 	return next_input;
 }
 
 lrm_op_t *
 construct_op(crm_data_t *rsc_op, const char *rsc_id, const char *operation)
 {
 	lrm_op_t *op = NULL;
 	const char *op_delay = NULL;
 	const char *op_timeout = NULL;
 	const char *op_interval = NULL;
 	
 	const char *transition = NULL;
 	CRM_DEV_ASSERT(rsc_id != NULL);
 
 	crm_malloc0(op, sizeof(lrm_op_t));
 	op->op_type   = crm_strdup(operation);
 	op->op_status = LRM_OP_PENDING;
 	op->rc = -1;
 	op->rsc_id = crm_strdup(rsc_id);
 	op->interval = 0;
 	op->timeout  = 0;
 	op->start_delay = 0;
 	op->app_name = crm_strdup(CRM_SYSTEM_CRMD);
 
 	if(rsc_op == NULL) {
 		CRM_DEV_ASSERT(safe_str_eq(CRMD_ACTION_STOP, operation));
 		op->user_data = NULL;
 		op->user_data_len = 0;
 		/* the stop_all_resources() case
 		 * by definition there is no DC (or they'd be shutting
 		 *   us down).
 		 * So we should put our version here.
 		 */
 		op->params = g_hash_table_new_full(
 			g_str_hash, g_str_equal,
 			g_hash_destroy_str, g_hash_destroy_str);
 		
 		g_hash_table_insert(op->params,
 				    crm_strdup(XML_ATTR_CRM_VERSION),
 				    crm_strdup(CRM_FEATURE_SET));
 
 		crm_debug_2("Constructed %s op for %s", operation, rsc_id);
 		return op;
 	}
 
 	op->params = xml2list(rsc_op);
 #if CRM_DEPRECATED_SINCE_2_0_3
 	if(g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION) == NULL) {
 		g_hash_table_destroy(op->params);
 		op->params = xml2list_202(rsc_op);
 	}
 #endif
 		
 	if(op->params == NULL) {
 		CRM_DEV_ASSERT(safe_str_eq(CRMD_ACTION_STOP, operation));
 	}
 
 	op_delay = g_hash_table_lookup(op->params, crm_meta_name("start_delay"));
 	op_timeout = g_hash_table_lookup(op->params, crm_meta_name("timeout"));
 	op_interval = g_hash_table_lookup(op->params, crm_meta_name("interval"));
 #if CRM_DEPRECATED_SINCE_2_0_5
 	if(op_delay == NULL) {
 		op_delay = g_hash_table_lookup(op->params, "start_delay");
 	}
 	if(op_timeout == NULL) {
 		op_timeout = g_hash_table_lookup(op->params, "timeout");
 	}
 	if(op_interval == NULL) {
 		op_interval = g_hash_table_lookup(op->params, "interval");
 	}
 #endif
 	
 	op->interval = crm_parse_int(op_interval, "0");
 	op->timeout  = crm_parse_int(op_timeout,  "0");
 	op->start_delay = crm_parse_int(op_delay, "0");
 
 	/* sanity */
 	if(op->interval < 0) {
 		op->interval = 0;
 	}
 	if(op->timeout < 0) {
 		op->timeout = 0;
 	}
 	if(op->start_delay < 0) {
 		op->start_delay = 0;
 	}
 
 	transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY);
 	CRM_CHECK(transition != NULL, return op);
 	
 	op->user_data = crm_strdup(transition);
 	op->user_data_len = 1+strlen(op->user_data);
 
 	if(op->interval != 0) {
 		if(safe_str_eq(operation, CRMD_ACTION_START)
 		   || safe_str_eq(operation, CRMD_ACTION_STOP)) {
 			crm_err("Start and Stop actions cannot have an interval");
 			op->interval = 0;
 		}
 	}
 
 	crm_debug_2("Constructed %s op for %s: interval=%d",
 		    operation, rsc_id, op->interval);	
 	
 	return op;
 }
 
 void
 send_direct_ack(const char *to_host, const char *to_sys,
 		lrm_op_t* op, const char *rsc_id)
 {
 	HA_Message *reply = NULL;
 	crm_data_t *update, *iter;
 	crm_data_t *fragment;
 	
 	CRM_DEV_ASSERT(op != NULL);
 	if(crm_assert_failed) {
 		return;
 	}
 	if(op->rsc_id == NULL) {
 		CRM_DEV_ASSERT(rsc_id != NULL);
 		op->rsc_id = crm_strdup(rsc_id);
 	}
 	if(to_sys == NULL) {
 		to_sys = CRM_SYSTEM_TENGINE;
 	}
 	update = create_node_state(
 		fsa_our_uname, NULL, NULL, NULL, NULL, NULL, FALSE, __FUNCTION__);
 
 	iter = create_xml_node(update, XML_CIB_TAG_LRM);
 	crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);
 	iter = create_xml_node(iter,   XML_LRM_TAG_RESOURCES);
 	iter = create_xml_node(iter,   XML_LRM_TAG_RESOURCE);
 
 	crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);
 
 	build_operation_update(iter, op, __FUNCTION__, 0);
 	fragment = create_cib_fragment(update, XML_CIB_TAG_STATUS);
 
 	reply = create_request(CRM_OP_INVOKE_LRM, fragment, to_host,
 			       to_sys, CRM_SYSTEM_LRMD, NULL);
 
 	crm_log_xml_debug_2(update, "ACK Update");
 
 	crm_info("ACK'ing resource op %s_%s_%d from %s: %s",
 		 op->rsc_id, op->op_type, op->interval, op->user_data,
 		 cl_get_string(reply, XML_ATTR_REFERENCE));
 
 	if(relay_message(reply, TRUE) == FALSE) {
 		crm_log_message_adv(LOG_ERR, "Unable to route reply", reply);
 		crm_msg_del(reply);
 	}
 	free_xml(fragment);
 	free_xml(update);
 }
 
 static void
 stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
 {
 	lrm_rsc_t *rsc = user_data;
 	struct recurring_op_s *op = (struct recurring_op_s*)value;
 	
 	if(op->interval != 0 && safe_str_eq(op->rsc_id, rsc->id)) {
 		cancel_op(rsc, key, op->call_id, FALSE);
 	}
 }
 
 enum crmd_fsa_input
 do_lrm_rsc_op(lrm_rsc_t *rsc, const char *operation,
 	      crm_data_t *msg, HA_Message *request)
 {
 	int call_id  = 0;
 	char *op_id  = NULL;
 	lrm_op_t* op = NULL;
 
 	fsa_data_t *msg_data = NULL;
 	const char *transition = NULL;	
 
 	CRM_CHECK(rsc != NULL, return I_NULL);
 	
 	if(msg != NULL) {
 		transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY);
 		if(transition == NULL) {
 			crm_err("Missing transition");
 			crm_log_message(LOG_ERR, msg);
 		}
 	}
 
 	op = construct_op(msg, rsc->id, operation);
 
 	/* stop the monitor before stopping the resource */
 	if(crm_str_eq(operation, CRMD_ACTION_STOP, TRUE)
 	   || crm_str_eq(operation, CRMD_ACTION_MIGRATE, TRUE)) {
 		g_hash_table_foreach(pending_ops, stop_recurring_action_by_rsc, rsc);
 	}
 	
 	/* now do the op */
 	crm_info("Performing op=%s_%s_%d key=%s)",
 		 rsc->id, operation, op->interval, transition);
 
 	if(fsa_state != S_NOT_DC && fsa_state != S_TRANSITION_ENGINE) {
 		if(safe_str_neq(operation, "fail")
 		   && safe_str_neq(operation, CRMD_ACTION_STOP)) {
 			crm_info("Discarding attempt to perform action %s on %s"
 				 " in state %s", operation, rsc->id,
 				 fsa_state2string(fsa_state));
 			op->rc = 99;
 			op->op_status = LRM_OP_ERROR;
 			send_direct_ack(NULL, NULL, op, rsc->id);
 			free_lrm_op(op);
 			crm_free(op_id);
 			return I_NULL;
 		}
 	}
 
 	op_id = generate_op_key(rsc->id, op->op_type, op->interval);
 
 	if(op->interval > 0) {
 		/* cancel it so we can then restart it without conflict */
 		cancel_op_key(rsc, op_id, FALSE);
 		op->target_rc = CHANGED;
 
 	} else {
 		op->target_rc = EVERYTIME;
 	}
 
 	g_hash_table_replace(resources,crm_strdup(rsc->id), crm_strdup(op_id));
 	call_id = rsc->ops->perform_op(rsc, op);
 
 	if(call_id <= 0) {
 		crm_err("Operation %s on %s failed: %d",
 			operation, rsc->id, call_id);
 		register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
 
 	} else {
 		/* record all operations so we can wait
 		 * for them to complete during shutdown
 		 */
 		char *call_id_s = make_stop_id(rsc->id, call_id);
 		struct recurring_op_s *pending = NULL;
 		crm_malloc0(pending, sizeof(struct recurring_op_s));
 		crm_debug("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);
 		
 		pending->call_id  = call_id;
 		pending->interval = op->interval;
 		pending->op_key   = crm_strdup(op_id);
 		pending->rsc_id   = crm_strdup(rsc->id);
 		g_hash_table_replace(pending_ops, call_id_s, pending);
 	}
 
 	crm_free(op_id);
 	free_lrm_op(op);		
 	return I_NULL;
 }
 
 void
 free_recurring_op(gpointer value)
 {
 	struct recurring_op_s *op = (struct recurring_op_s*)value;
 	crm_free(op->rsc_id);
 	crm_free(op->op_key);
 	crm_free(op);
 }
 
 
 void
 free_lrm_op(lrm_op_t *op) 
 {
 	g_hash_table_destroy(op->params);
 	crm_free(op->user_data);
 	crm_free(op->output);
 	crm_free(op->rsc_id);
 	crm_free(op->op_type);
 	crm_free(op->app_name);
 	crm_free(op);	
 }
 
 
 static void dup_attr(gpointer key, gpointer value, gpointer user_data)
 {
 	g_hash_table_replace(user_data, crm_strdup(key), crm_strdup(value));
 }
 
 lrm_op_t *
 copy_lrm_op(const lrm_op_t *op)
 {
 	lrm_op_t *op_copy = NULL;
 
 	CRM_DEV_ASSERT(op != NULL);
 	if(crm_assert_failed) {
 		return NULL;
 	}
 	CRM_ASSERT(op->rsc_id != NULL);
 
 	crm_malloc0(op_copy, sizeof(lrm_op_t));
 
 	op_copy->op_type = crm_strdup(op->op_type);
  	/* input fields */
 	op_copy->params = g_hash_table_new_full(
 		g_str_hash, g_str_equal,
 		g_hash_destroy_str, g_hash_destroy_str);
 	
 	if(op->params != NULL) {
 		g_hash_table_foreach(op->params, dup_attr, op_copy->params);
 	}
 	op_copy->timeout   = op->timeout;
 	op_copy->interval  = op->interval; 
 	op_copy->target_rc = op->target_rc; 
 
 	/* in the CRM, this is always a string */
 	if(op->user_data != NULL) {
 		op_copy->user_data = crm_strdup(op->user_data); 
 	}
 	
 	/* output fields */
 	op_copy->op_status = op->op_status; 
 	op_copy->rc        = op->rc; 
 	op_copy->call_id   = op->call_id; 
 	op_copy->output    = NULL;
 	op_copy->rsc_id    = crm_strdup(op->rsc_id);
 	if(op->app_name != NULL) {
 		op_copy->app_name  = crm_strdup(op->app_name);
 	}
 	if(op->output != NULL) {
 		op_copy->output = crm_strdup(op->output);
 	}
 	
 	return op_copy;
 }
 
 
 lrm_rsc_t *
 copy_lrm_rsc(const lrm_rsc_t *rsc)
 {
 	lrm_rsc_t *rsc_copy = NULL;
 
 	if(rsc == NULL) {
 		return NULL;
 	}
 	
 	crm_malloc0(rsc_copy, sizeof(lrm_rsc_t));
 
 	rsc_copy->id       = crm_strdup(rsc->id);
 	rsc_copy->type     = crm_strdup(rsc->type);
 	rsc_copy->class    = NULL;
 	rsc_copy->provider = NULL;
 
 	if(rsc->class != NULL) {
 		rsc_copy->class    = crm_strdup(rsc->class);
 	}
 	if(rsc->provider != NULL) {
 		rsc_copy->provider = crm_strdup(rsc->provider);
 	}
 /* 	GHashTable* 	params; */
 	rsc_copy->params = NULL;
 	rsc_copy->ops    = NULL;
 
 	return rsc_copy;
 }
 
 static void
 cib_rsc_callback(const HA_Message *msg, int call_id, int rc,
 		 crm_data_t *output, void *user_data)
 {
 	if(rc != cib_ok) {
 		crm_err("Resource update %d failed: %s",
 			call_id, cib_error2string(rc));	
 	} else {
 		crm_debug("Resource update %d complete", call_id);	
 	}
 }
 
 
 void
 do_update_resource(lrm_op_t* op)
 {
 /*
   <status>
     <nodes_status id=uname>
       <lrm>
         <lrm_resources>
           <lrm_resource id=...>
           </...>
 */
 	int rc = cib_ok;
 	lrm_rsc_t *rsc = NULL;
 	crm_data_t *update, *iter;
 	
 	CRM_CHECK(op != NULL, return);
 
 	update = create_node_state(
 		fsa_our_uname, NULL, NULL, NULL, NULL, NULL, FALSE, __FUNCTION__);
 
 	iter = create_xml_node(update, XML_CIB_TAG_LRM);
 	crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid);
 	iter = create_xml_node(iter,   XML_LRM_TAG_RESOURCES);
 	iter = create_xml_node(iter,   XML_LRM_TAG_RESOURCE);
 
 	crm_xml_add(iter, XML_ATTR_ID, op->rsc_id);
 		
 	rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, op->rsc_id);
 	
 	crm_xml_add(iter, XML_ATTR_TYPE, rsc->type);
 	crm_xml_add(iter, XML_AGENT_ATTR_CLASS, rsc->class);
 	crm_xml_add(iter, XML_AGENT_ATTR_PROVIDER,rsc->provider);
 	
 	lrm_free_rsc(rsc);
 	
 	build_operation_update(iter, op, __FUNCTION__, 0);
 
 	/* make it an asyncronous call and be done with it
 	 *
 	 * Best case:
 	 *   the resource state will be discovered during
 	 *   the next signup or election.
 	 *
 	 * Bad case:
 	 *   we are shutting down and there is no DC at the time,
 	 *   but then why were we shutting down then anyway?
 	 *   (probably because of an internal error)
 	 *
 	 * Worst case:
 	 *   we get shot for having resources "running" when the really weren't
 	 *
 	 * the alternative however means blocking here for too long, which
 	 * isnt acceptable
 	 */
 	fsa_cib_update(XML_CIB_TAG_STATUS, update, cib_quorum_override, rc);
 			
 	if(rc > 0) {
 		/* the return code is a call number, not an error code */
 		crm_debug("Sent resource state update message: %d", rc);
 		add_cib_op_callback(rc, FALSE, NULL, cib_rsc_callback);
 		
 	} else {
 		crm_err("Resource state update failed: %s",
 			cib_error2string(rc));	
 	}
 	
 	free_xml(update);
 }
 
 enum crmd_fsa_input
 do_lrm_event(long long action,
 	     enum crmd_fsa_cause cause,
 	     enum crmd_fsa_state cur_state,
 	     enum crmd_fsa_input cur_input,
 	     fsa_data_t *msg_data)
 {
 	lrm_op_t *op = NULL;
 	
 	CRM_CHECK(msg_data->fsa_cause == C_LRM_OP_CALLBACK, return I_NULL);
 
 	op = fsa_typed_data(fsa_dt_lrm);
 	process_lrm_event(op);
 
 	return I_NULL;
 }
 
 
 gboolean
 process_lrm_event(lrm_op_t *op)
 {
 	char *op_id = NULL;
 	char *op_key = NULL;
 	int log_level = LOG_ERR;
 	CRM_CHECK(op != NULL, return I_NULL);
 	CRM_CHECK(op->rsc_id != NULL, return I_NULL);
 
 	if(op->rc == EXECRA_RUNNING_MASTER || op->rc == EXECRA_NOT_RUNNING) {
 		/* Leave it up to the TE/PE to decide if this is an error */ 
 		op->op_status = LRM_OP_DONE;
 	}
 
 	op_key = generate_op_key(op->rsc_id, op->op_type, op->interval);
 	
 	switch(op->op_status) {
 		case LRM_OP_ERROR:
 		case LRM_OP_PENDING:
 		case LRM_OP_NOTSUPPORTED:
 			break;
 		case LRM_OP_CANCELLED:
 			log_level = LOG_INFO;
 			break;
 		case LRM_OP_DONE:
 			log_level = LOG_INFO;
 			break;
 		case LRM_OP_TIMEOUT:
 			log_level = LOG_DEBUG_3;
 			crm_err("LRM operation %s (%d) %s (timeout=%dms)",
 				op_key, op->call_id,
 				op_status2text(op->op_status), op->timeout);
 			/* set op->rc because the lrm doesn't bother */
 			op->rc = -1;
 			break;
 		default:
 			crm_err("Mapping unknown status (%d) to ERROR",
 				op->op_status);
 			op->op_status = LRM_OP_ERROR;
 	}
 
 	do_crm_log(log_level, "LRM operation %s (call=%d, rc=%d) %s %s",
 		   op_key, op->call_id, op->rc, op_status2text(op->op_status),
 		   op->op_status==LRM_OP_ERROR?execra_code2string(op->rc):"");
 
 	if(op->op_status == LRM_OP_ERROR && op->output != NULL) {
 		crm_info("Result: %s", op->output);
 	}
 	
 	if(op->op_status != LRM_OP_CANCELLED) {
 		do_update_resource(op);
 		if(op->interval != 0) {
 			goto out;
 		}
 		
 	} else if(op->interval == 0) {
 		/* no known valid reason for this to happen */
 		crm_err("Op %s (call=%d): Cancelled", op_key, op->call_id);
 
 	} else if(op->user_data != NULL) {
-		delete_op_entry(op, NULL, NULL, 0);
+		delete_op_entry(op, NULL, NULL, op->call_id);
 		
 	} else {
 		crm_err("Op %s (call=%d): No user data", op_key, op->call_id);
 	}	
 
 	op_id = make_stop_id(op->rsc_id, op->call_id);
 	if(g_hash_table_remove(pending_ops, op_id)) {
 		crm_debug("Op %s (call=%d): Confirmed", op_key, op->call_id);
 		goto out;
 	}
 
 	crm_err("Op %s (call=%d): Not matched", op_key, op->call_id);
 
   out:
 	crm_free(op_key);
 	crm_free(op_id);
 	return TRUE;
 }
 
 char *
 make_stop_id(const char *rsc, int call_id)
 {
 	char *op_id = NULL;
 	crm_malloc0(op_id, strlen(rsc) + 34);
 	if(op_id != NULL) {
 		snprintf(op_id, strlen(rsc) + 34, "%s:%d", rsc, call_id);
 	}
 	return op_id;
 }
diff --git a/cts/LSBDummy.in b/cts/LSBDummy.in
index bc91e172b6..4386e6a414 100755
--- a/cts/LSBDummy.in
+++ b/cts/LSBDummy.in
@@ -1,11 +1,11 @@
 #!/bin/sh
 #
 # WARNING: This script is for CTS testing only
 #
 . @HB_RA_DIR@/hto-mapfuncs
 
 OCF_TYPE=Dummy
 OCF_RESOURCE_INSTANCE=LSBDummy
 export OCF_TYPE OCF_RESOURCE_INSTANCE
 
-ra_execocf $2
+ra_execocf $1
diff --git a/lib/crm/common/xml.c b/lib/crm/common/xml.c
index 9f4a577964..229cc82842 100644
--- a/lib/crm/common/xml.c
+++ b/lib/crm/common/xml.c
@@ -1,2553 +1,2557 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <lha_internal.h>
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 #include <time.h>
 #include <string.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <ctype.h>
 
 #include <clplumbing/ipc.h>
 #include <clplumbing/cl_log.h> 
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <clplumbing/md5.h>
 #if HAVE_BZLIB_H
 #  include <bzlib.h>
 #endif
 
 #define XML_BUFFER_SIZE	4096
 
 int is_comment_start(const char *input, size_t offset, size_t max);
 int is_comment_end(const char *input, size_t offset, size_t max);
 gboolean drop_comments(const char *input, size_t *offset, size_t max);
 
 void dump_array(
 	int log_level, const char *message, const char **array, int depth);
 
 int print_spaces(char *buffer, int spaces);
 
 int log_data_element(const char *function, const char *prefix, int log_level,
 		     int depth, const crm_data_t *data, gboolean formatted);
 
 int dump_data_element(
 	int depth, char **buffer, const crm_data_t *data, gboolean formatted);
 
 crm_data_t *parse_xml(const char *input, size_t *offset);
 int get_tag_name(const char *input, size_t offset, size_t max);
 int get_attr_name(const char *input, size_t offset, size_t max);
 int get_attr_value(const char *input, size_t offset, size_t max);
 gboolean can_prune_leaf(crm_data_t *xml_node);
 
 void diff_filter_context(int context, int upper_bound, int lower_bound,
 		    crm_data_t *xml_node, crm_data_t *parent);
 int in_upper_context(int depth, int context, crm_data_t *xml_node);
 
 crm_data_t *
 find_xml_node(crm_data_t *root, const char * search_path, gboolean must_find)
 {
 	if(must_find || root != NULL) {
 		crm_validate_data(root);
 	}
 	
 	if(search_path == NULL) {
 		crm_warn("Will never find <NULL>");
 		return NULL;
 	}
 	
 	xml_child_iter_filter(
 		root, a_child, search_path,
 /* 		crm_debug_5("returning node (%s).", crm_element_name(a_child)); */
 		crm_log_xml(LOG_DEBUG_5, "found:", a_child);
 		crm_log_xml(LOG_DEBUG_6, "in:",    root);
 		crm_validate_data(a_child);
 		return a_child;
 		);
 
 	if(must_find) {
 		crm_warn("Could not find %s in %s.", search_path, crm_element_name(root));
 	} else if(root != NULL) {
 		crm_debug_3("Could not find %s in %s.", search_path, crm_element_name(root));
 	} else {
 		crm_debug_3("Could not find %s in <NULL>.", search_path);
 	}
 	
 	
 	return NULL;
 }
 
 crm_data_t*
 find_xml_node_nested(crm_data_t *root, const char **search_path, int len)
 {
 	int	j;
 	gboolean is_found = TRUE;
 	crm_data_t *match =  NULL;
 	crm_data_t *lastMatch = root;
 
 	crm_validate_data(root);
 	
 	if(search_path == NULL || search_path[0] == 0) {
 		crm_warn("Will never find NULL");
 		return NULL;
 	}
 	
 	dump_array(LOG_DEBUG_5, "Looking for.", search_path, len);
 
 	for (j=0; j < len; ++j) {
 		if (search_path[j] == NULL) {
 /* a NULL also means stop searching */
 			break;
 		}
 
 		match = find_xml_node(lastMatch, search_path[j], FALSE);
 		if(match == NULL) {
 			is_found = FALSE;
 			break;
 		} else {
 			lastMatch = match;
 		}
 	}
 
 	if (is_found) {
 		crm_debug_5("returning node (%s).",
 			   crm_element_name(lastMatch));
 
 		crm_log_xml_debug_5(lastMatch, "found\t%s");
 		crm_log_xml_debug_5(root, "in \t%s");
 		
 		crm_validate_data(lastMatch);
 		return lastMatch;
 	}
 
 	dump_array(LOG_DEBUG_2,
 		   "Could not find the full path to the node you specified.",
 		   search_path, len);
 
 	crm_debug_2("Closest point was node (%s) starting from %s.",
 		    crm_element_name(lastMatch), crm_element_name(root));
 
 	return NULL;
     
 }
 
 
 
 const char *
 get_xml_attr_nested(crm_data_t *parent,
 		    const char **node_path, int length,
 		    const char *attr_name, gboolean error)
 {
 	const char *attr_value = NULL;
 	crm_data_t *attr_parent = NULL;
 
 	if(error || parent != NULL) {
 		crm_validate_data(parent);
 	}
 	
 	if(parent == NULL) {
 		crm_debug_3("Can not find attribute %s in NULL parent",attr_name);
 		return NULL;
 	} 
 
 	if(attr_name == NULL || attr_name[0] == 0) {
 		crm_err("Can not find attribute with no name in %s",
 		       crm_element_name(parent));
 		return NULL;
 	}
 	
 	if(length == 0) {
 		attr_parent = parent;
 		
 	} else {
 		attr_parent = find_xml_node_nested(parent, node_path, length);
 		if(attr_parent == NULL && error) {
 			crm_err("No node at the path you specified.");
 			return NULL;
 		}
 	}
 	
 	attr_value = crm_element_value(attr_parent, attr_name);
 	if((attr_value == NULL || attr_value[0] == 0) && error) {
 		crm_err("No value present for %s at %s",
 			attr_name, crm_element_name(attr_parent));
 		return NULL;
 	}
 	
 	return attr_value;
 }
 
 
 crm_data_t*
 find_entity(crm_data_t *parent, const char *node_name, const char *id)
 {
 	crm_validate_data(parent);
 	xml_child_iter_filter(
 		parent, a_child, node_name,
 		if(id == NULL || crm_str_eq(id, ID(a_child), TRUE)) {
 			crm_debug_4("returning node (%s).", 
 				    crm_element_name(a_child));
 			return a_child;
 		}
 		);
 	crm_debug_3("node <%s id=%s> not found in %s.",
 		    node_name, id, crm_element_name(parent));
 	return NULL;
 }
 
 void
 copy_in_properties(crm_data_t* target, const crm_data_t *src)
 {
 	int value_len = 0;
 	char *incr_value = NULL;
 	char *new_value = NULL;
 	
 	crm_validate_data(src);
 	crm_validate_data(target);
 
 	if(src == NULL) {
 		crm_warn("No node to copy properties from");
 
 	} else if (target == NULL) {
 		crm_err("No node to copy properties into");
 
 	} else {
 		xml_prop_iter(
 			src, local_prop_name, local_prop_value,
 
 			/* if the value is name followed by "++" we need
 			 *   to increment the existing value
 			 */
 			new_value = NULL;
 			incr_value = NULL;
 
 			if(strstr(local_prop_value, "++") > local_prop_value) {
 				int old_int = 0;
 				const char *old_value = NULL;
 				value_len = strlen(local_prop_value);
 				crm_malloc0(incr_value, value_len+2);
 				sprintf(incr_value, "%s++", local_prop_name);
 
 				if(safe_str_eq(local_prop_value, incr_value)) {
 					old_value = crm_element_value(
 						target, local_prop_name);
 					old_int = crm_parse_int(old_value, "0");
 					new_value = crm_itoa(old_int + 1);
 					local_prop_value = new_value;
 				}
 			}
 
 			crm_xml_add(target, local_prop_name, local_prop_value);
 			crm_free(incr_value);
 			crm_free(new_value);
 			);
 		crm_validate_data(target);
 	}
 	
 	return;
 }
 
 crm_data_t*
 add_node_copy(crm_data_t *parent, const crm_data_t *src_node) 
 {
 	const char *name = NULL;
 	crm_data_t *child = NULL;
 	CRM_CHECK(src_node != NULL, return NULL);
 
 	crm_validate_data(src_node);
 
 	name = crm_element_name(src_node);
 	CRM_CHECK(name != NULL, return NULL);
 	
 	child = create_xml_node(parent, name);
 	copy_in_properties(child, src_node);
 
 	xml_child_iter(src_node, src_child,
 		       add_node_copy(child, src_child);
 		);
 	
 	return child;
 }
 
 
 int
 add_node_nocopy(crm_data_t *parent, const char *name, crm_data_t *child)
 {
 
 	int next = 0;
 	crm_validate_data(parent);
 	crm_validate_data(child);	
 
 	if(name == NULL) {
 		name = crm_element_name(child);
 	}
 	CRM_ASSERT(name != NULL && name[0] != 0);
 	
 	if (parent->nfields >= parent->nalloc
 		&& ha_msg_expand(parent) != HA_OK ){
 		crm_err("Parent expansion failed");
 		return HA_FAIL;
 	}
 
 	next = parent->nfields;
 	parent->names[next] = crm_strdup(name);
 	parent->nlens[next] = strlen(name);
 	parent->values[next] = child;
 	parent->vlens[next] = sizeof(struct ha_msg);
 	parent->types[next] = FT_UNCOMPRESS;
 	parent->nfields++;	
 	
 	return HA_OK;
 }
 
 const char *
 crm_xml_add(crm_data_t* node, const char *name, const char *value)
 {
 	const char *parent_name = NULL;
 
 	if(node != NULL) {
 		parent_name = crm_element_name(node);
 	}
 	
 	crm_debug_5("[%s] Setting %s to %s", crm_str(parent_name), name, value);
 
 	if (name == NULL || name[0] == 0) {
 		
 	} else if(node == NULL) {
 		
 	} else if(parent_name == NULL && strcasecmp(name, F_XML_TAGNAME) != 0) {
 		
 	} else if (value == NULL || value[0] == 0) {
 		xml_remove_prop(node, name);
 		return NULL;
 		
 	} else {
 		const char *new_value = NULL;
 		crm_validate_data(node);
 		ha_msg_mod(node, name, value);
 		new_value = crm_element_value(node, name);
 		return new_value;
 	}
 	
 	return NULL;
 }
 
 const char *
 crm_xml_add_int(crm_data_t* node, const char *name, int value)
 {
 	const char *parent_name = NULL;
 
 	if(node != NULL) {
 		parent_name = crm_element_name(node);
 	}
 
 	crm_debug_5("[%s] Setting %s to %d", crm_str(parent_name), name, value);
 
 	if (name == NULL || name[0] == 0) {
 		
 	} else if(node == NULL) {
 		
 	} else if(parent_name == NULL && strcasecmp(name, F_XML_TAGNAME) != 0) {
 		
 	} else {
 		crm_validate_data(node);
 		ha_msg_mod_int(node, name, value);
 		return crm_element_value(node, name);
 	}
 
 	return NULL;
 }
 
 crm_data_t*
 create_xml_node(crm_data_t *parent, const char *name)
 {
 	const char *local_name = NULL;
 	const char *parent_name = NULL;
 	crm_data_t *ret_value = NULL;
 	
 
 	if (name == NULL || name[0] == 0) {
 		ret_value = NULL;
 	} else {
 		local_name = name;
 		ret_value = ha_msg_new(3);
 		CRM_CHECK(ret_value != NULL, return NULL);
 		
 		crm_xml_add(ret_value, F_XML_TAGNAME, name);
 		crm_validate_data(ret_value);
 		if(parent) {
 			crm_validate_data(parent);
 			parent_name = crm_element_name(parent);
 			crm_debug_5("Attaching %s to parent %s",
 				   local_name, parent_name);
 			CRM_CHECK(HA_OK == ha_msg_addstruct(
 					  parent, name, ret_value), return NULL);
 			crm_msg_del(ret_value);
 
 			crm_validate_data(parent);
 			ret_value = parent->values[parent->nfields-1];
 		}
 	}
 
 	crm_debug_5("Created node [%s [%s]]",
 		  crm_str(parent_name), crm_str(local_name));
 	return ret_value;
 }
 
 void
 free_xml_from_parent(crm_data_t *parent, crm_data_t *a_node)
 {
 	CRM_CHECK(parent != NULL, return);
 	CRM_CHECK(a_node != NULL, return);
 
 	crm_validate_data(parent);
 	cl_msg_remove_value(parent, a_node);	
 	crm_validate_data(parent);
 }
 
 void
 add_xml_tstamp(crm_data_t *a_node)
 {
 	char *since_epoch = NULL;
 	time_t a_time = time(NULL);
 	
 	crm_validate_data(a_node);
 
 	if(a_time == (time_t)-1) {
 		cl_perror("set_node_tstamp(): Invalid time returned");
 		return;
 	}
 	
 	crm_malloc0(since_epoch, 128);
 	if(since_epoch != NULL) {
 		sprintf(since_epoch, "%ld", (unsigned long)a_time);
 		ha_msg_mod(a_node, XML_ATTR_TSTAMP, since_epoch);
 		crm_validate_data(a_node);
 		crm_free(since_epoch);
 	}
 }
 
 crm_data_t*
 copy_xml(const crm_data_t *src_node)
 {
 	return add_node_copy(NULL, src_node);
 }
 
 crm_data_t*
 string2xml(const char *input)
 {
 	crm_data_t *output = parse_xml(input, NULL);
 	if(output != NULL) {
 		crm_validate_data(output);
 	}
 	return output;
 }
 
 crm_data_t *
 stdin2xml(void) 
 {
  	size_t data_length = 0;
  	size_t read_chars = 0;
   
   	char *xml_buffer = NULL;
   	crm_data_t *xml_obj = NULL;
   
  	do {
  		crm_realloc(xml_buffer, XML_BUFFER_SIZE + data_length + 1);
  		read_chars = fread(xml_buffer + data_length, 1, XML_BUFFER_SIZE, stdin);
  		data_length += read_chars;
  	} while (read_chars > 0);
   	
  	xml_buffer[data_length] = '\0';
 
 	xml_obj = string2xml(xml_buffer);
 	crm_free(xml_buffer);
 
 	crm_log_xml_debug_3(xml_obj, "Created fragment");
 	return xml_obj;
 }
 
 
 crm_data_t*
 file2xml(FILE *input, gboolean compressed)
 {
 	char *buffer = NULL;
 	gboolean work_done = FALSE;
 	crm_data_t *new_obj = NULL;
 	size_t length = 0, read_len = 0;
 
 	if(input == NULL) {
 		/* Use perror here as we likely just called fopen() which return NULL */
 		cl_perror("File open failed, cannot read contents");
 		return NULL;
 	}
 
 	if(compressed) {
 #if HAVE_BZLIB_H
 		int rc = 0;
 		BZFILE *bz_file = BZ2_bzReadOpen(&rc, input, 0, 0, NULL, 0);
 		if ( rc != BZ_OK ) {
 			BZ2_bzReadClose ( &rc, bz_file);
 			return NULL;
 		}
 		
 		rc = BZ_OK;
 		while ( rc == BZ_OK ) {
 			crm_realloc(buffer, XML_BUFFER_SIZE + length + 1);
 			read_len = BZ2_bzRead (
 				&rc, bz_file, buffer + length, XML_BUFFER_SIZE);
 
 			crm_debug_5("Read %ld bytes from file: %d",
 				    (long)read_len, rc);
 
 			if ( rc == BZ_OK || rc == BZ_STREAM_END) {
 				length += read_len;
 			}
 		}
 
 		buffer[length] = '\0';
 		read_len = length;
 
 		if ( rc != BZ_STREAM_END ) {
 			crm_err("Couldnt read compressed xml from file");
 			crm_free(buffer);
 			buffer = NULL;
 		}
 
 		BZ2_bzReadClose (&rc, bz_file);
 		if(buffer == NULL) {
 			return NULL;
 		}
 
 		work_done = TRUE;
 #else
 		crm_err("Cannot read compressed files:"
 			" bzlib was not available at compile time");
 #endif
 	}	
 	
 	if(work_done == FALSE) {
 		int start = 0;
 		start  = ftell(input);
 		fseek(input, 0L, SEEK_END);
 		length = ftell(input);
 		fseek(input, 0L, start);
 		
 		CRM_ASSERT(start == ftell(input));
 		
 		crm_debug_3("Reading %ld bytes from file", (long)length);
 		crm_malloc0(buffer, (length+1));
 		read_len = fread(buffer, 1, length, input);
 	}
 
 	/* see how big the file is */
 	if(read_len != length) {
 		crm_err("Calculated and read bytes differ: %ld vs. %ld",
 			(long)length, (long)read_len);
 	} else if(length > 0) {
 		new_obj = string2xml(buffer);
 	} else {
 		crm_warn("File contained no XML");
 	}
 	
 	crm_free(buffer);
 	return new_obj;
 }
 
 void
 dump_array(int log_level, const char *message, const char **array, int depth)
 {
 	int j;
 	
 	if(message != NULL) {
 		do_crm_log(log_level, "%s", message);
 	}
 
 	do_crm_log(log_level,  "Contents of the array:");
 	if(array == NULL || array[0] == NULL || depth == 0) {
 		do_crm_log(log_level, "\t<empty>");
 		return;
 	}
 	
 	for (j=0; j < depth && array[j] != NULL; j++) {
 		if (array[j] == NULL) { break; }
 		do_crm_log(log_level, "\t--> (%s).", array[j]);
 	}
 }
 
 int
 write_xml_file(crm_data_t *xml_node, const char *filename, gboolean compress) 
 {
 	int res = 0;
 	time_t now;
 	char *buffer = NULL;
 	char *now_str = NULL;
 	gboolean is_done = FALSE;
 	FILE *file_output_strm = NULL;
 	static mode_t cib_mode = S_IRUSR|S_IWUSR;
 	
 	CRM_CHECK(filename != NULL, return -1);
 
 	crm_debug_3("Writing XML out to %s", filename);
 	crm_validate_data(xml_node);
 	if (xml_node == NULL) {
 		crm_err("Cannot write NULL to %s", filename);
 		return -1;
 	}
 
 	crm_validate_data(xml_node);
 	crm_log_xml_debug_4(xml_node, "Writing out");
 	crm_validate_data(xml_node);
 	
 	now = time(NULL);
 	now_str = ctime(&now);
 	now_str[24] = EOS; /* replace the newline */
 	crm_xml_add(xml_node, XML_CIB_ATTR_WRITTEN, now_str);
 	crm_validate_data(xml_node);
 	
 	buffer = dump_xml_formatted(xml_node);
 	CRM_CHECK(buffer != NULL && strlen(buffer) > 0, return -1);
 
 	/* establish the file with correct permissions */
 	file_output_strm = fopen(filename, "w");
 	if(file_output_strm == NULL) {
 		cl_perror("Cannot open %s for writing", filename);
 		crm_free(buffer);
 		return -1;
 	}
 	
 	fclose(file_output_strm);
 	chmod(filename, cib_mode);
 
 	/* now write it */
 	file_output_strm = fopen(filename, "w");
 	if(file_output_strm == NULL) {
 		cl_perror("Cannot open %s for writing", filename);
 		crm_free(buffer);
 		return -1;		
 	} 
 
 	if(compress) {
 #if HAVE_BZLIB_H
 		int rc = BZ_OK;
 		BZFILE *bz_file = NULL;
 		unsigned int in = 0, out = 0;
 		is_done = TRUE;
 		bz_file = BZ2_bzWriteOpen(&rc, file_output_strm, 5,0,0);
 		if(rc != BZ_OK) {
 			is_done = FALSE;
 			crm_err("bzWriteOpen failed: %d", rc);
 		}
 		if(is_done) {
 			BZ2_bzWrite(&rc,bz_file,buffer,strlen(buffer));
 			if(rc != BZ_OK) {
 				crm_err("bzWrite() failed: %d", rc);
 				is_done = FALSE;
 			}
 		}
 		if(is_done) {
 			BZ2_bzWriteClose(&rc, bz_file, 0, &in, &out);
 			if(rc != BZ_OK) {
 				crm_err("bzWriteClose() failed: %d",rc);
 				is_done = FALSE;
 			} else {
 				crm_debug_2("%s: In: %d, out: %d",
 					    filename, in, out);
 			}
 		}
 #else
 		crm_err("Cannot write compressed files:"
 			" bzlib was not available at compile time");		
 #endif
 	}
 	
 	if(is_done == FALSE) {
 		res = fprintf(file_output_strm, "%s", buffer);
 		if(res < 0) {
 			cl_perror("Cannot write output to %s", filename);
 		}
 		
 		if(fflush(file_output_strm) == EOF || fsync(fileno(file_output_strm)) < 0) {
 			cl_perror("fflush or fsync error on %s", filename);
 			fclose(file_output_strm);
 			crm_free(buffer);
 			return -1;
 		}
 	}
 	fclose(file_output_strm);
 	crm_free(buffer);
 
 	crm_debug_3("Saved %d bytes to the Cib as XML", res);
 
 	return res;
 }
 
 void
 print_xml_formatted(int log_level, const char *function,
 		    const crm_data_t *msg, const char *text)
 {
 	if(msg == NULL) {
 		do_crm_log(log_level, "%s: %s: NULL", function, crm_str(text));
 		return;
 	}
 
 	crm_validate_data(msg);
 	log_data_element(function, text, log_level, 0, msg, TRUE);
 	return;
 }
 
 crm_data_t *
 get_message_xml(HA_Message *msg, const char *field) 
 {
 	crm_data_t *xml_node = NULL;
 	crm_data_t *tmp_node = NULL;
 	crm_validate_data(msg);
 	tmp_node = cl_get_struct(msg, field);
 	if(tmp_node != NULL) {
 		xml_node = copy_xml(tmp_node);
 	}
 	return xml_node;
 }
 
 gboolean
 add_message_xml(HA_Message *msg, const char *field, const crm_data_t *xml) 
 {
 	crm_validate_data(xml);
 	crm_validate_data(msg);
 	CRM_CHECK(field != NULL, return FALSE);
 	ha_msg_addstruct_compress(msg, field, xml);
 	return TRUE;
 }
 
 
 char *
 dump_xml_formatted(const crm_data_t *an_xml_node)
 {
 	char *buffer     = NULL;
 	char *mutable_ptr = NULL;
 	if(an_xml_node == NULL) {
 		return NULL;
 	}
   	crm_malloc0(buffer, 3*get_stringlen(an_xml_node));
 	mutable_ptr = buffer;
 	
 	crm_validate_data(an_xml_node);
 	CRM_CHECK(dump_data_element(
 			  0, &mutable_ptr, an_xml_node, TRUE) >= 0,
 		  crm_crit("Could not dump the whole message"));
 	crm_debug_4("Dumped: %s", buffer);
 	return buffer;
 }
 	
 char *
 dump_xml_unformatted(const crm_data_t *an_xml_node)
 {
 	char *buffer     = NULL;
 	char *mutable_ptr = NULL;
   	crm_malloc0(buffer, 2*get_stringlen(an_xml_node));
 	mutable_ptr = buffer;
 	
 	crm_validate_data(an_xml_node);
 	CRM_CHECK(dump_data_element(
 			  0, &mutable_ptr, an_xml_node, FALSE) >= 0,
 		  crm_crit("Could not dump the whole message"));
 
 	crm_debug_4("Dumped: %s", buffer);
 	return buffer;
 }
 
 #define update_buffer_head(buffer, len) if(len < 0) {	\
 		(*buffer) = EOS; return -1;		\
 	} else {					\
 		buffer += len;				\
 	}
 
 
 int
 print_spaces(char *buffer, int depth) 
 {
 	int lpc = 0;
 	int spaces = 2*depth;
 	/* <= so that we always print 1 space - prevents problems with syslog */
 	for(lpc = 0; lpc <= spaces; lpc++) {
 		if(sprintf(buffer, "%c", ' ') < 1) {
 			return -1;
 		}
 		buffer += 1;
 	}
 	return lpc;
 }
 
 int
 log_data_element(
 	const char *function, const char *prefix, int log_level, int depth,
 	const crm_data_t *data, gboolean formatted) 
 {
 	int printed = 0;
 	int child_result = 0;
 	int has_children = 0;
 	char print_buffer[1000];
 	char *buffer = print_buffer;
 	const char *name = crm_element_name(data);
 	const char *hidden = NULL;	
 
 	crm_debug_5("Dumping %s...", name);
 	crm_validate_data(data);
 	if(data == NULL) {
 		crm_warn("No data to dump as XML");
 		return 0;
 
 	} else if(name == NULL && depth == 0) {
 		xml_child_iter(
 			data, a_child, 
 			child_result = log_data_element(
 				function, prefix, log_level, depth, a_child, formatted);
 			if(child_result < 0) {
 				return child_result;
 			}
 			);
 		return 0;
 
 	} else if(name == NULL) {
 		crm_err("Cannot dump NULL element at depth %d", depth);
 		return -1;
 	}
 	
 	if(formatted) {
 		printed = print_spaces(buffer, depth);
 		update_buffer_head(buffer, printed);
 	}
 	
 	printed = sprintf(buffer, "<%s", name);
 	update_buffer_head(buffer, printed);
 
 	hidden = crm_element_value(data, "hidden");
 	xml_prop_iter(
 		data, prop_name, prop_value,
 
 		if(prop_name == NULL) {
 			CRM_ASSERT(prop_name != NULL);
 			
 		} else if(safe_str_eq(F_XML_TAGNAME, prop_name)) {
 			continue;
 
 		} else if(hidden != NULL
 			  && prop_name[0] != 0
 			  && strstr(hidden, prop_name) != NULL) {
 			prop_value = "*****";
 		}
 		
 		crm_debug_5("Dumping <%s %s=\"%s\"...",
 			    name, prop_name, prop_value);
 		printed = sprintf(buffer, " %s=\"%s\"", prop_name, prop_value);
 		update_buffer_head(buffer, printed);
 		);
 
 	xml_child_iter(
 		data, child, 
 		if(child != NULL) {
 			has_children++;
 			break;
 		}
 		);
 
 	printed = sprintf(buffer, "%s>", has_children==0?"/":"");
 	update_buffer_head(buffer, printed);
 	do_crm_log(log_level, "%s: %s%s",
 		   function, prefix?prefix:"", print_buffer);
 	buffer = print_buffer;
 	
 	if(has_children == 0) {
 		return 0;
 	}
 	
 	xml_child_iter(
 		data, a_child, 
 		child_result = log_data_element(
 			function, prefix, log_level, depth+1, a_child, formatted);
 
 		if(child_result < 0) { return -1; }
 		);
 
 	if(formatted) {
 		printed = print_spaces(buffer, depth);
 		update_buffer_head(buffer, printed);
 	}
 	do_crm_log(log_level, "%s: %s%s</%s>",
 		   function, prefix?prefix:"", print_buffer, name);
 	crm_debug_5("Dumped %s...", name);
 
 	return has_children;
 }
 
 
 int
 dump_data_element(
 	int depth, char **buffer,  const crm_data_t *data, gboolean formatted) 
 {
 	int printed = 0;
 	int child_result = 0;
 	int has_children = 0;
 	const char *name = NULL;
 
 	CRM_ASSERT(data != NULL);
 	CRM_ASSERT(buffer != NULL && *buffer != NULL);
 
 	name = crm_element_name(data);
 	if(name == NULL && depth == 0) {
 		name = "__fake__";
 	}
 	CRM_ASSERT(name != NULL);
 	crm_debug_5("Dumping %s...", name);
 
 	if(formatted) {
 		printed = print_spaces(*buffer, depth);
 		update_buffer_head(*buffer, printed);
 	}
 	
 	printed = sprintf(*buffer, "<%s", name);
 	update_buffer_head(*buffer, printed);
 	has_children = xml_has_children(data);
 
 	xml_prop_iter(data, prop_name, prop_value,
 		      crm_debug_5("Dumping <%s %s=\"%s\"...",
 				  name, prop_name, prop_value);
 		      printed = sprintf(*buffer, " %s=\"%s\"", prop_name, prop_value);
 		      update_buffer_head(*buffer, printed);
 		);
 
 	printed = sprintf(*buffer, "%s>%s",
 			  has_children==0?"/":"", formatted?"\n":"");
 	update_buffer_head(*buffer, printed);
 
 	if(has_children == 0) {
 		return 0;
 	}
 	
 	xml_child_iter(
 		data, child, 
 		child_result = dump_data_element(
 			depth+1, buffer, child, formatted);
 
 		if(child_result < 0) { return -1; }
 		);
 
 	if(formatted) {
 		printed = print_spaces(*buffer, depth);
 		update_buffer_head(*buffer, printed);
 	}
 	printed = sprintf(*buffer, "</%s>%s", name, formatted?"\n":"");
 	update_buffer_head(*buffer, printed);
 	crm_debug_5("Dumped %s...", name);
 
 	return has_children;
 }
 
 gboolean
 xml_has_children(const crm_data_t *xml_root)
 {
 	if(xml_root != NULL) {
 		xml_child_iter(
 			xml_root, a_child, 
 			return TRUE;
 			);
 	}
 	return FALSE;
 }
 
 void
 xml_validate(const crm_data_t *xml_root)
 {
 	int lpc = 0;
 	CRM_ASSERT(xml_root != NULL);
 	CRM_ASSERT(cl_is_allocated(xml_root) == 1);
 	CRM_ASSERT(xml_root->nfields < 500);
 	
 	for (lpc = 0; lpc < xml_root->nfields; lpc++) {
 		void *child = xml_root->values[lpc];
 		CRM_ASSERT(cl_is_allocated(xml_root->names[lpc]) == 1);
 
 		if(child == NULL) {
 			
 		} else if(xml_root->types[lpc] == FT_STRUCT
 			  || xml_root->types[lpc] == FT_UNCOMPRESS) {
 			crm_validate_data(child);
 			
 		} else if(xml_root->types[lpc] == FT_STRING) {
 			CRM_ASSERT(cl_is_allocated(child) == 1);
 /* 		} else { */
 /* 			CRM_CHECK(FALSE); */
 		}
 	}
 }
 
 const char *
 crm_element_value(const crm_data_t *data, const char *name)
 {
 	const char *value = NULL;
 	crm_validate_data(data);
 	value = cl_get_string(data, name);
 #if XML_PARANOIA_CHECKS
 	CRM_CHECK(value == NULL || cl_is_allocated(value) == 1, return NULL);
 #endif
 	return value;
 }
 
 char *
 crm_element_value_copy(const crm_data_t *data, const char *name)
 {
 	char *value_copy = NULL;
 	const char *value = crm_element_value(data, name);
 	if(value != NULL) {
 		value_copy = crm_strdup(value);
 	}
 	return value_copy;
 }
 
 const char *
 crm_element_name(const crm_data_t *data)
 {
 #if CRM_DEV_BUILD
 	crm_validate_data(data);
 #endif
 	return cl_get_string(data, F_XML_TAGNAME);
 }
 
 void
 xml_remove_prop(crm_data_t *obj, const char *name)
 {
 	if(crm_element_value(obj, name) != NULL) {
 		cl_msg_remove(obj, name);
 	}
 }
 
 int
 get_tag_name(const char *input, size_t offset, size_t max) 
 {
 	char ch = 0;
 	size_t lpc = offset;
 
 	const char *error = NULL;
 	gboolean do_special = FALSE;
 	
 	for(; error == NULL && lpc < max; lpc++) {
 		ch = input[lpc];
 		crm_debug_5("Processing char %c [%d]", ch, (int)lpc);
 
 		switch(ch) {
 			case 0:
 				error = "unexpected EOS";
 				break;
 			case '?':
 				if(lpc == 0) {
 					/* weird xml tag that we dont care about */
 					do_special = TRUE;
 				} else {
 					goto out;
 				}
 				break;
 			case '/':
 			case '>':
 			case '\t':
 			case '\n':
 			case ' ':
 				if(!do_special) {
 					goto out;
 				}
 				break;
 			default:
 				if(do_special) {
 
 				} else if('a' <= ch && ch <= 'z') {
 				} else if('A' <= ch && ch <= 'Z') {
 				} else if(ch == '_') {
 				} else if(ch == '-') {
 				} else {
 					error = "bad character, not in [a-zA-Z_-]";
 				}
  				break;
 		}
 	}
 	crm_err("Error parsing token near %.15s: %s", input, crm_str(error));
 	return -1;
   out:
 	CRM_ASSERT(lpc > offset);
 	return lpc - offset;
 }
 
 int
 get_attr_name(const char *input, size_t offset, size_t max) 
 {
 	char ch = 0;
 	size_t lpc = offset;
 	const char *error = NULL;
 	
 	for(; error == NULL && lpc < max; lpc++) {
 		ch = input[lpc];
 		crm_debug_5("Processing char %c[%d]", ch, (int)lpc);
 
 		switch(ch) {
 			case 0:
 				error = "unexpected EOS";
  				break;
 			case '\t':
 			case '\n':
 			case ' ':
 				error = "unexpected whitespace";
  				break;
 			case '=':
 				return lpc - offset;
 			default:
 				if('a' <= ch && ch <= 'z') {
 				} else if('A' <= ch && ch <= 'Z') {
 				} else if('0' <= ch && ch <= '9') {
 				} else if(ch == '_') {
 				} else if(ch == '-') {
 				} else {
 					error = "bad character, not in [a-zA-Z0-9_-]";
 				}
  				break;
 		}
 	}
 	crm_err("Error parsing token near %.15s: %s", input, crm_str(error));
 	return -1;
 }
 
 int
 get_attr_value(const char *input, size_t offset, size_t max) 
 {
 	char ch = 0;
 	size_t lpc = offset;
 	const char *error = NULL;
 	
 	for(; error == NULL && lpc < max; lpc++) {
 		ch = input[lpc];
 		crm_debug_5("Processing char %c [%d]", ch, (int)lpc);
 		
 		switch(ch) {
 			case 0:
 				error = "unexpected EOS";
  				break;
 			case '\\':
 				if(input[lpc+1] == '"') {
 					/* skip over the next char */ 
 					lpc++;
 					break;
 				}
 				/*fall through*/
 			case '"':
 				return lpc - offset;
 			default:
  				break;
 		}
 	}
 	crm_err("Error parsing token near %.15s: %s", input, crm_str(error));
 	return -1;
 }
 
 int
 is_comment_start(const char *input, size_t offset, size_t max)
 {
 	size_t remaining = max - offset;
 	CRM_CHECK(input != NULL, return 0);
 	CRM_CHECK(offset < max, return 0);
 	input += offset;
 	
 	if(remaining > 4
 	   && input[0] == '<'
 	   && input[1] == '!'
 	   && input[2] == '-'
 	   && input[3] == '-') {
 		crm_debug_6("Found comment start: <!--");
 		return 4;
 		
 	} else if(remaining > 2
 	   && input[0] == '<'
 	   && input[1] == '!') {
 		crm_debug_6("Found comment start: <!");
 		return 2;
 
 	} else if(remaining > 2
 	   && input[0] == '<'
 	   && input[1] == '?') {
 		crm_debug_6("Found comment start: <?");
 		return 2;
 	}
 	if(remaining > 3) {
 		crm_debug_6("Not comment start: %c%c%c%c", input[0], input[1], input[2], input[3]);
 	} else {
 		crm_debug_6("Not comment start");
 	}
 	
 	return 0;
 }
 
 
 int
 is_comment_end(const char *input, size_t offset, size_t max)
 {
 	size_t remaining = max - offset;
 	CRM_CHECK(input != NULL, return 0);
 	input += offset;
 	
 	if(remaining > 2
 	   && input[0] == '-'
 	   && input[1] == '-'
 	   && input[2] == '>') {
 		crm_debug_6("Found comment end: -->");
 		return 3;
 		
 	} else if(remaining > 1
 	   && input[0] == '?'
 	   && input[1] == '>') {
 		crm_debug_6("Found comment end: ?>");
 		return 2;
 	}
 	if(remaining > 2) {
 		crm_debug_6("Not comment end: %c%c%c", input[0], input[1], input[2]);
 	} else {
 		crm_debug_6("Not comment end");
 	}
 	return 0;
 }
 
 static gboolean
 drop_whitespace(const char *input, size_t *offset, size_t max)
 {
 	char ch = 0;
 	size_t lpc = *offset;
 	gboolean more = TRUE;
 	const char *our_input = input;
 
 	if(input == NULL) {
 		return FALSE;
 	}
 	while(lpc < max && more) {
 		ch = our_input[lpc];
 		crm_debug_6("Processing char %c[%d]", ch, (int)lpc);
 		if(isspace(ch)) {
 			lpc++;
 
 		} else {
 			more = FALSE;
 		}
 	}
 
 	crm_debug_4("Finished processing whitespace");
 	if(lpc > *offset) {
 		crm_debug_5("Skipped %d whitespace chars", (int)(lpc - *offset));
 	}
 	(*offset) = lpc;	
 	return FALSE;
 }
 
 gboolean
 drop_comments(const char *input, size_t *offset, size_t max)
 {
 	gboolean more = TRUE;
 	gboolean in_directive = FALSE;
 	int in_comment = FALSE;
 	size_t lpc = 0;
 	int tag_len = 0;
 	char ch = 0;
 	if(input == NULL) {
 		return FALSE;
 	}
 
 	CRM_ASSERT(offset != NULL);
 	lpc = *offset;
 	while(lpc < max && more) {
 		ch = input[lpc];
 		crm_debug_6("Processing char [%d]: %c ", (int)lpc, ch);
 		switch(ch) {
 			case 0:
 				if(in_comment == FALSE) {
 					more = FALSE;
 				} else {
 					crm_err("unexpected EOS");
 					crm_warn("Parsing error at or before: %s", input+lpc);
 				}
 				break;
 			case '<':
 				tag_len = is_comment_start(input, lpc, max);
 				if(tag_len > 0) {
 					if(in_comment) {
 						crm_err("Nested XML comments are not supported!");
 						crm_warn("Parsing error at or before: %s", input+lpc);
 					}
 					in_comment = TRUE;
 					lpc+=tag_len;
 					if(tag_len == 2 && input[lpc-1] == '!') {
 						in_directive = TRUE;
 					}
 				} else if(in_comment == FALSE){
 					more = FALSE;
 					
 				} else {
 					lpc++;
 				}
 				break;
 			case '>':
 				lpc++;
 				if(in_directive) {
 					in_directive = FALSE;
 					in_comment = FALSE;
 				}
 				break;
 			case '-':
 			case '?':
 				tag_len = is_comment_end(input, lpc, max);
 				if(tag_len > 0) {
 					lpc+=tag_len;
 					in_comment = FALSE;
 
 				} else {
 					lpc++;
 				}
 				break;
 			case ' ':
 			case '\t':
 			case '\n':
 			case '\r':
 				lpc++;
 				break;
 			default:
 				lpc++;
 				break;
 		}
 	}
 	crm_debug_4("Finished processing comments");
 	crm_debug_5("Skipped %d comment chars", (int)(lpc - *offset));
 	*offset = lpc;
 	return FALSE;
 }
 
 crm_data_t*
 parse_xml(const char *input, size_t *offset)
 {
 	char ch = 0;
 	int len = 0;
 	size_t lpc = 0, max = 0;
 	char *tag_name = NULL;
 	char *attr_name = NULL;
 	char *attr_value = NULL;
 	gboolean more = TRUE;
 	gboolean were_comments = TRUE;
 	const char *error = NULL;
 	const char *our_input = input;
 	crm_data_t *new_obj = NULL;
 
 	if(input == NULL) {
 		return NULL;
 	}
 	if(offset != NULL) {
 		our_input = input + (*offset);
 	}
 
 	max = strlen(our_input);
 	were_comments = drop_comments(our_input, &lpc, max);
 	CRM_CHECK(our_input[lpc] == '<', return NULL);
 	lpc++;
 
 	len = get_tag_name(our_input, lpc, max);
 	crm_debug_5("Tag length: %d", (int)len);
 	CRM_CHECK(len > 0, return NULL);
 
 	crm_malloc0(tag_name, len+1);
 	strncpy(tag_name, our_input + lpc, len+1);
 	tag_name[len] = EOS;
 	crm_debug_4("Processing tag %s", tag_name);
 	
 	new_obj = ha_msg_new(1);
 	
 	ha_msg_add(new_obj, F_XML_TAGNAME, tag_name);
 	lpc += len;
 
 	for(; more && error == NULL && lpc < max; lpc++) {
 		ch = our_input[lpc];
 		crm_debug_5("Processing char %c[%d]", ch, (int)lpc);
 		switch(ch) {
 			case 0:
 				error = "unexpected EOS";
 				break;
 			case '/':
 				if(our_input[lpc+1] == '>') {
 					more = FALSE;
 				}
 				break;
 			case '<':
 				if(our_input[lpc+1] == '!') {
 					lpc--; /* allow the '<' to be processed */
 					drop_comments(our_input, &lpc, max);
 					lpc--; /* allow the '<' to be processed */
 					
 				} else if(our_input[lpc+1] != '/') {
 					crm_data_t *child = NULL;
 					crm_debug_4("Start parsing child at %d...", (int)lpc);
 					
 					lpc--;
 					child = parse_xml(our_input, &lpc);
 					if(child == NULL) {
 						error = "error parsing child";
 						break;
 					} 
 					add_node_nocopy(new_obj, NULL, child);
 /* 					ha_msg_addstruct_compress( */
 /* 						new_obj, crm_element_name(child), child); */
 					
 					crm_debug_4("Finished parsing child: %s",
 						    crm_element_name(child));
 					if(our_input[lpc] == '<') {
 						lpc--; /* allow the '<' to be processed */
 					}
 					
 				} else {
 					lpc += 2; /* </ */
 					len = get_tag_name(our_input, lpc, max);
 					if(len < 0) {
 						error = "couldnt find tag";
 						
 					} else if(strncmp(our_input+lpc, tag_name, len) == 0) {
 						more = FALSE;
 						lpc += len;
 						if(our_input[lpc] != '>') {
 							error = "clase tag cannot contain attrs";
 						}
 						crm_debug_4("Finished parsing ourselves: %s",
 							    crm_element_name(new_obj));
 						
 					} else {
 						error = "Mismatching close tag";
 						crm_err("Expected: %s", tag_name);
 					}
 				}
 				break;
 			case '=':
 				lpc++; /* = */
 				/*fall through*/
 			case '"':
 				lpc++; /* " */
 				len = get_attr_value(our_input, lpc, max);
 				if(len < 0) {
 					error = "couldnt find attr_value";
 				} else {
 					crm_malloc0(attr_value, len+1);
 					strncpy(attr_value, our_input+lpc, len+1);
 					attr_value[len] = EOS;
 					lpc += len;
 					
 					crm_debug_4("creating nvpair: <%s %s=\"%s\"...",
 						    tag_name, attr_name, attr_value);
 					
 					ha_msg_add(new_obj, attr_name, attr_value);
 					crm_free(attr_name);
 					crm_free(attr_value);
 				}
 				break;
 			case '>':
 				while(lpc < max && our_input[lpc+1] != '<') {
 					lpc++;
 				}
 				break;
 			case ' ':
 			case '\t':
 			case '\n':
 			case '\r':
 				break;
 			default:
 				len = get_attr_name(our_input, lpc, max);
 				if(len < 0) {
 					error = "couldnt find attr_name";
 				} else {
 					crm_malloc0(attr_name, len+1);
 					strncpy(attr_name, our_input+lpc, len+1);
 					attr_name[len] = EOS;
 					lpc += len;
 					crm_debug_4("found attr name: %s", attr_name);
 					lpc--; /* make sure the '=' is seen next time around */
 				}
 				break;
 		}
 	}
 	
 	if(error) {
 		crm_err("Error parsing token: %s", error);
 		crm_err("Error at or before: %.20s", our_input+lpc-3);
 		return NULL;
 	}
 	
 	if(offset == NULL) {
 		drop_comments(our_input, &lpc, max);
 		drop_whitespace(our_input, &lpc, max);
 		if(lpc < max) {
-			crm_err("Ignoring trailing characters in XML input.");
-			crm_err("Parsed %d characters of a possible %d.  Trailing text was: ...\'%20s\'",
-				(int)lpc, (int)max, our_input+lpc);
+		    if(crm_log_level < LOG_ERR) {
+			fprintf(stderr, "%s: Ignoring trailing characters in XML input.  Supply -V for more details.\n", __PRETTY_FUNCTION__);
+		    } else {
+			cl_log(LOG_ERR, "%s: Ignoring trailing characters in XML input.", __PRETTY_FUNCTION__);
+		    }
+		    cl_log(LOG_ERR, "%s: Parsed %d characters of a possible %d.  Trailing text was: \'%.20s\'...",
+			   __PRETTY_FUNCTION__, (int)lpc, (int)max, our_input+lpc);
 		}
 	}
 	
 	crm_debug_4("Finished processing %s tag", tag_name);
 	crm_free(tag_name);
 	if(offset != NULL) {
 		(*offset) += lpc;
 	}
 
 	return new_obj;
 }
 
 void
 log_xml_diff(unsigned int log_level, crm_data_t *diff, const char *function)
 {
 	crm_data_t *added = find_xml_node(diff, "diff-added", FALSE);
 	crm_data_t *removed = find_xml_node(diff, "diff-removed", FALSE);
 	gboolean is_first = TRUE;
 
 	if(crm_log_level < log_level) {
 		/* nothing will ever be printed */
 		return;
 	}
 	
 	xml_child_iter(
 		removed, child, 
 		log_data_element(function, "-", log_level, 0, child, TRUE);
 		if(is_first) {
 			is_first = FALSE;
 		} else {
 			do_crm_log(log_level, " --- ");
 		}
 		
 		);
 	is_first = TRUE;
 	xml_child_iter(
 		added, child, 
 		log_data_element(function, "+", log_level, 0, child, TRUE);
 		if(is_first) {
 			is_first = FALSE;
 		} else {
 			do_crm_log(log_level, " --- ");
 		}
 		);
 }
 
 void
 purge_diff_markers(crm_data_t *a_node)
 {
 	CRM_CHECK(a_node != NULL, return);
 
 	xml_remove_prop(a_node, XML_DIFF_MARKER);
 	xml_child_iter(a_node, child,
 		       purge_diff_markers(child);
 		);
 }
 
 gboolean
 apply_xml_diff(crm_data_t *old, crm_data_t *diff, crm_data_t **new)
 {
 	gboolean result = TRUE;
 	crm_data_t *added = find_xml_node(diff, "diff-added", FALSE);
 	crm_data_t *removed = find_xml_node(diff, "diff-removed", FALSE);
 
 	int root_nodes_seen = 0;
 
 	CRM_CHECK(new != NULL, return FALSE);
 
 	crm_debug_2("Substraction Phase");
 	xml_child_iter(removed, child_diff, 
 		       CRM_CHECK(root_nodes_seen == 0, result = FALSE);
 		       if(root_nodes_seen == 0) {
 			       *new = subtract_xml_object(old, child_diff, NULL);
 		       }
 		       root_nodes_seen++;
 		);
 	if(root_nodes_seen == 0) {
 		*new = copy_xml(old);
 		
 	} else if(root_nodes_seen > 1) {
 		crm_err("(-) Diffs cannot contain more than one change set..."
 			" saw %d", root_nodes_seen);
 		result = FALSE;
 	}
 
 	root_nodes_seen = 0;
 	crm_debug_2("Addition Phase");
 	if(result) {
 		xml_child_iter(added, child_diff, 
 			       CRM_CHECK(root_nodes_seen == 0, result = FALSE);
 			       if(root_nodes_seen == 0) {
 				       add_xml_object(NULL, *new, child_diff);
 			       }
 			       root_nodes_seen++;
 			);
 	}
 
 	if(root_nodes_seen > 1) {
 		crm_err("(+) Diffs cannot contain more than one change set..."
 			" saw %d", root_nodes_seen);
 		result = FALSE;
 
 	} else if(result) {
 		int lpc = 0;
 		crm_data_t *intermediate = NULL;
 		crm_data_t *diff_of_diff = NULL;
 		crm_data_t *calc_added = NULL;
 		crm_data_t *calc_removed = NULL;
 
 		const char *value = NULL;
 		const char *name = NULL;
 		const char *version_attrs[] = {
 			XML_ATTR_NUMUPDATES,
 			XML_ATTR_GENERATION,
 			XML_ATTR_GENERATION_ADMIN
 		};
 		
 		crm_debug_2("Verification Phase");
 		intermediate = diff_xml_object(old, *new, FALSE);
 		calc_added = find_xml_node(intermediate, "diff-added", FALSE);
 		calc_removed = find_xml_node(intermediate, "diff-removed", FALSE);
 
 		/* add any version details to the diff so they match */
 		for(lpc = 0; lpc < DIMOF(version_attrs); lpc++) {
 			name = version_attrs[lpc];
 
 			value = crm_element_value(added, name);
 			crm_xml_add(calc_added, name, value);
 			
 			value = crm_element_value(removed, name);
 			crm_xml_add(calc_removed, name, value);	
 		}
 		
 		diff_of_diff = diff_xml_object(intermediate, diff, TRUE);
 		if(diff_of_diff != NULL) {
 			crm_info("Diff application failed!");
 			crm_log_xml_debug(old, "diff:original");
 			crm_log_xml_debug(diff, "diff:input");
 			result = FALSE;
 		}
 		
 		free_xml(diff_of_diff);
 		free_xml(intermediate);
 		diff_of_diff = NULL;
 		intermediate = NULL;
 	}
 	
 	if(result) {
 		purge_diff_markers(*new);
 	}
 
 	return result;
 }
 
 
 crm_data_t *
 diff_xml_object(crm_data_t *old, crm_data_t *new, gboolean suppress)
 {
 	crm_data_t *diff = NULL;
 	crm_data_t *tmp1 = NULL;
 	crm_data_t *added = NULL;
 	crm_data_t *removed = NULL;
 
 	tmp1 = subtract_xml_object(old, new, "removed:top");
 	if(tmp1 != NULL) {
 		if(suppress && can_prune_leaf(tmp1)) {
 			ha_msg_del(tmp1);
 
 		} else {
 			diff = create_xml_node(NULL, "diff");
 			removed = create_xml_node(diff, "diff-removed");
 			added = create_xml_node(diff, "diff-added");
 			add_node_nocopy(removed, NULL, tmp1);
 		}
 	}
 	
 	tmp1 = subtract_xml_object(new, old, "added:top");
 	if(tmp1 != NULL) {
 		if(suppress && can_prune_leaf(tmp1)) {
 			ha_msg_del(tmp1);
 			return diff;
 			
 		}
 
 		if(diff == NULL) {
 			diff = create_xml_node(NULL, "diff");
 		}
 		if(removed == NULL) {
 			removed = create_xml_node(diff, "diff-removed");
 		}
 		if(added == NULL) {
 			added = create_xml_node(diff, "diff-added");
 		}
 		add_node_nocopy(added, NULL, tmp1);
 	}
 
 	return diff;
 }
 
 gboolean
 can_prune_leaf(crm_data_t *xml_node)
 {
 	gboolean can_prune = TRUE;
 /* 	return FALSE; */
 	
 	xml_prop_iter(xml_node, prop_name, prop_value,
 		      if(safe_str_eq(prop_name, XML_ATTR_ID)) {
 			      continue;
 		      }		      
 		      can_prune = FALSE;
 		);
 	xml_child_iter(xml_node, child, 
 		       if(can_prune_leaf(child)) {
 			       cl_msg_remove_value(xml_node, child);
 			       __counter--;
 		       } else {
 			       can_prune = FALSE;
 		       }
 		);
 	return can_prune;
 }
 
 
 void
 diff_filter_context(int context, int upper_bound, int lower_bound,
 		    crm_data_t *xml_node, crm_data_t *parent) 
 {
 	crm_data_t *us = NULL;
 	crm_data_t *new_parent = parent;
 	const char *name = crm_element_name(xml_node);
 
 	CRM_CHECK(xml_node != NULL && name != NULL, return);
 	
 	us = create_xml_node(parent, name);
 	xml_prop_iter(xml_node, prop_name, prop_value,
 		      lower_bound = context;
 		      crm_xml_add(us, prop_name, prop_value);
 		);
 	if(lower_bound >= 0 || upper_bound >= 0) {
 		crm_xml_add(us, XML_ATTR_ID, ID(xml_node));
 		new_parent = us;
 
 	} else {
 		upper_bound = in_upper_context(0, context, xml_node);
 		if(upper_bound >= 0) {
 			crm_xml_add(us, XML_ATTR_ID, ID(xml_node));
 			new_parent = us;
 		} else {
 			free_xml(us);
 			us = NULL;
 		}
 	}
 
 	xml_child_iter(us, child, 
 		       diff_filter_context(
 			       context, upper_bound-1, lower_bound-1,
 			       child, new_parent);
 		);
 }
 
 int
 in_upper_context(int depth, int context, crm_data_t *xml_node)
 {
 	gboolean has_attributes = FALSE;
 	if(context == 0) {
 		return 0;
 	}
 	
 	xml_prop_iter(xml_node, prop_name, prop_value,
 		      has_attributes = TRUE;
 		      break;
 		);
 	
 	if(has_attributes) {
 		return depth;
 
 	} else if(depth < context) {
 		xml_child_iter(xml_node, child, 
 			       if(in_upper_context(depth+1, context, child)) {
 				       return depth;
 			       }
 			);
 	}
 	return 0;       
 }
 
 
 crm_data_t *
 subtract_xml_object(crm_data_t *left, crm_data_t *right, const char *marker)
 {
 	gboolean skip = FALSE;
 	gboolean differences = FALSE;
 	crm_data_t *diff = NULL;
 	crm_data_t *child_diff = NULL;
 	crm_data_t *right_child = NULL;
 
 	const char *id = NULL;
 	const char *name = NULL;
 	const char *value = NULL;
 	const char *right_val = NULL;
 
 	int lpc = 0;
 	const char *filter[] = {
 		XML_ATTR_ORIGIN,
 		XML_DIFF_MARKER,
 		XML_CIB_ATTR_WRITTEN,		
 	};
 	static int filter_len = DIMOF(filter);
 	
 	crm_log_xml(LOG_DEBUG_5, "left:",  left);
 	crm_log_xml(LOG_DEBUG_5, "right:", right);
 	
 	if(left == NULL) {
 		return NULL;
 
 	}
 	id = ID(left);
 	if(right == NULL) {
 		crm_data_t *deleted = NULL;
 
 		crm_debug_5("Processing <%s id=%s> (complete copy)",
 			    crm_element_name(left), id);
 		deleted = copy_xml(left);
 		crm_xml_add(deleted, XML_DIFF_MARKER, marker);
 
 		return deleted;
 	}
 
 	name = crm_element_name(left);
 
 	/* sanity checks */
 	CRM_CHECK(name != NULL, return NULL);
 
 /* 	these checks are costly haven't caught anything for a while	*/
 /* 	CRM_CHECK(safe_str_eq(crm_element_name(left),			*/
 /* 			      crm_element_name(right)), return NULL);	*/
 /* 	CRM_CHECK(safe_str_eq(id, ID(right)), return NULL);		*/
 	
 	diff = create_xml_node(NULL, name);
 
 	/* changes to name/value pairs */
 	crm_debug_5("Processing <%s id=%s>", crm_str(name), id);
 
 	xml_prop_iter(left, prop_name, left_value,
 		      if(crm_str_eq(prop_name, XML_ATTR_ID, TRUE)) {
 			      continue;
 		      }
 
 		      skip = FALSE;
 		      for(lpc = 0; skip == FALSE && lpc < filter_len; lpc++){
 			      if(crm_str_eq(prop_name, filter[lpc], TRUE)) {
 				      skip = TRUE;
 			      }
 		      }
 		      
 		      if(skip) { continue; }
 		      
 		      right_val = crm_element_value(right, prop_name);
 		      if(right_val == NULL) {
 			      differences = TRUE;
 			      crm_xml_add(diff, prop_name, left_value);
 			      crm_debug_6("\t%s: %s", crm_str(prop_name),
 					  crm_str(left_value));
 				      
 		      } else if(safe_str_eq(left_value, right_val)) {
 			      crm_debug_5("\t%s: %s (removed)",
 					  crm_str(prop_name),
 					  crm_str(left_value));
 		      } else {
 			      differences = TRUE;
 			      crm_xml_add(diff, prop_name, left_value);
 			      crm_debug_5("\t%s: %s->%s",
 					  crm_str(prop_name),
 					  crm_str(left_value),
 					  right_val);
 		      }
 		);
 
 	/* changes to child objects */
 	crm_debug_3("Processing children of <%s id=%s>",crm_str(name),id);
 	xml_child_iter(
 		left, left_child,  
 		right_child = find_entity(
 			right, crm_element_name(left_child), ID(left_child));
 		child_diff = subtract_xml_object(
 			left_child, right_child, marker);
 		if(child_diff != NULL) {
 			differences = TRUE;
 			add_node_nocopy(diff, NULL, child_diff);
 		}
 		
 		);
 
 	if(differences == FALSE) {
 		/* check for XML_DIFF_MARKER in a child */ 
 		xml_child_iter(
 			right, right_child,  
 			value = crm_element_value(right_child, XML_DIFF_MARKER);
 			if(value != NULL && safe_str_eq(value, "removed:top")) {
 				crm_debug_3("Found the root of the deletion: %s", name);
 				crm_log_xml_debug_3(right_child, "deletion");
 				differences = TRUE;
 				break;
 			}
 			);
 	}
 	
 	if(differences == FALSE) {
 		free_xml(diff);
 		crm_debug_5("\tNo changes to <%s id=%s>", crm_str(name), id);
 		return NULL;
 	}
 	crm_xml_add(diff, XML_ATTR_ID, id);
 	return diff;
 }
 
 int
 add_xml_object(crm_data_t *parent, crm_data_t *target, const crm_data_t *update)
 {
 	const char *object_id = NULL;
 	const char *object_name = NULL;
 
 	crm_log_xml(LOG_DEBUG_5, "update:", update);
 	crm_log_xml(LOG_DEBUG_5, "target:", target);
 
 	CRM_CHECK(update != NULL, return 0);
 
 	object_name = crm_element_name(update);
 	object_id = ID(update);
 
 	CRM_CHECK(object_name != NULL, return 0);
 	
 	if(target == NULL && object_id == NULL) {
 		/*  placeholder object */
 		target = find_xml_node(parent, object_name, FALSE);
 
 	} else if(target == NULL) {
 		target = find_entity(parent, object_name, object_id);
 	}
 
 	if(target == NULL) {
 		target = create_xml_node(parent, object_name);
 		CRM_CHECK(target != NULL, return 0);
 		crm_debug_2("Added  <%s%s%s/>", crm_str(object_name),
 			    object_id?" id=":"", object_id?object_id:"");
 
 	} else {
 		crm_debug_3("Found node <%s%s%s/> to update",
 			    crm_str(object_name),
 			    object_id?" id=":"", object_id?object_id:"");
 	}
 
 	copy_in_properties(target, update);
 
 	xml_child_iter(
 		update, a_child,  
 		crm_debug_4("Updating child <%s id=%s>",
 			    crm_element_name(a_child), ID(a_child));
 		add_xml_object(target, NULL, a_child);
 		);
 
 	crm_debug_3("Finished with <%s id=%s>",
 		    crm_str(object_name), crm_str(object_id));
 	return 0;
 }
 
 gboolean
 update_xml_child(crm_data_t *child, crm_data_t *to_update)
 {
 	gboolean can_update = TRUE;
 	
 	CRM_CHECK(child != NULL, return FALSE);
 	CRM_CHECK(to_update != NULL, return FALSE);
 	
 	if(safe_str_neq(crm_element_name(to_update), crm_element_name(child))) {
 		can_update = FALSE;
 
 	} else if(safe_str_neq(ID(to_update), ID(child))) {
 		can_update = FALSE;
 
 	} else if(can_update) {
 		crm_log_xml_debug_2(child, "Update match found...");
 		add_xml_object(NULL, child, to_update);
 	}
 	
 	xml_child_iter(
 		child, child_of_child, 
 		/* only update the first one */
 		if(can_update) {
 			break;
 		}
 		can_update = update_xml_child(child_of_child, to_update);
 		);
 	
 	return can_update;
 }
 
 
 int
 find_xml_children(crm_data_t **children, crm_data_t *root,
 		  const char *tag, const char *field, const char *value,
 		  gboolean search_matches)
 {
 	int match_found = 0;
 	
 	CRM_CHECK(root != NULL, return FALSE);
 	CRM_CHECK(children != NULL, return FALSE);
 	
 	if(tag != NULL && safe_str_neq(tag, crm_element_name(root))) {
 
 	} else if(value != NULL
 		  && safe_str_neq(value, crm_element_value(root, field))) {
 
 	} else {
 		if(*children == NULL) {
 			*children = create_xml_node(NULL, __FUNCTION__);
 		}
 		add_node_copy(*children, root);
 		match_found = 1;
 	}
 
 	if(search_matches || match_found == 0) {
 		xml_child_iter(
 			root, child, 
 			match_found += find_xml_children(
 				children, child, tag, field, value,
 				search_matches);
 			);
 	}
 	
 	return match_found;
 }
 
 gboolean
 replace_xml_child(crm_data_t *parent, crm_data_t *child, crm_data_t *update, gboolean delete_only)
 {
 	gboolean can_delete = FALSE;
 
 	const char *up_id = NULL;
 	const char *child_id = NULL;
 	const char *right_val = NULL;
 	
 	CRM_CHECK(child != NULL, return FALSE);
 	CRM_CHECK(update != NULL, return FALSE);
 
 	up_id = ID(update);
 	child_id = ID(child);
 	
-	if(child_id == up_id || safe_str_eq(child_id, up_id)) {
+	if(up_id == NULL || safe_str_eq(child_id, up_id)) {
 		can_delete = TRUE;
-	}
+	} 
 	if(safe_str_neq(crm_element_name(update), crm_element_name(child))) {
 		can_delete = FALSE;
 	}
 	if(can_delete && delete_only) {
 		xml_prop_iter(update, prop_name, left_value,
 			      right_val = crm_element_value(child, prop_name);
 			      if(safe_str_neq(left_value, right_val)) {
 				      can_delete = FALSE;
 			      }
 			);
 	}
 	
 	if(can_delete && parent != NULL) {
 		crm_log_xml_debug_4(child, "Delete match found...");
 		if(delete_only) {
 			cl_msg_remove_value(parent, child);
 		} else {
 			/* preserve the order */
 			cl_msg_replace_value(parent, child, update,
 					     sizeof(struct ha_msg), FT_STRUCT);
 		}
 		child = NULL;
 		return TRUE;
 		
 	} else if(can_delete) {
 		crm_log_xml_debug(child, "Cannot delete the search root");
 		can_delete = FALSE;
 	}
 	
 	
 	xml_child_iter(
 		child, child_of_child, 
 		/* only delete the first one */
 		if(can_delete) {
 			break;
 		}
 		can_delete = replace_xml_child(child, child_of_child, update, delete_only);
 		);
 	
 	return can_delete;
 }
 
 void
 hash2nvpair(gpointer key, gpointer value, gpointer user_data) 
 {
 	const char *name    = key;
 	const char *s_value = value;
 
 	crm_data_t *xml_node  = user_data;
 	crm_data_t *xml_child = create_xml_node(xml_node, XML_CIB_TAG_NVPAIR);
 
 	crm_xml_add(xml_child, XML_ATTR_ID, name);
 	crm_xml_add(xml_child, XML_NVPAIR_ATTR_NAME, name);
 	crm_xml_add(xml_child, XML_NVPAIR_ATTR_VALUE, s_value);
 
 	crm_debug_3("dumped: name=%s value=%s", name, s_value);
 }
 
 void
 hash2field(gpointer key, gpointer value, gpointer user_data) 
 {
 	const char *name    = key;
 	const char *s_value = value;
 
 	crm_data_t *xml_node  = user_data;
 
 	if(crm_element_value(xml_node, name) == NULL) {
 		crm_xml_add(xml_node, name, s_value);
 		crm_debug_3("dumped: %s=%s", name, s_value);
 	} else {
 		crm_debug_2("duplicate: %s=%s", name, s_value);
 	}
 }
 
 void
 hash2metafield(gpointer key, gpointer value, gpointer user_data) 
 {
 	char *crm_name = NULL;
 
 	if(key == NULL || value == NULL) {
 		return;
 	}
 	
 	crm_name = crm_concat(CRM_META, key, '_');
 	hash2field(crm_name, value, user_data);
 	crm_free(crm_name);
 }
 
 
 #if CRM_DEPRECATED_SINCE_2_0_3
 GHashTable *
 xml2list_202(crm_data_t *parent)
 {
 	crm_data_t *nvpair_list = NULL;
 	GHashTable *nvpair_hash = g_hash_table_new_full(
 		g_str_hash, g_str_equal,
 		g_hash_destroy_str, g_hash_destroy_str);
 
 	CRM_CHECK(parent != NULL, return nvpair_hash);
 
 	nvpair_list = find_xml_node(parent, XML_TAG_ATTRS, FALSE);
 	if(nvpair_list == NULL) {
 		crm_debug("No attributes in %s",
 			  crm_element_name(parent));
 		crm_log_xml_debug_2(
 			parent,"No attributes for resource op");
 	}
 	
 	xml_child_iter_filter(
 		nvpair_list, node_iter, XML_CIB_TAG_NVPAIR,
 		
 		const char *key   = crm_element_value(
 			node_iter, XML_NVPAIR_ATTR_NAME);
 		const char *value = crm_element_value(
 			node_iter, XML_NVPAIR_ATTR_VALUE);
 		
 		crm_debug_2("Added %s=%s", key, value);
 		
 		g_hash_table_insert(
 			nvpair_hash, crm_strdup(key), crm_strdup(value));
 		);
 	
 	return nvpair_hash;
 }
 #endif
 
 GHashTable *
 xml2list(crm_data_t *parent)
 {
 	crm_data_t *nvpair_list = NULL;
 	GHashTable *nvpair_hash = g_hash_table_new_full(
 		g_str_hash, g_str_equal,
 		g_hash_destroy_str, g_hash_destroy_str);
 	
 	CRM_CHECK(parent != NULL, return nvpair_hash);
 
 	nvpair_list = find_xml_node(parent, XML_TAG_ATTRS, FALSE);
 	if(nvpair_list == NULL) {
 		crm_debug_2("No attributes in %s",
 			    crm_element_name(parent));
 		crm_log_xml_debug_2(
 			parent,"No attributes for resource op");
 	}
 	
 	crm_log_xml_debug_3(nvpair_list, "Unpacking");
 
 	xml_prop_iter(
 		nvpair_list, key, value, 
 		
 		crm_debug_4("Added %s=%s", key, value);
 		
 		g_hash_table_insert(
 			nvpair_hash, crm_strdup(key), crm_strdup(value));
 		);
 	
 	return nvpair_hash;
 }
 
 
 static void
 assign_uuid(crm_data_t *xml_obj) 
 {
 	cl_uuid_t new_uuid;
 	char *new_uuid_s = NULL;
 	const char *tag_name = crm_element_name(xml_obj);
 	const char *tag_id = ID(xml_obj);
 	
 	crm_malloc0(new_uuid_s, 38);
 	cl_uuid_generate(&new_uuid);
 	cl_uuid_unparse(&new_uuid, new_uuid_s);
 	
 	crm_warn("Updating object from <%s id=%s/> to <%s id=%s/>",
 		 tag_name, tag_id?tag_id:"__empty__", tag_name, new_uuid_s);
 	
 	crm_xml_add(xml_obj, XML_ATTR_ID, new_uuid_s);
 	crm_log_xml_debug(xml_obj, "Updated object");	
 	crm_free(new_uuid_s);
 }
 
 static gboolean
 tag_needs_id(const char *tag_name) 
 {
 	int lpc = 0;
 	const char *allowed_list[] = {
 		XML_TAG_CIB,
 		XML_TAG_FRAGMENT,
 		XML_CIB_TAG_NODES,
 		XML_CIB_TAG_RESOURCES,
 		XML_CIB_TAG_CONSTRAINTS,
 		XML_CIB_TAG_STATUS,
 		XML_LRM_TAG_RESOURCES,
 		"configuration",
 		"crm_config",
 		"attributes",
 		"operations",
 		"diff",
 		"diff-added",
 		"diff-removed",
 	};
 	
 	for(lpc = 0; lpc < DIMOF(allowed_list); lpc++) {
 		if(crm_str_eq(tag_name, allowed_list[lpc], TRUE)) {
 			/* this tag is never meant to have an ID */
 			return FALSE;
 		}
 	}
 	return TRUE;
 }
 
 static gboolean
 non_unique_allowed(const char *tag_name) 
 {
 	int lpc = 0;
 	const char *non_unique[] = {
 		XML_LRM_TAG_RESOURCE,
 		XML_LRM_TAG_RSC_OP,
 	};
 
 	for(lpc = 0; lpc < DIMOF(non_unique); lpc++) {
 		if(safe_str_eq(tag_name, non_unique[lpc])) {
 			/* this tag can have a non-unique ID */
 			return TRUE;
 		}
 	}
 	return FALSE;
 }
 
 gboolean
 do_id_check(crm_data_t *xml_obj, GHashTable *id_hash,
 	    gboolean silent_add, gboolean silent_rename) 
 {
 	char *lookup_id = NULL;
 	gboolean modified = FALSE;
 
 	char *old_id = NULL;
 	const char *tag_id = NULL;
 	const char *tag_name = NULL;
 	const char *lookup_value = NULL;
 
 	gboolean created_hash = FALSE;
 
 	if(xml_obj == NULL) {
 		return FALSE;
 
 	} else if(id_hash == NULL) {
 		created_hash = TRUE;
 		id_hash = g_hash_table_new_full(
 			g_str_hash, g_str_equal,
 			g_hash_destroy_str, g_hash_destroy_str);
 	}
 
 	xml_child_iter(
 		xml_obj, xml_child, 
 		if(do_id_check(xml_child, id_hash, silent_add, silent_rename)) {
 			modified = TRUE;
 		}
 		);
 
 	tag_id = ID(xml_obj);
 	tag_name = TYPE(xml_obj);
 	
 	if(tag_needs_id(tag_name) == FALSE) {
 		crm_debug_5("%s does not need an ID", tag_name);
 		goto finish_id_check;
 
 	} else if(tag_id != NULL && non_unique_allowed(tag_name)){
 		crm_debug_5("%s does not need top be unique", tag_name);
 		goto finish_id_check;
 	}
 	
 	lookup_id = NULL;
 	if(tag_id != NULL) {
 		lookup_id = crm_concat(tag_name, tag_id, '-');
 		lookup_value = g_hash_table_lookup(id_hash, lookup_id);
 		if(lookup_value == NULL) {
 			g_hash_table_insert(id_hash, lookup_id, crm_strdup(tag_id));
 			goto finish_id_check;
 		}
 		modified |= (!silent_rename);
 		
 	} else {
 		modified |= (!silent_add);
 	}
 
 	if(tag_id != NULL) {
 		old_id = crm_strdup(tag_id);
 	}
 	
 	crm_free(lookup_id);
 	assign_uuid(xml_obj);
 	tag_id = ID(xml_obj);
 	
 	if(modified == FALSE) {
 		/* nothing to report */
 		
 	} else if(old_id != NULL && safe_str_neq(tag_id, old_id)) {
 		crm_err("\"id\" collision detected... Multiple '%s' entries"
 			" with id=\"%s\", assigned id=\"%s\"",
 			tag_name, old_id, tag_id);
 
 	} else if(old_id == NULL && tag_id != NULL) {
 		crm_err("Detected <%s.../> object without an ID. Assigned: %s",
 			tag_name, tag_id);
 	}
 	crm_free(old_id);
 
   finish_id_check:
 	if(created_hash) {
 		g_hash_table_destroy(id_hash);
 	}
 
 	return modified;
 }
 
 typedef struct name_value_s 
 {
 	const char *name;
 	const void *value;
 } name_value_t;
 
 static gint
 sort_pairs(gconstpointer a, gconstpointer b)
 {
 	const name_value_t *pair_a = a;
 	const name_value_t *pair_b = b;
 	
 	if(a == NULL && b == NULL) {
 		return 0;
 	} else if(a == NULL) {
 		return 1;
 	} else if(b == NULL) {
 		return -1;
 	}
 
 	if(pair_a->name == NULL && pair_b->name == NULL) {
 		return 0;
 	} else if(pair_a->name == NULL) {
 		return 1;
 	} else if(pair_b->name == NULL) {
 		return -1;
 	}
 	return strcmp(pair_a->name, pair_b->name);
 }
 
 static void
 dump_pair(gpointer data, gpointer user_data)
 {
 	name_value_t *pair = data;
 	crm_data_t *parent = user_data;
 	crm_xml_add(parent, pair->name, pair->value);
 }
 
 static void
 free_pair(gpointer data, gpointer user_data)
 {
 	name_value_t *pair = data;
 	crm_free(pair);
 }
 
 static crm_data_t *
 sorted_xml(const crm_data_t *input)
 {
 	GListPtr sorted = NULL;
 	GListPtr unsorted = NULL;
 	name_value_t *pair = NULL;
 	crm_data_t *result = NULL;
 	const char *name = crm_element_name(input);
 
 	CRM_CHECK(input != NULL, return NULL);
 	
 	name = crm_element_name(input);
 	CRM_CHECK(name != NULL, return NULL);
 
 	result = create_xml_node(NULL, name);
 	
 	xml_prop_iter(input, p_name, p_value,
 		      crm_malloc0(pair, sizeof(name_value_t));
 		      pair->name  = p_name;
 		      pair->value = p_value;
 		      unsorted = g_list_prepend(unsorted, pair);
 		      pair = NULL;
 		);
 
 	sorted = g_list_sort(unsorted, sort_pairs);
 	g_list_foreach(sorted, dump_pair, result);
 	g_list_foreach(sorted, free_pair, NULL);
 	g_list_free(sorted);
 	return result;
 }
 
 /* "c048eae664dba840e1d2060f00299e9d" */
 char *
 calculate_xml_digest(crm_data_t *input, gboolean sort)
 {
 	int i = 0;
 	int digest_len = 16;
 	char *digest = NULL;
 	unsigned char *raw_digest = NULL;
 	crm_data_t *sorted = NULL;
 	char *buffer = NULL;
 	size_t buffer_len = 0;
 
 	if(sort) {
 		sorted = sorted_xml(input);
 	} else {
 		sorted = copy_xml(input);
 	}
 	buffer = dump_xml_formatted(sorted);
 	buffer_len = strlen(buffer);
 	
 	CRM_CHECK(buffer != NULL && buffer_len > 0,
 		  free_xml(sorted); return NULL);
 
 	crm_malloc0(digest, (2 * digest_len + 1));
 	crm_malloc0(raw_digest, (digest_len + 1));
 	MD5((unsigned char *)buffer, buffer_len, raw_digest);
 	for(i = 0; i < digest_len; i++) {
  		sprintf(digest+(2*i), "%02x", raw_digest[i]);
  	}
         crm_debug_2("Digest %s: %s\n", digest, buffer);
 	crm_log_xml(LOG_DEBUG_3,  "digest:source", sorted);
 	crm_free(buffer);
 	crm_free(raw_digest);
 	free_xml(sorted);
 	return digest;
 }
 
 
 #if HAVE_LIBXML2
 #  include <libxml/parser.h>
 #  include <libxml/tree.h>
 #endif
 
 gboolean
 validate_with_dtd(
 	crm_data_t *xml_blob, gboolean to_logs, const char *dtd_file) 
 {
 	gboolean valid = TRUE;
 #if HAVE_LIBXML2
 	char *buffer = NULL;
 
  	xmlDocPtr doc = NULL;
 	xmlDtdPtr dtd = NULL;
 	xmlValidCtxtPtr cvp = NULL;
 	
 	CRM_CHECK(xml_blob != NULL, return FALSE);
 	CRM_CHECK(dtd_file != NULL, return FALSE);
 
 	buffer = dump_xml_formatted(xml_blob);
 	CRM_CHECK(buffer != NULL, return FALSE);
 
  	doc = xmlParseMemory(buffer, strlen(buffer));
 	CRM_CHECK(doc != NULL, valid = FALSE; goto cleanup);
 	
 	dtd = xmlParseDTD(NULL, (const xmlChar *)dtd_file);
 	CRM_CHECK(dtd != NULL, goto cleanup);
 
 	cvp = xmlNewValidCtxt();
 	CRM_CHECK(cvp != NULL, goto cleanup);
 
 	if(to_logs) {
 		cvp->userData = (void *) LOG_ERR;
 		cvp->error    = (xmlValidityErrorFunc) cl_log;
 		cvp->warning  = (xmlValidityWarningFunc) cl_log;
 	} else {
 		cvp->userData = (void *) stderr;
 		cvp->error    = (xmlValidityErrorFunc) fprintf;
 		cvp->warning  = (xmlValidityWarningFunc) fprintf;
 	}
 	
 	if (!xmlValidateDtd(cvp, doc, dtd)) {
 		crm_err("CIB does not validate against %s", dtd_file);
 		crm_log_xml_debug(xml_blob, "invalid");
 		valid = FALSE;
 	}
 	
   cleanup:
 	if(cvp) {
 		xmlFreeValidCtxt(cvp);
 	}
 	if(dtd) {
 		xmlFreeDtd(dtd);
 	}
 	if(doc) {
 		xmlFreeDoc(doc);
 	}
 	if(buffer) {
 		crm_free(buffer);
 	}
 	
 #endif	
 	return valid;
 }