diff --git a/cib/io.c b/cib/io.c
index 268d9c8a9c..9a008c21f0 100644
--- a/cib/io.c
+++ b/cib/io.c
@@ -1,658 +1,659 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 #include <string.h>
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <dirent.h>
 
 #include <crm/crm.h>
 
 #include <cibio.h>
 #include <crm/cib.h>
 #include <crm/common/util.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/util.h>
 #include <crm/common/cluster.h>
 
 #define CIB_SERIES "cib"
 
 extern const char *cib_root;
 static int cib_wrap=100;
 
 
 #define CIB_WRITE_PARANOIA	0
 
 const char * local_resource_path[] =
 {
 	XML_CIB_TAG_STATUS,
 };
 
 const char * resource_path[] =
 {
 	XML_CIB_TAG_RESOURCES,
 };
 
 const char * node_path[] =
 {
 	XML_CIB_TAG_NODES,
 };
 
 const char * constraint_path[] =
 {
 	XML_CIB_TAG_CONSTRAINTS,
 };
 
 gboolean initialized = FALSE;
 xmlNode *node_search = NULL;
 xmlNode *resource_search = NULL;
 xmlNode *constraint_search = NULL;
 xmlNode *status_search = NULL;
 
 extern gboolean cib_writes_enabled;
 extern GTRIGSource *cib_writer;
 extern enum cib_errors cib_status;
 
 int set_connected_peers(xmlNode *xml_obj);
 void GHFunc_count_peers(gpointer key, gpointer value, gpointer user_data);
 int write_cib_contents(gpointer p);
 extern void cib_cleanup(void);
 
 
 static gboolean
 validate_cib_digest(xmlNode *local_cib, const char *sigfile)
 {
 	int s_res = -1;
 	struct stat buf;
 	char *digest = NULL;
 	char *expected = NULL;
 	gboolean passed = FALSE;
 	FILE *expected_strm = NULL;
 	int start = 0, length = 0, read_len = 0;
 	
 	CRM_ASSERT(sigfile != NULL);
 	s_res = stat(sigfile, &buf);
 	
 	if (s_res != 0) {
 		crm_warn("No on-disk digest present");
 		return TRUE;
 	}
 
 	if(local_cib != NULL) {
 	    digest = calculate_xml_digest(local_cib, FALSE, FALSE);
 	}
 	
 	expected_strm = fopen(sigfile, "r");
 	if(expected_strm == NULL) {
 		crm_perror(LOG_ERR,"Could not open signature file %s for reading", sigfile);
 		goto bail;
 	}
 
 	start  = ftell(expected_strm);
 	fseek(expected_strm, 0L, SEEK_END);
 	length = ftell(expected_strm);
 	fseek(expected_strm, 0L, start);
 	
+	CRM_ASSERT(length >= 0);
 	CRM_ASSERT(start == ftell(expected_strm));
 
 	crm_debug_3("Reading %d bytes from file", length);
 	crm_malloc0(expected, (length+1));
 	read_len = fread(expected, 1, length, expected_strm);
 	CRM_ASSERT(read_len == length);
 	fclose(expected_strm);
 
   bail:
 	if(expected == NULL) {
 		crm_err("On-disk digest is empty");
 		
 	} else if(safe_str_eq(expected, digest)) {
 		crm_debug_2("Digest comparision passed: %s", digest);
 		passed = TRUE;
 
 	} else {
 		crm_err("Digest comparision failed: expected %s (%s), calculated %s",
 			expected, sigfile, digest);
 	}
 
  	crm_free(digest);
  	crm_free(expected);
 	return passed;
 }
 
 static int
 write_cib_digest(xmlNode *local_cib, const char *digest_file, char *digest)
 {
 	int rc = 0;
 	char *local_digest = NULL;
 	FILE *digest_strm = fopen(digest_file, "w");
 
 	if(digest_strm == NULL) {
 		crm_perror(LOG_ERR,"Cannot open signature file %s for writing", digest_file);
 		return -1;
 	}
 
 	if(digest == NULL) {
 		local_digest = calculate_xml_digest(local_cib, FALSE, FALSE);
 		CRM_ASSERT(digest != NULL);
 		digest = local_digest;
 	}
 	
 	rc = fprintf(digest_strm, "%s", digest);
 	if(rc < 0) {
 		crm_perror(LOG_ERR,"Cannot write to signature file %s", digest_file);
 	}
 
 	CRM_ASSERT(digest_strm != NULL);
 	if(fflush(digest_strm) != 0) {
 	    crm_perror(LOG_ERR,"Couldnt flush the contents of %s", digest_file);
 	    rc = -1;
 	}
 	
 	if(fsync(fileno(digest_strm)) < 0) {
 	    crm_perror(LOG_ERR,"Couldnt sync the contents of %s", digest_file);
 	    rc = -1;
 	}
 	
 	fclose(digest_strm);
 	crm_free(local_digest);
 	return rc;
 }
 
 static gboolean
 validate_on_disk_cib(const char *filename, xmlNode **on_disk_cib)
 {
 	int s_res = -1;
 	struct stat buf;
 	gboolean passed = TRUE;
 	xmlNode *root = NULL;
 
 	CRM_ASSERT(filename != NULL);
 	
 	s_res = stat(filename, &buf);
 	if (s_res == 0) {
 		char *sigfile = NULL;
 		size_t		fnsize;
 		crm_debug_2("Reading cluster configuration from: %s", filename);
 		root = filename2xml(filename);
 		
 		fnsize =  strlen(filename) + 5;
 		crm_malloc0(sigfile, fnsize);
 		snprintf(sigfile, fnsize, "%s.sig", filename);
 		if(validate_cib_digest(root, sigfile) == FALSE) {
 			passed = FALSE;
 		}
 		crm_free(sigfile);
 	}
 	
 	if(on_disk_cib != NULL) {
 		*on_disk_cib = root;
 	} else {
 		free_xml(root);
 	}
 	
 	return passed;
 }
 
 static int
 cib_rename(const char *old, const char *new) 
 {
     int rc = 0;
     char *automatic = NULL;
     if(new == NULL) {
 	automatic = crm_concat(cib_root, "cib.auto.XXXXXX", '/');
 	automatic = mktemp(automatic);
 	new = automatic;
 
 	crm_err("Archiving corrupt or unusable file %s as %s", old, automatic);
     }
 
     rc = rename(old, new);
     if(rc < 0) {
 	crm_perror(LOG_ERR, "Couldn't rename %s as %s - Disabling disk writes and continuing", old, new);
 	cib_writes_enabled = FALSE;
     }
     crm_free(automatic);
     return rc;
 }
 
 /*
  * It is the callers responsibility to free the output of this function
  */
 
 static xmlNode*
 retrieveCib(const char *filename, const char *sigfile, gboolean archive_invalid)
 {
     struct stat buf;
     xmlNode *root = NULL;
     
     crm_info("Reading cluster configuration from: %s (digest: %s)",
 	     filename, sigfile);
 
     if(stat(filename, &buf) != 0) {
 	crm_warn("Cluster configuration not found: %s", filename);
 	return NULL;
     }
 
     root = filename2xml(filename);    
     if(root == NULL) {
 	crm_err("%s exists but does NOT contain valid XML. ", filename);
 	crm_warn("Continuing but %s will NOT used.", filename);
 	
     } else if(validate_cib_digest(root, sigfile) == FALSE) {
 	crm_err("Checksum of %s failed!  Configuration contents ignored!", filename);
 	crm_err("Usually this is caused by manual changes, "
 		"please refer to http://clusterlabs.org/wiki/FAQ#cib_changes_detected");
 	crm_warn("Continuing but %s will NOT used.", filename);
 	free_xml(root);
 	root = NULL;
 
 	if(archive_invalid) {
 	    /* Archive the original files so the contents are not lost */
 	    cib_rename(filename, NULL);
 	    cib_rename(sigfile, NULL);
 	}
     }
     return root;
 }
 
 xmlNode*
 readCibXmlFile(const char *dir, const char *file, gboolean discard_status)
 {
 	int seq = 0;
 	char *filename = NULL, *sigfile = NULL;
 	const char *name = NULL;
 	const char *value = NULL;
 	const char *validation = NULL;
 	const char *use_valgrind = getenv("HA_VALGRIND_ENABLED");
 	
 	xmlNode *root = NULL;
 	xmlNode *status = NULL;
 
 	if(!crm_is_writable(dir, file, CRM_DAEMON_USER, NULL, FALSE)) {
 		cib_status = cib_bad_permissions;
 		return NULL;
 	}
 	
 	filename = crm_concat(dir, file, '/');
 	sigfile  = crm_concat(filename, "sig", '.');
 
 	cib_status = cib_ok;
 	root = retrieveCib(filename, sigfile, TRUE);
 
 	if(root == NULL) {
 	    crm_warn("Primary configuration corrupt or unusable, trying backup...");
 	    seq = get_last_sequence(cib_root, CIB_SERIES);
 	}
 	
 	while(root == NULL) {
 	    struct stat buf;
 	    char *backup_file = NULL;
 	    crm_free(sigfile);
 
 	    if(seq == 0) {
 		seq += cib_wrap; /* unwrap */
 	    }
 
 	    backup_file = generate_series_filename(cib_root, CIB_SERIES, seq-1, FALSE);
 	    sigfile = crm_concat(filename, "sig", '.');
 	    
 	    if(stat(backup_file, &buf) != 0) {
 		crm_debug("Backup file %s not found", backup_file);
 		break;
 	    }
 	    crm_warn("Attempting to load: %s", backup_file);
 	    root = retrieveCib(backup_file, sigfile, FALSE);
 	    seq--;
 	}
 
 	if(root == NULL) {
 	    root = createEmptyCib();
 	    crm_xml_add(root, XML_ATTR_GENERATION, "0");
 	    crm_xml_add(root, XML_ATTR_NUMUPDATES, "0");
 	    crm_xml_add(root, XML_ATTR_GENERATION_ADMIN, "0");
 	    crm_xml_add(root, XML_ATTR_VALIDATION, LATEST_SCHEMA_VERSION);
 	    crm_warn("Continuing with an empty configuration.");
 	}	
 
 	if(cib_writes_enabled && use_valgrind) {
 	    if(crm_is_true(use_valgrind) || strstr(use_valgrind, "cib")) {
 		cib_writes_enabled = FALSE;
 		crm_err("*********************************************************");
 		crm_err("*** Disabling disk writes to avoid confusing Valgrind ***");
 		crm_err("*********************************************************");
 	    }
 	}
 	
 	status = find_xml_node(root, XML_CIB_TAG_STATUS, FALSE);
 	if(discard_status && status != NULL) {
 		/* strip out the status section if there is one */
 		free_xml_from_parent(root, status);
 		status = NULL;
 	}
 	if(status == NULL) {
 		create_xml_node(root, XML_CIB_TAG_STATUS);		
 	}
 	
 	/* Do this before DTD validation happens */
 
 	/* fill in some defaults */
 	name = XML_ATTR_GENERATION_ADMIN;
 	value = crm_element_value(root, name);
 	if(value == NULL) {
 		crm_warn("No value for %s was specified in the configuration.",
 			 name);
 		crm_warn("The reccomended course of action is to shutdown,"
 			 " run crm_verify and fix any errors it reports.");
 		crm_warn("We will default to zero and continue but may get"
 			 " confused about which configuration to use if"
 			 " multiple nodes are powered up at the same time.");
 		crm_xml_add_int(root, name, 0);
 	}
 	
 	name = XML_ATTR_GENERATION;
 	value = crm_element_value(root, name);
 	if(value == NULL) {
 		crm_xml_add_int(root, name, 0);
 	}
 	
 	name = XML_ATTR_NUMUPDATES;
 	value = crm_element_value(root, name);
 	if(value == NULL) {
 		crm_xml_add_int(root, name, 0);
 	}
 	
 	/* unset these and require the DC/CCM to update as needed */
 	xml_remove_prop(root, XML_ATTR_DC_UUID);
 
 	if(discard_status) {
 		crm_log_xml_debug(root, "[on-disk]");
 	}
 
 	validation = crm_element_value(root, XML_ATTR_VALIDATION);
 	if(validate_xml(root, NULL, TRUE) == FALSE) {
 	    crm_err("CIB does not validate with %s", crm_str(validation));
 	    cib_status = cib_dtd_validation;
 		
 	} else if(validation == NULL) {
 	    int version = 0;
 	    update_validation(&root, &version, FALSE, FALSE);
 	    if(version > 0) {
 		crm_notice("Enabling %s validation on"
 			   " the existing (sane) configuration",
 			   get_schema_name(version));
 	    } else {
 		crm_err("CIB does not validate with any known DTD or schema");
 		cib_status = cib_dtd_validation;
 	    }
 	}
 
 	crm_free(filename);
 	crm_free(sigfile);
 	return root;
 }
 
 /*
  * The caller should never free the return value
  */
 xmlNode*
 get_the_CIB(void)
 {
 	return the_cib;
 }
 
 gboolean
 uninitializeCib(void)
 {
 	xmlNode *tmp_cib = the_cib;
 	
 	
 	if(tmp_cib == NULL) {
 		crm_debug("The CIB has already been deallocated.");
 		return FALSE;
 	}
 	
 	initialized = FALSE;
 	the_cib = NULL;
 	node_search = NULL;
 	resource_search = NULL;
 	constraint_search = NULL;
 	status_search = NULL;
 
 	crm_debug("Deallocating the CIB.");
 	
 	free_xml(tmp_cib);
 
 	crm_debug("The CIB has been deallocated.");
 	
 	return TRUE;
 }
 
 /*
  * This method will not free the old CIB pointer or the new one.
  * We rely on the caller to have saved a pointer to the old CIB
  *   and to free the old/bad one depending on what is appropriate.
  */
 gboolean
 initializeCib(xmlNode *new_cib)
 {
 	if(new_cib == NULL) {
 		return FALSE;
 	}
 	
 	the_cib = new_cib;
 	initialized = TRUE;
 	return TRUE;
 }
 
 static void
 sync_directory(const char *name) 
 {
     int fd = 0;
     DIR *directory = NULL;
     
     directory = opendir(name);
     if(directory == NULL) {
 	crm_perror(LOG_ERR, "Could not open %s for syncing", name);
 	return;
     }
 
     fd = dirfd(directory);
     if(fd < 0) {
     	crm_perror(LOG_ERR,"Could not obtain file descriptor for %s", name);
 
     } else if(fsync(fd) < 0) {
     	crm_perror(LOG_ERR,"Could not sync %s", name);
     }
 
     if(closedir(directory) < 0) {
     	crm_perror(LOG_ERR,"Could not close %s after fsync", name);
     }
 }
 
 /*
  * This method will free the old CIB pointer on success and the new one
  * on failure.
  */
 int
 activateCibXml(xmlNode *new_cib, gboolean to_disk, const char *op)
 {
 	xmlNode *saved_cib = the_cib;
 
 	CRM_ASSERT(new_cib != saved_cib);
 	if(initializeCib(new_cib) == FALSE) {
 		free_xml(new_cib);
 		crm_err("Ignoring invalid or NULL CIB");
 
 		if(saved_cib != NULL) {
 			crm_warn("Reverting to last known CIB");
 			if (initializeCib(saved_cib) == FALSE) {
 				/* oh we are so dead  */
 				crm_crit("Couldn't re-initialize the old CIB!");
 				cl_flush_logs();
 				exit(1);
 			}
 			
 		} else {
 			crm_crit("Could not write out new CIB and no saved"
 				 " version to revert to");
 		}
 		return cib_ACTIVATION;		
 	} 
 
 	free_xml(saved_cib);
 	if(cib_writes_enabled && cib_status == cib_ok && to_disk) {
 	    crm_debug("Triggering CIB write for %s op", op);
 	    G_main_set_trigger(cib_writer);
 	}
 	
 	return cib_ok;    
 }
 
 int
 write_cib_contents(gpointer p) 
 {
 	gboolean need_archive = FALSE;
 	struct stat buf;
 	char *digest = NULL;
 	int exit_rc = LSB_EXIT_OK;
 	xmlNode *cib_status_root = NULL;
 	
 	/* we can scribble on "the_cib" here and not affect the parent */
 	const char *epoch = crm_element_value(the_cib, XML_ATTR_GENERATION);
 	const char *admin_epoch = crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN);
 
 	char *tmp1 = crm_concat(cib_root, "cib.XXXXXX", '/');
 	char *tmp2 = crm_concat(cib_root, "cib.XXXXXX", '/');
 
 	char *primary_file = crm_concat(cib_root, "cib.xml", '/');
 	char *digest_file = crm_concat(primary_file, "sig", '.');
 	
 	/* Always write out with num_updates=0 */
 	crm_xml_add(the_cib, XML_ATTR_NUMUPDATES, "0");
 	
 	if(crm_log_level > LOG_INFO) {
 	    crm_log_level--;
 	}
 
 	need_archive = (stat(primary_file, &buf) == 0);
 	if (need_archive) {
 	    char *backup_file = NULL;
 	    char *backup_digest = NULL;
 	    int seq = get_last_sequence(cib_root, CIB_SERIES);
 
 	    /* check the admin didnt modify it underneath us */
 	    if(validate_on_disk_cib(primary_file, NULL) == FALSE) {
 		crm_err("%s was manually modified while the cluster was active!", primary_file);
 		exit_rc = LSB_EXIT_GENERIC;
 		goto cleanup;
 	    }
 
 	    backup_file = generate_series_filename(cib_root, CIB_SERIES, seq, FALSE);
 	    backup_digest = crm_concat(backup_file, "sig", '.');
 	    
 	    link(primary_file, backup_file);
 	    link(digest_file, backup_digest);
 	    write_last_sequence(cib_root, CIB_SERIES, seq+1, cib_wrap);
 	    sync_directory(cib_root);
 
 	    crm_info("Archived previous version as %s", backup_file);	
 	    
 	    crm_free(backup_digest);
 	    crm_free(backup_file);
 	}
 
 	/* Given that we discard the status section on startup
 	 *   there is no point writing it out in the first place
 	 *   since users just get confused by it
 	 *
 	 * So delete the status section before we write it out
 	 */
 	crm_debug("Writing CIB to disk");	    
 	if(p == NULL) {
 	    cib_status_root = find_xml_node(the_cib, XML_CIB_TAG_STATUS, TRUE);
 	    CRM_DEV_ASSERT(cib_status_root != NULL);
 	    
 	    if(cib_status_root != NULL) {
 		free_xml_from_parent(the_cib, cib_status_root);
 	    }
 	}
 
 	tmp1 = mktemp(tmp1); /* cib    */
 	tmp2 = mktemp(tmp2); /* digest */
 	
 	if(write_xml_file(the_cib, tmp1, FALSE) <= 0) {
 	    crm_err("Changes couldn't be written to %s", tmp1);
 		exit_rc = LSB_EXIT_GENERIC;
 		goto cleanup;
 	}
 	
 	/* Must calculate the digest after writing as write_xml_file() updates the last-written field */
 	digest = calculate_xml_digest(the_cib, FALSE, FALSE); 
 	crm_info("Wrote version %s.%s.0 of the CIB to disk (digest: %s)",
 		 admin_epoch?admin_epoch:"0",
 		 epoch?epoch:"0", digest);	
 
 	if(write_cib_digest(the_cib, tmp2, digest) <= 0) {
 	    crm_err("Digest couldn't be written to %s", tmp2);
 		exit_rc = LSB_EXIT_GENERIC;
 		goto cleanup;
 	}
 	crm_debug("Wrote digest %s to disk", digest);
 	CRM_ASSERT(retrieveCib(tmp1, tmp2, FALSE) != NULL);
 	sync_directory(cib_root);
 
 	crm_debug("Activating %s", tmp1);
 	cib_rename(tmp1, primary_file);
 	cib_rename(tmp2, digest_file);
 	sync_directory(cib_root);
 
   cleanup:
 	crm_free(primary_file);
 	crm_free(digest_file);
 	crm_free(digest);
 	crm_free(tmp2);
 	crm_free(tmp1);
 
 	if(p == NULL) {
 		/* fork-and-write mode */
 		exit(exit_rc);
 	}
 
 	/* stand-alone mode */
 	return exit_rc;
 }
 
 void GHFunc_count_peers(gpointer key, gpointer value, gpointer user_data)
 {
 	int *active = user_data;
 	if(safe_str_eq(value, ONLINESTATUS)) {
 		(*active)++;
 		
 	} else if(safe_str_eq(value, JOINSTATUS)) {
 		(*active)++;
 	}
 }
 
diff --git a/cib/notify.c b/cib/notify.c
old mode 100755
new mode 100644
index 16eeaef8a1..7315e010d4
--- a/cib/notify.c
+++ b/cib/notify.c
@@ -1,418 +1,415 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 
 
 
 #include <time.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/msg.h>
 #include <crm/common/xml.h>
 #include <cibio.h>
 #include <callbacks.h>
 #include <notify.h>
 
 
 int pending_updates = 0;
 extern GHashTable *client_list;
 
 void cib_notify_client(gpointer key, gpointer value, gpointer user_data);
 void attach_cib_generation(xmlNode *msg, const char *field, xmlNode *a_cib);
 
 void do_cib_notify(
 	int options, const char *op, xmlNode *update,
 	enum cib_errors result, xmlNode *result_data, const char *msg_type);
 
 static void
 need_pre_notify(gpointer key, gpointer value, gpointer user_data)
 {
     cib_client_t *client = value;
     if(client->pre_notify) {
 	gboolean *needed = user_data;
 	*needed = TRUE;
     }
 }
 
 static void
 need_post_notify(gpointer key, gpointer value, gpointer user_data)
 {
     cib_client_t *client = value;
     if(client->post_notify) {
 	gboolean *needed = user_data;
 	*needed = TRUE;
     }
 }
 
 void
 cib_notify_client(gpointer key, gpointer value, gpointer user_data)
 {
 
 	IPC_Channel *ipc_client = NULL;
 	xmlNode *update_msg = user_data;
 	cib_client_t *client = value;
 	const char *type = NULL;
 	gboolean is_pre = FALSE;
 	gboolean is_post = FALSE;	
 	gboolean is_confirm = FALSE;
 	gboolean is_replace = FALSE;
 	gboolean is_diff = FALSE;
 	gboolean do_send = FALSE;
 
 	int qlen = 0;
 	int max_qlen = 500;
 	
 	CRM_DEV_ASSERT(client != NULL);
 	CRM_DEV_ASSERT(update_msg != NULL);
 
 	type = crm_element_value(update_msg, F_SUBTYPE);
 	CRM_DEV_ASSERT(type != NULL);
 
 	if(client == NULL) {
 		crm_warn("Skipping NULL client");
 		return;
 
 	} else if(client->channel == NULL) {
 		crm_warn("Skipping client with NULL channel");
 		return;
 
 	} else if(client->name == NULL) {
 		crm_debug_2("Skipping unnammed client / comamnd channel");
 		return;
 	}
 	
 	if(safe_str_eq(type, T_CIB_PRE_NOTIFY)) {
 		is_pre = TRUE;
 		
 	} else if(safe_str_eq(type, T_CIB_POST_NOTIFY)) {
 		is_post = TRUE;
 
 	} else if(safe_str_eq(type, T_CIB_UPDATE_CONFIRM)) {
 		is_confirm = TRUE;
 
 	} else if(safe_str_eq(type, T_CIB_DIFF_NOTIFY)) {
 		is_diff = TRUE;
 
 	} else if(safe_str_eq(type, T_CIB_REPLACE_NOTIFY)) {
 		is_replace = TRUE;
 	}	
 
 	ipc_client = client->channel;
 	if (FALSE == crm_str_eq(client->channel_name, "remote", FALSE)) {
 	    qlen = ipc_client->send_queue->current_qlen;
 	    max_qlen = ipc_client->send_queue->max_qlen;
 	}
 
 #if 1
 	/* get_chan_status() causes memory to be allocated that isnt free'd
 	 *   until the message is read (which messes up the memory stats) 
 	 */
 	if(client->pre_notify && is_pre) {
 		if(qlen < (int)(0.4 * max_qlen)) {
 			do_send = TRUE;
 		} else {
 			crm_warn("Throttling pre-notifications due to"
 				 " high load: queue=%d (max=%d)",
 				 qlen, max_qlen);
 		}
 		 
 	} else if(client->post_notify && is_post) {
 		if(qlen < (int)(0.7 * max_qlen)) {
 			do_send = TRUE;
 		} else {
 			crm_warn("Throttling post-notifications due to"
 				 " extreme load: queue=%d (max=%d)",
 				 qlen, max_qlen);
 		}
 
 		/* these are critical */
 	} else
 #endif
 		if(client->diffs && is_diff) {
 		do_send = TRUE;
 
 	} else if(client->confirmations && is_confirm) {
 		do_send = TRUE;
 
 	} else if(client->replace && is_replace) {
 		do_send = TRUE;
 	}
 
 	if(do_send) {
 		if (crm_str_eq(client->channel_name, "remote", FALSE)) {
 		    crm_debug("Sent %s notification to client %s/%s",
 			      is_confirm?"Confirmation":is_post?"Post":"Pre",
 			      client->name, client->id);
 		    cib_send_remote_msg(client->channel, update_msg, client->encrypted);
 
 		} else if(ipc_client->send_queue->current_qlen >= ipc_client->send_queue->max_qlen) {
 			/* We never want the CIB to exit because our client is slow */
 			crm_crit("%s-notification of client %s/%s failed - queue saturated",
 				 is_confirm?"Confirmation":is_post?"Post":"Pre",
 				 client->name, client->id);
 			
 		} else if(send_ipc_message(ipc_client, update_msg) == FALSE) {
 			crm_warn("Notification of client %s/%s failed",
 				 client->name, client->id);
 		}
 	}
 }
 
 void
 cib_pre_notify(
 	int options, const char *op, xmlNode *existing, xmlNode *update) 
 {
 	xmlNode *update_msg = NULL;
 	const char *type = NULL;
 	const char *id = NULL;
 	gboolean needed = FALSE;
 
 	g_hash_table_foreach(client_list, need_pre_notify, &needed);
 	if(needed == FALSE) {
 	    return;
 	}
 
 	/* TODO: consider pre-notification for removal */
 	update_msg = create_xml_node(NULL, "pre-notify");
 
 	if(update != NULL) {
 		id = crm_element_value(update, XML_ATTR_ID);
 	}
 	
 	crm_xml_add(update_msg, F_TYPE, T_CIB_NOTIFY);
 	crm_xml_add(update_msg, F_SUBTYPE, T_CIB_PRE_NOTIFY);
 	crm_xml_add(update_msg, F_CIB_OPERATION, op);
 
 	if(id != NULL) {
 		crm_xml_add(update_msg, F_CIB_OBJID, id);
 	}
 
 	if(update != NULL) {
 		crm_xml_add(update_msg, F_CIB_OBJTYPE, crm_element_name(update));
 	} else if(existing != NULL) {
 		crm_xml_add(update_msg, F_CIB_OBJTYPE, crm_element_name(existing));
 	}
 
 	type = crm_element_value(update_msg, F_CIB_OBJTYPE);	
 	attach_cib_generation(update_msg, "cib_generation", the_cib);
 	
 	if(existing != NULL) {
 		add_message_xml(update_msg, F_CIB_EXISTING, existing);
 	}
 	if(update != NULL) {
 		add_message_xml(update_msg, F_CIB_UPDATE, update);
 	}
 
 	g_hash_table_foreach(client_list, cib_notify_client, update_msg);
 	
 	if(update == NULL) {
 		crm_debug_2("Performing operation %s (on section=%s)",
 			    op, type);
 
 	} else {
 		crm_debug_2("Performing %s on <%s%s%s>",
 			    op, type, id?" id=":"", id?id:"");
 	}
 		
 	free_xml(update_msg);
 }
 
 void
 cib_post_notify(int options, const char *op, xmlNode *update,
 		enum cib_errors result, xmlNode *new_obj) 
 {
 	gboolean needed = FALSE;
 	g_hash_table_foreach(client_list, need_post_notify, &needed);
 	if(needed == FALSE) {
 	    return;
 	}
 
 	do_cib_notify(
 		options, op, update, result, new_obj, T_CIB_UPDATE_CONFIRM);
 }
 
 void
 cib_diff_notify(
 	int options, const char *client, const char *call_id, const char *op,
 	xmlNode *update, enum cib_errors result, xmlNode *diff) 
 {
 	int add_updates = 0;
 	int add_epoch  = 0;
 	int add_admin_epoch = 0;
 
 	int del_updates = 0;
 	int del_epoch  = 0;
 	int del_admin_epoch = 0;
 
 	int log_level = LOG_DEBUG_2;
 	
 	if(diff == NULL) {
 		return;
 	}
 
 	if(result != cib_ok) {
 		log_level = LOG_WARNING;
 	}
 	
 	cib_diff_version_details(
 		diff, &add_admin_epoch, &add_epoch, &add_updates, 
 		&del_admin_epoch, &del_epoch, &del_updates);
 
 	if(add_updates != del_updates) {
 		do_crm_log(log_level,
 			      "Update (client: %s%s%s): %d.%d.%d -> %d.%d.%d (%s)",
 			      client, call_id?", call:":"", call_id?call_id:"",
 			      del_admin_epoch, del_epoch, del_updates,
 			      add_admin_epoch, add_epoch, add_updates,
 			      cib_error2string(result));
 
 	} else if(diff != NULL) {
 		do_crm_log(log_level,
 			      "Local-only Change (client:%s%s%s): %d.%d.%d (%s)",
 			      client, call_id?", call: ":"", call_id?call_id:"",
 			      add_admin_epoch, add_epoch, add_updates,
 			      cib_error2string(result));
 	}
 	
 	do_cib_notify(options, op, update, result, diff, T_CIB_DIFF_NOTIFY);
 }
 
 void
 do_cib_notify(
 	int options, const char *op, xmlNode *update,
 	enum cib_errors result, xmlNode *result_data, const char *msg_type) 
 {
 	xmlNode *update_msg = NULL;
-	const char *type = NULL;
 	const char *id = NULL;
 
 	update_msg = create_xml_node(NULL, "notify");
 
 	if(result_data != NULL) {
 		id = crm_element_value(result_data, XML_ATTR_ID);
 	}
 	
 	crm_xml_add(update_msg, F_TYPE, T_CIB_NOTIFY);
 	crm_xml_add(update_msg, F_SUBTYPE, msg_type);
 	crm_xml_add(update_msg, F_CIB_OPERATION, op);
 	crm_xml_add_int(update_msg, F_CIB_RC, result);
 	
 	if(id != NULL) {
 		crm_xml_add(update_msg, F_CIB_OBJID, id);
 	}
 
 	if(update != NULL) {
 		crm_debug_4("Setting type to update->name: %s",
 			    crm_element_name(update));
 		crm_xml_add(update_msg, F_CIB_OBJTYPE, crm_element_name(update));
-		type = crm_element_name(update);
 
 	} else if(result_data != NULL) {
 		crm_debug_4("Setting type to new_obj->name: %s",
 			    crm_element_name(result_data));
 		crm_xml_add(update_msg, F_CIB_OBJTYPE, crm_element_name(result_data));
-		type = crm_element_name(result_data);
 		
 	} else {
 		crm_debug_4("Not Setting type");
 	}
 
 	attach_cib_generation(update_msg, "cib_generation", the_cib);
 	if(update != NULL) {
 		add_message_xml(update_msg, F_CIB_UPDATE, update);
 	}
 	if(result_data != NULL) {
 		add_message_xml(update_msg, F_CIB_UPDATE_RESULT, result_data);
 	}
 
 	crm_debug_3("Notifying clients");
 	g_hash_table_foreach(client_list, cib_notify_client, update_msg);
 	free_xml(update_msg);
 	crm_debug_3("Notify complete");
 }
 
 
 void
 attach_cib_generation(xmlNode *msg, const char *field, xmlNode *a_cib) 
 {
 	xmlNode *generation = create_xml_node(
 		NULL, XML_CIB_TAG_GENERATION_TUPPLE);
 
 	if(a_cib != NULL) {
 		copy_in_properties(generation, a_cib);
 	}
 	add_message_xml(msg, field, generation);
 	free_xml(generation);
 }
 
 void
 cib_replace_notify(const char *origin, xmlNode *update, enum cib_errors result, xmlNode *diff) 
 {
 	xmlNode *replace_msg = NULL;
 	
 	int add_updates = 0;
 	int add_epoch  = 0;
 	int add_admin_epoch = 0;
 
 	int del_updates = 0;
 	int del_epoch  = 0;
 	int del_admin_epoch = 0;
 
 	if(diff == NULL) {
 		return;
 	}
 
 	cib_diff_version_details(
 		diff, &add_admin_epoch, &add_epoch, &add_updates, 
 		&del_admin_epoch, &del_epoch, &del_updates);
 
 	if(add_updates != del_updates) {
 		crm_info("Replaced: %d.%d.%d -> %d.%d.%d from %s",
 			 del_admin_epoch, del_epoch, del_updates,
 			 add_admin_epoch, add_epoch, add_updates,
 			 crm_str(origin));
 	} else if(diff != NULL) {
 		crm_info("Local-only Replace: %d.%d.%d from %s",
 			 add_admin_epoch, add_epoch, add_updates,
 			 crm_str(origin));
 	}
 	
 	replace_msg = create_xml_node(NULL, "notify-replace");
 	crm_xml_add(replace_msg, F_TYPE, T_CIB_NOTIFY);
 	crm_xml_add(replace_msg, F_SUBTYPE, T_CIB_REPLACE_NOTIFY);
 	crm_xml_add(replace_msg, F_CIB_OPERATION, CIB_OP_REPLACE);
 	crm_xml_add_int(replace_msg, F_CIB_RC, result);
 	attach_cib_generation(replace_msg, "cib-replace-generation", update);
 
 	crm_log_xml(LOG_DEBUG_2,"CIB Replaced", replace_msg);
 	
 	g_hash_table_foreach(client_list, cib_notify_client, replace_msg);
 	free_xml(replace_msg);
 }
diff --git a/crmd/te_actions.c b/crmd/te_actions.c
index f0933e974d..a2f4e79a90 100644
--- a/crmd/te_actions.c
+++ b/crmd/te_actions.c
@@ -1,565 +1,560 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/msg.h>
 #include <crm/common/xml.h>
 #include <tengine.h>
 
 #include <crmd_fsa.h>
 #include <crmd_messages.h>
 #include <crm/common/cluster.h>
 
 char *te_uuid = NULL;
 
 void send_rsc_command(crm_action_t *action);
 
 static void
 te_start_action_timer(crm_graph_t *graph, crm_action_t *action) 
 {
 	crm_malloc0(action->timer, sizeof(crm_action_timer_t));
 	action->timer->timeout   = action->timeout;
 	action->timer->reason    = timeout_action;
 	action->timer->action    = action;
 	action->timer->source_id = g_timeout_add(
 		action->timer->timeout + graph->network_delay,
 		action_timer_callback, (void*)action->timer);
 
 	CRM_ASSERT(action->timer->source_id != 0);
 }
 
 
 static gboolean
 te_pseudo_action(crm_graph_t *graph, crm_action_t *pseudo) 
 {
 	crm_info("Pseudo action %d fired and confirmed", pseudo->id);
 	pseudo->confirmed = TRUE;
 	update_graph(graph, pseudo);
 	trigger_graph();
 	return TRUE;
 }
 
 void
 send_stonith_update(crm_action_t *action, const char *target, const char *uuid)
 {
 	enum cib_errors rc = cib_ok;
 	
 	/* zero out the node-status & remove all LRM status info */
 	xmlNode *node_state = create_xml_node(NULL, XML_CIB_TAG_STATE);
 	
 	CRM_CHECK(target != NULL, return);
 	CRM_CHECK(uuid != NULL, return);
 	
 	crm_xml_add(node_state, XML_ATTR_UUID,  uuid);
 	crm_xml_add(node_state, XML_ATTR_UNAME, target);
 	crm_xml_add(node_state, XML_CIB_ATTR_HASTATE,   DEADSTATUS);
 	crm_xml_add(node_state, XML_CIB_ATTR_INCCM,     XML_BOOLEAN_NO);
 	crm_xml_add(node_state, XML_CIB_ATTR_CRMDSTATE, OFFLINESTATUS);
 	crm_xml_add(node_state, XML_CIB_ATTR_JOINSTATE, CRMD_JOINSTATE_DOWN);
 	crm_xml_add(node_state, XML_CIB_ATTR_EXPSTATE,  CRMD_JOINSTATE_DOWN);
 	crm_xml_add(node_state, XML_ATTR_ORIGIN,   __FUNCTION__);
 	
 	rc = fsa_cib_conn->cmds->update(
 		fsa_cib_conn, XML_CIB_TAG_STATUS, node_state,
 		cib_quorum_override|cib_scope_local|cib_can_create);	
 	
 	/* Delay processing the trigger until the update completes */
 	crm_info("Sending fencing update %d for %s", rc, target);
 	add_cib_op_callback(fsa_cib_conn, rc, FALSE, crm_strdup(target), cib_fencing_updated);
 
 	/* Make sure it sticks */
 	/* fsa_cib_conn->cmds->bump_epoch(fsa_cib_conn, cib_quorum_override|cib_scope_local);	 */
 	
 	erase_status_tag(target, XML_CIB_TAG_LRM, cib_scope_local);
 	erase_status_tag(target, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
 	
 	free_xml(node_state);
 	
 #if 0
 	/* Make sure the membership cache is accurate */ 
 	crm_update_peer(0, 0, 0, -1, 0, uuid, target, NULL, CRM_NODE_LOST);
 #endif
 	
 	return;
 }
 
 static gboolean
 te_fence_node(crm_graph_t *graph, crm_action_t *action)
 {
 	int rc = 0;
 	const char *id = NULL;
 	const char *uuid = NULL;
 	const char *target = NULL;
 	const char *type = NULL;
 	gboolean invalid_action = FALSE;
 	
 	id = ID(action->xml);
 	target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 	uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
 	type = crm_meta_value(action->params, "stonith_action");
 	
 	CRM_CHECK(id != NULL,     invalid_action = TRUE);
 	CRM_CHECK(uuid != NULL,   invalid_action = TRUE);
 	CRM_CHECK(type != NULL,   invalid_action = TRUE);
 	CRM_CHECK(target != NULL, invalid_action = TRUE);
 
 	if(invalid_action) {
 	    crm_log_xml_warn(action->xml, "BadAction");
 	    return FALSE;
 	}
 
 	te_log_action(LOG_INFO,
 		      "Executing %s fencing operation (%s) on %s (timeout=%d)",
 		      type, id, target, transition_graph->stonith_timeout);
 
 	/* Passing NULL means block until we can connect... */
 	te_connect_stonith(NULL);
 
-	if(type == NULL) {
-	    type = "reboot";
-	}
-
 	rc = stonith_api->cmds->fence(
 	    stonith_api, 0, target, action->params, type, transition_graph->stonith_timeout/1000);
 
 	stonith_api->cmds->register_callback(
 	    stonith_api, rc, transition_graph->stonith_timeout/1000, FALSE,
 	    generate_transition_key(transition_graph->id, action->id, 0, te_uuid),
 	    "tengine_stonith_callback", tengine_stonith_callback);
 	
 	return TRUE;
 }
 
 static int get_target_rc(crm_action_t *action) 
 {
 	const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);
 
 	if(target_rc_s != NULL) {
 		return crm_parse_int(target_rc_s, "0");
 	}
 	return 0;
 }
 
 static gboolean
 te_crm_command(crm_graph_t *graph, crm_action_t *action)
 {
 	char *counter = NULL;
 	xmlNode *cmd = NULL;
 	gboolean is_local = FALSE;
 
 	const char *id = NULL;
 	const char *task = NULL;
 	const char *value = NULL;
 	const char *on_node = NULL;
 
 	gboolean rc = TRUE;
 	gboolean no_wait = FALSE;
 
 	id      = ID(action->xml);
 	task    = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 	on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 
 	CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
 		  te_log_action(LOG_ERR, "Corrupted command (id=%s) %s: no node",
 				crm_str(id), crm_str(task));
 		  return FALSE);
 	
 	te_log_action(LOG_INFO, "Executing crm-event (%s): %s on %s%s%s",
 		      crm_str(id), crm_str(task), on_node,
 		      is_local?" (local)":"", no_wait?" - no waiting":"");
 
 	if(safe_str_eq(on_node, fsa_our_uname)) {
 	    is_local = TRUE;
 	}
 	
 	value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
 	if(crm_is_true(value)) {
 	    no_wait = TRUE;
 	}
 
 	if(is_local && safe_str_eq(task, CRM_OP_SHUTDOWN)) {
 	    /* defer until everything else completes */
 	    te_log_action(LOG_INFO, "crm-event (%s) is a local shutdown", crm_str(id));
 	    graph->completion_action = tg_shutdown;
 	    graph->abort_reason = "local shutdown";
 	    action->confirmed = TRUE;
 	    update_graph(graph, action);
 	    trigger_graph();
 	    return TRUE;
 	}
 	
 	cmd = create_request(task, NULL, on_node, CRM_SYSTEM_CRMD,
 			     CRM_SYSTEM_TENGINE, NULL);
 	
 	counter = generate_transition_key(
 	    transition_graph->id, action->id, get_target_rc(action), te_uuid);
 	crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter);
 
 	rc = send_cluster_message(on_node, crm_msg_crmd, cmd, TRUE);
 	crm_free(counter);
 	free_xml(cmd);
 	
-	value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
 	if(rc == FALSE) {
 		crm_err("Action %d failed: send", action->id);
 		return FALSE;
 		
 	} else if(no_wait) {
 		action->confirmed = TRUE;
 		update_graph(graph, action);
 		trigger_graph();
 		
 	} else {
 	    if(action->timeout <= 0) {
 		crm_err("Action %d: %s on %s had an invalid timeout (%dms).  Using %dms instead",
 			action->id, task, on_node, action->timeout, graph->network_delay);
 		action->timeout = graph->network_delay;
 	    }
 	    te_start_action_timer(graph, action);
 	}
 
 	return TRUE;
 }
 
 gboolean
 cib_action_update(crm_action_t *action, int status, int op_rc)
 {
 	char *op_id  = NULL;
 	char *code   = NULL;
 	char *digest = NULL;
 	xmlNode *tmp      = NULL;
 	xmlNode *params   = NULL;
 	xmlNode *state    = NULL;
 	xmlNode *rsc      = NULL;
 	xmlNode *xml_op   = NULL;
 	xmlNode *action_rsc = NULL;
 
 	enum cib_errors rc = cib_ok;
 
 	const char *name   = NULL;
 	const char *value  = NULL;
 	const char *rsc_id = NULL;
 	const char *task   = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 	const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 	const char *task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 	const char *target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
 
 	int call_options = cib_quorum_override|cib_scope_local;
 
 	if(status == LRM_OP_PENDING) {
 	    crm_debug("%s %d: Recording pending operation %s on %s",
 		     crm_element_name(action->xml), action->id, task_uuid, target);
 	} else {
 	    crm_warn("%s %d: %s on %s timed out",
 		     crm_element_name(action->xml), action->id, task_uuid, target);
 	}
 	
 	action_rsc = find_xml_node(action->xml, XML_CIB_TAG_RESOURCE, TRUE);
 	if(action_rsc == NULL) {
 		return FALSE;
 	}
 	
 	rsc_id = ID(action_rsc);
 	CRM_CHECK(rsc_id != NULL,
 		  crm_log_xml_err(action->xml, "Bad:action");
 		  return FALSE);
 	
 /*
   update the CIB
 
 <node_state id="hadev">
       <lrm>
         <lrm_resources>
           <lrm_resource id="rsc2" last_op="start" op_code="0" target="hadev"/>
 */
 
 	state = create_xml_node(NULL, XML_CIB_TAG_STATE);
 
 	crm_xml_add(state, XML_ATTR_UUID,  target_uuid);
 	crm_xml_add(state, XML_ATTR_UNAME, target);
 	
 	rsc = create_xml_node(state, XML_CIB_TAG_LRM);
 	crm_xml_add(rsc, XML_ATTR_ID, target_uuid);
 
 	rsc = create_xml_node(rsc,   XML_LRM_TAG_RESOURCES);
 	rsc = create_xml_node(rsc,   XML_LRM_TAG_RESOURCE);
 	crm_xml_add(rsc, XML_ATTR_ID, rsc_id);
 
 	name = XML_ATTR_TYPE;
 	value = crm_element_value(action_rsc, name);
 	crm_xml_add(rsc, name, value);
 	name = XML_AGENT_ATTR_CLASS;
 	value = crm_element_value(action_rsc, name);
 	crm_xml_add(rsc, name, value);
 	name = XML_AGENT_ATTR_PROVIDER;
 	value = crm_element_value(action_rsc, name);
 	crm_xml_add(rsc, name, value);
 
 	xml_op = create_xml_node(rsc, XML_LRM_TAG_RSC_OP);	
 	crm_xml_add(xml_op, XML_ATTR_ID, task);
 	
 	op_id = generate_op_key(rsc_id, task, action->interval);
 	crm_xml_add(xml_op, XML_ATTR_ID, op_id);
 	crm_free(op_id);
 	
 	crm_xml_add_int(xml_op, XML_LRM_ATTR_CALLID, -1);
 	crm_xml_add(xml_op, XML_LRM_ATTR_TASK, task);
 	crm_xml_add(xml_op, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
 	crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS, status);
 	crm_xml_add_int(xml_op, XML_LRM_ATTR_INTERVAL, action->interval);
 	crm_xml_add_int(xml_op, XML_LRM_ATTR_RC, op_rc);
 	crm_xml_add(xml_op, XML_ATTR_ORIGIN, __FUNCTION__);
 
 	if(crm_str_eq(task, CRMD_ACTION_MIGRATED, TRUE)) {
 	    char *key = crm_meta_name("migrate_source_uuid");
 	    xmlNode *attrs = first_named_child(action->xml, XML_TAG_ATTRS);
 	    const char *host = crm_element_value(attrs, key);
 	    CRM_CHECK(host != NULL, crm_log_xml_err(action->xml, "Bad Op"));
 	    crm_xml_add(xml_op, CRMD_ACTION_MIGRATED, host);
 	    crm_free(key);
 	}	
 	
 	code = generate_transition_key(
 	    transition_graph->id, action->id, get_target_rc(action), te_uuid);
 	crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY, code);
 	crm_free(code);
 
 	code = generate_transition_magic(
 		crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY), status, op_rc);
 	crm_xml_add(xml_op,  XML_ATTR_TRANSITION_MAGIC, code);
 	crm_free(code);
 
 	tmp = find_xml_node(action->xml, "attributes", TRUE);
 	params = create_xml_node(NULL, XML_TAG_PARAMS);
 	copy_in_properties(params, tmp);
 	
 	filter_action_parameters(params, CRM_FEATURE_SET);
 	digest = calculate_xml_digest(params, TRUE, FALSE);
 
 	/* info for now as this area has been problematic to debug */
 	crm_debug("Calculated digest %s for %s (%s)\n", 
 		  digest, ID(xml_op),
 		  crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
 	crm_log_xml(LOG_DEBUG,  "digest:source", params);
 
 	crm_xml_add(xml_op, XML_LRM_ATTR_OP_DIGEST, digest);
 	crm_free(digest);
 	free_xml(params);
 	
 	crm_debug_3("Updating CIB with \"%s\" (%s): %s %s on %s",
 		  status<0?"new action":XML_ATTR_TIMEOUT,
 		  crm_element_name(action->xml), crm_str(task), rsc_id, target);
 	
 	rc = fsa_cib_conn->cmds->update(
 		fsa_cib_conn, XML_CIB_TAG_STATUS, state, call_options);
 
 	crm_debug_2("Updating CIB with %s action %d: %s on %s (call_id=%d)",
 		  op_status2text(status), action->id, task_uuid, target, rc);
 
 	add_cib_op_callback(fsa_cib_conn, rc, FALSE, NULL, cib_action_updated);
 	free_xml(state);
 
 	action->sent_update = TRUE;
 	
 	if(rc < cib_ok) {
 		return FALSE;
 	}
 
 	return TRUE;
 }
 
 
 static gboolean
 te_rsc_command(crm_graph_t *graph, crm_action_t *action) 
 {
 	/* never overwrite stop actions in the CIB with
 	 *   anything other than completed results
 	 *
 	 * Writing pending stops makes it look like the
 	 *   resource is running again
 	 */
 	xmlNode *cmd = NULL;
 	xmlNode *rsc_op  = NULL;
 
 	gboolean rc = TRUE;
 	gboolean no_wait = FALSE;
 	gboolean is_local = FALSE;
 	
 	char *counter = NULL;
 	const char *task    = NULL;
 	const char *value   = NULL;
 	const char *on_node = NULL;
 	const char *task_uuid = NULL;
 
 	CRM_ASSERT(action != NULL);
 	CRM_ASSERT(action->xml != NULL);
 
 	action->executed = FALSE;
 	on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 
 	CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
 		  te_log_action(LOG_ERR, "Corrupted command(id=%s) %s: no node",
 				ID(action->xml), crm_str(task));
 		  return FALSE);
 
 	rsc_op  = action->xml;
 	task    = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
 	task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 	on_node = crm_element_value(rsc_op, XML_LRM_ATTR_TARGET);
 	counter = generate_transition_key(
 	    transition_graph->id, action->id, get_target_rc(action), te_uuid);
 	crm_xml_add(rsc_op, XML_ATTR_TRANSITION_KEY, counter);
 
 	if(safe_str_eq(on_node, fsa_our_uname)) {
 	    is_local = TRUE;
 	}
 
 	value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
 	if(crm_is_true(value)) {
 	    no_wait = TRUE;
 	}
 	
 	crm_info("Initiating action %d: %s %s on %s%s%s",
 		 action->id, task, task_uuid, on_node,
 		 is_local?" (local)":"", no_wait?" - no waiting":"");
 
 	cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, on_node,
 			     CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);
 	
 	if(is_local) {
 	    /* shortcut local resource commands */
 	    ha_msg_input_t data = {
 		.msg = cmd,
 		.xml = rsc_op,
 	    };
 	    
 	    fsa_data_t msg = {
 		.id = 0,
 		.data = &data,
 		.data_type = fsa_dt_ha_msg,
 		.fsa_input = I_NULL,
 		.fsa_cause = C_FSA_INTERNAL,
 		.actions = A_LRM_INVOKE,
 		.origin = __FUNCTION__,
 	    };
 
 	    do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, fsa_state, I_NULL, &msg);
 
 	} else {
 	    rc = send_cluster_message(on_node, crm_msg_lrmd, cmd, TRUE);
 	}
 	
 	crm_free(counter);
 	free_xml(cmd);
 	
 	action->executed = TRUE;
 	if(rc == FALSE) {
 		crm_err("Action %d failed: send", action->id);
 		return FALSE;
 		
 	} else if(no_wait) {
 		action->confirmed = TRUE;
 		update_graph(transition_graph, action);
 		trigger_graph();
 
 	} else {
 	    if(action->timeout <= 0) {
 		crm_err("Action %d: %s %s on %s had an invalid timeout (%dms).  Using %dms instead",
 			action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
 		action->timeout = graph->network_delay;
 	    }
 	    te_start_action_timer(graph, action);
 	}
 
 	value = crm_meta_value(action->params, XML_OP_ATTR_PENDING);
 	if(crm_is_true(value)) {
 	    /* write a "pending" entry to the CIB, inhibit notification */
 	    crm_info("Recording pending op %s in the CIB", task_uuid);
 	    cib_action_update(action, LRM_OP_PENDING, EXECRA_STATUS_UNKNOWN);
 	}
 	
 	return TRUE;
 }
 
 crm_graph_functions_t te_graph_fns = {
 	te_pseudo_action,
 	te_rsc_command,
 	te_crm_command,
 	te_fence_node
 };
 
 void
 notify_crmd(crm_graph_t *graph)
 {
 	int log_level = LOG_DEBUG;
 	const char *type = "unknown";
 	enum crmd_fsa_input event = I_NULL;
 	
 	crm_debug("Processing transition completion in state %s", fsa_state2string(fsa_state));
 	
 	CRM_CHECK(graph->complete, graph->complete = TRUE);
 
 	switch(graph->completion_action) {
 		case tg_stop:
 		    type = "stop";
 		    /* fall through */
 		case tg_done:
 		    type = "done";
 		    log_level = LOG_INFO;
 		    if(fsa_state == S_TRANSITION_ENGINE) {
 			event = I_TE_SUCCESS;
 		    }
 		    break;
 		    
 		case tg_restart:
 		    type = "restart";
 		    if(fsa_state == S_TRANSITION_ENGINE) {
 			event = I_PE_CALC;
 
 		    } else if(fsa_state == S_POLICY_ENGINE) {
 			register_fsa_action(A_PE_INVOKE);
 		    }
 		    break;
 
 		case tg_shutdown:
 		    type = "shutdown";
 		    if(is_set(fsa_input_register, R_SHUTDOWN)) {
 			event = I_STOP;			
 			
 		    } else {
 			event = I_TERMINATE;
 		    }
 	}
 
 	te_log_action(log_level, "Transition %d status: %s - %s",
 		      graph->id, type, crm_str(graph->abort_reason));
 
 	graph->abort_reason = NULL;
 	graph->completion_action = tg_done;
 	clear_bit_inplace(fsa_input_register, R_IN_TRANSITION);
 
 	if(event != I_NULL) {
 	    register_fsa_input(C_FSA_INTERNAL, event, NULL);
 	}
 	
 }
diff --git a/crmd/te_events.c b/crmd/te_events.c
index 28542f5a01..3e24b5a670 100644
--- a/crmd/te_events.c
+++ b/crmd/te_events.c
@@ -1,495 +1,495 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/msg.h>
 #include <crm/common/xml.h>
 #include <tengine.h>
 
 #include <lrm/lrm_api.h>
 #include <crmd_fsa.h>
 
 char *failed_stop_offset = NULL;
 char *failed_start_offset = NULL;
 
 int match_graph_event(int action_id, xmlNode *event, const char *event_node,
 		      int op_status, int op_rc, int target_rc);
 
 gboolean
 need_abort(xmlNode *update)
 {
     xmlNode *xml = NULL;
     if(update == NULL) {
 	return FALSE;
     }
     
-    xml_prop_iter(update, name, value,
+    xml_prop_name_iter(update, name,
 		  if(safe_str_eq(name, XML_ATTR_HAVE_QUORUM)) {
 		      goto do_abort; /* possibly not required */
 		  } else if(safe_str_eq(name, XML_ATTR_GENERATION)) {
 		      goto do_abort;
 		  } else if(safe_str_eq(name, XML_ATTR_GENERATION_ADMIN)) {
 		      goto do_abort;
 		  }
 		  continue;
       do_abort:
 		  abort_transition(INFINITY, tg_restart, "Non-status change", NULL);
 		  crm_info("Aborting on change to %s", name);
 		  return TRUE;
 	);
     
     xml = get_object_root(XML_CIB_TAG_CONFIGURATION, update);
     if(xml != NULL) {
 	abort_transition(INFINITY, tg_restart, "Non-status change", xml);
 	return TRUE;
     }
     
     return FALSE;
 }
 
 gboolean
 fail_incompletable_actions(crm_graph_t *graph, const char *down_node) 
 {
 	const char *target = NULL;
 	xmlNode *last_action = NULL;
 
 	slist_iter(
 		synapse, synapse_t, graph->synapses, lpc,
 		if (synapse->confirmed) {
 			continue;
 		}
 
 		slist_iter(
 			action, crm_action_t, synapse->actions, lpc,
 
 			if(action->type == action_type_pseudo || action->confirmed) {
 			    continue;
 			} else if(action->type == action_type_crm) {
 			    const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 			    if(safe_str_eq(task, CRM_OP_FENCE)) {
 				continue;
 			    }
 			}
 			
 			target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
 			if(safe_str_eq(target, down_node)) {
 				action->failed = TRUE;
 				last_action = action->xml;
 				update_graph(graph, action);
 				crm_notice("Action %d (%s) is scheduled for %s (offline)",
 					   action->id, ID(action->xml), down_node);
 			}
 			);
 		);
 
 	if(last_action != NULL) {
 		crm_warn("Node %s shutdown resulted in un-runnable actions", down_node);
 		abort_transition(INFINITY, tg_restart, "Node failure", last_action);
 		return TRUE;
 	}
 	
 	return FALSE;
 }
 
 static gboolean
 update_failcount(xmlNode *event, const char *event_node, int rc, int target_rc, gboolean force) 
 {
 	int interval = 0;
 	char *task = NULL;
 	char *rsc_id = NULL;
 	char *attr_name = NULL;
 	const char *id  = ID(event);
 	const char *on_uname  = get_uname(event_node);
 	const char *value = NULL;
 
 	if(rc == 99) {
 		/* this is an internal code for "we're busy, try again" */
 		return FALSE;
 
 	} else if(rc == target_rc) {
 	    return FALSE;
 	}
 
 	if(failed_stop_offset == NULL) {
 	    failed_stop_offset = crm_strdup(INFINITY_S);
 	}
 
 	if(failed_start_offset == NULL) {
 	    failed_start_offset = crm_strdup(INFINITY_S);
 	}
 	
 	CRM_CHECK(on_uname != NULL, return TRUE);
 
 	CRM_CHECK(parse_op_key(id, &rsc_id, &task, &interval),
 		  crm_err("Couldn't parse: %s", ID(event));
 		  goto bail);
 	CRM_CHECK(task != NULL, goto bail);
 	CRM_CHECK(rsc_id != NULL, goto bail);
 
 	if(safe_str_eq(task, CRMD_ACTION_START)) {
 	    interval = 1;
 	    value = failed_start_offset;
 
 	} else if(safe_str_eq(task, CRMD_ACTION_STOP)) {
 	    interval = 1;
 	    value = failed_stop_offset;
 	}
 
 	if(value == NULL || safe_str_neq(value, INFINITY_S)) {
 	    value = XML_NVPAIR_ATTR_VALUE"++";
 	}
 
 	if(interval > 0 || force) {
 		char *now = crm_itoa(time(NULL));
 		
 		crm_warn("Updating failcount for %s on %s after failed %s:"
 			 " rc=%d (update=%s, time=%s)", rsc_id, on_uname, task, rc, value, now);
 
 		attr_name = crm_concat("fail-count", rsc_id, '-');
 		update_attrd(on_uname, attr_name, value);
 		crm_free(attr_name);
 
 		attr_name = crm_concat("last-failure", rsc_id, '-');
 		update_attrd(on_uname, attr_name, now);
 		crm_free(attr_name);
 
 		crm_free(now);
 	}
 
   bail:
 	crm_free(rsc_id);
 	crm_free(task);
 	return TRUE;
 }
 
 static int
 status_from_rc(crm_action_t *action, int orig_status, int rc, int target_rc)
 {
 	int status = orig_status;
 	if(target_rc == rc) {
 	    crm_debug_2("Target rc: == %d", rc);
 	    if(status != LRM_OP_DONE) {
 		crm_debug_2("Re-mapping op status to"
 			    " LRM_OP_DONE for rc=%d", rc);
 		status = LRM_OP_DONE;
 	    }
 
 	} else {
 	    status = LRM_OP_ERROR;
 	}
 	
 	/* 99 is the code we use for direct nack's */
 	if(rc != 99 && status != LRM_OP_DONE) {
 		const char *task, *uname;
 		task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 		uname  = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 		crm_warn("Action %d (%s) on %s failed (target: %d vs. rc: %d): %s",
 			 action->id, task, uname, target_rc, rc, op_status2text(status));
 	}
 
 	return status;
 }
 
 /*
  * returns the ID of the action if a match is found
  * returns -1 if a match was not found
  * returns -2 if a match was found but the action failed (and was
  *            not allowed to)
  */
 int
 match_graph_event(int action_id, xmlNode *event, const char *event_node,
 		  int op_status, int op_rc, int target_rc)
 {
 	const char *target = NULL;
 	const char *allow_fail = NULL;
 	const char *this_event = NULL;
 	crm_action_t *action = NULL;
 
 	action = get_action(action_id, FALSE);
 	if(action == NULL) {
 		return -1;
 	}
 	
 	op_status = status_from_rc(action, op_status, op_rc, target_rc);
 	if(op_status != LRM_OP_DONE) {
 	    update_failcount(event, event_node, op_rc, target_rc, FALSE);
 	}
 	
 	/* Process OP status */
 	switch(op_status) {
 		case LRM_OP_PENDING:
 			crm_debug("Ignoring pending operation");
 			return action->id;
 			break;
 		case LRM_OP_DONE:
 			break;
 		case LRM_OP_ERROR:
 		case LRM_OP_TIMEOUT:
 		case LRM_OP_NOTSUPPORTED:
 			action->failed = TRUE;
 			break;
 		case LRM_OP_CANCELLED:
 			/* do nothing?? */
 			crm_err("Dont know what to do for cancelled ops yet");
 			break;
 		default:
 			action->failed = TRUE;
 			crm_err("Unsupported action result: %d", op_status);
 	}
 
 	/* stop this event's timer if it had one */
 	stop_te_timer(action->timer);
 	action->confirmed = TRUE;
 	
 	update_graph(transition_graph, action);
 	trigger_graph();
 	
 	if(action->failed) {
 		allow_fail = crm_meta_value(action->params, XML_ATTR_TE_ALLOWFAIL);
 		if(crm_is_true(allow_fail)) {
 			action->failed = FALSE;
 		}
 	}
 
 	if(action->failed) {
 		abort_transition(action->synapse->priority+1,
 				 tg_restart, "Event failed", event);
 	}
 
 	this_event = ID(event);
 	target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 	te_log_action(LOG_INFO, "Action %s (%d) confirmed on %s (rc=%d)",
 		      crm_str(this_event), action->id, crm_str(target),
 		      op_status);
 
 	return action->id;
 }
 
 crm_action_t *
 get_action(int id, gboolean confirmed)
 {
 	slist_iter(
 		synapse, synapse_t, transition_graph->synapses, lpc,
 
 		slist_iter(
 			action, crm_action_t, synapse->actions, lpc2,
 
 			if(action->id == id) {
 				if(confirmed) {
 					stop_te_timer(action->timer);
 					action->confirmed = TRUE;
 				}
 				return action;
 			}
 			)
 		);
 	return NULL;
 }
 
 crm_action_t *
 get_cancel_action(const char *id, const char *node)
 {
     const char *task = NULL;
     const char *target = NULL;
 
     slist_iter(
 	synapse, synapse_t, transition_graph->synapses, lpc,
 	
 	slist_iter(
 	    action, crm_action_t, synapse->actions, lpc2,
 	    
 	    task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 	    if(safe_str_neq(CRMD_ACTION_CANCEL, task)) {
 		continue;
 	    }
 	    
 	    task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 	    if(safe_str_neq(task, id)) {
 		continue;
 	    }
 
 	    target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
 	    if(safe_str_neq(target, node)) {
 		continue;
 	    }
 	    
 	    return action;
 	    );
 	);
 	return NULL;
 }
 
 crm_action_t *
 match_down_event(int id, const char *target, const char *filter)
 {
 	const char *this_action = NULL;
 	const char *this_node   = NULL;
 	crm_action_t *match = NULL;
 	
 	slist_iter(
 		synapse, synapse_t, transition_graph->synapses, lpc,
 
 		/* lookup event */
 		slist_iter(
 			action, crm_action_t, synapse->actions, lpc2,
 
 			if(id > 0 && action->id == id) {
 				match = action;
 				break;
 			}
 			
 			this_action = crm_element_value(
 				action->xml, XML_LRM_ATTR_TASK);
 
 			if(action->type != action_type_crm) {
 				continue;
 
 			} else if(safe_str_eq(this_action, CRM_OP_LRM_REFRESH)){
 				continue;
 				
 			} else if(filter != NULL
 				  && safe_str_neq(this_action, filter)) {
 				continue;
 			}
 			
 			this_node = crm_element_value(
 				action->xml, XML_LRM_ATTR_TARGET_UUID);
 
 			if(this_node == NULL) {
 				crm_log_xml_err(action->xml, "No node uuid");
 			}
 			
 			if(safe_str_neq(this_node, target)) {
 				crm_debug("Action %d : Node mismatch: %s",
 					 action->id, this_node);
 				continue;
 			}
 
 			match = action;
 			id = action->id;
 			break;
 			);
 		if(match != NULL) {
 			/* stop this event's timer if it had one */
 			break;
 		}
 		);
 	
 	if(match != NULL) {
 		/* stop this event's timer if it had one */
 		crm_debug("Match found for action %d: %s on %s", id,
 			  crm_element_value(match->xml, XML_LRM_ATTR_TASK_KEY),
 			  target);
 
 	} else if(id > 0) {
 		crm_err("No match for action %d", id);
 
 	} else {
 		crm_warn("No match for shutdown action on %s", target);
 	}
 
 	return match;
 }
 
 
 gboolean
 process_graph_event(xmlNode *event, const char *event_node)
 {
 	int rc = -1;
 	int status = -1;
 
 	int action = -1;
 	int target_rc = -1;
 	int transition_num = -1;
 	char *update_te_uuid = NULL;
 
 	gboolean stop_early = FALSE;
 	gboolean passed = FALSE;
 	const char *id = NULL;
 	const char *magic = NULL;
 	
 	CRM_ASSERT(event != NULL);
 
 	id = ID(event);
 	magic = crm_element_value(event, XML_ATTR_TRANSITION_MAGIC);
 
 	if(magic == NULL) {
 		/* non-change */
 		return FALSE;
 	}
 	
 	CRM_CHECK(decode_transition_magic(
 			  magic, &update_te_uuid, &transition_num, &action,
 			  &status, &rc, &target_rc),
 		  crm_err("Invalid event %s detected", id);
 		  abort_transition(INFINITY, tg_restart,"Bad event", event);
 		  return FALSE;
 		);
 
 	if(status == LRM_OP_PENDING) {
 	    goto bail;
 	}
 	
 	if(transition_num == -1) {
 		crm_err("Action %s (%s) initiated outside of a transition",
 			id, magic);
 		abort_transition(INFINITY, tg_restart,"Unexpected event",event);
 
 	} else if(action < 0 || crm_str_eq(update_te_uuid, te_uuid, TRUE) == FALSE) {
 		crm_info("Action %s/%d (%s) initiated by a different transitioner",
 			 id, action, magic);
 		abort_transition(INFINITY, tg_restart,"Foreign event", event);
 		stop_early = TRUE; /* This could be an lrm status refresh */
 		
 	} else if(transition_graph->id != transition_num) {
 		crm_info("Detected action %s from a different transition:"
 			" %d vs. %d", id, transition_num, transition_graph->id);
 		abort_transition(INFINITY, tg_restart,"Old event", event);
 		stop_early = TRUE; /* This could be an lrm status refresh */
 		
 	} else if(transition_graph->complete) {
 		crm_info("Action %s arrived after a completed transition", id);
 		abort_transition(INFINITY, tg_restart, "Inactive graph", event);
 
 	} else if(match_graph_event(
 		      action, event, event_node, status, rc, target_rc) < 0) {
 		crm_err("Unknown graph action %s", id);
 		abort_transition(INFINITY, tg_restart, "Unknown event", event);
 
 	} else {
 		passed = TRUE;
 		crm_debug_2("Processed update to %s: %s", id, magic);
 	}
 
 	if(passed == FALSE) {
 	    if(update_failcount(event, event_node, rc, target_rc, transition_num == -1)) {
 		/* Turns out this wasn't an lrm status refresh update aferall */
 		stop_early = FALSE;
 	    }
 	}
 
   bail:
 	crm_free(update_te_uuid);
 	return stop_early;
 }
 
diff --git a/doc/Clusters_from_Scratch/en-US/Author_Group.xml b/doc/Clusters_from_Scratch/en-US/Author_Group.xml
index ed2f0918d4..2c9fd520b3 100644
--- a/doc/Clusters_from_Scratch/en-US/Author_Group.xml
+++ b/doc/Clusters_from_Scratch/en-US/Author_Group.xml
@@ -1,17 +1,10 @@
 <?xml version='1.0' encoding='utf-8' ?>
 <!DOCTYPE authorgroup PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
-<!ENTITY % BOOK_ENTITIES SYSTEM "Clusters_from_Scratch.ent">
-%BOOK_ENTITIES;
 ]>
-<authorgroup lang="en-US">
-	<author>
-		<firstname>Dude</firstname>
-		<surname>McPants</surname>
-		<affiliation>
-			<orgname>Somewhere</orgname>
-			<orgdiv>Someone</orgdiv>
-		</affiliation>
-		<email>Dude.McPants@example.com</email>
-	</author>
+<authorgroup>
+  <author>
+    <firstname>Andrew</firstname><surname>Beekhof</surname>
+    <affiliation><orgname>Red Hat</orgname></affiliation>
+    <email>andrew@beekhof.net</email>
+  </author>
 </authorgroup>
-
diff --git a/doc/Clusters_from_Scratch/en-US/Revision_History.xml b/doc/Clusters_from_Scratch/en-US/Revision_History.xml
index 4d92648a08..b36cde468b 100644
--- a/doc/Clusters_from_Scratch/en-US/Revision_History.xml
+++ b/doc/Clusters_from_Scratch/en-US/Revision_History.xml
@@ -1,27 +1,19 @@
 <?xml version='1.0' encoding='utf-8' ?>
 <!DOCTYPE appendix PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
 <!ENTITY % BOOK_ENTITIES SYSTEM "Clusters_from_Scratch.ent">
 %BOOK_ENTITIES;
 ]>
 <appendix id="appe-Clusters_from_Scratch-Revision_History" lang="en-US">
 	<title>Revision History</title>
 	<simpara>
 		<revhistory>
 			<revision>
-				<revnumber>0</revnumber>
-				<date>Fri Apr 23 2010</date>
-				<author>
-					<firstname>Dude</firstname>
-					<surname>McPants</surname>
-					<email>Dude.McPants@example.com</email>
-				</author>
-				<revdescription>
-					<simplelist>
-						<member>Initial creation of book by publican</member>
-					</simplelist>
-				</revdescription>
+			  <revnumber>1</revnumber>
+			  <date>Mon May 17 2010</date>
+			  <author><firstname>Andrew</firstname><surname>Beekhof</surname><email>andrew@beekhof.net</email></author>
+			  <revdescription><simplelist><member>Import from Pages.app</member></simplelist></revdescription>
 			</revision>
 		</revhistory>
 	</simpara>
 </appendix>
 
diff --git a/doc/Makefile.am b/doc/Makefile.am
index 493bac2bf6..a112f27312 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -1,129 +1,133 @@
 #
 # doc: Pacemaker code
 #
 # Copyright (C) 2008 Andrew Beekhof
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 MAINTAINERCLEANFILES    = Makefile.in
 
 helpdir		= $(datadir)/$(PACKAGE)
 
 ascii		= crm_cli.txt crm_fencing.txt
 help_DATA	= crm_cli.txt
-docbook		= Pacemaker_Explained
+docbook		= Pacemaker_Explained Clusters_from_Scratch
 man_MANS	= cibadmin.8 crm_resource.8
 doc_DATA	= README.hb2openais $(ascii) $(generated_docs)
 
 publican_docs    =
 generated_docs	= 
 
 SVG := $(wildcard */en-US/images/pcmk-*.svg)
 
 PNGS = $(SVG:%.svg=%-small.png) $(SVG:%.svg=%.png)  $(SVG:%.svg=%-large.png)	\
 	publican-clusterlabs/en-US/images/image_left.png			\
 	publican-clusterlabs/en-US/images/image_right.png			\
 	publican-clusterlabs/en-US/images/h1-bg.png
 
 graphics: $(PNGS)
 
 %.png: %.svg
 	inkscape --file=$< --export-dpi=90 -C --export-png=$@
 
 %-small.png: %.svg
 	inkscape --file=$< --export-dpi=45 -C --export-png=$@
 
 %-large.png: %.svg
 	inkscape --file=$< --export-dpi=180 -C --export-png=$@
 
 XML_FILES := $(wildcard *.xml)
 PNG_FILES := $(wildcard images/*.png)
 
 if BUILD_ASCIIDOC
 generated_docs	+= $(ascii:%.txt=%.html)
 endif
 
 if BUILD_DOCBOOK
 publican_docs	+= $(docbook)
 generated_docs	+= index.html
 endif
 
 EXTRA_DIST	= $(man_MANS) $(docbook:%=%.xml) 
 
 index.html:  $(docbook_txt)
 	echo "Building documentation index"
 	echo "<html><body><p>The following <a href=\"http://www.clusterlabs.org/wiki/Pacemaker\">Pacemaker</a> documentation was generated on `date` from version: $(BUILD_VERSION)</p>" > index.html	
 	echo "<ol>" >> index.html
 	for doc in $(generated_docs); do 					\
 		echo "<li><a href=\"$$doc\">$$doc</a></li>" >> index.html; 	\
 	done
 if BUILD_DOCBOOK
 	for book in $(docbook); do 					\
-	    for lang in `ls -1 $(docbook)/publish`; do 			\
+	    for lang in `ls -1 $$book/publish`; do 			\
 	      echo "<li>$$book ($$lang)<ul>" >> index.html; 	\
 	      find $$book/publish/$$lang -name "*.pdf" -exec echo -n "<li><a href=\"{}\">" \; -exec basename {} \; -exec echo "</a></li>" \; | sed s:$$book/publish/:: >> index.html ;		\
 	      find $$book/publish/$$lang -name "*.txt" -exec echo -n "<li><a href=\"{}\">" \; -exec basename {} \; -exec echo "</a></li>" \; | sed s:$$book/publish/:: >> index.html ;		\
 	      find $$book/publish/$$lang -name html -exec echo "<li><a href=\"{}/$$book/index.html\">$$book HTML</a></li>" \; | sed s:$$book/publish/:: >> index.html ;		 		\
 	      find $$book/publish/$$lang -name html-single -exec echo "<li><a href=\"{}/$$book/index.html\">$$book HTML (single page)</a></li>" \; | sed s:$$book/publish/:: >> index.html ;	\
 	      echo "</ul></li>" >> index.html; 				\
 	    done;							\
 	done
 endif
 	echo "</ol>" >> index.html	
 	echo "<p>You can find <a href=\"http://www.clusterlabs.org/wiki/Documentation\">additional documentation</a> and details about the Pacemaker project at <a href=\"http://www.clusterlabs.org\">http://www.clusterlabs.org</a></p>" >> index.html	
 	echo "</body></html>" >> index.html	
 
-
 %.html: %.txt
 	$(ASCIIDOC) --unsafe --backend=xhtml11 $<
 
 %.txt: %/en-US/*.xml
 	cd $* && $(PUBLICAN) build --publish --langs=all --formats=pdf,html,html-single,txt
 	touch $@
 
 if BUILD_DOCBOOK
 docbook_txt = $(docbook:%=%.txt)
 
 all-local: $(docbook_txt) */publican.cfg
 
 #install-data-local: all-local
 install-data-local: all-local
 	for book in $(docbook); do 							\
 	    filelist=`find $$book/publish/* -print`;					\
 	    for f in $$filelist; do							\
 		p=`echo $$f | sed s:publish/:: | sed s:Pacemaker/::`;			\
 		if [ -d $$f ]; then							\
 			$(INSTALL) -d 775 $(DESTDIR)$(docdir)/$$p;			\
 		else									\
 			$(INSTALL) -m 644 $$f $(DESTDIR)$(docdir)/$$p;			\
 		fi									\
 	    done;									\
 	done
 endif
 
+brand:
+	find publican-clusterlabs -name "*.noarch.rpm" -exec rm -f \{\} \;
+	cd publican-clusterlabs && $(PUBLICAN) package --binary
+	find publican-clusterlabs -name "*.noarch.rpm" -exec sudo rpm -Uvh --force \{\} \;
+
 push:   all-local $(generated_docs)
 	echo Uploading current documentation set to clusterlabs.org
-	rsync -rtz --progress $(generated_docs) $(ascii) root@oss.clusterlabs.org:/srv/www/extras/doc/
+	rsync -rtz --progress $(generated_docs) $(ascii) root@oss.clusterlabs.org:/var/www/html/doc/
 if BUILD_DOCBOOK
 	for book in $(docbook); do 									\
 		echo Uploading $$book...;								\
 		echo "Generated on `date` from version: $(BUILD_VERSION)" > $$book/publish/build-$(PACKAGE_SERIES).txt;	\
-		rsync -rtz --progress $$book/publish/* root@oss.clusterlabs.org:/srv/www/extras/doc/;	\
+		rsync -rtz --progress $$book/publish/* root@oss.clusterlabs.org:/var/www/html/doc/;	\
 	done
 endif
 
 clean-local:
 	-rm -rf $(generated_docs) $(docbook_txt)
 	for book in $(docbook); do rm -rf $$book/tmp $$book/publish; done
diff --git a/doc/Pacemaker_Explained/en-US/Book_Info.xml b/doc/Pacemaker_Explained/en-US/Book_Info.xml
index b3c37d7606..9f1f55f2fc 100644
--- a/doc/Pacemaker_Explained/en-US/Book_Info.xml
+++ b/doc/Pacemaker_Explained/en-US/Book_Info.xml
@@ -1,40 +1,39 @@
 <?xml version='1.0' encoding='utf-8' ?>
 <!DOCTYPE bookinfo PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
 ]>
 <bookinfo>
   <title>Configuration Explained</title>
   <subtitle>An A-Z guide to Pacemaker's Configuration Options</subtitle>
   <productname>Pacemaker</productname>
-  <productnumber>1.0</productnumber>
+  <productnumber>1.1</productnumber>
   <edition>1</edition>
   <pubsnumber>0</pubsnumber>
   <abstract>
     <para>
       The purpose of this document is to definitively explain the concepts used to configure Pacemaker.
       To achieve this best, it will focus exclusively on the XML syntax used to configure the CIB.
     </para>
     <para>
       For those that are allergic to XML, Pacemaker comes with a cluster shell and a Python based GUI exists, however these tools will not be covered at all in this document
       <footnote>
 	<para>It is hoped however, that having understood the concepts explained here, that the functionality of these tools will also be more readily understood.</para>
       </footnote>
       , precisely because they hide the XML.
     </para>
     <para>
       Additionally, this document is NOT a step-by-step how-to guide for configuring a specific clustering scenario.
       Although such guides exist, the purpose of this document is to provide an understanding of the building blocks that can be used to construct any type of Pacemaker cluster.
     </para>
   </abstract>
   <corpauthor>
     <inlinemediaobject>
       <imageobject>
-	<imagedata fileref="Common_Content/images/title_logo.svg" format="SVG">
-	</imagedata>
+	<imagedata fileref="Common_Content/images/title_logo.svg" format="SVG"/>
       </imageobject>
     </inlinemediaobject>
   </corpauthor>
   <xi:include href="Common_Content/Legal_Notice.xml" xmlns:xi="http://www.w3.org/2001/XInclude">
   </xi:include>
   <xi:include href="Author_Group.xml" xmlns:xi="http://www.w3.org/2001/XInclude">
   </xi:include>
 </bookinfo>
diff --git a/doc/publican-clusterlabs/README b/doc/publican-clusterlabs/README
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/doc/publican-clusterlabs/en-US/css/overrides.css b/doc/publican-clusterlabs/en-US/css/overrides.css
index df07539534..5a5f639b97 100644
--- a/doc/publican-clusterlabs/en-US/css/overrides.css
+++ b/doc/publican-clusterlabs/en-US/css/overrides.css
@@ -1,69 +1,69 @@
 a:link {
 	color:#0076d6;
 }
 
 a:visited {
 	color:grey;
 }
 
 h1 {
 	color:#172969;
 }
 
 .producttitle {
-	background: #800 url(../images/h1-bg.png) top left repeat;
+	background: #2C4081 url(../images/h1-bg.png) top left repeat;
 }
 
 .section h1.title {
 	color:#2C4081;
 }
 
 
 h2,h3,h4,h5,h6 {
 	color:#2C4081;
 }
 
 table {
 	border:1px solid #732f2f;
 }
 
 table th {
 	background-color:#2C4081;
 }
 
 table tr.even td {
 	background-color: #e3e0d6;
 }
 
 #title a {
 	height: 54px;
 }
 
 .term{
 	color:#172969;
 }
 
 .revhistory table th {
 	color: #172969;
 }
 
 .edition {
 	color: #172969;
 }
 
 span.remark{
 	background-color: #ffff00;
 }
 
 .example {
 	border: 1px solid #732f2f;
 	background-color: #e3e0d6;
 }
 
 .programlisting {
 	border: 1px solid #732f2f;
 }
 
 .screen {
 	border: 1px solid #732f2f;
 }
diff --git a/doc/publican-clusterlabs/xsl/common.xsl b/doc/publican-clusterlabs/xsl/common.xsl
index 809dce68a2..5f255dfcc7 100644
--- a/doc/publican-clusterlabs/xsl/common.xsl
+++ b/doc/publican-clusterlabs/xsl/common.xsl
@@ -1,68 +1,68 @@
 <?xml version='1.0'?>
  
 <!--
         Copyright 2009 Andrew Beekhof
         License: GPL
         Author: Andrew Beekhof <andrew@beekhof.net>
 -->
 
 <!DOCTYPE xsl:stylesheet [
 <!ENTITY lowercase "'abcdefghijklmnopqrstuvwxyz'">
 <!ENTITY uppercase "'ABCDEFGHIJKLMNOPQRSTUVWXYZ'">
  ]>
 
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
                 version='1.0'
                 xmlns="http://www.w3.org/TR/xhtml1/transitional"
                 xmlns:fo="http://www.w3.org/1999/XSL/Format"
                 exclude-result-prefixes="#default">
 
-<xsl:param name="title.color">#843A39</xsl:param>
+<xsl:param name="title.color">#2C4081</xsl:param>
 <!-- http://docbook.sourceforge.net/release/xsl/current/doc/html/generate.toc.html -->
 <xsl:param name="generate.toc">
 appendix  toc,title
 article/appendix  nop
 article   toc,title
 book      toc,title,figure,table,example,equation
 chapter   toc,title
 part      toc,title
 preface   toc,title
 qandadiv  nop
 qandaset  nop
 reference toc,title
 sect1	  nop
 sect2	  nop
 sect3	  nop
 sect4	  nop
 sect5	  nop
 section   nop
 set       toc,title
 <!-- publican defaults
 set toc
 book toc,qandadiv
 article toc
 chapter nop
 qandadiv nop
 qandaset nop
 sect1 nop
 sect2 nop
 sect3 nop
 sect4 nop
 sect5 nop
 section nop
 part nop
 -->
 </xsl:param>
 
 <xsl:template name="tr.attributes">
   <xsl:param name="row" select="."/>
   <xsl:param name="rownum" select="0"/>
 
   <xsl:if test="$rownum mod 2 = 0">
     <xsl:attribute name="class">even</xsl:attribute>
   </xsl:if>
 
 </xsl:template>
 
 
 </xsl:stylesheet>
diff --git a/extra/resources/ping b/extra/resources/ping
index 710c288ac0..4e334478fb 100755
--- a/extra/resources/ping
+++ b/extra/resources/ping
@@ -1,306 +1,307 @@
 #!/bin/sh
 #
 #
 #	Ping OCF RA that utilizes the system ping 
 #
 # Copyright (c) 2009 Andrew Beekhof
 #                    All Rights Reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of version 2 of the GNU General Public License as
 # published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it would be useful, but
 # WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 #
 # Further, this software is distributed without any warranty that it is
 # free of the rightful claim of any third person regarding infringement
 # or the like.  Any license provided herein, whether implied or
 # otherwise, applies only to this software file.  Patent licenses, if
 # any, provided herein do not apply to combinations of this program with
 # other software, or any other product whatsoever.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write the Free Software Foundation,
 # Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
 #
 
 #######################################################################
 # Initialization:
 
 . ${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs
 
 #######################################################################
 
 meta_data() {
 	cat <<END
 <?xml version="1.0"?>
 <!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
 <resource-agent name="ping">
 <version>1.0</version>
 
 <longdesc lang="en">
 Every time the monitor action is run, this resource agent records (in the CIB) the current number of ping nodes the host can connect to.
 It is essentially the same as pingd except that it uses the system ping tool to obtain the results.
 </longdesc>
 <shortdesc lang="en">node connectivity</shortdesc>
 
 <parameters>
 
 <parameter name="pidfile" unique="0">
 <longdesc lang="en">PID file</longdesc>
 <shortdesc lang="en">PID file</shortdesc>
 <content type="string" default="$HA_VARRUN/ping-${OCF_RESOURCE_INSTANCE}" />
 </parameter>
 
 <parameter name="dampen" unique="0">
 <longdesc lang="en">
 The time to wait (dampening) further changes occur
 </longdesc>
 <shortdesc lang="en">Dampening interval</shortdesc>
 <content type="integer" default="5s"/>
 </parameter>
 
 <parameter name="name" unique="0">
 <longdesc lang="en">
 The name of the attributes to set.  This is the name to be used in the constraints.
 </longdesc>
 <shortdesc lang="en">Attribute name</shortdesc>
 <content type="string" default="${OCF_RESOURCE_INSTANCE}"/>
 </parameter>
 
 <parameter name="multiplier" unique="0">
 <longdesc lang="en">
 The number by which to multiply the number of connected ping nodes by
 </longdesc>
 <shortdesc lang="en">Value multiplier</shortdesc>
 <content type="integer" default=""/>
 </parameter>
 
 <parameter name="host_list" unique="0">
 <longdesc lang="en">
 The list of ping nodes to count.  Defaults to all configured ping nodes.  Rarely needs to be specified.
 </longdesc>
 <shortdesc lang="en">Host list</shortdesc>
 <content type="string" default=""/>
 </parameter>
 
 <parameter name="attempts" unique="0">
 <longdesc lang="en">
 Number of ping attempts, per host, before declaring it dead
 </longdesc>
 <shortdesc lang="en">no. of ping attempts</shortdesc>
 <content type="integer" default="2"/>
 </parameter>
 
 <parameter name="timeout" unique="0">
 <longdesc lang="en">
 How long, in seconds, to wait before declaring a ping lost
 </longdesc>
 <shortdesc lang="en">ping timeout in seconds</shortdesc>
 <content type="integer" default="2"/>
 </parameter>
 
 <parameter name="options" unique="0">
 <longdesc lang="en">
 A catch all for any other options that need to be passed to ping.
 </longdesc>
 <shortdesc lang="en">Extra Options</shortdesc>
 <content type="string" default=""/>
 </parameter>
 
 <parameter name="debug" unique="0">
 <longdesc lang="en">
 Enables to use default attrd_updater verbose logging on every call.
 </longdesc>
 <shortdesc lang="en">Verbose logging</shortdesc>
 <content type="string" default="false"/>
 </parameter>
 
 </parameters>
 
 <actions>
 <action name="start"   timeout="60" />
 <action name="stop"    timeout="20" />
 <action name="reload"  timeout="100" />
 <action name="monitor" depth="0"  timeout="60" interval="10"/>
 <action name="meta-data"  timeout="5" />
 <action name="validate-all"  timeout="30" />
 </actions>
 </resource-agent>
 END
 }
 
 #######################################################################
 
 ping_conditional_log() {
     level=$1; shift 
     if [ ${OCF_RESKEY_debug} = "true" ]; then
 	ocf_log $level "$*"
     fi
 }
 
 ping_usage() {
 	cat <<END
 usage: $0 {start|stop|monitor|migrate_to|migrate_from|validate-all|meta-data}
 
 Expects to have a fully populated OCF RA-compliant environment set.
 END
 }
 
 ping_start() {
     ping_monitor
     if [ $? =  $OCF_SUCCESS ]; then
 	return $OCF_SUCCESS
     fi
     touch ${OCF_RESKEY_pidfile}
     ping_update
 }
 
 ping_stop() {
 
 	rm -f ${OCF_RESKEY_pidfile}
     
     attrd_updater -D -n $OCF_RESKEY_name -d $OCF_RESKEY_dampen $attrd_options
     
     return $OCF_SUCCESS
 }
 
 ping_monitor() {
     if [ -f ${OCF_RESKEY_pidfile} ]; then
 	ping_update
 	return $OCF_SUCCESS
     fi
     return $OCF_NOT_RUNNING
 }
 
 ping_validate() {
     
     # Is the state directory writable? 
     state_dir=`dirname "$OCF_RESKEY_pidfile"`
     touch "$state_dir/$$"
     if [ $? != 0 ]; then
 	ocf_log err "Invalid location for 'state': $state_dir is not writable"
 	return $OCF_ERR_ARGS
     fi
     rm "$state_dir/$$"
 
 # Pidfile better be an absolute path
     case $OCF_RESKEY_pidfile in
 	/*) ;;
 	*) ocf_log warn "You should use an absolute path for pidfile not: $OCF_RESKEY_pidfile" ;;
     esac
     
 # Check the ping interval
     if ocf_is_decimal "$OCF_RESKEY_CRM_meta_interval" && [ $OCF_RESKEY_CRM_meta_interval -gt 0 ]; then
 	:
     else
 	ocf_log err "Invalid ping interval $OCF_RESKEY_interval. It should be positive integer!"
 	exit $OCF_ERR_CONFIGURED
     fi
 
 # Check the host list
     if [ "x" = "x$OCF_RESKEY_host_list" ]; then 
 	ocf_log err "Empty host_list.  Please specify some nodes to ping"
 	exit $OCF_ERR_CONFIGURED
     fi
 
     check_binary ping
 
     return $OCF_SUCCESS
 }
 
 ping_update() {
     active=0
     for host in $OCF_RESKEY_host_list; do
 	p_exe=ping
 
 	case `uname` in
 	    Linux) p_args="-n -q -W $OCF_RESKEY_timeout -c $OCF_RESKEY_attempts";;
 	    Darwin) p_args="-n -q -t $OCF_RESKEY_timeout -c $OCF_RESKEY_attempts -o";;
 	    *) ocf_log err "Unknown host type: `uname`"; exit $OCF_ERR_INSTALLED;;
 	esac
 
 	case $host in
 	    *:*) p_exe=ping6
 	esac
 	
 	p_out=`$p_exe $p_args $OCF_RESKEY_options $host 2>&1`; rc=$?
 
 	case $rc in
 	    0) active=`expr $active + 1`;;
 	    1) ping_conditional_log warn "$host is inactive: $p_out";;
 	    *) ocf_log err "Unexpected result for '$p_exe $p_args $OCF_RESKEY_options $host' $rc: $p_out";;
 	esac
     done
     score=`expr $active \* $OCF_RESKEY_multiplier`
     attrd_updater -n $OCF_RESKEY_name -v $score -d $OCF_RESKEY_dampen $attrd_options
     rc=$?
     case $rc in 
 	0) ping_conditional_log debug "Updated $OCF_RESKEY_name = $score" ;;
 	*) ocf_log warn "Could not update $OCF_RESKEY_name = $score: rc=$rc";;
     esac
     return $rc
 }
 
 : ${OCF_RESKEY_name:="pingd"}
 : ${OCF_RESKEY_dampen:="5s"}
 : ${OCF_RESKEY_attempts:="5"}
 : ${OCF_RESKEY_multiplier:="1"}
 : ${OCF_RESKEY_debug:="false"}
 : ${OCF_RESKEY_CRM_meta_timeout:="60"}
 : ${OCF_RESKEY_CRM_meta_interval="10"}
 : ${OCF_RESKEY_CRM_meta_globally_unique:="true"}
 
 if [ -z ${OCF_RESKEY_timeout} ]; then
-    OCF_RESKEY_timeout=`expr $OCF_RESKEY_CRM_meta_timeout / $OCF_RESKEY_attempts`
+    host_count=`echo $OCF_RESKEY_host_list | awk '{print NF}'`
+    OCF_RESKEY_timeout=`expr $OCF_RESKEY_CRM_meta_timeout / $host_count / $OCF_RESKEY_attempts`
     OCF_RESKEY_timeout=`expr $OCF_RESKEY_timeout / 1100` # Convert to seconds and finish 10% early
 fi
 
 if [ ${OCF_RESKEY_timeout} -lt 1 ]; then
     OCF_RESKEY_timeout=5
 elif [ ${OCF_RESKEY_timeout} -gt 1000 ]; then
     # ping actually complains if this value is too high, 5 minutes is plenty
     OCF_RESKEY_timeout=300
 fi
 
 if [ ${OCF_RESKEY_CRM_meta_globally_unique} = "false" ]; then
     : ${OCF_RESKEY_pidfile:="$HA_VARRUN/ping-${OCF_RESKEY_name}"}
 else 
     : ${OCF_RESKEY_pidfile:="$HA_VARRUN/ping-${OCF_RESOURCE_INSTANCE}"}
 fi
 
 attrd_options='-q'
 if ocf_is_true ${OCF_RESKEY_debug} ; then
     attrd_options=''
 fi
 
 # Check the debug option    
     case "${OCF_RESKEY_debug}" in
 	true|True|TRUE|1)    OCF_RESKEY_debug=true;;
 	false|False|FALSE|0) OCF_RESKEY_debug=false;;
 	*)
             ocf_log warn "Value for 'debug' is incorrect. Please specify 'true' or 'false' not: ${OCF_RESKEY_debug}"
 	    OCF_RESKEY_debug=false
 	    ;;
     esac
 
 case $__OCF_ACTION in
 meta-data)	meta_data
 		exit $OCF_SUCCESS
 		;;
 start)		ping_start;;
 stop)		ping_stop;;
 monitor)	ping_monitor;;
 reload)		ping_start;;
 validate-all)	ping_validate;;
 usage|help)	ping_usage
 		exit $OCF_SUCCESS
 		;;
 *)		ping_usage
 		exit $OCF_ERR_UNIMPLEMENTED
 		;;
 esac
 exit $?
 
diff --git a/fencing/commands.c b/fencing/commands.c
index 8488849bb2..09188dd415 100644
--- a/fencing/commands.c
+++ b/fencing/commands.c
@@ -1,920 +1,923 @@
 /* 
  * Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <sys/stat.h>
 #include <unistd.h>
 #include <sys/utsname.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <ctype.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/cluster.h>
 
 #include <crm/stonith-ng.h>
 #include <crm/common/xml.h>
 #include <crm/common/msg.h>
 #include <internal.h>
 
 #include <clplumbing/proctrack.h>
 
 GHashTable *device_list = NULL;
 static int active_children = 0;
 
 static void exec_child_done(ProcTrack* proc, int status, int signo, int rc, int waslogged);
 static void exec_child_new(ProcTrack* p) { active_children++; }
 static const char *exec_child_name(ProcTrack* p) {
     async_command_t *cmd = proctrack_data(p);
     return cmd->client?cmd->client:cmd->remote;
 }
 
 static ProcTrack_ops StonithdProcessTrackOps = {
 	exec_child_done,
 	exec_child_new,
 	exec_child_name,
 };
 
 
 static async_command_t *create_async_command(xmlNode *msg, const char *action) 
 {
     async_command_t *cmd = NULL;
     CRM_CHECK(action != NULL, crm_log_xml_warn(msg, "NoAction"); return NULL);
 
     crm_malloc0(cmd, sizeof(async_command_t));
     crm_element_value_int(msg, F_STONITH_CALLID, &(cmd->id));
     crm_element_value_int(msg, F_STONITH_CALLOPTS, &cmd->options);
 
     cmd->origin = crm_element_value_copy(msg, F_ORIG);
     cmd->remote = crm_element_value_copy(msg, F_STONITH_REMOTE);
     cmd->client = crm_element_value_copy(msg, F_STONITH_CLIENTID);
     cmd->op     = crm_element_value_copy(msg, F_STONITH_OPERATION);
     cmd->action = crm_strdup(action);
     cmd->victim = crm_element_value_copy(msg, F_STONITH_TARGET);
     cmd->pt_ops = &StonithdProcessTrackOps;
 
     CRM_CHECK(cmd->op != NULL,     crm_log_xml_warn(msg, "NoOp");     return NULL);
     CRM_CHECK(cmd->client != NULL || cmd->remote != NULL, crm_log_xml_warn(msg, "NoClient"));
     
     return cmd;
 }
 
 static void free_async_command(async_command_t *cmd) 
 {
     if(cmd->node_attrs) {
 	g_hash_table_destroy(cmd->node_attrs);
     }    
     crm_free(cmd->action);
     crm_free(cmd->victim);
     crm_free(cmd->remote);
     crm_free(cmd->client);
     crm_free(cmd->origin);
     crm_free(cmd->op);
     crm_free(cmd);    
 }
 
 static void free_device(gpointer data)
 {
     stonith_device_t *device = data;
 
     g_hash_table_destroy(device->params);
     slist_destroy(char, item, device->targets, crm_free(item));
     crm_free(device->namespace);
     crm_free(device->agent);
     crm_free(device->id);
     crm_free(device);
 }
 
 static GHashTable *build_port_aliases(const char *hostmap, GListPtr *targets) 
 {
     char *name = NULL;
     char *value = NULL;
     int last = 0, lpc = 0, max = 0;
     GHashTable *aliases = g_hash_table_new_full(g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
     
     if(hostmap == NULL) {
 	return aliases;
     }
     
     max = strlen(hostmap);
     for(; lpc < max; lpc++) {
 	if(hostmap[lpc] == 0) {
 	    break;
 	    
 	} else if(isalpha(hostmap[lpc])) {
 	    /* keep going */
 	    
 	} else if(hostmap[lpc] == '=') {
 	    crm_malloc0(name, 1 + lpc - last);
 	    strncpy(name, hostmap + last, lpc - last);
 	    last = lpc + 1;
 	    
 	} else if(name && isspace(hostmap[lpc])) {
 	    crm_malloc0(value, 1 + lpc - last);
 	    strncpy(value, hostmap + last, lpc - last);
 	    last = lpc + 1;
 
 	    crm_debug("Adding alias '%s'='%s'", name, value);
 	    g_hash_table_replace(aliases, name, value);
 	    if(targets) {
 		*targets = g_list_append(*targets, crm_strdup(value));
 	    }
 	    value=NULL;
 	    name=NULL;
 	    
 	} else if(isspace(hostmap[lpc])) {
 	    last = lpc;
 	}   
     }
     return aliases;
 }
 
 static void parse_host_line(const char *line, GListPtr *output) 
 {
     int lpc = 0;
     int max = 0;
     int last = 0;
 
     if(line) {
 	max = strlen(line);
     } else {
 	return;
     }
 
     /* Check for any complaints about additional parameters that the device doesn't understand */
     if(strstr(line, "invalid") || strstr(line, "variable")) {
 	crm_debug("Skipping: %s", line);
 	return;
     }
     
     crm_debug_2("Processing: %s", line);
     /* Skip initial whitespace */
     for(lpc = 0; lpc <= max && isspace(line[lpc]); lpc++) {
 	last = lpc+1;
     }
 
     /* Now the actual content */
     for(lpc = 0; lpc <= max; lpc++) {
 	gboolean a_space = isspace(line[lpc]);
 	if(a_space && lpc < max && isspace(line[lpc+1])) {
 	    /* fast-forward to the end of the spaces */
 	
 	} else if(a_space || line[lpc] == ',' || line[lpc] == 0) {
 	    int rc = 0;
 	    char *entry = NULL;
 	    
 	    crm_malloc0(entry, 1 + lpc - last);
 	    rc = sscanf(line+last, "%[a-zA-Z0-9_-.]", entry);
 	    if(rc != 1) {
 		crm_warn("Could not parse (%d %d): %s", last, lpc, line+last);
 		
 	    } else if(safe_str_neq(entry, "on") && safe_str_neq(entry, "off")) {
 		crm_debug_2("Adding '%s'", entry);
 		*output = g_list_append(*output, entry);
 		entry = NULL;
 	    }
 	    
 	    crm_free(entry);
 	    last = lpc + 1;
 	}
     }
 }
 
 static GListPtr parse_host_list(const char *hosts) 
 {
     int lpc = 0;
     int max = 0;
     int last = 0;
     GListPtr output = NULL;
 
     if(hosts) {
 	max = strlen(hosts);
     }
     
     for(lpc = 0; lpc < max; lpc++) {
 	if(hosts[lpc] == '\n' || hosts[lpc] == 0) {
 	    char *line = NULL;
 
 	    crm_malloc0(line, 2 + lpc - last);
 	    snprintf(line, 1 + lpc - last, "%s", hosts+last);
 	    parse_host_line(line, &output);
 	    crm_free(line);
 
 	    last = lpc + 1;	    
 	}
     }
     
     return output;
 }
 
 static stonith_device_t *build_device_from_xml(xmlNode *msg) 
 {
     xmlNode *dev = get_xpath_object("//"F_STONITH_DEVICE, msg, LOG_ERR);
     stonith_device_t *device = NULL;
 
     crm_malloc0(device, sizeof(stonith_device_t));
     device->id = crm_element_value_copy(dev, XML_ATTR_ID);
     device->agent = crm_element_value_copy(dev, "agent");
     device->namespace = crm_element_value_copy(dev, "namespace");
     device->params = xml2list(dev);
     /* TODO: Hook up priority */
     
     return device;
 }
 
 static int stonith_device_register(xmlNode *msg) 
 {
     const char *value = NULL;
     stonith_device_t *device = build_device_from_xml(msg);
 
     value = g_hash_table_lookup(device->params, STONITH_ATTR_HOSTLIST);
     if(value) {
 	device->targets = parse_host_list(value);
     }
 	
     value = g_hash_table_lookup(device->params, STONITH_ATTR_HOSTMAP);
     device->aliases = build_port_aliases(value, &(device->targets));
 
     g_hash_table_replace(device_list, device->id, device);
 
     crm_info("Added '%s' to the device list (%d active devices)", device->id, g_hash_table_size(device_list));
     return stonith_ok;
 }
 
 static int stonith_device_remove(xmlNode *msg) 
 {
     xmlNode *dev = get_xpath_object("//"F_STONITH_DEVICE, msg, LOG_ERR);
     const char *id = crm_element_value(dev, XML_ATTR_ID);
     if(g_hash_table_remove(device_list, id)) {
 	crm_info("Removed '%s' from the device list (%d active devices)",
 		 id, g_hash_table_size(device_list));
     } else {
 	crm_info("Device '%s' not found (%d active devices)",
 		 id, g_hash_table_size(device_list));
     }
     
     return stonith_ok;
 }
 
 static gboolean string_in_list(GListPtr list, const char *item)
 {
     int lpc = 0;
     int max = g_list_length(list);
     for(lpc = 0; lpc < max; lpc ++) {
 	const char *value = g_list_nth_data(list, lpc);
 	if(safe_str_eq(item, value)) {
 	    return TRUE;
 	}
     }
     return FALSE;
 }
 
 static const char *get_victim_name(stonith_device_t *dev, const char *host) 
 {
     if(dev == NULL) {
 	return NULL;
 
     } else if(host && dev->aliases) {
 	char *alias = g_hash_table_lookup(dev->aliases, host);
 	if(alias) {
 	    return alias;
 	}
     }
     
     return host;
 }
 
 static int stonith_device_action(xmlNode *msg, char **output) 
 {
     int rc = stonith_ok;
     xmlNode *dev = get_xpath_object("//"F_STONITH_DEVICE, msg, LOG_ERR);
     const char *id = crm_element_value(dev, F_STONITH_DEVICE);
     const char *action = crm_element_value(dev, F_STONITH_ACTION);
 
     async_command_t *cmd = NULL;
     stonith_device_t *device = NULL;
     GHashTable *node_attrs = xml2list(dev);
 
     if(id) {
 	crm_debug_2("Looking for '%s'", id);
 	device = g_hash_table_lookup(device_list, id);
 	
     } else {
 	CRM_CHECK(safe_str_eq(action, "metadata"), crm_log_xml_warn(msg, "StrangeOp"));
 	
 	device = build_device_from_xml(msg);
 	if(device != NULL && device->id == NULL) {
 	    device->id = crm_strdup(device->agent);
 	}
     }
 
     if(device) {
 	int exec_rc = 0;
 	const char *victim = NULL;
 
 	cmd = create_async_command(msg, action);
 	if(cmd == NULL) {
 	    return st_err_internal;
 	}
 
 	cmd->node_attrs = node_attrs;
 	victim = get_victim_name(device, cmd->victim);
 	if(cmd->victim && victim == NULL) {
 	    crm_err("Unknown or unhandled port '%s' for device '%s'", cmd->victim, device->id);
 	    free_async_command(cmd);
 	    return st_err_unknown_port;
 	}
 	cmd->device = crm_strdup(device->id);
 	crm_debug("Calling '%s' with action '%s'%s%s",
 		  device->id,  action, victim?" on port ":"", victim?victim:"");
 	
 	exec_rc = run_stonith_agent(
 	    device->agent, device->params, cmd->node_attrs, action, victim, &rc, output, cmd);
 	if(exec_rc < 0 || rc != 0) {
 	    crm_warn("Operation %s on %s failed (%d/%d): %.100s",
 		     action, device->id, exec_rc, rc, *output);
 	    
 	} else if(exec_rc > 0) {
 	    crm_info("Operation %s on %s active with pid: %d", action, device->id, exec_rc);
 	    rc = exec_rc;
 	    
 	} else {
 	    crm_info("Operation %s on %s passed: %.100s", action, device->id, *output);
 	}
 	
     } else {
 	crm_notice("Device %s not found", id);
 	rc = st_err_unknown_device;
     }
 
     if(id == NULL) {
 	free_device(device);
     }
     return rc;
 }
 
 static gboolean can_fence_host_with_device(stonith_device_t *dev, const char *host)
 {
     gboolean can = FALSE;
-    const char *victim = get_victim_name(dev, host);
-    const char *check_type = g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTCHECK);
+    const char *victim = NULL;
+    const char *check_type = NULL;
 
     if(dev == NULL) {
 	return FALSE;
 
     } else if(host == NULL) {
 	return TRUE;
     }
 
+    victim = get_victim_name(dev, host);
+    check_type = g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTCHECK);
+    
     if(check_type == NULL) {
 	check_type = "dynamic-list";
     }
     
     if(safe_str_eq(check_type, "none")) {
 	can = TRUE;
 
     } else if(safe_str_eq(check_type, "static-list")) {
 
 	/* Presence in the hostmap is sufficient
 	 * Only use if all hosts on which the device can be active can always fence all listed hosts
 	 */
 
 	if(string_in_list(dev->targets, victim)) {
 	    can = TRUE;
 	}
 
     } else if(safe_str_eq(check_type, "dynamic-list")) {
 	time_t now = time(NULL);
 
 	/* Host/alias must be in the list output to be eligable to be fenced
 	 *
 	 * Will cause problems if down'd nodes aren't listed or (for virtual nodes)
 	 *  if the guest is still listed despite being moved to another machine
 	 */
 	
 	if(dev->targets == NULL || dev->targets_age + 60 < now) {
 	    char *output = NULL;
 	    int rc = stonith_ok;
 	    int exec_rc = stonith_ok;
 	    
 	    /* Some use hostlist instead of the "standard" list */
 	    const char *list_cmd = g_hash_table_lookup(dev->params, STONITH_ATTR_LIST_OP);
 	    if(list_cmd == NULL) {
 		list_cmd = "list";
 	    }
 	    
 	    /* Check for the target's presence in the output of the 'list' command */
 	    slist_destroy(char, item, dev->targets, crm_free(item));
 	    dev->targets = NULL;
 	    
 	    exec_rc = run_stonith_agent(dev->agent, dev->params, NULL, list_cmd, NULL, &rc, &output, NULL);
 	    if(exec_rc < 0 || rc != 0) {
 		crm_notice("Disabling port list queries for %s", dev->id);	    
 		dev->targets_age = -1;
 		
 	    } else {
 		crm_info("Refreshing port list for %s", dev->id);
 		dev->targets = parse_host_list(output);
 		dev->targets_age = now;
 	    }
 	
 	    crm_free(output);
 	}
 	
 	if(string_in_list(dev->targets, victim)) {
 	    can = TRUE;
 	}
 
     } else if(safe_str_eq(check_type, "status")) {
 	int rc = 0;
 	int exec_rc = 0;
 
 	/* Some use stat instead of the "standard" status */
 	const char *status = g_hash_table_lookup(dev->params, STONITH_ATTR_STATUS_OP);
 	if(status == NULL) {
 	    status = "status";
 	}
 
 	/* Run the status operation for the device/target combination
 	 * Will cause problems if the device doesn't return 2 for down'd nodes or
 	 *  (for virtual nodes) if the device doesn't return 1 for guests that
 	 *  have been moved to another host
 	 */
 
 	/* TODO: Get node_attrs in here */
 	
 	exec_rc = run_stonith_agent(
 	    dev->agent, dev->params, NULL, status, victim, &rc, NULL, NULL);
 
 	if(exec_rc != 0) {
 	    crm_err("Could not invoke %s: rc=%d", dev->id, exec_rc);
 
 	} else if(rc == 1 /* unkown */) {
 	    crm_debug_2("Host %s is not known by %s", victim, dev->id);
 	    
 	} else if(rc == 0 /* active */ || rc == 2 /* inactive */) {
 	    can = TRUE;
 
 	} else {
 	    crm_err("Unkown result calling %s for %s with %s: rc=%d", status, victim, dev->id, rc);
 	}
 
     } else {
 	crm_err("Unknown check type: %s", check_type);
     }
 
     crm_info("%s can%s fence %s: %s", dev->id, can?"":" not", victim, check_type);
     return can;
 }
 
 
 struct device_search_s 
 {
 	const char *host;
 	GListPtr capable;
 };
 
 static void search_devices(
     gpointer key, gpointer value, gpointer user_data) 
 {
     stonith_device_t *dev = value;
     struct device_search_s *search = user_data;
     if(can_fence_host_with_device(dev, search->host)) {
 	search->capable = g_list_append(search->capable, value);
     }
 }
 
 static int stonith_query(xmlNode *msg, xmlNode **list) 
 {
     struct device_search_s search;
     xmlNode *dev = get_xpath_object("//@"F_STONITH_TARGET, msg, LOG_DEBUG_3);
 	
     search.host = NULL;
     search.capable = NULL;
 
     if(dev) {
 	search.host = crm_element_value(dev, F_STONITH_TARGET);
     }
     
     crm_log_xml_info(msg, "Query");
 	
     g_hash_table_foreach(device_list, search_devices, &search);
     if(search.host) {
 	crm_info("Found %d matching devices for '%s'",
 		 g_list_length(search.capable), search.host);
     } else {
 	crm_info("%d devices installed", g_list_length(search.capable));
     }
     
     /* Pack the results into data */
     if(list) {
 	*list = create_xml_node(NULL, __FUNCTION__);
 	crm_xml_add(*list, F_STONITH_TARGET, search.host);
 	crm_xml_add_int(*list, "st-available-devices", g_list_length(search.capable));
 	slist_iter(device, stonith_device_t, search.capable, lpc,
 		   dev = create_xml_node(*list, F_STONITH_DEVICE);
 		   crm_xml_add(dev, XML_ATTR_ID, device->id);
 		   crm_xml_add(dev, "namespace", device->namespace);
 		   crm_xml_add(dev, "agent", device->agent);
 		   if(search.host == NULL) {
 		       xmlNode *attrs = create_xml_node(dev, XML_TAG_ATTRS);
 		       g_hash_table_foreach(device->params, hash2field, attrs);
 		   }
 	    );
     }
     
     return g_list_length(search.capable);
 }
 
 static void log_operation(async_command_t *cmd, int rc, int pid, const char *next, const char *output) 
 {
     if(rc == 0) {
 	next = NULL;
     }
     
     if(cmd->victim != NULL) {
 	do_crm_log(rc==0?LOG_INFO:LOG_ERR,
 		   "Operation '%s' [%d] for host '%s' with device '%s' returned: %d%s%s (call %d from %s)",
 		   cmd->action, pid, cmd->victim, cmd->device, rc, next?". Trying: ":"", next?next:"",
 		   cmd->id, cmd->client);
     } else {
 	do_crm_log(rc==0?LOG_INFO:LOG_NOTICE,
 		   "Operation '%s' [%d] for device '%s' returned: %d%s%s",
 		   cmd->action, pid, cmd->device, rc, next?". Trying: ":"", next?next:"");
     }
 
     if(output) {
 	/* Logging the whole string confuses syslog when the string is xml */ 
 	char *local_copy = crm_strdup(output);
 	int lpc = 0, last = 0, more = strlen(local_copy);
 	for(lpc = 0; lpc < more; lpc++) {
 	    if(local_copy[lpc] == '\n' || local_copy[lpc] == 0) {
 		local_copy[lpc] = 0;
 		crm_debug("%s output: %s", cmd->device, local_copy+last);
 		last = lpc+1;
 	    }
 	}
 	crm_debug("%s output: %s (total %d bytes)", cmd->device, local_copy+last, more);
 	crm_free(local_copy);
     }
 }
 
 #define READ_MAX 500
 static void
 exec_child_done(ProcTrack* proc, int status, int signum, int rc, int waslogged)
 {
     int len = 0;
     int more = 0;
     gboolean bcast = FALSE;
     
     char *output = NULL;  
     xmlNode *data = NULL;
     xmlNode *reply = NULL;
 
     int pid = proctrack_pid(proc);
     async_command_t *cmd = proctrack_data(proc);
     
     CRM_CHECK(cmd != NULL, return);
     active_children--;
 
     if( signum ) {
 	rc = st_err_signal;
 	if( proctrack_timedout(proc) ) {
 	    crm_warn("Child '%d' performing action '%s' with '%s' timed out",
 		     pid, cmd->action, cmd->device);
 	    rc = st_err_timeout;
 	}
     }
 
     do {
 	char buffer[READ_MAX];
 
 	errno = 0;
 	memset(&buffer, 0, READ_MAX);
 	more = read(cmd->stdout, buffer, READ_MAX-1);
 	do_crm_log(status!=0?LOG_DEBUG:LOG_DEBUG_2,
 		   "Got %d more bytes: %s", more, buffer);
 
 	if(more > 0) {
 	    crm_realloc(output, len + more + 1);
 	    sprintf(output+len, "%s", buffer);
 	    len += more;
 	}
 	
     } while (more == (READ_MAX-1) || (more < 0 && errno == EINTR));
 
     if(cmd->stdout) {
 	close(cmd->stdout);
 	cmd->stdout = 0;
     }
 
     while(rc != 0 && cmd->device_next) {
 	int exec_rc = 0;
 	stonith_device_t *dev = cmd->device_next->data;
 	const char *victim = get_victim_name(dev, cmd->victim);
 
 	log_operation(cmd, rc, pid, dev->id, output);
 	
 	cmd->device = dev->id;
 	cmd->device_next = cmd->device_next->next;
 
 	exec_rc = run_stonith_agent(dev->agent, dev->params, cmd->node_attrs, cmd->action, victim, &rc, NULL, cmd);
 	if(exec_rc > 0) {
 	    goto done;
 	}
 	pid = exec_rc;
     }
 
     reply = stonith_construct_async_reply(cmd, output, data, rc);
     if(safe_str_eq(cmd->action, "metadata")) {
 	/* Too verbose to log */
 	crm_free(output); output = NULL;
 
     } else if(crm_str_eq(cmd->action, "reboot", TRUE)
 	   || crm_str_eq(cmd->action, "off", TRUE)
 	   || crm_str_eq(cmd->action, "on", TRUE)) {
 	bcast = TRUE;
     }
 
     log_operation(cmd, rc, pid, NULL, output);
     crm_log_xml_debug_3(reply, "Reply");
     
     if(bcast) {
 	/* Send reply as T_STONITH_NOTIFY so everyone does notifications
 	 * Potentially limit to unsucessful operations to the originator?
 	 */
 	crm_xml_add(reply, F_STONITH_OPERATION, T_STONITH_NOTIFY);
 	send_cluster_message(NULL, crm_msg_stonith_ng, reply, FALSE);
 
     } else if(cmd->origin) {
 	send_cluster_message(cmd->origin, crm_msg_stonith_ng, reply, FALSE);
 
     } else {
 	do_local_reply(reply, cmd->client, cmd->options & st_opt_sync_call, FALSE);
     }
     
     free_async_command(cmd);
   done:
 
     reset_proctrack_data(proc);
     crm_free(output);
     free_xml(reply);
     free_xml(data);
 }
 
 static gint sort_device_priority(gconstpointer a, gconstpointer b)
 {
     const stonith_device_t *dev_a = a;
     const stonith_device_t *dev_b = a;
     if(dev_a->priority > dev_b->priority) {
 	return -1;
     } else if(dev_a->priority < dev_b->priority) {
 	return 1;
     }
     return 0;
 }
 
 static int stonith_fence(xmlNode *msg) 
 {
     int rc = 0;
     struct device_search_s search;
     stonith_device_t *device = NULL;
     async_command_t *cmd = create_async_command(msg, crm_element_value(msg, F_STONITH_ACTION));
     xmlNode *dev = get_xpath_object("//@"F_STONITH_TARGET, msg, LOG_ERR);
     GHashTable *node_attrs = xml2list(dev);
     
     if(cmd == NULL) {
 	return st_err_internal;
     }
     
     search.capable = NULL;
     search.host = crm_element_value(dev, F_STONITH_TARGET);
 
     crm_log_xml_info(msg, "Exec");
     
     g_hash_table_foreach(device_list, search_devices, &search);
     crm_info("Found %d matching devices for '%s'", g_list_length(search.capable), search.host);
 
     if(g_list_length(search.capable) == 0) {
 	return st_err_none_available;
     }
 
     /* Order based on priority */
     search.capable = g_list_sort(search.capable, sort_device_priority);
     
     device = search.capable->data;
     cmd->device = device->id;
 
     if(g_list_length(search.capable) > 1) {
 	cmd->device_list = search.capable;
 	cmd->node_attrs = node_attrs;
     }
     
     return run_stonith_agent(device->agent, device->params, node_attrs, cmd->action, cmd->victim, &rc, NULL, cmd);
 }
 
 xmlNode *stonith_construct_reply(xmlNode *request, char *output, xmlNode *data, int rc) 
 {
     int lpc = 0;
     xmlNode *reply = NULL;
 	
     const char *name = NULL;
     const char *value = NULL;
     const char *names[] = {
 	F_STONITH_OPERATION,
 	F_STONITH_CALLID,
 	F_STONITH_CLIENTID,
 	F_STONITH_REMOTE,
 	F_STONITH_CALLOPTS
     };
 
     crm_debug_4("Creating a basic reply");
     reply = create_xml_node(NULL, T_STONITH_REPLY);
 
     crm_xml_add(reply, "st_origin", __FUNCTION__);
     crm_xml_add(reply, F_TYPE, T_STONITH_NG);
     crm_xml_add(reply, "st_output", output);
     crm_xml_add_int(reply, F_STONITH_RC, rc);
 
     CRM_CHECK(request != NULL, crm_warn("Can't create a sane reply"); return reply);
     for(lpc = 0; lpc < DIMOF(names); lpc++) {
 	name = names[lpc];
 	value = crm_element_value(request, name);
 	crm_xml_add(reply, name, value);
     }
 
     if(data != NULL) {
 	crm_debug_4("Attaching reply output");
 	add_message_xml(reply, F_STONITH_CALLDATA, data);
     }
     return reply;
 }
 
 xmlNode *stonith_construct_async_reply(async_command_t *cmd, char *output, xmlNode *data, int rc) 
 {
     xmlNode *reply = NULL;
 
     crm_debug_4("Creating a basic reply");
     reply = create_xml_node(NULL, T_STONITH_REPLY);
 
     crm_xml_add(reply, "st_origin", __FUNCTION__);
     crm_xml_add(reply, F_TYPE, T_STONITH_NG);
 
     crm_xml_add(reply, F_STONITH_OPERATION, cmd->op);
     crm_xml_add(reply, F_STONITH_REMOTE, cmd->remote);
     crm_xml_add(reply, F_STONITH_CLIENTID, cmd->client);
     crm_xml_add_int(reply, F_STONITH_CALLID, cmd->id);
     crm_xml_add_int(reply, F_STONITH_CALLOPTS, cmd->options);
     
     crm_xml_add_int(reply, F_STONITH_RC, rc);
     crm_xml_add(reply, "st_output", output);
 
     if(data != NULL) {
 	crm_info("Attaching reply output");
 	add_message_xml(reply, F_STONITH_CALLDATA, data);
     }
     return reply;
 }
 
 void
 stonith_command(stonith_client_t *client, xmlNode *request, const char *remote)
 {
     int call_options = 0;
     int rc = st_err_generic;
 
     gboolean is_reply = FALSE;
 
     xmlNode *reply = NULL;
     xmlNode *data = NULL;
 
     char *output = NULL;
     const char *op = crm_element_value(request, F_STONITH_OPERATION);
     const char *client_id = crm_element_value(request, F_STONITH_CLIENTID);
     
     crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
     if(get_xpath_object("//"T_STONITH_REPLY, request, LOG_DEBUG_3)) {
 	is_reply = TRUE;
     }
     
     if(device_list == NULL) {
 	device_list = g_hash_table_new_full(
 	    g_str_hash, g_str_equal, NULL, free_device);
     }
     
     crm_debug("Processing %s%s from %s", op, is_reply?" reply":"",
 	      client?client->name:remote);
 
     if(crm_str_eq(op, CRM_OP_REGISTER, TRUE)) {
 	return;
 	    
     } else if(crm_str_eq(op, STONITH_OP_DEVICE_ADD, TRUE)) {
 	rc = stonith_device_register(request);
 	do_stonith_notify(call_options, op, rc, request, NULL);
 
     } else if(crm_str_eq(op, STONITH_OP_DEVICE_DEL, TRUE)) {
 	rc = stonith_device_remove(request);
 	do_stonith_notify(call_options, op, rc, request, NULL);
 	
 
     } else if(crm_str_eq(op, STONITH_OP_EXEC, TRUE)) {
 	rc = stonith_device_action(request, &output);
 
     } else if(is_reply && crm_str_eq(op, STONITH_OP_QUERY, TRUE)) {
 	process_remote_stonith_query(request);
 	return;
 	
     } else if(crm_str_eq(op, STONITH_OP_QUERY, TRUE)) {
 	create_remote_stonith_op(client_id, request, TRUE); /* Record it for the future notification */
 	rc = stonith_query(request, &data);
 
     } else if(is_reply && crm_str_eq(op, T_STONITH_NOTIFY, TRUE)) {
 	process_remote_stonith_exec(request);
 	return;
 	
     } else if(crm_str_eq(op, T_STONITH_NOTIFY, TRUE)) {
 	const char *flag_name = NULL;
 
 	flag_name = crm_element_value(request, F_STONITH_NOTIFY_ACTIVATE);
 	if(flag_name) {
 	    crm_debug("Setting %s callbacks for %s (%s): ON",
 		      flag_name, client->name, client->id);
 	    client->flags |= get_stonith_flag(flag_name);
 	}
 	
 	flag_name = crm_element_value(request, F_STONITH_NOTIFY_DEACTIVATE);
 	if(flag_name) {
 	    crm_debug("Setting %s callbacks for %s (%s): off",
 		      flag_name, client->name, client->id);
 	    client->flags |= get_stonith_flag(flag_name);
 	}
 	return;
 
     /* } else if(is_reply && crm_str_eq(op, STONITH_OP_FENCE, TRUE)) { */
     /* 	process_remote_stonith_exec(request); */
     /* 	return; */
 
     } else if(is_reply == FALSE && crm_str_eq(op, STONITH_OP_FENCE, TRUE)) {
 
 	if(remote) {
 	    rc = stonith_fence(request);
 
 	} else if(call_options & st_opt_local_first) {
 	    rc = stonith_fence(request);
 	    if(rc < 0) {
 		initiate_remote_stonith_op(client, request);
 	    }
 
 	} else {
 	    initiate_remote_stonith_op(client, request);
 	}
 	return;
 
     } else {
 	crm_err("Unknown %s%s from %s", op, is_reply?" reply":"",
 		 client?client->name:remote);
 	crm_log_xml_warn(request, "UnknownOp");
     }
 
     crm_info("Processed %s%s from %s: rc=%d", op, is_reply?" reply":"",
 	     client?client->name:remote, rc);
     
     if(is_reply) {
 	/* Nothing */	
 	
     } else if(remote) {
 	reply = stonith_construct_reply(request, output, data, rc);
 	send_cluster_message(remote, crm_msg_stonith_ng, reply, FALSE);
 
     } else if(rc <= 0 || crm_str_eq(op, STONITH_OP_QUERY, TRUE)) {
 	reply = stonith_construct_reply(request, output, data, rc);
 	do_local_reply(reply, client_id, call_options & st_opt_sync_call, remote!=NULL);
 	free_xml(reply);
     }    
 
     crm_free(output);
     free_xml(data);
 }
diff --git a/include/crm/common/xml.h b/include/crm/common/xml.h
index bb21f250ab..0ab69a4b4a 100644
--- a/include/crm/common/xml.h
+++ b/include/crm/common/xml.h
@@ -1,298 +1,312 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 #ifndef CRM_COMMON_XML__H
 #define CRM_COMMON_XML__H
 
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 
 #include <crm/crm.h>
 #include <ha_msg.h>
 
 #include <libxml/tree.h> 
 #include <libxml/xpath.h>
 typedef xmlNode crm_data_t;
 
 #define CRM_BZ2_BLOCKS		4
 #define CRM_BZ2_WORK		20
 #define CRM_BZ2_THRESHOLD	10 * 1024
 
 #define XML_PARANOIA_CHECKS 0
 
 extern gboolean add_message_xml(
 	xmlNode *msg, const char *field, xmlNode *xml);
 extern xmlNode *get_message_xml(xmlNode *msg, const char *field);
 extern GHashTable *xml2list(xmlNode *parent);
 #if CRM_DEPRECATED_SINCE_2_0_3
 extern GHashTable *xml2list_202(xmlNode *parent);
 #endif
 extern void hash2nvpair(gpointer key, gpointer value, gpointer user_data);
 extern void hash2field(gpointer key, gpointer value, gpointer user_data);
 extern void hash2metafield(gpointer key, gpointer value, gpointer user_data);
 extern void hash2smartfield(gpointer key, gpointer value, gpointer user_data);
 
 /*
  * Replacement function for xmlCopyPropList which at the very least,
  * doesnt work the way *I* would expect it to.
  *
  * Copy all the attributes/properties from src into target.
  *
  * Not recursive, does not return anything. 
  *
  */
 extern void copy_in_properties(xmlNode *target, xmlNode *src);
 extern void expand_plus_plus(xmlNode* target, const char *name, const char *value);
 extern void fix_plus_plus_recursive(xmlNode* target);
 
 void free_xml_from_parent(xmlNode *parent, xmlNode *a_node);
 #define zap_xml_from_parent(parent, xml_obj) free_xml_from_parent(parent, xml_obj); xml_obj = NULL
 
 
 /*
  * Create a node named "name" as a child of "parent"
  * If parent is NULL, creates an unconnected node.
  *
  * Returns the created node
  *
  */
 extern xmlNode *create_xml_node(xmlNode *parent, const char *name);
 
 /*
  * Make a copy of name and value and use the copied memory to create
  * an attribute for node.
  *
  * If node, name or value are NULL, nothing is done.
  *
  * If name or value are an empty string, nothing is done.
  *
  * Returns FALSE on failure and TRUE on success.
  *
  */
 extern const char *crm_xml_add(
 	xmlNode *node, const char *name, const char *value);
 
 extern const char *crm_xml_replace(
 	xmlNode *node, const char *name, const char *value);
 
 extern const char *crm_xml_add_int(
 	xmlNode* node, const char *name, int value);
 
 /*
  * Unlink the node and set its doc pointer to NULL so free_xml()
  * will act appropriately
  */
 extern void unlink_xml_node(xmlNode *node);
 
 /*
  * 
  */
 extern void purge_diff_markers(xmlNode *a_node);
 
 /*
  * Returns a deep copy of src_node
  *
  */
 extern xmlNode *copy_xml(xmlNode *src_node);
 
 /*
  * Add a copy of xml_node to new_parent
  */
 extern xmlNode *add_node_copy(
 	xmlNode *new_parent, xmlNode *xml_node);
 
 extern int add_node_nocopy(xmlNode *parent, const char *name, xmlNode *child);
 
 
 /*
  * XML I/O Functions
  *
  * Whitespace between tags is discarded.
  */
 extern xmlNode *filename2xml(const char *filename);
 
 extern xmlNode *stdin2xml(void);
 
 extern xmlNode *string2xml(const char *input);
 
 extern int write_xml_file(
 	xmlNode *xml_node, const char *filename, gboolean compress);
 
 extern char *dump_xml_formatted(xmlNode *msg);
 
 extern char *dump_xml_unformatted(xmlNode *msg);
 
 extern void print_xml_formatted(
 	int log_level, const char *function,
 	xmlNode *an_xml_node, const char *text);
 
 /*
  * Diff related Functions
  */ 
 extern xmlNode *diff_xml_object(
 	xmlNode *left, xmlNode *right, gboolean suppress);
 
 extern void print_xml_diff(FILE *where, xmlNode *diff);
 extern void log_xml_diff(unsigned int log_level, xmlNode *diff, const char *function);
 
 extern gboolean apply_xml_diff(
 	xmlNode *old, xmlNode *diff, xmlNode **new);
 
 
 /*
  * Searching & Modifying
  */
 extern xmlNode *find_xml_node(
 	xmlNode *cib, const char * node_path, gboolean must_find);
 
 extern xmlNode *find_entity(
 	xmlNode *parent, const char *node_name, const char *id);
 
 extern xmlNode *subtract_xml_object(
 	xmlNode *left, xmlNode *right, const char *marker);
 
 extern int add_xml_object(
 	xmlNode *parent, xmlNode *target, xmlNode *update);
 
 extern void xml_remove_prop(xmlNode *obj, const char *name);
 
 extern gboolean replace_xml_child(
 	xmlNode *parent, xmlNode *child, xmlNode *update, gboolean delete_only);
 
 extern gboolean update_xml_child(xmlNode *child, xmlNode *to_update);
 
 extern int find_xml_children(
 	xmlNode **children, xmlNode *root,
 	const char *tag, const char *field, const char *value,
 	gboolean search_matches);
 
 extern int crm_element_value_int(xmlNode *data, const char *name, int *dest);
 extern char *crm_element_value_copy(xmlNode *data, const char *name);
 extern const char *crm_element_value_const(const xmlNode *data, const char *name);
 extern xmlNode *get_xpath_object(const char *xpath, xmlNode *xml_obj, int error_level);
 extern xmlNode *get_xpath_object_relative(const char *xpath, xmlNode *xml_obj, int error_level);
 
 static inline const char *crm_element_name(const xmlNode *data)
 {
     return (data ? (const char *)data->name : NULL);
 }
 
 extern const char *crm_element_value(xmlNode *data, const char *name);
 
 extern void xml_validate(const xmlNode *root);
 
 extern gboolean xml_has_children(const xmlNode *root);	 		
 
 extern char *calculate_xml_digest(xmlNode *local_cib, gboolean sort, gboolean do_filter);
 
 extern gboolean validate_xml(xmlNode *xml_blob, const char *validation, gboolean to_logs);
 extern gboolean validate_xml_verbose(xmlNode *xml_blob);
 extern int update_validation(xmlNode **xml_blob, int *best, gboolean transform, gboolean to_logs);
 extern int get_schema_version(const char *name);
 extern const char *get_schema_name(int version);
 
 #if XML_PARANOIA_CHECKS
 #  define crm_validate_data(obj) xml_validate(obj)
 #else
 #  define crm_validate_data(obj) CRM_DEV_ASSERT(obj != NULL)
 #endif
 
 #  define xml_child_iter(parent, child, code) do {			\
 	if(parent != NULL) {						\
 		xmlNode *child = NULL;					\
 		xmlNode *__crm_xml_iter = parent->children;		\
 		while(__crm_xml_iter != NULL) {				\
 			child = __crm_xml_iter;				\
 			__crm_xml_iter = __crm_xml_iter->next;		\
 			if(child->type == XML_ELEMENT_NODE) {		\
 			    code;					\
 			}						\
 		}							\
 	}								\
     } while(0)
 
 #  define xml_child_iter_filter(parent, child, filter, code) do {	\
 	if(parent != NULL) {						\
 	    xmlNode *child = NULL;					\
 	    xmlNode *__crm_xml_iter = parent->children;			\
 	    while(__crm_xml_iter != NULL) {				\
 		child = __crm_xml_iter;					\
 		__crm_xml_iter = __crm_xml_iter->next;			\
 		if(child->type == XML_ELEMENT_NODE) {			\
 		    if(filter == NULL					\
 		       || crm_str_eq(filter, (const char *)child->name, TRUE)) { \
 			code;						\
 		    }							\
 		}							\
 	    }								\
 	}								\
     } while(0)
 
 #  define xml_prop_iter(parent, prop_name, prop_value, code) do {	\
 	if(parent != NULL) {						\
 	    xmlAttrPtr prop_iter = parent->properties;			\
 	    const char *prop_name = NULL;				\
 	    const char *prop_value = NULL;				\
 	    while(prop_iter != NULL) {					\
 		prop_name = (const char *)prop_iter->name;		\
-		prop_value = crm_element_value(parent, prop_name);\
+		prop_value = crm_element_value(parent, prop_name);	\
+		prop_iter = prop_iter->next;				\
+		if(prop_name) {						\
+		    code;						\
+		}							\
+	    }								\
+	}								\
+    } while(0)
+
+#  define xml_prop_name_iter(parent, prop_name, code) do {		\
+	if(parent != NULL) {						\
+	    xmlAttrPtr prop_iter = parent->properties;			\
+	    const char *prop_name = NULL;				\
+	    while(prop_iter != NULL) {					\
+		prop_name = (const char *)prop_iter->name;		\
 		prop_iter = prop_iter->next;				\
 		if(prop_name) {						\
 		    code;						\
 		}							\
 	    }								\
 	}								\
     } while(0)
 
 #  define free_xml(a_node) do {						\
 	if((a_node) != NULL) {						\
 	    xmlNode *a_doc_top = NULL;					\
 	    xmlDoc *a_doc = (a_node)->doc;				\
 	    if (a_doc != NULL) {					\
 		a_doc_top = xmlDocGetRootElement(a_doc);		\
 	    }								\
 	    if(a_doc != NULL && a_doc_top == (a_node)) {		\
 		xmlFreeDoc(a_doc);					\
 									\
 	    } else {							\
 		/* make sure the node is unlinked first */		\
 		xmlUnlinkNode(a_node);					\
 		xmlFreeNode(a_node);					\
 	    }								\
 	}								\
     } while(0)
 
 
 extern xmlNode *first_named_child(xmlNode *parent, const char *name);
 
 extern xmlNode *convert_ipc_message(IPC_Message *msg, const char *field);
 extern xmlNode *convert_ha_message(xmlNode *parent, HA_Message *msg, const char *field);
 
 extern HA_Message *convert_xml_message(xmlNode *msg);
 extern xmlNode *sorted_xml(xmlNode *input, xmlNode *parent, gboolean recursive);
 extern xmlXPathObjectPtr xpath_search(xmlNode *xml_top, const char *path);
 extern gboolean cli_config_update(xmlNode **xml, int *best_version, gboolean to_logs);
 extern xmlNode *expand_idref(xmlNode *input, xmlNode *top);
 
 extern xmlNode *getXpathResult(xmlXPathObjectPtr xpathObj, int index);
 
 #endif
diff --git a/lib/ais/plugin.c b/lib/ais/plugin.c
index 34849434de..041c2ecd01 100644
--- a/lib/ais/plugin.c
+++ b/lib/ais/plugin.c
@@ -1,1747 +1,1745 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 #include <sys/types.h>
 #include <sys/uio.h>
 #include <sys/socket.h>
 #include <sys/un.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <unistd.h>
 #include <fcntl.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <errno.h>
 #include <signal.h>
 #include <string.h>
 
 #ifdef AIS_COROSYNC
 #  include <corosync/totem/totempg.h>
 #  include <corosync/engine/objdb.h>
 #  include <corosync/engine/config.h>
 #endif
 
 #include <config.h>
 #include <crm/ais.h>
 #include "plugin.h"
 #include "utils.h"
 
 #include <glib/ghash.h>
 
 #include <sys/resource.h>
 #include <sys/utsname.h>
 #include <sys/socket.h>
 #include <sys/wait.h>
 #include <sys/stat.h>
 #include <pthread.h>
 #include <bzlib.h>
 #include <pwd.h>
 
 struct corosync_api_v1 *pcmk_api = NULL;
 
 uint32_t plugin_has_votes = 0;
 uint32_t plugin_expected_votes = 2;
 
 int use_mgmtd = 0;
 int plugin_log_level = LOG_DEBUG;
 char *local_uname = NULL;
 int local_uname_len = 0;
 char *local_cname = NULL;
 int local_cname_len = 0;
 uint32_t local_nodeid = 0;
 char *ipc_channel_name = NULL;
 static uint64_t local_born_on = 0;
 
 uint64_t membership_seq = 0;
 pthread_t pcmk_wait_thread;
 
 gboolean wait_active = TRUE;
 gboolean have_reliable_membership_id = FALSE;
 GHashTable *ipc_client_list = NULL;
 GHashTable *membership_list = NULL;
 GHashTable *membership_notify_list = NULL;
 
 #define MAX_RESPAWN		100
 #define LOOPBACK_ID		16777343
 #define crm_flag_none		0x00000000
 #define crm_flag_members	0x00000001
 
 struct crm_identify_msg_s
 {
 	coroipc_request_header_t	header __attribute__((aligned(8)));
 	uint32_t		id;
 	uint32_t		pid;
 	 int32_t		votes;
 	uint32_t		processes;
 	char			uname[256];
 	char			version[256];
 	uint64_t		born_on;
 } __attribute__((packed));
 
 static crm_child_t pcmk_children[] = {
     { 0, crm_proc_none,     crm_flag_none,    0, 0, FALSE, "none",     NULL,		NULL,			   NULL, NULL },
     { 0, crm_proc_ais,      crm_flag_none,    0, 0, FALSE, "ais",      NULL,		NULL,			   NULL, NULL },
     { 0, crm_proc_lrmd,     crm_flag_none,    3, 0, TRUE,  "lrmd",     NULL,		HB_DAEMON_DIR"/lrmd",      NULL, NULL },
     { 0, crm_proc_cib,      crm_flag_members, 2, 0, TRUE,  "cib",      CRM_DAEMON_USER, CRM_DAEMON_DIR"/cib",      NULL, NULL },
     { 0, crm_proc_crmd,     crm_flag_members, 6, 0, TRUE,  "crmd",     CRM_DAEMON_USER, CRM_DAEMON_DIR"/crmd",     NULL, NULL },
     { 0, crm_proc_attrd,    crm_flag_none,    4, 0, TRUE,  "attrd",    CRM_DAEMON_USER, CRM_DAEMON_DIR"/attrd",    NULL, NULL },
     { 0, crm_proc_stonithd, crm_flag_none,    0, 0, TRUE,  "stonithd", NULL,		"/bin/false",		   NULL, NULL },
     { 0, crm_proc_pe,       crm_flag_none,    5, 0, TRUE,  "pengine",  CRM_DAEMON_USER, CRM_DAEMON_DIR"/pengine",  NULL, NULL },
     { 0, crm_proc_mgmtd,    crm_flag_none,    7, 0, TRUE,  "mgmtd",    NULL,		HB_DAEMON_DIR"/mgmtd",     NULL, NULL },
     { 0, crm_proc_stonith_ng, crm_flag_none,  1, 0, TRUE,  "stonith-ng", NULL,		CRM_DAEMON_DIR"/stonithd", NULL, NULL },
 };
 
 void send_cluster_id(void);
 int send_cluster_msg_raw(const AIS_Message *ais_msg);
 char *pcmk_generate_membership_data(void);
 gboolean check_message_sanity(const AIS_Message *msg, const char *data);
 
 #ifdef AIS_COROSYNC
 typedef const void ais_void_ptr;
 int pcmk_shutdown(void);
 void pcmk_peer_update(enum totem_configuration_type configuration_type,
 		      const unsigned int *member_list, size_t member_list_entries,
 		      const unsigned int *left_list, size_t left_list_entries,
 		      const unsigned int *joined_list, size_t joined_list_entries,
 		      const struct memb_ring_id *ring_id);
 #else
 
 typedef void ais_void_ptr;
 extern totempg_groups_handle openais_group_handle;
 int pcmk_shutdown(struct objdb_iface_ver0 *objdb);
 void pcmk_peer_update(enum totem_configuration_type configuration_type,
 		      unsigned int *member_list, int member_list_entries,
 		      unsigned int *left_list, int left_list_entries,
 		      unsigned int *joined_list, int joined_list_entries,
 		      struct memb_ring_id *ring_id);
 #endif
 
 int pcmk_startup (struct corosync_api_v1 *corosync_api);
 int pcmk_config_init(struct corosync_api_v1 *corosync_api);
 
 int pcmk_ipc_exit (void *conn);
 int pcmk_ipc_connect (void *conn);
 void pcmk_ipc(void *conn, ais_void_ptr *msg);
 
 void pcmk_exec_dump(void);
 void pcmk_cluster_swab(void *msg);
 void pcmk_cluster_callback(ais_void_ptr *message, unsigned int nodeid);
 
 void pcmk_nodeid(void *conn, ais_void_ptr *msg);
 void pcmk_nodes(void *conn, ais_void_ptr *msg);
 void pcmk_notify(void *conn, ais_void_ptr *msg);
 void pcmk_remove_member(void *conn, ais_void_ptr *msg);
 void pcmk_quorum(void *conn, ais_void_ptr *msg);
 
 void pcmk_cluster_id_swab(void *msg);
 void pcmk_cluster_id_callback(ais_void_ptr *message, unsigned int nodeid);
 void ais_remove_peer(char *node_id);
 
 static uint32_t get_process_list(void) 
 {
     int lpc = 0;
     uint32_t procs = crm_proc_ais;
     for (lpc = 0; lpc < SIZEOF(pcmk_children); lpc++) {
 	if(pcmk_children[lpc].pid != 0) {
 	    procs |= pcmk_children[lpc].flag;
 	}
     }
     return procs;
 }
 
 
 static struct corosync_lib_handler pcmk_lib_service[] =
 {
     { /* 0 */
 	.lib_handler_fn		= pcmk_ipc,
 	.flow_control		= COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED,
 #ifdef AIS_WHITETANK
 	.response_size		= sizeof (coroipc_response_header_t),
 	.response_id		= CRM_MESSAGE_IPC_ACK,
 #endif
     },
     { /* 1 */
 	.lib_handler_fn		= pcmk_nodes,
 	.flow_control		= COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED,
 #ifdef AIS_WHITETANK
 	.response_size		= sizeof (coroipc_response_header_t),
 	.response_id		= CRM_MESSAGE_IPC_ACK,
 #endif
     },
     { /* 2 */
 	.lib_handler_fn		= pcmk_notify,
 	.flow_control		= COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED,
 #ifdef AIS_WHITETANK
 	.response_size		= sizeof (coroipc_response_header_t),
 	.response_id		= CRM_MESSAGE_IPC_ACK,
 #endif
     },
     { /* 3 */
 	.lib_handler_fn		= pcmk_nodeid,
 	.flow_control		= COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED,
 #ifdef AIS_WHITETANK
 	.response_size		= sizeof (struct crm_ais_nodeid_resp_s),
 	.response_id		= crm_class_nodeid,
 #endif
     },
     { /* 4 */
 	.lib_handler_fn		= pcmk_remove_member,
 	.flow_control		= COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED,
 #ifdef AIS_WHITETANK
 	.response_size		= sizeof (coroipc_response_header_t),
 	.response_id		= CRM_MESSAGE_IPC_ACK,
 #endif
     },
     { /* 5 */
 	.lib_handler_fn		= pcmk_quorum,
 	.flow_control		= COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED,
 #ifdef AIS_WHITETANK
 	.response_size		= sizeof (coroipc_response_header_t),
 	.response_id		= CRM_MESSAGE_IPC_ACK,
 #endif
     },
 };
 
 static struct corosync_exec_handler pcmk_exec_service[] =
 {
     { /* 0 */
 	.exec_handler_fn	= pcmk_cluster_callback,
 	.exec_endian_convert_fn = pcmk_cluster_swab
     },
     { /* 1 */
 	.exec_handler_fn	= pcmk_cluster_id_callback,
 	.exec_endian_convert_fn = pcmk_cluster_id_swab
     }
 };
 
 /*
  * Exports the interface for the service
  */
 struct corosync_service_engine pcmk_service_handler = {
     .name			= (unsigned char *)"Pacemaker Cluster Manager "PACKAGE_VERSION,
     .id				= PCMK_SERVICE_ID,
     .private_data_size		= 0,
     .flow_control		= COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED, 
     .lib_init_fn		= pcmk_ipc_connect,
     .lib_exit_fn		= pcmk_ipc_exit,
     .exec_init_fn		= pcmk_startup,
     .exec_exit_fn		= pcmk_shutdown,
     .config_init_fn		= pcmk_config_init,
 #ifdef AIS_COROSYNC
     .priority			= 50,    
     .lib_engine			= pcmk_lib_service,
     .lib_engine_count		= sizeof (pcmk_lib_service) / sizeof (struct corosync_lib_handler),
     .exec_engine		= pcmk_exec_service,
     .exec_engine_count		= sizeof (pcmk_exec_service) / sizeof (struct corosync_exec_handler),
 #else
     .lib_service		= pcmk_lib_service,
     .lib_service_count		= sizeof (pcmk_lib_service) / sizeof (struct corosync_lib_handler),
     .exec_service		= pcmk_exec_service,
     .exec_service_count		= sizeof (pcmk_exec_service) / sizeof (struct corosync_exec_handler),
 #endif
     .confchg_fn			= pcmk_peer_update,
     .exec_dump_fn		= pcmk_exec_dump,
 /* 	void (*sync_init) (void); */
 /* 	int (*sync_process) (void); */
 /* 	void (*sync_activate) (void); */
 /* 	void (*sync_abort) (void); */
 };
 
 
 /*
  * Dynamic Loader definition
  */
 struct corosync_service_engine *pcmk_get_handler_ver0 (void);
 
 #ifdef AIS_COROSYNC
 struct corosync_service_engine_iface_ver0 pcmk_service_handler_iface = {
     .corosync_get_service_engine_ver0 = pcmk_get_handler_ver0
 };
 #else
 struct openais_service_handler_iface_ver0 pcmk_service_handler_iface = {
     .openais_get_service_handler_ver0 = pcmk_get_handler_ver0
 };
 #endif
 
 static struct lcr_iface openais_pcmk_ver0[1] = {
     {
 	.name				= "pacemaker",
 	.version			= 0,
 	.versions_replace		= 0,
 	.versions_replace_count		= 0,
 	.dependencies			= 0,
 	.dependency_count		= 0,
 	.constructor			= NULL,
 	.destructor			= NULL,
 	.interfaces			= NULL
     }
 };
 
 static struct lcr_comp pcmk_comp_ver0 = {
     .iface_count				= 1,
     .ifaces					= openais_pcmk_ver0
 };
 
 struct corosync_service_engine *pcmk_get_handler_ver0 (void)
 {
     return (&pcmk_service_handler);
 }
 
 __attribute__ ((constructor)) static void register_this_component (void) {
     lcr_interfaces_set (&openais_pcmk_ver0[0], &pcmk_service_handler_iface);
 
     lcr_component_register (&pcmk_comp_ver0);
 }
 
 static int plugin_has_quorum(void) 
 {
     if((plugin_expected_votes >> 1) < plugin_has_votes) {
 	return 1;
     }
     return 0;
 }
 
 static void update_expected_votes(int value) 
 {
     if(value < plugin_has_votes) {
 	/* Never drop below the number of connected nodes */
 	ais_info("Cannot update expected quorum votes %d -> %d:"
 		 " value cannot be less that the current number of votes",
 		 plugin_expected_votes, value);
 
     } else if(plugin_expected_votes != value) {
 	ais_info("Expected quorum votes %d -> %d",
 		 plugin_expected_votes, value);
 	plugin_expected_votes = value;
     }
 }
 
 /* Create our own local copy of the config so we can navigate it */
 static void process_ais_conf(void)
 {
     char *value = NULL;
     gboolean any_log = FALSE;
     hdb_handle_t top_handle = 0;
     hdb_handle_t local_handle = 0;
     
     ais_info("Reading configure");
     top_handle = config_find_init(pcmk_api, "logging");
     local_handle = config_find_next(pcmk_api, "logging", top_handle);
     
     get_config_opt(pcmk_api, local_handle, "debug", &value, "on");
     if(ais_get_boolean(value)) {
 	plugin_log_level = LOG_DEBUG;
 	pcmk_env.debug = "1";
 	
     } else {
 	plugin_log_level = LOG_INFO;
 	pcmk_env.debug = "0";
     }    
     
     get_config_opt(pcmk_api, local_handle, "to_logfile", &value, "off");
     if(ais_get_boolean(value)) {
 	get_config_opt(pcmk_api, local_handle, "logfile", &value, NULL);
 
 	if(value == NULL) {
 	    ais_err("Logging to a file requested but no log file specified");
 
 	} else {
 	    uid_t pcmk_uid = geteuid();
 	    uid_t pcmk_gid = getegid();
 
-	    pcmk_env.logfile = value;
+	    FILE *logfile = fopen(value, "a");
+	    if(logfile) {
+		int logfd = fileno(logfile);
 
-	    if(pcmk_uid >= 0 && pcmk_gid >= 0) {
+		pcmk_env.logfile = value;
+	
 		/* Ensure the file has the correct permissions */
-		FILE *logfile = fopen(value, "a");
-		int logfd = fileno(logfile);
-		
 		fchown(logfd, pcmk_uid, pcmk_gid);
 		fchmod(logfd, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP);
 		
 		fprintf(logfile, "Set r/w permissions for uid=%d, gid=%d on %s\n",
 			pcmk_uid, pcmk_gid, value);
 		fflush(logfile);
-		fsync(fileno(logfile));
+		fsync(logfd);
 		fclose(logfile);
 		any_log = TRUE;
 
 	    } else {
-		ais_err("Couldn't setup correct logfile permissions, some logs may be lost");
+		ais_err("Couldn't create logfile: %s", value);
 	    }
 	}
     }
 
     get_config_opt(pcmk_api, local_handle, "to_syslog", &value, "on");
     if(any_log && ais_get_boolean(value) == FALSE) {
 	ais_info("User configured file based logging and explicitly disabled syslog.");
 	value = "none";
 
     } else {
 	if(ais_get_boolean(value) == FALSE) {
 	    ais_err("Please enable some sort of logging, either 'to_file: on' or  'to_syslog: on'.");
 	    ais_err("If you use file logging, be sure to also define a value for 'logfile'");	    
 	}
 	get_config_opt(pcmk_api, local_handle, "syslog_facility", &value, "daemon");
     }
     pcmk_env.syslog = value;
 
     config_find_done(pcmk_api, local_handle);
     
     top_handle = config_find_init(pcmk_api, "service");
     local_handle = config_find_next(pcmk_api, "service", top_handle);
     while(local_handle) {
 	value = NULL;
 	pcmk_api->object_key_get(local_handle, "name", strlen("name"), (void**)&value, NULL);
 	if(ais_str_eq("pacemaker", value)) {
 	    break;
 	}
 	local_handle = config_find_next(pcmk_api, "service", top_handle);
     }
 
     get_config_opt(pcmk_api, local_handle, "clustername", &local_cname, "pcmk");
     local_cname_len = strlen(local_cname);
     
     get_config_opt(pcmk_api, local_handle, "use_logd", &value, "no");
     pcmk_env.use_logd = value;
 
     get_config_opt(pcmk_api, local_handle, "use_mgmtd", &value, "no");
     if(ais_get_boolean(value) == FALSE) {
 	int lpc = 0;
 	for (; lpc < SIZEOF(pcmk_children); lpc++) {
 	    if(crm_proc_mgmtd & pcmk_children[lpc].flag) {
 		/* Disable mgmtd startup */
 		pcmk_children[lpc].start_seq = 0;
 		break;
 	    }
 	}
     }
     
     config_find_done(pcmk_api, local_handle);
 }
 
 int pcmk_config_init(struct corosync_api_v1 *unused)
 {
     return 0;
 }
 
 static void *pcmk_wait_dispatch (void *arg)
 {
     struct timespec waitsleep = {
 	.tv_sec = 1,
 	.tv_nsec = 0
     };
     
     while(wait_active) {
 	int lpc = 0;
 	for (; lpc < SIZEOF(pcmk_children); lpc++) {
 	    if(pcmk_children[lpc].pid > 0) {
 		int status;
 		pid_t pid = wait4(
 		    pcmk_children[lpc].pid, &status, WNOHANG, NULL);
 
 		if(pid == 0) {
 		    continue;
 		    
 		} else if(pid < 0) {
 		    ais_perror("Call to wait4(%s) failed", pcmk_children[lpc].name);
 		    continue;
 		}
 
 		/* cleanup */
 		pcmk_children[lpc].pid = 0;
 		pcmk_children[lpc].conn = NULL;
 		pcmk_children[lpc].async_conn = NULL;
 
 		if(WIFSIGNALED(status)) {
 		    int sig = WTERMSIG(status);
 		    ais_err("Child process %s terminated with signal %d"
 			     " (pid=%d, core=%s)",
 			     pcmk_children[lpc].name, sig, pid,
 			     WCOREDUMP(status)?"true":"false");
 
 		} else if (WIFEXITED(status)) {
 		    int rc = WEXITSTATUS(status);
 		    do_ais_log(rc==0?LOG_NOTICE:LOG_ERR, "Child process %s exited (pid=%d, rc=%d)",
 			       pcmk_children[lpc].name, pid, rc);
 
 		    if(rc == 100) {
 			ais_notice("Child process %s no longer wishes"
 				   " to be respawned", pcmk_children[lpc].name);
 			pcmk_children[lpc].respawn = FALSE;
 		    }
 		}
 
 		pcmk_children[lpc].respawn_count += 1;
 		if(pcmk_children[lpc].respawn_count > MAX_RESPAWN) {
 		    ais_err("Child respawn count exceeded by %s",
 			       pcmk_children[lpc].name);
 		    pcmk_children[lpc].respawn = FALSE;
 		}
 		if(pcmk_children[lpc].respawn) {
 		    ais_notice("Respawning failed child process: %s",
 			       pcmk_children[lpc].name);
 		    spawn_child(&(pcmk_children[lpc]));
-		} else {
-		    send_cluster_id();
 		}
+		send_cluster_id();
 	    }
 	}
 	sched_yield ();
 	nanosleep (&waitsleep, 0);
     }
     return 0;
 }
 
 static uint32_t pcmk_update_nodeid(void)  
 {
     int last = local_nodeid;
 #if AIS_COROSYNC
     local_nodeid = pcmk_api->totem_nodeid_get();
 #else
     local_nodeid = totempg_my_nodeid_get();
 #endif
 
     if(last != local_nodeid) {
 	if(last == 0) {
 	    ais_info("Local node id: %u", local_nodeid);
 
 	} else {
 	    char *last_s = NULL;
 	    ais_malloc0(last_s, 32);
 	    ais_warn("Detected local node id change: %u -> %u", last, local_nodeid);
 	    snprintf(last_s, 31, "%u", last);
 	    ais_remove_peer(last_s);
 	    ais_free(last_s);
 	}
 	update_member(local_nodeid, 0, 0, 1, 0, local_uname, CRM_NODE_MEMBER, NULL);
     }
     
     return local_nodeid;
 }
 
 static void build_path(const char *path_c, mode_t mode)
 {
     int offset = 1, len = 0;
     char *path = ais_strdup(path_c);
 
     AIS_CHECK(path != NULL, return);
     for(len = strlen(path); offset < len; offset++) {
 	if(path[offset] == '/') {
 	    path[offset] = 0;
 	    if(mkdir(path, mode) < 0 && errno != EEXIST) {
 		ais_perror("Could not create directory '%s'", path);
 		break;
 	    }
 	    path[offset] = '/';
 	}
     }
     if(mkdir(path, mode) < 0 && errno != EEXIST) {
 	ais_perror("Could not create directory '%s'", path);
     }
     ais_free(path);
 }
 
 int pcmk_startup(struct corosync_api_v1 *init_with)
 {
     int rc = 0;
     int lpc = 0;
     int start_seq = 1;
     struct utsname us;
     struct rlimit cores;
     static int max = SIZEOF(pcmk_children);
 
     uid_t pcmk_uid = 0;
     gid_t pcmk_gid = 0;
 
     uid_t root_uid = -1;
     uid_t cs_uid = geteuid();
     pcmk_user_lookup("root", &root_uid, NULL);
 
     pcmk_api = init_with;
 
 #ifdef AIS_WHITETANK 
     log_init ("crm");
 #endif
 
     pcmk_env.debug    = "0";
     pcmk_env.logfile  = NULL;
     pcmk_env.use_logd = "false";
     pcmk_env.syslog   = "daemon";
 
     if(cs_uid != root_uid) {
 	ais_err("Corosync must be configured to start as 'root',"
 		" otherwise Pacemaker cannot manage services."
 		"  Expected %d got %d", root_uid, cs_uid);
 	return -1;
     }    
     
     process_ais_conf();
     
     membership_list = g_hash_table_new_full(
 	g_direct_hash, g_direct_equal, NULL, destroy_ais_node);
     membership_notify_list = g_hash_table_new(g_direct_hash, g_direct_equal);
     ipc_client_list = g_hash_table_new(g_direct_hash, g_direct_equal);
     
     ais_info("CRM: Initialized");
     log_printf(LOG_INFO, "Logging: Initialized %s\n", __PRETTY_FUNCTION__);
     
     rc = getrlimit(RLIMIT_CORE, &cores);
     if(rc < 0) {
 	ais_perror("Cannot determine current maximum core size.");
     }
     
     if(cores.rlim_max <= 0) {
 	cores.rlim_max = RLIM_INFINITY;
 	
 	rc = setrlimit(RLIMIT_CORE, &cores);
 	if(rc < 0) {
 	    ais_perror("Core file generation will remain disabled."
 		       " Core files are an important diagnositic tool,"
 		       " please consider enabling them by default.");
 	}
 	
     } else {
 	ais_info("Maximum core file size is: %lu", cores.rlim_max);
 #if 0
 	/* system() is not thread-safe, can't call from here
 	 * Actually, its a pretty hacky way to try and achieve this anyway
 	 */
 	if(system("echo 1 > /proc/sys/kernel/core_uses_pid") != 0) {
 	    ais_perror("Could not enable /proc/sys/kernel/core_uses_pid");
 	}
 #endif
     }
 
     if(pcmk_user_lookup(CRM_DAEMON_USER, &pcmk_uid, &pcmk_gid) < 0) {
 	ais_err("Cluster user %s does not exist, aborting Pacemaker startup", CRM_DAEMON_USER);
 	return TRUE;
     }
     
     mkdir(CRM_STATE_DIR, 0750);
     chown(CRM_STATE_DIR, pcmk_uid, pcmk_gid);
     
     /* Used by stonithd */
     build_path(HA_STATE_DIR"/heartbeat", 0755); 
 
     /* Used by RAs - Leave owned by root */
     build_path(CRM_RSCTMP_DIR, 0755); 
 
     rc = uname(&us);
     AIS_ASSERT(rc == 0);
     local_uname = ais_strdup(us.nodename);
     local_uname_len = strlen(local_uname);	
     
     ais_info("Service: %d", PCMK_SERVICE_ID);
     ais_info("Local hostname: %s", local_uname);
     pcmk_update_nodeid();    
     
     pthread_create (&pcmk_wait_thread, NULL, pcmk_wait_dispatch, NULL);
     for (start_seq = 1; start_seq < max; start_seq++) {
 	/* dont start anything with start_seq < 1 */
 	for (lpc = 0; lpc < max; lpc++) {
 	    if(start_seq == pcmk_children[lpc].start_seq) {
 		spawn_child(&(pcmk_children[lpc]));
 	    }
 	}
     }
-    
     return 0;
 }
 
 /*
   static void ais_print_node(const char *prefix, struct totem_ip_address *host) 
   {
   int len = 0;
   char *buffer = NULL;
 
   ais_malloc0(buffer, INET6_ADDRSTRLEN+1);
 	
   inet_ntop(host->family, host->addr, buffer, INET6_ADDRSTRLEN);
 
   len = strlen(buffer);
   ais_info("%s: %.*s", prefix, len, buffer);
   ais_free(buffer);
   }
 */
 
 #if 0
 /* copied here for reference from exec/totempg.c */
 char *totempg_ifaces_print (unsigned int nodeid)
 {
     static char iface_string[256 * INTERFACE_MAX];
     char one_iface[64];
     struct totem_ip_address interfaces[INTERFACE_MAX];
     char **status;
     unsigned int iface_count;
     unsigned int i;
     int res;
 
     iface_string[0] = '\0';
 
     res = totempg_ifaces_get (nodeid, interfaces, &status, &iface_count);
     if (res == -1) {
 	return ("no interface found for nodeid");
     }
 
     for (i = 0; i < iface_count; i++) {
 	sprintf (one_iface, "r(%d) ip(%s), ",
 		 i, totemip_print (&interfaces[i]));
 	strcat (iface_string, one_iface);
     }
     return (iface_string);
 }
 #endif
 
 static void ais_mark_unseen_peer_dead(
     gpointer key, gpointer value, gpointer user_data)
 {
     int *changed = user_data;
     crm_node_t *node = value;
     if(node->last_seen != membership_seq
        && ais_str_eq(CRM_NODE_LOST, node->state) == FALSE) {
 	ais_info("Node %s was not seen in the previous transition", node->uname);
 	*changed += update_member(node->id, 0, membership_seq, node->votes,
 				  node->processes, node->uname, CRM_NODE_LOST, NULL);
     }
 }
 
 void pcmk_peer_update (
     enum totem_configuration_type configuration_type,
 #ifdef AIS_COROSYNC 
     const unsigned int *member_list, size_t member_list_entries,
     const unsigned int *left_list, size_t left_list_entries,
     const unsigned int *joined_list, size_t joined_list_entries,
     const struct memb_ring_id *ring_id
 #else
     unsigned int *member_list, int member_list_entries,
     unsigned int *left_list, int left_list_entries,
     unsigned int *joined_list, int joined_list_entries,
     struct memb_ring_id *ring_id
 #endif
     )
 {
     int lpc = 0;
     int changed = 0;
     int do_update = 0;
     
     AIS_ASSERT(ring_id != NULL);
     switch(configuration_type) {
 	case TOTEM_CONFIGURATION_REGULAR:
 	    do_update = 1;
 	    break;
 	case TOTEM_CONFIGURATION_TRANSITIONAL:
 	    break;
     }
 
     membership_seq = ring_id->seq;
     ais_notice("%s membership event on ring %lld: memb=%ld, new=%ld, lost=%ld",
 	       do_update?"Stable":"Transitional", ring_id->seq,
 	       (long)member_list_entries, (long)joined_list_entries, (long)left_list_entries);
 
     if(do_update == 0) {
 	for(lpc = 0; lpc < joined_list_entries; lpc++) {
 	    const char *prefix = "new: ";
 	    uint32_t nodeid = joined_list[lpc];
 	    ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid);
 	}
 	for(lpc = 0; lpc < member_list_entries; lpc++) {
 	    const char *prefix = "memb:";
 	    uint32_t nodeid = member_list[lpc];
 	    ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid);
 	}
 	for(lpc = 0; lpc < left_list_entries; lpc++) {
 	    const char *prefix = "lost:";
 	    uint32_t nodeid = left_list[lpc];
 	    ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid);
 	}
 	return;
     }
     
     for(lpc = 0; lpc < joined_list_entries; lpc++) {
 	const char *prefix = "NEW: ";
 	uint32_t nodeid = joined_list[lpc];
 	crm_node_t *node = NULL;
 	changed += update_member(
 	    nodeid, 0, membership_seq, -1, 0, NULL, CRM_NODE_MEMBER, NULL);
 
 	ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid);
 
 	node = g_hash_table_lookup(membership_list, GUINT_TO_POINTER(nodeid));	
 	if(node->addr == NULL) {
 	    const char *addr = totempg_ifaces_print(nodeid);
 	    node->addr = ais_strdup(addr);
 	    ais_debug("Node %u has address %s", nodeid, node->addr);	    
 	}
     }
 
     for(lpc = 0; lpc < member_list_entries; lpc++) {
 	const char *prefix = "MEMB:";
 	uint32_t nodeid = member_list[lpc];
 	changed += update_member(
 	    nodeid, 0, membership_seq, -1, 0, NULL, CRM_NODE_MEMBER, NULL);
 
 	ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid);
     }
     
     for(lpc = 0; lpc < left_list_entries; lpc++) {
 	const char *prefix = "LOST:";
 	uint32_t nodeid = left_list[lpc];
 	changed += update_member(
 	    nodeid, 0, membership_seq, -1, 0, NULL, CRM_NODE_LOST, NULL);
 	ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid);
     }    
 
     if(changed && joined_list_entries == 0 && left_list_entries == 0) {
 	ais_err("Something strange happened: %d", changed);
 	changed = 0;
     }
     
     ais_debug_2("Reaping unseen nodes...");
     g_hash_table_foreach(membership_list, ais_mark_unseen_peer_dead, &changed);
 
     if(member_list_entries > 1) {
 	/* Used to set born-on in send_cluster_id())
 	 * We need to wait until we have at least one peer since first
 	 * membership id is based on the one before we stopped and isn't reliable
 	 */
 	have_reliable_membership_id = TRUE;
     }
     
     if(changed) {
 	ais_debug("%d nodes changed", changed);
 	pcmk_update_nodeid();
 	send_member_notification();
     }
     
     send_cluster_id();
 }
 
 int pcmk_ipc_exit (void *conn)
 {
     int lpc = 0;
     const char *client = NULL;
     void *async_conn = conn;
     
     for (; lpc < SIZEOF(pcmk_children); lpc++) {
 	if(pcmk_children[lpc].conn == conn) {
 	    if(wait_active == FALSE) {
 		/* Make sure the shutdown loop exits */
 		pcmk_children[lpc].pid = 0;
 	    }
 	    pcmk_children[lpc].conn = NULL;
 	    pcmk_children[lpc].async_conn = NULL;
 	    client = pcmk_children[lpc].name;
 	    break;
 	}
     }
 
     g_hash_table_remove(membership_notify_list, async_conn);
     g_hash_table_remove(ipc_client_list, async_conn);
 
     do_ais_log(client?LOG_INFO:(LOG_DEBUG+1), "Client %s (conn=%p, async-conn=%p) left",
 	     client?client:"unknown-transient", conn, async_conn);
 
     return (0);
 }
 
 int pcmk_ipc_connect (void *conn)
 {
     /* OpenAIS hasn't finished setting up the connection at this point
      * Sending messages now messes up the protocol!
      */
     return (0);
 }
 
 /*
  * Executive message handlers
  */
 void pcmk_cluster_swab(void *msg)
 {
     AIS_Message *ais_msg = msg;
 
     ais_debug_3("Performing endian conversion...");
     ais_msg->id                = swab32 (ais_msg->id);
     ais_msg->size              = swab32 (ais_msg->size);
     ais_msg->is_compressed     = swab32 (ais_msg->is_compressed);
     ais_msg->compressed_size   = swab32 (ais_msg->compressed_size);
     
     ais_msg->host.id      = swab32 (ais_msg->host.id);
     ais_msg->host.pid     = swab32 (ais_msg->host.pid);
     ais_msg->host.type    = swab32 (ais_msg->host.type);
     ais_msg->host.size    = swab32 (ais_msg->host.size);
     ais_msg->host.local   = swab32 (ais_msg->host.local);
     
     ais_msg->sender.id    = swab32 (ais_msg->sender.id);
     ais_msg->sender.pid   = swab32 (ais_msg->sender.pid);
     ais_msg->sender.type  = swab32 (ais_msg->sender.type);
     ais_msg->sender.size  = swab32 (ais_msg->sender.size);
     ais_msg->sender.local = swab32 (ais_msg->sender.local);
 
     ais_msg->header.size   = swab32 (ais_msg->header.size);
     ais_msg->header.id     = swab32 (ais_msg->header.id);
     ais_msg->header.error  = swab32 (ais_msg->header.error);
 }
 
 void pcmk_cluster_callback (
     ais_void_ptr *message, unsigned int nodeid)
 {
     const AIS_Message *ais_msg = message;
 
     ais_debug_2("Message from node %u (%s)",
 		nodeid, nodeid==local_nodeid?"local":"remote");
 /*  Shouldn't be required...
     update_member(
  	ais_msg->sender.id, membership_seq, -1, 0, ais_msg->sender.uname, NULL);
 */
 
     if(ais_msg->host.size == 0
        || ais_str_eq(ais_msg->host.uname, local_uname)) {
 	route_ais_message(ais_msg, FALSE);
 
     } else {
 	ais_debug_3("Discarding Msg[%d] (dest=%s:%s, from=%s:%s)",
 		    ais_msg->id, ais_dest(&(ais_msg->host)),
 		    msg_type2text(ais_msg->host.type),
 		    ais_dest(&(ais_msg->sender)),
 		    msg_type2text(ais_msg->sender.type));
     }
 }
 
 void pcmk_cluster_id_swab(void *msg)
 {
     struct crm_identify_msg_s *ais_msg = msg;
 
     ais_debug_3("Performing endian conversion...");
     ais_msg->id        = swab32 (ais_msg->id);
     ais_msg->pid       = swab32 (ais_msg->pid);
     ais_msg->votes     = swab32 (ais_msg->votes);
     ais_msg->processes = swab32 (ais_msg->processes);
     ais_msg->born_on   = swab64 (ais_msg->born_on);
 
     ais_msg->header.size   = swab32 (ais_msg->header.size);
     ais_msg->header.id     = swab32 (ais_msg->header.id);
 }
 
 void pcmk_cluster_id_callback (ais_void_ptr *message, unsigned int nodeid)
 {
     int changed = 0;
     const struct crm_identify_msg_s *msg = message;
     if(nodeid != msg->id) {
 	ais_err("Invalid message: Node %u claimed to be node %d",
 		nodeid, msg->id);
 	return;
     }
     ais_debug("Node update: %s (%s)", msg->uname, msg->version);
     changed = update_member(
 	nodeid, msg->born_on, membership_seq, msg->votes, msg->processes, msg->uname, NULL, msg->version);
 
     if(changed) {
 	send_member_notification();
     }
 }
 
 struct res_overlay {
 	coroipc_response_header_t header __attribute((aligned(8)));
 	char buf[4096];
 };
 
 struct res_overlay *res_overlay = NULL;
 
 static void send_ipc_ack(void *conn)
 {
     if(res_overlay == NULL) {
 	ais_malloc0(res_overlay, sizeof(struct res_overlay));
     }
     
     res_overlay->header.id = CRM_MESSAGE_IPC_ACK;
     res_overlay->header.size = sizeof (coroipc_response_header_t);
     res_overlay->header.error = CS_OK;
 #ifdef AIS_COROSYNC
     pcmk_api->ipc_response_send (conn, res_overlay, res_overlay->header.size);
 #else
     openais_response_send (conn, res_overlay, res_overlay->header.size);
 #endif
 }
 
 
 /* local callbacks */
 void pcmk_ipc(void *conn, ais_void_ptr *msg)
 {
     AIS_Message *mutable;
     int type = 0, size = 0;
     gboolean transient = TRUE;
     const AIS_Message *ais_msg = (const AIS_Message*)msg;
     void *async_conn = conn;
     ais_debug_2("Message from client %p", conn);
 
     if(check_message_sanity(msg, ((const AIS_Message*)msg)->data) == FALSE) {
 	/* The message is corrupted - ignore */
 	send_ipc_ack(conn);  msg = NULL; 
 	return;
     }
 
     /* Make a copy of the message here and ACK it
      * The message is only valid until a response is sent
      * but the response must also be sent _before_ we send anything else
      */
 
     mutable = ais_msg_copy(ais_msg);
     AIS_ASSERT(check_message_sanity(mutable, mutable->data));
     
     size = mutable->header.size;
     /* ais_malloc0(ais_msg, size); */
     /* memcpy(ais_msg, msg, size); */    
     
     type = mutable->sender.type;
     ais_debug_3("type: %d local: %d conn: %p host type: %d ais: %d sender pid: %d child pid: %d size: %d",
 		type, mutable->host.local, pcmk_children[type].conn, mutable->host.type, crm_msg_ais,
 		mutable->sender.pid, pcmk_children[type].pid, ((int)SIZEOF(pcmk_children)));
     
     if(type > crm_msg_none && type < SIZEOF(pcmk_children)) {
 	/* known child process */
 	transient = FALSE;
     }
     
     /* If this check fails, the order of pcmk_children probably 
      *   doesn't match that of the crm_ais_msg_types enum
      */
     AIS_CHECK(transient || mutable->sender.pid == pcmk_children[type].pid,
 	      ais_err("Sender: %d, child[%d]: %d", mutable->sender.pid, type, pcmk_children[type].pid);
 	      return);
     
     if(transient == FALSE
        && type > crm_msg_none
        && mutable->host.local
        && pcmk_children[type].conn == NULL
        && mutable->host.type == crm_msg_ais) {
 	AIS_CHECK(mutable->sender.type != mutable->sender.pid,
 		  ais_err("Pid=%d, type=%d", mutable->sender.pid, mutable->sender.type));
 	
 	ais_info("Recorded connection %p for %s/%d",
 		 conn, pcmk_children[type].name, pcmk_children[type].pid);
 	pcmk_children[type].conn = conn;
 	pcmk_children[type].async_conn = async_conn;
 
 	/* Make sure they have the latest membership */
 	if(pcmk_children[type].flags & crm_flag_members) {
 	    char *update = pcmk_generate_membership_data();
 	    g_hash_table_replace(membership_notify_list, async_conn, async_conn);
 	    ais_info("Sending membership update "U64T" to %s",
 		     membership_seq, pcmk_children[type].name);
  	    send_client_msg(async_conn, crm_class_members, crm_msg_none,update);
 	}	
 
     } else if(transient) {
 	AIS_CHECK(mutable->sender.type == mutable->sender.pid,
 		  ais_err("Pid=%d, type=%d", mutable->sender.pid, mutable->sender.type));
 	g_hash_table_replace(ipc_client_list, async_conn, GUINT_TO_POINTER(mutable->sender.pid));
     }
 
     mutable->sender.id = local_nodeid;
     mutable->sender.size = local_uname_len;
     memset(mutable->sender.uname, 0, MAX_NAME);
     memcpy(mutable->sender.uname, local_uname, mutable->sender.size);
 
     route_ais_message(mutable, TRUE);
     send_ipc_ack(conn);  msg = NULL;
     ais_free(mutable);
 }
 
 int pcmk_shutdown (
 #ifdef AIS_COROSYNC
     void
 #else
     struct objdb_iface_ver0 *objdb
 #endif
     )
 {
     int lpc = 0;
     static int phase = 0;
     static int max_wait = 0;
     static time_t next_log = 0;
     static int max = SIZEOF(pcmk_children);
     
     if(phase == 0) {
 	ais_notice("Shuting down Pacemaker");    
 	phase = max;
     }
 
     wait_active = FALSE; /* stop the wait loop */
 
     for (; phase > 0; phase--) {
 	/* dont stop anything with start_seq < 1 */
    
 	for (lpc = max - 1; lpc >= 0; lpc--) {
 	    if(phase != pcmk_children[lpc].start_seq) {
 		continue;
 	    }
 		
 #ifdef AIS_WHITETANK
 	  retry:
 #endif
 	    
 	    if(pcmk_children[lpc].pid) {
 		pid_t pid = 0;
 		int status = 0;
 		time_t now = time(NULL);
 
 		if(pcmk_children[lpc].respawn) {
 		    max_wait = 5; /* 5 * 30s = 2.5 minutes... plenty once the crmd is gone */
 		    next_log = now + 30;
 		    pcmk_children[lpc].respawn = FALSE;
 		    stop_child(&(pcmk_children[lpc]), SIGTERM);   
 		}
 
 		pid = wait4(pcmk_children[lpc].pid, &status, WNOHANG, NULL);
 		if(pid < 0) {
 		    ais_perror("Call to wait4(%s/%d) failed - treating it as stopped",
 			       pcmk_children[lpc].name, pcmk_children[lpc].pid);
 		    
 		} else if(pid == 0) {
 		    if(now >= next_log) {
 			max_wait--;
 			next_log = now + 30;
 			ais_notice("Still waiting for %s (pid=%d, seq=%d) to terminate...",
 				   pcmk_children[lpc].name, pcmk_children[lpc].pid, pcmk_children[lpc].start_seq);
 			if(max_wait <= 0 && phase < pcmk_children[crm_msg_crmd].start_seq) {
 			    ais_err("Child %s taking too long to terminate, sending SIGKILL", pcmk_children[lpc].name);
 			    stop_child(&(pcmk_children[lpc]), SIGKILL);
 			}
 		    }		    
 #ifdef AIS_WHITETANK
 		    {
 			struct timespec waitsleep = {
 			    .tv_sec = 1,
 			    .tv_nsec = 0
 			};
 			
 			sched_yield ();
 			nanosleep (&waitsleep, 0);
 			goto retry;
 		    }
 #else
 		    /* Return control to corosync */
 		    return -1;
 #endif
 		}
 	    }
 	    
 	    /* cleanup */
 	    ais_notice("%s confirmed stopped", pcmk_children[lpc].name);
 	    pcmk_children[lpc].async_conn = NULL;
 	    pcmk_children[lpc].conn = NULL;
 	    pcmk_children[lpc].pid = 0;
 	}
     }
     
     send_cluster_id();
     ais_notice("Shutdown complete");
     /* TODO: Add back the logsys flush call once its written */
     
 #ifdef AIS_WHITETANK
     /* Bug bnc#482847, bnc#482905
      *
      * All cluster services are now down, we could allow OpenAIS to continue
      * unloading plugins, but its kinda new at that and there are a bunch of
      * race conditions that get exercised.
      *
      * Take the easy way out for now (on whitetank) and eventually fix for
      * CoroSync which is where everyone wants to be eventually anyway
      */
     ais_notice("Forcing clean exit of OpenAIS");
     exit(0);
 #endif
     return 0;
 }
 
 struct member_loop_data 
 {
 	char *string;
 };
 
 void member_vote_count_fn(gpointer key, gpointer value, gpointer user_data)
 {
     crm_node_t *node = value;
     if (ais_str_eq(CRM_NODE_MEMBER, node->state)) {
         plugin_has_votes += node->votes;
     }
 }
 
 
 void member_loop_fn(gpointer key, gpointer value, gpointer user_data)
 {
     crm_node_t *node = value;
     struct member_loop_data *data = user_data;    
 
     ais_debug_2("Dumping node %u", node->id);
     data->string = append_member(data->string, node);
 }
 
 char *pcmk_generate_membership_data(void)
 {
     int size = 0;
     struct member_loop_data data;
     size = 256; 
     ais_malloc0(data.string, size);
 
     /* Ensure the list of active processes is up-to-date */
-    update_member(local_nodeid, 0, 0, -1, get_process_list(), local_uname, CRM_NODE_MEMBER, NULL);
+    update_member(local_nodeid, 0, 0, -1, get_process_list(), local_uname, NULL, NULL);
     
     plugin_has_votes = 0;
     g_hash_table_foreach(membership_list, member_vote_count_fn, NULL);
     if(plugin_has_votes > plugin_expected_votes) {
 	update_expected_votes(plugin_has_votes);
     }
     
     snprintf(data.string, size,
 	     "<nodes id=\""U64T"\" quorate=\"%s\" expected=\"%u\" actual=\"%u\">",
 	     membership_seq, plugin_has_quorum()?"true":"false",
 	     plugin_expected_votes, plugin_has_votes);
 
     g_hash_table_foreach(membership_list, member_loop_fn, &data);
     size = strlen(data.string);
     data.string = realloc(data.string, size + 9) ;/* 9 = </nodes> + nul */
     sprintf(data.string + size, "</nodes>");
     return data.string;
 }
 
 void pcmk_nodes(void *conn, ais_void_ptr *msg)
 {
     char *data = pcmk_generate_membership_data();
     void *async_conn = conn;
 
     /* send the ACK before we send any other messages
      * - but after we no longer need to access the message
      */
     send_ipc_ack(conn);  msg = NULL;
 
     if(async_conn) {
 	send_client_msg(async_conn, crm_class_members, crm_msg_none, data);
     }
     ais_free(data);
 }
 
 void pcmk_remove_member(void *conn, ais_void_ptr *msg)
 {
     const AIS_Message *ais_msg = msg;
     char *data = get_ais_data(ais_msg);
     send_ipc_ack(conn);  msg = NULL;
     
     if(data != NULL) {
 	char *bcast = ais_concat("remove-peer", data, ':');
 	send_cluster_msg(crm_msg_ais, NULL, bcast);
 	ais_info("Sent: %s", bcast);
 	ais_free(bcast);
     }
     
     ais_free(data);
 }
 
 static void send_quorum_details(void *conn) 
 {
     int size = 256;
     char *data = NULL;
     ais_malloc0(data, size);
     
     snprintf(data, size, "<quorum id=\""U64T"\" quorate=\"%s\" expected=\"%u\" actual=\"%u\"/>",
 	    membership_seq, plugin_has_quorum()?"true":"false",
 	    plugin_expected_votes, plugin_has_votes);
 
     send_client_msg(conn, crm_class_quorum, crm_msg_none, data);
     ais_free(data);
 }
 
 void pcmk_quorum(void *conn, ais_void_ptr *msg)
 {
     char *dummy = NULL;
     const AIS_Message *ais_msg = msg;
     char *data = get_ais_data(ais_msg);
     send_ipc_ack(conn);  msg = NULL;
 
     /* Make sure the current number of votes is accurate */
     dummy = pcmk_generate_membership_data();
     ais_free(dummy);
 
     /* Calls without data just want the current quorum details */
     if(data != NULL && strlen(data) > 0) {
 	int value = ais_get_int(data, NULL);
 	update_expected_votes(value);
     }
 
     send_quorum_details(conn);
     ais_free(data);
 }
 
 void pcmk_notify(void *conn, ais_void_ptr *msg)
 {
     const AIS_Message *ais_msg = msg;
     char *data = get_ais_data(ais_msg);
     void *async_conn = conn;
 
     int enable = 0;
     int sender = ais_msg->sender.pid;
     
     send_ipc_ack(conn);  msg = NULL;
 
     if(ais_str_eq("true", data)) {
 	enable = 1;
     }
     
     ais_info("%s node notifications for child %d (%p)",
 	     enable?"Enabling":"Disabling", sender, async_conn);
     if(enable) {
 	g_hash_table_replace(membership_notify_list, async_conn, async_conn);
     } else {
 	g_hash_table_remove(membership_notify_list, async_conn);
     }
     ais_free(data);
 }
 
 void pcmk_nodeid(void *conn, ais_void_ptr *msg)
 {
     static int counter = 0;
     struct crm_ais_nodeid_resp_s resp;
     ais_debug_2("Sending local nodeid: %d to %p[%d]", local_nodeid, conn, counter);
     
     resp.header.id = crm_class_nodeid;
     resp.header.size = sizeof (struct crm_ais_nodeid_resp_s);
     resp.header.error = CS_OK;
     resp.id = local_nodeid;
     resp.counter = counter++;
     memset(resp.uname, 0, MAX_NAME);
     memcpy(resp.uname, local_uname, local_uname_len);
     memset(resp.cname, 0, MAX_NAME);
     memcpy(resp.cname, local_cname, local_cname_len);
     
 #ifdef AIS_COROSYNC
     pcmk_api->ipc_response_send (conn, &resp, resp.header.size);
 #else
     openais_response_send (conn, &resp, resp.header.size);
 #endif
 }
 
 static gboolean
 ghash_send_update(gpointer key, gpointer value, gpointer data)
 {
     if(send_client_msg(value, crm_class_members, crm_msg_none, data) != 0) {
 	/* remove it */
 	return TRUE;
     }
     return FALSE;
 }
 
 void send_member_notification(void)
 {
     char *update = pcmk_generate_membership_data();
 
     ais_info("Sending membership update "U64T" to %d children",
 	     membership_seq,
 	     g_hash_table_size(membership_notify_list));
 
     g_hash_table_foreach_remove(membership_notify_list, ghash_send_update, update);
     ais_free(update);
 }
 
 gboolean check_message_sanity(const AIS_Message *msg, const char *data) 
 {
     gboolean sane = TRUE;
     gboolean repaired = FALSE;
     int dest = msg->host.type;
     int tmp_size = msg->header.size - sizeof(AIS_Message);
 
     if(sane && msg->header.size == 0) {
 	ais_err("Message with no size");
 	sane = FALSE;
     }
 
     if(sane && msg->header.error != CS_OK) {
 	ais_err("Message header contains an error: %d", msg->header.error);
 	sane = FALSE;
     }
 
     AIS_CHECK(msg->header.size > sizeof(AIS_Message),
 	      ais_err("Message %d size too small: %d < %zu",
 		      msg->header.id, msg->header.size, sizeof(AIS_Message));
 	      return FALSE);
 
     if(sane && ais_data_len(msg) != tmp_size) {
 	ais_warn("Message payload size is incorrect: expected %d, got %d", ais_data_len(msg), tmp_size);
 	sane = TRUE;
     }
 
     if(sane && ais_data_len(msg) == 0) {
 	ais_err("Message with no payload");
 	sane = FALSE;
     }
 
     if(sane && data && msg->is_compressed == FALSE) {
 	int str_size = strlen(data) + 1;
 	if(ais_data_len(msg) != str_size) {
 	    int lpc = 0;
 	    ais_err("Message payload is corrupted: expected %d bytes, got %d",
 		    ais_data_len(msg), str_size);
 	    sane = FALSE;
 	    for(lpc = (str_size - 10); lpc < msg->size; lpc++) {
 		if(lpc < 0) {
 		    lpc = 0;
 		}
 		ais_debug_2("bad_data[%d]: %d / '%c'", lpc, data[lpc], data[lpc]);
 	    }
 	}
     }
     
     if(sane == FALSE) {
 	AIS_CHECK(sane,
 		  ais_err("Invalid message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)",
 			  msg->id, ais_dest(&(msg->host)), msg_type2text(dest),
 			  ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
 			  msg->sender.pid, msg->is_compressed, ais_data_len(msg),
 			  msg->header.size));
 	
     } else if(repaired) {
 	ais_err("Repaired message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)",
 		msg->id, ais_dest(&(msg->host)), msg_type2text(dest),
 		ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
 		msg->sender.pid, msg->is_compressed, ais_data_len(msg),
 		msg->header.size);
     } else {
 	ais_debug_3("Verified message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)",
 		    msg->id, ais_dest(&(msg->host)), msg_type2text(dest),
 		    ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
 		    msg->sender.pid, msg->is_compressed, ais_data_len(msg),
 		    msg->header.size);
     }
     return sane;
 }
 
 static int delivered_transient = 0;
 static void deliver_transient_msg(gpointer key, gpointer value, gpointer user_data)
 {
     int pid = GPOINTER_TO_INT(value); 
     AIS_Message *mutable = user_data;
 
     if(pid == mutable->host.type) {
 	int rc = send_client_ipc(key, mutable);
 	delivered_transient++;
 	
 	ais_info("Sent message to %s.%d (rc=%d)",
 		 ais_dest(&(mutable->host)), pid, rc);
 	if(rc != 0) {
 	    ais_warn("Sending message to %s.%d failed (rc=%d)",
 		     ais_dest(&(mutable->host)), pid, rc);
 	    log_ais_message(LOG_DEBUG, mutable);
 	}
     }
 }
 
 gboolean route_ais_message(const AIS_Message *msg, gboolean local_origin) 
 {
     int rc = 0;
     int dest = msg->host.type;
     const char *reason = "unknown";
     AIS_Message *mutable = ais_msg_copy(msg);
     static int service_id =  SERVICE_ID_MAKE(PCMK_SERVICE_ID, 0);
 
     ais_debug_3("Msg[%d] (dest=%s:%s, from=%s:%s.%d, remote=%s, size=%d)",
 		mutable->id, ais_dest(&(mutable->host)), msg_type2text(dest),
 		ais_dest(&(mutable->sender)), msg_type2text(mutable->sender.type),
 		mutable->sender.pid, local_origin?"false":"true", ais_data_len((mutable)));
 
     if(local_origin == FALSE) {
        if(mutable->host.size == 0
 	  || ais_str_eq(local_uname, mutable->host.uname)) {
 	   mutable->host.local = TRUE;
        }
     }
 
     if(check_message_sanity(mutable, mutable->data) == FALSE) {
 	/* Dont send this message to anyone */
 	rc = 1;
 	goto bail;
     }
     
     if(mutable->host.local) {
 	void *conn = NULL;
 	const char *lookup = NULL;
 
 	if(dest == crm_msg_ais) {
 	    process_ais_message(mutable);
 	    goto bail;
 
 	} else if(dest == crm_msg_lrmd) {
 	    /* lrmd messages are routed via the crm */
 	    dest = crm_msg_crmd;
 
 	} else if(dest == crm_msg_te) {
 	    /* te messages are routed via the crm */
 	    dest = crm_msg_crmd;
 
 	} else if(dest >= SIZEOF(pcmk_children)) {
 	    /* Transient client */	    
 
 	    delivered_transient = 0;
 	    g_hash_table_foreach(ipc_client_list, deliver_transient_msg, mutable);
 	    if(delivered_transient) {
 		ais_debug_2("Sent message to %d transient clients: %d", delivered_transient, dest);
 		goto bail;
 
 	    } else {
 		/* try the crmd */
 		ais_debug_2("Sending message to transient client %d via crmd", dest);
 		dest = crm_msg_crmd;		
 	    }
 	    
 	} else if(dest == 0) {
 	    ais_err("Invalid destination: %d", dest);
 	    log_ais_message(LOG_ERR, mutable);
 	    log_printf(LOG_ERR, "%s", get_ais_data(mutable));
 	    rc = 1;
 	    goto bail;
 	}
 	
 	lookup = msg_type2text(dest);
 	conn = pcmk_children[dest].async_conn;
 
 	/* the cluster fails in weird and wonderfully obscure ways when this is not true */
 	AIS_ASSERT(ais_str_eq(lookup, pcmk_children[dest].name));
 
 	if(mutable->header.id == service_id) {
 	    mutable->header.id = 0; /* reset this back to zero for IPC messages */
 
 	} else if(mutable->header.id != 0) {
 	    ais_err("reset header id back to zero from %d", mutable->header.id);
 	    mutable->header.id = 0; /* reset this back to zero for IPC messages */
 	}
 	
 	reason = "ipc delivery failed";
 	rc = send_client_ipc(conn, mutable);
 
     } else if(local_origin) {
 	/* forward to other hosts */
 	ais_debug_3("Forwarding to cluster");
 	reason = "cluster delivery failed";
 	rc = send_cluster_msg_raw(mutable);    
     }
     
     if(rc != 0) {
 	ais_warn("Sending message to %s.%s failed: %s (rc=%d)",
 		 ais_dest(&(mutable->host)), msg_type2text(dest), reason, rc);
 	log_ais_message(LOG_DEBUG, mutable);
     }
     
   bail:
     ais_free(mutable);
     return rc==0?TRUE:FALSE;
 }
 
 int send_cluster_msg_raw(const AIS_Message *ais_msg) 
 {
     int rc = 0;
     struct iovec iovec;
     static uint32_t msg_id = 0;
     AIS_Message *mutable = ais_msg_copy(ais_msg);
 
     AIS_ASSERT(local_nodeid != 0);
     AIS_ASSERT(ais_msg->header.size == (sizeof(AIS_Message) + ais_data_len(ais_msg)));
 
     if(mutable->id == 0) {
 	msg_id++;
 	AIS_CHECK(msg_id != 0 /* detect wrap-around */,
 		  msg_id++; ais_err("Message ID wrapped around"));
 	mutable->id = msg_id;
     }
     
     mutable->header.error = CS_OK;
     mutable->header.id = SERVICE_ID_MAKE(PCMK_SERVICE_ID, 0);	
 
     mutable->sender.id = local_nodeid;
     mutable->sender.size = local_uname_len;
     memset(mutable->sender.uname, 0, MAX_NAME);
     memcpy(mutable->sender.uname, local_uname, mutable->sender.size);
 
     iovec.iov_base = (char *)mutable;
     iovec.iov_len = mutable->header.size;
 
     ais_debug_3("Sending message (size=%u)", (unsigned int)iovec.iov_len);
 #if AIS_COROSYNC
     rc = pcmk_api->totem_mcast(&iovec, 1, TOTEMPG_SAFE);
 #else
     rc = totempg_groups_mcast_joined(openais_group_handle, &iovec, 1, TOTEMPG_SAFE);
 #endif
 
     if(rc == 0 && mutable->is_compressed == FALSE) {
 	ais_debug_2("Message sent: %.80s", mutable->data);
     }
     
     AIS_CHECK(rc == 0, ais_err("Message not sent (%d): %.120s", rc, mutable->data));
 
     ais_free(mutable);
     return rc;	
 }
 
 #define min(x,y) (x)<(y)?(x):(y)
 
 void send_cluster_id(void) 
 {
     int rc = 0;
     int lpc = 0;
     int len = 0;
     time_t now = time(NULL);
     struct iovec iovec;
     struct crm_identify_msg_s *msg = NULL;
 
     static time_t started = 0;
     static uint64_t first_seq = 0;
     
     AIS_ASSERT(local_nodeid != 0);
 
     if(started == 0) {
 	started = now;
 	first_seq = membership_seq;
     }
 
     if(local_born_on == 0) {
 	if(started + 15 < now) {
 	    ais_debug("Born-on set to: "U64T" (age)", first_seq);
 	    local_born_on = first_seq;
 
 	} else if(have_reliable_membership_id) {
 	    ais_debug("Born-on set to: "U64T" (peer)", membership_seq);
 	    local_born_on = membership_seq;
 
 	} else {
 	    ais_debug("Leaving born-on unset: "U64T, membership_seq);
 	}
     }
     
     ais_malloc0(msg, sizeof(struct crm_identify_msg_s));
     msg->header.size = sizeof(struct crm_identify_msg_s);
 
     msg->id = local_nodeid;
     /* msg->header.error = CS_OK; */
     msg->header.id = SERVICE_ID_MAKE(PCMK_SERVICE_ID, 1);	
 
     len = min(local_uname_len, MAX_NAME-1);
     memset(msg->uname, 0, MAX_NAME);
     memcpy(msg->uname, local_uname, len);
 
     len = min(strlen(VERSION), MAX_NAME-1);
     memset(msg->version, 0, MAX_NAME);
     memcpy(msg->version, VERSION, len);
     
     msg->votes = 1;
     msg->pid = getpid();
     msg->processes = get_process_list();
     msg->born_on = local_born_on;
 
     ais_debug("Local update: id=%u, born="U64T", seq="U64T"",
 	      local_nodeid, local_born_on, membership_seq);
     update_member(
 	local_nodeid, local_born_on, membership_seq, msg->votes, msg->processes, NULL, NULL, VERSION);
 
     iovec.iov_base = (char *)msg;
     iovec.iov_len = msg->header.size;
     
 #if AIS_COROSYNC
     rc = pcmk_api->totem_mcast(&iovec, 1, TOTEMPG_SAFE);
 #else
     rc = totempg_groups_mcast_joined(openais_group_handle, &iovec, 1, TOTEMPG_SAFE);
 #endif
 
     AIS_CHECK(rc == 0, ais_err("Message not sent (%d)", rc));
 
     ais_free(msg);
 }
 
 static gboolean
 ghash_send_removal(gpointer key, gpointer value, gpointer data)
 {
     send_quorum_details(value);
     if(send_client_msg(value, crm_class_rmpeer, crm_msg_none, data) != 0) {
 	/* remove it */
 	return TRUE;
     }
     return FALSE;
 }
 
 void ais_remove_peer(char *node_id)
 {
     uint32_t id = ais_get_int(node_id, NULL);    
     crm_node_t *node = g_hash_table_lookup(membership_list, GUINT_TO_POINTER(id));
     if(node == NULL) {
 	ais_info("Peer %u is unknown", id);
 
     } else if(ais_str_eq(CRM_NODE_MEMBER, node->state)) {
 	ais_warn("Peer %u/%s is still active", id, node->uname);
 
     } else if(g_hash_table_remove(membership_list, GUINT_TO_POINTER(id))) {
 	plugin_expected_votes--;
 	ais_notice("Removed dead peer %u from the membership list", id);
 	ais_info("Sending removal of %u to %d children",
 		 id, g_hash_table_size(membership_notify_list));
 	
 	g_hash_table_foreach_remove(membership_notify_list, ghash_send_removal, node_id);
 	
     } else {
 	ais_warn("Peer %u/%s was not removed", id, node->uname);
     }
 }
 
 gboolean process_ais_message(const AIS_Message *msg) 
 {
     int len = ais_data_len(msg);
     char *data = get_ais_data(msg);
     do_ais_log(LOG_DEBUG,
 	       "Msg[%d] (dest=%s:%s, from=%s:%s.%d, remote=%s, size=%d): %.90s",
 	       msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type),
 	       ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
 	       msg->sender.pid,
 	       msg->sender.uname==local_uname?"false":"true",
 	       ais_data_len(msg), data);
 
     if(data && len > 12 && strncmp("remove-peer:", data, 12) == 0) {
 	char *node = data+12;
 	ais_remove_peer(node);
     }
     
     ais_free(data);
     return TRUE;
 }
 
 static void member_dump_fn(gpointer key, gpointer value, gpointer user_data)
 {
     crm_node_t *node = value;
     ais_info(" node id:%u, uname=%s state=%s processes=%.16x born="U64T" seen="U64T" addr=%s version=%s", 
 	     node->id, node->uname?node->uname:"-unknown-",
 	     node->state, node->processes, node->born, node->last_seen,
 	     node->addr?node->addr:"-unknown-", node->version?node->version:"-unknown-");
 }
 
 void pcmk_exec_dump(void) 
 {
     /* Called after SIG_USR2 */
     process_ais_conf();
     ais_info("Local id: %u, uname: %s, born: "U64T, local_nodeid, local_uname, local_born_on);
     ais_info("Membership id: "U64T", quorate: %s, expected: %u, actual: %u",
 	     membership_seq, plugin_has_quorum()?"true":"false",
 	     plugin_expected_votes, plugin_has_votes);
 
     g_hash_table_foreach(membership_list, member_dump_fn, NULL);
 }
diff --git a/lib/ais/utils.c b/lib/ais/utils.c
index fb8a701fa1..b49c75bb76 100644
--- a/lib/ais/utils.c
+++ b/lib/ais/utils.c
@@ -1,710 +1,714 @@
 /*
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 #include <sys/types.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include <fcntl.h>
 #include <errno.h>
 #include <stdio.h>
 #include <signal.h>
 #include <sys/types.h>
 #include <sys/time.h>
 #include <sys/resource.h>
 
 #include <pwd.h>
 #include <glib.h>
 #include <bzlib.h>
 
 #include <crm/ais.h>
 #include "./utils.h"
 #include "./plugin.h"
 
 struct pcmk_env_s pcmk_env;
 
 void log_ais_message(int level, const AIS_Message *msg) 
 {
     char *data = get_ais_data(msg);
     log_printf(level,
 	       "Msg[%d] (dest=%s:%s, from=%s:%s.%d, remote=%s, size=%d): %.90s",
 	       msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type),
 	       ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
 	       msg->sender.pid,
 	       msg->sender.uname==local_uname?"false":"true",
 	       ais_data_len(msg), data);
 /*     do_ais_log(level, */
 /* 	       "Msg[%d] (dest=%s:%s, from=%s:%s.%d, remote=%s, size=%d): %.90s", */
 /* 	       msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type), */
 /* 	       ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), */
 /* 	       msg->sender.pid, */
 /* 	       msg->sender.uname==local_uname?"false":"true", */
 /* 	       ais_data_len(msg), data); */
     ais_free(data);
 }
 /*
 static gboolean ghash_find_by_uname(gpointer key, gpointer value, gpointer user_data) 
 {
     crm_node_t *node = value;
     int id = GPOINTER_TO_INT(user_data);
 
     if (node->id == id) {
 	return TRUE;
     }
     return FALSE;
 }
 */
 
 static int
 ais_string_to_boolean(const char * s)
 {
     int rc = 0;
     if(s == NULL) {
 	return rc;
     }
     
     if(strcasecmp(s, "true") == 0
        || strcasecmp(s, "on") == 0
        || strcasecmp(s, "yes") == 0
        || strcasecmp(s, "y") == 0
        || strcasecmp(s, "1") == 0) {
 	rc = 1;
     }
     return rc;
 }
 
 static char *opts_default[] = { NULL, NULL };
 static char *opts_vgrind[]  = { NULL, NULL, NULL };
 
 gboolean spawn_child(crm_child_t *child)
 {
     int lpc = 0;
     uid_t uid = 0;
     struct rlimit oflimits;
     struct passwd *pwentry = NULL;
     gboolean use_valgrind = FALSE;
     const char *devnull = "/dev/null";
     const char *env_valgrind = getenv("HA_VALGRIND_ENABLED");
     
     if(child->command == NULL) {
 	ais_info("Nothing to do for child \"%s\"", child->name);
 	return TRUE;
     }
     
     if(env_valgrind == NULL) {
 	use_valgrind = FALSE;
 
     } else if(ais_string_to_boolean(env_valgrind)) {
 	use_valgrind = TRUE;
 
     } else if(strstr(env_valgrind, child->name)) {
 	use_valgrind = TRUE;
     }
 
     if(use_valgrind && strlen(VALGRIND_BIN) == 0) {
 	ais_warn("Cannot enable valgrind for %s:"
 		 " The location of the valgrind binary is unknown", child->name);
 	use_valgrind = FALSE;
     }
 
     child->pid = fork();
     AIS_ASSERT(child->pid != -1);
 
     if(child->pid > 0) {
 	/* parent */
 	ais_info("Forked child %d for process %s%s", child->pid, child->name,
 		 use_valgrind?" (valgrind enabled: "VALGRIND_BIN")":"");
 
     } else {
 	/* Setup the two alternate arg arrarys */ 
 	opts_vgrind[0] = ais_strdup(VALGRIND_BIN);
 	opts_vgrind[1] = ais_strdup(child->command);
 	opts_default[0] = opts_vgrind[1];
 	
 #if 0
 	/* Dont set the group for now - it prevents connection to the cluster */
 	if(gid && setgid(gid) < 0) {
 	    ais_perror("Could not set group to %d", gid);
 	}
 #endif
 
 	if(child->uid) {
 	    if(pcmk_user_lookup(child->uid, &uid, NULL) < 0) {
 		ais_err("Invalid uid (%s) specified for %s",
 			child->uid, child->name);
 		return TRUE;
 	    }
 	}
 	
 	if(uid && setuid(uid) < 0) {
 	    ais_perror("Could not set user to %d (%s)", uid, child->uid);
 	}
 	
 	/* Close all open file descriptors */
 	getrlimit(RLIMIT_NOFILE, &oflimits);
 	for (; lpc < oflimits.rlim_cur; lpc++) {
 	    close(lpc);
 	}
 	
 	(void)open(devnull, O_RDONLY);	/* Stdin:  fd 0 */
 	(void)open(devnull, O_WRONLY);	/* Stdout: fd 1 */
 	(void)open(devnull, O_WRONLY);	/* Stderr: fd 2 */
 
 	setenv("HA_COMPRESSION",	"bz2",             1);
 	setenv("HA_cluster_type",	"openais",	   1);
 	setenv("HA_debug",		pcmk_env.debug,    1);
 	setenv("HA_logfacility",	pcmk_env.syslog,   1);
 	setenv("HA_LOGFACILITY",	pcmk_env.syslog,   1);
 	setenv("HA_use_logd",		pcmk_env.use_logd, 1);
 	if(pcmk_env.logfile) {
 	    setenv("HA_debugfile", pcmk_env.logfile, 1);
 	}
 	
 	if(use_valgrind) {
 	    (void)execvp(VALGRIND_BIN, opts_vgrind);
 	} else {
 	    (void)execvp(child->command, opts_default);
 	}
 	ais_perror("FATAL: Cannot exec %s", child->command);
 	exit(100);
     }
     return TRUE; /* never reached */
 }
 
 gboolean
 stop_child(crm_child_t *child, int signal)
 {
     if(signal == 0) {
 	signal = SIGTERM;
     }
 
     if(child->command == NULL) {
 	ais_info("Nothing to do for child \"%s\"", child->name);
 	return TRUE;
     }
     
     ais_debug("Stopping CRM child \"%s\"", child->name);
     
     if (child->pid <= 0) {
 	ais_debug_2("Client %s not running", child->name);
 	return TRUE;
     }
     
     errno = 0;
     if(kill(child->pid, signal) == 0) {
 	ais_notice("Sent -%d to %s: [%d]", signal, child->name, child->pid);
 	
     } else {
 	ais_perror("Sent -%d to %s: [%d]", signal, child->name, child->pid);
     }
     
     return TRUE;
 }
 
 void destroy_ais_node(gpointer data) 
 {
     crm_node_t *node = data;
     ais_info("Destroying entry for node %u", node->id);
 
     ais_free(node->addr);
     ais_free(node->uname);
     ais_free(node->state);
     ais_free(node);
 }
 
 int update_member(unsigned int id, uint64_t born, uint64_t seq, int32_t votes,
 		  uint32_t procs, const char *uname, const char *state, const char *version) 
 {
     int changed = 0;
     crm_node_t *node = NULL;
     
     node = g_hash_table_lookup(membership_list, GUINT_TO_POINTER(id));	
 
     if(node == NULL) {	
 	ais_malloc0(node, sizeof(crm_node_t));
 	ais_info("Creating entry for node %u born on "U64T"", id, seq);
 	node->id = id;
 	node->addr = NULL;
 	node->state = ais_strdup("unknown");
 	
 	g_hash_table_insert(membership_list, GUINT_TO_POINTER(id), node);
 	node = g_hash_table_lookup(membership_list, GUINT_TO_POINTER(id));
     }
 
     if(seq != 0) {
 	node->last_seen = seq;
     }
 
     if(born != 0 && node->born != born) {
 	changed = TRUE;
 	node->born = born;
 	ais_info("%p Node %u (%s) born on: "U64T, node, id, uname, born);
     }
 
     if(version != NULL) {
 	ais_free(node->version);
 	node->version = ais_strdup(version);
     }
     
     if(uname != NULL) {
 	if(node->uname == NULL || ais_str_eq(node->uname, uname) == FALSE) {
 	    ais_info("%p Node %u now known as %s (was: %s)",
 		     node, id, uname, node->uname);
 	    ais_free(node->uname);
 	    node->uname = ais_strdup(uname);
 	    changed = TRUE;
 	}
     }
 
     if(procs != 0 && procs != node->processes) {
 	ais_info("Node %s now has process list: %.32x (%u)",
 		 node->uname, procs, procs);
 	node->processes = procs;
 	changed = TRUE;
     }
 
     if(votes >= 0 && votes != node->votes) {
 	ais_info("Node %s now has %d quorum votes (was %d)",
 		 node->uname, votes, node->votes);
 	node->votes = votes;
 	changed = TRUE;
     }
     
     if(state != NULL) {
 	if(node->state == NULL || ais_str_eq(node->state, state) == FALSE) {
 	    ais_free(node->state);
 	    node->state = ais_strdup(state);
 	    ais_info("Node %u/%s is now: %s",
 		     id, node->uname?node->uname:"unknown", state);
 	    changed = TRUE;
 	}
     }
     
     AIS_ASSERT(node != NULL);
     return changed;
 }
 
 void delete_member(uint32_t id, const char *uname) 
 {
     if(uname == NULL) {
 	g_hash_table_remove(membership_list, GUINT_TO_POINTER(id));
 	return;
     }
     ais_err("Deleting by uname is not yet supported");
 }
 
 const char *member_uname(uint32_t id) 
 {
      crm_node_t *node = g_hash_table_lookup(
 	 membership_list, GUINT_TO_POINTER(id));	
      if(node == NULL) {
 	 return ".unknown.";
      }
      if(node->uname == NULL) {
 	 return ".pending.";
      }
      return node->uname;
 }
 
 char *append_member(char *data, crm_node_t *node)
 {
     int size = 1; /* nul */
     int offset = 0;
     static int fixed_len = 4 + 8 + 7 + 6 + 6 + 7 + 11;
 
     if(data) {
 	size = strlen(data);
     }
     offset = size;
 
     size += fixed_len;
     size += 32; /* node->id */
     size += 100; /* node->seq, node->born */
     size += strlen(node->state);
     if(node->uname) {
 	size += (7 + strlen(node->uname));
     }
     if(node->addr) {
 	size += (6 + strlen(node->addr));
     }
     if(node->version) {
 	size += (9 + strlen(node->version));
     }
     data = realloc(data, size);
 
     offset += snprintf(data + offset, size - offset, "<node id=\"%u\" ", node->id);
     if(node->uname) {
 	offset += snprintf(data + offset, size - offset, "uname=\"%s\" ", node->uname);
     }
     offset += snprintf(data + offset, size - offset, "state=\"%s\" ", node->state);
     offset += snprintf(data + offset, size - offset, "born=\""U64T"\" ", node->born);
     offset += snprintf(data + offset, size - offset, "seen=\""U64T"\" ", node->last_seen);
     offset += snprintf(data + offset, size - offset, "votes=\"%d\" ", node->votes);
     offset += snprintf(data + offset, size - offset, "processes=\"%u\" ", node->processes);
     if(node->addr) {
 	offset += snprintf(data + offset, size - offset, "addr=\"%s\" ", node->addr);
     }
     if(node->version) {
 	offset += snprintf(data + offset, size - offset, "version=\"%s\" ", node->version);
     }
     offset += snprintf(data + offset, size - offset, "/>");
 
     return data;
 }
 
 void swap_sender(AIS_Message *msg) 
 {
     int tmp = 0;
     char tmp_s[256];
     tmp = msg->host.type;
     msg->host.type = msg->sender.type;
     msg->sender.type = tmp;
 
     tmp = msg->host.type;
     msg->host.size = msg->sender.type;
     msg->sender.type = tmp;
 
     memcpy(tmp_s, msg->host.uname, 256);
     memcpy(msg->host.uname, msg->sender.uname, 256);
     memcpy(msg->sender.uname, tmp_s, 256);
 }
 
 char *get_ais_data(const AIS_Message *msg)
 {
     int rc = BZ_OK;
     char *uncompressed = NULL;
     unsigned int new_size = msg->size + 1;
     
     if(msg->is_compressed == FALSE) {
 	uncompressed = strdup(msg->data);
 
     } else {
 	ais_malloc0(uncompressed, new_size);
 	
 	rc = BZ2_bzBuffToBuffDecompress(
 	    uncompressed, &new_size, (char*)msg->data, msg->compressed_size, 1, 0);
 	if(rc != BZ_OK) {
 	    ais_info("rc=%d, new=%u expected=%u", rc, new_size, msg->size);
 	}
 	AIS_ASSERT(rc == BZ_OK);
 	AIS_ASSERT(new_size == msg->size);
     }
     
     return uncompressed;
 }
 
 int send_cluster_msg(
     enum crm_ais_msg_types type, const char *host, const char *data) 
 {
     int rc = 0;
     int data_len = 0;
     AIS_Message *ais_msg = NULL;
     int total_size = sizeof(AIS_Message);
 
     AIS_ASSERT(local_nodeid != 0);
 
     if(data != NULL) {
 	data_len = 1 + strlen(data);
 	total_size += data_len;
     } 
     ais_malloc0(ais_msg, total_size);
 	
     ais_msg->header.size = total_size;
     ais_msg->header.error = CS_OK;
     ais_msg->header.id = 0;
     
     ais_msg->size = data_len;
-    memcpy(ais_msg->data, data, data_len);
     ais_msg->sender.type = crm_msg_ais;
+    if(data != NULL) {
+	memcpy(ais_msg->data, data, data_len);
+    }
 
     ais_msg->host.type = type;
     ais_msg->host.id = 0;
     if(host) {
 	ais_msg->host.size = strlen(host);
 	memset(ais_msg->host.uname, 0, MAX_NAME);
 	memcpy(ais_msg->host.uname, host, ais_msg->host.size);
 /* 	ais_msg->host.id = nodeid_lookup(host); */
 		
     } else {
 	ais_msg->host.type = type;
 	ais_msg->host.size = 0;
 	memset(ais_msg->host.uname, 0, MAX_NAME);
     }
     
     rc = send_cluster_msg_raw(ais_msg);
     ais_free(ais_msg);
     
     return rc;	
 }
 
 extern struct corosync_api_v1 *pcmk_api;
 
 int send_client_ipc(void *conn, const AIS_Message *ais_msg) 
 {
     int rc = -1;
     if (conn == NULL) {
 	rc = -2;
 	    
     } else if (!libais_connection_active(conn)) {
 	ais_warn("Connection no longer active");
 	rc = -3;
 	    
 /* 	} else if ((queue->size - 1) == queue->used) { */
 /* 	    ais_err("Connection is throttled: %d", queue->size); */
 
     } else {
 #ifdef AIS_WHITETANK
 	rc = openais_dispatch_send (conn, (void*)ais_msg, ais_msg->header.size);
 #endif
 #ifdef AIS_COROSYNC
 	rc = pcmk_api->ipc_dispatch_send (conn, ais_msg, ais_msg->header.size);
 #endif
     }
     return rc;
 }
 
 int send_client_msg(
     void *conn, enum crm_ais_msg_class class, enum crm_ais_msg_types type, const char *data) 
 {
     int rc = 0;
     int data_len = 0;
     int total_size = sizeof(AIS_Message);
     AIS_Message *ais_msg = NULL;
     static int msg_id = 0;
 
     AIS_ASSERT(local_nodeid != 0);
 
     msg_id++;
     AIS_ASSERT(msg_id != 0 /* wrap-around */);
 
     if(data != NULL) {
 	data_len = 1 + strlen(data);
     }
     total_size += data_len;
     
     ais_malloc0(ais_msg, total_size);
 	
     ais_msg->id = msg_id;
     ais_msg->header.id = class;
     ais_msg->header.size = total_size;
     ais_msg->header.error = CS_OK;
     
     ais_msg->size = data_len;
-    memcpy(ais_msg->data, data, data_len);
+    if(data != NULL) {
+	memcpy(ais_msg->data, data, data_len);
+    }
     
     ais_msg->host.size = 0;
     ais_msg->host.type = type;
     memset(ais_msg->host.uname, 0, MAX_NAME);
     ais_msg->host.id = 0;
 
     ais_msg->sender.type = crm_msg_ais;
     ais_msg->sender.size = local_uname_len;
     memset(ais_msg->sender.uname, 0, MAX_NAME);
     memcpy(ais_msg->sender.uname, local_uname, ais_msg->sender.size);
     ais_msg->sender.id = local_nodeid;
 
     rc = send_client_ipc(conn, ais_msg);
 
     if(rc != 0) {
 	ais_warn("Sending message to %s failed: %d", msg_type2text(type), rc);
 	log_ais_message(LOG_DEBUG, ais_msg);
 	return FALSE;
     }
 
     ais_free(ais_msg);
     return rc;    
 }
 
 char *
 ais_concat(const char *prefix, const char *suffix, char join) 
 {
 	int len = 0;
 	char *new_str = NULL;
 	AIS_ASSERT(prefix != NULL);
 	AIS_ASSERT(suffix != NULL);
 	len = strlen(prefix) + strlen(suffix) + 2;
 
 	ais_malloc0(new_str, (len));
 	sprintf(new_str, "%s%c%s", prefix, join, suffix);
 	new_str[len-1] = 0;
 	return new_str;
 }
 
 hdb_handle_t config_find_init(struct corosync_api_v1 *config, char *name) 
 {
     hdb_handle_t local_handle = 0;
 #ifdef AIS_COROSYNC
     config->object_find_create(OBJECT_PARENT_HANDLE, name, strlen(name), &local_handle);
     ais_info("Local handle: %lld for %s", local_handle, name);
 #endif
     
 #ifdef AIS_WHITETANK 
     config->object_find_reset (OBJECT_PARENT_HANDLE);
 #endif
     return local_handle;
 }
 
 hdb_handle_t config_find_next(struct corosync_api_v1 *config, char *name, hdb_handle_t top_handle) 
 {
     int rc = 0;
     hdb_handle_t local_handle = 0;
 
 #ifdef AIS_COROSYNC
     rc = config->object_find_next (top_handle, &local_handle);
 #endif
     
 #ifdef AIS_WHITETANK 
     rc = config->object_find(OBJECT_PARENT_HANDLE, name, strlen (name), &local_handle);
 #endif
 
     if(rc < 0) {
 	ais_info("No additional configuration supplied for: %s", name);
 	local_handle = 0;
     } else {
 	ais_info("Processing additional %s options...", name);
     }
     return local_handle;
 }
 
 void config_find_done(struct corosync_api_v1 *config, hdb_handle_t local_handle) 
 {
 #ifdef AIS_COROSYNC
     config->object_find_destroy (local_handle);
 #endif
 }
 
 int get_config_opt(
     struct corosync_api_v1 *config,
     hdb_handle_t object_service_handle,
     char *key, char **value, const char *fallback)
 {
     char *env_key = NULL;
     *value = NULL;
 
     if(object_service_handle > 0) {
 	config->object_key_get(
 	    object_service_handle, key, strlen(key), (void**)value, NULL);
     }
     
     if (*value) {
 	ais_info("Found '%s' for option: %s", *value, key);
 	return 0;
     }
 
     env_key = ais_concat("HA", key, '_');
     *value = getenv(env_key);
     ais_free(env_key);
 
     if (*value) {
 	ais_info("Found '%s' in ENV for option: %s", *value, key);
 	return 0;
     }
 
     if(fallback) {
 	ais_info("Defaulting to '%s' for option: %s", fallback, key);
 	*value = ais_strdup(fallback);
 
     } else {
 	ais_info("No default for option: %s", key);
     }
     
     return -1;
 }
 
 int
 ais_get_boolean(const char * value)
 {
 	if(value == NULL) {
 		return 0;
 
 	} else if (strcasecmp(value, "true") == 0
 		   ||	strcasecmp(value, "on") == 0
 		   ||	strcasecmp(value, "yes") == 0
 		   ||	strcasecmp(value, "y") == 0
 		   ||	strcasecmp(value, "1") == 0){
 		return 1;
 	}
 	return 0;
 }
 
 long long
 ais_get_int(const char *text, char **end_text)
 {
     long long result = -1;
     char *local_end_text = NULL;
     
     errno = 0;
     
     if(text != NULL) {
 #ifdef ANSI_ONLY
 	if(end_text != NULL) {
 	    result = strtol(text, end_text, 10);
 	} else {
 	    result = strtol(text, &local_end_text, 10);
 	}
 #else
 	if(end_text != NULL) {
 	    result = strtoll(text, end_text, 10);
 	} else {
 	    result = strtoll(text, &local_end_text, 10);
 	}
 #endif
 	
 	if(errno == EINVAL) {
 	    ais_err("Conversion of %s failed", text);
 	    result = -1;
 	    
 	} else if(errno == ERANGE) {
 	    ais_err("Conversion of %s was clipped: %lld", text, result);
 
 	} else if(errno != 0) {
 	    ais_perror("Conversion of %s failed:", text);
 	}
 			
 	if(local_end_text != NULL && local_end_text[0] != '\0') {
 	    ais_err("Characters left over after parsing '%s': '%s'", text, local_end_text);
 	}
     }
     return result;
 }
 
 #define PW_BUFFER_LEN 500
 
 int pcmk_user_lookup(const char *name, uid_t *uid, gid_t *gid)
 {
     int rc = -1;
     char *buffer = NULL;
     struct passwd pwd;
     struct passwd *pwentry = NULL;
 
     ais_malloc0(buffer, PW_BUFFER_LEN);
     getpwnam_r(name, &pwd, buffer, PW_BUFFER_LEN, &pwentry);
     if(pwentry) {
 	rc = 0;
 	if(uid) { *uid = pwentry->pw_uid; }
 	if(gid) { *gid = pwentry->pw_gid; }
 	ais_debug("Cluster user %s has uid=%d gid=%d",
 		  name, pwentry->pw_uid, pwentry->pw_gid);
 
     } else {
 	ais_err("Cluster user %s does not exist", name);
     }
 
     ais_free(buffer);
     return rc;
 }
diff --git a/lib/cib/cib_ops.c b/lib/cib/cib_ops.c
index e6d9f720b6..a6b806e7ad 100644
--- a/lib/cib/cib_ops.c
+++ b/lib/cib/cib_ops.c
@@ -1,1013 +1,1013 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <time.h>
 
 #include <sys/param.h>
 #include <sys/types.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/msg.h>
 #include <crm/common/xml.h>
 
 enum cib_errors 
 cib_process_query(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	xmlNode *obj_root = NULL;
 	enum cib_errors result = cib_ok;
 	
 	crm_debug_2("Processing \"%s\" event for section=%s",
 		  op, crm_str(section));
 
 	if(options & cib_xpath) {
 	    return cib_process_xpath(op, options, section, req, input,
 				     existing_cib, result_cib, answer);
 	}
 
 	CRM_CHECK(*answer == NULL, free_xml(*answer));
 	*answer = NULL;
 	
 	if (safe_str_eq(XML_CIB_TAG_SECTION_ALL, section)) {
 		section = NULL;
 	}
 
 	obj_root = get_object_root(section, existing_cib);
 	
 	if(obj_root == NULL) {
 		result = cib_NOTEXISTS;
 
 	} else {
 		*answer = obj_root;
 	}
 
 	if(result == cib_ok && *answer == NULL) {
 		crm_err("Error creating query response");
 		result = cib_output_data;
 	}
 	
 	return result;
 }
 
 enum cib_errors 
 cib_process_erase(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	enum cib_errors result = cib_ok;
 
 	crm_debug_2("Processing \"%s\" event", op);
 	*answer = NULL;
 	free_xml(*result_cib);
 	*result_cib = createEmptyCib();
 
 	copy_in_properties(*result_cib, existing_cib);	
 	cib_update_counter(*result_cib, XML_ATTR_GENERATION, FALSE);
 	
 	return result;
 }
 
 enum cib_errors 
 cib_process_upgrade(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
     int rc = 0;
     int new_version = 0;
     int current_version = 0;
 
     const char *value = NULL;
 
     *answer = NULL;
     crm_debug_2("Processing \"%s\" event", op);
     
     value = crm_element_value_copy(existing_cib, XML_ATTR_VALIDATION);
     if(value != NULL) {
 	current_version = get_schema_version(value);
     }
 
     rc = update_validation(result_cib, &new_version, TRUE, TRUE);
     if(new_version > current_version) {
 	return cib_ok;
     }
     
     return rc;
 }
 
 enum cib_errors 
 cib_process_bump(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	enum cib_errors result = cib_ok;
 
 	crm_debug_2("Processing \"%s\" event for epoch=%s",
 		  op, crm_str(crm_element_value(existing_cib, XML_ATTR_GENERATION)));
 	
 	*answer = NULL;
 	cib_update_counter(*result_cib, XML_ATTR_GENERATION, FALSE);
 	
 	return result;
 }
 
 
 enum cib_errors 
 cib_update_counter(xmlNode *xml_obj, const char *field, gboolean reset)
 {
 	char *new_value = NULL;
 	char *old_value = NULL;
 	int  int_value  = -1;
 	
 	if(reset == FALSE && crm_element_value(xml_obj, field) != NULL) {
 		old_value = crm_element_value_copy(xml_obj, field);
 	}
 	if(old_value != NULL) {
 		crm_malloc0(new_value, 128);
 		int_value = atoi(old_value);
 		sprintf(new_value, "%d", ++int_value);
 	} else {
 		new_value = crm_strdup("1");
 	}
 
 	crm_debug_4("%s %d(%s)->%s",
 		  field, int_value, crm_str(old_value), crm_str(new_value));
 	crm_xml_add(xml_obj, field, new_value);
 
 	crm_free(new_value);
 	crm_free(old_value);
 
 	return cib_ok;
 }
 
 enum cib_errors 
 cib_process_replace(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	const char *tag = NULL;
 	gboolean verbose       = FALSE;
 	enum cib_errors result = cib_ok;
 	
 	crm_debug_2("Processing \"%s\" event for section=%s",
 		    op, crm_str(section));
 
 	if(options & cib_xpath) {
 	    return cib_process_xpath(op, options, section, req, input,
 				     existing_cib, result_cib, answer);
 	}
 
 	*answer = NULL;
 
 	if (input == NULL) {
 		return cib_NOOBJECT;
 	}
 
 	tag = crm_element_name(input);
 
 	if (options & cib_verbose) {
 		verbose = TRUE;
 	}
 	if(safe_str_eq(XML_CIB_TAG_SECTION_ALL, section)) {
 		section = NULL;
 
 	} else if(safe_str_eq(tag, section)) {
 		section = NULL;
 	}
 	
 	if(safe_str_eq(tag, XML_TAG_CIB)) {
 		int updates = 0;
 		int epoch  = 0;
 		int admin_epoch = 0;
 		
 		int replace_updates = 0;
 		int replace_epoch  = 0;
 		int replace_admin_epoch = 0;
 		const char *reason = NULL;
 		
 		cib_version_details(
 			existing_cib, &admin_epoch, &epoch, &updates);
 		cib_version_details(input, &replace_admin_epoch,
 				    &replace_epoch, &replace_updates);
 
 		if(replace_admin_epoch < admin_epoch) {
 			reason = XML_ATTR_GENERATION_ADMIN;
 
 		} else if(replace_admin_epoch > admin_epoch) {
 			/* no more checks */
 
 		} else if(replace_epoch < epoch) {
 			reason = XML_ATTR_GENERATION;
 
 		} else if(replace_epoch > epoch) {
 			/* no more checks */
 
 		} else if(replace_updates < updates) {
 			reason = XML_ATTR_NUMUPDATES;
 		}
 
 		if(reason != NULL) {
 			crm_warn("Replacement %d.%d.%d not applied to %d.%d.%d:"
 				 " current %s is greater than the replacement",
 				 replace_admin_epoch, replace_epoch,
 				 replace_updates, admin_epoch, epoch, updates,
 				 reason);
 			result = cib_old_data;
 		}
 
 		free_xml(*result_cib);
 		*result_cib = copy_xml(input);
 		
 	} else {
 		xmlNode *obj_root = NULL;
 		gboolean ok = TRUE;
 		obj_root = get_object_root(section, *result_cib);
 		ok = replace_xml_child(NULL, obj_root, input, FALSE);
 		if(ok == FALSE) {
 			crm_debug_2("No matching object to replace");
 			result = cib_NOTEXISTS;
 		}
 	}
 
 	return result;
 }
 
 enum cib_errors 
 cib_process_delete(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	xmlNode *obj_root = NULL;
 	crm_debug_2("Processing \"%s\" event", op);
 
 	if(options & cib_xpath) {
 	    return cib_process_xpath(op, options, section, req, input,
 				     existing_cib, result_cib, answer);
 	}
 
 	if(input == NULL) {
 		crm_err("Cannot perform modification with no data");
 		return cib_NOOBJECT;
 	}
 	
 	obj_root = get_object_root(section, *result_cib);
 	
 	crm_validate_data(input);
 	crm_validate_data(*result_cib);
 
 	if(replace_xml_child(NULL, obj_root, input, TRUE) == FALSE) {
 		crm_debug_2("No matching object to delete");
 	}
 	
 	return cib_ok;
 }
 
 enum cib_errors 
 cib_process_modify(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	xmlNode *obj_root = NULL;
 	crm_debug_2("Processing \"%s\" event", op);
 
 	if(options & cib_xpath) {
 	    return cib_process_xpath(op, options, section, req, input,
 				     existing_cib, result_cib, answer);
 	}
 
 	if(input == NULL) {
 		crm_err("Cannot perform modification with no data");
 		return cib_NOOBJECT;
 	}
 	
 	obj_root = get_object_root(section, *result_cib);
 	
 	crm_validate_data(input);
 	crm_validate_data(*result_cib);
 
 	if(obj_root == NULL) {
 	    xmlNode *tmp_section = NULL;
 	    const char *path = get_object_parent(section);
 	    if(path == NULL) {
 		return cib_bad_section;		
 	    }
 
 	    tmp_section = create_xml_node(NULL, section);
 	    cib_process_xpath(
 		CIB_OP_CREATE, 0, path, NULL, tmp_section, NULL, result_cib, answer);
 	    free_xml(tmp_section);
 	    
 	    obj_root = get_object_root(section, *result_cib);
 	}
 
 	CRM_CHECK(obj_root != NULL, return cib_unknown);
 	
 	if(update_xml_child(obj_root, input) == FALSE) {
 	    if(options & cib_can_create) {
 		add_node_copy(obj_root, input);
 	    } else {
 		return cib_NOTEXISTS;		
 	    }
 	}
 	
 	return cib_ok;
 }
 
 static int
 update_cib_object(xmlNode *parent, xmlNode *update)
 {
 	const char *replace = NULL;
 	const char *object_name = NULL;
 	const char *object_id = NULL;
 	xmlNode *target = NULL;
 	int result = cib_ok;
 
 	CRM_CHECK(update != NULL, return cib_NOOBJECT);
 	CRM_CHECK(parent != NULL, return cib_NOPARENT);
 
 	object_name = crm_element_name(update);
 	CRM_CHECK(object_name != NULL, return cib_NOOBJECT);
 
 	object_id = ID(update);
 	crm_debug_3("Processing: <%s id=%s>",
 		    crm_str(object_name), crm_str(object_id));
 	
 	if(object_id == NULL) {
 		/*  placeholder object */
 		target = find_xml_node(parent, object_name, FALSE);
 
 	} else {
 		target = find_entity(parent, object_name, object_id);
 	}
 
 	if(target == NULL) {
 		target = create_xml_node(parent, object_name);
 	} 
 
 	crm_debug_2("Found node <%s id=%s> to update",
 		    crm_str(object_name), crm_str(object_id));
 	
 	replace = crm_element_value(update, XML_CIB_ATTR_REPLACE);
 	if(replace != NULL) {
 	    xmlNode *remove = NULL;
 	    int last = 0, lpc = 0, len = 0;
 
 	    len = strlen(replace);
 	    while(lpc <= len) {
 		if(replace[lpc] == ',' || replace[lpc] == 0) {
 		    char *replace_item = NULL;
 		    if ( last == lpc ) {
 			/* nothing to do */
 			last = lpc+1;
 			goto incr;
 		    }
 
 		    crm_malloc0(replace_item, lpc - last + 1);
 		    strncpy(replace_item, replace+last, lpc-last);
 		    
 		    remove = find_xml_node(target, replace_item, FALSE);
 		    if(remove != NULL) {
 			crm_debug_3("Replacing node <%s> in <%s>",
 				    replace_item, crm_element_name(target));
 			zap_xml_from_parent(target, remove);
 		    }
 		    crm_free(replace_item);
 		    last = lpc+1;
 		}
 	      incr:
 		lpc++;
 	    }
 	    xml_remove_prop(update, XML_CIB_ATTR_REPLACE);
 	    xml_remove_prop(target, XML_CIB_ATTR_REPLACE);
 	}
 	
 	copy_in_properties(target, update);
 
 	crm_debug_3("Processing children of <%s id=%s>",
 		    crm_str(object_name), crm_str(object_id));
 	
 	xml_child_iter(
 		update, a_child,  
 		int tmp_result = 0;
 		crm_debug_3("Updating child <%s id=%s>",
 			    crm_element_name(a_child), ID(a_child));
 		
 		tmp_result = update_cib_object(target, a_child);
 		
 		/*  only the first error is likely to be interesting */
 		if(tmp_result != cib_ok) {
 			crm_err("Error updating child <%s id=%s>",
 				crm_element_name(a_child), ID(a_child));
 			
 			if(result == cib_ok) {
 				result = tmp_result;
 			}
 		}
 		);
 	
 	crm_debug_3("Finished with <%s id=%s>",
 		  crm_str(object_name), crm_str(object_id));
 
 	return result;
 }
 
 static int
 add_cib_object(xmlNode *parent, xmlNode *new_obj)
 {
 	enum cib_errors result = cib_ok;
 	const char *object_name = NULL;
 	const char *object_id = NULL;
 	xmlNode *equiv_node = NULL;
 	
 	if(new_obj != NULL) {
 		object_name = crm_element_name(new_obj);
 	}
 	object_id = crm_element_value(new_obj, XML_ATTR_ID);
 
 	crm_debug_3("Processing: <%s id=%s>",
 		    crm_str(object_name), crm_str(object_id));
 	
 	if(new_obj == NULL || object_name == NULL) {
 		result = cib_NOOBJECT;
 
 	} else if(parent == NULL) {
 		result = cib_NOPARENT;
 
 	} else if(object_id == NULL) {
 		/*  placeholder object */
 		equiv_node = find_xml_node(parent, object_name, FALSE);
 		
 	} else {
 		equiv_node = find_entity(parent, object_name, object_id);
 	}
 
 	if(result != cib_ok) {
 		; /* do nothing */
 		
 	} else if(equiv_node != NULL) {
 		result = cib_EXISTS;
 
 	} else {
 		result = update_cib_object(parent, new_obj);
 	}
 
 	return result;
 }
 
 
 
 enum cib_errors 
 cib_process_create(
     const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
     xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
     xmlNode *failed = NULL;
     enum cib_errors result = cib_ok;
     xmlNode *update_section = NULL;
 	
     crm_debug_2("Processing \"%s\" event for section=%s", op, crm_str(section));
     if(safe_str_eq(XML_CIB_TAG_SECTION_ALL, section)) {
 	section = NULL;
 
     } else if(safe_str_eq(XML_TAG_CIB, section)) {
 	section = NULL;
 
     } else if(safe_str_eq(crm_element_name(input), XML_TAG_CIB)) {
 	section = NULL;
     }
 
     CRM_CHECK(strcasecmp(CIB_OP_CREATE, op) == 0, return cib_operation);
 	
     if(input == NULL) {
 	crm_err("Cannot perform modification with no data");
 	return cib_NOOBJECT;
     }
 	
     if(section == NULL) {
 	return cib_process_modify(op, options, section, req, input, existing_cib, result_cib, answer);
     }
 
     failed = create_xml_node(NULL, XML_TAG_FAILED);
 
     update_section = get_object_root(section, *result_cib);
     if(safe_str_eq(crm_element_name(input), section)) {
 	xml_child_iter(input, a_child, 
 		       result = add_cib_object(update_section, a_child);
 		       if(update_results(failed, a_child, op, result)) {
 			   break;
 		       }
 	    );
 
     } else {
 	result = add_cib_object(update_section, input);
 	update_results(failed, input, op, result);
     }
 	
     if(xml_has_children(failed)) {
 	CRM_CHECK(result != cib_ok, result = cib_unknown);
     }
 
     if (result != cib_ok) {
 	crm_log_xml_err(failed, "CIB Update failures");
 	*answer = failed;
 
     } else {
 	free_xml(failed);
     }
 
     return result;
 }
 
 enum cib_errors 
 cib_process_diff(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
 	unsigned int log_level = LOG_DEBUG;
 	const char *reason = NULL;
 	gboolean apply_diff = TRUE;
 	enum cib_errors result = cib_ok;
 
 	int this_updates = 0;
 	int this_epoch  = 0;
 	int this_admin_epoch = 0;
 
 	int diff_add_updates = 0;
 	int diff_add_epoch  = 0;
 	int diff_add_admin_epoch = 0;
 
 	int diff_del_updates = 0;
 	int diff_del_epoch  = 0;
 	int diff_del_admin_epoch = 0;
 
 	crm_debug_2("Processing \"%s\" event", op);
 
 	cib_diff_version_details(
 		input,
 		&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates, 
 		&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
 
 	crm_element_value_int(existing_cib, XML_ATTR_GENERATION, &this_epoch);
 	crm_element_value_int(existing_cib, XML_ATTR_NUMUPDATES, &this_updates);
 	crm_element_value_int(existing_cib, XML_ATTR_GENERATION_ADMIN, &this_admin_epoch);
 
 	if(this_epoch < 0) { this_epoch = 0; }
 	if(this_updates < 0) { this_updates = 0; }
 	if(this_admin_epoch < 0) { this_admin_epoch = 0; }
 
 	if(diff_del_admin_epoch == diff_add_admin_epoch
 	   && diff_del_epoch == diff_add_epoch
 	   && diff_del_updates == diff_add_updates) {
 		if(options & cib_force_diff) {
 			apply_diff = FALSE;
 			log_level = LOG_ERR;
 			reason = "+ and - versions in the diff did not change in global update";
 			crm_log_xml_warn(input, "Bad global update");
 			
 		} else if(diff_add_admin_epoch == -1 && diff_add_epoch == -1 && diff_add_updates == -1) {
 			crm_err("Massaging diff versions");
 			diff_add_epoch = this_epoch;
 			diff_add_updates = this_updates + 1;
 			diff_add_admin_epoch = this_admin_epoch;
 			diff_del_epoch = this_epoch;
 			diff_del_updates = this_updates;
 			diff_del_admin_epoch = this_admin_epoch;
 			crm_log_xml_err(input, __FUNCTION__);
 
 		} else {
 			apply_diff = FALSE;
 			log_level = LOG_ERR;
 			reason = "+ and - versions in the diff did not change";
 			log_cib_diff(LOG_ERR, input, __FUNCTION__);
 		}
 	}
 
 	if(apply_diff && diff_del_admin_epoch > this_admin_epoch) {
 		result = cib_diff_resync;
 		apply_diff = FALSE;
 		log_level = LOG_INFO;
 		reason = "current \""XML_ATTR_GENERATION_ADMIN"\" is less than required";
 		
 	} else if(apply_diff && diff_del_admin_epoch < this_admin_epoch) {
 		apply_diff = FALSE;
 		log_level = LOG_WARNING;
 		reason = "current \""XML_ATTR_GENERATION_ADMIN"\" is greater than required";
 
 	} else if(apply_diff && diff_del_epoch > this_epoch) {
 		result = cib_diff_resync;
 		apply_diff = FALSE;
 		log_level = LOG_INFO;
 		reason = "current \""XML_ATTR_GENERATION"\" is less than required";
 		
 	} else if(apply_diff && diff_del_epoch < this_epoch) {
 		apply_diff = FALSE;
 		log_level = LOG_WARNING;
 		reason = "current \""XML_ATTR_GENERATION"\" is greater than required";
 
 	} else if(apply_diff && diff_del_updates > this_updates) {
 		result = cib_diff_resync;
 		apply_diff = FALSE;
 		log_level = LOG_INFO;
 		reason = "current \""XML_ATTR_NUMUPDATES"\" is less than required";
 		
 	} else if(apply_diff && diff_del_updates < this_updates) {
 		apply_diff = FALSE;
 		log_level = LOG_WARNING;
 		reason = "current \""XML_ATTR_NUMUPDATES"\" is greater than required";
 	}
 
 	if(apply_diff) {
 		free_xml(*result_cib);
 		*result_cib = NULL;
 		if(apply_xml_diff(existing_cib, input, result_cib) == FALSE) {
 		    log_level = LOG_NOTICE;
 		    reason = "Failed application of an update diff";
 		    
 		    if(options & cib_force_diff) {
 			result = cib_diff_resync;
 		    }
 		}
 	}
 	
 	if(reason != NULL) {
 		do_crm_log(
 			log_level,
 			"Diff %d.%d.%d -> %d.%d.%d not applied to %d.%d.%d: %s",
 			diff_del_admin_epoch,diff_del_epoch,diff_del_updates,
 			diff_add_admin_epoch,diff_add_epoch,diff_add_updates,
 			this_admin_epoch,this_epoch,this_updates, reason);
 
 		if(result == cib_ok) {
 		    result = cib_diff_failed;
 		}
 		
 	} else if(apply_diff) {
 		crm_debug_2("Diff %d.%d.%d -> %d.%d.%d was applied to %d.%d.%d",
 			    diff_del_admin_epoch,diff_del_epoch,diff_del_updates,
 			    diff_add_admin_epoch,diff_add_epoch,diff_add_updates,
 			    this_admin_epoch,this_epoch,this_updates);
 	    
 	}
 	return result;
 }
 
 gboolean
 apply_cib_diff(xmlNode *old, xmlNode *diff, xmlNode **new)
 {
 	gboolean result = TRUE;
 	const char *value = NULL;
 
 	int this_updates = 0;
 	int this_epoch  = 0;
 	int this_admin_epoch = 0;
 
 	int diff_add_updates = 0;
 	int diff_add_epoch  = 0;
 	int diff_add_admin_epoch = 0;
 
 	int diff_del_updates = 0;
 	int diff_del_epoch  = 0;
 	int diff_del_admin_epoch = 0;
 
 	CRM_CHECK(diff != NULL, return FALSE);
 	CRM_CHECK(old != NULL, return FALSE);
 	
 	value = crm_element_value(old, XML_ATTR_GENERATION_ADMIN);
 	this_admin_epoch = crm_parse_int(value, "0");
 	crm_debug_3("%s=%d (%s)", XML_ATTR_GENERATION_ADMIN,
 		  this_admin_epoch, value);
 	
 	value = crm_element_value(old, XML_ATTR_GENERATION);
 	this_epoch = crm_parse_int(value, "0");
 	crm_debug_3("%s=%d (%s)", XML_ATTR_GENERATION, this_epoch, value);
 	
 	value = crm_element_value(old, XML_ATTR_NUMUPDATES);
 	this_updates = crm_parse_int(value, "0");
 	crm_debug_3("%s=%d (%s)", XML_ATTR_NUMUPDATES, this_updates, value);
 	
 	cib_diff_version_details(
 		diff,
 		&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates, 
 		&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
 
 	value = NULL;
 	if(result && diff_del_admin_epoch != this_admin_epoch) {
 		value = XML_ATTR_GENERATION_ADMIN;
 		result = FALSE;
 		crm_debug_3("%s=%d", value, diff_del_admin_epoch);
 
 	} else if(result && diff_del_epoch != this_epoch) {
 		value = XML_ATTR_GENERATION;
 		result = FALSE;
 		crm_debug_3("%s=%d", value, diff_del_epoch);
 
 	} else if(result && diff_del_updates != this_updates) {
 		value = XML_ATTR_NUMUPDATES;
 		result = FALSE;
 		crm_debug_3("%s=%d", value, diff_del_updates);
 	}
 
 	if(result) {
 		xmlNode *tmp = NULL;
 		xmlNode *diff_copy = copy_xml(diff);
 		
 		tmp = find_xml_node(diff_copy, "diff-removed", TRUE);
 		if(tmp != NULL) {
 			xml_remove_prop(tmp, XML_ATTR_GENERATION_ADMIN);
 			xml_remove_prop(tmp, XML_ATTR_GENERATION);
 			xml_remove_prop(tmp, XML_ATTR_NUMUPDATES);
 		}
 		
 		tmp = find_xml_node(diff_copy, "diff-added", TRUE);
 		if(tmp != NULL) {
 			xml_remove_prop(tmp, XML_ATTR_GENERATION_ADMIN);
 			xml_remove_prop(tmp, XML_ATTR_GENERATION);
 			xml_remove_prop(tmp, XML_ATTR_NUMUPDATES);
 		}
 		
 		result = apply_xml_diff(old, diff_copy, new);
 		free_xml(diff_copy);
 		
 	} else {
 		crm_err("target and diff %s values didnt match", value);
 	}
 	
 	
 	return result;
 }
 
 gboolean
 cib_config_changed(xmlNode *last, xmlNode *next, xmlNode **diff)
 {
     gboolean config_changes = FALSE;
     xmlXPathObject *xpathObj = NULL;
 
     CRM_ASSERT(diff != NULL);
 
     *diff = diff_xml_object(last, next, FALSE);
     if(*diff == NULL) {
 	char *digest_last = calculate_xml_digest(last, FALSE, TRUE);
 	char *digest_next = calculate_xml_digest(next, FALSE, TRUE);
 
 	/* Detect ordering changes - important for groups and resource sets */
 	if(safe_str_neq(digest_last, digest_next)) {
 	    config_changes = TRUE;
 	    crm_info("Detected ordering change: %s vs %s", digest_last, digest_next);
 
 	    /* Create a fake diff so that notifications will be sent */
 	    *diff = create_xml_node(NULL, "diff");
 	    create_xml_node(*diff, "diff-removed");
 	    create_xml_node(*diff, "diff-added");
 	}
 	crm_free(digest_last);
 	crm_free(digest_next);
 	goto done;
     }
     
     xpathObj = xpath_search(*diff, "//"XML_CIB_TAG_CONFIGURATION);
     if(xpathObj && xpathObj->nodesetval->nodeNr > 0) {
 	config_changes = TRUE;
 	goto done;
 
     } else if(xpathObj) {
 	xmlXPathFreeObject(xpathObj);
     }
     
     xpathObj = xpath_search(*diff, "//"XML_TAG_CIB);
     if(xpathObj) {
 	int lpc = 0, max = xpathObj->nodesetval->nodeNr;
 	for(lpc = 0; lpc < max; lpc++) {
 	    xmlNode *top = getXpathResult(xpathObj, lpc);
-	    xml_prop_iter(top, name, value,
+	    xml_prop_name_iter(top, name,
 			  if(crm_str_eq(XML_ATTR_NUMUPDATES, name, TRUE) == FALSE) {
 			      config_changes = TRUE;
 			      goto done;
 			  }
 		);
 	}	    
     }
 
   done:
     if(xpathObj) {
 	xmlXPathFreeObject(xpathObj);
     }
     return config_changes;
 }
 
 xmlNode *
 diff_cib_object(xmlNode *old_cib, xmlNode *new_cib, gboolean suppress)
 {
 	xmlNode *dest = NULL;
 	xmlNode *src = NULL;
 	const char *name = NULL;
 	const char *value = NULL;
 
 	xmlNode *diff = diff_xml_object(old_cib, new_cib, suppress);
 	
 	/* add complete version information */
 	src = old_cib;
 	dest = find_xml_node(diff, "diff-removed", FALSE);
 	if(src != NULL && dest != NULL) {
 		name = XML_ATTR_GENERATION_ADMIN;
 		value = crm_element_value(src, name);
 		if(value == NULL) {
 			value = "0";
 		}
 		crm_xml_add(dest, name, value);
 
 		name = XML_ATTR_GENERATION;
 		value = crm_element_value(src, name);
 		if(value == NULL) {
 			value = "0";
 		}
 		crm_xml_add(dest, name, value);
 
 		name = XML_ATTR_NUMUPDATES;
 		value = crm_element_value(src, name);
 		if(value == NULL) {
 			value = "0";
 		}
 		crm_xml_add(dest, name, value);
 	}
 	
 	src = new_cib;
 	dest = find_xml_node(diff, "diff-added", FALSE);
 	if(src != NULL && dest != NULL) {
 		name = XML_ATTR_GENERATION_ADMIN;
 		value = crm_element_value(src, name);
 		if(value == NULL) {
 			value = "0";
 		}
 		crm_xml_add(dest, name, value);
 
 		name = XML_ATTR_GENERATION;
 		value = crm_element_value(src, name);
 		if(value == NULL) {
 			value = "0";
 		}
 		crm_xml_add(dest, name, value);
 
 		name = XML_ATTR_NUMUPDATES;
 		value = crm_element_value(src, name);
 		if(value == NULL) {
 			value = "0";
 		}
 		crm_xml_add(dest, name, value);
 	}
 	return diff;
 }
 
 enum cib_errors 
 cib_process_xpath(
 	const char *op, int options, const char *section, xmlNode *req, xmlNode *input,
 	xmlNode *existing_cib, xmlNode **result_cib, xmlNode **answer)
 {
     int lpc = 0;
     int max = 0;
     int rc = cib_ok;
     gboolean is_query = safe_str_eq(op, CIB_OP_QUERY);
     
     xmlXPathObjectPtr xpathObj = NULL;
     crm_debug_2("Processing \"%s\" event", op);
 
     if(is_query) {
 	xpathObj = xpath_search(existing_cib, section);
     } else {
 	xpathObj = xpath_search(*result_cib, section);
     }
     
     if(xpathObj != NULL && xpathObj->nodesetval != NULL) {
 	max = xpathObj->nodesetval->nodeNr;
     }
 
     if(max < 1 && safe_str_eq(op, CIB_OP_DELETE)) {
 	crm_debug("%s was already removed", section);
 
     } else if(max < 1) {
 	crm_debug("%s: %s does not exist", op, section);
 	rc = cib_NOTEXISTS;
 
     } else if(is_query) {
 	if(max > 1) {
 	    *answer = create_xml_node(NULL, "xpath-query");
 	}
     }
 
     for(lpc = 0; lpc < max; lpc++) {
 	xmlChar *path = NULL;
 	xmlNode *match = getXpathResult(xpathObj, lpc);
 	if(match == NULL) {
 	    continue;
 	}
 	
 	path = xmlGetNodePath(match);
 	crm_debug("Processing %s op for %s (%s)", op, section, path);
 	free(path);
 	
 	if(safe_str_eq(op, CIB_OP_DELETE)) {
 	    free_xml_from_parent(NULL, match);
 	    break;
 
 	} else if(safe_str_eq(op, CIB_OP_MODIFY)) {
 	    if(update_xml_child(match, input) == FALSE) {
 		rc = cib_NOTEXISTS;		
 	    } else {
 		rc = cib_ok;
 		if((options & cib_multiple) == 0) {
 		    break;
 		}
 	    }
 	    
 	} else if(safe_str_eq(op, CIB_OP_CREATE)) {
 	    add_node_copy(match, input);
 	    break;
 
 	} else if(safe_str_eq(op, CIB_OP_QUERY)) {
 
 	    if(options & cib_no_children) {
 		const char *tag = TYPE(match);
 		*answer = create_xml_node(NULL, tag);
 		copy_in_properties(*answer, match);
 		break;
 		
 	    } else if(*answer) {
 		add_node_copy(*answer, match);
 
 	    } else {
 		*answer = match;
 	    }
 	    
 	} else if(safe_str_eq(op, CIB_OP_REPLACE)) {
 	    xmlNode *parent = match->parent;
 	    free_xml_from_parent(NULL, match);
 	    if(input != NULL) {
 		add_node_copy(parent, input);
 	    }
 	    
 	    if((options & cib_multiple) == 0) {
 		break;
 	    }
 	}
     }
 	
     if(xpathObj) {
 	xmlXPathFreeObject(xpathObj);
     }
 	    
     return rc;
 }
 
 /* remove this function */
 gboolean
 update_results(
 	xmlNode *failed, xmlNode *target, const char* operation, int return_code)
 {
 	xmlNode *xml_node = NULL;
 	gboolean was_error = FALSE;
 	const char *error_msg = NULL;
 	
     
 	if (return_code != cib_ok) {
 		error_msg = cib_error2string(return_code);
 
 		xml_node = create_xml_node(failed, XML_FAIL_TAG_CIB);
 
 		was_error = TRUE;
 
 		add_node_copy(xml_node, target);
 		
 		crm_xml_add(xml_node, XML_FAILCIB_ATTR_ID,      ID(target));
 		crm_xml_add(xml_node, XML_FAILCIB_ATTR_OBJTYPE, TYPE(target));
 		crm_xml_add(xml_node, XML_FAILCIB_ATTR_OP,      operation);
 		crm_xml_add(xml_node, XML_FAILCIB_ATTR_REASON,  error_msg);
 
 		crm_warn("Action %s failed: %s (cde=%d)",
 			  operation, error_msg, return_code);
 	}
 
 	return was_error;
 }
diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c
index 4935d4c2ab..b9fe4be5a3 100644
--- a/lib/cib/cib_utils.c
+++ b/lib/cib/cib_utils.c
@@ -1,894 +1,894 @@
 /*
  * Copyright (c) 2004 International Business Machines
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  * 
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  *
  */
 #include <crm_internal.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdarg.h>
 #include <string.h>
 #include <sys/utsname.h>
 
 #include <glib.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <lib/cib/cib_private.h>
 
 struct config_root_s 
 {
 	const char *name;
 	const char *parent;
 	const char *path;
 };
 
  /*
   * "//crm_config" will also work in place of "/cib/configuration/crm_config"
   * The / prefix means find starting from the root, whereas the // prefix means
   * find anywhere and risks multiple matches
   */
 struct config_root_s known_paths[] = {
     { NULL,			NULL,                 "//cib" },
     { XML_TAG_CIB,		NULL,                 "//cib" },
     { XML_CIB_TAG_STATUS,       "/cib",               "//cib/status" },
     { XML_CIB_TAG_CONFIGURATION,"/cib",               "//cib/configuration" },
     { XML_CIB_TAG_CRMCONFIG,    "/cib/configuration", "//cib/configuration/crm_config" },
     { XML_CIB_TAG_NODES,        "/cib/configuration", "//cib/configuration/nodes" },
     { XML_CIB_TAG_DOMAINS,      "/cib/configuration", "//cib/configuration/domains" },
     { XML_CIB_TAG_RESOURCES,    "/cib/configuration", "//cib/configuration/resources" },
     { XML_CIB_TAG_CONSTRAINTS,  "/cib/configuration", "//cib/configuration/constraints" },
     { XML_CIB_TAG_OPCONFIG,	"/cib/configuration", "//cib/configuration/op_defaults" },
     { XML_CIB_TAG_RSCCONFIG,	"/cib/configuration", "//cib/configuration/rsc_defaults" },
     { XML_CIB_TAG_SECTION_ALL,  NULL,                 "//cib" },
 };
 
 const char *
 cib_error2string(enum cib_errors return_code)
 {
 	const char *error_msg = NULL;
 	switch(return_code) {
 		case cib_bad_permissions:
 			error_msg = "bad permissions for the on-disk configuration. shutdown heartbeat and repair.";
 			break;
 		case cib_bad_digest:
 			error_msg = "the on-disk configuration was manually altered. shutdown heartbeat and repair.";
 			break;
 		case cib_bad_config:
 			error_msg = "the on-disk configuration is not valid";
 			break;
 		case cib_msg_field_add:
 			error_msg = "failed adding field to cib message";
 			break;			
 		case cib_id_check:
 			error_msg = "missing id or id-collision detected";
 			break;			
 		case cib_operation:
 			error_msg = "invalid operation";
 			break;
 		case cib_create_msg:
 			error_msg = "couldnt create cib message";
 			break;
 		case cib_client_gone:
 			error_msg = "client left before we could send reply";
 			break;
 		case cib_not_connected:
 			error_msg = "not connected";
 			break;
 		case cib_not_authorized:
 			error_msg = "not authorized";
 			break;
 		case cib_send_failed:
 			error_msg = "send failed";
 			break;
 		case cib_reply_failed:
 			error_msg = "reply failed";
 			break;
 		case cib_return_code:
 			error_msg = "no return code";
 			break;
 		case cib_output_ptr:
 			error_msg = "nowhere to store output";
 			break;
 		case cib_output_data:
 			error_msg = "corrupt output data";
 			break;
 		case cib_connection:
 			error_msg = "connection failed";
 			break;
 		case cib_callback_register:
 			error_msg = "couldnt register callback channel";
 			break;
 		case cib_authentication:
 			error_msg = "";
 			break;
 		case cib_registration_msg:
 			error_msg = "invalid registration msg";
 			break;
 		case cib_callback_token:
 			error_msg = "callback token not found";
 			break;
 		case cib_missing:
 			error_msg = "cib object missing";
 			break;
 		case cib_variant:
 			error_msg = "unknown/corrupt cib variant";
 			break;
 		case CIBRES_MISSING_ID:
 			error_msg = "The id field is missing";
 			break;
 		case CIBRES_MISSING_TYPE:
 			error_msg = "The type field is missing";
 			break;
 		case CIBRES_MISSING_FIELD:
 			error_msg = "A required field is missing";
 			break;
 		case CIBRES_OBJTYPE_MISMATCH:
 			error_msg = "CIBRES_OBJTYPE_MISMATCH";
 			break;
 		case cib_EXISTS:
 			error_msg = "The object already exists";
 			break;
 		case cib_NOTEXISTS:
 			error_msg = "The object/attribute does not exist";
 			break;
 		case CIBRES_CORRUPT:
 			error_msg = "The CIB is corrupt";
 			break;
 		case cib_NOOBJECT:
 			error_msg = "The update was empty";
 			break;
 		case cib_NOPARENT:
 			error_msg = "The parent object does not exist";
 			break;
 		case cib_NODECOPY:
 			error_msg = "Failed while copying update";
 			break;
 		case CIBRES_OTHER:
 			error_msg = "CIBRES_OTHER";
 			break;
 		case cib_ok:
 			error_msg = "ok";
 			break;
 		case cib_unknown:
 			error_msg = "Unknown error";
 			break;
 		case cib_STALE:
 			error_msg = "Discarded old update";
 			break;
 		case cib_ACTIVATION:
 			error_msg = "Activation Failed";
 			break;
 		case cib_NOSECTION:
 			error_msg = "Required section was missing";
 			break;
 		case cib_NOTSUPPORTED:
 			error_msg = "The action/feature is not supported";
 			break;
 		case cib_not_master:
 			error_msg = "Local service is not the master instance";
 			break;
 		case cib_client_corrupt:
 			error_msg = "Service client not valid";
 			break;
 		case cib_remote_timeout:
 			error_msg = "Remote node did not respond";
 			break;
 		case cib_master_timeout:
 			error_msg = "No master service is currently active";
 			break;
 		case cib_revision_unsupported:
 			error_msg = "The required CIB revision number is not supported";
 			break;
 		case cib_revision_unknown:
 			error_msg = "The CIB revision number could not be determined";
 			break;
 		case cib_missing_data:
 			error_msg = "Required data for this CIB API call not found";
 			break;
 		case cib_no_quorum:
 			error_msg = "Write requires quorum";
 			break;
 		case cib_diff_failed:
 			error_msg = "Application of an update diff failed";
 			break;
 		case cib_diff_resync:
 			error_msg = "Application of an update diff failed, requesting a full refresh";
 			break;
 		case cib_bad_section:
 			error_msg = "Invalid CIB section specified";
 			break;
 		case cib_old_data:
 			error_msg = "Update was older than existing configuration";
 			break;
 		case cib_dtd_validation:
 			error_msg = "Update does not conform to the configured schema/DTD";
 			break;
 		case cib_invalid_argument:
 			error_msg = "Invalid argument";
 			break;
 		case cib_transform_failed:
 			error_msg = "Schema transform failed";
 			break;
 	}
 			
 	if(error_msg == NULL) {
 		crm_err("Unknown CIB Error Code: %d", return_code);
 		error_msg = "<unknown error>";
 	}
 	
 	return error_msg;
 }
 
 int
 cib_section2enum(const char *a_section) 
 {
 	if(a_section == NULL || strcasecmp(a_section, "all") == 0) {
 		return cib_section_all;
 
 	} else if(strcasecmp(a_section, XML_CIB_TAG_NODES) == 0) {
 		return cib_section_nodes;
 
 	} else if(strcasecmp(a_section, XML_CIB_TAG_STATUS) == 0) {
 		return cib_section_status;
 
 	} else if(strcasecmp(a_section, XML_CIB_TAG_CONSTRAINTS) == 0) {
 		return cib_section_constraints;
 		
 	} else if(strcasecmp(a_section, XML_CIB_TAG_RESOURCES) == 0) {
 		return cib_section_resources;
 
 	} else if(strcasecmp(a_section, XML_CIB_TAG_CRMCONFIG) == 0) {
 		return cib_section_crmconfig;
 
 	}
 	crm_err("Unknown CIB section: %s", a_section);
 	return cib_section_none;
 }
 
 
 int
 cib_compare_generation(xmlNode *left, xmlNode *right)
 {
 	int lpc = 0;
 	const char *attributes[] = {
 		XML_ATTR_GENERATION_ADMIN,
 		XML_ATTR_GENERATION,
 		XML_ATTR_NUMUPDATES,
 	};
 
 	crm_log_xml_debug_3(left, "left");
 	crm_log_xml_debug_3(right, "right");
 	
 	for(lpc = 0; lpc < DIMOF(attributes); lpc++) {
 		int int_elem_l = -1;
 		int int_elem_r = -1;
 		const char *elem_r = NULL;
 		const char *elem_l = crm_element_value(left, attributes[lpc]);
 
 		if(right != NULL) {
 			elem_r = crm_element_value(right, attributes[lpc]);
 		}
 	
 		if(elem_l != NULL) { int_elem_l = crm_parse_int(elem_l, NULL); }
 		if(elem_r != NULL) { int_elem_r = crm_parse_int(elem_r, NULL); }
 
 		if(int_elem_l < int_elem_r) {
 			crm_debug_2("%s (%s < %s)", attributes[lpc],
 				    crm_str(elem_l), crm_str(elem_r));
 			return -1;
 			
 		} else if(int_elem_l > int_elem_r) {
 			crm_debug_2("%s (%s > %s)", attributes[lpc],
 				    crm_str(elem_l), crm_str(elem_r));
 			return 1;
 		}
 	}
 	
 	return 0;
 }
 
 xmlNode*
 get_cib_copy(cib_t *cib)
 {
 	xmlNode *xml_cib;
 	int options = cib_scope_local|cib_sync_call;
 	if(cib->cmds->query(cib, NULL, &xml_cib, options) != cib_ok) {
 		crm_err("Couldnt retrieve the CIB");
 		return NULL;
 	} else if(xml_cib == NULL) {
 		crm_err("The CIB result was empty");
 		return NULL;
 	}
 
 	if(safe_str_eq(crm_element_name(xml_cib), XML_TAG_CIB)) {
 		return xml_cib;
 	}
 	free_xml(xml_cib);
 	return NULL;
 }
 
 xmlNode*
 cib_get_generation(cib_t *cib)
 {
 	xmlNode *the_cib = get_cib_copy(cib);
 	xmlNode *generation = create_xml_node(
 		NULL, XML_CIB_TAG_GENERATION_TUPPLE);
 
 	if(the_cib != NULL) {
 		copy_in_properties(generation, the_cib);
 		free_xml(the_cib);
 	}
 	
 	return generation;
 }
 
 
 void
 log_cib_diff(int log_level, xmlNode *diff, const char *function)
 {
 	int add_updates = 0;
 	int add_epoch  = 0;
 	int add_admin_epoch = 0;
 
 	int del_updates = 0;
 	int del_epoch  = 0;
 	int del_admin_epoch = 0;
 
 	if(diff == NULL) {
 		return;
 	}
 	
 	cib_diff_version_details(
 		diff, &add_admin_epoch, &add_epoch, &add_updates, 
 		&del_admin_epoch, &del_epoch, &del_updates);
 
 	if(add_updates != del_updates) {
 		do_crm_log(log_level, "%s: Diff: --- %d.%d.%d", function,
 			   del_admin_epoch, del_epoch, del_updates);
 		do_crm_log(log_level, "%s: Diff: +++ %d.%d.%d", function,
 			   add_admin_epoch, add_epoch, add_updates);
 	} else if(diff != NULL) {
 		do_crm_log(log_level,
 			   "%s: Local-only Change: %d.%d.%d", function,
 			   add_admin_epoch, add_epoch, add_updates);
 	}
 	
 	log_xml_diff(log_level, diff, function);
 }
 
 gboolean
 cib_version_details(
 	xmlNode *cib, int *admin_epoch, int *epoch, int *updates)
 {
 	if(cib == NULL) {
 	    *admin_epoch = -1;
 	    *epoch  = -1;
 	    *updates = -1;
 	    return FALSE;
 		
 	} else {
 	    crm_element_value_int(cib, XML_ATTR_GENERATION, epoch);
 	    crm_element_value_int(cib, XML_ATTR_NUMUPDATES, updates);
 	    crm_element_value_int(cib, XML_ATTR_GENERATION_ADMIN, admin_epoch);
 	}
 	return TRUE;	
 }
 
 gboolean
 cib_diff_version_details(
 	xmlNode *diff, int *admin_epoch, int *epoch, int *updates, 
 	int *_admin_epoch, int *_epoch, int *_updates)
 {
 	xmlNode *tmp = NULL;
 
 	tmp = find_xml_node(diff, "diff-added", FALSE);
 	cib_version_details(tmp, admin_epoch, epoch, updates);
 
 	tmp = find_xml_node(diff, "diff-removed", FALSE);
 	cib_version_details(tmp, _admin_epoch, _epoch, _updates);
 	return TRUE;
 }
 
 /*
  * The caller should never free the return value
  */
 
 const char *get_object_path(const char *object_type)
 {
     int lpc = 0;
     int max = DIMOF(known_paths);
     for(; lpc < max; lpc++) {
 	if((object_type == NULL && known_paths[lpc].name == NULL)
 	   || safe_str_eq(object_type, known_paths[lpc].name)) {
 	    return known_paths[lpc].path;
 	}
     }
     return NULL;
 }
 
 const char *get_object_parent(const char *object_type)
 {
     int lpc = 0;
     int max = DIMOF(known_paths);
     for(; lpc < max; lpc++) {
 	if(safe_str_eq(object_type, known_paths[lpc].name)) {
 	    return known_paths[lpc].parent;
 	}
     }
     return NULL;
 }
 
 xmlNode*
 get_object_root(const char *object_type, xmlNode *the_root)
 {
     const char *xpath = get_object_path(object_type);
 
     if(xpath == NULL) {
 	return the_root; /* or return NULL? */
     }
 
     return get_xpath_object(xpath, the_root, LOG_DEBUG_2);
 }
 
 xmlNode*
 create_cib_fragment_adv(
 	xmlNode *update, const char *update_section, const char *source)
 {
 	xmlNode *cib = NULL;
 	gboolean whole_cib = FALSE;
 	xmlNode *object_root  = NULL;
 	char *local_section = NULL;
 
 /* 	crm_debug("Creating a blank fragment: %s", update_section); */
 	
 	if(update == NULL && update_section == NULL) {
 		crm_debug_3("Creating a blank fragment");
 		update = createEmptyCib();
 		crm_xml_add(cib, XML_ATTR_ORIGIN, source);
 		return update;
 
 	} else if(update == NULL) {
 		crm_err("No update to create a fragment for");
 		return NULL;
 		
 	}
 
 	CRM_CHECK(update_section != NULL, return NULL);
 	if(safe_str_eq(crm_element_name(update), XML_TAG_CIB)) {
 		whole_cib = TRUE;
 	}
 	
 	if(whole_cib == FALSE) {
 		cib = createEmptyCib();
 		crm_xml_add(cib, XML_ATTR_ORIGIN, source);
 		object_root = get_object_root(update_section, cib);
 		add_node_copy(object_root, update);
 
 	} else {
 		cib = copy_xml(update);
 		crm_xml_add(cib, XML_ATTR_ORIGIN, source);
 	}
 
 	crm_free(local_section);
 	crm_debug_3("Verifying created fragment");
 	return cib;
 }
 
 /*
  * It is the callers responsibility to free both the new CIB (output)
  *     and the new CIB (input)
  */
 xmlNode*
 createEmptyCib(void)
 {
-	xmlNode *cib_root = NULL, *config = NULL, *status = NULL;
+	xmlNode *cib_root = NULL, *config = NULL;
 	
 	cib_root = create_xml_node(NULL, XML_TAG_CIB);
 
 	config = create_xml_node(cib_root, XML_CIB_TAG_CONFIGURATION);
-	status = create_xml_node(cib_root, XML_CIB_TAG_STATUS);
+	create_xml_node(cib_root, XML_CIB_TAG_STATUS);
 
 /* 	crm_xml_add(cib_root, "version", "1"); */
 	create_xml_node(config, XML_CIB_TAG_CRMCONFIG);
 	create_xml_node(config, XML_CIB_TAG_NODES);
 	create_xml_node(config, XML_CIB_TAG_RESOURCES);
 	create_xml_node(config, XML_CIB_TAG_CONSTRAINTS);
 	
 	return cib_root;
 }
 
 static unsigned int dtd_throttle = 0;
 
 enum cib_errors
 cib_perform_op(const char *op, int call_options, cib_op_t *fn, gboolean is_query,
 	       const char *section, xmlNode *req, xmlNode *input,
 	       gboolean manage_counters, gboolean *config_changed,
 	       xmlNode *current_cib, xmlNode **result_cib, xmlNode **diff, xmlNode **output)
 {
 
     int rc = cib_ok; 
     gboolean check_dtd = TRUE;
     xmlNode *scratch = NULL;
     xmlNode *local_diff = NULL;
     const char *current_dtd = "unknown";
     
     CRM_CHECK(output != NULL, return cib_output_data);
     CRM_CHECK(result_cib != NULL, return cib_output_data);
     CRM_CHECK(config_changed != NULL, return cib_output_data);
     
     *output = NULL;
     *result_cib = NULL;
     *config_changed = FALSE;
 
     if(fn == NULL) {
 	return cib_operation;
     }
     
     if(is_query) {
 	rc = (*fn)(op, call_options, section, req, input, current_cib, result_cib, output);
 	return rc;
     }
     
     scratch = copy_xml(current_cib);
     rc = (*fn)(op, call_options, section, req, input, current_cib, &scratch, output);    
 
     CRM_CHECK(current_cib != scratch, return cib_unknown);
 
     if(rc == cib_ok && scratch == NULL) {	
 	rc = cib_unknown;
     }
     
     if(rc == cib_ok && current_cib) {
 	int old = 0;
 	int new = 0;
 	crm_element_value_int(scratch, XML_ATTR_GENERATION_ADMIN, &new);
 	crm_element_value_int(current_cib, XML_ATTR_GENERATION_ADMIN, &old);
 	    
 	if(old > new) {
 	    crm_err("%s went backwards: %d -> %d (Opts: 0x%x)",
 		    XML_ATTR_GENERATION_ADMIN, old, new, call_options);
 	    crm_log_xml_warn(req, "Bad Op");
 	    crm_log_xml_warn(input, "Bad Data");
 	    rc = cib_old_data;
 
 	} else if(old == new) {
 	    crm_element_value_int(scratch, XML_ATTR_GENERATION, &new);
 	    crm_element_value_int(current_cib, XML_ATTR_GENERATION, &old);
 	    if(old > new) {
 		crm_err("%s went backwards: %d -> %d (Opts: 0x%x)",
 			XML_ATTR_GENERATION, old, new, call_options);
 		crm_log_xml_warn(req, "Bad Op");
 		crm_log_xml_warn(input, "Bad Data");
 		rc = cib_old_data;
 	    }
 	}
     }
 	 
     if(rc == cib_ok) {
 	fix_plus_plus_recursive(scratch);
 	current_dtd = crm_element_value(scratch, XML_ATTR_VALIDATION);
 	    
 	if(manage_counters) {
 	    *config_changed = cib_config_changed(current_cib, scratch, &local_diff);
 
 	    if(*config_changed) {
 		cib_update_counter(scratch, XML_ATTR_NUMUPDATES, TRUE);
 		cib_update_counter(scratch, XML_ATTR_GENERATION, FALSE);
 
 	    } else if(local_diff != NULL){
 		cib_update_counter(scratch, XML_ATTR_NUMUPDATES, FALSE);
 		if(dtd_throttle++ % 20) {
 		    check_dtd = FALSE; /* Throttle the amount of costly validation we perform due to status updates
 					* a) we don't really care whats in the status section
 					* b) we don't validate any of it's contents at the moment anyway
 					*/
 		}
 	    }
 	}
     }
 
     if(diff != NULL && local_diff != NULL) {
 	/* Only fix the diff if we'll return it... */
 
 	xmlNode *cib = NULL;
 	xmlNode *diff_child = NULL;
 	const char *tag = NULL;
 	const char *value = NULL;
 
 	tag = "diff-removed";
 	diff_child = find_xml_node(local_diff, tag, FALSE);
 	if(diff_child == NULL) {
 	    diff_child = create_xml_node(local_diff, tag);
 	}
 
 	tag = XML_TAG_CIB;
 	cib = find_xml_node(diff_child, tag, FALSE);
 	if(cib == NULL) {
 	    cib = create_xml_node(diff_child, tag);
 	}
 		    
 	tag = XML_ATTR_GENERATION_ADMIN;
 	value = crm_element_value(current_cib, tag);
 	crm_xml_add(diff_child, tag, value);
 	if(*config_changed) {
 	    crm_xml_add(cib, tag, value);		    
 	}
 
 	tag = XML_ATTR_GENERATION;
 	value = crm_element_value(current_cib, tag);
 	crm_xml_add(diff_child, tag, value);
 	if(*config_changed) {
 	    crm_xml_add(cib, tag, value);
 	}
 		    
 	tag = XML_ATTR_NUMUPDATES;
 	value = crm_element_value(current_cib, tag);
 	crm_xml_add(cib, tag, value);
 	crm_xml_add(diff_child, tag, value);
 		    
 	tag = "diff-added";
 	diff_child = find_xml_node(local_diff, tag, FALSE);
 	if(diff_child == NULL) {
 	    diff_child = create_xml_node(local_diff, tag);
 	}
 		    
 	tag = XML_TAG_CIB;
 	cib = find_xml_node(diff_child, tag, FALSE);
 	if(cib == NULL) {
 	    cib = create_xml_node(diff_child, tag);
 	}
 		    
 	tag = XML_ATTR_GENERATION_ADMIN;
 	value = crm_element_value(scratch, tag);
 	crm_xml_add(diff_child, tag, value);
 	if(*config_changed) {
 	    crm_xml_add(cib, tag, value);		    
 	}
 
 	tag = XML_ATTR_GENERATION;
 	value = crm_element_value(scratch, tag);
 	crm_xml_add(diff_child, tag, value);
 	if(*config_changed) {
 	    crm_xml_add(cib, tag, value);
 	}
 
 	tag = XML_ATTR_NUMUPDATES;
 	value = crm_element_value(scratch, tag);
 	crm_xml_add(cib, tag, value);		    
 	crm_xml_add(diff_child, tag, value);
 
 	*diff = local_diff;
 	local_diff = NULL;		    
     }
 
     if(rc == cib_ok
        && check_dtd
        && validate_xml(scratch, NULL, TRUE) == FALSE) {
 	crm_warn("Updated CIB does not validate against %s schema/dtd", crm_str(current_dtd));
 	rc = cib_dtd_validation;
     }
 
     *result_cib = scratch;
     free_xml(local_diff);
     return rc;
 }
 
 int get_channel_token(IPC_Channel *ch, char **token) 
 {
     int rc = cib_ok;
     xmlNode *reg_msg = NULL;
     const char *msg_type = NULL;
     const char *tmp_ticket = NULL;
     
     CRM_CHECK(ch != NULL, return cib_missing);
     CRM_CHECK(token != NULL, return cib_output_ptr);
     
     crm_debug_4("Waiting for msg on command channel");
     
     reg_msg = xmlfromIPC(ch, MAX_IPC_DELAY);
     
     if(ch->ops->get_chan_status(ch) != IPC_CONNECT) {
 	crm_err("No reply message - disconnected");
 	free_xml(reg_msg);
 	return cib_not_connected;
 	
     } else if(reg_msg == NULL) {
 	crm_err("No reply message - empty");
 	return cib_reply_failed;
     }
     
     msg_type = crm_element_value(reg_msg, F_CIB_OPERATION);
     tmp_ticket = crm_element_value(reg_msg, F_CIB_CLIENTID);
     
     if(safe_str_neq(msg_type, CRM_OP_REGISTER) ) {
 	crm_err("Invalid registration message: %s", msg_type);
 	rc = cib_registration_msg;
 	
     } else if(tmp_ticket == NULL) {
 	rc = cib_callback_token;
 
     } else {
 	*token = crm_strdup(tmp_ticket);
     }
 
     free_xml(reg_msg);
     return rc;
 }
 
 
 xmlNode *
 cib_create_op(
     int call_id, const char *token, const char *op, const char *host, const char *section,
     xmlNode *data, int call_options) 
 {
 	int  rc = HA_OK;
 	xmlNode *op_msg = create_xml_node(NULL, "cib_command");
 	CRM_CHECK(op_msg != NULL, return NULL);
 	CRM_CHECK(token != NULL, return NULL);
 
 	crm_xml_add(op_msg, F_XML_TAGNAME, "cib_command");
 	
 	crm_xml_add(op_msg, F_TYPE, T_CIB);
 	crm_xml_add(op_msg, F_CIB_CALLBACK_TOKEN, token);
 	crm_xml_add(op_msg, F_CIB_OPERATION, op);
 	crm_xml_add(op_msg, F_CIB_HOST, host);
 	crm_xml_add(op_msg, F_CIB_SECTION, section);
 	crm_xml_add_int(op_msg, F_CIB_CALLID, call_id);
 	crm_debug_4("Sending call options: %.8lx, %d",
 		    (long)call_options, call_options);
 	crm_xml_add_int(op_msg, F_CIB_CALLOPTS, call_options);
 
 	if(data != NULL) {
 		add_message_xml(op_msg, F_CIB_CALLDATA, data);
 	}
 	
 	if (rc != HA_OK) {
 		crm_err("Failed to create CIB operation message");
 		crm_log_xml(LOG_ERR, "op", op_msg);
 		free_xml(op_msg);
 		return NULL;
 	}
 
 	if(call_options & cib_inhibit_bcast) {
 		CRM_CHECK((call_options & cib_scope_local), return NULL);
 	}
 	return op_msg;
 }
 
 void
 cib_native_callback(cib_t *cib, xmlNode *msg, int call_id, int rc)
 {
 	xmlNode *output = NULL;
 	cib_callback_client_t *blob = NULL;
 	cib_callback_client_t local_blob;
 
 	local_blob.id = NULL;
 	local_blob.callback = NULL;
 	local_blob.user_data = NULL;
 	local_blob.only_success = FALSE;
 
 	if(msg != NULL) {
 	    crm_element_value_int(msg, F_CIB_RC, &rc);
 	    crm_element_value_int(msg, F_CIB_CALLID, &call_id);
 	    output = get_message_xml(msg, F_CIB_CALLDATA);
 	}
 
 	blob = g_hash_table_lookup(
 		cib_op_callback_table, GINT_TO_POINTER(call_id));
 	
 	if(blob != NULL) {
 		local_blob = *blob;
 		blob = NULL;
 		
 		remove_cib_op_callback(call_id, FALSE);
 
 	} else {
 		crm_debug_2("No callback found for call %d", call_id);
 		local_blob.callback = NULL;
 	}
 
 	if(cib == NULL) {
 	    crm_debug("No cib object supplied");
 	}
 	
 	if(rc == cib_diff_resync) {
 	    /* This is an internal value that clients do not and should not care about */
 	    rc = cib_ok;
 	}
 
 	if(local_blob.callback != NULL
 	   && (rc == cib_ok || local_blob.only_success == FALSE)) {
 	    crm_debug_2("Invoking callback %s for call %d", crm_str(local_blob.id), call_id);
 	    local_blob.callback(msg, call_id, rc, output, local_blob.user_data);
 		
 	} else if(cib && cib->op_callback == NULL && rc != cib_ok) {
 	    crm_warn("CIB command failed: %s", cib_error2string(rc));
 	    crm_log_xml(LOG_DEBUG, "Failed CIB Update", msg);
 	}
 	
 	if(cib && cib->op_callback != NULL) {
 		crm_debug_2("Invoking global callback for call %d", call_id);
 		cib->op_callback(msg, call_id, rc, output);
 	}
 	crm_debug_4("OP callback activated.");
 }
 
 
 void
 cib_native_notify(gpointer data, gpointer user_data)
 {
 	xmlNode *msg = user_data;
 	cib_notify_client_t *entry = data;
 	const char *event = NULL;
 
 	if(msg == NULL) {
 		crm_warn("Skipping callback - NULL message");
 		return;
 	}
 
 	event = crm_element_value(msg, F_SUBTYPE);
 	
 	if(entry == NULL) {
 		crm_warn("Skipping callback - NULL callback client");
 		return;
 
 	} else if(entry->callback == NULL) {
 		crm_warn("Skipping callback - NULL callback");
 		return;
 
 	} else if(safe_str_neq(entry->event, event)) {
 		crm_debug_4("Skipping callback - event mismatch %p/%s vs. %s",
 			  entry, entry->event, event);
 		return;
 	}
 	
 	crm_debug_4("Invoking callback for %p/%s event...", entry, event);
 	entry->callback(event, msg);
 	crm_debug_4("Callback invoked...");
 }
 
 gboolean determine_host(cib_t *cib_conn, char **node_uname, char **node_uuid) 
 {
     CRM_CHECK(node_uname != NULL, return FALSE);
     
     if(*node_uname == NULL) {
 	struct utsname name;
 	if(uname(&name) < 0) {
 	    crm_perror(LOG_ERR,"uname(2) call failed");
 	    return FALSE;
 	}
 	*node_uname = crm_strdup(name.nodename);
 	crm_info("Detected uname: %s", *node_uname);
     }
     
     if(cib_conn && *node_uname != NULL
        && node_uuid != NULL && *node_uuid == NULL) {
 	int rc = query_node_uuid(cib_conn, *node_uname, node_uuid);
 	if(rc != cib_ok) {
 	    fprintf(stderr,"Could not map uname=%s to a UUID: %s\n",
 		    *node_uname, cib_error2string(rc));
 	    return FALSE;
 	}
 	crm_info("Mapped %s to %s", *node_uname, crm_str(*node_uuid));
     }
     return TRUE;
 }
 
diff --git a/lib/common/ais.c b/lib/common/ais.c
index df4b25fac6..9bf4f95e31 100644
--- a/lib/common/ais.c
+++ b/lib/common/ais.c
@@ -1,752 +1,752 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 #include <bzlib.h>
 #include <crm/ais.h>
 #include <crm/common/cluster.h>
 #include <sys/utsname.h>
 #include "stack.h"
 #ifdef AIS_COROSYNC
 #  include <corosync/corodefs.h>
 #endif
 
 enum crm_ais_msg_types text2msg_type(const char *text) 
 {
 	int type = crm_msg_none;
 
 	CRM_CHECK(text != NULL, return type);
 	if(safe_str_eq(text, "ais")) {
 		type = crm_msg_ais;
 	} else if(safe_str_eq(text, "crm_plugin")) {
 		type = crm_msg_ais;
 	} else if(safe_str_eq(text, CRM_SYSTEM_CIB)) {
 		type = crm_msg_cib;
 	} else if(safe_str_eq(text, CRM_SYSTEM_CRMD)) {
 		type = crm_msg_crmd;
 	} else if(safe_str_eq(text, CRM_SYSTEM_DC)) {
 		type = crm_msg_crmd;
 	} else if(safe_str_eq(text, CRM_SYSTEM_TENGINE)) {
 		type = crm_msg_te;
 	} else if(safe_str_eq(text, CRM_SYSTEM_PENGINE)) {
 		type = crm_msg_pe;
 	} else if(safe_str_eq(text, CRM_SYSTEM_LRMD)) {
 		type = crm_msg_lrmd;
 	} else if(safe_str_eq(text, CRM_SYSTEM_STONITHD)) {
 		type = crm_msg_stonithd;
 	} else if(safe_str_eq(text, "stonith-ng")) {
 		type = crm_msg_stonith_ng;
 	} else if(safe_str_eq(text, "attrd")) {
 		type = crm_msg_attrd;
 
 	} else {
 	    /* This will normally be a transient client rather than
 	     * a cluster daemon.  Set the type to the pid of the client
 	     */
 	    int scan_rc = sscanf(text, "%d", &type);
 	    if(scan_rc != 1) {
 		/* Ensure its sane */
 		type = crm_msg_none;
 	    }
 	}
 	return type;
 }
 
 char *get_ais_data(const AIS_Message *msg)
 {
     int rc = BZ_OK;
     char *uncompressed = NULL;
     unsigned int new_size = msg->size + 1;
     
     if(msg->is_compressed == FALSE) {
 	crm_debug_2("Returning uncompressed message data");
 	uncompressed = strdup(msg->data);
 
     } else {
 	crm_debug_2("Decompressing message data");
 	crm_malloc0(uncompressed, new_size);
 	
 	rc = BZ2_bzBuffToBuffDecompress(
 	    uncompressed, &new_size, (char*)msg->data, msg->compressed_size, 1, 0);
 	
-	CRM_ASSERT(rc = BZ_OK);
+	CRM_ASSERT(rc == BZ_OK);
 	CRM_ASSERT(new_size == msg->size);
     }
     
     return uncompressed;
 }
 
 
 #if SUPPORT_AIS
 int ais_fd_sync = -1;
 int ais_fd_async = -1; /* never send messages via this channel */
 void *ais_ipc_ctx = NULL;
 #ifdef AIS_COROSYNC
 #  ifndef TRADITIONAL_AIS_IPC
    hdb_handle_t ais_ipc_handle = 0;
 #  endif
 #endif
 GFDSource *ais_source = NULL;
 GFDSource *ais_source_sync = NULL;
 static char *ais_cluster_name = NULL;
 
 gboolean get_ais_nodeid(uint32_t *id, char **uname)
 {
     struct iovec iov;
     int retries = 0;
     int rc = CS_OK;
     coroipc_response_header_t header;
     struct crm_ais_nodeid_resp_s answer;
 
     header.error = CS_OK;
     header.id = crm_class_nodeid;
     header.size = sizeof(coroipc_response_header_t);
 
     CRM_CHECK(id != NULL, return FALSE);
     CRM_CHECK(uname != NULL, return FALSE);
 
     iov.iov_base = &header;
     iov.iov_len = header.size;
     
   retry:
     errno = 0;
 #ifdef TRADITIONAL_AIS_IPC
     rc = saSendReceiveReply(ais_fd_sync, &header, header.size, &answer, sizeof (struct crm_ais_nodeid_resp_s));
 #else
 #  ifdef AIS_WHITETANK
     rc = openais_msg_send_reply_receive(
 	ais_ipc_ctx, &iov, 1, &answer, sizeof (answer));
 #  else
     rc = coroipcc_msg_send_reply_receive(
 	ais_ipc_handle, &iov, 1, &answer, sizeof (answer));
 #  endif
 #endif
     if(rc == CS_OK) {
 	CRM_CHECK(answer.header.size == sizeof (struct crm_ais_nodeid_resp_s),
 		  crm_err("Odd message: id=%d, size=%d, error=%d",
 			  answer.header.id, answer.header.size, answer.header.error));
 	CRM_CHECK(answer.header.id == crm_class_nodeid, crm_err("Bad response id: %d", answer.header.id));
     }
 
     if(rc == CS_ERR_TRY_AGAIN && retries < 20) {
 	retries++;
 	crm_info("Peer overloaded: Re-sending message (Attempt %d of 20)", retries);
 	sleep(retries); /* Proportional back off */
 	goto retry;
     }
 
     if(rc != CS_OK) {    
 	crm_err("Sending nodeid request: FAILED (rc=%d): %s", rc, ais_error2text(rc));
 	return FALSE;
 	
     } else if(answer.header.error != CS_OK) {
 	crm_err("Bad response from peer: (rc=%d): %s", rc, ais_error2text(rc));
 	return FALSE;
     }
 
     crm_info("Server details: id=%u uname=%s cname=%s",
 	     answer.id, answer.uname, answer.cname);
     
     *id = answer.id;
     *uname = crm_strdup(answer.uname);
     ais_cluster_name = crm_strdup(answer.cname);
 
     return TRUE;
 }
 
 gboolean crm_get_cluster_name(char **cname)
 {
     CRM_CHECK(cname != NULL, return FALSE);
     if(ais_cluster_name) {
 	*cname = crm_strdup(ais_cluster_name);
 	return TRUE;
     }
     return FALSE;
 }
 
 gboolean
 send_ais_text(int class, const char *data,
 	      gboolean local, const char *node, enum crm_ais_msg_types dest)
 {
     static int msg_id = 0;
     static int local_pid = 0;
 
     int retries = 0;
     int rc = CS_OK;
     int buf_len = sizeof(coroipc_response_header_t);
 
     char *buf = NULL;
     struct iovec iov;
     coroipc_response_header_t *header;
     AIS_Message *ais_msg = NULL;
     enum crm_ais_msg_types sender = text2msg_type(crm_system_name);
 
     /* There are only 6 handlers registered to crm_lib_service in plugin.c */
     CRM_CHECK(class < 6, crm_err("Invalid message class: %d", class); return FALSE); 
 
     if(data == NULL) {
 	data = "";
     }
     
     if(local_pid == 0) {
 	local_pid = getpid();
     }
 
     if(sender == crm_msg_none) {
 	sender = local_pid;
     }
     
     crm_malloc0(ais_msg, sizeof(AIS_Message));
     
     ais_msg->id = msg_id++;
     ais_msg->header.id = class;
     ais_msg->header.error = CS_OK;
     
     ais_msg->host.type = dest;
     ais_msg->host.local = local;
     if(node) {
 	ais_msg->host.size = strlen(node);
 	memset(ais_msg->host.uname, 0, MAX_NAME);
 	memcpy(ais_msg->host.uname, node, ais_msg->host.size);
 	ais_msg->host.id = 0;
 	
     } else {
 	ais_msg->host.size = 0;
 	memset(ais_msg->host.uname, 0, MAX_NAME);
 	ais_msg->host.id = 0;
     }
     
     ais_msg->sender.type = sender;
     ais_msg->sender.pid = local_pid;
     ais_msg->sender.size = 0;
     memset(ais_msg->sender.uname, 0, MAX_NAME);
     ais_msg->sender.id = 0;
 
     ais_msg->size = 1 + strlen(data);
 
     if(ais_msg->size < CRM_BZ2_THRESHOLD) {
   failback:
 	crm_realloc(ais_msg, sizeof(AIS_Message) + ais_msg->size);
 	memcpy(ais_msg->data, data, ais_msg->size);
 	
     } else {
 	char *compressed = NULL;
 	char *uncompressed = crm_strdup(data);
 	unsigned int len = (ais_msg->size * 1.1) + 600; /* recomended size */
 	
 	crm_debug_5("Compressing message payload");
 	crm_malloc(compressed, len);
 	
 	rc = BZ2_bzBuffToBuffCompress(
 	    compressed, &len, uncompressed, ais_msg->size, CRM_BZ2_BLOCKS, 0, CRM_BZ2_WORK);
 
 	crm_free(uncompressed);
 	
 	if(rc != BZ_OK) {
 	    crm_err("Compression failed: %d", rc);
 	    crm_free(compressed);
 	    goto failback;  
 	}
 
 	crm_realloc(ais_msg, sizeof(AIS_Message) + len + 1);
 	memcpy(ais_msg->data, compressed, len);
 	ais_msg->data[len] = 0;
 	crm_free(compressed);
 
 	ais_msg->is_compressed = TRUE;
 	ais_msg->compressed_size = len;
 
 	crm_debug_2("Compression details: %d -> %d",
 		  ais_msg->size, ais_data_len(ais_msg));
     } 
 
     ais_msg->header.size = sizeof(AIS_Message) + ais_data_len(ais_msg);
 
     crm_debug_3("Sending%s message %d to %s.%s (data=%d, total=%d)",
 		ais_msg->is_compressed?" compressed":"",
 		ais_msg->id, ais_dest(&(ais_msg->host)), msg_type2text(dest),
 		ais_data_len(ais_msg), ais_msg->header.size);
 
     iov.iov_base = ais_msg;
     iov.iov_len = ais_msg->header.size;
   retry:
     errno = 0;
     crm_realloc(buf, buf_len);
     
 #ifdef TRADITIONAL_AIS_IPC
     rc = saSendReceiveReply(ais_fd_sync, ais_msg, ais_msg->header.size, buf, buf_len);
 #else
 #  ifdef AIS_WHITETANK
     rc = openais_msg_send_reply_receive(ais_ipc_ctx, &iov, 1, buf, buf_len);
 #  else
     rc = coroipcc_msg_send_reply_receive(ais_ipc_handle, &iov, 1, buf, buf_len);
 #  endif
 #endif
     header = (coroipc_response_header_t *)buf;
 
     if(rc == CS_ERR_TRY_AGAIN && retries < 20) {
 	retries++;
 	crm_info("Peer overloaded: Re-sending message (Attempt %d of 20)", retries);
 	sleep(retries); /* Proportional back off */
 	goto retry;
 
     } else if(rc == CS_OK) {
 
 	CRM_CHECK_AND_STORE(header->size == sizeof (coroipc_response_header_t),
 			    crm_err("Odd message: id=%d, size=%d, class=%d, error=%d",
 				    header->id, header->size, class, header->error));
 
 	if(buf_len < header->size) {
 	    crm_err("Increasing buffer length to %d and retrying", header->size);
 	    buf_len = header->size + 1;
 	    goto retry;
 
 	} else if(header->id == crm_class_nodeid && header->size == sizeof (struct crm_ais_nodeid_resp_s)){
 	    struct crm_ais_nodeid_resp_s *answer = (struct crm_ais_nodeid_resp_s *)header;
 	    crm_err("Server details: id=%u uname=%s counter=%u", answer->id, answer->uname, answer->counter);
 
 	} else {
 	    CRM_CHECK_AND_STORE(header->id == CRM_MESSAGE_IPC_ACK,
 				crm_err("Bad response id (%d) for request (%d)", header->id, ais_msg->header.id));
 	    CRM_CHECK(header->error == CS_OK, rc = header->error);
 	}
     }
     
     if(rc != CS_OK) {    
 	crm_perror(LOG_ERR,"Sending message %d: FAILED (rc=%d): %s",
 		  ais_msg->id, rc, ais_error2text(rc));
 	ais_fd_async = -1;
     } else {
 	crm_debug_4("Message %d: sent", ais_msg->id);
     }
 
     crm_free(buf);
     crm_free(ais_msg);
     return (rc == CS_OK);
 }
 
 gboolean
 send_ais_message(xmlNode *msg, 
 		 gboolean local, const char *node, enum crm_ais_msg_types dest)
 {
     gboolean rc = TRUE;
     char *data = NULL;
 
     if(ais_fd_async < 0 || ais_source == NULL) {
 	crm_err("Not connected to AIS");
 	return FALSE;
     }
 
     data = dump_xml_unformatted(msg);
     rc = send_ais_text(0, data, local, node, dest);
     crm_free(data);
     return rc;
 }
 
 void terminate_ais_connection(void) 
 {
 #ifndef TRADITIONAL_AIS_IPC
     if(ais_ipc_ctx) {
 #  ifdef AIS_WHITETANK
 	openais_service_disconnect(ais_ipc_ctx);
 #  else
 	coroipcc_service_disconnect(ais_ipc_handle);
 #  endif
     }
 #else
     if(ais_fd_sync > 0) {
 	close(ais_fd_sync);
     }
     if(ais_fd_async > 0) {
 	close(ais_fd_async);
     }
 #endif
     crm_notice("Disconnected from AIS");
 /*     G_main_del_fd(ais_source); */
 /*     G_main_del_fd(ais_source_sync);     */
 }
 
 int ais_membership_timer = 0;
 gboolean ais_membership_force = FALSE;
 
 gboolean ais_dispatch(int sender, gpointer user_data)
 {
     char *data = NULL;
     char *buffer = NULL;
     char *uncompressed = NULL;
     
     int rc = CS_OK;
     xmlNode *xml = NULL;
     AIS_Message *msg = NULL;
     gboolean (*dispatch)(AIS_Message*,char*,int) = user_data;
 
 #ifdef TRADITIONAL_AIS_IPC
     coroipc_response_header_t *header = NULL;
     static int header_len = sizeof(coroipc_response_header_t);
 
     crm_malloc0(header, header_len);
     buffer = (char*)header;
     
     errno = 0;
     rc = saRecvRetry(sender, header, header_len);
     
     if (rc != CS_OK) {
 	crm_perror(LOG_ERR, "Receiving message header failed: (%d/%d) %s", rc, errno, ais_error2text(rc));
 	goto bail;
 
     } else if(header->size == header_len) {
 	crm_err("Empty message: id=%d, size=%d, error=%d, header_len=%d",
 		header->id, header->size, header->error, header_len);
 	goto done;
 	
     } else if(header->size == 0 || header->size < header_len) {
 	crm_err("Mangled header: size=%d, header=%d, error=%d",
 		header->size, header_len, header->error);
 	goto done;
 	
     } else if(header->error != CS_OK) {
 	crm_err("Header contined error: %d", header->error);
     }
     
     crm_debug_2("Looking for %d (%d - %d) more bytes",
 		header->size - header_len, header->size, header_len);
 
     crm_realloc(header, header->size);
     /* Use a char* so we can store the remainder into an offset */
     buffer = (char*)header;
 
     errno = 0;
     rc = saRecvRetry(sender, buffer+header_len, header->size - header_len);
 #else
 #  ifdef AIS_WHITETANK
     crm_malloc0(buffer, 1000000);
     rc = openais_dispatch_recv (ais_ipc_ctx, buffer, 0);
 #  else
     rc = coroipcc_dispatch_get (ais_ipc_handle, (void**)&buffer, 0);
 #  endif
 #endif
 
     if (rc == 0) {
 	/* Zero is a legal "no message afterall" value */
 	goto done;
 	
     } else if (rc != CS_OK) {
 	crm_perror(LOG_ERR,"Receiving message body failed: (%d) %s", rc, ais_error2text(rc));
 	goto bail;
     }
 
     msg = (AIS_Message*)buffer;
     crm_debug_3("Got new%s message (size=%d, %d, %d)",
 		msg->is_compressed?" compressed":"",
 		ais_data_len(msg), msg->size, msg->compressed_size);
     
     data = msg->data;
     if(msg->is_compressed && msg->size > 0) {
 	int rc = BZ_OK;
 	unsigned int new_size = msg->size + 1;
 
 	if(check_message_sanity(msg, NULL) == FALSE) {
 	    goto badmsg;
 	}
 
 	crm_debug_5("Decompressing message data");
 	crm_malloc0(uncompressed, new_size);
 	rc = BZ2_bzBuffToBuffDecompress(
 	    uncompressed, &new_size, data, msg->compressed_size, 1, 0);
 
 	if(rc != BZ_OK) {
 	    crm_err("Decompression failed: %d", rc);
 	    goto badmsg;
 	}
 	
 	CRM_ASSERT(rc == BZ_OK);
 	CRM_ASSERT(new_size == msg->size);
 
 	data = uncompressed;
 
     } else if(check_message_sanity(msg, data) == FALSE) {
 	goto badmsg;
 
     } else if(safe_str_eq("identify", data)) {
 	int pid = getpid();
 	char *pid_s = crm_itoa(pid);
 	send_ais_text(0, pid_s, TRUE, NULL, crm_msg_ais);
 	crm_free(pid_s);
 	goto done;
     }
 
     if(msg->header.id != crm_class_members) {
 	crm_update_peer(msg->sender.id, 0,0,0,0, msg->sender.uname, msg->sender.uname, NULL, NULL);
     }
     
     if(msg->header.id == crm_class_rmpeer) {
 	uint32_t id = crm_int_helper(data, NULL);
 	crm_info("Removing peer %s/%u", data, id);
 	reap_crm_member(id);
 	goto done;
 
     } else if(msg->header.id == crm_class_members
 	|| msg->header.id == crm_class_quorum) {
 	const char *value = NULL;
 	gboolean quorate = FALSE;	
 	
 	xml = string2xml(data);
 	if(xml == NULL) {
 	    crm_err("Invalid membership update: %s", data);
 	    goto badmsg;
 	}
 	
 	value = crm_element_value(xml, "quorate");
 	CRM_CHECK(value != NULL, crm_log_xml_err(xml, "No quorum value:"); goto badmsg);
 	if(crm_is_true(value)) {
 	    quorate = TRUE;
 	}
 
 	value = crm_element_value(xml, "id");
 	CRM_CHECK(value != NULL, crm_log_xml_err(xml, "No membership id"); goto badmsg);
 	crm_peer_seq = crm_int_helper(value, NULL);
 
 	if(quorate != crm_have_quorum) {
 	    crm_notice("Membership %s: quorum %s", value, quorate?"acquired":"lost");
 	    crm_have_quorum = quorate;
 
 	} else {
 	    crm_info("Membership %s: quorum %s", value, quorate?"retained":"still lost");
 	}
 	
 	xml_child_iter(xml, node, crm_update_ais_node(node, crm_peer_seq));
     }
 
     if(dispatch != NULL) {
 	dispatch(msg, data, sender);
     }
     
   done:
     crm_free(uncompressed);
     free_xml(xml);
     
 #ifdef AIS_COROSYNC
 #  ifndef TRADITIONAL_AIS_IPC
     coroipcc_dispatch_put (ais_ipc_handle);
     buffer = NULL;
 #  endif
 #endif
 
     crm_free(buffer);
     return TRUE;
 
   badmsg:
     crm_err("Invalid message (id=%d, dest=%s:%s, from=%s:%s.%d):"
 	    " min=%d, total=%d, size=%d, bz2_size=%d",
 	    msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type),
 	    ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
 	    msg->sender.pid, (int)sizeof(AIS_Message),
 	    msg->header.size, msg->size, msg->compressed_size);
     goto done;
     
   bail:
     crm_err("AIS connection failed");
 #ifdef AIS_COROSYNC
 #  ifndef TRADITIONAL_AIS_IPC
     buffer = NULL;
 #  endif
 #endif
     crm_free(buffer);
     return FALSE;
 }
 
 static void
 ais_destroy(gpointer user_data)
 {
     crm_err("AIS connection terminated");
     ais_fd_sync = -1;
     exit(1);
 }
 
 gboolean init_ais_connection(
     gboolean (*dispatch)(AIS_Message*,char*,int),
     void (*destroy)(gpointer), char **our_uuid, char **our_uname, int *nodeid)
 {
     int pid = 0;
     int retries = 0;
     int rc = CS_OK;
     char *pid_s = NULL;
     struct utsname name;
     uint32_t local_nodeid = 0;
     char *local_uname = NULL;
     
   retry:
     crm_info("Creating connection to our AIS plugin");
 #ifdef TRADITIONAL_AIS_IPC
     rc = saServiceConnect (&ais_fd_sync, &ais_fd_async, PCMK_SERVICE_ID);
 #else
 #  ifdef AIS_WHITETANK
     rc = openais_service_connect(PCMK_SERVICE_ID, &ais_ipc_ctx);
     if(ais_ipc_ctx) {
 	ais_fd_async = openais_fd_get(ais_ipc_ctx);
     }
 #  else
     rc = coroipcc_service_connect(
 	COROSYNC_SOCKET_NAME, PCMK_SERVICE_ID,
 	AIS_IPC_MESSAGE_SIZE, AIS_IPC_MESSAGE_SIZE, AIS_IPC_MESSAGE_SIZE,
 	&ais_ipc_handle);
     if(ais_ipc_handle) {
 	coroipcc_fd_get(ais_ipc_handle, &ais_fd_async);
     }
 #  endif
 #endif
     if(ais_fd_async <= 0 && rc == CS_OK) {
 	crm_err("No context created, but connection reported 'ok'");
 	rc = CS_ERR_LIBRARY;
     }
     if (rc != CS_OK) {
 	crm_info("Connection to our AIS plugin (%d) failed: %s (%d)", PCMK_SERVICE_ID, ais_error2text(rc), rc);
     }
 
     switch(rc) {
 	case CS_OK:
 	    break;
 	case CS_ERR_TRY_AGAIN:
 	    if(retries < 30) {
 		sleep(1);
 		retries++;
 		goto retry;
 	    }
 	    crm_err("Retry count exceeded");
 	    return FALSE;
 	default:
 	    return FALSE;
     }
 
     if(destroy == NULL) {
 	destroy = ais_destroy;
     } 
    
     crm_info("AIS connection established");
     
     pid = getpid();
     pid_s = crm_itoa(pid);
     send_ais_text(0, pid_s, TRUE, NULL, crm_msg_ais);
     crm_free(pid_s);
 
     crm_peer_init();
     get_ais_nodeid(&local_nodeid, &local_uname);
 
     if(uname(&name) < 0) {
 	crm_perror(LOG_ERR,"uname(2) call failed");
 	exit(100);
     }
 
     if(safe_str_neq(name.nodename, local_uname)) {
 	crm_crit("Node name mismatch!  OpenAIS supplied %s, our lookup returned %s", local_uname, name.nodename);
 	crm_notice("Node name mismatches usually occur when assigned automatically by DHCP servers");
 	crm_notice("If this node was part of the cluster with a different name,"
 		   " you will need to remove the old entry with crm_node --remove");
     }
     
     if(our_uuid != NULL) {
 	*our_uuid = crm_strdup(local_uname);
     }
     if(our_uname != NULL) {
 	*our_uname = local_uname;
     }
 
     if(nodeid != NULL) {
 	*nodeid = local_nodeid;
     }
     
     if(local_nodeid != 0) {
 	/* Ensure the local node always exists */
 	crm_update_peer(local_nodeid, 0, 0, 0, 0, local_uname, local_uname, NULL, NULL);
     }
 
     if(dispatch) {
 	ais_source = G_main_add_fd(
 	    G_PRIORITY_HIGH, ais_fd_async, FALSE, ais_dispatch, dispatch, destroy);
     }
     return TRUE;
 }
 
 gboolean check_message_sanity(const AIS_Message *msg, const char *data) 
 {
     gboolean sane = TRUE;
     gboolean repaired = FALSE;
     int dest = msg->host.type;
     int tmp_size = msg->header.size - sizeof(AIS_Message);
 
     if(sane && msg->header.size == 0) {
 	crm_warn("Message with no size");
 	sane = FALSE;
     }
 
     if(sane && msg->header.error != CS_OK) {
 	crm_warn("Message header contains an error: %d", msg->header.error);
 	sane = FALSE;
     }
 
     if(sane && ais_data_len(msg) != tmp_size) {
 	crm_warn("Message payload size is incorrect: expected %d, got %d", ais_data_len(msg), tmp_size);
 	sane = TRUE;
     }
 
     if(sane && ais_data_len(msg) == 0) {
 	crm_warn("Message with no payload");
 	sane = FALSE;
     }
 
     if(sane && data && msg->is_compressed == FALSE) {
 	int str_size = strlen(data) + 1;
 	if(ais_data_len(msg) != str_size) {
 	    int lpc = 0;
 	    crm_warn("Message payload is corrupted: expected %d bytes, got %d",
 		    ais_data_len(msg), str_size);
 	    sane = FALSE;
 	    for(lpc = (str_size - 10); lpc < msg->size; lpc++) {
 		if(lpc < 0) {
 		    lpc = 0;
 		}
 		crm_debug("bad_data[%d]: %d / '%c'", lpc, data[lpc], data[lpc]);
 	    }
 	}
     }
     
     if(sane == FALSE) {
 	crm_err("Invalid message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)",
 		msg->id, ais_dest(&(msg->host)), msg_type2text(dest),
 		ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
 		msg->sender.pid, msg->is_compressed, ais_data_len(msg),
 		msg->header.size);
 	
     } else if(repaired) {
 	crm_err("Repaired message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)",
 		msg->id, ais_dest(&(msg->host)), msg_type2text(dest),
 		ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
 		msg->sender.pid, msg->is_compressed, ais_data_len(msg),
 		msg->header.size);
     } else {
 	crm_debug_3("Verfied message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)",
 		    msg->id, ais_dest(&(msg->host)), msg_type2text(dest),
 		    ais_dest(&(msg->sender)), msg_type2text(msg->sender.type),
 		    msg->sender.pid, msg->is_compressed, ais_data_len(msg),
 		    msg->header.size);
     }
     
     return sane;
 }
 #endif
 
diff --git a/lib/common/ipc.c b/lib/common/ipc.c
index ae1b6a6479..0b3aaccebe 100644
--- a/lib/common/ipc.c
+++ b/lib/common/ipc.c
@@ -1,672 +1,670 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <stdio.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
 #include <errno.h>
 #include <fcntl.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/ipc.h>
 #include <crm/common/cluster.h>
 
 xmlNode *xmlfromIPC(IPC_Channel *ch, int timeout) 
 {
     xmlNode *xml = NULL;
     HA_Message *msg = NULL;
     
 #if HAVE_MSGFROMIPC_TIMEOUT
     int ipc_rc = IPC_OK;
 
     msg = msgfromIPC_timeout(ch, MSG_ALLOWINTR, timeout, &ipc_rc);
     
     if(ipc_rc == IPC_TIMEOUT) {
 	crm_warn("No message received in the required interval (%ds)", timeout);
 	return NULL;
 	
     } else if(ipc_rc == IPC_BROKEN) {
 	crm_debug("Peer disconnected");
 	return NULL;
 	
     } else if(ipc_rc != IPC_OK) {
 	crm_err("msgfromIPC_timeout failed: rc=%d", ipc_rc);
 	return NULL;
 
     } else if(msg == NULL) {
 	crm_err("Empty reply from msgfromIPC_timeout");
 	return NULL;
     }
 #else
     static gboolean do_show_error = TRUE;
 
     if(timeout && do_show_error) {
 	crm_err("Timeouts are not supported by the current heartbeat libraries");
 	do_show_error = FALSE;
     }
 
     msg = msgfromIPC_noauth(ch);
     if(msg == NULL) {
 	crm_debug("Empty reply from msgfromIPC_noauth");
 	return NULL;
     }
 #endif
 
     xml = convert_ha_message(NULL, msg, __FUNCTION__);
     CRM_CHECK(xml != NULL, crm_err("Invalid ipc message"));
     crm_msg_del(msg);
     return xml;
 }
 
 static int xml2ipcchan(xmlNode *m, IPC_Channel *ch)
 {
 	HA_Message  *msg = NULL;
 	IPC_Message *imsg = NULL;
 	
 	if (m == NULL || ch == NULL) {
 		cl_log(LOG_ERR, "Invalid msg2ipcchan argument");
 		errno = EINVAL;
 		return HA_FAIL;
 	}
 
 	msg = convert_xml_message(m);
 	if ((imsg = hamsg2ipcmsg(msg, ch)) == NULL) {
 		cl_log(LOG_ERR, "hamsg2ipcmsg() failure");
 		crm_msg_del(msg);
 		return HA_FAIL;
 	}
 	crm_msg_del(msg);
 	
 	if (ch->ops->send(ch, imsg) != IPC_OK) {
 		if (ch->ch_status == IPC_CONNECT) {
 			snprintf(ch->failreason,MAXFAILREASON, 
 				 "send failed,farside_pid=%d, sendq length=%ld(max is %ld)",
 				 ch->farside_pid, (long)ch->send_queue->current_qlen, 
 				 (long)ch->send_queue->max_qlen);	
 		}
 		imsg->msg_done(imsg);
 		return HA_FAIL;
 	}
 	return HA_OK;
 }
 
 /* frees msg */
 gboolean 
 send_ipc_message(IPC_Channel *ipc_client, xmlNode *msg)
 {
 	gboolean all_is_good = TRUE;
 	int fail_level = LOG_WARNING;
 
 	if(ipc_client != NULL && ipc_client->conntype == IPC_CLIENT) {
 		fail_level = LOG_ERR;
 	}
 
 	if (msg == NULL) {
 		crm_err("cant send NULL message");
 		all_is_good = FALSE;
 
 	} else if (ipc_client == NULL) {
 		crm_err("cant send message without an IPC Channel");
 		all_is_good = FALSE;
 
 	} else if(ipc_client->ops->get_chan_status(ipc_client) != IPC_CONNECT) {
 		do_crm_log(fail_level, "IPC Channel to %d is not connected",
 			      (int)ipc_client->farside_pid);
 		all_is_good = FALSE;
 	}
 
 	if(all_is_good && xml2ipcchan(msg, ipc_client) != HA_OK) {
 		do_crm_log(fail_level, "Could not send IPC message to %d",
 			(int)ipc_client->farside_pid);
 		all_is_good = FALSE;
 
 		if(ipc_client->ops->get_chan_status(ipc_client) != IPC_CONNECT) {
 			do_crm_log(fail_level,
 				      "IPC Channel to %d is no longer connected",
 				      (int)ipc_client->farside_pid);
 
 		} else if(ipc_client->conntype == IPC_CLIENT) {
 			if(ipc_client->send_queue->current_qlen >= ipc_client->send_queue->max_qlen) {
 				crm_err("Send queue to %d (size=%d) full.",
 					ipc_client->farside_pid,
 					(int)ipc_client->send_queue->max_qlen);
 			}
 		}
 	}
 	/* crm_log_xml(all_is_good?LOG_MSG:LOG_WARNING,"IPC[outbound]",msg); */
 	
 	return all_is_good;
 }
 
 void
 default_ipc_connection_destroy(gpointer user_data)
 {
 	return;
 }
 
 int
 init_server_ipc_comms(
 	char *channel_name,
 	gboolean (*channel_client_connect)(IPC_Channel *newclient,gpointer user_data),
 	void (*channel_connection_destroy)(gpointer user_data))
 {
 	/* the clients wait channel is the other source of events.
 	 * This source delivers the clients connection events.
 	 * listen to this source at a relatively lower priority.
 	 */
     
 	char    commpath[SOCKET_LEN];
 	IPC_WaitConnection *wait_ch;
 	
 	sprintf(commpath, CRM_STATE_DIR "/%s", channel_name);
 
 	wait_ch = wait_channel_init(commpath);
 
 	if (wait_ch == NULL) {
 		return 1;
 	}
 	
 	G_main_add_IPC_WaitConnection(
 		G_PRIORITY_LOW, wait_ch, NULL, FALSE,
 		channel_client_connect, channel_name,
 		channel_connection_destroy);
 
 	crm_debug_3("Listening on: %s", commpath);
 
 	return 0;
 }
 
 GCHSource*
 init_client_ipc_comms(const char *channel_name,
 		      gboolean (*dispatch)(
 			      IPC_Channel* source_data, gpointer user_data),
 		      void *client_data, IPC_Channel **ch)
 {
 	IPC_Channel *a_ch = NULL;
 	GCHSource *the_source = NULL;
 	void *callback_data = client_data;
 
 	a_ch = init_client_ipc_comms_nodispatch(channel_name);
 	if(ch != NULL) {
 		*ch = a_ch;
 		if(callback_data == NULL) {
 			callback_data = a_ch;
 		}
 	}
 
 	if(a_ch == NULL) {
 		crm_warn("Setup of client connection failed,"
 			 " not adding channel to mainloop");
 		
 		return NULL;
 	}
 
 	if(dispatch == NULL) {
 		crm_warn("No dispatch method specified..."
 			 "maybe you meant init_client_ipc_comms_nodispatch()?");
 	} else {
 		crm_debug_3("Adding dispatch method to channel");
 
 		the_source = G_main_add_IPC_Channel(
 			G_PRIORITY_HIGH, a_ch, FALSE, dispatch, callback_data, 
 			default_ipc_connection_destroy);
 	}
 	
 	return the_source;
 }
 
 IPC_Channel *
 init_client_ipc_comms_nodispatch(const char *channel_name)
 {
 	IPC_Channel *ch;
 	GHashTable  *attrs;
 	static char  path[] = IPC_PATH_ATTR;
 
 	char *commpath = NULL;
 	int local_socket_len = 2; /* 2 = '/' + '\0' */
 
 	local_socket_len += strlen(channel_name);
 	local_socket_len += strlen(CRM_STATE_DIR);
 
 	crm_malloc0(commpath, local_socket_len);
 
 	sprintf(commpath, CRM_STATE_DIR "/%s", channel_name);
 	commpath[local_socket_len - 1] = '\0';
 	crm_debug("Attempting to talk on: %s", commpath);
 	
 	attrs = g_hash_table_new(g_str_hash,g_str_equal);
 	g_hash_table_insert(attrs, path, commpath);
 
 	ch = ipc_channel_constructor(IPC_ANYTYPE, attrs);
 	g_hash_table_destroy(attrs);
 
 	if (ch == NULL) {
 		crm_err("Could not access channel on: %s", commpath);
 		crm_free(commpath);
 		return NULL;
 		
 	} else if (ch->ops->initiate_connection(ch) != IPC_OK) {
 		crm_debug("Could not init comms on: %s", commpath);
 		ch->ops->destroy(ch);
 		crm_free(commpath);
 		return NULL;
 	}
 
 	ch->ops->set_recv_qlen(ch, 512);
 	ch->ops->set_send_qlen(ch, 512);
  	ch->should_send_block = TRUE;
 
 	crm_debug_3("Processing of %s complete", commpath);
 
 	crm_free(commpath);
 	return ch;
 }
 
 IPC_WaitConnection *
 wait_channel_init(char daemonsocket[])
 {
 	IPC_WaitConnection *wait_ch;
 	mode_t mask;
 	char path[] = IPC_PATH_ATTR;
 	GHashTable * attrs;
 
 	
 	attrs = g_hash_table_new(g_str_hash,g_str_equal);
 	g_hash_table_insert(attrs, path, daemonsocket);
     
 	mask = umask(0);
 	wait_ch = ipc_wait_conn_constructor(IPC_ANYTYPE, attrs);
 	if (wait_ch == NULL) {
 		crm_perror(LOG_ERR,"Can't create wait channel of type %s",
 			  IPC_ANYTYPE);
 		exit(1);
 	}
 	mask = umask(mask);
     
 	g_hash_table_destroy(attrs);
     
 	return wait_ch;
 }
 
 gboolean
 subsystem_msg_dispatch(IPC_Channel *sender, void *user_data)
 {
 	int lpc = 0;
 	xmlNode *msg = NULL;
 	xmlNode *data = NULL;
 	gboolean all_is_well = TRUE;
 	const char *sys_to;
 	const char *task;
 	gboolean (*process_function)
 	    (xmlNode *msg, xmlNode *data, IPC_Channel *sender) = NULL;
 
 	while(IPC_ISRCONN(sender)) {
 		gboolean process = FALSE;
 		if(sender->ops->is_message_pending(sender) == 0) {
 			break;
 		}
 
 		msg = xmlfromIPC(sender, MAX_IPC_DELAY);
 		if (msg == NULL) {
 		    break;
 		}
 
 		lpc++;
 		crm_log_xml(LOG_MSG, __FUNCTION__, msg);
 
 		sys_to = crm_element_value(msg, F_CRM_SYS_TO);
 		task   = crm_element_value(msg, F_CRM_TASK);
 
 		if(safe_str_eq(task, CRM_OP_HELLO)) {
 			process = TRUE;
 
 		} else if(sys_to == NULL) {
 			crm_err("Value of %s was NULL!!", F_CRM_SYS_TO);
 			
 		} else if(task == NULL) {
 			crm_err("Value of %s was NULL!!", F_CRM_TASK);
 			
 		} else {
 			process = TRUE;
 		}
 
 		if(process == FALSE) {
 		    free_xml(msg); msg = NULL;
 		    continue;
 		}
 		
 		data = get_message_xml(msg, F_CRM_DATA);		
 		process_function = user_data;
 		if(FALSE == process_function(msg, data, sender)) {
 		    crm_warn("Received a message destined for %s"
 			     " by mistake", sys_to);
 		}
 	
 		free_xml(msg); msg = NULL;
 		
 		if(sender->ch_status == IPC_CONNECT) {
 		    break;
 		}
 	}
 
 	crm_debug_2("Processed %d messages", lpc);
 	if (sender->ch_status != IPC_CONNECT) {
 		crm_err("The server %d has left us: Shutting down...NOW",
 			sender->farside_pid);
 
 		exit(1); /* shutdown properly later */
 		
 		return !all_is_well;
 	}
 	return all_is_well;
 }
 
 gboolean
 is_ipc_empty(IPC_Channel *ch)
 {
 	if(ch == NULL) {
 		return TRUE;
 
 	} else if(ch->send_queue->current_qlen == 0
 		  && ch->recv_queue->current_qlen == 0) {
 		return TRUE;
 	}	
 	return FALSE;
 }
 
 void
 send_hello_message(IPC_Channel *ipc_client,
 		   const char *uuid,
 		   const char *client_name,
 		   const char *major_version,
 		   const char *minor_version)
 {
 	xmlNode *hello_node = NULL;
 	xmlNode *hello = NULL;
 	if (uuid == NULL || strlen(uuid) == 0
 	    || client_name == NULL || strlen(client_name) == 0
 	    || major_version == NULL || strlen(major_version) == 0
 	    || minor_version == NULL || strlen(minor_version) == 0) {
 		crm_err("Missing fields, Hello message will not be valid.");
 		return;
 	}
 
 	hello_node = create_xml_node(NULL, XML_TAG_OPTIONS);
 	crm_xml_add(hello_node, "major_version", major_version);
 	crm_xml_add(hello_node, "minor_version", minor_version);
 	crm_xml_add(hello_node, "client_name",   client_name);
 	crm_xml_add(hello_node, "client_uuid",   uuid);
 
 	crm_debug_4("creating hello message");
 	hello = create_request(
 		CRM_OP_HELLO, hello_node, NULL, NULL, client_name, uuid);
 
 	send_ipc_message(ipc_client, hello);
 	crm_debug_4("hello message sent");
 	
 	free_xml(hello_node);
 	free_xml(hello);
 }
 
 
 gboolean
 process_hello_message(xmlNode *hello,
 		      char **uuid,
 		      char **client_name,
 		      char **major_version,
 		      char **minor_version)
 {
 	const char *local_uuid;
 	const char *local_client_name;
 	const char *local_major_version;
 	const char *local_minor_version;
 
 	*uuid = NULL;
 	*client_name = NULL;
 	*major_version = NULL;
 	*minor_version = NULL;
 
 	if(hello == NULL) {
 		return FALSE;
 	}
 	
 	local_uuid = crm_element_value(hello, "client_uuid");
 	local_client_name = crm_element_value(hello, "client_name");
 	local_major_version = crm_element_value(hello, "major_version");
 	local_minor_version = crm_element_value(hello, "minor_version");
 
 	if (local_uuid == NULL || strlen(local_uuid) == 0) {
 		crm_err("Hello message was not valid (field %s not found)",
 		       "uuid");
 		return FALSE;
 
 	} else if (local_client_name==NULL || strlen(local_client_name)==0){
 		crm_err("Hello message was not valid (field %s not found)",
 			"client name");
 		return FALSE;
 
 	} else if(local_major_version == NULL
 		  || strlen(local_major_version) == 0){
 		crm_err("Hello message was not valid (field %s not found)",
 			"major version");
 		return FALSE;
 
 	} else if (local_minor_version == NULL
 		   || strlen(local_minor_version) == 0){
 		crm_err("Hello message was not valid (field %s not found)",
 			"minor version");
 		return FALSE;
 	}
     
 	*uuid          = crm_strdup(local_uuid);
 	*client_name   = crm_strdup(local_client_name);
 	*major_version = crm_strdup(local_major_version);
 	*minor_version = crm_strdup(local_minor_version);
 
 	crm_debug_3("Hello message ok");
 	return TRUE;
 }
 
 xmlNode *
 create_request_adv(const char *task, xmlNode *msg_data,
 		   const char *host_to,  const char *sys_to,
 		   const char *sys_from, const char *uuid_from,
 		   const char *origin)
 {
 	char *true_from = NULL;
 	xmlNode *request = NULL;
 	char *reference = generateReference(task, sys_from);
 
 	if (uuid_from != NULL) {
 		true_from = generate_hash_key(sys_from, uuid_from);
 	} else if(sys_from != NULL) {
 		true_from = crm_strdup(sys_from);
 	} else {
 		crm_err("No sys from specified");
 	}
 	
 	/* host_from will get set for us if necessary by CRMd when routed */
 	request = create_xml_node(NULL, __FUNCTION__);
 	crm_xml_add(request, F_CRM_ORIGIN,	origin);
 	crm_xml_add(request, F_TYPE,		T_CRM);
 	crm_xml_add(request, F_CRM_VERSION,	CRM_FEATURE_SET);
 	crm_xml_add(request, F_CRM_MSG_TYPE,     XML_ATTR_REQUEST);
 	crm_xml_add(request, XML_ATTR_REFERENCE, reference);
 	crm_xml_add(request, F_CRM_TASK,		task);
 	crm_xml_add(request, F_CRM_SYS_TO,       sys_to);
 	crm_xml_add(request, F_CRM_SYS_FROM,     true_from);
 
 	/* HOSTTO will be ignored if it is to the DC anyway. */
 	if(host_to != NULL && strlen(host_to) > 0) {
 		crm_xml_add(request, F_CRM_HOST_TO,  host_to);
 	}
 
 	if (msg_data != NULL) {
 		add_message_xml(request, F_CRM_DATA, msg_data);
 	}
 	crm_free(reference);
 	crm_free(true_from);
 	
 	return request;
 }
 
 ha_msg_input_t *
 new_ha_msg_input(xmlNode *orig) 
 {
 	ha_msg_input_t *input_copy = NULL;
 	crm_malloc0(input_copy, sizeof(ha_msg_input_t));
 	input_copy->msg = orig;
 	input_copy->xml = get_message_xml(input_copy->msg, F_CRM_DATA);
 	return input_copy;
 }
 
 void
 delete_ha_msg_input(ha_msg_input_t *orig) 
 {
 	if(orig == NULL) {
 		return;
 	}
  	free_xml(orig->msg);
 	crm_free(orig);
 }
 
 xmlNode *
 validate_crm_message(
 	xmlNode *msg, const char *sys, const char *uuid, const char *msg_type)
 {
-	const char *from = NULL;
 	const char *to = NULL;
 	const char *type = NULL;
 	const char *crm_msg_reference = NULL;
 	xmlNode *action = NULL;
 	const char *true_sys;
 	char *local_sys = NULL;
 	
 	
 	if (msg == NULL) {
 		return NULL;
 	}
 
-	from = crm_element_value(msg, F_CRM_SYS_FROM);
 	to   = crm_element_value(msg, F_CRM_SYS_TO);
 	type = crm_element_value(msg, F_CRM_MSG_TYPE);
 	
 	crm_msg_reference = crm_element_value(msg, XML_ATTR_REFERENCE);
 	action = msg;
 	true_sys = sys;
 
 	if (uuid != NULL) {
 		local_sys = generate_hash_key(sys, uuid);
 		true_sys = local_sys;
 	}
 
 	if (to == NULL) {
 		crm_info("No sub-system defined.");
 		action = NULL;
 	} else if (true_sys != NULL && strcasecmp(to, true_sys) != 0) {
 		crm_debug_3("The message is not for this sub-system (%s != %s).",
 			  to, true_sys);
 		action = NULL;
 	}
 
 	crm_free(local_sys);
 	
 	if (type == NULL) {
 		crm_info("No message type defined.");
 		return NULL;
 		
 	} else if (msg_type != NULL && strcasecmp(msg_type, type) != 0) {
 		crm_info("Expecting a (%s) message but received a (%s).",
 		       msg_type, type);
 		action = NULL;
 	}
 
 	if (crm_msg_reference == NULL) {
 		crm_info("No message crm_msg_reference defined.");
 		action = NULL;
 	}
 /*
  	if(action != NULL) 
 		crm_debug_3(
 		       "XML is valid and node with message type (%s) found.",
 		       type);
 	crm_debug_3("Returning node (%s)", crm_element_name(action));
 */
 	
 	return action;
 }
 
 /*
  * This method adds a copy of xml_response_data
  */
 xmlNode *
 create_reply_adv(xmlNode *original_request,
 		 xmlNode *xml_response_data, const char *origin)
 {
 	xmlNode *reply = NULL;
 
 	const char *host_from= crm_element_value(original_request, F_CRM_HOST_FROM);
 	const char *sys_from = crm_element_value(original_request, F_CRM_SYS_FROM);
 	const char *sys_to   = crm_element_value(original_request, F_CRM_SYS_TO);
 	const char *type     = crm_element_value(original_request, F_CRM_MSG_TYPE);
 	const char *operation= crm_element_value(original_request, F_CRM_TASK);
 	const char *crm_msg_reference = crm_element_value(
 		original_request, XML_ATTR_REFERENCE);
 	
 	if (type == NULL) {
 		crm_err("Cannot create new_message,"
 			" no message type in original message");
 		CRM_ASSERT(type != NULL);
 		return NULL;
 #if 0
 	} else if (strcasecmp(XML_ATTR_REQUEST, type) != 0) {
 		crm_err("Cannot create new_message,"
 			" original message was not a request");
 		return NULL;
 #endif
 	}
 	reply = create_xml_node(NULL, __FUNCTION__);
 	crm_xml_add(reply, F_CRM_ORIGIN,		origin);
 	crm_xml_add(reply, F_TYPE,		T_CRM);
 	crm_xml_add(reply, F_CRM_VERSION,	CRM_FEATURE_SET);
 	crm_xml_add(reply, F_CRM_MSG_TYPE,	XML_ATTR_RESPONSE);
 	crm_xml_add(reply, XML_ATTR_REFERENCE,	crm_msg_reference);
 	crm_xml_add(reply, F_CRM_TASK,		operation);
 
 	/* since this is a reply, we reverse the from and to */
 	crm_xml_add(reply, F_CRM_SYS_TO,		sys_from);
 	crm_xml_add(reply, F_CRM_SYS_FROM,	sys_to);
 	
 	/* HOSTTO will be ignored if it is to the DC anyway. */
 	if(host_from != NULL && strlen(host_from) > 0) {
 		crm_xml_add(reply, F_CRM_HOST_TO, host_from);
 	}
 
 	if (xml_response_data != NULL) {
 		add_message_xml(reply, F_CRM_DATA, xml_response_data);
 	}
 
 	return reply;
 }
diff --git a/lib/common/utils.c b/lib/common/utils.c
index 9bc9d2c691..f4f54378fd 100644
--- a/lib/common/utils.c
+++ b/lib/common/utils.c
@@ -1,2362 +1,2363 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #ifndef _GNU_SOURCE
 #  define _GNU_SOURCE
 #endif
 
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <sys/stat.h>
 #include <stdio.h>
 #include <unistd.h>
 #include <string.h>
 
 #include <stdlib.h>
 #include <limits.h>
 #include <ctype.h>
 #include <pwd.h>
 #include <grp.h>
 #include <time.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/util.h>
 #include <crm/common/ipc.h>
 #include <crm/common/iso8601.h>
 
 
 #if HAVE_HB_CONFIG_H
 #include <heartbeat/hb_config.h> /* for HB_COREDIR */
 #endif
 
 #if HAVE_GLUE_CONFIG_H
 #include <glue_config.h> /* for HB_COREDIR */
 #endif
 
 #ifndef MAXLINE
 #    define MAXLINE 512
 #endif
 
 #ifdef HAVE_GETOPT_H
 #  include <getopt.h>
 #endif
 
 static uint ref_counter = 0;
 unsigned int crm_log_level = LOG_INFO;
 gboolean crm_config_error = FALSE;
 gboolean crm_config_warning = FALSE;
 const char *crm_system_name = "unknown";
 
 int node_score_red = 0;
 int node_score_green = 0;
 int node_score_yellow = 0;
 int node_score_infinity = INFINITY;
 
 void crm_set_env_options(void);
 
 gboolean
 check_time(const char *value) 
 {
 	if(crm_get_msec(value) < 5000) {
 		return FALSE;
 	}
 	return TRUE;
 }
 
 gboolean
 check_timer(const char *value) 
 {
 	if(crm_get_msec(value) < 0) {
 		return FALSE;
 	}
 	return TRUE;
 }
 
 gboolean
 check_boolean(const char *value) 
 {
 	int tmp = FALSE;
 	if(crm_str_to_boolean(value, &tmp) != 1) {
 		return FALSE;
 	}
 	return TRUE;
 }
 
 gboolean
 check_number(const char *value) 
 {
 	errno = 0;
 	if(value == NULL) {
 		return FALSE;
 		
 	} else if(safe_str_eq(value, MINUS_INFINITY_S)) {
 		
 	} else if(safe_str_eq(value, INFINITY_S)) {
 
 	} else {
 		crm_int_helper(value, NULL);
 	}
 
 	if(errno != 0) {
 		return FALSE;
 	}
 	return TRUE;
 }
 
 int
 char2score(const char *score) 
 {
 	int score_f = 0;
 	
 	if(score == NULL) {
 		
 	} else if(safe_str_eq(score, MINUS_INFINITY_S)) {
 		score_f = -node_score_infinity;
 		
 	} else if(safe_str_eq(score, INFINITY_S)) {
 		score_f = node_score_infinity;
 		
 	} else if(safe_str_eq(score, "+"INFINITY_S)) {
 		score_f = node_score_infinity;
 		
 	} else if(safe_str_eq(score, "red")) {
 		score_f = node_score_red;
 
 	} else if(safe_str_eq(score, "yellow")) {
 		score_f = node_score_yellow;
 
 	} else if(safe_str_eq(score, "green")) {
 		score_f = node_score_green;
 
 	} else {
 		score_f = crm_parse_int(score, NULL);
 		if(score_f > 0 && score_f > node_score_infinity) {
 			score_f = node_score_infinity;
 			
 		} else if(score_f < 0 && score_f < -node_score_infinity) {
 			score_f = -node_score_infinity;
 		}
 	}
 	
 	return score_f;
 }
 
 char *
 score2char(int score) 
 {
 	if(score >= node_score_infinity) {
 		return crm_strdup(INFINITY_S);
 
 	} else if(score <= -node_score_infinity) {
 		return crm_strdup("-"INFINITY_S);
 	} 
 	return crm_itoa(score);
 }
 
 
 const char *
 cluster_option(GHashTable* options, gboolean(*validate)(const char*),
 	       const char *name, const char *old_name, const char *def_value)
 {
 	const char *value = NULL;
 	CRM_ASSERT(name != NULL);
 
 	if(options != NULL) {
 		value = g_hash_table_lookup(options, name);
 	}
 
 	if(value == NULL && old_name && options != NULL) {
 		value = g_hash_table_lookup(options, old_name);
 		if(value != NULL) {
 			crm_config_warn("Using deprecated name '%s' for"
 				       " cluster option '%s'", old_name, name);
 			g_hash_table_insert(
 				options, crm_strdup(name), crm_strdup(value));
 			value = g_hash_table_lookup(options, old_name);
 		}
 	}
 
 	if(value == NULL) {
 		crm_debug_2("Using default value '%s' for cluster option '%s'",
 			    def_value, name);
 
 		if(options == NULL) {
 			return def_value;
 		}
 		
 		g_hash_table_insert(
 			options, crm_strdup(name), crm_strdup(def_value));
 		value = g_hash_table_lookup(options, name);
 	}
 	
 	if(validate && validate(value) == FALSE) {
 		crm_config_err("Value '%s' for cluster option '%s' is invalid."
 			      "  Defaulting to %s", value, name, def_value);
 		g_hash_table_replace(options, crm_strdup(name),
 				     crm_strdup(def_value));
 		value = g_hash_table_lookup(options, name);
 	}
 	
 	return value;
 }
 
 
 const char *
 get_cluster_pref(GHashTable *options, pe_cluster_option *option_list, int len, const char *name)
 {
 	int lpc = 0;
 	const char *value = NULL;
 	gboolean found = FALSE;
 	for(lpc = 0; lpc < len; lpc++) {
 		if(safe_str_eq(name, option_list[lpc].name)) {
 			found = TRUE;
 			value = cluster_option(options, 
 					       option_list[lpc].is_valid,
 					       option_list[lpc].name,
 					       option_list[lpc].alt_name,
 					       option_list[lpc].default_value);
 		}
 	}
 	CRM_CHECK(found, crm_err("No option named: %s", name));
 	CRM_ASSERT(value != NULL);
 	return value;
 }
 
 void
 config_metadata(const char *name, const char *version, const char *desc_short, const char *desc_long,
 		pe_cluster_option *option_list, int len)
 {
 	int lpc = 0;
 
 	fprintf(stdout, "<?xml version=\"1.0\"?>"
 		"<!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n"
 		"<resource-agent name=\"%s\">\n"
 		"  <version>%s</version>\n"
 		"  <longdesc lang=\"en\">%s</longdesc>\n"
 		"  <shortdesc lang=\"en\">%s</shortdesc>\n"
 		"  <parameters>\n", name, version, desc_long, desc_short);
 	
 	for(lpc = 0; lpc < len; lpc++) {
 		if(option_list[lpc].description_long == NULL
 			&& option_list[lpc].description_short == NULL) {
 			continue;
 		}
 		fprintf(stdout, "    <parameter name=\"%s\" unique=\"0\">\n"
 			"      <shortdesc lang=\"en\">%s</shortdesc>\n"
 			"      <content type=\"%s\" default=\"%s\"/>\n"
 			"      <longdesc lang=\"en\">%s%s%s</longdesc>\n"
 			"    </parameter>\n",
 			option_list[lpc].name,
 			option_list[lpc].description_short,
 			option_list[lpc].type,
 			option_list[lpc].default_value,
 			option_list[lpc].description_long?option_list[lpc].description_long:option_list[lpc].description_short,
 			option_list[lpc].values?"  Allowed values: ":"",
 			option_list[lpc].values?option_list[lpc].values:"");
 	}
 	fprintf(stdout, "  </parameters>\n</resource-agent>\n");
 }
 
 void
 verify_all_options(GHashTable *options, pe_cluster_option *option_list, int len)
 {
 	int lpc = 0;
 	for(lpc = 0; lpc < len; lpc++) {
 		cluster_option(options, 
 			       option_list[lpc].is_valid,
 			       option_list[lpc].name,
 			       option_list[lpc].alt_name,
 			       option_list[lpc].default_value);
 	}
 }
 
 char *
 generateReference(const char *custom1, const char *custom2)
 {
 
 	const char *local_cust1 = custom1;
 	const char *local_cust2 = custom2;
 	int reference_len = 4;
 	char *since_epoch = NULL;
 
 	reference_len += 20; /* too big */
 	reference_len += 40; /* too big */
 	
 	if(local_cust1 == NULL) { local_cust1 = "_empty_"; }
 	reference_len += strlen(local_cust1);
 	
 	if(local_cust2 == NULL) { local_cust2 = "_empty_"; }
 	reference_len += strlen(local_cust2);
 	
 	crm_malloc0(since_epoch, reference_len);
 
 	if(since_epoch != NULL) {
 		sprintf(since_epoch, "%s-%s-%ld-%u",
 			local_cust1, local_cust2,
 			(unsigned long)time(NULL), ref_counter++);
 	}
 
 	return since_epoch;
 }
 
 gboolean
 decodeNVpair(const char *srcstring, char separator, char **name, char **value)
 {
 	int lpc = 0;
 	int len = 0;
 	const char *temp = NULL;
 
 	CRM_ASSERT(name != NULL && value != NULL);
 	*name = NULL;
 	*value = NULL;
 
 	crm_debug_4("Attempting to decode: [%s]", srcstring);
 	if (srcstring != NULL) {
 		len = strlen(srcstring);
 		while(lpc <= len) {
 			if (srcstring[lpc] == separator) {
 				crm_malloc0(*name, lpc+1);
 				if(*name == NULL) {
 					break; /* and return FALSE */
 				}
 				strncpy(*name, srcstring, lpc);
 				(*name)[lpc] = '\0';
 
 /* this sucks but as the strtok manpage says..
  * it *is* a bug
  */
 				len = len-lpc; len--;
 				if(len <= 0) {
 					*value = NULL;
 				} else {
 
 					crm_malloc0(*value, len+1);
 					if(*value == NULL) {
 						crm_free(*name);
 						break; /* and return FALSE */
 					}
 					temp = srcstring+lpc+1;
 					strncpy(*value, temp, len);
 					(*value)[len] = '\0';
 				}
 				return TRUE;
 			}
 			lpc++;
 		}
 	}
 
 	if(*name != NULL) {
 		crm_free(*name);
 	}
 	*name = NULL;
 	*value = NULL;
     
 	return FALSE;
 }
 
 char *
 crm_concat(const char *prefix, const char *suffix, char join) 
 {
 	int len = 0;
 	char *new_str = NULL;
 	CRM_ASSERT(prefix != NULL);
 	CRM_ASSERT(suffix != NULL);
 	len = strlen(prefix) + strlen(suffix) + 2;
 
 	crm_malloc0(new_str, (len));
 	sprintf(new_str, "%s%c%s", prefix, join, suffix);
 	new_str[len-1] = 0;
 	return new_str;
 }
 
 
 char *
 generate_hash_key(const char *crm_msg_reference, const char *sys)
 {
 	char *hash_key = crm_concat(sys?sys:"none", crm_msg_reference, '_');
 	crm_debug_3("created hash key: (%s)", hash_key);
 	return hash_key;
 }
 
 char *
 generate_hash_value(const char *src_node, const char *src_subsys)
 {
 	char *hash_value = NULL;
 	
 	if (src_node == NULL || src_subsys == NULL) {
 		return NULL;
 	}
     
 	if (strcasecmp(CRM_SYSTEM_DC, src_subsys) == 0) {
 		hash_value = crm_strdup(src_subsys);
 		CRM_ASSERT(hash_value);
 		return hash_value;
 	}
 
 	hash_value = crm_concat(src_node, src_subsys, '_');
 	crm_info("created hash value: (%s)", hash_value);
 	return hash_value;
 }
 
 char *
 crm_itoa(int an_int)
 {
 	int len = 32;
 	char *buffer = NULL;
 	
 	crm_malloc0(buffer, (len+1));
 	if(buffer != NULL) {
 		snprintf(buffer, len, "%d", an_int);
 	}
 	
 	return buffer;
 }
 
 extern int LogToLoggingDaemon(int priority, const char * buf, int bstrlen, gboolean use_pri_str);
 
 #ifdef HAVE_G_LOG_SET_DEFAULT_HANDLER
 GLogFunc glib_log_default;
 
 static void
 crm_glib_handler(const gchar *log_domain, GLogLevelFlags flags, const gchar *message, gpointer user_data)
 {
 	int log_level = LOG_WARNING;
 	GLogLevelFlags msg_level = (flags & G_LOG_LEVEL_MASK);
 
 	switch(msg_level) {
 	    case G_LOG_LEVEL_CRITICAL:
 		/* log and record how we got here */
 		crm_abort(__FILE__,__PRETTY_FUNCTION__,__LINE__, message, TRUE, TRUE);
 		return;
 
 	    case G_LOG_LEVEL_ERROR:	log_level = LOG_ERR;    break;
 	    case G_LOG_LEVEL_MESSAGE:	log_level = LOG_NOTICE; break;
 	    case G_LOG_LEVEL_INFO:	log_level = LOG_INFO;   break;
 	    case G_LOG_LEVEL_DEBUG:	log_level = LOG_DEBUG;  break;
 		
 	    case G_LOG_LEVEL_WARNING:
 	    case G_LOG_FLAG_RECURSION:
 	    case G_LOG_FLAG_FATAL:
 	    case G_LOG_LEVEL_MASK:
 		log_level = LOG_WARNING;
 		break;
 	}
 
 	do_crm_log(log_level, "%s: %s", log_domain, message);
 }
 #endif
 
 void crm_log_deinit(void) {
 #ifdef HAVE_G_LOG_SET_DEFAULT_HANDLER
     g_log_set_default_handler(glib_log_default, NULL);
 #endif
 }
 
 gboolean
 crm_log_init(
     const char *entity, int level, gboolean coredir, gboolean to_stderr,
     int argc, char **argv)
 {
 	/* Redirect messages from glib functions to our handler */
 /*  	cl_malloc_forced_for_glib(); */
 #ifdef HAVE_G_LOG_SET_DEFAULT_HANDLER
 	glib_log_default = g_log_set_default_handler(crm_glib_handler, NULL);
 #endif
 	
 	/* and for good measure... - this enum is a bit field (!) */
 	g_log_set_always_fatal((GLogLevelFlags)0); /*value out of range*/
 	
 	crm_system_name = entity;
 	setenv("PCMK_service", crm_system_name, 1);
 	cl_log_set_entity(entity);
 	if(argc == 0) {
 	    /* Nuke any syslog activity */
 	    unsetenv("HA_logfacility");
 
 	} else if(getenv("HA_logfacility") == NULL) {
 	    /* Set a default */
 	    cl_log_set_facility(HA_LOG_FACILITY);
 	} /* else: picked up by crm_set_env_options() */
 
 	if(coredir) {
 	    const char *user = getenv("USER");
 	    if(safe_str_neq(user, "root") && safe_str_neq(user, CRM_DAEMON_USER)) {
 		crm_info("Not switching to corefile directory");
 		coredir = FALSE;
 	    }
 	}
 	if(coredir) {
 	    int user = getuid();
 	    struct passwd *pwent = NULL;
 	    const char *base = HA_COREDIR;
 	    
 	    pwent = getpwuid(user);
 
 	    
 	    if (pwent == NULL) {
 		crm_perror(LOG_ERR, "Cannot get name for uid: %d", user);
 
 	    } else if(safe_str_neq(pwent->pw_name, "root")
 		      && safe_str_neq(pwent->pw_name, "nobody")
 		      && safe_str_neq(pwent->pw_name, CRM_DAEMON_USER)) {
 		crm_debug("Don't change active directory for regular user: %s", pwent->pw_name);
 		
 	    } else if (chdir(base) < 0) {
 		crm_perror(LOG_ERR, "Cannot change active directory to %s", base);
 
 	    } else if (chdir(pwent->pw_name) < 0) {
 		crm_perror(LOG_ERR, "Cannot change active directory to %s/%s", base, pwent->pw_name);
 
 	    } else {
 		crm_info("Changed active directory to %s/%s", base, pwent->pw_name);
 	    }
 	}
 	
 	set_crm_log_level(level);
 	crm_set_env_options();
 
 	cl_log_args(argc, argv);
 	cl_log_enable_stderr(to_stderr);
 
 	crm_signal(DEBUG_INC, alter_debug);
 	crm_signal(DEBUG_DEC, alter_debug);
 
 	return TRUE;
 }
 
 /* returns the old value */
 unsigned int
 set_crm_log_level(unsigned int level)
 {
 	unsigned int old = crm_log_level;
 
 	while(crm_log_level < 100 && crm_log_level < level) {
 		alter_debug(DEBUG_INC);
 	}
 	while(crm_log_level > 0 && crm_log_level > level) {
 		alter_debug(DEBUG_DEC);
 	}
 	
 	return old;
 }
 
 unsigned int
 get_crm_log_level(void)
 {
 	return crm_log_level;
 }
 
 static int
 crm_version_helper(const char *text, char **end_text)
 {
 	int atoi_result = -1;
 	CRM_ASSERT(end_text != NULL);
 
 	errno = 0;
 	
 	if(text != NULL && text[0] != 0) {
 	    atoi_result = (int)strtol(text, end_text, 10);
 		
 	    if(errno == EINVAL) {
 		crm_err("Conversion of '%s' %c failed", text, text[0]);
 		atoi_result = -1;
 	    }
 	}
 	return atoi_result;
 }
 
 
 /*
  * version1 < version2 : -1
  * version1 = version2 :  0
  * version1 > version2 :  1
  */
 int
 compare_version(const char *version1, const char *version2)
 {
 	int rc = 0;
 	int lpc = 0;
 	char *ver1_copy = NULL, *ver2_copy = NULL;
 	char *rest1 = NULL, *rest2 = NULL;
 
 	if(version1 == NULL && version2 == NULL) {
 		return 0;
 	} else if(version1 == NULL) {
 		return -1;
 	} else if(version2 == NULL) {
 		return 1;
 	}
 	
 	ver1_copy = crm_strdup(version1);
 	ver2_copy = crm_strdup(version2);
 	rest1 = ver1_copy;
 	rest2 = ver2_copy;
 
 	while(1) {
 		int digit1 = 0;
 		int digit2 = 0;
 
 		lpc++;
 
 		if(rest1 == rest2) {
 		    break;
 		}
 		
 		if(rest1 != NULL) {
 		    digit1 = crm_version_helper(rest1, &rest1);
 		}
 
 		if(rest2 != NULL) {
 		    digit2 = crm_version_helper(rest2, &rest2);
 		}
 
 		if(digit1 < digit2){
 			rc = -1;
 			crm_debug_5("%d < %d", digit1, digit2);
 			break;
 			
 		} else if (digit1 > digit2){
 			rc = 1;
 			crm_debug_5("%d > %d", digit1, digit2);
 			break;
 		}
 
 		if(rest1 != NULL && rest1[0] == '.') {
 		    rest1++;
 		}
 		if(rest1 != NULL && rest1[0] == 0) {
 		    rest1 = NULL;
 		}
 
 		if(rest2 != NULL && rest2[0] == '.') {
 		    rest2++;
 		}
 		if(rest2 != NULL && rest2[0] == 0) {
 		    rest2 = NULL;
 		}
 	}
 	
 	crm_free(ver1_copy);
 	crm_free(ver2_copy);
 
 	if(rc == 0) {
 	    crm_debug_3("%s == %s (%d)", version1, version2, lpc);
 	} else if(rc < 0) {
 	    crm_debug_3("%s < %s (%d)", version1, version2, lpc);
 	} else if(rc > 0) {
 	    crm_debug_3("%s > %s (%d)", version1, version2, lpc);
 	}
 
 	return rc;
 }
 
 gboolean do_stderr = FALSE;
 
 void
 alter_debug(int nsig) 
 {
 	crm_signal(DEBUG_INC, alter_debug);
 	crm_signal(DEBUG_DEC, alter_debug);
 	
 	switch(nsig) {
 		case DEBUG_INC:
 			if (crm_log_level < 100) {
 				crm_log_level++;
 			}
 			break;
 
 		case DEBUG_DEC:
 			if (crm_log_level > 0) {
 				crm_log_level--;
 			}
 			break;	
 
 		default:
 			fprintf(stderr, "Unknown signal %d\n", nsig);
 			cl_log(LOG_ERR, "Unknown signal %d", nsig);
 			break;	
 	}
 }
 
 
 void g_hash_destroy_str(gpointer data)
 {
 	crm_free(data);
 }
 
 #include <sys/types.h>
 /* #include <stdlib.h> */
 /* #include <limits.h> */
 
 long long
 crm_int_helper(const char *text, char **end_text)
 {
     long long result = -1;
     char *local_end_text = NULL;
     int saved_errno = 0;
     
     errno = 0;
     
     if(text != NULL) {
 #ifdef ANSI_ONLY
 	if(end_text != NULL) {
 	    result = strtol(text, end_text, 10);
 	} else {
 	    result = strtol(text, &local_end_text, 10);
 	}
 #else
 	if(end_text != NULL) {
 	    result = strtoll(text, end_text, 10);
 	} else {
 	    result = strtoll(text, &local_end_text, 10);
 	}
 #endif
 
 	saved_errno = errno;
 /* 		CRM_CHECK(errno != EINVAL); */
 	if(errno == EINVAL) {
 	    crm_err("Conversion of %s failed", text);
 	    result = -1;
 	    
 	} else if(errno == ERANGE) {
 	    crm_err("Conversion of %s was clipped: %lld", text, result);
 
 	} else if(errno != 0) {
 	    crm_perror(LOG_ERR,"Conversion of %s failed:", text);
 	}
 			
 	if(local_end_text != NULL && local_end_text[0] != '\0') {
 	    crm_err("Characters left over after parsing '%s': '%s'", text, local_end_text);
 	}
 
 	errno = saved_errno;
     }
     return result;
 }
 
 int
 crm_parse_int(const char *text, const char *default_text)
 {
 	int atoi_result = -1;
 	if(text != NULL) {
 		atoi_result = crm_int_helper(text, NULL);
 		if(errno == 0) {
 			return atoi_result;
 		}
 	}
 	
 	if(default_text != NULL) {
 		atoi_result = crm_int_helper(default_text, NULL);
 		if(errno == 0) {
 			return atoi_result;
 		}
 
 	} else {
 		crm_err("No default conversion value supplied");
 	}
 
 	return -1;
 }
 
 gboolean
 safe_str_neq(const char *a, const char *b)
 {
 	if(a == b) {
 		return FALSE;
 
 	} else if(a==NULL || b==NULL) {
 		return TRUE;
 
 	} else if(strcasecmp(a, b) == 0) {
 		return FALSE;
 	}
 	return TRUE;
 }
 
 char *
 crm_strdup_fn(const char *src, const char *file, const char *fn, int line)
 {
 	char *dup = NULL;
 	CRM_CHECK(src != NULL,
 		  crm_err("Could not perform copy at %s:%d (%s)", file, line, fn);
 		  return NULL);
 	crm_malloc0(dup, strlen(src) + 1);
 	return strcpy(dup, src);
 }
 
 
 
 #define ENV_PREFIX "HA_"
 void
 crm_set_env_options(void) 
 {
 	cl_inherit_logging_environment(500);
 	cl_log_set_logd_channel_source(NULL, NULL);
 
 	if(debug_level > 0 && (debug_level+LOG_INFO) > (int)crm_log_level) {
 	    set_crm_log_level(LOG_INFO + debug_level);
 	}
 }
 
 gboolean
 crm_is_true(const char * s)
 {
 	gboolean ret = FALSE;
 	if(s != NULL) {
 		crm_str_to_boolean(s, &ret);
 	}
 	return ret;
 }
 
 int
 crm_str_to_boolean(const char * s, int * ret)
 {
 	if(s == NULL) {
 		return -1;
 
 	} else if (strcasecmp(s, "true") == 0
 		   ||	strcasecmp(s, "on") == 0
 		   ||	strcasecmp(s, "yes") == 0
 		   ||	strcasecmp(s, "y") == 0
 		   ||	strcasecmp(s, "1") == 0){
 		*ret = TRUE;
 		return 1;
 
 	} else if (strcasecmp(s, "false") == 0
 		   ||	strcasecmp(s, "off") == 0
 		   ||	strcasecmp(s, "no") == 0
 		   ||	strcasecmp(s, "n") == 0
 		   ||	strcasecmp(s, "0") == 0){
 		*ret = FALSE;
 		return 1;
 	}
 	return -1;
 }
 
 #ifndef NUMCHARS
 #    define	NUMCHARS	"0123456789."
 #endif
 
 #ifndef WHITESPACE
 #    define	WHITESPACE	" \t\n\r\f"
 #endif
 
 unsigned long long
 crm_get_interval(const char * input)
 {
     ha_time_t *interval = NULL;
     char *input_copy = crm_strdup(input);
     char *input_copy_mutable = input_copy;
     unsigned long long msec = 0;
     
     if(input == NULL) {
 	return 0;
 
     } else if(input[0] != 'P') {
 	crm_free(input_copy);
 	return crm_get_msec(input);
     }
     
     interval = parse_time_duration(&input_copy_mutable);
     msec = date_in_seconds(interval);
     free_ha_date(interval);
     crm_free(input_copy);
     return msec * 1000;
 }
 
 long long
 crm_get_msec(const char * input)
 {
 	const char *cp = input;
 	const char *units;
 	long long multiplier = 1000;
 	long long divisor = 1;
 	long long msec = -1;
 	char *end_text = NULL;
 	/* double dret; */
 
 	if(input == NULL) {
 		return msec;
 	}
 	
 	cp += strspn(cp, WHITESPACE);
 	units = cp + strspn(cp, NUMCHARS);
 	units += strspn(units, WHITESPACE);
 
 	if (strchr(NUMCHARS, *cp) == NULL) {
 		return msec;
 	}
 
 	if (strncasecmp(units, "ms", 2) == 0
 	    || strncasecmp(units, "msec", 4) == 0) {
 		multiplier = 1;
 		divisor = 1;
 	} else if (strncasecmp(units, "us", 2) == 0
 		  || strncasecmp(units, "usec", 4) == 0) {
 		multiplier = 1;
 		divisor = 1000;
 	} else if (strncasecmp(units, "s", 1) == 0
 		  || strncasecmp(units, "sec", 3) == 0) {
 		multiplier = 1000;
 		divisor = 1;
 	} else if (strncasecmp(units, "m", 1) == 0
 		  || strncasecmp(units, "min", 3) == 0) {
 		multiplier = 60*1000;
 		divisor = 1;
 	} else if (strncasecmp(units, "h", 1) == 0
 		  || strncasecmp(units, "hr", 2) == 0) {
 		multiplier = 60*60*1000;
 		divisor = 1;
 	} else if (*units != EOS && *units != '\n' && *units != '\r') {
 		return msec;
 	}
 	
 	msec = crm_int_helper(cp, &end_text);
 	msec *= multiplier;
 	msec /= divisor;
 	/* dret += 0.5; */
 	/* msec = (long long)dret; */
 	return msec;
 }
 
 const char *
 op_status2text(op_status_t status)
 {
 	switch(status) {
 		case LRM_OP_PENDING:
 			return "pending";
 			break;
 		case LRM_OP_DONE:
 			return "complete";
 			break;
 		case LRM_OP_ERROR:
 			return "Error";
 			break;
 		case LRM_OP_TIMEOUT:
 			return "Timed Out";
 			break;
 		case LRM_OP_NOTSUPPORTED:
 			return "NOT SUPPORTED";
 			break;
 		case LRM_OP_CANCELLED:
 			return "Cancelled";
 			break;
 	}
 	crm_err("Unknown status: %d", status);
 	return "UNKNOWN!";
 }
 
 char *
 generate_op_key(const char *rsc_id, const char *op_type, int interval)
 {
 	int len = 35;
 	char *op_id = NULL;
 
 	CRM_CHECK(rsc_id  != NULL, return NULL);
 	CRM_CHECK(op_type != NULL, return NULL);
 	
 	len += strlen(op_type);
 	len += strlen(rsc_id);
 	crm_malloc0(op_id, len);
 	CRM_CHECK(op_id != NULL, return NULL);
 	sprintf(op_id, "%s_%s_%d", rsc_id, op_type, interval);
 	return op_id;
 }
 
 gboolean
 parse_op_key(const char *key, char **rsc_id, char **op_type, int *interval)
 {
 	char *mutable_key = NULL;
 	char *mutable_key_ptr = NULL;
 	int len = 0, offset = 0, ch = 0;
 
 	CRM_CHECK(key != NULL, return FALSE);
 	
 	*interval = 0;
 	len = strlen(key);
 	offset = len-1;
 
 	crm_debug_3("Source: %s", key);
 	
 	while(offset > 0 && isdigit(key[offset])) {
 		int digits = len-offset;
 		ch = key[offset] - '0';
 		CRM_CHECK(ch < 10, return FALSE);
 		CRM_CHECK(ch >= 0, return FALSE);
 		while(digits > 1) {
 			digits--;
 			ch = ch * 10;
 		}
 		*interval +=  ch;
 		offset--;
 	}
 
 	crm_debug_3("  Interval: %d", *interval);
 	CRM_CHECK(key[offset] == '_', return FALSE);
 
 	mutable_key = crm_strdup(key);
 	mutable_key_ptr = mutable_key_ptr;
 	mutable_key[offset] = 0;
 	offset--;
 
 	while(offset > 0 && key[offset] != '_') {
 		offset--;
 	}
 
 	CRM_CHECK(key[offset] == '_',
 		  crm_free(mutable_key); return FALSE);
 
 	mutable_key_ptr = mutable_key+offset+1;
 
 	crm_debug_3("  Action: %s", mutable_key_ptr);
 	
 	*op_type = crm_strdup(mutable_key_ptr);
 
 	mutable_key[offset] = 0;
 	offset--;
 
 	CRM_CHECK(mutable_key != mutable_key_ptr,
 		  crm_free(mutable_key); return FALSE);
 	
 	crm_debug_3("  Resource: %s", mutable_key);
 	*rsc_id = crm_strdup(mutable_key);
 
 	crm_free(mutable_key);
 	return TRUE;
 }
 
 char *
 generate_notify_key(const char *rsc_id, const char *notify_type, const char *op_type)
 {
 	int len = 12;
 	char *op_id = NULL;
 
 	CRM_CHECK(rsc_id  != NULL, return NULL);
 	CRM_CHECK(op_type != NULL, return NULL);
 	CRM_CHECK(notify_type != NULL, return NULL);
 	
 	len += strlen(op_type);
 	len += strlen(rsc_id);
 	len += strlen(notify_type);
 	crm_malloc0(op_id, len);
 	if(op_id != NULL) {
 		sprintf(op_id, "%s_%s_notify_%s_0", rsc_id, notify_type, op_type);
 	}
 	return op_id;
 }
 
 char *
 generate_transition_magic_v202(const char *transition_key, int op_status)
 {
 	int len = 80;
 	char *fail_state = NULL;
 
 	CRM_CHECK(transition_key != NULL, return NULL);
 	
 	len += strlen(transition_key);
 	
 	crm_malloc0(fail_state, len);
 	if(fail_state != NULL) {
 		snprintf(fail_state, len, "%d:%s", op_status,transition_key);
 	}
 	return fail_state;
 }
 
 char *
 generate_transition_magic(const char *transition_key, int op_status, int op_rc)
 {
 	int len = 80;
 	char *fail_state = NULL;
 
 	CRM_CHECK(transition_key != NULL, return NULL);
 	
 	len += strlen(transition_key);
 	
 	crm_malloc0(fail_state, len);
 	if(fail_state != NULL) {
 		snprintf(fail_state, len, "%d:%d;%s",
 			 op_status, op_rc, transition_key);
 	}
 	return fail_state;
 }
 
 gboolean
 decode_transition_magic(
 	const char *magic, char **uuid, int *transition_id, int *action_id,
 	int *op_status, int *op_rc, int *target_rc)
 {
     int res = 0;
     char *key = NULL;
     gboolean result = TRUE;
 
     CRM_CHECK(magic != NULL, return FALSE);
     CRM_CHECK(op_rc != NULL, return FALSE);
     CRM_CHECK(op_status != NULL, return FALSE);
     
-    crm_malloc0(key, strlen(magic));
+    crm_malloc0(key, strlen(magic)+1);
     res = sscanf(magic, "%d:%d;%s", op_status, op_rc, key);
     if(res != 3) {
 	crm_crit("Only found %d items in: %s", res, magic);
 	result = FALSE;
 	goto bail;
     }
     
     CRM_CHECK(decode_transition_key(key, uuid, transition_id, action_id, target_rc),
 	      result = FALSE;
 	      goto bail;
 	);
     
   bail:
     crm_free(key);
     return result;
 }
 
 char *
 generate_transition_key(int transition_id, int action_id, int target_rc, const char *node)
 {
 	int len = 40;
 	char *fail_state = NULL;
 
 	CRM_CHECK(node != NULL, return NULL);
 	
 	len += strlen(node);
 	
 	crm_malloc0(fail_state, len);
 	if(fail_state != NULL) {
 		snprintf(fail_state, len, "%d:%d:%d:%s",
 			 action_id, transition_id, target_rc, node);
 	}
 	return fail_state;
 }
 
 
 gboolean
 decode_transition_key(
 	const char *key, char **uuid, int *transition_id, int *action_id, int *target_rc)
 {
 	int res = 0;
 	gboolean done = FALSE;
 
 	CRM_CHECK(uuid != NULL, return FALSE);
 	CRM_CHECK(target_rc != NULL, return FALSE);
 	CRM_CHECK(action_id != NULL, return FALSE);
 	CRM_CHECK(transition_id != NULL, return FALSE);
 	
-	crm_malloc0(*uuid, strlen(key));
+	crm_malloc0(*uuid, strlen(key)+1);
 	res = sscanf(key, "%d:%d:%d:%s", action_id, transition_id, target_rc, *uuid);
 	switch(res) {
 	    case 4:
 		/* Post Pacemaker 0.6 */
 		done = TRUE;
 		break;
 	    case 3:
 	    case 2:
 		/* this can be tricky - the UUID might start with an integer */
 
 		/* Until Pacemaker 0.6 */
 		done = TRUE;
 		*target_rc = -1;
 		res = sscanf(key, "%d:%d:%s", action_id, transition_id, *uuid);
 		if(res == 2) {
 		    *action_id = -1;
 		    res = sscanf(key, "%d:%s", transition_id, *uuid);
 		    CRM_CHECK(res == 2, done = FALSE);
 
 		} else if(res != 3) {
 		    CRM_CHECK(res == 3, done = FALSE);
 		}
 		break;
 		
 	    case 1:
 		/* Prior to Heartbeat 2.0.8 */
 		done = TRUE;
 		*action_id = -1;
 		*target_rc = -1;
 		res = sscanf(key, "%d:%s", transition_id, *uuid);
 		CRM_CHECK(res == 2, done = FALSE);
 		break;
 	    default:
 		crm_crit("Unhandled sscanf result (%d) for %s", res, key);
 		
 	}
 
 	if(strlen(*uuid) != 36) {
 	    crm_warn("Bad UUID (%s) in sscanf result (%d) for %s", *uuid, res, key);		    
 	}
 	
 	if(done == FALSE) {
 	    crm_err("Cannot decode '%s' rc=%d", key, res);
 	    
 	    crm_free(*uuid);
 	    *uuid = NULL;
 	    *target_rc = -1;
 	    *action_id = -1;
 	    *transition_id = -1;
 	}
 	
 	return done;
 }
 
 void
 filter_action_parameters(xmlNode *param_set, const char *version) 
 {
 	char *key = NULL;
 	char *timeout = NULL;
 	char *interval = NULL;
 #if CRM_DEPRECATED_SINCE_2_0_5
 	const char *filter_205[] = {
 		XML_ATTR_TE_TARGET_RC,
 		XML_ATTR_LRM_PROBE,
 		XML_RSC_ATTR_START,
 		XML_RSC_ATTR_NOTIFY,
 		XML_RSC_ATTR_UNIQUE,
 		XML_RSC_ATTR_MANAGED,
 		XML_RSC_ATTR_PRIORITY,
 		XML_RSC_ATTR_MULTIPLE,
 		XML_RSC_ATTR_STICKINESS,
 		XML_RSC_ATTR_FAIL_STICKINESS,
 		XML_RSC_ATTR_TARGET_ROLE,
 
 /* ignore clone fields */
 		XML_RSC_ATTR_INCARNATION, 
 		XML_RSC_ATTR_INCARNATION_MAX, 
 		XML_RSC_ATTR_INCARNATION_NODEMAX,
 		XML_RSC_ATTR_MASTER_MAX,
 		XML_RSC_ATTR_MASTER_NODEMAX,
 		
 /* old field names */
 		"role",
 		"crm_role",
 		"te-target-rc",
 		
 /* ignore notify fields */
  		"notify_stop_resource",
  		"notify_stop_uname",
  		"notify_start_resource",
  		"notify_start_uname",
  		"notify_active_resource",
  		"notify_active_uname",
  		"notify_inactive_resource",
  		"notify_inactive_uname",
  		"notify_promote_resource",
  		"notify_promote_uname",
  		"notify_demote_resource",
  		"notify_demote_uname",
  		"notify_master_resource",
  		"notify_master_uname",
  		"notify_slave_resource",
  		"notify_slave_uname"		
 	};
 #endif
 	
 	const char *attr_filter[] = {
 		XML_ATTR_ID,
 		XML_ATTR_CRM_VERSION,
 		XML_LRM_ATTR_OP_DIGEST,
 	};
 
 	gboolean do_delete = FALSE;
 	int lpc = 0;
 	static int meta_len = 0;
 	if(meta_len == 0) {
 		meta_len  = strlen(CRM_META);
 	}	
 	
 	if(param_set == NULL) {
 		return;
 	}
 
 #if CRM_DEPRECATED_SINCE_2_0_5
  	if(version == NULL || compare_version("1.0.5", version) > 0) {
 		for(lpc = 0; lpc < DIMOF(filter_205); lpc++) {
 			xml_remove_prop(param_set, filter_205[lpc]); 
 		}
 	}
 #endif
 
 	for(lpc = 0; lpc < DIMOF(attr_filter); lpc++) {
 		xml_remove_prop(param_set, attr_filter[lpc]); 
 	}
 
 	key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
 	interval = crm_element_value_copy(param_set, key);
 	crm_free(key);
 
 	key = crm_meta_name(XML_ATTR_TIMEOUT);
 	timeout = crm_element_value_copy(param_set, key);
 	
-	xml_prop_iter(param_set, prop_name, prop_value,      
+	xml_prop_name_iter(param_set, prop_name,      
 		      do_delete = FALSE;
 		      if(strncasecmp(prop_name, CRM_META, meta_len) == 0) {
 			      do_delete = TRUE;
 		      }
 
 		      if(do_delete) {
 			      xml_remove_prop(param_set, prop_name);
 		      }
 		);
 
 	if(crm_get_msec(interval) > 0 && compare_version(version, "1.0.8") > 0) {
 		/* Re-instate the operation's timeout value */
 		if(timeout != NULL) {
 			crm_xml_add(param_set, key, timeout);
 		}
 	}
 
 	crm_free(interval);
 	crm_free(timeout);
 	crm_free(key);
 }
 
 void
 filter_reload_parameters(xmlNode *param_set, const char *restart_string) 
 {
 	int len = 0;
 	char *name = NULL;
 	char *match = NULL;
 	
 	if(param_set == NULL) {
 		return;
 	}
 
-	xml_prop_iter(param_set, prop_name, prop_value,      
+	xml_prop_name_iter(param_set, prop_name,
 		      name = NULL;
 		      len = strlen(prop_name) + 3;
 
 		      crm_malloc0(name, len);
 		      sprintf(name, " %s ", prop_name);
 		      name[len-1] = 0;
 		      
 		      match = strstr(restart_string, name);
 		      if(match == NULL) {
 			      crm_debug_3("%s not found in %s",
 					  prop_name, restart_string);
 			      xml_remove_prop(param_set, prop_name);
 		      }
 		      crm_free(name);
 		);
 }
 
 void
 crm_abort(const char *file, const char *function, int line,
 	  const char *assert_condition, gboolean do_core, gboolean do_fork)
 {
 	int rc = 0;
 	int pid = 0;
 	int status = 0;
 
 	if(do_core == FALSE) {
 	    do_crm_log(LOG_ERR, "%s: Triggered assert at %s:%d : %s",
 		       function, file, line, assert_condition);
 	    return;
 
 	} else if(do_fork) {
 	    pid=fork();
 
 	} else {
 	    do_crm_log(LOG_ERR, "%s: Triggered fatal assert at %s:%d : %s",
 		       function, file, line, assert_condition);
 	}
 	
 	switch(pid) {
 		case -1:
 			do_crm_log(LOG_CRIT, "%s: Cannot create core for non-fatal assert at %s:%d : %s",
 				   function, file, line, assert_condition);
 			return;
 
 		default:	/* Parent */
 			do_crm_log(LOG_ERR, 
 				   "%s: Forked child %d to record non-fatal assert at %s:%d : %s",
 				   function, pid, file, line, assert_condition);
 			do {
 			    rc = waitpid(pid, &status, 0);
 			    if(rc < 0 && errno != EINTR) {
 				crm_perror(LOG_ERR,"%s: Cannot wait on forked child %d", function, pid);
 			    }
 			    
 			} while(rc < 0 && errno == EINTR);
 			    
 			return;
 
 		case 0:	/* Child */
 			abort();
 			break;
 	}
 }
 
 char *
 generate_series_filename(
 	const char *directory, const char *series, int sequence, gboolean bzip)
 {
 	int len = 40;
 	char *filename = NULL;
 	const char *ext = "raw";
 
 	CRM_CHECK(directory  != NULL, return NULL);
 	CRM_CHECK(series != NULL, return NULL);
 	
 	len += strlen(directory);
 	len += strlen(series);
 	crm_malloc0(filename, len);
 	CRM_CHECK(filename != NULL, return NULL);
 
 	if(bzip) {
 		ext = "bz2";
 	}
 	sprintf(filename, "%s/%s-%d.%s", directory, series, sequence, ext);
 	
 	return filename;
 }
 
 int
 get_last_sequence(const char *directory, const char *series)
 {
 	FILE *file_strm = NULL;
 	int start = 0, length = 0, read_len = 0;
 	char *series_file = NULL;
 	char *buffer = NULL;
 	int seq = 0;
 	int len = 36;
 
 	CRM_CHECK(directory  != NULL, return 0);
 	CRM_CHECK(series != NULL, return 0);
 	
 	len += strlen(directory);
 	len += strlen(series);
 	crm_malloc0(series_file, len);
 	CRM_CHECK(series_file != NULL, return 0);
 	sprintf(series_file, "%s/%s.last", directory, series);
 	
 	file_strm = fopen(series_file, "r");
 	if(file_strm == NULL) {
 		crm_debug("Series file %s does not exist", series_file);
 		crm_free(series_file);
 		return 0;
 	}
 	
 	/* see how big the file is */
 	start  = ftell(file_strm);
 	fseek(file_strm, 0L, SEEK_END);
 	length = ftell(file_strm);
 	fseek(file_strm, 0L, start);
 	
+	CRM_ASSERT(length >= 0);
 	CRM_ASSERT(start == ftell(file_strm));
 
 	crm_debug_3("Reading %d bytes from file", length);
 	crm_malloc0(buffer, (length+1));
 	read_len = fread(buffer, 1, length, file_strm);
 
 	if(read_len != length) {
 		crm_err("Calculated and read bytes differ: %d vs. %d",
 			length, read_len);
 		crm_free(buffer);
 		buffer = NULL;
 		
 	} else  if(length <= 0) {
 		crm_info("%s was not valid", series_file);
 		crm_free(buffer);
 		buffer = NULL;
 	}
 	
 	crm_free(series_file);
 	seq = crm_parse_int(buffer, "0");
 	crm_free(buffer);
 	fclose(file_strm);
 	return seq;
 }
 
 void
 write_last_sequence(
 	const char *directory, const char *series, int sequence, int max)
 {
 	int rc = 0;
 	int len = 36;
 	FILE *file_strm = NULL;
 	char *series_file = NULL;
 
 	CRM_CHECK(directory  != NULL, return);
 	CRM_CHECK(series != NULL, return);
 
 	if(max == 0) {
 		return;
 	}
 	while(max > 0 && sequence > max) {
 		sequence -= max;
 	}
 	
 	len += strlen(directory);
 	len += strlen(series);
 	crm_malloc0(series_file, len);
 	sprintf(series_file, "%s/%s.last", directory, series);
 	
 	file_strm = fopen(series_file, "w");
 	if(file_strm == NULL) {
 		crm_err("Cannout open series file %s for writing", series_file);
 		goto bail;
 	}
 
 	rc = fprintf(file_strm, "%d", sequence);
 	if(rc < 0) {
 		crm_perror(LOG_ERR,"Cannot write to series file %s", series_file);
 	}
 
   bail:
 	if(file_strm != NULL) {
 		fflush(file_strm);
 		fclose(file_strm);
 	}
 	
 	crm_free(series_file);
 }
 
 #define	LOCKSTRLEN	11
 
 int crm_pid_active(long pid)
 {
     int rc = 0;
     int running = 0;
     char proc_path[PATH_MAX], exe_path[PATH_MAX], myexe_path[PATH_MAX];
 
     if(pid <= 0) {
 	return -1;
 
     } else if (kill(pid, 0) < 0 && errno == ESRCH) {
 	return 0;
     }
 
 #ifndef HAVE_PROC_PID
     return 1;
 #endif
 	
     /* check to make sure pid hasn't been reused by another process */
     snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", pid);
 	
     rc = readlink(proc_path, exe_path, PATH_MAX-1);
     if(rc < 0) {
 	crm_perror(LOG_ERR, "Could not read from %s", proc_path);
 	goto bail;
     }
 
     exe_path[rc] = 0;
     snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", (long unsigned int)getpid());
     rc = readlink(proc_path, myexe_path, PATH_MAX-1);
     if(rc < 0) {
 	crm_perror(LOG_ERR, "Could not read from %s", proc_path);
 	goto bail;
     }
     
     myexe_path[rc] = 0;
     if(strcmp(exe_path, myexe_path) == 0) {
 	running = 1;
     }
 
   bail:
     return running;
 }
 
 
 int
 crm_read_pidfile(const char *filename)
 {
     int fd;
     long pid = -1;
     char  buf[LOCKSTRLEN+1];
     if ((fd = open(filename, O_RDONLY)) < 0) {
 	goto bail;
     }
     
     if (read(fd, buf, sizeof(buf)) < 1) {
 	goto bail;
     } 
     
     if (sscanf(buf, "%lu", &pid) > 0) {
 	if (pid <= 0){
 	    pid = -LSB_STATUS_STOPPED;
 	}
     }
     
   bail:
-    close(fd);
+    if(fd >= 0) { close(fd); }
     return pid;
 }
 
 int
 crm_lock_pidfile(const char *filename)
 {
     struct stat sbuf;
     int fd = 0, rc = 0;
     long pid = 0, mypid = 0;
     char lf_name[256], tf_name[256], buf[LOCKSTRLEN+1];
 
     mypid = (unsigned long) getpid();
     snprintf(lf_name, sizeof(lf_name), "%s",filename);
     snprintf(tf_name, sizeof(tf_name), "%s.%lu", filename, mypid);
 	
     if ((fd = open(lf_name, O_RDONLY)) >= 0) {
 	if (fstat(fd, &sbuf) >= 0 && sbuf.st_size < LOCKSTRLEN) {
 	    sleep(1); /* if someone was about to create one,
 		       * give'm a sec to do so
 		       * Though if they follow our protocol,
 		       * this won't happen.  They should really
 		       * put the pid in, then link, not the
 		       * other way around.
 		       */
 	}
 	if (read(fd, buf, sizeof(buf)) > 0) {
 	    if (sscanf(buf, "%lu", &pid) > 0) {
 		if (pid > 1 && pid != getpid() && crm_pid_active(pid)) {
 		    /* locked by existing process - give up */
 		    close(fd);
 		    return -1;
 		}
 	    }
 	}
 	unlink(lf_name);
 	close(fd);
     }
     
     if ((fd = open(tf_name, O_CREAT | O_WRONLY | O_EXCL, 0644)) < 0) {
 	/* Hmmh, why did we fail? Anyway, nothing we can do about it */
 	return -3;
     }
 
     /* Slight overkill with the %*d format ;-) */
     snprintf(buf, sizeof(buf), "%*lu\n", LOCKSTRLEN-1, mypid);
 
     if (write(fd, buf, LOCKSTRLEN) != LOCKSTRLEN) {
 	/* Again, nothing we can do about this */
 	rc = -3;
 	close(fd);
 	goto out;
     }
     close(fd);
 
     switch (link(tf_name, lf_name)) {
 	case 0:
 	    if (stat(tf_name, &sbuf) < 0) {
 		/* something weird happened */
 		rc = -3;
 
 	    } else if (sbuf.st_nlink < 2) {
 		/* somehow, it didn't get through - NFS trouble? */
 		rc = -2;
 
 	    } else {
 		rc = 0;
 	    }
 	    break;
 
 	case EEXIST:
 	    rc = -1;
 	    break;
 
 	default:
 	    rc = -3;
     }
   out:
     unlink(tf_name);
     return rc;
 }
 
 void
 crm_make_daemon(const char *name, gboolean daemonize, const char *pidfile)
 {
     long pid;
     const char *devnull = "/dev/null";
 
     if(daemonize == FALSE) {
 	return;
     }
 	
     pid = fork();
     if (pid < 0) {
 	fprintf(stderr, "%s: could not start daemon\n", name);
 	crm_perror(LOG_ERR,"fork");
 	exit(LSB_EXIT_GENERIC);
 
     } else if (pid > 0) {
 	exit(LSB_EXIT_OK);
     }
 
     if (crm_lock_pidfile(pidfile) < 0 ) {
 	pid = crm_read_pidfile(pidfile);
 	if(crm_pid_active(pid) > 0) {
 	    crm_warn("%s: already running [pid %ld] (%s).\n", name, pid, pidfile);
 	    exit(LSB_EXIT_OK);
 	}
     }
 	
     umask(022);
     close(STDIN_FILENO);
     (void)open(devnull, O_RDONLY);		/* Stdin:  fd 0 */
     close(STDOUT_FILENO);
     (void)open(devnull, O_WRONLY);		/* Stdout: fd 1 */
     close(STDERR_FILENO);
     (void)open(devnull, O_WRONLY);		/* Stderr: fd 2 */
 }
 
 gboolean
 crm_is_writable(const char *dir, const char *file,
 		const char *user, const char *group, gboolean need_both)
 {
 	int s_res = -1;
 	struct stat buf;
 	char *full_file = NULL;
 	const char *target = NULL;
 	
 	gboolean pass = TRUE;
 	gboolean readwritable = FALSE;
 
 	CRM_ASSERT(dir != NULL);
 	if(file != NULL) {
 		full_file = crm_concat(dir, file, '/');
 		target = full_file;
 		s_res = stat(full_file, &buf);
 		if( s_res == 0 && S_ISREG(buf.st_mode) == FALSE ) {
 			crm_err("%s must be a regular file", target);
 			pass = FALSE;
 			goto out;
 		}
 	}
 	
 	if (s_res != 0) {
 		target = dir;
 		s_res = stat(dir, &buf);
 		if(s_res != 0) {
 			crm_err("%s must exist and be a directory", dir);
 			pass = FALSE;
 			goto out;
 
 		} else if( S_ISDIR(buf.st_mode) == FALSE ) {
 			crm_err("%s must be a directory", dir);
 			pass = FALSE;
 		}
 	}
 
 	if(user) {
 		struct passwd *sys_user = NULL;
 		sys_user = getpwnam(user);
 		readwritable = (sys_user != NULL
 				&& buf.st_uid == sys_user->pw_uid
 				&& (buf.st_mode & (S_IRUSR|S_IWUSR)));
 		if(readwritable == FALSE) {
 			crm_err("%s must be owned and r/w by user %s",
 				target, user);
 			if(need_both) {
 				pass = FALSE;
 			}
 		}
 	}	
 
 	if(group) {
 		struct group *sys_grp = getgrnam(group);
 		readwritable = (
 			sys_grp != NULL
 			&& buf.st_gid == sys_grp->gr_gid
 			&& (buf.st_mode & (S_IRGRP|S_IWGRP)));		
 		if(readwritable == FALSE) {
 			if(need_both || user == NULL) {
 				pass = FALSE;
 				crm_err("%s must be owned and r/w by group %s",
 					target, group);
 			} else {
 				crm_warn("%s should be owned and r/w by group %s",
 					 target, group);
 			}
 		}
 	}
 
   out:
 	crm_free(full_file);
 	return pass;
 }
 
 static unsigned long long crm_bit_filter = 0; /* 0x00000002ULL; */
 static unsigned int bit_log_level = LOG_DEBUG_5;
 
 long long
 crm_clear_bit(const char *function, long long word, long long bit)
 {
 	unsigned int level = bit_log_level;
 	if(bit & crm_bit_filter) {
 	    level = LOG_ERR;
 	}
 
 	do_crm_log_unlikely(level, "Bit 0x%.16llx cleared by %s", bit, function);
 	word &= ~bit;
 
 	return word;
 }
 
 long long
 crm_set_bit(const char *function, long long word, long long bit)
 {
 	unsigned int level = bit_log_level;
 	if(bit & crm_bit_filter) {
 	    level = LOG_ERR;
 	}
 
 	do_crm_log_unlikely(level, "Bit 0x%.16llx set by %s", bit, function);
 	word |= bit;
 	return word;
 }
 
 static const char *cluster_type = NULL;
 
 gboolean is_openais_cluster(void)
 {
     if(cluster_type == NULL) {
 	cluster_type = getenv("HA_cluster_type");
 	if(cluster_type == NULL) {
 	    cluster_type = "Heartbeat";
 	}
     }
     
     if(safe_str_eq("openais", cluster_type)) {
 #if SUPPORT_AIS
 	return TRUE;
 #else
 	crm_crit("The installation of Pacemaker only supports Heartbeat"
 		 " but you're trying to run it on %s.  Terminating.",
 		 cluster_type);
 	exit(100);
 #endif
     }
     return FALSE;
 }
 
 gboolean is_heartbeat_cluster(void)
 {
 #if SUPPORT_HEARTBEAT
     return !is_openais_cluster();
 #else
     if(is_openais_cluster() == FALSE) {
 	crm_crit("The installation of Pacemaker only supports OpenAIS"
 		 " but you're trying to run it on %s.  Terminating.",
 		 cluster_type);
 	exit(100);
     }
     return FALSE;
 #endif
 }
 
 gboolean crm_str_eq(const char *a, const char *b, gboolean use_case) 
 {
     if(a == b) {
 	return TRUE;
 	
     } else if(a == NULL || b == NULL) {
 	/* shouldn't be comparing NULLs */
 	return FALSE;
 	    
     } else if(use_case && a[0] != b[0]) {
 	return FALSE;		
 	
     } else if(strcasecmp(a, b) == 0) {
 	return TRUE;
     }
     return FALSE;
 }
 
 char *crm_meta_name(const char *field) 
 {
     int lpc = 0;
     int max = 0;
     char *crm_name = NULL;
 
     CRM_CHECK(field != NULL, return NULL);
     crm_name = crm_concat(CRM_META, field, '_');
 
     /* Massage the names so they can be used as shell variables */ 
     max = strlen(crm_name);
     for(; lpc < max; lpc++) {
 	switch(crm_name[lpc]) {
 	    case '-':
 		crm_name[lpc] = '_';
 		break;
 	}
     }
     return crm_name;
 }
 
 const char *crm_meta_value(GHashTable *hash, const char *field) 
 {
     char *key = NULL;
     const char *value = NULL;
 
     key = crm_meta_name(field);
     if(key) {
 	value = g_hash_table_lookup(hash, key);
 	crm_free(key);
     }
     
     return value;
 }
 
 static struct crm_option *crm_long_options = NULL;
 static const char *crm_app_description = NULL;
 static const char *crm_short_options = NULL;
 static const char *crm_app_usage = NULL;
 
 static struct option *crm_create_long_opts(struct crm_option *long_options) 
 {
     struct option *long_opts = NULL;
 
 #ifdef HAVE_GETOPT_H
     int index = 0, lpc = 0;
 
     /*
      * A previous, possibly poor, choice of '?' as the short form of --help
      * means that getopt_long() returns '?' for both --help and for "unknown option"
      *
      * This dummy entry allows us to differentiate between the two in crm_get_option()
      * and exit with the correct error code
      */
     crm_realloc(long_opts, (index+1) * sizeof(struct option));
     long_opts[index].name = "__dummmy__";
     long_opts[index].has_arg = 0;
     long_opts[index].flag = 0;
     long_opts[index].val = '_';
     index++;
     
     for(lpc = 0; long_options[lpc].name != NULL; lpc++) {
 	if(long_options[lpc].name[0] == '-') {
 	    continue;
 	}
 	
 	crm_realloc(long_opts, (index+1) * sizeof(struct option));
 	/*fprintf(stderr, "Creating %d %s = %c\n", index,
 	 * long_options[lpc].name, long_options[lpc].val);	*/
 	long_opts[index].name = long_options[lpc].name;
 	long_opts[index].has_arg = long_options[lpc].has_arg;
 	long_opts[index].flag = long_options[lpc].flag;
 	long_opts[index].val = long_options[lpc].val;
 	index++;
     }
 
     /* Now create the list terminator */
     crm_realloc(long_opts, (index+1) * sizeof(struct option));
     long_opts[index].name = NULL;
     long_opts[index].has_arg = 0;
     long_opts[index].flag = 0;
     long_opts[index].val = 0;
 #endif
     
     return long_opts;
 }
 
 void crm_set_options(const char *short_options, const char *app_usage, struct crm_option *long_options, const char *app_desc) 
 {
     if(short_options) {
 	crm_short_options = short_options;
     }
     if(long_options) {
 	crm_long_options = long_options;
     }
     if(app_desc) {
 	crm_app_description = app_desc;
     }
     if(app_usage) {
 	crm_app_usage = app_usage;
     }
 }
 
 int crm_get_option(int argc, char **argv, int *index) 
 {
 #ifdef HAVE_GETOPT_H
     static struct option *long_opts = NULL;
     if(long_opts == NULL && crm_long_options) {
 	long_opts = crm_create_long_opts(crm_long_options);
     }
     
     if(long_opts) {
 	int flag = getopt_long(argc, argv, crm_short_options, long_opts, index);
 	switch(flag) {
 	    case 0: return long_opts[*index].val;
 	    case -1:  /* End of option processing */ break;
 	    case ':': crm_debug_2("Missing argument"); crm_help('?', 1); break;
 	    case '?': crm_help('?', *index?0:1); break;
 	}
 	return flag;
     }
 #endif
 
     if(crm_short_options) {
 	return getopt(argc, argv, crm_short_options);
     }
     
     return -1;
 }
 
 void crm_help(char cmd, int exit_code) 
 {
     int i = 0;
     FILE *stream = (exit_code ? stderr : stdout);
 
     if(cmd == 'v' || cmd == '$') {
 	fprintf(stream, "%s %s for %s (Build: %s)\n",
 		crm_system_name, VERSION,
 		#if !SUPPORT_HEARTBEAT
 		"OpenAIS",
 		#elif !SUPPORT_AIS
 		"Heartbeat",
 		#else
 		"OpenAIS and Heartbeat",
 		#endif
 		BUILD_VERSION);
 	fprintf(stream, "\nWritten by Andrew Beekhof\n");
 	goto out;
     }
     
     fprintf(stream, "%s - %s\n", crm_system_name, crm_app_description);
 
     if(crm_app_usage) {
 	fprintf(stream, "Usage: %s %s\n", crm_system_name, crm_app_usage);
     }
 	
     if(crm_long_options) {
 	fprintf(stream, "Options:\n");
 	for(i = 0; crm_long_options[i].name != NULL; i++) {
 	    if(crm_long_options[i].flags & pcmk_option_hidden) {
 
 	    } else if(crm_long_options[i].flags & pcmk_option_paragraph) {
 		fprintf(stream, "%s\n\n", crm_long_options[i].desc);
 
 	    } else if(crm_long_options[i].flags & pcmk_option_example) {
 		fprintf(stream, "\t#%s\n\n", crm_long_options[i].desc);
 
 	    } else if(crm_long_options[i].val == '-' && crm_long_options[i].desc) {
 		fprintf(stream, "%s\n", crm_long_options[i].desc);
 		
 	    } else {
 		fprintf(stream, " -%c, --%s%c%s\t%s\n", crm_long_options[i].val, crm_long_options[i].name,
 			crm_long_options[i].has_arg?'=':' ',crm_long_options[i].has_arg?"value":"",
 			crm_long_options[i].desc?crm_long_options[i].desc:"");
 	    }
 	}
 	    
     } else if(crm_short_options) {
 	fprintf(stream, "Usage: %s - %s\n", crm_system_name, crm_app_description);
 	for(i = 0; crm_short_options[i] != 0; i++) {
 	    int has_arg = FALSE;
 		
 	    if(crm_short_options[i+1] == ':') {
 		has_arg = TRUE;
 	    }
 		
 	    fprintf(stream, " -%c %s\n", crm_short_options[i], has_arg?"{value}":"");
 	    if(has_arg) {
 		i++;
 	    }
 	}
     }
 
     fprintf(stream, "\nReport bugs to %s\n", PACKAGE_BUGREPORT);
 
   out:
     if(exit_code >= 0) {
 	exit(exit_code);
     }
 }
 
 #include <../../tools/attrd.h>
 gboolean attrd_update(IPC_Channel *cluster, char command, const char *host, const char *name, const char *value, const char *section, const char *set, const char *dampen) 
 {
     gboolean success = FALSE;
     const char *reason = "Cluster connection failed";
 
     /* remap common aliases */
     if(safe_str_eq(section, "reboot")) {
 	section = XML_CIB_TAG_STATUS;
 	
     } else if(safe_str_eq(section, "forever")) {
 	section = XML_CIB_TAG_NODES;
     }
     
     if(cluster == NULL) {
 	reason = "No connection to the cluster";
 
     } else {
 	xmlNode *update = create_xml_node(NULL, __FUNCTION__);
 	crm_xml_add(update, F_TYPE, T_ATTRD);
 	crm_xml_add(update, F_ORIG, crm_system_name);
 
 	if(name == NULL && command == 'U') {
 	    command = 'R';
 	}
 	
 	switch(command) {
 	    case 'D':
 	    case 'U':
 	    case 'v':
 		crm_xml_add(update, F_ATTRD_TASK, "update");
 		crm_xml_add(update, F_ATTRD_ATTRIBUTE, name);
 		break;
 	    case 'R':
 		crm_xml_add(update, F_ATTRD_TASK, "refresh");
 		break;
 	    case 'q':
 		crm_xml_add(update, F_ATTRD_TASK, "query");
 		break;
 	}
 	
 	crm_xml_add(update, F_ATTRD_VALUE, value);
 	crm_xml_add(update, F_ATTRD_DAMPEN, dampen);
 	crm_xml_add(update, F_ATTRD_SECTION, section);
 	crm_xml_add(update, F_ATTRD_HOST, host);
 	crm_xml_add(update, F_ATTRD_SET, set);
 
 	success = send_ipc_message(cluster, update);
 	free_xml(update);
     }
  
     if(success) {
 	crm_debug("Sent update: %s=%s for %s", name, value, host?host:"localhost");
 	return TRUE;
     }
 
     crm_info("Could not send update: %s=%s for %s", name, value, host?host:"localhost");
     return FALSE;
 }
 
 gboolean attrd_lazy_update(char command, const char *host, const char *name, const char *value, const char *section, const char *set, const char *dampen) 
 {
     int max = 5;
     gboolean updated = FALSE;
     static IPC_Channel *cluster = NULL;
 
     while(updated == 0 && max > 0) {
 	if(cluster == NULL) {
 	    crm_info("Connecting to cluster... %d retries remaining", max);
 	    cluster = init_client_ipc_comms_nodispatch(T_ATTRD);
 	}
 
 	if(cluster != NULL) {
 	    updated = attrd_update(cluster, command, host, name, value, section, set, dampen);
 	}
 	
 	if(updated == 0) {
 	    cluster = NULL;
 	    sleep(2);
 	    max--;
 	}
     }
 
     return updated;
 }
 
 gboolean attrd_update_no_mainloop(int *connection, char command, const char *host, const char *name, const char *value, const char *section, const char *set, const char *dampen) 
 {
     int max = 5;
     gboolean updated = FALSE;
     static IPC_Channel *cluster = NULL;
 
     if(connection && *connection == 0 && cluster) {
 	crm_info("Forcing a new connection to the cluster");
 	cluster = NULL;
     }
     
     while(updated == 0 && max > 0) {
 	if(cluster == NULL) {
 	    crm_info("Connecting to cluster... %d retries remaining", max);
 	    cluster = init_client_ipc_comms_nodispatch(T_ATTRD);
 	}
 
 	if(connection) {
 	    if(cluster != NULL) {
 		*connection = cluster->ops->get_recv_select_fd(cluster);
 	    } else {
 		*connection = 0;
 	    }
 	}
 	
 	if(cluster != NULL) {
 	    updated = attrd_update(cluster, command, host, name, value, section, set, dampen);
 	}
 	
 	if(updated == 0) {
 	    cluster = NULL;
 	    sleep(2);
 	    max--;
 	}
     }
     return updated;
 }
 
 #define FAKE_TE_ID	"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
 static void
 append_digest(lrm_op_t *op, xmlNode *update, const char *version, const char *magic, int level) 
 {
     /* this will enable us to later determine that the
      *   resource's parameters have changed and we should force
      *   a restart
      */
     char *digest = NULL;
     xmlNode *args_xml = NULL;
 
     if(op->params == NULL) {
 	return;
     }
     
     args_xml = create_xml_node(NULL, XML_TAG_PARAMS);
     g_hash_table_foreach(op->params, hash2field, args_xml);
     filter_action_parameters(args_xml, version);
     digest = calculate_xml_digest(args_xml, TRUE, FALSE);
 
 #if 0
     if(level < crm_log_level
        && op->interval == 0
        && crm_str_eq(op->op_type, CRMD_ACTION_START, TRUE)) {
 	char *digest_source = dump_xml_unformatted(args_xml);
 	do_crm_log(level, "Calculated digest %s for %s (%s). Source: %s\n", 
 		   digest, ID(update), magic, digest_source);
 	crm_free(digest_source);
     }
 #endif
     crm_xml_add(update, XML_LRM_ATTR_OP_DIGEST, digest);
 
     free_xml(args_xml);
     crm_free(digest);
 }
 
 xmlNode *
 create_operation_update(
     xmlNode *parent, lrm_op_t *op, const char *caller_version, int target_rc, const char *origin, int level)
 {
     char *magic = NULL;
     const char *task = NULL;
     xmlNode *xml_op = NULL;
     char *op_id = NULL;
     char *local_user_data = NULL;
 
     CRM_CHECK(op != NULL, return NULL);
     do_crm_log(level, "%s: Updating resouce %s after %s %s op (interval=%d)",
 	       origin, op->rsc_id, op_status2text(op->op_status), op->op_type, op->interval);
 
     if(op->op_status == LRM_OP_CANCELLED) {
 	crm_debug_3("Ignoring cancelled op");
 	return NULL;
     }
 
     crm_debug_3("DC version: %s", caller_version);
 
     task = op->op_type;
     /* remap the task name under various scenarios
      * this makes life easier for the PE when its trying determin the current state 
      */
     if(crm_str_eq(task, "reload", TRUE)) {
 	if(op->op_status == LRM_OP_DONE) {
 	    task = CRMD_ACTION_START;
 	} else {
 	    task = CRMD_ACTION_STATUS;
 	}
 
     } else if(crm_str_eq(task, CRMD_ACTION_MIGRATE, TRUE)) {
 	/* if the migrate_from fails it will have enough info to do the right thing */
 	if(op->op_status == LRM_OP_DONE) {
 	    task = CRMD_ACTION_STOP;
 	} else {
 	    task = CRMD_ACTION_STATUS;
 	}
 
     } else if(op->op_status == LRM_OP_DONE
 	      && crm_str_eq(task, CRMD_ACTION_MIGRATED, TRUE)) {
 	task = CRMD_ACTION_START;
 
     } else if(crm_str_eq(task, CRMD_ACTION_NOTIFY, TRUE)) {
 	const char *n_type = crm_meta_value(op->params, "notify_type");
 	const char *n_task = crm_meta_value(op->params, "notify_operation");
 	CRM_DEV_ASSERT(n_type != NULL);
 	CRM_DEV_ASSERT(n_task != NULL);
 	op_id = generate_notify_key(op->rsc_id, n_type, n_task);
 
 	/* these are not yet allowed to fail */
 	op->op_status = LRM_OP_DONE;
 	op->rc = 0;
 		
     }
 
     if (op_id == NULL) {
 	op_id = generate_op_key(op->rsc_id, task, op->interval);
     }
 
     xml_op = find_entity(parent, XML_LRM_TAG_RSC_OP, op_id);
     if(xml_op != NULL) {
 	crm_log_xml(LOG_DEBUG, "Replacing existing entry", xml_op);
 		
     } else {
 	xml_op = create_xml_node(parent, XML_LRM_TAG_RSC_OP);
     }
 	
     if(op->user_data == NULL) {
 	crm_debug("Generating fake transition key for:"
 		  " %s_%s_%d %d from %s",
 		  op->rsc_id, op->op_type, op->interval, op->call_id,
 		  op->app_name);
 	local_user_data = generate_transition_key(-1, op->call_id, target_rc, FAKE_TE_ID);
 	op->user_data = local_user_data;
     }
 	
     magic = generate_transition_magic(op->user_data, op->op_status, op->rc);
 	
     crm_xml_add(xml_op, XML_ATTR_ID,			op_id);
     crm_xml_add(xml_op, XML_LRM_ATTR_TASK,		task);
     crm_xml_add(xml_op, XML_ATTR_ORIGIN,		origin);
     crm_xml_add(xml_op, XML_ATTR_CRM_VERSION,		caller_version);
     crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY,	op->user_data);
     crm_xml_add(xml_op, XML_ATTR_TRANSITION_MAGIC,	magic);
 
     crm_xml_add_int(xml_op, XML_LRM_ATTR_CALLID,	op->call_id);
     crm_xml_add_int(xml_op, XML_LRM_ATTR_RC,		op->rc);
     crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS,	op->op_status);
     crm_xml_add_int(xml_op, XML_LRM_ATTR_INTERVAL,	op->interval);
 
     if(compare_version("2.1", caller_version) <= 0) {
 	if(op->t_run || op->t_rcchange || op->exec_time || op->queue_time) {
 	    crm_debug_2("Timing data (%s_%s_%d): last=%lu change=%lu exec=%lu queue=%lu",
 			op->rsc_id, op->op_type, op->interval,
 			op->t_run, op->t_rcchange, op->exec_time, op->queue_time);
 	
 	    crm_xml_add_int(xml_op, "last-run",       op->t_run);
 	    crm_xml_add_int(xml_op, "last-rc-change", op->t_rcchange);
 	    crm_xml_add_int(xml_op, "exec-time",      op->exec_time);
 	    crm_xml_add_int(xml_op, "queue-time",     op->queue_time);
 	}
     }
 	
     append_digest(op, xml_op, caller_version, magic, LOG_DEBUG);
 
     if(op->op_status != LRM_OP_DONE
        && crm_str_eq(op->op_type, CRMD_ACTION_MIGRATED, TRUE)) {
 	const char *host = crm_meta_value(op->params, "migrate_source_uuid");
 	crm_xml_add(xml_op, CRMD_ACTION_MIGRATED, host);
     }	
 	
     if(local_user_data) {
 	crm_free(local_user_data);
 	op->user_data = NULL;
     }
     crm_free(magic);	
     crm_free(op_id);
     return xml_op;
 }
diff --git a/lib/common/xml.c b/lib/common/xml.c
index 4ed6e7ddb4..ff97b9e6ce 100644
--- a/lib/common/xml.c
+++ b/lib/common/xml.c
@@ -1,2805 +1,2805 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 #include <sys/param.h>
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 #include <time.h>
 #include <string.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <ctype.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <libxml/xmlreader.h>
 
 #include <clplumbing/md5.h>
 #if HAVE_BZLIB_H
 #  include <bzlib.h>
 #endif
 
 #define XML_BUFFER_SIZE	4096
 #define XML_PARSER_DEBUG 0
 
 xmlDoc *getDocPtr(xmlNode *node);
 
 struct schema_s 
 {
 	int type;
 	const char *name;
 	const char *location;
 	const char *transform;
 	int after_transform;
 };
 
 struct schema_s known_schemas[] = {
 /* 0 */    { 0, NULL, NULL, NULL, 1 },
 /* 1 */    { 1, "pacemaker-0.6",    CRM_DTD_DIRECTORY"/crm.dtd",		CRM_DTD_DIRECTORY"/upgrade06.xsl", 4 },
 /* 2 */    { 1, "transitional-0.6", CRM_DTD_DIRECTORY"/crm-transitional.dtd",	CRM_DTD_DIRECTORY"/upgrade06.xsl", 4 },
 /* 3 */    { 2, "pacemaker-0.7",    CRM_DTD_DIRECTORY"/pacemaker-1.0.rng",	NULL, 0 },
 /* 4 */    { 2, "pacemaker-1.0",    CRM_DTD_DIRECTORY"/pacemaker-1.0.rng",	NULL, 6 },
 /* 5 */    { 2, "pacemaker-1.1",    CRM_DTD_DIRECTORY"/pacemaker-1.1.rng",	NULL, 6 },
 /* 6 */    { 2, "pacemaker-1.2",    CRM_DTD_DIRECTORY"/pacemaker-1.2.rng",	NULL, 0 },
 /* 7 */    { 0, "none", NULL, NULL, 0 },
 };
 
 static int all_schemas = DIMOF(known_schemas);
 static int max_schemas = DIMOF(known_schemas) - 2; /* skip back past 'none' */
 
 static const char *filter[] = {
     XML_ATTR_ORIGIN,
     XML_DIFF_MARKER,
     XML_CIB_ATTR_WRITTEN,		
 };
 
 static void add_ha_nocopy(HA_Message *parent, HA_Message *child, const char *field) 
 {
     int next = parent->nfields;
     if (parent->nfields >= parent->nalloc && ha_msg_expand(parent) != HA_OK ) {
 	crm_err("Parent expansion failed");
 	return;
     }
     
     parent->names[next] = crm_strdup(field);
     parent->nlens[next] = strlen(field);
     parent->values[next] = child;
     parent->vlens[next] = sizeof(HA_Message);
     parent->types[next] = FT_UNCOMPRESS;
     parent->nfields++;	
 }
 
 int print_spaces(char *buffer, int spaces, int max);
 
 int log_data_element(const char *function, const char *prefix, int log_level,
 		     int depth, xmlNode *data, gboolean formatted);
 
 int get_tag_name(const char *input, size_t offset, size_t max);
 int get_attr_name(const char *input, size_t offset, size_t max);
 int get_attr_value(const char *input, size_t offset, size_t max);
 gboolean can_prune_leaf(xmlNode *xml_node);
 
 void diff_filter_context(int context, int upper_bound, int lower_bound,
 		    xmlNode *xml_node, xmlNode *parent);
 int in_upper_context(int depth, int context, xmlNode *xml_node);
 int write_file(const char *string, const char *filename);
 
 xmlNode *
 find_xml_node(xmlNode *root, const char * search_path, gboolean must_find)
 {
 	const char *name = "NULL";
 	if(must_find || root != NULL) {
 		crm_validate_data(root);
 	}
 	if(root != NULL) {
 	    name = crm_element_name(root);
 	}
 	
 	if(search_path == NULL) {
 		crm_warn("Will never find <NULL>");
 		return NULL;
 	}
 	
 	xml_child_iter_filter(
 		root, a_child, search_path,
 /* 		crm_debug_5("returning node (%s).", crm_element_name(a_child)); */
 		crm_validate_data(a_child);
 		return a_child;
 		);
 
 	if(must_find) {
 		crm_warn("Could not find %s in %s.", search_path, name);
 	} else if(root != NULL) {
 		crm_debug_3("Could not find %s in %s.", search_path, name);
 	} else {
 		crm_debug_3("Could not find %s in <NULL>.", search_path);
 	}
 	
 	
 	return NULL;
 }
 
 xmlNode*
 find_entity(xmlNode *parent, const char *node_name, const char *id)
 {
 	crm_validate_data(parent);
 	xml_child_iter_filter(
 		parent, a_child, node_name,
 		if(id == NULL || crm_str_eq(id, ID(a_child), TRUE)) {
 			crm_debug_4("returning node (%s).", 
 				    crm_element_name(a_child));
 			return a_child;
 		}
 		);
 	crm_debug_3("node <%s id=%s> not found in %s.",
 		    node_name, id, crm_element_name(parent));
 	return NULL;
 }
 
 void
 copy_in_properties(xmlNode* target, xmlNode *src)
 {
 	crm_validate_data(src);
 	crm_validate_data(target);
 
 	if(src == NULL) {
 		crm_warn("No node to copy properties from");
 
 	} else if (target == NULL) {
 		crm_err("No node to copy properties into");
 
 	} else {
 		xml_prop_iter(
 			src, local_prop_name, local_prop_value,
 			expand_plus_plus(target, local_prop_name, local_prop_value)
 			);
 		crm_validate_data(target);
 	}
 	
 	return;
 }
 
 void fix_plus_plus_recursive(xmlNode* target)
 {
     xml_prop_iter(target, name, value, expand_plus_plus(target, name, value));
     xml_child_iter(target, child, fix_plus_plus_recursive(child));
 }
 
 
 void
 expand_plus_plus(xmlNode* target, const char *name, const char *value)
 {
     int offset = 1;
     int name_len = 0;
     int int_value = 0;
     int value_len = 0;
 
     const char *old_value = NULL;
 
     if(value == NULL || name == NULL) {
 	return;
     }
     
     old_value = crm_element_value(target, name);
 
     if(old_value == NULL) {
 	/* if no previous value, set unexpanded */
 	goto set_unexpanded;
 
     } else if(strstr(value, name) != value) {
 	goto set_unexpanded;
     }
 
     name_len = strlen(name);
     value_len = strlen(value);
     if(value_len < (name_len + 2)
 	      || value[name_len] != '+'
 	      || (value[name_len+1] != '+' && value[name_len+1] != '=')) {
 	goto set_unexpanded;
     }
 
     /* if we are expanding ourselves,
      * then no previous value was set and leave int_value as 0
      */
     if(old_value != value) {
 	int_value = char2score(old_value);
     }
     
     if(value[name_len+1] != '+') {
 	const char *offset_s = value+(name_len+2);
 	offset = char2score(offset_s);
     }
     int_value += offset;
 
     if(int_value > INFINITY) {
 	int_value = INFINITY;
     }
     
     crm_xml_add_int(target, name, int_value);
     return;
 
   set_unexpanded:
     if(old_value == value) {
 	/* the old value is already set, nothing to do */
 	return;
     }
     crm_xml_add(target, name, value);
     return;
 }
 
 xmlDoc *getDocPtr(xmlNode *node)
 {
     xmlDoc *doc = NULL;
     CRM_CHECK(node != NULL, return NULL);
 
     doc = node->doc;
     if(doc == NULL) {
 	doc = xmlNewDoc((const xmlChar*)"1.0");
 	xmlDocSetRootElement(doc, node);
 	xmlSetTreeDoc(node, doc);
     }
     return doc;
 }
 
 xmlNode*
 add_node_copy(xmlNode *parent, xmlNode *src_node) 
 {
 	xmlNode *child = NULL;
 	xmlDoc *doc = getDocPtr(parent);
 	CRM_CHECK(src_node != NULL, return NULL);
 
 	child = xmlDocCopyNode(src_node, doc, 1);
 	xmlAddChild(parent, child);
 	return child;
 }
 
 
 int
 add_node_nocopy(xmlNode *parent, const char *name, xmlNode *child)
 {
 	add_node_copy(parent, child);
 	free_xml(child);
 	return HA_OK;
 }
 
 const char *
 crm_xml_add(xmlNode* node, const char *name, const char *value)
 {
     xmlAttr *attr = NULL;
     CRM_CHECK_AND_STORE(node != NULL, return NULL);
     CRM_CHECK_AND_STORE(name != NULL, return NULL);
 
     if(value == NULL) {
 	return NULL;
     }
 
 #if XML_PARANOIA_CHECKS
     {
 	const char *old_value = NULL;
 	old_value = crm_element_value(node, name);
 	
 	/* Could be re-setting the same value */
 	CRM_CHECK_AND_STORE(old_value != value,
 			    crm_err("Cannot reset %s with crm_xml_add(%s)",
 				    name, value);
 			    return value);
     }
 #endif
     
     attr = xmlSetProp(node, (const xmlChar*)name, (const xmlChar*)value);
     CRM_CHECK(attr && attr->children && attr->children->content, return NULL);
     return (char *)attr->children->content;
 }
 
 const char *
 crm_xml_replace(xmlNode* node, const char *name, const char *value)
 {
     xmlAttr *attr = NULL;
     const char *old_value = NULL;
     CRM_CHECK(node != NULL, return NULL);
     CRM_CHECK(name != NULL && name[0] != 0, return NULL);
 
     old_value = crm_element_value(node, name);
 
     /* Could be re-setting the same value */
     CRM_CHECK_AND_STORE(old_value != value, return value);
 
     if (old_value != NULL && value == NULL) {
 	xml_remove_prop(node, name);
 	return NULL;
 
     } else if(value == NULL) {
 	return NULL;
     }
     
     attr = xmlSetProp(node, (const xmlChar*)name, (const xmlChar*)value);
     CRM_CHECK(attr && attr->children && attr->children->content, return NULL);
     return (char *)attr->children->content;
 }
 
 const char *
 crm_xml_add_int(xmlNode* node, const char *name, int value)
 {
     char *number = crm_itoa(value);
     const char *added = crm_xml_add(node, name, number);
     crm_free(number);
     return added;
 }
 
 xmlNode*
 create_xml_node(xmlNode *parent, const char *name)
 {
     xmlDoc *doc = NULL;
     xmlNode *node = NULL;	
 
     if (name == NULL || name[0] == 0) {
 	return NULL;
     }
     
     if(parent == NULL) {
 	doc = xmlNewDoc((const xmlChar*)"1.0");
 	node = xmlNewDocRawNode(doc, NULL, (const xmlChar*)name, NULL);
 	xmlDocSetRootElement(doc, node);
 	
     } else {
 	doc = getDocPtr(parent);
 	node = xmlNewDocRawNode(doc, NULL, (const xmlChar*)name, NULL);
 	xmlAddChild(parent, node);
     }
     return node;
 }
 
 void
 free_xml_from_parent(xmlNode *parent, xmlNode *a_node)
 {
 	CRM_CHECK(a_node != NULL, return);
 
 	xmlUnlinkNode(a_node);
 	xmlFreeNode(a_node);
 }
 
 xmlNode*
 copy_xml(xmlNode *src)
 {
     xmlDoc *doc = xmlNewDoc((const xmlChar*)"1.0");
     xmlNode *copy = xmlDocCopyNode(src, doc, 1);
     xmlDocSetRootElement(doc, copy);
     xmlSetTreeDoc(copy, doc);
     return copy;
 }
 
 
 static void crm_xml_err(void * ctx, const char * msg, ...) G_GNUC_PRINTF(2,3);
 extern size_t strlcat(char * dest, const char *source, size_t len);
 
 int
 write_file(const char *string, const char *filename) 
 {
 	int rc = 0;
 	FILE *file_output_strm = NULL;
 	
 	CRM_CHECK(filename != NULL, return -1);
 
 	if (string == NULL) {
 		crm_err("Cannot write NULL to %s", filename);
 		return -1;
 	}
 
 	file_output_strm = fopen(filename, "w");
 	if(file_output_strm == NULL) {
 		crm_perror(LOG_ERR,"Cannot open %s for writing", filename);
 		return -1;
 	} 
 
 	rc = fprintf(file_output_strm, "%s", string);
 	if(rc < 0) {
 	    crm_perror(LOG_ERR,"Cannot write output to %s", filename);
 	}		
 	
 	if(fflush(file_output_strm) != 0) {
 	    crm_perror(LOG_ERR,"fflush for %s failed:", filename);
 	    rc = -1;
 	}
 	
 	if(fsync(fileno(file_output_strm)) < 0) {
 	    crm_perror(LOG_ERR,"fsync for %s failed:", filename);
 	    rc = -1;
 	}
 	    
 	fclose(file_output_strm);
 	return rc;
 }
 
 static void crm_xml_err(void * ctx, const char * msg, ...)
 {
     int len = 0;
     va_list args;
     char *buf = NULL;
     static int buffer_len = 0;
     static char *buffer = NULL;
     
     va_start(args, msg);
     len = vasprintf(&buf, msg, args);
 
     if(strchr(buf, '\n')) {
 	buf[len - 1] = 0;
 	if(buffer) {
 	    crm_err("XML Error: %s%s", buffer, buf);
 	    free(buffer);
 	} else {
 	    crm_err("XML Error: %s", buf);	    
 	}
 	buffer = NULL;
 	buffer_len = 0;
 	
     } else if(buffer == NULL) {
 	buffer_len = len;
 	buffer = buf;
 	buf = NULL;
 
     } else {
 	buffer_len += len;
 	buffer = realloc(buffer, buffer_len);
 	strlcat(buffer, buf, buffer_len);
     }
     
     va_end(args);
     free(buf);	
 }
 
 xmlNode*
 string2xml(const char *input)
 {
 	xmlNode *xml = NULL;
 	xmlDocPtr output = NULL;
 	xmlParserCtxtPtr ctxt = NULL;
 	xmlErrorPtr last_error = NULL;
 	
 	if(input == NULL) {
 	    crm_err("Can't parse NULL input");
 	    return NULL;
 	}
 	
 	/* create a parser context */
 	ctxt = xmlNewParserCtxt();
 	CRM_CHECK(ctxt != NULL, return NULL);
 
 	/* xmlCtxtUseOptions(ctxt, XML_PARSE_NOBLANKS|XML_PARSE_RECOVER); */
 
 	xmlCtxtResetLastError(ctxt);
 	xmlSetGenericErrorFunc(ctxt, crm_xml_err);
 	/* initGenericErrorDefaultFunc(crm_xml_err); */
 	output = xmlCtxtReadDoc(ctxt, (const xmlChar*)input, NULL, NULL, XML_PARSE_NOBLANKS|XML_PARSE_RECOVER);
 	if(output) {
 	    xml = xmlDocGetRootElement(output);
 	}
 	last_error = xmlCtxtGetLastError(ctxt);
 	if(last_error && last_error->code != XML_ERR_OK) {
 	    /* crm_abort(__FILE__,__PRETTY_FUNCTION__,__LINE__, "last_error->code != XML_ERR_OK", TRUE, TRUE); */
 	    /*
 	     * http://xmlsoft.org/html/libxml-xmlerror.html#xmlErrorLevel
 	     * http://xmlsoft.org/html/libxml-xmlerror.html#xmlParserErrors
 	     */
 	    crm_warn("Parsing failed (domain=%d, level=%d, code=%d): %s",
 		    last_error->domain, last_error->level,
 		    last_error->code, last_error->message);
 
 	    if(last_error->code != XML_ERR_DOCUMENT_END) {
 		crm_err("Couldn't%s parse %d chars: %s", xml?" fully":"", (int)strlen(input), input);
 		if(xml != NULL) {
 		    crm_log_xml_err(xml, "Partial");
 		}
 
 	    } else {
 		int len = strlen(input);
 		crm_warn("String start: %.50s", input);
 		crm_warn("String start+%d: %s", len-50, input+len-50);
 		crm_abort(__FILE__,__PRETTY_FUNCTION__,__LINE__, "String parsing error", TRUE, TRUE);
 	    }
 	}
 
 	xmlFreeParserCtxt(ctxt);
 	return xml;
 }
 
 xmlNode *
 stdin2xml(void) 
 {
  	size_t data_length = 0;
  	size_t read_chars = 0;
   
   	char *xml_buffer = NULL;
   	xmlNode *xml_obj = NULL;
   
  	do {
  		crm_realloc(xml_buffer, XML_BUFFER_SIZE + data_length + 1);
  		read_chars = fread(xml_buffer + data_length, 1, XML_BUFFER_SIZE, stdin);
  		data_length += read_chars;
  	} while (read_chars > 0);
 
 	if(data_length == 0) {
 	    crm_warn("No XML supplied on stdin");
 	    return NULL;
 	}
 
  	xml_buffer[data_length] = '\0';
 
 	xml_obj = string2xml(xml_buffer);
 	crm_free(xml_buffer);
 
 	crm_log_xml_debug_3(xml_obj, "Created fragment");
 	return xml_obj;
 }
 
 static char *
 decompress_file(const char *filename)
 {
     char *buffer = NULL;
 #if HAVE_BZLIB_H
     int rc = 0;
     size_t length = 0, read_len = 0;
     
     BZFILE *bz_file = NULL;
     FILE *input = fopen(filename, "r");
 
     if(input == NULL) {
 	crm_perror(LOG_ERR,"Could not open %s for reading", filename);
 	return NULL;
     }
     
     bz_file = BZ2_bzReadOpen(&rc, input, 0, 0, NULL, 0);
 
     if ( rc != BZ_OK ) {
 	BZ2_bzReadClose ( &rc, bz_file);
 	return NULL;
     }
     
     rc = BZ_OK;
     while ( rc == BZ_OK ) {
 	crm_realloc(buffer, XML_BUFFER_SIZE + length + 1);
 	read_len = BZ2_bzRead (
 	    &rc, bz_file, buffer + length, XML_BUFFER_SIZE);
 	
 	crm_debug_5("Read %ld bytes from file: %d",
 		    (long)read_len, rc);
 	
 	if ( rc == BZ_OK || rc == BZ_STREAM_END) {
 	    length += read_len;
 	}
     }
     
     buffer[length] = '\0';
     read_len = length;
     
     if ( rc != BZ_STREAM_END ) {
 	crm_err("Couldnt read compressed xml from file");
 	crm_free(buffer);
 	buffer = NULL;
     }
     
     BZ2_bzReadClose (&rc, bz_file);
     fclose(input);
     
 #else
     crm_err("Cannot read compressed files:"
 	    " bzlib was not available at compile time");
 #endif
     return buffer;
 }
 
 xmlNode *
 filename2xml(const char *filename)
 {
     xmlNode *xml = NULL;
     xmlDocPtr output = NULL;
     xmlParserCtxtPtr ctxt = NULL;
     xmlErrorPtr last_error = NULL;
     static int xml_options = XML_PARSE_NOBLANKS|XML_PARSE_RECOVER;
     
     /* create a parser context */
     ctxt = xmlNewParserCtxt();
     CRM_CHECK(ctxt != NULL, return NULL);
     
     /* xmlCtxtUseOptions(ctxt, XML_PARSE_NOBLANKS|XML_PARSE_RECOVER); */
     
     xmlCtxtResetLastError(ctxt);
     xmlSetGenericErrorFunc(ctxt, crm_xml_err);
     /* initGenericErrorDefaultFunc(crm_xml_err); */
 
     if(filename == NULL) {
 	/* STDIN_FILENO == fileno(stdin) */
 	output = xmlCtxtReadFd(ctxt, STDIN_FILENO, "unknown.xml", NULL, xml_options);
 
     } else if(strstr(filename, ".bz2") == NULL) {
 	output = xmlCtxtReadFile(ctxt, filename, NULL, xml_options);
 
     } else {
 	char *input = decompress_file(filename);
 	output = xmlCtxtReadDoc(ctxt, (const xmlChar*)input, NULL, NULL, xml_options);
 	crm_free(input);
     }
 
     if(output) {
 	xml = xmlDocGetRootElement(output);
     }
     
     last_error = xmlCtxtGetLastError(ctxt);
     if(last_error && last_error->code != XML_ERR_OK) {
 	/* crm_abort(__FILE__,__PRETTY_FUNCTION__,__LINE__, "last_error->code != XML_ERR_OK", TRUE, TRUE); */
 	/*
 	 * http://xmlsoft.org/html/libxml-xmlerror.html#xmlErrorLevel
 	 * http://xmlsoft.org/html/libxml-xmlerror.html#xmlParserErrors
 	 */
 	crm_err("Parsing failed (domain=%d, level=%d, code=%d): %s",
 		last_error->domain, last_error->level,
 		last_error->code, last_error->message);
 	
 	if(last_error && last_error->code != XML_ERR_OK) {
 	    crm_err("Couldn't%s parse %s", xml?" fully":"", filename);
 	    if(xml != NULL) {
 		crm_log_xml_err(xml, "Partial");
 	    }
 	}
     }
     
     xmlFreeParserCtxt(ctxt);
     return xml;
 }
 
 int
 write_xml_file(xmlNode *xml_node, const char *filename, gboolean compress) 
 {
 	int res = 0;
 	time_t now;
 	char *buffer = NULL;
 	char *now_str = NULL;
 	unsigned int out = 0;
 	FILE *file_output_strm = NULL;
 	static mode_t cib_mode = S_IRUSR|S_IWUSR;
 	
 	CRM_CHECK(filename != NULL, return -1);
 
 	crm_debug_3("Writing XML out to %s", filename);
 	crm_validate_data(xml_node);
 	if (xml_node == NULL) {
 		crm_err("Cannot write NULL to %s", filename);
 		return -1;
 	}
 
 	file_output_strm = fopen(filename, "w");
 	if(file_output_strm == NULL) {
 		crm_perror(LOG_ERR,"Cannot open %s for writing", filename);
 		return -1;
 	} 
 
 	/* establish the correct permissions */
 	fchmod(fileno(file_output_strm), cib_mode);
 	
 	crm_log_xml_debug_4(xml_node, "Writing out");
 	
 	now = time(NULL);
 	now_str = ctime(&now);
 	now_str[24] = EOS; /* replace the newline */
 	crm_xml_add(xml_node, XML_CIB_ATTR_WRITTEN, now_str);
 	crm_validate_data(xml_node);
 	
 	buffer = dump_xml_formatted(xml_node);
 	CRM_CHECK(buffer != NULL && strlen(buffer) > 0,
 		  crm_log_xml_warn(xml_node, "dump:failed");
 		  goto bail);	
 
 	if(compress) {
 #if HAVE_BZLIB_H
 	    int rc = BZ_OK;
 	    unsigned int in = 0;
 	    BZFILE *bz_file = NULL;
 	    bz_file = BZ2_bzWriteOpen(&rc, file_output_strm, 5, 0, 30);
 	    if(rc != BZ_OK) {
 		crm_err("bzWriteOpen failed: %d", rc);
 	    } else {
 		BZ2_bzWrite(&rc,bz_file,buffer,strlen(buffer));
 		if(rc != BZ_OK) {
 		    crm_err("bzWrite() failed: %d", rc);
 		}
 	    }
 	    
 	    if(rc == BZ_OK) {
 		BZ2_bzWriteClose(&rc, bz_file, 0, &in, &out);
 		if(rc != BZ_OK) {
 		    crm_err("bzWriteClose() failed: %d",rc);
 		    out = -1;
 		} else {
 		    crm_debug_2("%s: In: %d, out: %d", filename, in, out);
 		}
 	    }
 #else
 	    crm_err("Cannot write compressed files:"
 		    " bzlib was not available at compile time");		
 #endif
 	}
 	
 	if(out <= 0) {
 	    res = fprintf(file_output_strm, "%s", buffer);
 	    if(res < 0) {
 		crm_perror(LOG_ERR,"Cannot write output to %s", filename);
 		goto bail;
 	    }		
 	}
 	
   bail:
 	
 	if(fflush(file_output_strm) != 0) {
 	    crm_perror(LOG_ERR,"fflush for %s failed:", filename);
 	    res = -1;
 	}
 	
 	if(fsync(fileno(file_output_strm)) < 0) {
 	    crm_perror(LOG_ERR,"fsync for %s failed:", filename);
 	    res = -1;
 	}
 	    
 	fclose(file_output_strm);
 	
 	crm_debug_3("Saved %d bytes to the Cib as XML", res);
 	crm_free(buffer);
 
 	return res;
 }
 
 void
 print_xml_formatted(int log_level, const char *function,
 		    xmlNode *msg, const char *text)
 {
 	if(msg == NULL) {
 		do_crm_log(log_level, "%s: %s: NULL", function, crm_str(text));
 		return;
 	}
 
 	crm_validate_data(msg);
 	log_data_element(function, text, log_level, 0, msg, TRUE);
 	return;
 }
 
 static HA_Message*
 convert_xml_message_struct(HA_Message *parent, xmlNode *src_node, const char *field) 
 {
     xmlNode *child = NULL;
     xmlNode *__crm_xml_iter = src_node->children;
     xmlAttrPtr prop_iter = src_node->properties;
     const char *name = NULL;
     const char *value = NULL;
 
     HA_Message *result = ha_msg_new(3);
     ha_msg_add(result, F_XML_TAGNAME, (const char *)src_node->name);
     
     while(prop_iter != NULL) {
 	name = (const char *)prop_iter->name;
 	value = (const char *)xmlGetProp(src_node, prop_iter->name);
 	prop_iter = prop_iter->next;
 	ha_msg_add(result, name, value);
     }
 
     while(__crm_xml_iter != NULL) {
 	child = __crm_xml_iter;
 	__crm_xml_iter = __crm_xml_iter->next;
 	convert_xml_message_struct(result, child, NULL);
     }
 
     if(parent == NULL) {
 	return result;
     }
     
     if(field) {
-	HA_Message *holder = holder = ha_msg_new(3);
+	HA_Message *holder = ha_msg_new(3);
 	CRM_ASSERT(holder != NULL);
 	
 	ha_msg_add(holder, F_XML_TAGNAME, field);
 	add_ha_nocopy(holder, result, (const char*)src_node->name);
 	
 	ha_msg_addstruct_compress(parent, field, holder);
 	ha_msg_del(holder);
 
     } else {
 	add_ha_nocopy(parent, result, (const char*)src_node->name);
     }
     return result;
 }
 
 static void
 convert_xml_child(HA_Message *msg, xmlNode *xml) 
 {
     int orig = 0;
     int rc = BZ_OK;
     unsigned int len = 0;
     
     char *buffer = NULL;
     char *compressed = NULL;
     const char *name = NULL;
 
     name = (const char *)xml->name;
     buffer = dump_xml_unformatted(xml);
     orig = strlen(buffer);
     if(orig < CRM_BZ2_THRESHOLD) {
 	ha_msg_add(msg, name, buffer);
 	goto done;
     }
     
     len = (orig * 1.1) + 600; /* recomended size */
     
     crm_malloc(compressed, len);
     rc = BZ2_bzBuffToBuffCompress(compressed, &len, buffer, orig, CRM_BZ2_BLOCKS, 0, CRM_BZ2_WORK);
     
     if(rc != BZ_OK) {
 	crm_err("Compression failed: %d", rc);
 	crm_free(compressed);
 	convert_xml_message_struct(msg, xml, name);
 	goto done;
     }
     
     crm_free(buffer);
     buffer = compressed;
     crm_debug_2("Compression details: %d -> %d", orig, len);
     ha_msg_addbin(msg, name, buffer, len);
   done:
     crm_free(buffer);
 
 
 #  if 0
     {
 	unsigned int used = orig;
 	char *uncompressed = NULL;
 	
 	crm_debug("Trying to decompress %d bytes", len);
 	crm_malloc0(uncompressed, orig);
 	rc = BZ2_bzBuffToBuffDecompress(
 	    uncompressed, &used, compressed, len, 1, 0);
 	CRM_CHECK(rc == BZ_OK, ;);
 	CRM_CHECK(used == orig, ;);
 	crm_debug("rc=%d, used=%d", rc, used);
 	if(rc != BZ_OK) {
 	    exit(100);
 	}
 	crm_debug("Original %s, decompressed %s", buffer, uncompressed);
 	crm_free(uncompressed);
     }
 #  endif 
 }
 
 HA_Message*
 convert_xml_message(xmlNode *xml) 
 {
     HA_Message *result = NULL;
 
     result = ha_msg_new(3);
     ha_msg_add(result, F_XML_TAGNAME, (const char *)xml->name);
 
     xml_prop_iter(xml, name, value, ha_msg_add(result, name, value));
     xml_child_iter(xml, child, convert_xml_child(result, child));
 
     return result;
 }
 
 static void
 convert_ha_field(xmlNode *parent, HA_Message *msg, int lpc) 
 {
     int type = 0;
     const char *name = NULL;
     const char *value = NULL;
     xmlNode *xml = NULL;
     
     int rc = BZ_OK;
     size_t orig_len = 0;
     unsigned int used = 0;
     char *uncompressed = NULL;
     char *compressed = NULL;
     int size = orig_len * 10;
     
     CRM_CHECK(parent != NULL, return);
     CRM_CHECK(msg != NULL, return);
 	
     name = msg->names[lpc];
     type = cl_get_type(msg, name);
 
     switch(type) {
 	case FT_STRUCT:
 	    convert_ha_message(parent, msg->values[lpc], name);
 	    break;
 	case FT_COMPRESS:
 	case FT_UNCOMPRESS:
 	    convert_ha_message(parent, cl_get_struct(msg, name), name);
 	    break;
 	case FT_STRING:
 	    value = cl_get_string(msg, name);
 	    CRM_CHECK_AND_STORE(value != NULL, return);
 	    crm_debug_5("Converting %s/%d/%s", name, type, value[0] == '<' ? "xml":"field");
 
 	    if( value[0] != '<' ) {
 		crm_xml_add(parent, name, value);
 		break;
 	    }
 	    
 	    /* unpack xml string */
 	    xml = string2xml(value);
 	    if(xml == NULL) {
 		crm_err("Conversion of field '%s' failed", name);
 		return;
 	    }
 
 	    add_node_nocopy(parent, NULL, xml);
 	    break;
 
 	case FT_BINARY:
 	    value = cl_get_binary(msg, name, &orig_len);
 	    size = orig_len * 10 + 1; /* +1 because an exact 10x compression factor happens occasionally */
 
 	    if(orig_len < 3
 	       || value[0] != 'B'
 	       || value[1] != 'Z'
 	       || value[2] != 'h') {
 		if(strstr(name, "uuid") == NULL) {
 		    crm_err("Skipping non-bzip binary field: %s", name);
 		}
 		return;
 	    }
 
 	    crm_malloc0(compressed, orig_len);
 	    memcpy(compressed, value, orig_len);
 	    
 	    crm_debug_2("Trying to decompress %d bytes", (int)orig_len);
 	  retry:
 	    crm_realloc(uncompressed, size);
 	    memset(uncompressed, 0, size);
 	    used = size - 1; /* always leave room for a trailing '\0'
 			      * BZ2_bzBuffToBuffDecompress wont say anything if
 			      * the uncompressed data is exactly 'size' bytes 
 			      */
 	    
 	    rc = BZ2_bzBuffToBuffDecompress(
 		uncompressed, &used, compressed, orig_len, 1, 0);
 	    
 	    if(rc == BZ_OUTBUFF_FULL) {
 		size = size * 2;
 		/* dont try to allocate more memory than we have */
 		if(size > 0) {
 		    goto retry;
 		}
 	    }
 	    
 	    if(rc != BZ_OK) { 
 		crm_err("Decompression of %s (%d bytes) into %d failed: %d",
 			name, (int)orig_len, size, rc);
 		
 	    } else {
 		CRM_ASSERT(used < size);
 		CRM_CHECK(uncompressed[used] == 0, uncompressed[used] = 0);
 		xml = string2xml(uncompressed);
 	    }
 
 	    if(xml != NULL) {
 		add_node_copy(parent, xml);
 		free_xml(xml);
 	    }
 	    
 	    crm_free(uncompressed);
 	    crm_free(compressed);		
 	    break;
     }
 }
 
 xmlNode *
 convert_ha_message(xmlNode *parent, HA_Message *msg, const char *field) 
 {
     int lpc = 0;
     xmlNode *child = NULL;
     const char *tag = NULL;
     
     CRM_CHECK_AND_STORE(msg != NULL, crm_err("Empty message for %s", field); return parent);
     
     tag = cl_get_string(msg, F_XML_TAGNAME);
     if(tag == NULL) {
 	tag = field;
 
     } else if(parent && safe_str_neq(field, tag)) {
 	/* For compatability with 0.6.x */
 	crm_debug("Creating intermediate parent %s between %s and %s", field, crm_element_name(parent), tag);
 	parent = create_xml_node(parent, field);
     }
     
     if(parent == NULL) {
 	parent = create_xml_node(NULL, tag);
 	child = parent;
 	
     } else {
 	child = create_xml_node(parent, tag);
     }
 
     for (lpc = 0; lpc < msg->nfields; lpc++) {
 	convert_ha_field(child, msg, lpc);
     }
     
     return parent;
 }
 
 xmlNode *convert_ipc_message(IPC_Message *msg, const char *field)
 {
     HA_Message *hmsg = wirefmt2msg((char *)msg->msg_body, msg->msg_len, 0);
     xmlNode *xml = convert_ha_message(NULL, hmsg, __FUNCTION__);
     crm_msg_del(hmsg);
     return xml;
 }
 
 xmlNode *
 get_message_xml(xmlNode *msg, const char *field) 
 {
     xmlNode *tmp = first_named_child(msg, field);
     return first_named_child(tmp, NULL);
 }
 
 gboolean
 add_message_xml(xmlNode *msg, const char *field, xmlNode *xml) 
 {
     xmlNode *holder = create_xml_node(msg, field);
     add_node_copy(holder, xml);
     return TRUE;
 }
 
 static char *
 dump_xml(xmlNode *an_xml_node, gboolean formatted, gboolean for_digest)
 {
     int len = 0;
     char *buffer = NULL;
     xmlBuffer *xml_buffer = NULL;
     xmlDoc *doc = getDocPtr(an_xml_node);
 
     /* doc will only be NULL if an_xml_node is */
     CRM_CHECK(doc != NULL, return NULL);
 
     xml_buffer = xmlBufferCreate();
     CRM_ASSERT(xml_buffer != NULL);
     
     len = xmlNodeDump(xml_buffer, doc, an_xml_node, 0, formatted);
 
     if(len > 0) {
 	if(for_digest) {
 	    /* for compatability with the old result which is used for digests */
 	    len += 3;
 	    crm_malloc0(buffer, len);
 	    snprintf(buffer, len, " %s\n", (char *)xml_buffer->content);
 	} else {
 	    buffer = crm_strdup((char *)xml_buffer->content);	    
 	}
 
     } else {
 	crm_err("Conversion failed");
     }
 
     xmlBufferFree(xml_buffer);
     return buffer;    
 }
 
 char *
 dump_xml_formatted(xmlNode *an_xml_node)
 {
     return dump_xml(an_xml_node, TRUE, FALSE);
 }
 
 char *
 dump_xml_unformatted(xmlNode *an_xml_node)
 {
     return dump_xml(an_xml_node, FALSE, FALSE);
 }
     
 #define update_buffer() do {				\
 	if(printed < 0) {				\
 	    crm_perror(LOG_ERR,"snprintf failed");		\
 	    goto print;					\
 	} else if(printed >= (buffer_len - offset)) {	\
 	    crm_err("Output truncated: available=%d, needed=%d", buffer_len - offset, printed);	\
 	    offset += printed;				\
 	    goto print;					\
 	} else if(offset >= buffer_len) {		\
 	    crm_err("Buffer exceeded");			\
 	    offset += printed;				\
 	    goto print;					\
 	} else {					\
 	    offset += printed;				\
 	}						\
 } while(0)
 
 int
 print_spaces(char *buffer, int depth, int max) 
 {
 	int lpc = 0;
 	int spaces = 2*depth;
 	max--;
 	
 	/* <= so that we always print 1 space - prevents problems with syslog */
 	for(lpc = 0; lpc <= spaces && lpc < max; lpc++) {
 		if(sprintf(buffer+lpc, "%c", ' ') < 1) {
 			return -1;
 		}
 	}
 	return lpc;
 }
 
 int
 log_data_element(
 	const char *function, const char *prefix, int log_level, int depth,
 	xmlNode *data, gboolean formatted) 
 {
 	int child_result = 0;
 
 	int offset = 0;
 	int printed = 0;
 	char *buffer = NULL;
 	int buffer_len = 1000;
 
 	const char *name = NULL;
 	const char *hidden = NULL;
 
 	if(data == NULL) {
 	    crm_warn("No data to dump as XML");
 	    return 0;
 	}
 	
 	name = crm_element_name(data);
 	CRM_ASSERT(name != NULL);
 	
 	crm_debug_5("Dumping %s", name);
 	crm_malloc0(buffer, buffer_len);
 	
 	if(formatted) {
 	    offset = print_spaces(buffer, depth, buffer_len - offset);
 	}
 
 	printed = snprintf(buffer + offset, buffer_len - offset, "<%s", name);
 	update_buffer();
 	
 	hidden = crm_element_value(data, "hidden");
 	xml_prop_iter(
 		data, prop_name, prop_value,
 
 		if(prop_name == NULL
 		   || safe_str_eq(F_XML_TAGNAME, prop_name)) {
 			continue;
 
 		} else if(hidden != NULL
 			  && prop_name[0] != 0
 			  && strstr(hidden, prop_name) != NULL) {
 			prop_value = "*****";
 		}
 		
 		crm_debug_5("Dumping <%s %s=\"%s\"...",
 			    name, prop_name, prop_value);
 		printed = snprintf(buffer + offset, buffer_len - offset,
 				   " %s=\"%s\"", prop_name, prop_value);
 		update_buffer();
 		);
 
 	printed = snprintf(buffer + offset, buffer_len - offset,
 			   " %s>", xml_has_children(data)?"":"/");
 	update_buffer();
 	
   print:
 	do_crm_log(log_level, "%s: %s%s", function, prefix?prefix:"", buffer);
 	
 	if(xml_has_children(data) == FALSE) {
 		crm_free(buffer);
 		return 0;
 	}
 	
 	xml_child_iter(
 		data, a_child, 
 		child_result = log_data_element(
 			function, prefix, log_level, depth+1, a_child, formatted);
 		);
 
 	if(formatted) {
 		offset = print_spaces(buffer, depth, buffer_len);
 	}
 	do_crm_log(log_level, "%s: %s%s</%s>", function, prefix?prefix:"", buffer, name);
 	crm_free(buffer);
 	return 1;
 }
 
 gboolean
 xml_has_children(const xmlNode *xml_root)
 {
 	if(xml_root != NULL && xml_root->children != NULL) {
 	    return TRUE;
 	}
 	return FALSE;
 }
 
 void
 xml_validate(const xmlNode *xml_root)
 {
 	CRM_ASSERT(xml_root != NULL);
 }
 
 int
 crm_element_value_int(xmlNode *data, const char *name, int *dest)
 {
     const char *value = crm_element_value(data, name);
     CRM_CHECK(dest != NULL, return -1);
     if(value) {
 	*dest = crm_int_helper(value, NULL);
 	return 0;
     }
     return -1;
 }
 
 const char *
 crm_element_value_const(const xmlNode *data, const char *name)
 {
     return crm_element_value((xmlNode*)data, name);
 }
 
 char *
 crm_element_value_copy(xmlNode *data, const char *name)
 {
 	char *value_copy = NULL;
 	const char *value = crm_element_value(data, name);
 	if(value != NULL) {
 		value_copy = crm_strdup(value);
 	}
 	return value_copy;
 }
 
 void
 xml_remove_prop(xmlNode *obj, const char *name)
 {
     xmlUnsetProp(obj, (const xmlChar*)name);
 }
 
 void
 log_xml_diff(unsigned int log_level, xmlNode *diff, const char *function)
 {
 	xmlNode *added = find_xml_node(diff, "diff-added", FALSE);
 	xmlNode *removed = find_xml_node(diff, "diff-removed", FALSE);
 	gboolean is_first = TRUE;
 
 	if(crm_log_level < log_level) {
 		/* nothing will ever be printed */
 		return;
 	}
 	
 	xml_child_iter(
 		removed, child, 
 		log_data_element(function, "-", log_level, 0, child, TRUE);
 		if(is_first) {
 			is_first = FALSE;
 		} else {
 			do_crm_log(log_level, " --- ");
 		}
 		
 		);
 	is_first = TRUE;
 	xml_child_iter(
 		added, child, 
 		log_data_element(function, "+", log_level, 0, child, TRUE);
 		if(is_first) {
 			is_first = FALSE;
 		} else {
 			do_crm_log(log_level, " +++ ");
 		}
 		);
 }
 
 void
 purge_diff_markers(xmlNode *a_node)
 {
 	CRM_CHECK(a_node != NULL, return);
 
 	xml_remove_prop(a_node, XML_DIFF_MARKER);
 	xml_child_iter(a_node, child,
 		       purge_diff_markers(child);
 		);
 }
 
 gboolean
 apply_xml_diff(xmlNode *old, xmlNode *diff, xmlNode **new)
 {
 	gboolean result = TRUE;
 	const char *digest = crm_element_value(diff, XML_ATTR_DIGEST);
 	xmlNode *added = find_xml_node(diff, "diff-added", FALSE);
 	xmlNode *removed = find_xml_node(diff, "diff-removed", FALSE);
 
 	int root_nodes_seen = 0;
 
 	CRM_CHECK(new != NULL, return FALSE);
 
 	crm_debug_2("Substraction Phase");
 	xml_child_iter(removed, child_diff, 
 		       CRM_CHECK(root_nodes_seen == 0, result = FALSE);
 		       if(root_nodes_seen == 0) {
 			       *new = subtract_xml_object(old, child_diff, NULL);
 		       }
 		       root_nodes_seen++;
 		);
 	if(root_nodes_seen == 0) {
 		*new = copy_xml(old);
 		
 	} else if(root_nodes_seen > 1) {
 		crm_err("(-) Diffs cannot contain more than one change set..."
 			" saw %d", root_nodes_seen);
 		result = FALSE;
 	}
 
 	root_nodes_seen = 0;
 	crm_debug_2("Addition Phase");
 	if(result) {
 		xml_child_iter(added, child_diff, 
 			       CRM_CHECK(root_nodes_seen == 0, result = FALSE);
 			       if(root_nodes_seen == 0) {
 				       add_xml_object(NULL, *new, child_diff);
 			       }
 			       root_nodes_seen++;
 			);
 	}
 
 	if(root_nodes_seen > 1) {
 		crm_err("(+) Diffs cannot contain more than one change set..."
 			" saw %d", root_nodes_seen);
 		result = FALSE;
 
 	} else if(result && digest) {
 	    char *new_digest = calculate_xml_digest(*new, FALSE, TRUE);
 	    if(safe_str_neq(new_digest, digest)) {
 		crm_info("Digest mis-match: expected %s, calculated %s",
 			 digest, new_digest);
  		result = FALSE;
 	    } else {
 		crm_debug_2("Digest matched: expected %s, calculated %s",
 			    digest, new_digest);
 	    }
 	    crm_free(new_digest);
 	    
 	} else if(result) {
 		int lpc = 0;
 		xmlNode *intermediate = NULL;
 		xmlNode *diff_of_diff = NULL;
 		xmlNode *calc_added = NULL;
 		xmlNode *calc_removed = NULL;
 
 		const char *value = NULL;
 		const char *name = NULL;
 		const char *version_attrs[] = {
 			XML_ATTR_NUMUPDATES,
 			XML_ATTR_GENERATION,
 			XML_ATTR_GENERATION_ADMIN
 		};
 
 		crm_debug_2("Verification Phase");
 		intermediate = diff_xml_object(old, *new, FALSE);
 		calc_added = find_xml_node(intermediate, "diff-added", FALSE);
 		calc_removed = find_xml_node(intermediate, "diff-removed", FALSE);
 
 		/* add any version details to the diff so they match */
 		for(lpc = 0; lpc < DIMOF(version_attrs); lpc++) {
 			name = version_attrs[lpc];
 
 			value = crm_element_value(added, name);
 			crm_xml_add(calc_added, name, value);
 			
 			value = crm_element_value(removed, name);
 			crm_xml_add(calc_removed, name, value);	
 		}
 
 		diff_of_diff = diff_xml_object(intermediate, diff, TRUE);
 		if(diff_of_diff != NULL) {
 			crm_info("Diff application failed!");
 			crm_log_xml_debug(old, "diff:original");
 			crm_log_xml_debug(diff, "diff:input");
 			result = FALSE;
 		}
 		
 		free_xml(diff_of_diff);
 		free_xml(intermediate);
 		diff_of_diff = NULL;
 		intermediate = NULL;
 	}
 	
 	if(result) {
 		purge_diff_markers(*new);
 	}
 
 	return result;
 }
 
 
 xmlNode *
 diff_xml_object(xmlNode *old, xmlNode *new, gboolean suppress)
 {
 	xmlNode *diff = NULL;
 	xmlNode *tmp1 = NULL;
 	xmlNode *added = NULL;
 	xmlNode *removed = NULL;
 
 	tmp1 = subtract_xml_object(old, new, "removed:top");
 	if(tmp1 != NULL) {
 		if(suppress && can_prune_leaf(tmp1)) {
 			free_xml(tmp1);
 
 		} else {
 			diff = create_xml_node(NULL, "diff");
 			removed = create_xml_node(diff, "diff-removed");
 			added = create_xml_node(diff, "diff-added");
 			add_node_nocopy(removed, NULL, tmp1);
 		}
 	}
 	
 	tmp1 = subtract_xml_object(new, old, "added:top");
 	if(tmp1 != NULL) {
 		if(suppress && can_prune_leaf(tmp1)) {
 			free_xml(tmp1);
 			return diff;
 			
 		}
 
 		if(diff == NULL) {
 			diff = create_xml_node(NULL, "diff");
 		}
 		if(removed == NULL) {
-			removed = create_xml_node(diff, "diff-removed");
+			create_xml_node(diff, "diff-removed");
 		}
 		if(added == NULL) {
 			added = create_xml_node(diff, "diff-added");
 		}
 		add_node_nocopy(added, NULL, tmp1);
 	}
 
 	return diff;
 }
 
 gboolean
 can_prune_leaf(xmlNode *xml_node)
 {
 	gboolean can_prune = TRUE;
 /* 	return FALSE; */
 	
-	xml_prop_iter(xml_node, prop_name, prop_value,
+	xml_prop_name_iter(xml_node, prop_name,
 		      if(safe_str_eq(prop_name, XML_ATTR_ID)) {
 			      continue;
 		      }		      
 		      can_prune = FALSE;
 		);
 	xml_child_iter(xml_node, child, 
 		       if(can_prune_leaf(child)) {
 				free_xml(child);
 		       } else {
 			       can_prune = FALSE;
 		       }
 		);
 	return can_prune;
 }
 
 
 void
 diff_filter_context(int context, int upper_bound, int lower_bound,
 		    xmlNode *xml_node, xmlNode *parent) 
 {
 	xmlNode *us = NULL;
 	xmlNode *new_parent = parent;
 	const char *name = crm_element_name(xml_node);
 
 	CRM_CHECK(xml_node != NULL && name != NULL, return);
 	
 	us = create_xml_node(parent, name);
 	xml_prop_iter(xml_node, prop_name, prop_value,
 		      lower_bound = context;
 		      crm_xml_add(us, prop_name, prop_value);
 		);
 	if(lower_bound >= 0 || upper_bound >= 0) {
 		crm_xml_add(us, XML_ATTR_ID, ID(xml_node));
 		new_parent = us;
 
 	} else {
 		upper_bound = in_upper_context(0, context, xml_node);
 		if(upper_bound >= 0) {
 			crm_xml_add(us, XML_ATTR_ID, ID(xml_node));
 			new_parent = us;
 		} else {
 			free_xml(us);
 			us = NULL;
 		}
 	}
 
 	xml_child_iter(us, child, 
 		       diff_filter_context(
 			       context, upper_bound-1, lower_bound-1,
 			       child, new_parent);
 		);
 }
 
 int
 in_upper_context(int depth, int context, xmlNode *xml_node)
 {
 	gboolean has_attributes = FALSE;
 	if(context == 0) {
 		return 0;
 	}
 	
-	xml_prop_iter(xml_node, prop_name, prop_value,
+	xml_prop_name_iter(xml_node, prop_name,
 		      has_attributes = TRUE;
 		      break;
 		);
 	
 	if(has_attributes) {
 		return depth;
 
 	} else if(depth < context) {
 		xml_child_iter(xml_node, child, 
 			       if(in_upper_context(depth+1, context, child)) {
 				       return depth;
 			       }
 			);
 	}
 	return 0;       
 }
 
 
 xmlNode *
 subtract_xml_object(xmlNode *left, xmlNode *right, const char *marker)
 {
 	gboolean skip = FALSE;
 	gboolean differences = FALSE;
 	xmlNode *diff = NULL;
 	xmlNode *child_diff = NULL;
 	xmlNode *right_child = NULL;
 
 	const char *id = NULL;
 	const char *name = NULL;
 	const char *value = NULL;
 	const char *right_val = NULL;
 
 	int lpc = 0;
 	static int filter_len = DIMOF(filter);
 	
 	if(left == NULL) {
 		return NULL;
 	}
 
 	id = ID(left);
 	if(right == NULL) {
 		xmlNode *deleted = NULL;
 
 		crm_debug_5("Processing <%s id=%s> (complete copy)",
 			    crm_element_name(left), id);
 		deleted = copy_xml(left);
 		crm_xml_add(deleted, XML_DIFF_MARKER, marker);
 
 		return deleted;
 	}
 
 	name = crm_element_name(left);
 	CRM_CHECK(name != NULL, return NULL);
 
 	diff = create_xml_node(NULL, name);
 
 	/* changes to name/value pairs */
 	xml_prop_iter(left, prop_name, left_value,
 		      if(crm_str_eq(prop_name, XML_ATTR_ID, TRUE)) {
 			      continue;
 		      }
 
 		      skip = FALSE;
 		      for(lpc = 0; skip == FALSE && lpc < filter_len; lpc++){
 			      if(crm_str_eq(prop_name, filter[lpc], TRUE)) {
 				      skip = TRUE;
 			      }
 		      }
 		      
 		      if(skip) { continue; }
 		      
 		      right_val = crm_element_value(right, prop_name);
 		      if(right_val == NULL) {
 			  /* new */
 			  differences = TRUE;
 			  crm_xml_add(diff, prop_name, left_value);
 				      
 		      } else if(strcmp(left_value, right_val) == 0) {
 			  /* unchanged */
 
 		      } else {
 			  /* changed */
 			  differences = TRUE;
 			  crm_xml_add(diff, prop_name, left_value);
 		      }
 		);
 
 	/* changes to child objects */
 	xml_child_iter(
 		left, left_child,  
 		right_child = find_entity(
 			right, crm_element_name(left_child), ID(left_child));
 		child_diff = subtract_xml_object(
 			left_child, right_child, marker);
 		if(child_diff != NULL) {
 			differences = TRUE;
 			add_node_nocopy(diff, NULL, child_diff);
 		}
 		
 		);
 
 	if(differences == FALSE) {
 		/* check for XML_DIFF_MARKER in a child */ 
 		xml_child_iter(
 			right, right_child,  
 			value = crm_element_value(right_child, XML_DIFF_MARKER);
 			if(value != NULL && safe_str_eq(value, "removed:top")) {
 				crm_debug_3("Found the root of the deletion: %s", name);
 				differences = TRUE;
 				break;
 			}
 			);
 	}
 	
 	if(differences == FALSE) {
 		free_xml(diff);
 		crm_debug_5("\tNo changes to <%s id=%s>", crm_str(name), id);
 		return NULL;
 	}
 	crm_xml_add(diff, XML_ATTR_ID, id);
 	return diff;
 }
 
 int
 add_xml_object(xmlNode *parent, xmlNode *target, xmlNode *update)
 {
 	const char *object_id = NULL;
 	const char *object_name = NULL;
 
 #if XML_PARSE_DEBUG
 	crm_log_xml(LOG_DEBUG_5, "update:", update);
 	crm_log_xml(LOG_DEBUG_5, "target:", target);
 #endif
 
 	CRM_CHECK(update != NULL, return 0);
 
 	object_name = crm_element_name(update);
 	object_id = ID(update);
 
 	CRM_CHECK(object_name != NULL, return 0);
 	
 	if(target == NULL && object_id == NULL) {
 		/*  placeholder object */
 		target = find_xml_node(parent, object_name, FALSE);
 
 	} else if(target == NULL) {
 		target = find_entity(parent, object_name, object_id);
 	}
 
 	if(target == NULL) {
 		target = create_xml_node(parent, object_name);
 		CRM_CHECK(target != NULL, return 0);
 #if XML_PARSER_DEBUG
 		crm_debug_2("Added  <%s%s%s/>", crm_str(object_name),
 			    object_id?" id=":"", object_id?object_id:"");
 
 	} else {
 		crm_debug_3("Found node <%s%s%s/> to update",
 			    crm_str(object_name),
 			    object_id?" id=":"", object_id?object_id:"");
 #endif
 	}
 
 	copy_in_properties(target, update);
 
 	xml_child_iter(
 		update, a_child,  
 #if XML_PARSER_DEBUG
 		crm_debug_4("Updating child <%s id=%s>",
 			    crm_element_name(a_child), ID(a_child));
 #endif
 		add_xml_object(target, NULL, a_child);
 		);
 
 #if XML_PARSER_DEBUG
 	crm_debug_3("Finished with <%s id=%s>",
 		    crm_str(object_name), crm_str(object_id));
 #endif
 	return 0;
 }
 
 gboolean
 update_xml_child(xmlNode *child, xmlNode *to_update)
 {
 	gboolean can_update = TRUE;
 	
 	CRM_CHECK(child != NULL, return FALSE);
 	CRM_CHECK(to_update != NULL, return FALSE);
 	
 	if(safe_str_neq(crm_element_name(to_update), crm_element_name(child))) {
 		can_update = FALSE;
 
 	} else if(safe_str_neq(ID(to_update), ID(child))) {
 		can_update = FALSE;
 
 	} else if(can_update) {
 #if XML_PARSER_DEBUG
 		crm_log_xml_debug_2(child, "Update match found...");
 #endif
 		add_xml_object(NULL, child, to_update);
 	}
 	
 	xml_child_iter(
 		child, child_of_child, 
 		/* only update the first one */
 		if(can_update) {
 			break;
 		}
 		can_update = update_xml_child(child_of_child, to_update);
 		);
 	
 	return can_update;
 }
 
 
 int
 find_xml_children(xmlNode **children, xmlNode *root,
 		  const char *tag, const char *field, const char *value,
 		  gboolean search_matches)
 {
 	int match_found = 0;
 	
 	CRM_CHECK(root != NULL, return FALSE);
 	CRM_CHECK(children != NULL, return FALSE);
 	
 	if(tag != NULL && safe_str_neq(tag, crm_element_name(root))) {
 
 	} else if(value != NULL
 		  && safe_str_neq(value, crm_element_value(root, field))) {
 
 	} else {
 		if(*children == NULL) {
 			*children = create_xml_node(NULL, __FUNCTION__);
 		}
 		add_node_copy(*children, root);
 		match_found = 1;
 	}
 
 	if(search_matches || match_found == 0) {
 		xml_child_iter(
 			root, child, 
 			match_found += find_xml_children(
 				children, child, tag, field, value,
 				search_matches);
 			);
 	}
 	
 	return match_found;
 }
 
 gboolean
 replace_xml_child(xmlNode *parent, xmlNode *child, xmlNode *update, gboolean delete_only)
 {
 	gboolean can_delete = FALSE;
 
 	const char *up_id = NULL;
 	const char *child_id = NULL;
 	const char *right_val = NULL;
 	
 	CRM_CHECK(child != NULL, return FALSE);
 	CRM_CHECK(update != NULL, return FALSE);
 
 	up_id = ID(update);
 	child_id = ID(child);
 	
 	if(up_id == NULL || safe_str_eq(child_id, up_id)) {
 		can_delete = TRUE;
 	} 
 	if(safe_str_neq(crm_element_name(update), crm_element_name(child))) {
 		can_delete = FALSE;
 	}
 	if(can_delete && delete_only) {
 		xml_prop_iter(update, prop_name, left_value,
 			      right_val = crm_element_value(child, prop_name);
 			      if(safe_str_neq(left_value, right_val)) {
 				      can_delete = FALSE;
 			      }
 			);
 	}
 	
 	if(can_delete && parent != NULL) {
 		crm_log_xml_debug_4(child, "Delete match found...");
 		if(delete_only || update == NULL) {
 		    free_xml_from_parent(NULL, child);
 		    
 		} else {	
 		    xmlNode *tmp = copy_xml(update);
 		    xmlDoc *doc = tmp->doc;
 		    xmlNode *old = xmlReplaceNode(child, tmp);
 		    free_xml_from_parent(NULL, old);
 		    xmlDocSetRootElement(doc, NULL);
 		    xmlFreeDoc(doc);
 		}
 		child = NULL;
 		return TRUE;
 		
 	} else if(can_delete) {
 		crm_log_xml_debug(child, "Cannot delete the search root");
 		can_delete = FALSE;
 	}
 	
 	
 	xml_child_iter(
 		child, child_of_child, 
 		/* only delete the first one */
 		if(can_delete) {
 			break;
 		}
 		can_delete = replace_xml_child(child, child_of_child, update, delete_only);
 		);
 	
 	return can_delete;
 }
 
 void
 hash2nvpair(gpointer key, gpointer value, gpointer user_data) 
 {
 	const char *name    = key;
 	const char *s_value = value;
 
 	xmlNode *xml_node  = user_data;
 	xmlNode *xml_child = create_xml_node(xml_node, XML_CIB_TAG_NVPAIR);
 
 	crm_xml_add(xml_child, XML_ATTR_ID, name);
 	crm_xml_add(xml_child, XML_NVPAIR_ATTR_NAME, name);
 	crm_xml_add(xml_child, XML_NVPAIR_ATTR_VALUE, s_value);
 
 	crm_debug_3("dumped: name=%s value=%s", name, s_value);
 }
 
 void
 hash2smartfield(gpointer key, gpointer value, gpointer user_data) 
 {
 	const char *name    = key;
 	const char *s_value = value;
 
 	xmlNode *xml_node  = user_data;
 
 	if(isdigit(name[0])) {
 	    xmlNode *tmp = create_xml_node(xml_node, XML_TAG_PARAM);
 	    crm_xml_add(tmp, XML_NVPAIR_ATTR_NAME, name);
 	    crm_xml_add(tmp, XML_NVPAIR_ATTR_VALUE, s_value);
 	    
 	} else if(crm_element_value(xml_node, name) == NULL) {
 		crm_xml_add(xml_node, name, s_value);
 		crm_debug_3("dumped: %s=%s", name, s_value);
 
 	} else {
 		crm_debug_2("duplicate: %s=%s", name, s_value);
 	}
 }
 
 void
 hash2field(gpointer key, gpointer value, gpointer user_data) 
 {
 	const char *name    = key;
 	const char *s_value = value;
 
 	xmlNode *xml_node  = user_data;
 
 	if(crm_element_value(xml_node, name) == NULL) {
 		crm_xml_add(xml_node, name, s_value);
 		crm_debug_3("dumped: %s=%s", name, s_value);
 
 	} else {
 		crm_debug_2("duplicate: %s=%s", name, s_value);
 	}
 }
 
 void
 hash2metafield(gpointer key, gpointer value, gpointer user_data) 
 {
     char *crm_name = NULL;
     
     if(key == NULL || value == NULL) {
 	return;
     } else if(((char*)key)[0] == '#') {
 	return;
     } else if(strstr(key, ":")) {
 	return;
     }
     
     crm_name = crm_meta_name(key);
     hash2field(crm_name, value, user_data);
     crm_free(crm_name);
 }
 
 
 GHashTable *
 xml2list(xmlNode *parent)
 {
 	xmlNode *nvpair_list = NULL;
 	GHashTable *nvpair_hash = g_hash_table_new_full(
 		g_str_hash, g_str_equal,
 		g_hash_destroy_str, g_hash_destroy_str);
 	
 	CRM_CHECK(parent != NULL, return nvpair_hash);
 
 	nvpair_list = find_xml_node(parent, XML_TAG_ATTRS, FALSE);
 	if(nvpair_list == NULL) {
 		crm_debug_2("No attributes in %s",
 			    crm_element_name(parent));
 		crm_log_xml_debug_2(
 			parent,"No attributes for resource op");
 	}
 	
 	crm_log_xml_debug_3(nvpair_list, "Unpacking");
 
 	xml_prop_iter(
 		nvpair_list, key, value, 
 		
 		crm_debug_4("Added %s=%s", key, value);
 		
 		g_hash_table_insert(
 			nvpair_hash, crm_strdup(key), crm_strdup(value));
 		);
 
 	xml_child_iter_filter(
 	    nvpair_list, child, XML_TAG_PARAM,
 
 	    const char *key = crm_element_value(child, XML_NVPAIR_ATTR_NAME);
 	    const char *value = crm_element_value(child, XML_NVPAIR_ATTR_VALUE);
 	    crm_debug_4("Added %s=%s", key, value);
 	    if(key != NULL && value != NULL) {
 		g_hash_table_insert(nvpair_hash, crm_strdup(key), crm_strdup(value));		
 	    }
 	    );
 	
 	return nvpair_hash;
 }
 
 
 typedef struct name_value_s 
 {
 	const char *name;
 	const void *value;
 } name_value_t;
 
 static gint
 sort_pairs(gconstpointer a, gconstpointer b)
 {
     int rc = 0;
 	const name_value_t *pair_a = a;
 	const name_value_t *pair_b = b;
 
 	CRM_ASSERT(a != NULL);
 	CRM_ASSERT(pair_a->name != NULL);
 
 	CRM_ASSERT(b != NULL);
 	CRM_ASSERT(pair_b->name != NULL);
 
 	rc = strcmp(pair_a->name, pair_b->name);
 	if(rc < 0) {
 	    return -1;
 	} else if(rc > 0) {
 	    return 1;
 	}
 	return 0;
 }
 
 static void
 dump_pair(gpointer data, gpointer user_data)
 {
 	name_value_t *pair = data;
 	xmlNode *parent = user_data;
 	crm_xml_add(parent, pair->name, pair->value);
 }
 
 xmlNode *
 sorted_xml(xmlNode *input, xmlNode *parent, gboolean recursive)
 {
 	GListPtr sorted = NULL;
 	GListPtr unsorted = NULL;
 	name_value_t *pair = NULL;
 	xmlNode *result = NULL;
-	const char *name = crm_element_name(input);
+	const char *name = NULL;
 
 	CRM_CHECK(input != NULL, return NULL);
 	
 	name = crm_element_name(input);
 	CRM_CHECK(name != NULL, return NULL);
 
 	result = create_xml_node(parent, name);
 	
 	xml_prop_iter(input, p_name, p_value,
 		      crm_malloc0(pair, sizeof(name_value_t));
 		      pair->name  = p_name;
 		      pair->value = p_value;
 		      unsorted = g_list_prepend(unsorted, pair);
 		      pair = NULL;
 		);
 
 	sorted = g_list_sort(unsorted, sort_pairs);
 	g_list_foreach(sorted, dump_pair, result);
 	slist_destroy(name_value_t, child, sorted, crm_free(child));
 
 	if(recursive) {
 	    xml_child_iter(input, child, sorted_xml(child, result, recursive));
 	} else {
 	    xml_child_iter(input, child, add_node_copy(result, child));
 	}
 	
 	return result;
 }
 
 static void
 filter_xml(xmlNode *data, const char **filter, int filter_len, gboolean recursive) 
 {
     int lpc = 0;
     
     for(lpc = 0; lpc < filter_len; lpc++) {
 	xml_remove_prop(data, filter[lpc]);
     }
 
     if(recursive == FALSE) {
 	return;
     }
     
     xml_child_iter(data, child, filter_xml(child, filter, filter_len, recursive));
 }
 
 /* "c048eae664dba840e1d2060f00299e9d" */
 char *
 calculate_xml_digest(xmlNode *input, gboolean sort, gboolean do_filter)
 {
 	int i = 0;
 	int digest_len = 16;
 	char *digest = NULL;
 	unsigned char *raw_digest = NULL;
 	xmlNode *sorted = NULL;
 	char *buffer = NULL;
 	size_t buffer_len = 0;
 
 	if(sort || do_filter) {
 	    sorted = sorted_xml(input, NULL, TRUE);
 	} else {
 	    sorted = copy_xml(input);
 	}
 
 	if(do_filter) {
 	    filter_xml(sorted, filter, DIMOF(filter), TRUE);
 	}
 
 	buffer = dump_xml(sorted, FALSE, TRUE);
 	buffer_len = strlen(buffer);
 	
 	CRM_CHECK(buffer != NULL && buffer_len > 0, free_xml(sorted); crm_free(buffer); return NULL);
 
 	crm_malloc(digest, (2 * digest_len + 1));
 	crm_malloc(raw_digest, (digest_len + 1));
 	MD5((unsigned char *)buffer, buffer_len, raw_digest);
 	for(i = 0; i < digest_len; i++) {
  		sprintf(digest+(2*i), "%02x", raw_digest[i]);
  	}
 	digest[(2*digest_len)] = 0;
 	crm_debug_2("Digest %s: %s\n", digest, buffer);
 	crm_log_xml(LOG_DEBUG_3,  "digest:source", sorted);
 	crm_free(buffer);
 	crm_free(raw_digest);
 	free_xml(sorted);
 	return digest;
 }
 
 
 #if HAVE_LIBXML2
 #  include <libxml/parser.h>
 #  include <libxml/tree.h>
 #  include <libxml/relaxng.h>
 #  include <libxslt/xslt.h>
 #  include <libxslt/transform.h>
 #endif
 
 static gboolean
 validate_with_dtd(
 	xmlDocPtr doc, gboolean to_logs, const char *dtd_file) 
 {
 	gboolean valid = TRUE;
 
 	xmlDtdPtr dtd = NULL;
 	xmlValidCtxtPtr cvp = NULL;
 	
 	CRM_CHECK(doc != NULL, return FALSE);
 	CRM_CHECK(dtd_file != NULL, return FALSE);
 
 	dtd = xmlParseDTD(NULL, (const xmlChar *)dtd_file);
 	CRM_CHECK(dtd != NULL, crm_err("Could not find/parse %s", dtd_file); goto cleanup);
 
 	cvp = xmlNewValidCtxt();
 	CRM_CHECK(cvp != NULL, goto cleanup);
 
 	if(to_logs) {
 		cvp->userData = (void *) LOG_ERR;
 		cvp->error    = (xmlValidityErrorFunc) cl_log;
 		cvp->warning  = (xmlValidityWarningFunc) cl_log;
 	} else {
 		cvp->userData = (void *) stderr;
 		cvp->error    = (xmlValidityErrorFunc) fprintf;
 		cvp->warning  = (xmlValidityWarningFunc) fprintf;
 	}
 	
 	if (!xmlValidateDtd(cvp, doc, dtd)) {
 		valid = FALSE;
 	}
 	
   cleanup:
 	if(cvp) {
 		xmlFreeValidCtxt(cvp);
 	}
 	if(dtd) {
 		xmlFreeDtd(dtd);
 	}
 	
 	return valid;
 }
 
 xmlNode *first_named_child(xmlNode *parent, const char *name) 
 {
     xml_child_iter_filter(parent, match, name, return match);
     return NULL;
 }
 
 #if 0
 static void relaxng_invalid_stderr(void * userData, xmlErrorPtr error)
 {
     /*
 Structure xmlError
 struct _xmlError {
     int	domain	: What part of the library raised this er
     int	code	: The error code, e.g. an xmlParserError
     char *	message	: human-readable informative error messag
     xmlErrorLevel	level	: how consequent is the error
     char *	file	: the filename
     int	line	: the line number if available
     char *	str1	: extra string information
     char *	str2	: extra string information
     char *	str3	: extra string information
     int	int1	: extra number information
     int	int2	: column number of the error or 0 if N/A
     void *	ctxt	: the parser context if available
     void *	node	: the node in the tree
 }
      */
     crm_err("Structured error: line=%d, level=%d %s",
 	    error->line, error->level, error->message);
 }
 #endif
 
 static gboolean
 validate_with_relaxng(
     xmlDocPtr doc, gboolean to_logs, const char *relaxng_file) 
 {
     gboolean valid = TRUE;
     int rc = 0;
 
     xmlRelaxNGPtr rng = NULL;
     xmlRelaxNGValidCtxtPtr valid_ctx = NULL;
     xmlRelaxNGParserCtxtPtr parser_ctx = NULL;
     
     CRM_CHECK(doc != NULL, return FALSE);
     CRM_CHECK(relaxng_file != NULL, return FALSE);
 
     xmlLoadExtDtdDefaultValue = 1;
     parser_ctx = xmlRelaxNGNewParserCtxt(relaxng_file);
     CRM_CHECK(parser_ctx != NULL, goto cleanup);
 
     if(to_logs) {
 	xmlRelaxNGSetParserErrors(parser_ctx,
 				  (xmlRelaxNGValidityErrorFunc) cl_log,
 				  (xmlRelaxNGValidityWarningFunc) cl_log,
 				  GUINT_TO_POINTER(LOG_ERR));
     } else {
 	xmlRelaxNGSetParserErrors(parser_ctx,
 				  (xmlRelaxNGValidityErrorFunc) fprintf,
 				  (xmlRelaxNGValidityWarningFunc) fprintf,
 				  stderr);
     }
 
     rng = xmlRelaxNGParse(parser_ctx);
     CRM_CHECK(rng != NULL, crm_err("Could not find/parse %s", relaxng_file); goto cleanup);
 
     valid_ctx = xmlRelaxNGNewValidCtxt(rng);
     CRM_CHECK(valid_ctx != NULL, goto cleanup);
 
     if(to_logs) {
 	xmlRelaxNGSetValidErrors(valid_ctx,
 				 (xmlRelaxNGValidityErrorFunc) cl_log,
 				 (xmlRelaxNGValidityWarningFunc) cl_log,
 				 GUINT_TO_POINTER(LOG_ERR));
     } else {
 	xmlRelaxNGSetValidErrors(valid_ctx,
 				 (xmlRelaxNGValidityErrorFunc) fprintf,
 				 (xmlRelaxNGValidityWarningFunc) fprintf,
 				 stderr);
     }
 
     /* xmlRelaxNGSetValidStructuredErrors( */
     /* 	valid_ctx, relaxng_invalid_stderr, valid_ctx); */
     
     xmlLineNumbersDefault(1);
     rc = xmlRelaxNGValidateDoc(valid_ctx, doc);
     if (rc > 0) {
 	valid = FALSE;
 
     } else if (rc < 0) {
 	crm_err("Internal libxml error during validation\n");
     }
 
   cleanup:
     if(parser_ctx != NULL) {
 	xmlRelaxNGFreeParserCtxt(parser_ctx);
 	xmlCleanupParser();
     }
 
     if(valid_ctx != NULL) {
 	xmlRelaxNGFreeValidCtxt(valid_ctx);
     }
     
     if (rng != NULL) {
 	xmlRelaxNGFree(rng);    
     }
     return valid;
 }
 
 static gboolean validate_with(xmlNode *xml, int method, gboolean to_logs) 
 {
     xmlDocPtr doc = NULL;
     gboolean valid = FALSE;
     int type = known_schemas[method].type;
     const char *file = known_schemas[method].location;    
 
     CRM_CHECK(xml != NULL, return FALSE);
     doc = getDocPtr(xml);
     
     crm_debug_2("Validating with: %s (type=%d)", crm_str(file), type);
     switch(type) {
 	case 0:
 	    valid = TRUE;
 	    break;
 	case 1:
 	    valid = validate_with_dtd(doc, to_logs, file);
 	    break;
 	case 2:
 	    valid = validate_with_relaxng(doc, to_logs, file);
 	    break;
 	default:
 	    crm_err("Unknown validator type: %d", type);
 	    break;
     }
 
     return valid;
 }
 
 #include <stdio.h>
 static void dump_file(const char *filename) 
 {
 
     FILE *fp = NULL;
     int ch, line = 0;
 
     CRM_CHECK(filename != NULL, return);
 
     fp = fopen(filename, "r");
     CRM_CHECK(fp != NULL, return);
 
     fprintf(stderr, "%4d ", ++line);
     do {
 	ch = getc(fp);
 	if(ch == EOF) {
 	    putc('\n', stderr);
 	    break;
 	} else if(ch == '\n') {
 	    fprintf(stderr, "\n%4d ", ++line);
 	} else {
 	    putc(ch, stderr);
 	}
     } while(1);
     
     fclose(fp);
 }
 
 gboolean validate_xml_verbose(xmlNode *xml_blob) 
 {
     xmlDoc *doc = NULL;
     xmlNode *xml = NULL;
     gboolean rc = FALSE;
 
     char *filename = NULL;
     static char *template = NULL;
     if(template == NULL) {
 	template = crm_strdup(CRM_STATE_DIR"/cib-invalid.XXXXXX");
     }
     
     filename = mktemp(template);
     write_xml_file(xml_blob, filename, FALSE);
     
     dump_file(filename);
     
     doc = xmlParseFile(filename);
     xml = xmlDocGetRootElement(doc);
     rc = validate_xml(xml, NULL, FALSE);
     free_xml(xml);
     
     return rc;
 }
 
 gboolean validate_xml(xmlNode *xml_blob, const char *validation, gboolean to_logs)
 {
     int lpc = 0;
     
     if(validation == NULL) {
 	validation = crm_element_value(xml_blob, XML_ATTR_VALIDATION);
     }
 
     if(validation == NULL) {
 	validation = crm_element_value(xml_blob, "ignore-dtd");
 	if(crm_is_true(validation)) {
 	    validation = "none";
 	} else {
 	    validation = "transitional-0.6";
 	}
     }
     
     if(safe_str_eq(validation, "none")) {
 	return TRUE;
     }
     
     for(; lpc < all_schemas; lpc++) {
 	if(safe_str_eq(validation, known_schemas[lpc].name)) {
 	    return validate_with(xml_blob, lpc, to_logs);
 	}
     }
 
     crm_err("Unknown validator: %s", validation);
     return FALSE;
 }
 
 static xmlNode *apply_transformation(xmlNode *xml, const char *transform) 
 {
     xmlNode *out = NULL;
     xmlDocPtr res = NULL;
     xmlDocPtr doc = NULL;
     xsltStylesheet *xslt = NULL;
 
     CRM_CHECK(xml != NULL, return FALSE);
     doc = getDocPtr(xml);
 
     xmlLoadExtDtdDefaultValue = 1;
     xmlSubstituteEntitiesDefault(1);
     
     xslt = xsltParseStylesheetFile((const xmlChar *)transform);
     CRM_CHECK(xslt != NULL, goto cleanup);
     
     res = xsltApplyStylesheet(xslt, doc, NULL);
     CRM_CHECK(res != NULL, goto cleanup);
 
     out = xmlDocGetRootElement(res);
     
   cleanup:
     if(xslt) {
 	xsltFreeStylesheet(xslt);
     }
 
     xsltCleanupGlobals();
     xmlCleanupParser();
     
     return out;
 }
 
 const char *get_schema_name(int version)
 {
     if(version < 0 || version >= all_schemas) {
 	return "unknown";
     }
     return known_schemas[version].name;
 }
 
 
 int get_schema_version(const char *name) 
 {
     int lpc = 0;
     for(; lpc < all_schemas; lpc++) {
 	if(safe_str_eq(name, known_schemas[lpc].name)) {
 	    return lpc;
 	}
     }
     return -1;
 }
 
 /* set which validation to use */
 #include <crm/cib.h>
 int update_validation(
     xmlNode **xml_blob, int *best, gboolean transform, gboolean to_logs) 
 {
     xmlNode *xml = NULL;
     char *value = NULL;
     int lpc = 0, match = -1, rc = cib_ok;
 
     CRM_CHECK(best != NULL, return cib_invalid_argument);
     CRM_CHECK(xml_blob != NULL, return cib_invalid_argument);
     CRM_CHECK(*xml_blob != NULL, return cib_invalid_argument);
     
     *best = 0;
     xml = *xml_blob;
     value = crm_element_value_copy(xml, XML_ATTR_VALIDATION);
 
     if(value != NULL) {
 	match = get_schema_version(value);
 	
 	lpc = match;
 	if(lpc >= 0 && transform == FALSE) {
 	    lpc++;
 
 	} else if(lpc < 0) {
 	    crm_debug("Unknown validation type");
 	    lpc = 0;
 	}
     }
 
     if(match >= max_schemas) {
 	/* nothing to do */
 	crm_free(value);
 	*best = match;
 	return cib_ok;
     }
     
     for(; lpc < max_schemas; lpc++) {
 	gboolean valid = TRUE;
 	crm_debug("Testing '%s' validation", known_schemas[lpc].name?known_schemas[lpc].name:"<unset>");
 	valid = validate_with(xml, lpc, to_logs);
 	
 	if(valid) {
 	    *best = lpc;
 	}
 	
 	if(valid && transform) {
 	    xmlNode *upgrade = NULL;
 	    int next = known_schemas[lpc].after_transform;
 	    if(next <= 0) {
 		next = lpc+1;
 	    }
 	    
 	    crm_notice("Upgrading %s-style configuration to %s with %s",
 		       known_schemas[lpc].name, known_schemas[next].name, known_schemas[lpc].transform?known_schemas[lpc].transform:"no-op");
 
 	    if(known_schemas[lpc].transform == NULL) {
 		if(validate_with(xml, next, to_logs)) {
 		    crm_debug("Configuration valid for schema: %s", known_schemas[next].name);
 		    lpc = next; *best = next;
 		    rc = cib_ok;
 
 		} else {
 		    crm_info("Configuration not valid for schema: %s", known_schemas[next].name);
 		}
 		
 	    } else {
 		upgrade = apply_transformation(xml, known_schemas[lpc].transform);
 
 		if(upgrade == NULL) {
 		    crm_err("Transformation %s failed", known_schemas[lpc].transform);
 		    rc = cib_transform_failed;
 		    
 		} else if(validate_with(upgrade, next, to_logs)) {
 		    crm_info("Transformation %s successful", known_schemas[lpc].transform);
 		    lpc = next; *best = next;
 		    free_xml(xml);
 		    xml = upgrade;
 		    rc = cib_ok;
 		    
 		} else {
 		    crm_err("Transformation %s did not produce a valid configuration", known_schemas[lpc].transform);
 		    crm_log_xml_info(upgrade, "transform:bad");
 		    free_xml(upgrade);
 		    rc = cib_dtd_validation;
 		}
 	    }
 	}
     }
     
     if(*best > match) {
 	crm_notice("Upgraded from %s to %s validation", value?value:"<none>", known_schemas[*best].name);
 	crm_xml_add(xml, XML_ATTR_VALIDATION, known_schemas[*best].name);
     }
 
     *xml_blob = xml;
     crm_free(value);
     return rc;
 }
 
 xmlNode *
 getXpathResult(xmlXPathObjectPtr xpathObj, int index) 
 {
     xmlNode *match = NULL;
     CRM_CHECK(index >= 0, return NULL);
     CRM_CHECK(xpathObj != NULL, return NULL);
 
     if(index >= xpathObj->nodesetval->nodeNr) {
 	crm_err("Requested index %d of only %d items", index, xpathObj->nodesetval->nodeNr);
 	return NULL;
     }
     
     match = xpathObj->nodesetval->nodeTab[index];
     CRM_CHECK(match != NULL, return NULL);
 
     /*
      * From xpath2.c
      *
      * All the elements returned by an XPath query are pointers to
      * elements from the tree *except* namespace nodes where the XPath
      * semantic is different from the implementation in libxml2 tree.
      * As a result when a returned node set is freed when
      * xmlXPathFreeObject() is called, that routine must check the
      * element type. But node from the returned set may have been removed
      * by xmlNodeSetContent() resulting in access to freed data.
      * This can be exercised by running
      *       valgrind xpath2 test3.xml '//discarded' discarded
      * There is 2 ways around it:
      *   - make a copy of the pointers to the nodes from the result set 
      *     then call xmlXPathFreeObject() and then modify the nodes
      * or
      *   - remove the reference to the modified nodes from the node set
      *     as they are processed, if they are not namespace nodes.
      */
     if (xpathObj->nodesetval->nodeTab[index]->type != XML_NAMESPACE_DECL) {
 	xpathObj->nodesetval->nodeTab[index] = NULL;
     }
 
     if(match->type == XML_DOCUMENT_NODE) {
 	/* Will happen if section = '/' */
 	match = match->children;
 
     } else if(match->type != XML_ELEMENT_NODE
 	      && match->parent
 	      && match->parent->type == XML_ELEMENT_NODE) {
 	/* reurning the parent instead */
 	match = match->parent;
 	
     } else if(match->type != XML_ELEMENT_NODE) {
 	/* We only support searching nodes */
 	crm_err("We only support %d not %d", XML_ELEMENT_NODE, match->type);
 	match = NULL;
     }
     return match;
 }
 
 /* the caller needs to check if the result contains a xmlDocPtr or xmlNodePtr */
 xmlXPathObjectPtr 
 xpath_search(xmlNode *xml_top, const char *path)
 {
     xmlDocPtr doc = NULL;
     xmlXPathObjectPtr xpathObj = NULL; 
     xmlXPathContextPtr xpathCtx = NULL; 
     const xmlChar *xpathExpr = (const xmlChar *)path;
 
     CRM_CHECK(path != NULL, return NULL);
     CRM_CHECK(xml_top != NULL, return NULL);
     CRM_CHECK(strlen(path) > 0, return NULL);
     
     doc = getDocPtr(xml_top);
 
     crm_debug_2("Evaluating: %s", path);
     xpathCtx = xmlXPathNewContext(doc);
     CRM_ASSERT(xpathCtx != NULL);
     
     xpathObj = xmlXPathEvalExpression(xpathExpr, xpathCtx);
     xmlXPathFreeContext(xpathCtx);
     return xpathObj;
 }
 
 gboolean
 cli_config_update(xmlNode **xml, int *best_version, gboolean to_logs) 
 {
     gboolean rc = TRUE;
     static int min_version = -1;
     static int max_version = -1;
 
     const char *value = crm_element_value(*xml, XML_ATTR_VALIDATION);
     int version = get_schema_version(value);
 
     if(min_version < 0) {
 	min_version = get_schema_version(MINIMUM_SCHEMA_VERSION);
     }
     if(max_version < 0) {
 	max_version = get_schema_version(LATEST_SCHEMA_VERSION);
     }
     
     if(version < min_version) {
 	xmlNode *converted = NULL;
 	
 	converted = copy_xml(*xml);
 	update_validation(&converted, &version, TRUE, to_logs);
 	
 	value = crm_element_value(converted, XML_ATTR_VALIDATION);
 	if(version < min_version) {
 	    if(to_logs) {
 		crm_config_err("Your current configuration could only be upgraded to %s... "
 			"the minimum requirement is %s.\n", crm_str(value), MINIMUM_SCHEMA_VERSION);
 	    } else {
 		fprintf(stderr, "Your current configuration could only be upgraded to %s... "
 			"the minimum requirement is %s.\n", crm_str(value), MINIMUM_SCHEMA_VERSION);
 	    }
 	    
 	    free_xml(converted);
 	    converted = NULL;
 	    rc = FALSE;
 	    
 	} else {
 	    free_xml(*xml);
 	    *xml = converted;
 
 	    if(version < max_version) {
 		crm_config_warn("Your configuration was internally updated to %s... "
 				"which is acceptable but not the most recent",
 				get_schema_name(version));
 		
 	    } else if(to_logs){
 		crm_info("Your configuration was internally updated to the latest version (%s)",
 				get_schema_name(version));
 	    } 
 	}
     } else if(version > max_version) {
 	if(to_logs){
 	    crm_config_warn("Configuration validation is currently disabled."
 			    " It is highly encouraged and prevents many common cluster issues.");
 
 	} else {
 	    fprintf(stderr, "Configuration validation is currently disabled."
 			    " It is highly encouraged and prevents many common cluster issues.\n");
 	}
     }
 
     if(best_version) {
 	*best_version = version;	    
     }
     
     return rc;
 }
 
 xmlNode *expand_idref(xmlNode *input, xmlNode *top) 
 {
     const char *tag = NULL;
     const char *ref = NULL;
     xmlNode *result = input;
     char *xpath_string = NULL;
 
     if(result == NULL) {
 	return NULL;
 
     } else if(top == NULL) {
 	top = input;
     }
 
     tag = crm_element_name(result);
     ref = crm_element_value(result, XML_ATTR_IDREF);
     
     if(ref != NULL) {
 	int xpath_max = 512, offset = 0;
 	crm_malloc0(xpath_string, xpath_max);
 
 	offset += snprintf(xpath_string + offset, xpath_max - offset, "//%s[@id='%s']", tag, ref);
 	result = get_xpath_object(xpath_string, top, LOG_ERR);
 	if(result == NULL) {
 	    char *nodePath = (char *)xmlGetNodePath(top);
 	    crm_err("No match for %s found in %s: Invalid configuration", xpath_string, crm_str(nodePath));
 	    crm_free(nodePath);
 	}
     }
     
     crm_free(xpath_string);
     return result;
 }
 
 xmlNode*
 get_xpath_object_relative(const char *xpath, xmlNode *xml_obj, int error_level)
 {
     int len = 0;
     xmlNode *result = NULL;
     char *xpath_full = NULL;
     char *xpath_prefix = NULL;
     
     if(xml_obj == NULL || xpath == NULL) {
 	return NULL;
     }
 
     xpath_prefix = (char *)xmlGetNodePath(xml_obj);
     len += strlen(xpath_prefix);
     len += strlen(xpath);
 
     xpath_full = crm_strdup(xpath_prefix);
     crm_realloc(xpath_full, len+1);
     strncat(xpath_full, xpath, len);
 
     result = get_xpath_object(xpath_full, xml_obj, error_level);
 
     crm_free(xpath_prefix);
     crm_free(xpath_full);
     return result;
 }
 
 xmlNode*
 get_xpath_object(const char *xpath, xmlNode *xml_obj, int error_level)
 {
     xmlNode *result = NULL;
     xmlXPathObjectPtr xpathObj = NULL;
     char *nodePath = NULL;
     char *matchNodePath = NULL;
 
     if(xpath == NULL) {
 	return xml_obj; /* or return NULL? */
     }
     
     xpathObj = xpath_search(xml_obj, xpath);
     nodePath = (char *)xmlGetNodePath(xml_obj);
     if(xpathObj == NULL || xpathObj->nodesetval == NULL || xpathObj->nodesetval->nodeNr < 1) {
 	do_crm_log(error_level, "No match for %s in %s", xpath, crm_str(nodePath));
 	crm_log_xml(LOG_DEBUG_2, "Bad Input", xml_obj);
 	
     } else if(xpathObj->nodesetval->nodeNr > 1) {
 	int lpc = 0, max = xpathObj->nodesetval->nodeNr;
 	do_crm_log(error_level, "Too many matches for %s in %s", xpath, crm_str(nodePath));
 
 	for(lpc = 0; lpc < max; lpc++) {
 	    xmlNode *match = getXpathResult(xpathObj, lpc);
 	    CRM_CHECK(match != NULL, continue);
 
 	    matchNodePath = (char *)xmlGetNodePath(match);
 	    do_crm_log(error_level, "%s[%d] = %s", xpath, lpc, crm_str(matchNodePath));
 	    crm_free(matchNodePath);
 	}
 	crm_log_xml(LOG_DEBUG_2, "Bad Input", xml_obj);
 
     } else {
 	result = getXpathResult(xpathObj, 0);
     }
     
     if(xpathObj) {
 	xmlXPathFreeObject(xpathObj);
     }
     crm_free(nodePath);
 
     return result;
 }
 
 const char *
 crm_element_value(xmlNode *data, const char *name)
 {
     xmlAttr *attr = NULL;
     
     if(data == NULL) {
 	crm_err("Couldn't find %s in NULL", name?name:"<null>");
 	return NULL;
 
     } else if(name == NULL) {
 	crm_err("Couldn't find NULL in %s", crm_element_name(data));
 	return NULL;
     }
     
     attr = xmlHasProp(data, (const xmlChar*)name);
     if(attr == NULL || attr->children == NULL) {
 	return NULL;
     }
     return (const char*)attr->children->content;
 }
diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c
index 53150af122..977e3fd185 100644
--- a/lib/fencing/st_client.c
+++ b/lib/fencing/st_client.c
@@ -1,1591 +1,1594 @@
 /*
  * Copyright (c) 2004 Andrew Beekhof <andrew@beekhof.net>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This library is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  * 
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  *
  */
 #include <crm_internal.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <stdarg.h>
 #include <string.h>
 #include <ctype.h>
 
 #include <sys/types.h>
 #include <sys/wait.h>
 
 #include <glib.h>
 
 #include <crm/crm.h>
 #include <crm/stonith-ng.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <stonith/stonith.h>
 
 #define FE_AGENT_FORK		-2
 #define FE_AGENT_ERROR		-3
 
 
 typedef struct stonith_private_s 
 {
 	char *token;
 	IPC_Channel	*command_channel;
 	IPC_Channel	*callback_channel;
 	GCHSource	*callback_source;
 	GHashTable      *stonith_op_callback_table;
 
 	void (*op_callback)(
 	    stonith_t *st, const xmlNode *msg, int call, int rc, xmlNode *output, void *userdata);
 		
 } stonith_private_t;
 
 typedef struct stonith_notify_client_s 
 {
 	const char *event;
 	const char *obj_id;   /* implement one day */
 	const char *obj_type; /* implement one day */
 	void (*notify)(stonith_t *st, const char *event, xmlNode *msg);
 	
 } stonith_notify_client_t;
 
 typedef struct stonith_callback_client_s 
 {
 	void (*callback)(
 	    stonith_t *st, const xmlNode *msg, int call, int rc, xmlNode *output, void *userdata);
 	const char *id;
 	void *user_data;
 	gboolean only_success;
 	struct timer_rec_s *timer;
 	
 } stonith_callback_client_t;
 
 struct notify_blob_s 
 {
 	stonith_t *stonith;
 	xmlNode *xml;
 };
 
 struct timer_rec_s 
 {
 	int call_id;
 	int timeout;
 	guint ref;	
 	stonith_t *stonith;
 };
 
 typedef enum stonith_errors (*stonith_op_t)(
     const char *, int, const char *, xmlNode *,
     xmlNode*, xmlNode*, xmlNode**, xmlNode**);
 
 static const char META_TEMPLATE[] =
 "<?xml version=\"1.0\"?>\n"
 "<!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n"
 "<resource-agent name=\"%s\">\n"
 "  <version>1.0</version>\n"
 "  <longdesc lang=\"en\">\n"
 "%s\n"
 "  </longdesc>\n"	
 "  <shortdesc lang=\"en\">%s</shortdesc>\n"
 "%s\n"
 "  <actions>\n"
 "    <action name=\"start\"   timeout=\"15\" />\n"
 "    <action name=\"stop\"    timeout=\"15\" />\n"
 "    <action name=\"status\"  timeout=\"15\" />\n"
 "    <action name=\"monitor\" timeout=\"15\" interval=\"15\" start-delay=\"15\" />\n"
 "    <action name=\"meta-data\"  timeout=\"15\" />\n"
 "  </actions>\n"
 "  <special tag=\"heartbeat\">\n"
 "    <version>2.0</version>\n"
 "  </special>\n"
 "</resource-agent>\n";
 
 gboolean stonith_dispatch(IPC_Channel *channel, gpointer user_data);
 void stonith_perform_callback(stonith_t *stonith, xmlNode *msg, int call_id, int rc);
 xmlNode *stonith_create_op(
     int call_id, const char *token, const char *op, xmlNode *data, int call_options);
 int stonith_send_command(
     stonith_t *stonith, const char *op, xmlNode *data,
     xmlNode **output_data, int call_options, int timeout);
 
 static void stonith_connection_destroy(gpointer user_data);
 static void stonith_send_notification(gpointer data, gpointer user_data);
 
 static void stonith_connection_destroy(gpointer user_data)
 {
     stonith_t *stonith = user_data;
     stonith_private_t *native = NULL;
     struct notify_blob_s blob;
 
     blob.stonith = stonith;
     blob.xml = create_xml_node(NULL, "notify");;
 
     native = stonith->private;
     native->callback_source = NULL;
 
     stonith->state = stonith_disconnected;
     crm_xml_add(blob.xml, F_TYPE, T_STONITH_NOTIFY);
     crm_xml_add(blob.xml, F_SUBTYPE, T_STONITH_NOTIFY_DISCONNECT);
 
     g_list_foreach(stonith->notify_list, stonith_send_notification, &blob);
     free_xml(blob.xml);
 }
 
 static int stonith_api_register_device(
     stonith_t *stonith, int call_options,
     const char *id, const char *namespace, const char *agent, GHashTable *params)
 {
     int rc = 0;
     xmlNode *data = create_xml_node(NULL, F_STONITH_DEVICE);
     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
 
     crm_xml_add(data, XML_ATTR_ID, id);
     crm_xml_add(data, "origin", __FUNCTION__);
     crm_xml_add(data, "agent", agent);
     crm_xml_add(data, "namespace", namespace);
 
     g_hash_table_foreach(params, hash2field, args);
     
     rc = stonith_send_command(stonith, STONITH_OP_DEVICE_ADD, data, NULL, call_options, 0);
     free_xml(data);
     
     return rc;
 }
 
 static int stonith_api_remove_device(
     stonith_t *stonith, int call_options, const char *name)
 {
     int rc = 0;
     xmlNode *data = NULL;
 
     data = create_xml_node(NULL, F_STONITH_DEVICE);
     crm_xml_add(data, "origin", __FUNCTION__);
     crm_xml_add(data, XML_ATTR_ID, name);
     rc = stonith_send_command(stonith, STONITH_OP_DEVICE_DEL, data, NULL, call_options, 0);
     free_xml(data);
     
     return rc;
 }
 
 static void append_arg(
     gpointer key, gpointer value, gpointer user_data) 
 {
     int len = 3; /* =, \n, \0 */ 
     int last = 0;
     char **args = user_data;
 
     CRM_CHECK(key != NULL, return);
     CRM_CHECK(value != NULL, return);
 
     if(strstr(key, "pcmk_")) {
 	return;
     } else if(strstr(key, CRM_META)) {
 	return;
     } else if(safe_str_eq(key, "crm_feature_set")) {
 	return;
     }
     
     len += strlen(key);
     len += strlen(value);
     if(*args != NULL) {
 	last = strlen(*args);
     }
     
     crm_realloc(*args, last+len);
     crm_debug_2("Appending: %s=%s", (char *)key, (char *)value);
     sprintf((*args)+last, "%s=%s\n", (char *)key, (char *)value);
 }
 
 static void append_const_arg(const char *key, const char *value, char **arg_list) 
 {
     char *glib_sucks_key = crm_strdup(key);
     char *glib_sucks_value = crm_strdup(value);
     
     append_arg(glib_sucks_key, glib_sucks_value, arg_list);
 
     crm_free(glib_sucks_value);
     crm_free(glib_sucks_key);
 }
 
 static void append_host_specific_args(const char *victim, const char *map, GHashTable *params, char **arg_list) 
 {
     char *name = NULL;
     int last = 0, lpc = 0, max = 0;
 
     if(map == NULL) {
 	/* The best default there is for now... */
 	crm_debug("Using default arg map: port=uname");
 	append_const_arg("port", victim, arg_list);	
 	return;
     }
     
     max = strlen(map);
     crm_debug("Processing arg map: %s", map);
     for(; lpc < max + 1; lpc++) {
 	if(isalpha(map[lpc])) {
 	    /* keep going */
 	    
 	} else if(map[lpc] == '=' || map[lpc] == ':') {
 	    crm_malloc0(name, 1 + lpc - last);
 	    strncpy(name, map + last, lpc - last);
 	    crm_debug("Got name: %s", name);
 	    last = lpc + 1;
 	    
 	} else if(map[lpc] == 0 || map[lpc] == ',' || isspace(map[lpc])) {
 	    char *param = NULL;
 	    const char *value = NULL;
 	    
 	    crm_malloc0(param, 1 + lpc - last);
 	    strncpy(param, map + last, lpc - last);
 	    last = lpc + 1;
 
 	    crm_debug("Got key: %s", param);
 	    if(name == NULL) {
 		crm_err("Misparsed '%s', found '%s' without a name", map, param);
 		crm_free(param);
 		continue;
 	    }
 
 	    if(safe_str_eq(param, "uname")) {
 		value = victim;
 	    } else {
 		char *key = crm_meta_name(param);
 		value = g_hash_table_lookup(params, key);
 		crm_free(key);
 	    }
 
 	    if(value) {
 		crm_debug("Setting '%s'='%s' (%s) for %s", name, value, param, victim);
 		append_const_arg(name, value, arg_list);
 
 	    } else {
 		crm_err("No node attribute '%s' for '%s'", name, victim);
 	    }
 	    
 	    crm_free(name); name=NULL;
 	    crm_free(param); 
 	    if(map[lpc] == 0) {
 		break;	
 	    }
 	    
 	} else if(isspace(map[lpc])) {
 	    last = lpc;
 	}   
     }
 }
 
 static char *make_args(GHashTable *dev_hash, GHashTable *node_hash, const char *action, const char *victim)
 {
     char *arg_list = NULL;
     const char *map = NULL;
     CRM_CHECK(action != NULL, return NULL);
 
     if(dev_hash) {
 	map = g_hash_table_lookup(dev_hash, STONITH_ATTR_ARGMAP);
 	g_hash_table_foreach(dev_hash, append_arg, &arg_list);
     }
     
     append_const_arg(STONITH_ATTR_ACTION_OP, action, &arg_list);
     if(victim) {
 	append_const_arg("nodename", victim, &arg_list);
 	append_host_specific_args(victim, map, node_hash, &arg_list);
     }
     
     crm_debug_3("Calculated: %s", arg_list);
     return arg_list;
 }
 
 /* Borrowed from libfence and extended */
 int run_stonith_agent(
     const char *agent, GHashTable *dev_hash, GHashTable *node_hash, const char *action, const char *victim,
     int *agent_result, char **output, async_command_t *track)
 {
     char *args = make_args(dev_hash, node_hash, action, victim);
     int pid, status, len, rc = -1;
     int p_read_fd, p_write_fd;  /* parent read/write file descriptors */
     int c_read_fd, c_write_fd;  /* child read/write file descriptors */
     int fd1[2];
     int fd2[2];
 
     c_read_fd = c_write_fd = p_read_fd = p_write_fd = -1;
 
     if (args == NULL || agent == NULL)
 	goto fail;
     len = strlen(args);
 
     if (pipe(fd1))
 	goto fail;
     p_read_fd = fd1[0];
     c_write_fd = fd1[1];
 
     if (pipe(fd2))
 	goto fail;
     c_read_fd = fd2[0];
     p_write_fd = fd2[1];
 
     crm_debug("forking");
     pid = fork();
     if (pid < 0) {
 	*agent_result = FE_AGENT_FORK;
 	goto fail;
     }
 
     if (pid) {
 	/* parent */
 	int ret;
 
 	fcntl(p_read_fd, F_SETFL, fcntl(p_read_fd, F_GETFL, 0) | O_NONBLOCK);
 
 	do {
 	    crm_debug("sending args");
 	    ret = write(p_write_fd, args, len);
 
 	} while (ret < 0 && errno == EINTR);
 
 	if (ret != len) {
-	    if(rc >= 0) {
+	    if(ret >= 0) {
 		rc = st_err_generic;
 	    }
 	    goto fail;
 	}
 	
 	close(p_write_fd);
 
 	if(track) {
 	    NewTrackedProc(pid, 0, PT_LOGNORMAL, track, track->pt_ops);
 	    
 #if 0
 	    ProcTrackKillInfo *info = NULL;
 	    crm_malloc0(info, sizeof(ProcTrackKillInfo) * 3);
 	    
 	    killseq[0].mstimeout = timeout; /* after timeout send TERM */
 	    killseq[0].signalno = SIGTERM;
 	    killseq[1].mstimeout = 5000; /* after 5 secs remove it */
 	    killseq[1].signalno = SIGKILL;
 	    killseq[2].mstimeout = 5000; /* if it's still there after 5, complain */
 	    killseq[2].signalno = 0;
 	    SetTrackedProcTimeouts(pid,killseq);
 #endif
 	    track->stdout = p_read_fd;
 	    
 	    crm_free(args);
 	    close(c_write_fd);
 	    close(c_read_fd);
 	    return pid;
 
 	} else {
 	    waitpid(pid, &status, 0);
 	    
 	    if(output != NULL) {
 		len = 0;
 		do {
 		    char buf[500];
 		    ret = read(p_read_fd, buf, 500);
 		    if(ret > 0) {
 			buf[ret] = 0;
 			crm_realloc(*output, len + ret + 1);
 			sprintf((*output)+len, "%s", buf);
 			crm_debug("%d: %s", ret, (*output)+len);
 			len += ret;
 		    }
 		    
 		} while (ret == 500 || (ret < 0 && errno == EINTR));
 	    }
 	    
 	    *agent_result = FE_AGENT_ERROR;
 	    if (WIFEXITED(status)) {
 		crm_debug("result = %d", WEXITSTATUS(status));
 		*agent_result = -WEXITSTATUS(status);
 		rc = 0;
 	    }
 	    if(node_hash) {
 		g_hash_table_destroy(node_hash);
 	    }
 	}
 
     } else {
 	/* child */
 
 	close(1);
 	if (dup(c_write_fd) < 0)
 	    goto fail;
 	close(2);
 	if (dup(c_write_fd) < 0)
 	    goto fail;
 	close(0);
 	if (dup(c_read_fd) < 0)
 	    goto fail;
 
 	/* keep c_write_fd open so parent can report all errors. */
 	close(c_read_fd);
 	close(p_read_fd);
 	close(p_write_fd);
 
 	execlp(agent, agent, NULL);
 	exit(EXIT_FAILURE);
     }
 
   fail:
     crm_free(args);
 
-    close(p_read_fd);
-    close(p_write_fd);
+    if(p_read_fd >= 0) { close(p_read_fd); }
+    if(p_write_fd >= 0) { close(p_write_fd); }
 
-    close(c_read_fd);
-    close(c_write_fd);
+    if(c_read_fd >= 0) { close(c_read_fd); }
+    if(c_write_fd >= 0) { close(c_write_fd); }
+	
     return rc;
 }
 
 static int stonith_api_device_metadata(
     stonith_t *stonith, int call_options, const char *agent, const char *namespace,
     char **output, int timeout)
 {
     int rc = 0;
     int bufferlen = 0;
 
     char *buffer = NULL;
     char *xml_meta_longdesc = NULL;
     char *xml_meta_shortdesc = NULL;
 
     char *meta_param = NULL;
     char *meta_longdesc = NULL;
     char *meta_shortdesc = NULL;
     const char *provider = get_stonith_provider(agent, namespace);
 
     Stonith *stonith_obj = NULL;	
     static const char *no_parameter_info = "<!-- no value -->";
 
     crm_info("looking up %s/%s metadata", agent, provider);
     
     /* By having this in a library, we can access it from stonith_admin
      *  when neither lrmd or stonith-ng are running
      * Important for the crm shell's validations...
      */
     
     if(safe_str_eq(provider, "redhat")) {
 	
 	int exec_rc = run_stonith_agent(
 	    agent, NULL, NULL, "metadata", NULL, &rc, &buffer, NULL);
 
 	if(exec_rc < 0 || rc != 0 || buffer == NULL) {
 	    /* failed */
 	    crm_debug("Query failed: %d %d: %s", exec_rc, rc, crm_str(buffer));
 
 	    /* provide a fake metadata entry */
 	    meta_longdesc = crm_strdup(no_parameter_info);
 	    meta_shortdesc = crm_strdup(no_parameter_info);
 	    meta_param = crm_strdup(
 "  <parameters>\n"
 "    <parameter name=\"action\">\n"
 "      <getopt mixed=\"-o\" />\n"
 "      <content type=\"string\" default=\"reboot\" />\n"
 "      <shortdesc lang=\"en\">Fencing action (null, off, on, [reboot], status, hostlist, devstatus)</shortdesc>\n"
 "    </parameter>\n"
 "  </parameters>");
 
 	    goto build;
 	}
 	
     } else {
 	stonith_obj = stonith_new(agent);
 	
 	meta_longdesc = crm_strdup(stonith_get_info(stonith_obj, ST_DEVICEDESCR));
 	if (meta_longdesc == NULL) {
 	    crm_warn("no long description in %s's metadata.", agent);
 	    meta_longdesc = crm_strdup(no_parameter_info);
 	}
 	
 	meta_shortdesc = crm_strdup(stonith_get_info(stonith_obj, ST_DEVICENAME));
 	crm_info("short description: %s", meta_shortdesc);
 	if (meta_shortdesc == NULL) {
 	    crm_warn("no short description in %s's metadata.", agent);
 	    meta_shortdesc = crm_strdup(no_parameter_info);
 	}
 	
 	meta_param = crm_strdup(stonith_get_info(stonith_obj, ST_CONF_XML));
 	if (meta_param == NULL) {
 	    crm_warn("no list of parameters in %s's metadata.", agent);
 	    meta_param = crm_strdup(no_parameter_info);
 	}
 
       build:
 	xml_meta_longdesc = (char *)xmlEncodeEntitiesReentrant(NULL, (const unsigned char *)meta_longdesc);
 	xml_meta_shortdesc = (char *)xmlEncodeEntitiesReentrant(NULL, (const unsigned char *)meta_shortdesc);
 	
 	bufferlen = strlen(META_TEMPLATE) + strlen(agent)
 	    + strlen(xml_meta_longdesc) + strlen(xml_meta_shortdesc)
 	    + strlen(meta_param) + 1;
 
 	crm_malloc0(buffer, bufferlen);
 	snprintf(buffer, bufferlen-1, META_TEMPLATE,
 		 agent, xml_meta_longdesc, xml_meta_shortdesc, meta_param);
 	
 	xmlFree(xml_meta_longdesc);
 	xmlFree(xml_meta_shortdesc);
 
 	if(stonith_obj) {
 	    stonith_delete(stonith_obj);
 	}
 	
 	crm_free(meta_shortdesc);
 	crm_free(meta_longdesc);
 	crm_free(meta_param);	
     }
 
     if(output) {
 	*output = buffer;
 
     } else {
 	crm_free(buffer);
     }
     
     return rc;
 }
 
 static int stonith_api_query(
     stonith_t *stonith, int call_options, const char *target, GListPtr *devices, int timeout)
 {
     int rc = 0, lpc = 0, max = 0;
 
     xmlNode *data = NULL;
     xmlNode *output = NULL;
     xmlXPathObjectPtr xpathObj = NULL;
 
     CRM_CHECK(devices != NULL, return st_err_missing);
 
     data = create_xml_node(NULL, F_STONITH_DEVICE);
     crm_xml_add(data, "origin", __FUNCTION__);
     crm_xml_add(data, F_STONITH_TARGET, target);
     rc = stonith_send_command(stonith, STONITH_OP_QUERY, data, &output, call_options, timeout);
 
     if(rc < 0) {
 	return rc;
     }
     
     xpathObj = xpath_search(output, "//@agent");
-    max = xpathObj->nodesetval->nodeNr;
+    if(xpathObj) {
+	max = xpathObj->nodesetval->nodeNr;
 
-    for(lpc = 0; lpc < max; lpc++) {
-	xmlNode *match = getXpathResult(xpathObj, lpc);
-	CRM_CHECK(match != NULL, continue);
-	
-	crm_info("%s[%d] = %s", "//@agent", lpc, xmlGetNodePath(match));
-	*devices = g_list_append(*devices, crm_element_value_copy(match, XML_ATTR_ID));
+	for(lpc = 0; lpc < max; lpc++) {
+	    xmlNode *match = getXpathResult(xpathObj, lpc);
+	    CRM_CHECK(match != NULL, continue);
+	    
+	    crm_info("%s[%d] = %s", "//@agent", lpc, xmlGetNodePath(match));
+	    *devices = g_list_append(*devices, crm_element_value_copy(match, XML_ATTR_ID));
+	}
     }
-
+    
     free_xml(output);
     free_xml(data);
     return max;
 }
 
 static int stonith_api_call(
     stonith_t *stonith, int call_options, const char *id, const char *action, const char *victim, int timeout)
 {
     int rc = 0;
     xmlNode *data = NULL;
 
     data = create_xml_node(NULL, F_STONITH_DEVICE);
     crm_xml_add(data, "origin", __FUNCTION__);
     crm_xml_add(data, F_STONITH_DEVICE, id);
     crm_xml_add(data, F_STONITH_ACTION, action);
     crm_xml_add(data, F_STONITH_TARGET, victim);
     crm_xml_add_int(data, "timeout", timeout);
 
     rc = stonith_send_command(stonith, STONITH_OP_EXEC, data, NULL, call_options, timeout);
     free_xml(data);
     
     return rc;
 }
 
 static int stonith_api_fence(
     stonith_t *stonith, int call_options, const char *node, GHashTable *parameters,
     const char *action, int timeout)
 {
     int rc = 0;
     xmlNode *data = NULL;
     xmlNode *params = NULL;
 
     data = create_xml_node(NULL, __FUNCTION__);
     crm_xml_add(data, F_STONITH_TARGET, node);
     crm_xml_add(data, F_STONITH_ACTION, action);
     crm_xml_add_int(data, "timeout", timeout);
     params = create_xml_node(data, XML_TAG_ATTRS);
     g_hash_table_foreach(parameters, hash2nvpair, params);
 
     rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, timeout);
     free_xml(data);
     
     return rc;
 }
 
 const char *
 stonith_error2string(enum stonith_errors return_code)
 {
     const char *error_msg = NULL;
     switch(return_code) {
 	case stonith_ok:
 	    error_msg = "OK";
 	    break;
 	case st_err_not_supported:
 	    error_msg = "Not supported";
 	    break;
 	case st_err_authentication:
 	    error_msg = "Not authenticated";
 	    break;
 	case st_err_generic:
 	    error_msg = "Generic error";
 	    break;
 	case st_err_internal:
 	    error_msg = "Internal error";
 	    break;
 	case st_err_unknown_device:
 	    error_msg = "Unknown device";
 	    break;
 	case st_err_unknown_operation:
 	    error_msg = "Unknown operation";
 	    break;
 	case st_err_unknown_port:
 	    error_msg = "Unknown victim";
 	    break;
 	case st_err_none_available:
 	    error_msg = "No available fencing devices";
 	    break;
 	case st_err_connection:
 	    error_msg = "Not connected";
 	    break;
 	case st_err_missing:
 	    error_msg = "Missing input";
 	    break;
 	case st_err_exists:
 	    error_msg = "Device exists";
 	    break;
 	case st_err_timeout:
 	    error_msg = "Operation timed out";
 	    break;
 	case st_err_signal:
 	    error_msg = "Killed by signal";
 	    break;
 	case st_err_ipc:
 	    error_msg = "IPC connection failed";
 	    break;
 	case st_err_peer:
 	    error_msg = "Error from peer";
 	    break;
     }
 			
     if(error_msg == NULL) {
 	crm_err("Unknown Stonith error code: %d", return_code);
 	error_msg = "<unknown error>";
     }
 	
     return error_msg;
 }
 
 gboolean is_redhat_agent(const char *agent) 
 {
     int rc = 0;
     struct stat prop;
     char buffer[FILENAME_MAX+1];
 
     snprintf(buffer,FILENAME_MAX,"%s/%s", RH_STONITH_DIR, agent);
     rc = stat(buffer, &prop);
     if (rc >= 0 && S_ISREG(prop.st_mode)) {
 	return TRUE;
     }
     return FALSE;
 }
 
 const char *get_stonith_provider(const char *agent, const char *provider) 
 {
     /* This function sucks */
     if(is_redhat_agent(agent)) {
 	return "redhat";
 
     } else {
 	Stonith *stonith_obj = stonith_new(agent);
 	if(stonith_obj) {
 	    stonith_delete(stonith_obj);
 	    return "heartbeat";
 	}
     }
     
     crm_err("No such device: %s", agent);
     return NULL;
 }
 
 static gint stonithlib_GCompareFunc(gconstpointer a, gconstpointer b)
 {
     int rc = 0;
     const stonith_notify_client_t *a_client = a;
     const stonith_notify_client_t *b_client = b;
 	
     CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0);
     rc = strcmp(a_client->event, b_client->event);
     if(rc == 0) {
 	if(a_client->notify == NULL || b_client->notify == NULL) {
 	    return 0;
 
 	} else if(a_client->notify == b_client->notify) {
 	    return 0;
 
 	} else if(((long)a_client->notify) < ((long)b_client->notify)) {
 	    crm_err("callbacks for %s are not equal: %p vs. %p",
 		    a_client->event, a_client->notify, b_client->notify);
 	    return -1;
 	} 
 	crm_err("callbacks for %s are not equal: %p vs. %p",
 		a_client->event, a_client->notify, b_client->notify);
 	return 1;
     }
     return rc;
 }
 
 static int get_stonith_token(IPC_Channel *ch, char **token) 
 {
     int rc = stonith_ok;
     xmlNode *reg_msg = NULL;
     const char *msg_type = NULL;
     const char *tmp_ticket = NULL;
     
     CRM_CHECK(ch != NULL, return st_err_missing);
     CRM_CHECK(token != NULL, return st_err_missing);
     
     crm_debug_4("Waiting for msg on command channel");
     
     reg_msg = xmlfromIPC(ch, MAX_IPC_DELAY);
     
     if(ch->ops->get_chan_status(ch) != IPC_CONNECT) {
 	crm_err("No reply message - disconnected");
 	free_xml(reg_msg);
 	return st_err_connection;
 	
     } else if(reg_msg == NULL) {
 	crm_err("No reply message - empty");
 	return st_err_ipc;
     }
     
     msg_type = crm_element_value(reg_msg, F_STONITH_OPERATION);
     tmp_ticket = crm_element_value(reg_msg, F_STONITH_CLIENTID);
     
     if(safe_str_neq(msg_type, CRM_OP_REGISTER) ) {
 	crm_err("Invalid registration message: %s", msg_type);
 	rc = st_err_internal;
 	
     } else if(tmp_ticket == NULL) {
 	crm_err("No registration token provided");
 	crm_log_xml_warn(reg_msg, "Bad reply")
 	rc = st_err_internal;
 
     } else {
 	crm_debug("Obtained registration token: %s", tmp_ticket);
 	*token = crm_strdup(tmp_ticket);
     }
 
     free_xml(reg_msg);
     return rc;
 }
 
 xmlNode *stonith_create_op(
     int call_id, const char *token, const char *op, xmlNode *data, int call_options) 
 {
     int  rc = HA_OK;
     xmlNode *op_msg = create_xml_node(NULL, "stonith_command");
     CRM_CHECK(op_msg != NULL, return NULL);
     CRM_CHECK(token != NULL, return NULL);
 
     crm_xml_add(op_msg, F_XML_TAGNAME, "stonith_command");
 	
     crm_xml_add(op_msg, F_TYPE, T_STONITH_NG);
     crm_xml_add(op_msg, F_STONITH_CALLBACK_TOKEN, token);
     crm_xml_add(op_msg, F_STONITH_OPERATION, op);
     crm_xml_add_int(op_msg, F_STONITH_CALLID, call_id);
     crm_debug_4("Sending call options: %.8lx, %d",
 		(long)call_options, call_options);
     crm_xml_add_int(op_msg, F_STONITH_CALLOPTS, call_options);
 
     if(data != NULL) {
 	add_message_xml(op_msg, F_STONITH_CALLDATA, data);
     }
 	
     if (rc != HA_OK) {
 	crm_err("Failed to create STONITH operation message");
 	crm_log_xml(LOG_ERR, "BadOp", op_msg);
 	free_xml(op_msg);
 	return NULL;
     }
 
     return op_msg;
 }
 
 static void stonith_destroy_op_callback(gpointer data)
 {
     stonith_callback_client_t *blob = data;
     if(blob->timer && blob->timer->ref > 0) {
 	g_source_remove(blob->timer->ref);
     }
     crm_free(blob->timer);
     crm_free(blob);
 }
 
 static int stonith_api_signoff(stonith_t* stonith)
 {
     stonith_private_t *native = stonith->private;
 
     crm_debug("Signing out of the STONITH Service");
 	
     /* close channels */
     if (native->command_channel != NULL) {
 	native->command_channel->ops->destroy(
 	    native->command_channel);
 	native->command_channel = NULL;
     }
 
     if (native->callback_source != NULL) {
 	G_main_del_IPC_Channel(native->callback_source);
 	native->callback_source = NULL;
     }
 
     if (native->callback_channel != NULL) {
 #ifdef BUG
 	native->callback_channel->ops->destroy(
 	    native->callback_channel);
 #endif
 	native->callback_channel = NULL;
     }
 
     stonith->state = stonith_disconnected;
     return stonith_ok;
 }
 
 static int stonith_api_signon(
     stonith_t* stonith, const char *name, int *async_fd, int *sync_fd)
 {
     int rc = stonith_ok;
     xmlNode *hello = NULL;
     char *uuid_ticket = NULL;
     stonith_private_t *native = stonith->private;
 	
     crm_debug_4("Connecting command channel");
 
     stonith->state = stonith_connected_command;
     native->command_channel = init_client_ipc_comms_nodispatch(stonith_channel);
 	
     if(native->command_channel == NULL) {
 	crm_debug("Connection to command channel failed");
 	rc = st_err_connection;
 		
     } else if(native->command_channel->ch_status != IPC_CONNECT) {
 	crm_err("Connection may have succeeded,"
 		" but authentication to command channel failed");
 	rc = st_err_authentication;
     }
 
     if(rc == stonith_ok) {
 	rc = get_stonith_token(native->command_channel, &uuid_ticket);
 	if(rc == stonith_ok) {
 	    native->token = uuid_ticket;
 	    uuid_ticket = NULL;
 
 	} else {
 	    stonith->state = stonith_disconnected;
 	    native->command_channel->ops->disconnect(native->command_channel);
 	    return rc;
 	}
     }
 
     native->callback_channel = init_client_ipc_comms_nodispatch(
 	stonith_channel_callback);
     
     if(native->callback_channel == NULL) {
 	crm_debug("Connection to callback channel failed");
 	rc = st_err_connection;
 		
     } else if(native->callback_channel->ch_status != IPC_CONNECT) {
 	crm_err("Connection may have succeeded,"
 		" but authentication to command channel failed");
 	rc = st_err_authentication;
     }
 	
     if(rc == stonith_ok) {
 	native->callback_channel->send_queue->max_qlen = 500;
 	rc = get_stonith_token(native->callback_channel, &uuid_ticket);
 	if(rc == stonith_ok) {
 	    crm_free(native->token);
 	    native->token = uuid_ticket;
 	}
     }
 	
     if(rc == stonith_ok) {
 	CRM_CHECK(native->token != NULL, ;);
 	hello = stonith_create_op(0, native->token, CRM_OP_REGISTER, NULL, 0);
 	crm_xml_add(hello, F_STONITH_CLIENTNAME, name);
 	    
 	if(send_ipc_message(native->command_channel, hello) == FALSE) {
 	    rc = st_err_internal;
 	}
 
 	free_xml(hello);
     }
 	
     if(rc == stonith_ok) {
 	gboolean do_mainloop = TRUE;
 	if(async_fd != NULL) {
 	    do_mainloop = FALSE;
 	    *async_fd = native->callback_channel->ops->get_recv_select_fd(native->callback_channel);
 	}
 
 	if(sync_fd != NULL) {
 	    do_mainloop = FALSE;
 	    *sync_fd = native->callback_channel->ops->get_send_select_fd(native->callback_channel);
 	}
 
 	if(do_mainloop) {
 	    crm_debug_4("Connecting callback channel");
 	    native->callback_source = G_main_add_IPC_Channel(
 		G_PRIORITY_HIGH, native->callback_channel, FALSE, stonith_dispatch,
 		stonith, default_ipc_connection_destroy);
 		
 	    if(native->callback_source == NULL) {
 		crm_err("Callback source not recorded");
 		rc = st_err_connection;
 
 	    } else {
 		set_IPC_Channel_dnotify(
 		    native->callback_source, stonith_connection_destroy);
 	    }
 	}
     } 
 
     if(rc == stonith_ok) {
 #if HAVE_MSGFROMIPC_TIMEOUT
 	stonith->call_timeout = MAX_IPC_DELAY;
 #endif
 	crm_debug("Connection to STONITH successful");
 	return stonith_ok;
     }
 
     crm_debug("Connection to STONITH failed: %s", stonith_error2string(rc));
     stonith->cmds->disconnect(stonith);
     return rc;
 }
 
 static int stonith_set_notification(stonith_t* stonith, const char *callback, int enabled) 
 {
     xmlNode *notify_msg = create_xml_node(NULL, __FUNCTION__);
     stonith_private_t *native = stonith->private;
 
     if(stonith->state != stonith_disconnected) {
 	crm_xml_add(notify_msg, F_STONITH_OPERATION, T_STONITH_NOTIFY);
 	if(enabled) {
 	    crm_xml_add(notify_msg, F_STONITH_NOTIFY_ACTIVATE, callback);
 	} else {
 	    crm_xml_add(notify_msg, F_STONITH_NOTIFY_DEACTIVATE, callback);
 	}	
 	send_ipc_message(native->callback_channel, notify_msg);
     }
 
     free_xml(notify_msg);
     return stonith_ok;
 }
 
 static int stonith_api_add_notification(
     stonith_t *stonith, const char *event,
     void (*callback)(stonith_t *stonith, const char *event, xmlNode *msg))
 {
     GList *list_item = NULL;
     stonith_notify_client_t *new_client = NULL;
 
     crm_debug_2("Adding callback for %s events (%d)",
 		event, g_list_length(stonith->notify_list));
 
     crm_malloc0(new_client, sizeof(stonith_notify_client_t));
     new_client->event = event;
     new_client->notify = callback;
 
     list_item = g_list_find_custom(
 	stonith->notify_list, new_client, stonithlib_GCompareFunc);
 	
     if(list_item != NULL) {
 	crm_warn("Callback already present");
 	crm_free(new_client);
 	return st_err_exists;
 		
     } else {
 	stonith->notify_list = g_list_append(
 	    stonith->notify_list, new_client);
 
 	stonith_set_notification(stonith, event, 1);
 		
 	crm_debug_3("Callback added (%d)", g_list_length(stonith->notify_list));
     }
     return stonith_ok;
 }
 
 
 static int stonith_api_del_notification(stonith_t *stonith, const char *event)
 {
     GList *list_item = NULL;
     stonith_notify_client_t *new_client = NULL;
 
     crm_debug("Removing callback for %s events", event);
 
     crm_malloc0(new_client, sizeof(stonith_notify_client_t));
     new_client->event = event;
     new_client->notify = NULL;
 
     list_item = g_list_find_custom(
 	stonith->notify_list, new_client, stonithlib_GCompareFunc);
 	
     stonith_set_notification(stonith, event, 0);
 
     if(list_item != NULL) {
 	stonith_notify_client_t *list_client = list_item->data;
 	stonith->notify_list =
 	    g_list_remove(stonith->notify_list, list_client);
 	crm_free(list_client);
 
 	crm_debug_3("Removed callback");
 
     } else {
 	crm_debug_3("Callback not present");
     }
     crm_free(new_client);
     return stonith_ok;
 }
 
 static gboolean stonith_async_timeout_handler(gpointer data)
 {
     struct timer_rec_s *timer = data;
     crm_debug("Async call %d timed out after %dms", timer->call_id, timer->timeout);
     stonith_perform_callback(timer->stonith, NULL, timer->call_id, st_err_timeout);
 
     /* Always return TRUE, never remove the handler
      * We do that in stonith_del_callback()
      */
     return TRUE;
 }
 
 static int stonith_api_add_callback(
     stonith_t *stonith, int call_id, int timeout, gboolean only_success,
     void *user_data, const char *callback_name,
     void (*callback)(
 	stonith_t *st, const xmlNode *msg, int call, int rc, xmlNode *output, void *userdata))
 {
     stonith_callback_client_t *blob = NULL;
     stonith_private_t *private = NULL;
     CRM_CHECK(stonith != NULL, return st_err_missing);
     CRM_CHECK(stonith->private != NULL, return st_err_missing);
     private = stonith->private;
 
     if(call_id == 0) {
 	private->op_callback = callback;
 
     } else if(call_id < 0) {
 	if(only_success == FALSE) {
 	    callback(stonith, NULL, call_id, call_id, NULL, user_data);
 	} else {
 	    crm_warn("STONITH call failed: %s", stonith_error2string(call_id));
 	}
 	return FALSE;
     }
 	
     crm_malloc0(blob, sizeof(stonith_callback_client_t));
     blob->id = callback_name;
     blob->only_success = only_success;
     blob->user_data = user_data;
     blob->callback = callback;
 
     if(timeout > 0) {
 	struct timer_rec_s *async_timer = NULL;
 	    
 	crm_malloc0(async_timer, sizeof(struct timer_rec_s));
 	blob->timer = async_timer;
 
 	async_timer->stonith = stonith;
 	async_timer->call_id = call_id;
 	async_timer->timeout = timeout*1100;
 	async_timer->ref = g_timeout_add(
 	    async_timer->timeout, stonith_async_timeout_handler, async_timer);
     }
 	
     g_hash_table_insert(private->stonith_op_callback_table, GINT_TO_POINTER(call_id), blob);
 	
     return TRUE;
 }
 
 static int stonith_api_del_callback(stonith_t *stonith, int call_id, gboolean all_callbacks) 
 {
     stonith_private_t *private = stonith->private;
     
     if(all_callbacks) {
 	private->op_callback = NULL;
 	g_hash_table_destroy(private->stonith_op_callback_table);
 	private->stonith_op_callback_table = g_hash_table_new_full(
 	    g_direct_hash, g_direct_equal,
 	    NULL, stonith_destroy_op_callback);
 
     } else if(call_id == 0) {
 	private->op_callback = NULL;
 
     } else {
 	g_hash_table_remove(private->stonith_op_callback_table, GINT_TO_POINTER(call_id));
     }
     return stonith_ok;
 }
 
 static void stonith_dump_pending_op(
     gpointer key, gpointer value, gpointer user_data) 
 {
     int call = GPOINTER_TO_INT(key);
     stonith_callback_client_t *blob = value;
 
     crm_debug("Call %d (%s): pending", call, crm_str(blob->id));
 }
 
 void stonith_dump_pending_callbacks(stonith_t *stonith)
 {
     stonith_private_t *private = stonith->private;
     if(private->stonith_op_callback_table == NULL) {
 	return;
     }
     return g_hash_table_foreach(
 	private->stonith_op_callback_table, stonith_dump_pending_op, NULL);
 }
 
 void stonith_perform_callback(stonith_t *stonith, xmlNode *msg, int call_id, int rc)
 {
     xmlNode *output = NULL;
     stonith_private_t *private = NULL;
     stonith_callback_client_t *blob = NULL;
     stonith_callback_client_t local_blob;
     CRM_CHECK(stonith != NULL, return);
     CRM_CHECK(stonith->private != NULL, return);
 
     private = stonith->private;
 
     local_blob.id = NULL;
     local_blob.callback = NULL;
     local_blob.user_data = NULL;
     local_blob.only_success = FALSE;
 
     if(msg != NULL) {
 	crm_element_value_int(msg, F_STONITH_RC, &rc);
 	crm_element_value_int(msg, F_STONITH_CALLID, &call_id);
 	output = get_message_xml(msg, F_STONITH_CALLDATA);
     }
     
     CRM_CHECK(call_id > 0, crm_warn("Strange or missing call-id"));
 
     blob = g_hash_table_lookup(
 	private->stonith_op_callback_table, GINT_TO_POINTER(call_id));
 	
     if(blob != NULL) {
 	local_blob = *blob;
 	blob = NULL;
 		
 	stonith_api_del_callback(stonith, call_id, FALSE);
 
     } else {
 	crm_debug_2("No callback found for call %d", call_id);
 	local_blob.callback = NULL;
     }
 
     if(stonith == NULL) {
 	crm_debug("No stonith object supplied");
     }
 	
     if(local_blob.callback != NULL
        && (rc == stonith_ok || local_blob.only_success == FALSE)) {
 	crm_debug_2("Invoking callback %s for call %d", crm_str(local_blob.id), call_id);
 	local_blob.callback(stonith, msg, call_id, rc, output, local_blob.user_data);
 		
     } else if(private->op_callback == NULL && rc != stonith_ok) {
 	crm_warn("STONITH command failed: %s", stonith_error2string(rc));
 	crm_log_xml(LOG_DEBUG, "Failed STONITH Update", msg);
     }
 	
     if(private->op_callback != NULL) {
 	crm_debug_2("Invoking global callback for call %d", call_id);
 	private->op_callback(stonith, msg, call_id, rc, output, NULL);
     }
     crm_debug_4("OP callback activated.");
 }
 
 static void stonith_send_notification(gpointer data, gpointer user_data)
 {
     struct notify_blob_s *blob = user_data;
     stonith_notify_client_t *entry = data;
     const char *event = NULL;
     
     if(blob->xml == NULL) {
 	crm_warn("Skipping callback - NULL message");
 	return;
     }
 
     event = crm_element_value(blob->xml, F_SUBTYPE);
 	
     if(entry == NULL) {
 	crm_warn("Skipping callback - NULL callback client");
 	return;
 
     } else if(entry->notify == NULL) {
 	crm_warn("Skipping callback - NULL callback");
 	return;
 
     } else if(safe_str_neq(entry->event, event)) {
 	crm_debug_4("Skipping callback - event mismatch %p/%s vs. %s",
 		    entry, entry->event, event);
 	return;
     }
 	
     crm_debug_4("Invoking callback for %p/%s event...", entry, event);
     entry->notify(blob->stonith, event, blob->xml);
     crm_debug_4("Callback invoked...");
 }
 
 int stonith_send_command(
     stonith_t *stonith, const char *op, xmlNode *data, xmlNode **output_data,
     int call_options, int timeout) 
 {
     int  rc = HA_OK;
 	
     xmlNode *op_msg   = NULL;
     xmlNode *op_reply = NULL;
 
     stonith_private_t *native = stonith->private;
 	
     if(stonith->state == stonith_disconnected) {
 	return st_err_connection;
     }
 
     if(output_data != NULL) {
 	*output_data = NULL;
     }
 	
     if(op == NULL) {
 	crm_err("No operation specified");
 	return st_err_missing;
     }
 
     stonith->call_id++;
     /* prevent call_id from being negative (or zero) and conflicting
      *    with the stonith_errors enum
      * use 2 because we use it as (stonith->call_id - 1) below
      */
     if(stonith->call_id < 1) {
 	stonith->call_id = 1;
     }
 	
     CRM_CHECK(native->token != NULL, ;);
     op_msg = stonith_create_op(stonith->call_id, native->token, op, data, call_options);
     if(op_msg == NULL) {
 	return st_err_missing;
     }
 	
     crm_debug_3("Sending %s message to STONITH service", op);
     if(send_ipc_message(native->command_channel, op_msg) == FALSE) {
 	crm_err("Sending message to STONITH service FAILED");
 	free_xml(op_msg);
 	return st_err_ipc;
 
     } else {
 	crm_debug_3("Message sent");
     }
 
     free_xml(op_msg);
 
     if((call_options & st_opt_discard_reply)) {
 	crm_debug_3("Discarding reply");
 	return stonith_ok;
 
     } else if(!(call_options & st_opt_sync_call)) {
 	crm_debug_3("Async call, returning");
 	CRM_CHECK(stonith->call_id != 0, return st_err_ipc);
 
 	return stonith->call_id;
     }
 	
     rc = IPC_OK;
     crm_debug_3("Waiting for a syncronous reply");
 
     rc = stonith_ok;
     while(IPC_ISRCONN(native->command_channel)) {
 	int reply_id = -1;
 	int msg_id = stonith->call_id;
 
 	op_reply = xmlfromIPC(native->command_channel, stonith->call_timeout);
 	if(op_reply == NULL) {
 	    rc = st_err_peer;
 	    break;
 	}
 
 	crm_element_value_int(op_reply, F_STONITH_CALLID, &reply_id);
 	if(reply_id <= 0) {
 	    rc = st_err_peer;
 	    break;
 
 	} else if(reply_id == msg_id) {
 	    crm_debug_3("Syncronous reply received");
 	    crm_log_xml(LOG_MSG, "Reply", op_reply);
 	    if(crm_element_value_int(op_reply, F_STONITH_RC, &rc) != 0) {
 		rc = st_err_peer;
 	    }
 		    
 	    if(output_data != NULL && is_not_set(call_options, st_opt_discard_reply)) {
 		*output_data = op_reply;
 		op_reply = NULL;
 	    }
 
 	    break;
 			
 	} else if(reply_id < msg_id) {
 	    crm_debug("Recieved old reply: %d (wanted %d)", reply_id, msg_id);
 	    crm_log_xml(LOG_MSG, "Old reply", op_reply);
 
 	} else if((reply_id - 10000) > msg_id) {
 	    /* wrap-around case */
 	    crm_debug("Recieved old reply: %d (wanted %d)", reply_id, msg_id);
 	    crm_log_xml(LOG_MSG, "Old reply", op_reply);
 
 	} else {
 	    crm_err("Received a __future__ reply:"
 		    " %d (wanted %d)", reply_id, msg_id);
 	}
 	free_xml(op_reply);
 	op_reply = NULL;
     }
 	
     if(IPC_ISRCONN(native->command_channel) == FALSE) {
 	crm_err("STONITH disconnected: %d", native->command_channel->ch_status);
 	stonith->state = stonith_disconnected;
     }
 
     if(op_reply == NULL && stonith->state == stonith_disconnected) {
 	rc = st_err_connection;
 
     } else if(rc == stonith_ok && op_reply == NULL) {
 	rc = st_err_peer;
     }
 	
     free_xml(op_reply);
     return rc;
 }
 
 static gboolean stonith_msgready(stonith_t* stonith)
 {
     stonith_private_t *private = NULL;
 	
     if (stonith == NULL) {
 	crm_err("No STONITH!");
 	return FALSE;
     }
 
     private = stonith->private;
 
     if(private->command_channel != NULL) {
 	/* drain the channel */
 	IPC_Channel *cmd_ch = private->command_channel;
 	xmlNode *cmd_msg = NULL;
 	while(cmd_ch->ch_status != IPC_DISCONNECT
 	      && cmd_ch->ops->is_message_pending(cmd_ch)) {
 	    /* this will happen when the STONITH exited from beneath us */
 	    cmd_msg = xmlfromIPC(cmd_ch, MAX_IPC_DELAY);
 	    free_xml(cmd_msg);
 	}
 
     } else {
 	crm_err("No command channel");
     }	
 
     if(private->callback_channel == NULL) {
 	crm_err("No callback channel");
 	return FALSE;
 
     } else if(private->callback_channel->ch_status == IPC_DISCONNECT) {
 	crm_info("Lost connection to the STONITH service [%d].",
 		 private->callback_channel->farside_pid);
 	return FALSE;
 
     } else if(private->callback_channel->ops->is_message_pending(
 		  private->callback_channel)) {
 	crm_debug_4("Message pending on command channel [%d]",
 		    private->callback_channel->farside_pid);
 	return TRUE;
     }
 
     crm_debug_3("No message pending");
     return FALSE;
 }
 
 static int stonith_rcvmsg(stonith_t* stonith)
 {
     const char *type = NULL;
     stonith_private_t *private = NULL;
     struct notify_blob_s blob;
 	
     if (stonith == NULL) {
 	crm_err("No STONITH!");
 	return FALSE;
     }
 
     blob.stonith = stonith;
     private = stonith->private;
 	
     /* if it is not blocking mode and no message in the channel, return */
     if (stonith_msgready(stonith) == FALSE) {
 	crm_debug_3("No message ready and non-blocking...");
 	return 0;
     }
 
     /* IPC_INTR is not a factor here */
     blob.xml = xmlfromIPC(private->callback_channel, MAX_IPC_DELAY);
     if (blob.xml == NULL) {
 	crm_warn("Received a NULL msg from STONITH service.");
 	return 0;
     }
 
     /* do callbacks */
     type = crm_element_value(blob.xml, F_TYPE);
     crm_debug_4("Activating %s callbacks...", type);
 
     if(safe_str_eq(type, T_STONITH_NG)) {
 	stonith_perform_callback(stonith, blob.xml, 0, 0);
 		
     } else if(safe_str_eq(type, T_STONITH_NOTIFY)) {
 	g_list_foreach(stonith->notify_list, stonith_send_notification, &blob);
 
     } else {
 	crm_err("Unknown message type: %s", type);
 	crm_log_xml_warn(blob.xml, "BadReply");
     }
 	
     free_xml(blob.xml);
 
     return 1;
 }
 
 gboolean stonith_dispatch(IPC_Channel *channel, gpointer user_data)
 {
     stonith_t *stonith = user_data;
     stonith_private_t *private = NULL;
     gboolean stay_connected = TRUE;
     
     CRM_CHECK(stonith != NULL, return FALSE);
     
     private = stonith->private;
     CRM_CHECK(private->callback_channel == channel, return FALSE);
     
     while(stonith_msgready(stonith)) {
 	/* invoke the callbacks but dont block */
 	int rc = stonith_rcvmsg(stonith);
 	if( rc < 0) {
 	    crm_err("Message acquisition failed: %d", rc);
 	    break;
 
 	} else if(rc == 0) {
 	    break;
 	}
     }
     
     if(private->callback_channel
        && private->callback_channel->ch_status != IPC_CONNECT) {
 	crm_crit("Lost connection to the STONITH service [%d/callback].",
 		 channel->farside_pid);
 	private->callback_source = NULL;
 	stay_connected = FALSE;
     }
     
     if(private->command_channel
        && private->command_channel->ch_status != IPC_CONNECT) {
 	crm_crit("Lost connection to the STONITH service [%d/command].",
 		 channel->farside_pid);
 	private->callback_source = NULL;
 	stay_connected = FALSE;
     }
 
     return stay_connected;
 }
 
 static int stonith_api_free (stonith_t* stonith)
 {
     int rc = stonith_ok;
 
     if(stonith->state != stonith_disconnected) {
 	rc = stonith->cmds->disconnect(stonith);
     }
 
     if(stonith->state == stonith_disconnected) {
 	stonith_private_t *private = stonith->private;
 	g_hash_table_destroy(private->stonith_op_callback_table);
 	crm_free(private->token);
 	crm_free(stonith->private);
 	crm_free(stonith->cmds);
 	crm_free(stonith);
     }
 	
     return rc;
 }
 
 void stonith_api_delete(stonith_t *stonith)
 {
     GList *list = stonith->notify_list;
     while(list != NULL) {
 	stonith_notify_client_t *client = g_list_nth_data(list, 0);
 	list = g_list_remove(list, client);
 	crm_free(client);
     }
 	
     stonith->cmds->free(stonith);
     stonith = NULL;
 }
 
 stonith_t *stonith_api_new(void) 
 {
     stonith_t* new_stonith = NULL;
     stonith_private_t* private = NULL;
     crm_malloc0(new_stonith, sizeof(stonith_t));
     crm_malloc0(private, sizeof(stonith_private_t));
     new_stonith->private = private;
     
     private->stonith_op_callback_table = g_hash_table_new_full(
 	g_direct_hash, g_direct_equal,
 	NULL, stonith_destroy_op_callback);
 
     new_stonith->call_id = 1;
     new_stonith->notify_list = NULL;
     new_stonith->state = stonith_disconnected;
     
     crm_malloc0(new_stonith->cmds, sizeof(stonith_api_operations_t));
 
     new_stonith->cmds->free       = stonith_api_free;
     new_stonith->cmds->connect    = stonith_api_signon;
     new_stonith->cmds->disconnect = stonith_api_signoff;
     
     new_stonith->cmds->call       = stonith_api_call;
     new_stonith->cmds->fence      = stonith_api_fence;
     new_stonith->cmds->metadata   = stonith_api_device_metadata;
 
     new_stonith->cmds->query           = stonith_api_query;
     new_stonith->cmds->remove_device   = stonith_api_remove_device;
     new_stonith->cmds->register_device = stonith_api_register_device;
     
     new_stonith->cmds->remove_callback       = stonith_api_del_callback;	
     new_stonith->cmds->register_callback     = stonith_api_add_callback;	
     new_stonith->cmds->remove_notification   = stonith_api_del_notification;
     new_stonith->cmds->register_notification = stonith_api_add_notification;
 
     return new_stonith;
 }
 
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
index f155c9e96a..efad53e187 100644
--- a/lib/pengine/unpack.c
+++ b/lib/pengine/unpack.c
@@ -1,1829 +1,1826 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 #include <crm_internal.h>
 
 #include <lrm/lrm_api.h>
 #include <glib.h>
 
 #include <crm/crm.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/msg.h>
 #include <crm/common/util.h>
 #include <crm/pengine/status.h>
 #include <crm/pengine/rules.h>
 #include <utils.h>
 #include <unpack.h>
 
 #define set_config_flag(data_set, option, flag) do {			\
 	const char *tmp = pe_pref(data_set->config_hash, option);	\
 	if(tmp) {							\
 	    if(crm_is_true(tmp)) {					\
 		set_bit_inplace(data_set->flags, flag);			\
 	    } else {							\
 		clear_bit_inplace(data_set->flags, flag);		\
 	    }								\
 	}								\
     } while(0)
 
 gboolean unpack_rsc_op(
     resource_t *rsc, node_t *node, xmlNode *xml_op,
     enum action_fail_response *failed, pe_working_set_t *data_set);
 
 static void pe_fence_node(pe_working_set_t *data_set, node_t *node, const char *reason)
 {
     CRM_CHECK(node, return);
     if(node->details->unclean == FALSE) {
 	if(is_set(data_set->flags, pe_flag_stonith_enabled)) {
 	    crm_warn("Node %s will be fenced %s", node->details->uname, reason);
 	} else {
 	    crm_warn("Node %s is unclean %s", node->details->uname, reason);
 	}
     }
     node->details->unclean = TRUE;
 }
 
 
 gboolean
 unpack_config(xmlNode *config, pe_working_set_t *data_set)
 {
 	const char *value = NULL;
 	GHashTable *config_hash = g_hash_table_new_full(
 		g_str_hash,g_str_equal, g_hash_destroy_str,g_hash_destroy_str);
 
 	data_set->config_hash = config_hash;
 
 	unpack_instance_attributes(
 		data_set->input, config, XML_CIB_TAG_PROPSET, NULL, config_hash,
 		CIB_OPTIONS_FIRST, FALSE, data_set->now);
 
 	verify_pe_options(data_set->config_hash);
 
 	set_config_flag(data_set, "enable-startup-probes", pe_flag_startup_probes);
 	crm_info("Startup probes: %s",
 		  is_set(data_set->flags, pe_flag_startup_probes)?"enabled":"disabled (dangerous)");
 	
 	value = pe_pref(data_set->config_hash, "stonith-timeout");
 	data_set->stonith_timeout = crm_get_msec(value);
 	crm_debug("STONITH timeout: %d", data_set->stonith_timeout);
 	
 	set_config_flag(data_set, "stonith-enabled", pe_flag_stonith_enabled);
 	crm_debug("STONITH of failed nodes is %s",
 		  is_set(data_set->flags, pe_flag_stonith_enabled)?"enabled":"disabled");	
 
 	data_set->stonith_action = pe_pref(data_set->config_hash, "stonith-action");
 	crm_debug_2("STONITH will %s nodes", data_set->stonith_action);	
 	
 	set_config_flag(data_set, "stop-all-resources", pe_flag_stop_everything);
 	crm_debug("Stop all active resources: %s",
 		  is_set(data_set->flags, pe_flag_stop_everything)?"true":"false");
 	
 	set_config_flag(data_set, "symmetric-cluster", pe_flag_symmetric_cluster);
 	if(is_set(data_set->flags, pe_flag_symmetric_cluster)) {
 		crm_debug("Cluster is symmetric"
 			 " - resources can run anywhere by default");
 	}
 	
 	value = pe_pref(data_set->config_hash, "default-resource-stickiness");
 	data_set->default_resource_stickiness = char2score(value);
 	crm_debug("Default stickiness: %d",
 		 data_set->default_resource_stickiness);
 
 	value = pe_pref(data_set->config_hash, "no-quorum-policy");
 
 	if(safe_str_eq(value, "ignore")) {
 		data_set->no_quorum_policy = no_quorum_ignore;
 		
 	} else if(safe_str_eq(value, "freeze")) {
 		data_set->no_quorum_policy = no_quorum_freeze;
 
 	} else if(safe_str_eq(value, "suicide")) {
 	    gboolean do_panic = FALSE;
 	    crm_element_value_int(data_set->input, XML_ATTR_QUORUM_PANIC, &do_panic);
 
 	    if(is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE){
 		crm_config_err("Setting no-quorum-policy=suicide makes no sense if stonith-enabled=false");
 	    }
 
 	    if(do_panic && is_set(data_set->flags, pe_flag_stonith_enabled)) {
 		data_set->no_quorum_policy = no_quorum_suicide;
 
 	    } else if(is_set(data_set->flags, pe_flag_have_quorum) == FALSE && do_panic == FALSE) {
 		crm_notice("Resetting no-quorum-policy to 'stop': The cluster has never had quorum");
 		data_set->no_quorum_policy = no_quorum_stop;
 	    }
 	    
 	} else {
 		data_set->no_quorum_policy = no_quorum_stop;
 	}
 	
 	switch (data_set->no_quorum_policy) {
 		case no_quorum_freeze:
 			crm_debug("On loss of CCM Quorum: Freeze resources");
 			break;
 		case no_quorum_stop:
 			crm_debug("On loss of CCM Quorum: Stop ALL resources");
 			break;
 		case no_quorum_suicide:
 			crm_notice("On loss of CCM Quorum: Fence all remaining nodes");
 			break;
 		case no_quorum_ignore:
 			crm_notice("On loss of CCM Quorum: Ignore");
 			break;
 	}
 
 	set_config_flag(data_set, "stop-orphan-resources", pe_flag_stop_rsc_orphans);
 	crm_debug_2("Orphan resources are %s",
 		    is_set(data_set->flags, pe_flag_stop_rsc_orphans)?"stopped":"ignored");	
 	
 	set_config_flag(data_set, "stop-orphan-actions", pe_flag_stop_action_orphans);
 	crm_debug_2("Orphan resource actions are %s",
 		    is_set(data_set->flags, pe_flag_stop_action_orphans)?"stopped":"ignored");	
 
 	set_config_flag(data_set, "remove-after-stop", pe_flag_remove_after_stop);
 	crm_debug_2("Stopped resources are removed from the status section: %s",
 		    is_set(data_set->flags, pe_flag_remove_after_stop)?"true":"false");	
 	
 	set_config_flag(data_set, "maintenance-mode", pe_flag_maintenance_mode);
 	crm_debug_2("Maintenance mode: %s",
 		    is_set(data_set->flags, pe_flag_maintenance_mode)?"true":"false");	
 
 	if(is_set(data_set->flags, pe_flag_maintenance_mode)) {
 	    clear_bit(data_set->flags, pe_flag_is_managed_default);
 	} else {
 	    set_config_flag(data_set, "is-managed-default", pe_flag_is_managed_default);
 	}
 	crm_debug_2("By default resources are %smanaged",
 		    is_set(data_set->flags, pe_flag_is_managed_default)?"":"not ");
 
 	set_config_flag(data_set, "start-failure-is-fatal", pe_flag_start_failure_fatal);
 	crm_debug_2("Start failures are %s",
 		    is_set(data_set->flags, pe_flag_start_failure_fatal)?"always fatal":"handled by failcount");
 
 	node_score_red    = char2score(pe_pref(data_set->config_hash, "node-health-red"));
 	node_score_green  = char2score(pe_pref(data_set->config_hash, "node-health-green"));
 	node_score_yellow = char2score(pe_pref(data_set->config_hash, "node-health-yellow"));
 
 	crm_info("Node scores: 'red' = %s, 'yellow' = %s, 'green' = %s",
 		 pe_pref(data_set->config_hash, "node-health-red"),
 		 pe_pref(data_set->config_hash, "node-health-yellow"),
 		 pe_pref(data_set->config_hash, "node-health-green"));
 
 	data_set->placement_strategy = pe_pref(data_set->config_hash, "placement-strategy");
 	crm_debug_2("Placement strategy: %s", data_set->placement_strategy);	
 	
 	return TRUE;
 }
 
 gboolean
 unpack_nodes(xmlNode * xml_nodes, pe_working_set_t *data_set)
 {
 	node_t *new_node   = NULL;
 	const char *id     = NULL;
 	const char *uname  = NULL;
 	const char *type   = NULL;
 	const char *score  = NULL;
 	gboolean unseen_are_unclean = TRUE;
 	const char *blind_faith = pe_pref(
 		data_set->config_hash, "startup-fencing");
 	
 	if(crm_is_true(blind_faith) == FALSE) {
 		unseen_are_unclean = FALSE;
 		crm_warn("Blind faith: not fencing unseen nodes");
 	}
 
 	xml_child_iter_filter(
 		xml_nodes, xml_obj, XML_CIB_TAG_NODE,
 
 		new_node = NULL;
 
 		id     = crm_element_value(xml_obj, XML_ATTR_ID);
 		uname  = crm_element_value(xml_obj, XML_ATTR_UNAME);
 		type   = crm_element_value(xml_obj, XML_ATTR_TYPE);
 		score  = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
 		crm_debug_3("Processing node %s/%s", uname, id);
 
 		if(id == NULL) {
 			crm_config_err("Must specify id tag in <node>");
 			continue;
 		}
 		if(type == NULL) {
 			crm_config_err("Must specify type tag in <node>");
 			continue;
 		}
 		if(pe_find_node(data_set->nodes, uname) != NULL) {
 		    crm_config_warn("Detected multiple node entries with uname=%s"
 				    " - this is rarely intended", uname);
 		}
 
 		crm_malloc0(new_node, sizeof(node_t));
 		if(new_node == NULL) {
 			return FALSE;
 		}
 		
 		new_node->weight = char2score(score);
 		new_node->fixed  = FALSE;
 		crm_malloc0(new_node->details,
 			   sizeof(struct node_shared_s));
 
 		if(new_node->details == NULL) {
 			crm_free(new_node);
 			return FALSE;
 		}
 
 		crm_debug_3("Creaing node for entry %s/%s", uname, id);
 		new_node->details->id		= id;
 		new_node->details->uname	= uname;
 		new_node->details->type		= node_ping;
 		new_node->details->online	= FALSE;
 		new_node->details->shutdown	= FALSE;
 		new_node->details->running_rsc	= NULL;
 		new_node->details->attrs        = g_hash_table_new_full(
 			g_str_hash, g_str_equal,
 			g_hash_destroy_str, g_hash_destroy_str);
 		new_node->details->utilization  = g_hash_table_new_full(
 			g_str_hash, g_str_equal,
 			g_hash_destroy_str, g_hash_destroy_str);
 		
 /* 		if(data_set->have_quorum == FALSE */
 /* 		   && data_set->no_quorum_policy == no_quorum_stop) { */
 /* 			/\* start shutting resources down *\/ */
 /* 			new_node->weight = -INFINITY; */
 /* 		} */
 		
 		if(is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE || unseen_are_unclean == FALSE) {
 			/* blind faith... */
 			new_node->details->unclean = FALSE; 
 
 		} else {
 			/* all nodes are unclean until we've seen their
 			 * status entry
 			 */
 			new_node->details->unclean = TRUE;
 		}
 		
 		if(type == NULL
 		   || safe_str_eq(type, "member")
 		   || safe_str_eq(type, NORMALNODE)) {
 			new_node->details->type = node_member;
 		}
 
 		add_node_attrs(xml_obj, new_node, FALSE, data_set);
 		unpack_instance_attributes(
 			data_set->input, xml_obj, XML_TAG_UTILIZATION, NULL,
 			new_node->details->utilization, NULL, FALSE, data_set->now);
 
 		data_set->nodes = g_list_append(data_set->nodes, new_node);    
 		crm_debug_3("Done with node %s",
 			    crm_element_value(xml_obj, XML_ATTR_UNAME));
 		);
   
 	return TRUE;
 }
 
 static void g_hash_destroy_node_list(gpointer data)
 {
     GListPtr domain = data;
     slist_destroy(node_t, node, domain, crm_free(node));
 }
 
 gboolean
 unpack_domains(xmlNode *xml_domains, pe_working_set_t *data_set)
 {
     GListPtr domain = NULL;
     const char *id     = NULL;
 
     crm_info("Unpacking domains");
     data_set->domains = g_hash_table_new_full(
 	g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_node_list);
     
     xml_child_iter_filter(
 	xml_domains, xml_domain, XML_CIB_TAG_DOMAIN,
 
 	domain = NULL;
 	id = crm_element_value(xml_domain, XML_ATTR_ID);
 
 	xml_child_iter_filter(
 	    xml_domain, xml_node, XML_CIB_TAG_NODE,
 	    
 	    node_t *copy = NULL;
 	    node_t *node = NULL;
 	    const char *uname = crm_element_value(xml_node, "name");
 	    const char *score = crm_element_value(xml_node, XML_RULE_ATTR_SCORE);
 
 	    if(uname == NULL) {
 		crm_config_err("Invalid domain %s: Must specify id tag in <node>", id);
 		continue;
 	    }
 
 	    node = pe_find_node(data_set->nodes, uname);
 	    if(node == NULL) {
 		node = pe_find_node_id(data_set->nodes, uname);
-		continue;
 	    }
 	    if(node == NULL) {
 		crm_config_warn("Invalid domain %s: Node %s does not exist", id, uname);
 		continue;
 	    }
 
 	    copy = node_copy(node);
 	    copy->weight = char2score(score);
 	    crm_debug("Adding %s to domain %s with score %s", node->details->uname, id, score);
 
 	    domain = g_list_append(domain, copy);
 	    );
 
 	if(domain) {
 	    crm_debug("Created domain %s with %d members", id, g_list_length(domain));
 	    g_hash_table_replace(data_set->domains, crm_strdup(id), domain);
 	}
 	);
   
 	return TRUE;
 }
 
 gboolean 
 unpack_resources(xmlNode * xml_resources, pe_working_set_t *data_set)
 {
 	xml_child_iter(
 		xml_resources, xml_obj, 
 
 		resource_t *new_rsc = NULL;
 		crm_debug_3("Beginning unpack... <%s id=%s... >",
 			    crm_element_name(xml_obj), ID(xml_obj));
 		if(common_unpack(xml_obj, &new_rsc, NULL, data_set)) {
 			data_set->resources = g_list_append(
 				data_set->resources, new_rsc);
 			
 			print_resource(LOG_DEBUG_3, "Added", new_rsc, FALSE);
 
 		} else {
 			crm_config_err("Failed unpacking %s %s",
 				      crm_element_name(xml_obj),
 				      crm_element_value(xml_obj, XML_ATTR_ID));
 			if(new_rsc != NULL && new_rsc->fns != NULL) {
 				new_rsc->fns->free(new_rsc);
 			}
 		}
 		);
 	
 	data_set->resources = g_list_sort(
 		data_set->resources, sort_rsc_priority);
 
 	if(is_set(data_set->flags, pe_flag_stonith_enabled) && is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {
 	    crm_config_err("Resource start-up disabled since no STONITH resources have been defined");
 	    crm_config_err("Either configure some or disable STONITH with the stonith-enabled option");
 	    crm_config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
 	}
 	
 	return TRUE;
 }
 
 
 /* remove nodes that are down, stopping */
 /* create +ve rsc_to_node constraints between resources and the nodes they are running on */
 /* anything else? */
 gboolean
 unpack_status(xmlNode * status, pe_working_set_t *data_set)
 {
 	const char *id    = NULL;
 	const char *uname = NULL;
 
 	xmlNode * lrm_rsc    = NULL;
 	xmlNode * attrs      = NULL;
 	node_t    *this_node  = NULL;
 	
 	crm_debug_3("Beginning unpack");
 	xml_child_iter_filter(
 		status, node_state, XML_CIB_TAG_STATE,
 
 		id    = crm_element_value(node_state, XML_ATTR_ID);
 		uname = crm_element_value(node_state,    XML_ATTR_UNAME);
 		attrs = find_xml_node(
 			node_state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
 
 		lrm_rsc = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
 		lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
 
 		crm_debug_3("Processing node %s", uname);
 		this_node = pe_find_node_id(data_set->nodes, id);
 
 		if(uname == NULL) {
 			/* error */
 			continue;
 
 		} else if(this_node == NULL) {
 			crm_config_warn("Node %s in status section no longer exists",
 				       uname);
 			continue;
 		}
 
 		/* Mark the node as provisionally clean
 		 * - at least we have seen it in the current cluster's lifetime
 		 */
 		this_node->details->unclean = FALSE;
 		add_node_attrs(attrs, this_node, TRUE, data_set);
 
 		if(crm_is_true(g_hash_table_lookup(this_node->details->attrs, "standby"))) {
 			crm_info("Node %s is in standby-mode",
 				 this_node->details->uname);
 			this_node->details->standby = TRUE;
 		}
 		
 		crm_debug_3("determining node state");
 		determine_online_status(node_state, this_node, data_set);
 
 		if(this_node->details->online
 		   && data_set->no_quorum_policy == no_quorum_suicide) {
 		    /* Everything else should flow from this automatically
 		     * At least until the PE becomes able to migrate off healthy resources 
 		     */
 		    pe_fence_node(data_set, this_node, "because the cluster does not have quorum");
 		}
 
 		if(this_node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) {
 			/* offline nodes run no resources...
 			 * unless stonith is enabled in which case we need to
 			 *   make sure rsc start events happen after the stonith
 			 */
 			crm_debug_3("Processing lrm resource entries");
 			unpack_lrm_resources(this_node, lrm_rsc, data_set);
 		}
 		);
 
 	return TRUE;
 	
 }
 
 static gboolean
 determine_online_status_no_fencing(pe_working_set_t *data_set, xmlNode * node_state, node_t *this_node)
 {
 	gboolean online = FALSE;
 	const char *join_state = crm_element_value(node_state, XML_CIB_ATTR_JOINSTATE);
 	const char *crm_state  = crm_element_value(node_state, XML_CIB_ATTR_CRMDSTATE);
 	const char *ccm_state  = crm_element_value(node_state, XML_CIB_ATTR_INCCM);
 	const char *ha_state   = crm_element_value(node_state, XML_CIB_ATTR_HASTATE);
 	const char *exp_state  = crm_element_value(node_state, XML_CIB_ATTR_EXPSTATE);
 
 	if(ha_state == NULL) {
 		ha_state = DEADSTATUS;
 	}
 	
 	if(!crm_is_true(ccm_state) || safe_str_eq(ha_state, DEADSTATUS)){
 		crm_debug_2("Node is down: ha_state=%s, ccm_state=%s",
 			    crm_str(ha_state), crm_str(ccm_state));
 		
 	} else if(!crm_is_true(ccm_state)
 		  || safe_str_eq(ha_state, DEADSTATUS)) {
 
 	} else if(safe_str_eq(crm_state, ONLINESTATUS)) {
 		if(safe_str_eq(join_state, CRMD_JOINSTATE_MEMBER)) {
 			online = TRUE;
 		} else {
 			crm_debug("Node is not ready to run resources: %s", join_state);
 		}
 		
 	} else if(this_node->details->expected_up == FALSE) {
 		crm_debug_2("CRMd is down: ha_state=%s, ccm_state=%s",
 			    crm_str(ha_state), crm_str(ccm_state));
 		crm_debug_2("\tcrm_state=%s, join_state=%s, expected=%s",
 			    crm_str(crm_state), crm_str(join_state),
 			    crm_str(exp_state));
 		
 	} else {
 		/* mark it unclean */
 		pe_fence_node(data_set, this_node, "because it is partially and/or un-expectedly down");
 		crm_info("\tha_state=%s, ccm_state=%s,"
 			 " crm_state=%s, join_state=%s, expected=%s",
 			 crm_str(ha_state), crm_str(ccm_state),
 			 crm_str(crm_state), crm_str(join_state),
 			 crm_str(exp_state));
 	}
 	return online;
 }
 
 static gboolean
 determine_online_status_fencing(pe_working_set_t *data_set, xmlNode * node_state, node_t *this_node)
 {
 	gboolean online = FALSE;
 	gboolean do_terminate = FALSE;
 	const char *join_state = crm_element_value(node_state, XML_CIB_ATTR_JOINSTATE);
 	const char *crm_state  = crm_element_value(node_state, XML_CIB_ATTR_CRMDSTATE);
 	const char *ccm_state  = crm_element_value(node_state, XML_CIB_ATTR_INCCM);
 	const char *ha_state   = crm_element_value(node_state, XML_CIB_ATTR_HASTATE);
 	const char *exp_state  = crm_element_value(node_state, XML_CIB_ATTR_EXPSTATE);
 	const char *terminate = g_hash_table_lookup(this_node->details->attrs, "terminate");
 
 	if(ha_state == NULL) {
 		ha_state = DEADSTATUS;
 	}
 	
 	if(crm_is_true(terminate)) {
 	    do_terminate = TRUE;
 
 	} else if(terminate != NULL && strlen(terminate) > 0) {
 	    /* could be a time() value */
 	    char t = terminate[0];
 	    if(t != '0' && isdigit(t)) {
 		do_terminate = TRUE;
 	    }
 	}
 	
 	if(crm_is_true(ccm_state)
 	   && safe_str_eq(ha_state, ACTIVESTATUS)
 	   && safe_str_eq(crm_state, ONLINESTATUS)) {
 
 	    if(safe_str_eq(join_state, CRMD_JOINSTATE_MEMBER)) {
 		online = TRUE;
 		if(do_terminate) {
 		    pe_fence_node(data_set, this_node, "because termination was requested");
 		    this_node->details->shutdown = TRUE;
 		}
 
 	    } else if(join_state == exp_state /* == NULL */) {
 		crm_info("Node %s is coming up", this_node->details->uname);
 		crm_debug("\tha_state=%s, ccm_state=%s,"
 			  " crm_state=%s, join_state=%s, expected=%s",
 			  crm_str(ha_state), crm_str(ccm_state),
 			  crm_str(crm_state), crm_str(join_state),
 			  crm_str(exp_state));
 
 	    } else if(safe_str_eq(join_state, CRMD_JOINSTATE_PENDING)) {
 		crm_info("Node %s is not ready to run resources",
 			 this_node->details->uname);
 		this_node->details->standby = TRUE;
 		this_node->details->pending = TRUE;
 		online = TRUE;
 		
 	    } else if(safe_str_eq(join_state, CRMD_JOINSTATE_NACK)) {
 		crm_warn("Node %s is not part of the cluster",
 			 this_node->details->uname);
 		this_node->details->standby = TRUE;
 		this_node->details->pending = TRUE;
 		online = TRUE;
 		
 	    } else if(safe_str_eq(join_state, exp_state)) {
 		crm_info("Node %s is still coming up: %s",
 			 this_node->details->uname, join_state);
 		crm_info("\tha_state=%s, ccm_state=%s, crm_state=%s",
 			 crm_str(ha_state), crm_str(ccm_state), crm_str(crm_state));
 		this_node->details->standby = TRUE;
 		this_node->details->pending = TRUE;
 		online = TRUE;
 		
 	    } else {
 		crm_warn("Node %s (%s) is un-expectedly down",
 			 this_node->details->uname, this_node->details->id);
 		crm_info("\tha_state=%s, ccm_state=%s,"
 			 " crm_state=%s, join_state=%s, expected=%s",
 			 crm_str(ha_state), crm_str(ccm_state),
 			 crm_str(crm_state), crm_str(join_state),
 			 crm_str(exp_state));
 		pe_fence_node(data_set, this_node, "because it is un-expectedly down");
 	    }
 		
 	} else if(crm_is_true(ccm_state) == FALSE
  		  && safe_str_eq(ha_state, DEADSTATUS)
 		  && safe_str_eq(crm_state, OFFLINESTATUS)
 		  && this_node->details->expected_up == FALSE) {
 		crm_debug("Node %s is down: join_state=%s, expected=%s",
 			  this_node->details->uname,
 			  crm_str(join_state), crm_str(exp_state));
 
 #if 0
 		/* While a nice optimization, it causes the cluster to block until the node
 		 *  comes back online.  Which is a serious problem if the cluster software
 		 *  is not configured to start at boot or stonith is configured to merely
 		 *  stop the node instead of restart it.
 		 * Easily triggered by setting terminate=true for the DC
 		 */
 	} else if(do_terminate) {
 	    crm_info("Node %s is %s after forced termination",
 		     this_node->details->uname, crm_is_true(ccm_state)?"coming up":"going down");
 	    crm_debug("\tha_state=%s, ccm_state=%s,"
 		      " crm_state=%s, join_state=%s, expected=%s",
 		      crm_str(ha_state), crm_str(ccm_state),
 		      crm_str(crm_state), crm_str(join_state),
 		      crm_str(exp_state));
 	    
 	    if(crm_is_true(ccm_state) == FALSE) {
 		this_node->details->standby = TRUE;
 		this_node->details->pending = TRUE;
 		online = TRUE;
 	    }
 #endif
 	    
 	} else if(this_node->details->expected_up) {
 		/* mark it unclean */
 		pe_fence_node(data_set, this_node, "because it is un-expectedly down");
 		crm_info("\tha_state=%s, ccm_state=%s,"
 			 " crm_state=%s, join_state=%s, expected=%s",
 			 crm_str(ha_state), crm_str(ccm_state),
 			 crm_str(crm_state), crm_str(join_state),
 			 crm_str(exp_state));
 
 	} else {
 		crm_info("Node %s is down", this_node->details->uname);
 		crm_debug("\tha_state=%s, ccm_state=%s,"
 			  " crm_state=%s, join_state=%s, expected=%s",
 			  crm_str(ha_state), crm_str(ccm_state),
 			  crm_str(crm_state), crm_str(join_state),
 			  crm_str(exp_state));
 	}
 	return online;
 }
 
 gboolean
 determine_online_status(
 	xmlNode * node_state, node_t *this_node, pe_working_set_t *data_set)
 {
 	gboolean online = FALSE;
 	const char *shutdown = NULL;
 	const char *exp_state = crm_element_value(node_state, XML_CIB_ATTR_EXPSTATE);
 	
 	if(this_node == NULL) {
 		crm_config_err("No node to check");
 		return online;
 	}
 
 	this_node->details->shutdown = FALSE;
 	this_node->details->expected_up = FALSE;
 	shutdown = g_hash_table_lookup(this_node->details->attrs, XML_CIB_ATTR_SHUTDOWN);
 
 	if(shutdown != NULL && safe_str_neq("0", shutdown)) {
 	    this_node->details->shutdown = TRUE;
 
 	} else if(safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) {
 	    this_node->details->expected_up = TRUE;
 	}
 	
 	if(is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
 		online = determine_online_status_no_fencing(
 		    data_set, node_state, this_node);
 		
 	} else {
 		online = determine_online_status_fencing(
 		    data_set, node_state, this_node);
 	}
 	
 	if(online) {
 		this_node->details->online = TRUE;
 
 	} else {
 		/* remove node from contention */
 		this_node->fixed = TRUE;
 		this_node->weight = -INFINITY;
 	}
 
 	if(online && this_node->details->shutdown) {
 		/* dont run resources here */
 		this_node->fixed = TRUE;
 		this_node->weight = -INFINITY;
 	}	
 
 	if(this_node->details->unclean) {
 		pe_proc_warn("Node %s is unclean", this_node->details->uname);
 
 	} else if(this_node->details->online) {
 	    crm_info("Node %s is %s", this_node->details->uname,
 		     this_node->details->shutdown?"shutting down":
 		     this_node->details->pending?"pending":
 		     this_node->details->standby?"standby":"online");
 	    
 	} else {
 		crm_debug_2("Node %s is offline", this_node->details->uname);
 	}
 
 	return online;
 }
 
 #define set_char(x) last_rsc_id[lpc] = x; complete = TRUE;
 
 char *
 clone_zero(const char *last_rsc_id)
 {
     int lpc = 0;
     char *zero = NULL;
 
     CRM_CHECK(last_rsc_id != NULL, return NULL);
     if(last_rsc_id != NULL) {
 	lpc = strlen(last_rsc_id);
     }
     
     while(--lpc > 0) {
 	switch(last_rsc_id[lpc]) {
 	    case 0:
 		return NULL;
 		break;
 	    case '0':
 	    case '1':
 	    case '2':
 	    case '3':
 	    case '4':
 	    case '5':
 	    case '6':
 	    case '7':
 	    case '8':
 	    case '9':
 		break;
 	    case ':':
 		crm_malloc0(zero, lpc + 3);
 		memcpy(zero, last_rsc_id, lpc);		
 		zero[lpc] = ':';
 		zero[lpc+1] = '0';
 		zero[lpc+2] = 0;
 		return zero;
 	}
     }
     return NULL;
 }
 
 char *
 increment_clone(char *last_rsc_id)
 {
 	int lpc = 0;
 	int len = 0;
 	char *tmp = NULL;
 	gboolean complete = FALSE;
 
 	CRM_CHECK(last_rsc_id != NULL, return NULL);
 	if(last_rsc_id != NULL) {
 		len = strlen(last_rsc_id);
 	}
 	
 	lpc = len-1;
 	while(complete == FALSE && lpc > 0) {
 		switch (last_rsc_id[lpc]) {
 			case 0:
 				lpc--;
 				break;
 			case '0':
 				set_char('1');
 				break;
 			case '1':
 				set_char('2');
 				break;
 			case '2':
 				set_char('3');
 				break;
 			case '3':
 				set_char('4');
 				break;
 			case '4':
 				set_char('5');
 				break;
 			case '5':
 				set_char('6');
 				break;
 			case '6':
 				set_char('7');
 				break;
 			case '7':
 				set_char('8');
 				break;
 			case '8':
 				set_char('9');
 				break;
 			case '9':
 				last_rsc_id[lpc] = '0';
 				lpc--;
 				break;
 			case ':':
 				tmp = last_rsc_id;
 				crm_malloc0(last_rsc_id, len + 2);
 				memcpy(last_rsc_id, tmp, len);
 				last_rsc_id[++lpc] = '1';
 				last_rsc_id[len] = '0';
 				last_rsc_id[len+1] = 0;
 				complete = TRUE;
 				crm_free(tmp);
 				break;
 			default:
 				crm_err("Unexpected char: %c (%d)",
 					last_rsc_id[lpc], lpc);
 				break;
 		}
 	}
 	return last_rsc_id;
 }
 
 static resource_t *
 create_fake_resource(const char *rsc_id, xmlNode *rsc_entry, pe_working_set_t *data_set) 
 {
 	resource_t *rsc = NULL;
 	xmlNode *xml_rsc  = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
 	copy_in_properties(xml_rsc, rsc_entry);
 	crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
 	crm_log_xml_debug(xml_rsc, "Orphan resource");
 	
 	common_unpack(xml_rsc, &rsc, NULL, data_set);
 	set_bit(rsc->flags, pe_rsc_orphan);
 	
 	data_set->resources = g_list_append(data_set->resources, rsc);
 	return rsc;
 }
 
 extern resource_t *create_child_clone(resource_t *rsc, int sub_id, pe_working_set_t *data_set);
 
 static resource_t *find_clone(pe_working_set_t *data_set, node_t *node, resource_t *parent, const char *rsc_id) 
 {
     int len = 0;
     resource_t *rsc = NULL;
     char *base = clone_zero(rsc_id);
     char *alt_rsc_id = crm_strdup(rsc_id);
 
     CRM_ASSERT(parent != NULL);
     CRM_ASSERT(parent->variant == pe_clone || parent->variant == pe_master);
 
     if(base) {
 	len = strlen(base);
     }
     if(len > 0) {
 	base[len-1] = 0;
     }
     
     crm_debug_3("Looking for %s on %s in %s %d",
 		rsc_id, node->details->uname, parent->id, is_set(parent->flags, pe_rsc_unique));
     
     if(is_set(parent->flags, pe_rsc_unique)) {
 	crm_debug_3("Looking for %s", rsc_id);
 	rsc = parent->fns->find_rsc(parent, rsc_id, FALSE, FALSE, NULL, TRUE);
 
     } else {
 	rsc = parent->fns->find_rsc(parent, base, FALSE, TRUE, node, TRUE);
 	if(rsc != NULL && rsc->running_on) {
 	    rsc = NULL;
 	    crm_debug_3("Looking for an existing orphan for %s: %s on %s", parent->id, rsc_id, node->details->uname);
 
 	    /* There is already an instance of this _anonymous_ clone active on "node".
 	     *
 	     * If there is a partially active orphan (only applies to clone groups) on
 	     * the same node, use that.
 	     * Otherwise create a new (orphaned) instance at "orphan_check:".
 	    */
 	    slist_iter(child, resource_t, parent->children, lpc,
 		       node_t *loc = child->fns->location(child, NULL, TRUE);
 		       if(loc && loc->details == node->details) {
 			   resource_t *tmp = child->fns->find_rsc(child, base, FALSE, TRUE, NULL, TRUE);
 			   if(tmp && tmp->running_on == NULL) {
 			       rsc = tmp;
 			       break;
 			   }
 		       }
 		);
 	    
 	    goto orphan_check;
 	}
 	
 	while(rsc == NULL) {
 	    crm_debug_3("Trying %s", alt_rsc_id);
 	    rsc = parent->fns->find_rsc(parent, alt_rsc_id, FALSE, FALSE, NULL, TRUE);
 	    if(rsc == NULL) {
 		break;
 
 	    } else if(rsc->running_on == NULL) {
 		break;
 	    }
 	    alt_rsc_id = increment_clone(alt_rsc_id);
 	    rsc = NULL;
 	}
     }
 
   orphan_check:
     if(rsc == NULL) {
 	/* Create an extra orphan */
 	resource_t *top = create_child_clone(parent, -1, data_set);
 	crm_debug("Created orphan for %s: %s on %s", parent->id, rsc_id, node->details->uname);
 	rsc = top->fns->find_rsc(top, base, FALSE, TRUE, NULL, TRUE);
 	CRM_ASSERT(rsc != NULL);
     }
 
     crm_free(rsc->clone_name); rsc->clone_name = NULL;
     if(safe_str_neq(rsc_id, rsc->id)) {
 	crm_info("Internally renamed %s on %s to %s%s",
 		 rsc_id, node->details->uname, rsc->id,
 		 is_set(rsc->flags, pe_rsc_orphan)?" (ORPHAN)":"");
 	rsc->clone_name = crm_strdup(rsc_id);
     }
     
     crm_free(alt_rsc_id);
     crm_free(base);
     return rsc;
 }
 
 static resource_t *
 unpack_find_resource(
 	pe_working_set_t *data_set, node_t *node, const char *rsc_id, xmlNode *rsc_entry)
 {
 	resource_t *rsc = NULL;
 	resource_t *clone_parent = NULL;
 	char *alt_rsc_id = crm_strdup(rsc_id);
 	
 	crm_debug_2("looking for %s", rsc_id);
 		
 	rsc = pe_find_resource(data_set->resources, alt_rsc_id);
 	/* no match */
 	if(rsc == NULL) {
 	    /* Even when clone-max=0, we still create a single :0 orphan to match against */
 	    char *tmp = clone_zero(alt_rsc_id);
 	    resource_t *clone0 = pe_find_resource(data_set->resources, tmp);
 	    clone_parent = uber_parent(clone0);
 	    crm_free(tmp);
 	    
 	    crm_debug_2("%s not found: %s", alt_rsc_id, clone_parent?clone_parent->id:"orphan");
 
 	} else {
 	    clone_parent = uber_parent(rsc);
 	}
 
 	if(clone_parent && clone_parent->variant > pe_group) {
 	    rsc = find_clone(data_set, node, clone_parent, rsc_id);
 	    CRM_ASSERT(rsc != NULL);
 	}
 	
 	crm_free(alt_rsc_id);
 	return rsc;
 }
 
 static resource_t *
 process_orphan_resource(xmlNode *rsc_entry, node_t *node, pe_working_set_t *data_set) 
 {
 	resource_t *rsc = NULL;
 	const char *rsc_id   = crm_element_value(rsc_entry, XML_ATTR_ID);
 	
 	crm_debug("Detected orphan resource %s on %s", rsc_id, node->details->uname);	
 	rsc = create_fake_resource(rsc_id, rsc_entry, data_set);
 	
 	if(is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
 	    clear_bit(rsc->flags, pe_rsc_managed);
 		
 	} else {
 		print_resource(LOG_DEBUG_3, "Added orphan", rsc, FALSE);
 			
 		CRM_CHECK(rsc != NULL, return NULL);
 		resource_location(rsc, NULL, -INFINITY, "__orphan_dont_run__", data_set);
 	}
 	return rsc;
 }
 
 static void
 process_rsc_state(resource_t *rsc, node_t *node,
 		  enum action_fail_response on_fail,
 		  xmlNode *migrate_op,
 		  pe_working_set_t *data_set) 
 {
 	if(on_fail == action_migrate_failure) {
 	    node_t *from = NULL;
 	    const char *uuid = crm_element_value(migrate_op, CRMD_ACTION_MIGRATED);
 	    
 	    on_fail = action_fail_recover;
 	    
 	    from = pe_find_node_id(data_set->nodes, uuid);
 	    if(from != NULL) {
 		process_rsc_state(rsc, from, on_fail, NULL, data_set);
 	    } else {
 		crm_log_xml_err(migrate_op, "Bad Op");
 	    }
 	}
 	
 	crm_debug_2("Resource %s is %s on %s: on_fail=%s",
 		    rsc->id, role2text(rsc->role),
 		node->details->uname, fail2text(on_fail));
 
 	/* process current state */
 	if(rsc->role != RSC_ROLE_UNKNOWN) { 
 		rsc->known_on = g_list_append(rsc->known_on, node);
 	}
 
 	if(node->details->unclean) {
 	    /* No extra processing needed
 	     * Also allows resources to be started again after a node is shot
 	     */
 	    on_fail = action_fail_ignore;
 	}
 	
 	switch(on_fail) {
 	    case action_fail_ignore:
 		/* nothing to do */
 		break;
 		
 	    case action_fail_fence:
 		/* treat it as if it is still running
 		 * but also mark the node as unclean
 		 */
 		pe_fence_node(data_set, node, "to recover from resource failure(s)");
 		break;
 		
 	    case action_fail_standby:
 		node->details->standby = TRUE;
 		node->details->standby_onfail = TRUE;
 		break;
 		    
 	    case action_fail_block:
 		/* is_managed == FALSE will prevent any
 		 * actions being sent for the resource
 		 */
 		clear_bit(rsc->flags, pe_rsc_managed);
 		break;
 		
 	    case action_fail_migrate:
 		/* make sure it comes up somewhere else
 		 * or not at all
 		 */
 		resource_location(rsc, node, -INFINITY,
 				  "__action_migration_auto__",data_set);
 		break;
 		
 	    case action_fail_stop:		
 		rsc->next_role = RSC_ROLE_STOPPED;
 		break;
 		
 	    case action_fail_recover:
 		if(rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) { 
 		    set_bit(rsc->flags, pe_rsc_failed);
 		    stop_action(rsc, node, FALSE);
 		}
 		break;
 		
 	    case action_migrate_failure:
-		/* anything extra? */
+		/* Unreachable, leave to satisfy compiler */
 		break;
 	}
 	
 	if(rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
 		if(is_set(rsc->flags, pe_rsc_orphan)) {
 		    if(is_set(rsc->flags, pe_rsc_managed)) {
 			crm_config_warn("Detected active orphan %s running on %s",
 					rsc->id, node->details->uname);
 		    } else {
 			crm_config_warn("Cluster configured not to stop active orphans."
 					" %s must be stopped manually on %s",
 					rsc->id, node->details->uname);
 		    }
 		}
 	    
 		native_add_running(rsc, node, data_set);
 		if(on_fail != action_fail_ignore) {
 		    set_bit(rsc->flags, pe_rsc_failed);
 		}
 			
 	} else if(rsc->clone_name) {
 		crm_debug_2("Resetting clone_name %s for %s (stopped)",
 			    rsc->clone_name, rsc->id);
 		crm_free(rsc->clone_name);
 		rsc->clone_name = NULL;
 
 	} else {
 		char *key = stop_key(rsc);
 		GListPtr possible_matches = find_actions(rsc->actions, key, node);
 		slist_iter(stop, action_t, possible_matches, lpc,
 			   stop->optional = TRUE;
 			);
 		crm_free(key);
 	}
 }
 
 /* create active recurring operations as optional */ 
 static void
 process_recurring(node_t *node, resource_t *rsc,
 		  int start_index, int stop_index,
 		  GListPtr sorted_op_list, pe_working_set_t *data_set)
 {
 	const char *task = NULL;
 	const char *status = NULL;
 	
 	crm_debug_3("%s: Start index %d, stop index = %d",
 		    rsc->id, start_index, stop_index);
 	slist_iter(rsc_op, xmlNode, sorted_op_list, lpc,
 		   int interval = 0;
 		   char *key = NULL;
 		   const char *id = ID(rsc_op);
 		   const char *interval_s = NULL;
 		   if(node->details->online == FALSE) {
 			   crm_debug_4("Skipping %s/%s: node is offline",
 				       rsc->id, node->details->uname);
 			   break;
 			   
 		   } else if(start_index < stop_index) {
 			   crm_debug_4("Skipping %s/%s: not active",
 				       rsc->id, node->details->uname);
 			   break;
 			   
 		   } else if(lpc <= start_index) {
 			   crm_debug_4("Skipping %s/%s: old",
 				       id, node->details->uname);
 			   continue;
 		   }
 		   	
 		   interval_s = crm_element_value(rsc_op,XML_LRM_ATTR_INTERVAL);
 		   interval = crm_parse_int(interval_s, "0");
 		   if(interval == 0) {
 			   crm_debug_4("Skipping %s/%s: non-recurring",
 				       id, node->details->uname);
 			   continue;
 		   }
 		   
 		   status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
 		   if(safe_str_eq(status, "-1")) {
 			   crm_debug_4("Skipping %s/%s: status",
 				       id, node->details->uname);
 			   continue;
 		   }
 		   task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
 		   /* create the action */
 		   key = generate_op_key(rsc->id, task, interval);
 		   crm_debug_3("Creating %s/%s", key, node->details->uname);
 		   custom_action(rsc, key, task, node, TRUE, TRUE, data_set);
 		);
 }
 
 void
 calculate_active_ops(GListPtr sorted_op_list, int *start_index, int *stop_index) 
 {
 	const char *task = NULL;
 	const char *status = NULL;
 
 	*stop_index = -1;
 	*start_index = -1;
 	
 	slist_iter(
 		rsc_op, xmlNode, sorted_op_list, lpc,
 
 		task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
 		status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
 
 		if(safe_str_eq(task, CRMD_ACTION_STOP)
 		   && safe_str_eq(status, "0")) {
 			*stop_index = lpc;
 			
 		} else if(safe_str_eq(task, CRMD_ACTION_START)) {
 			*start_index = lpc;
 			
 		} else if(*start_index <= *stop_index
 			  && safe_str_eq(task, CRMD_ACTION_STATUS)) {
 			const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);
 			if(safe_str_eq(rc, "0") || safe_str_eq(rc, "8")) {
 				*start_index = lpc;
 			}
 		}
 		);
 }
 
 	
 static void
 unpack_lrm_rsc_state(
 	node_t *node, xmlNode * rsc_entry, pe_working_set_t *data_set)
 {	
 	int stop_index = -1;
 	int start_index = -1;
 	enum rsc_role_e req_role = RSC_ROLE_UNKNOWN;
 
 	const char *task = NULL;
 	const char *rsc_id  = crm_element_value(rsc_entry, XML_ATTR_ID);
 
 	resource_t *rsc = NULL;
 	GListPtr op_list = NULL;
 	GListPtr sorted_op_list = NULL;
 
 	xmlNode *migrate_op = NULL;
 	
 	enum action_fail_response on_fail = FALSE;
 	enum rsc_role_e saved_role = RSC_ROLE_UNKNOWN;
 	
 	crm_debug_3("[%s] Processing %s on %s",
 		    crm_element_name(rsc_entry), rsc_id, node->details->uname);
 
 	/* extract operations */
 	op_list = NULL;
 	sorted_op_list = NULL;
 		
 	xml_child_iter_filter(
 		rsc_entry, rsc_op, XML_LRM_TAG_RSC_OP,
 		op_list = g_list_append(op_list, rsc_op);
 		);
 
 	if(op_list == NULL) {
 		/* if there are no operations, there is nothing to do */
 		return;
 	}
 
 	/* find the resource */
 	rsc = unpack_find_resource(data_set, node, rsc_id, rsc_entry);
 	if(rsc == NULL) {
 		rsc = process_orphan_resource(rsc_entry, node, data_set);
 	} 
 	CRM_ASSERT(rsc != NULL);
 	
 	/* process operations */
 	saved_role = rsc->role;
 	on_fail = action_fail_ignore;
 	rsc->role = RSC_ROLE_UNKNOWN;
 	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 	
 	slist_iter(
 		rsc_op, xmlNode, sorted_op_list, lpc,
 
 		task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
 		if(safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
 			migrate_op = rsc_op;
 		}
 		
 		unpack_rsc_op(rsc, node, rsc_op, &on_fail, data_set);
 		);
 
 	/* create active recurring operations as optional */ 
 	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
 	process_recurring(node, rsc, start_index, stop_index,
 			  sorted_op_list, data_set);
 	
 	/* no need to free the contents */
 	g_list_free(sorted_op_list);
 	
 	process_rsc_state(rsc, node, on_fail, migrate_op, data_set);
 
 	if(get_target_role(rsc, &req_role)) {
 	    if(rsc->next_role == RSC_ROLE_UNKNOWN || req_role < rsc->next_role) {
 		crm_debug("%s: Overwriting calculated next role %s"
 			  " with requested next role %s",
 			  rsc->id, role2text(rsc->next_role),
 			  role2text(req_role));
 		rsc->next_role = req_role;
 
 	    } else if(req_role > rsc->next_role) {
 		crm_info("%s: Not overwriting calculated next role %s"
 			 " with requested next role %s",
 			 rsc->id, role2text(rsc->next_role),
 			 role2text(req_role));
 	    }
 	}
 		
 	if(saved_role > rsc->role) {
 		rsc->role = saved_role;
 	}
 }
 
 gboolean
 unpack_lrm_resources(node_t *node, xmlNode * lrm_rsc_list, pe_working_set_t *data_set)
 {
 	CRM_CHECK(node != NULL, return FALSE);
 
 	crm_debug_3("Unpacking resources on %s", node->details->uname);
 
 	xml_child_iter_filter(
 		lrm_rsc_list, rsc_entry, XML_LRM_TAG_RESOURCE,
 		unpack_lrm_rsc_state(node, rsc_entry, data_set);
 		);
 	
 	return TRUE;
 }
 
 static void set_active(resource_t *rsc) 
 {
     resource_t *top = uber_parent(rsc);
     if(top && top->variant == pe_master) {
 	rsc->role = RSC_ROLE_SLAVE;
     } else {
 	rsc->role = RSC_ROLE_STARTED;
     }
 }
 
 gboolean
 unpack_rsc_op(resource_t *rsc, node_t *node, xmlNode *xml_op,
 	      enum action_fail_response *on_fail, pe_working_set_t *data_set) 
 {    
 	const char *id          = NULL;
 	const char *key        = NULL;
 	const char *task        = NULL;
 	const char *magic       = NULL;
  	const char *task_id     = NULL;
  	const char *actual_rc   = NULL;
 /* 	const char *target_rc   = NULL;	 */
 	const char *task_status = NULL;
 	const char *interval_s  = NULL;
-	const char *op_digest   = NULL;
 	const char *op_version  = NULL;
 
 	int interval = 0;
 	int task_status_i = -2;
 	int actual_rc_i = 0;
 	int target_rc = -1;
 	
 	action_t *action = NULL;
 	node_t *effective_node = NULL;
 	resource_t *failed = NULL;
 
 	gboolean expired = FALSE;
 	gboolean is_probe = FALSE;
 	
 	CRM_CHECK(rsc    != NULL, return FALSE);
 	CRM_CHECK(node   != NULL, return FALSE);
 	CRM_CHECK(xml_op != NULL, return FALSE);
 
 	id	    = ID(xml_op);
 	task        = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
  	task_id     = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
 	task_status = crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS);
-	op_digest   = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
 	op_version  = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
 	magic	    = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);
 	key	    = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);
 
 	CRM_CHECK(id != NULL, return FALSE);
 	CRM_CHECK(task != NULL, return FALSE);
 	CRM_CHECK(task_status != NULL, return FALSE);
 
 	task_status_i = crm_parse_int(task_status, NULL);
 
 	CRM_CHECK(task_status_i <= LRM_OP_ERROR, return FALSE);
 	CRM_CHECK(task_status_i >= LRM_OP_PENDING, return FALSE);
 
 	if(safe_str_eq(task, CRMD_ACTION_NOTIFY)) {
 		/* safe to ignore these */
 		return TRUE;
 	}
 
 	if(rsc->failure_timeout > 0) {
 	    int last_run = 0;
 
 	    if(crm_element_value_int(xml_op, "last-run", &last_run) == 0) {
 /* int last_change = crm_element_value_int(xml_op, "last_rc_change"); */
 		time_t now = get_timet_now(data_set);
 		if(now > (last_run + rsc->failure_timeout)) {
 		    expired = TRUE;
 		}
 	    }
 	}
 	
 	crm_debug_2("Unpacking task %s/%s (call_id=%s, status=%s) on %s (role=%s)",
 		    id, task, task_id, task_status, node->details->uname,
 		    role2text(rsc->role));
 
 	interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
 	interval = crm_parse_int(interval_s, "0");
 	
 	if(interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
 		is_probe = TRUE;
 	}
 	
 	if(node->details->unclean) {
 		crm_debug_2("Node %s (where %s is running) is unclean."
 			  " Further action depends on the value of the stop's on-fail attribue",
 			  node->details->uname, rsc->id);
 	}
 
 	actual_rc = crm_element_value(xml_op, XML_LRM_ATTR_RC);
 	CRM_CHECK(actual_rc != NULL, return FALSE);	
 	actual_rc_i = crm_parse_int(actual_rc, NULL);
 
 	if(key) {
 	    int dummy = 0;
 	    char *dummy_string = NULL;
 	    decode_transition_key(key, &dummy_string, &dummy, &dummy, &target_rc);
 	    crm_free(dummy_string);
 	}
 	
 	if(task_status_i == LRM_OP_DONE && target_rc >= 0) {
 	    if(target_rc == actual_rc_i) {
 		task_status_i = LRM_OP_DONE;
 		
 	    } else {
 		task_status_i = LRM_OP_ERROR;
 		crm_debug("%s on %s returned %d (%s) instead of the expected value: %d (%s)",
 			 id, node->details->uname,
 			 actual_rc_i, execra_code2string(actual_rc_i),
 			 target_rc, execra_code2string(target_rc));
 	    }
 
 	} else if(task_status_i == LRM_OP_ERROR) {
 	    /* let us decide that */
  	    task_status_i = LRM_OP_DONE;
 	}
 	
 	if(task_status_i == LRM_OP_NOTSUPPORTED) {
 	    actual_rc_i = EXECRA_UNIMPLEMENT_FEATURE;
 	}
 
 	if(task_status_i != actual_rc_i
 	   && rsc->failure_timeout > 0
 	   && get_failcount(node, rsc, NULL, data_set) == 0) {
 	    action_t *clear_op = NULL;
 	    clear_op = custom_action(
 		rsc, crm_concat(rsc->id, CRM_OP_CLEAR_FAILCOUNT, '_'),
 		CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE, data_set);
 	    
 	    add_hash_param(clear_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);	    
 	    crm_notice("Clearing expired failcount for %s on %s", rsc->id, node->details->uname);
 	}
 	
 	if(expired
 	   && actual_rc_i != EXECRA_NOT_RUNNING
 	   && actual_rc_i != EXECRA_RUNNING_MASTER
 	   && actual_rc_i != EXECRA_OK) {
 	    crm_notice("Ignoring expired failure %s (rc=%d, magic=%s) on %s",
 		       id, actual_rc_i, magic, node->details->uname);
 	    goto done;
 	}	
 
 	/* we could clean this up significantly except for old LRMs and CRMs that
 	 * didnt include target_rc and liked to remap status
 	 */
 	switch(actual_rc_i) {
 	    case EXECRA_NOT_RUNNING:
 		if(is_probe || target_rc == actual_rc_i) {
 		    task_status_i = LRM_OP_DONE;
 		    rsc->role = RSC_ROLE_STOPPED;
 		    
 		    /* clear any previous failure actions */
 		    *on_fail = action_fail_ignore;
 		    rsc->next_role = RSC_ROLE_UNKNOWN;
 		    
 		} else if(safe_str_neq(task, CRMD_ACTION_STOP)) {
 		    task_status_i = LRM_OP_ERROR;
 		}
 		break;
 		
 	    case EXECRA_RUNNING_MASTER:
 		if(is_probe) {
 		    task_status_i = LRM_OP_DONE;
 		    crm_notice("Operation %s found resource %s active in master mode on %s",
 			     id, rsc->id, node->details->uname);
 
 		} else if(target_rc == actual_rc_i) {
 		    /* nothing to do */
 
 		} else if(target_rc >= 0) {
 		    task_status_i = LRM_OP_ERROR;
 
 		    /* legacy code for pre-0.6.5 operations */
 		} else if(safe_str_neq(task, CRMD_ACTION_STATUS)
 			  || rsc->role != RSC_ROLE_MASTER) {
 		    task_status_i = LRM_OP_ERROR;
 		    if(rsc->role != RSC_ROLE_MASTER) {
 			crm_err("%s reported %s in master mode on %s",
 				id, rsc->id,
 				node->details->uname);
 		    }
 		}
 		rsc->role = RSC_ROLE_MASTER;
 		break;
 		
 	    case EXECRA_FAILED_MASTER:
 		rsc->role = RSC_ROLE_MASTER;
 		task_status_i = LRM_OP_ERROR;
 		break;
 
 	    case EXECRA_UNIMPLEMENT_FEATURE:
 		if(interval > 0) {
 		    task_status_i = LRM_OP_NOTSUPPORTED;
 		    break;
 		}
 		/* else: fall through */
 	    case EXECRA_INSUFFICIENT_PRIV:
 	    case EXECRA_NOT_INSTALLED:
 	    case EXECRA_INVALID_PARAM:
 		effective_node = node;
 		/* fall through */
 	    case EXECRA_NOT_CONFIGURED:
 		failed = rsc;
 		if(is_not_set(rsc->flags, pe_rsc_unique)) {
 		    failed = uber_parent(failed);
 		}
 		
 		do_crm_log(actual_rc_i==EXECRA_NOT_INSTALLED?LOG_NOTICE:LOG_ERR,
 			   "Hard error - %s failed with rc=%d: Preventing %s from re-starting %s %s",
 			id, actual_rc_i, failed->id,
 			effective_node?"on":"anywhere",
 			effective_node?effective_node->details->uname:"in the cluster");
 
 		resource_location(failed, effective_node, -INFINITY, "hard-error", data_set);
 		if(is_probe) {
 			/* treat these like stops */
 			task = CRMD_ACTION_STOP;
 			task_status_i = LRM_OP_DONE;
 			crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
 			if(actual_rc_i != EXECRA_NOT_INSTALLED
 			   || is_set(data_set->flags, pe_flag_symmetric_cluster)) {
 			    add_node_copy(data_set->failed, xml_op);
 			}
 		}
 		break;
 
 	    case EXECRA_OK:
 		if(is_probe && target_rc == 7) {
 		    task_status_i = LRM_OP_DONE;
 		    crm_notice("Operation %s found resource %s active on %s",
 			     id, rsc->id, node->details->uname);
 
 		    /* legacy code for pre-0.6.5 operations */
 		} else if(target_rc < 0
 		   && interval > 0
 		   && rsc->role == RSC_ROLE_MASTER) {
 		    /* catch status ops that return 0 instead of 8 while they
 		     *   are supposed to be in master mode
 		     */
 		    task_status_i = LRM_OP_ERROR;
 		}
 		
 		break;
 		
 	    default:
 		if(task_status_i == LRM_OP_DONE) {
 		    crm_info("Remapping %s (rc=%d) on %s to an ERROR",
 			     id, actual_rc_i, node->details->uname);
 		    task_status_i = LRM_OP_ERROR;
 		}
 	}
 
 	if(task_status_i == LRM_OP_ERROR
 	   || task_status_i == LRM_OP_TIMEOUT
 	   || task_status_i == LRM_OP_NOTSUPPORTED) {
 	    action = custom_action(rsc, crm_strdup(id), task, NULL, TRUE, FALSE, data_set);
 	    if(expired) {
 		crm_notice("Ignoring expired failure (calculated) %s (rc=%d, magic=%s) on %s",
 			   id, actual_rc_i, magic, node->details->uname);
 		goto done;
 
 	    } else if(action->on_fail == action_fail_ignore) {
 		crm_warn("Remapping %s (rc=%d) on %s to DONE: ignore",
 			 id, actual_rc_i, node->details->uname);
 		task_status_i = LRM_OP_DONE;
 	    } 
 	}
 	
 	switch(task_status_i) {
 		case LRM_OP_PENDING:
 			if(safe_str_eq(task, CRMD_ACTION_START)) {
 				set_bit(rsc->flags, pe_rsc_start_pending);
 				set_active(rsc);
 				
 			} else if(safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
 				rsc->role = RSC_ROLE_MASTER;
 			}
 			break;
 		
 		case LRM_OP_DONE:
 			crm_debug_3("%s/%s completed on %s",
 				    rsc->id, task, node->details->uname);
 
 			if(actual_rc_i == EXECRA_NOT_RUNNING) {
 			    /* nothing to do */
 			    
 			} else if(safe_str_eq(task, CRMD_ACTION_STOP)) {
 				rsc->role = RSC_ROLE_STOPPED;
 			    
 				/* clear any previous failure actions */
 				switch(*on_fail) {
 				    case action_fail_block:
 				    case action_fail_stop:
 				    case action_fail_fence:
 				    case action_fail_migrate:
 				    case action_fail_standby:
 					crm_debug_2("%s.%s is not cleared by a completed stop",
 						    rsc->id, fail2text(*on_fail));
 					break;
 
 				    case action_fail_ignore:
 				    case action_fail_recover:
 				    case action_migrate_failure:
 					*on_fail = action_fail_ignore;
 					rsc->next_role = RSC_ROLE_UNKNOWN;
 				}
 
 			} else if(safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
 				rsc->role = RSC_ROLE_MASTER;
 
 			} else if(safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
 				rsc->role = RSC_ROLE_SLAVE;
 				
 			} else if(rsc->role < RSC_ROLE_STARTED) {
 				crm_debug_3("%s active on %s",
 					    rsc->id, node->details->uname);
 				set_active(rsc);
 			}
 			break;
 
 		case LRM_OP_ERROR:
 		case LRM_OP_TIMEOUT:
 		case LRM_OP_NOTSUPPORTED:
 			crm_warn("Processing failed op %s on %s: %s (%d)",
 				 id, node->details->uname,
 				 execra_code2string(actual_rc_i), actual_rc_i);
 			crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
 			add_node_copy(data_set->failed, xml_op);
 
 			if(*on_fail < action->on_fail) {
 				*on_fail = action->on_fail;
 			}
 
 			if(safe_str_eq(task, CRMD_ACTION_STOP)) {
 			    resource_location(
 				rsc, node, -INFINITY, "__stop_fail__", data_set);
 			    
 			} else if(safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
 			    rsc->role = RSC_ROLE_MASTER;
 
 			} else if(safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
 			    /*
 			     * staying in role=master ends up putting the PE/TE into a loop
 			     * setting role=slave is not dangerous because no master will be
 			     * promoted until the failed resource has been fully stopped
 			     */
 			    crm_warn("Forcing %s to stop after a failed demote action", rsc->id);
 			    rsc->next_role = RSC_ROLE_STOPPED;
 			    rsc->role = RSC_ROLE_SLAVE;
 				
 			} else if(compare_version("2.0", op_version) > 0
 				  && safe_str_eq(task, CRMD_ACTION_START)) {
 			    crm_warn("Compatibility handling for failed op %s on %s",
 				     id, node->details->uname);
 			    resource_location(
 				rsc, node, -INFINITY, "__legacy_start__", data_set);
 			}
 
 			if(rsc->role < RSC_ROLE_STARTED) {
 			    set_active(rsc);
 			}
 
 			crm_debug_2("Resource %s: role=%s, unclean=%s, on_fail=%s, fail_role=%s",
 				    rsc->id, role2text(rsc->role),
 				    node->details->unclean?"true":"false",
 				    fail2text(action->on_fail),
 				    role2text(action->fail_role));
 
 			if(action->fail_role != RSC_ROLE_STARTED
 			   && rsc->next_role < action->fail_role) {
 				rsc->next_role = action->fail_role;
 			}
 
 			if(action->fail_role == RSC_ROLE_STOPPED) {
 				crm_err("Making sure %s doesn't come up again", rsc->id);
 				/* make sure it doesnt come up again */
 				pe_free_shallow_adv(rsc->allowed_nodes, TRUE);
 				rsc->allowed_nodes = node_list_dup(
 					data_set->nodes, FALSE, FALSE);
 				slist_iter(
 					node, node_t, rsc->allowed_nodes, lpc,
 					node->weight = -INFINITY;
 					);
 			}
 			
 			pe_free_action(action);
 			action = NULL;
 			break;
 		case LRM_OP_CANCELLED:
 			/* do nothing?? */
 			pe_err("Dont know what to do for cancelled ops yet");
 			break;
 	}
 
   done:
 	crm_debug_3("Resource %s after %s: role=%s",
 		    rsc->id, task, role2text(rsc->role));
 
 	pe_free_action(action);
 	
 	return TRUE;
 }
 
 gboolean
 add_node_attrs(xmlNode *xml_obj, node_t *node, gboolean overwrite, pe_working_set_t *data_set)
 {
  	g_hash_table_insert(node->details->attrs,
 			    crm_strdup("#"XML_ATTR_UNAME),
 			    crm_strdup(node->details->uname));
  	g_hash_table_insert(node->details->attrs,
 			    crm_strdup("#"XML_ATTR_ID),
 			    crm_strdup(node->details->id));
 	if(safe_str_eq(node->details->id, data_set->dc_uuid)) {
 		data_set->dc_node = node;
 		node->details->is_dc = TRUE;
 		g_hash_table_insert(node->details->attrs,
 				    crm_strdup("#"XML_ATTR_DC),
 				    crm_strdup(XML_BOOLEAN_TRUE));
 	} else {
 		g_hash_table_insert(node->details->attrs,
 				    crm_strdup("#"XML_ATTR_DC),
 				    crm_strdup(XML_BOOLEAN_FALSE));
 	}
 	
 	unpack_instance_attributes(
 		data_set->input, xml_obj, XML_TAG_ATTR_SETS, NULL,
 		node->details->attrs, NULL, overwrite, data_set->now);
 
 	return TRUE;
 }
 
 static GListPtr
 extract_operations(const char *node, const char *rsc, xmlNode *rsc_entry, gboolean active_filter)
 {	
     int stop_index = -1;
     int start_index = -1;
     
     GListPtr op_list = NULL;
     GListPtr sorted_op_list = NULL;
 
     /* extract operations */
     op_list = NULL;
     sorted_op_list = NULL;
     
     xml_child_iter_filter(
 	rsc_entry, rsc_op, XML_LRM_TAG_RSC_OP,
 	crm_xml_add(rsc_op, "resource", rsc);
 	crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
 	op_list = g_list_append(op_list, rsc_op);
 	);
     
     if(op_list == NULL) {
 	/* if there are no operations, there is nothing to do */
 	return NULL;
     }
     
     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
     
     /* create active recurring operations as optional */ 
     if(active_filter == FALSE) {
 	return sorted_op_list;
     }
     
     op_list = NULL;
     
     calculate_active_ops(sorted_op_list, &start_index, &stop_index);	
     slist_iter(rsc_op, xmlNode, sorted_op_list, lpc,
 	       if(start_index < stop_index) {
 		   crm_debug_4("Skipping %s: not active", ID(rsc_entry));
 		   break;
 		   
 	       } else if(lpc < start_index) {
 		   crm_debug_4("Skipping %s: old", ID(rsc_op));
 		   continue;
 	       }
 	       op_list = g_list_append(op_list, rsc_op);
 	);
     
     g_list_free(sorted_op_list);
     return op_list;
 }
 
 GListPtr find_operations(
     const char *rsc, const char *node, gboolean active_filter, pe_working_set_t *data_set) 
 {
     GListPtr output = NULL;
     GListPtr intermediate = NULL;
 
     xmlNode *tmp = NULL;
     xmlNode *status = find_xml_node(data_set->input, XML_CIB_TAG_STATUS, TRUE);
 
     const char *uname = NULL;
     node_t *this_node = NULL;
     
     xml_child_iter_filter(
 	status, node_state, XML_CIB_TAG_STATE,
 	
 	uname = crm_element_value(node_state, XML_ATTR_UNAME);
 	if(node != NULL && safe_str_neq(uname, node)) {
 	    continue;
 	}
 
 	this_node = pe_find_node(data_set->nodes, uname);
 	CRM_CHECK(this_node != NULL, continue);
 	
 	determine_online_status(node_state, this_node, data_set);
 	
 	if(this_node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) {
 	    /* offline nodes run no resources...
 	     * unless stonith is enabled in which case we need to
 	     *   make sure rsc start events happen after the stonith
 	     */
 	    tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
 	    tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);
 
 	    xml_child_iter_filter(
 		tmp, lrm_rsc, XML_LRM_TAG_RESOURCE,
 		const char *rsc_id  = crm_element_value(lrm_rsc, XML_ATTR_ID);
 		if(rsc != NULL && safe_str_neq(rsc_id, rsc)) {
 		    continue;
 		}
 
 		intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
 		output = g_list_concat(output, intermediate);
 		);
 	}
 	);
 
     return output;
 }
diff --git a/lib/plugins/lrm/raexecstonith.c b/lib/plugins/lrm/raexecstonith.c
index 0d215f6c9b..8588dfa603 100644
--- a/lib/plugins/lrm/raexecstonith.c
+++ b/lib/plugins/lrm/raexecstonith.c
@@ -1,300 +1,296 @@
 /* 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  *
  * File: raexecocf.c
  * Author: Sun Jiang Dong <sunjd@cn.ibm.com>
  * Copyright (c) 2004 International Business Machines
  *
  * This code implements the Resource Agent Plugin Module for LSB style.
  * It's a part of Local Resource Manager. Currently it's used by lrmd only.
  */
 
 #include <crm_internal.h>
 #include <stdio.h>		
 #include <string.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <libgen.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <sys/stat.h>
 #include <errno.h>
 #include <glib.h>
 
 #if HAVE_HB_CONFIG_H
 #include <heartbeat/hb_config.h>
 #endif
 
 #if HAVE_GLUE_CONFIG_H
 #include <glue_config.h>
 #endif
 
 #include <clplumbing/cl_log.h>
 #include <clplumbing/uids.h>
 #include <pils/plugin.h>
 #include <dirent.h>
 #include <libgen.h>  /* Add it for compiling on OSX */
 #include <libxml/entities.h>
 
 #include <lrm/raexec.h>
 #include <crm/stonith-ng.h>
 #include <stonith/stonith.h>
 
 # define PIL_PLUGINTYPE		RA_EXEC_TYPE
 # define PIL_PLUGINTYPE_S	"RAExec"
 # define PIL_PLUGINLICENSE	LICENSE_PUBDOM
 # define PIL_PLUGINLICENSEURL	URL_PUBDOM
 
 # define PIL_PLUGIN		stonith
 # define PIL_PLUGIN_S		"stonith"
 
 static PIL_rc close_stonithRA(PILInterface*, void* ud_interface);
 
 /* The begin of exported function list */
 static int execra(const char * rsc_id,
 		  const char * rsc_type,
 		  const char * provider,
 		  const char * op_type,
 		  const int    timeout,
 	 	  GHashTable * params);
 static uniform_ret_execra_t map_ra_retvalue(int ret_execra
 	, const char * op_type, const char * std_output);
 static int get_resource_list(GList ** rsc_info);
 static char* get_resource_meta(const char* rsc_type,  const char* provider);
 static int get_provider_list(const char* op_type, GList ** providers);
 
 /* The end of exported function list */
 
 /* Rource agent execution plugin operations */
 static struct RAExecOps raops =
 {	execra,
 	map_ra_retvalue,
 	get_resource_list,
 	get_provider_list,
 	get_resource_meta
 };
 
 PIL_PLUGIN_BOILERPLATE2("1.0", Debug);
 
 static const PILPluginImports*  PluginImports;
 static PILPlugin*               OurPlugin;
 static PILInterface*		OurInterface;
 static void*			OurImports;
 static void*			interfprivate;
 
 /*
  * Our plugin initialization and registration function
  * It gets called when the plugin gets loaded.
  */
 PIL_rc
 PIL_PLUGIN_INIT(PILPlugin * us, const PILPluginImports* imports);
 
 PIL_rc
 PIL_PLUGIN_INIT(PILPlugin * us, const PILPluginImports* imports)
 {
 	/* Force the compiler to do a little type checking */
 	(void)(PILPluginInitFun)PIL_PLUGIN_INIT;
 
 	PluginImports = imports;
 	OurPlugin = us;
 
 	/* Register ourself as a plugin */
 	imports->register_plugin(us, &OurPIExports);
 
 	/*  Register our interfaces */
  	return imports->register_interface(us, PIL_PLUGINTYPE_S,  PIL_PLUGIN_S,
 		&raops, close_stonithRA, &OurInterface, &OurImports,
 		interfprivate);
 }
 
 static PIL_rc
 close_stonithRA(PILInterface* pif, void* ud_interface)
 {
 	return PIL_OK;
 }
 
 static int
 execra(const char *rsc_id, const char *rsc_type, const char *provider,
        const char *op_type, const int timeout, GHashTable *params)
 {
     int rc = 0;
     stonith_t *stonith_api = NULL;
     provider = get_stonith_provider(rsc_type, provider);
     crm_log_init("lrm-stonith", LOG_INFO, FALSE, FALSE, 0, NULL);
     
     if ( 0 == STRNCMP_CONST(op_type, "meta-data")) {
 	char *meta = get_resource_meta(rsc_type, provider);
 	printf("%s", meta);
 	free(meta);
 	exit(0);
     }
 
     stonith_api = stonith_api_new();
     rc = stonith_api->cmds->connect(stonith_api, "lrmd", NULL, NULL);
     if(provider == NULL) {
 	crm_err("No such legacy stonith device: %s", rsc_type);
 	rc = st_err_unknown_device;
 	    
     } else if ( 0 == STRNCMP_CONST(op_type, "monitor") ) {
 	/* monitor isn't universally supported yet - allow another option to be specified */
 	const char *action = g_hash_table_lookup(params, STONITH_ATTR_MONITOR_OP);
 	if(action == NULL) {
 	    action = "monitor";
 	} else {
 	    crm_debug("Using action %s for %s", action, op_type);
 	}
 	
 	rc = stonith_api->cmds->call(
 	    stonith_api, st_opt_sync_call, rsc_id, action, NULL, timeout);
 	
     } else if ( 0 == STRNCMP_CONST(op_type, "start") ) {
 	const char *agent = rsc_type;
 
 	if(0 == STRNCMP_CONST(provider, "heartbeat")) {
 	    agent = "fence_legacy";
 	    g_hash_table_replace(params, strdup("plugin"), strdup(rsc_type));
 	}
 	
 	rc = stonith_api->cmds->register_device(
 	    stonith_api, st_opt_sync_call, rsc_id, provider, agent, params);
 
     } else if ( 0 == STRNCMP_CONST(op_type, "stop") ) {
 	rc = stonith_api->cmds->remove_device(
 	    stonith_api, st_opt_sync_call, rsc_id);
     }
 
     crm_debug("%s_%s returned %d", rsc_id, op_type, rc);
     stonith_api->cmds->disconnect(stonith_api);
     stonith_api_delete(stonith_api);
     
     /* cl_log(LOG_DEBUG, "stonithRA orignal exit code=%d", exit_value); */
     exit(map_ra_retvalue(rc, op_type, NULL));
 }
 
 static uniform_ret_execra_t
 map_ra_retvalue(int rc, const char * op_type, const char * std_output)
 {
     if(rc == st_err_unknown_device) {
 	if ( 0 == STRNCMP_CONST(op_type, "stop") ) {
 	    rc = 0;
 
 	} else if ( 0 == STRNCMP_CONST(op_type, "start") ) {
 	    rc = 5;
 
 	} else {
 	    rc = 7;
 	}
 	
     } else if (rc < 0 || rc > EXECRA_STATUS_UNKNOWN) {
 	crm_warn("Mapped the invalid return code %d.", rc);
 	rc = EXECRA_UNKNOWN_ERROR;
     }
     return rc;
 }
 
 static int
 get_resource_list(GList ** rsc_info)
 {
     int file_num;
     char **entry = NULL;
     char **type_list = NULL;
     struct dirent **namelist;
 
     if ( rsc_info == NULL ) {
 	cl_log(LOG_ERR, "Parameter error: get_resource_list");
 	return -2;
     }
 
     /* Include Heartbeat agents */
     type_list = stonith_types();
     for(entry = type_list; *entry; ++entry) {
 	cl_log(LOG_INFO, "Added: %s", *entry);
 	*rsc_info = g_list_append(*rsc_info, *entry);
     }
 
     /* Include Red Hat agents, basically: ls -1 @sbin_dir@/fence_* */
     file_num = scandir(RH_STONITH_DIR, &namelist, 0, alphasort);
     if (file_num > 0) {
 	struct stat prop;
 	char buffer[FILENAME_MAX+1];
 
 	while (file_num--) {
 	    if ('.' == namelist[file_num]->d_name[0]) {
 		free(namelist[file_num]);
 		continue;
 
 	    } else if(0 != strncmp(RH_STONITH_PREFIX,
 				   namelist[file_num]->d_name,
 				   strlen(RH_STONITH_PREFIX))) {
 		free(namelist[file_num]);
 		continue;
 	    }
 	    
 	    snprintf(buffer,FILENAME_MAX,"%s/%s",
 		     RH_STONITH_DIR, namelist[file_num]->d_name);
-	    stat(buffer, &prop);
-	    if (S_ISREG(prop.st_mode)) {
+	    if(stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) {
 		*rsc_info = g_list_append(*rsc_info, g_strdup(namelist[file_num]->d_name));
 	    }
 
 	    free(namelist[file_num]);
 	}
 	free(namelist);
     }
 
     return 0;
 }
 
 static int
 get_provider_list(const char* op_type, GList ** providers)
 {
     if(providers == NULL) {
 	return -1;
-
-    } else if(op_type == NULL) {
-	return -2;
     }
 
     if(op_type == NULL) {
 	*providers = g_list_append(*providers, g_strdup("redhat"));
 	*providers = g_list_append(*providers, g_strdup("heartbeat"));
 	return 2;
 
     } else {
 	const char *provider = get_stonith_provider(op_type, NULL);
 	if(provider) {
 	    *providers = g_list_append(*providers, g_strdup(provider));
 	    return 1;
 	}
     }
 	
     return 0;
 }
 
 static char *
 get_resource_meta(const char* rsc_type, const char* provider)
 {
     char *buffer = NULL;
     stonith_t *stonith_api = stonith_api_new();
     stonith_api->cmds->metadata(
 	stonith_api, st_opt_sync_call, rsc_type, provider, &buffer, 0);
     stonith_api_delete(stonith_api);
     cl_log(LOG_INFO, "stonithRA plugin: got metadata: %s", buffer);
 
 
     /* TODO: Convert to XML and ensure our standard actions exist */
     return buffer;
 }
 
diff --git a/pengine/allocate.c b/pengine/allocate.c
index 15dc08cff1..c6f6a2d7e3 100644
--- a/pengine/allocate.c
+++ b/pengine/allocate.c
@@ -1,1699 +1,1698 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/msg.h>
 
 
 #include <glib.h>
 
 #include <crm/pengine/status.h>
 #include <pengine.h>
 #include <allocate.h>
 #include <utils.h>
 #include <lib/pengine/utils.h>
 
 void set_alloc_actions(pe_working_set_t *data_set);
 void migrate_reload_madness(pe_working_set_t *data_set);
 
 resource_alloc_functions_t resource_class_alloc_functions[] = {
 	{
 		native_merge_weights,
 		native_color,
 		native_create_actions,
 		native_create_probe,
 		native_internal_constraints,
 		native_rsc_colocation_lh,
 		native_rsc_colocation_rh,
 		native_rsc_order_lh,
 		native_rsc_order_rh,
 		native_rsc_location,
 		native_expand,
 		complex_migrate_reload,
 		complex_stonith_ordering,
 		native_append_meta,
 	},
 	{
 		group_merge_weights,
 		group_color,
 		group_create_actions,
 		native_create_probe,
 		group_internal_constraints,
 		group_rsc_colocation_lh,
 		group_rsc_colocation_rh,
 		group_rsc_order_lh,
 		group_rsc_order_rh,
 		group_rsc_location,
 		group_expand,
 		complex_migrate_reload,
 		complex_stonith_ordering,
 		group_append_meta,
 	},
 	{
 		native_merge_weights,
 		clone_color,
 		clone_create_actions,
 		clone_create_probe,
 		clone_internal_constraints,
 		clone_rsc_colocation_lh,
 		clone_rsc_colocation_rh,
 		clone_rsc_order_lh,
 		clone_rsc_order_rh,
 		clone_rsc_location,
 		clone_expand,
 		complex_migrate_reload,
 		complex_stonith_ordering,
 		clone_append_meta,
 	},
 	{
 		native_merge_weights,
 		master_color,
 		master_create_actions,
 		clone_create_probe,
 		master_internal_constraints,
 		clone_rsc_colocation_lh,
 		master_rsc_colocation_rh,
 		clone_rsc_order_lh,
 		clone_rsc_order_rh,
 		clone_rsc_location,
 		clone_expand,
 		complex_migrate_reload,
 		complex_stonith_ordering,
 		master_append_meta,
 	}
 };
 
 static gboolean
 check_rsc_parameters(resource_t *rsc, node_t *node, xmlNode *rsc_entry,
 		     pe_working_set_t *data_set) 
 {
 	int attr_lpc = 0;
 	gboolean force_restart = FALSE;
 	gboolean delete_resource = FALSE;
 	
 	const char *value = NULL;
 	const char *old_value = NULL;
 	const char *attr_list[] = {
 		XML_ATTR_TYPE, 
 		XML_AGENT_ATTR_CLASS,
  		XML_AGENT_ATTR_PROVIDER
 	};
 
 	for(; attr_lpc < DIMOF(attr_list); attr_lpc++) {
 		value = crm_element_value(rsc->xml, attr_list[attr_lpc]);
 		old_value = crm_element_value(rsc_entry, attr_list[attr_lpc]);
 		if(value == old_value /* ie. NULL */
 		   || crm_str_eq(value, old_value, TRUE)) {
 			continue;
 		}
 		
 		force_restart = TRUE;
 		crm_notice("Forcing restart of %s on %s, %s changed: %s -> %s",
 			   rsc->id, node->details->uname, attr_list[attr_lpc],
 			   crm_str(old_value), crm_str(value));
 	}
 	if(force_restart) {
 		/* make sure the restart happens */
 		stop_action(rsc, node, FALSE);
 		set_bit(rsc->flags, pe_rsc_start_pending);
 		delete_resource = TRUE;
 	}
 	return delete_resource;
 }
 
 static void CancelXmlOp(resource_t *rsc, xmlNode *xml_op, node_t *active_node,
 			const char *reason, pe_working_set_t *data_set) 
 {
     int interval = 0;
     action_t *cancel = NULL;
 
     char *key = NULL;
     const char *task = NULL;
     const char *call_id = NULL;
-    const char *op_version = NULL;
     const char *interval_s = NULL;
     
     CRM_CHECK(xml_op != NULL, return);
     CRM_CHECK(active_node != NULL, return);
 
     task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
     call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
-    op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
     interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
     
     interval = crm_parse_int(interval_s, "0");
 
     /* we need to reconstruct the key because of the way we used to construct resource IDs */
     key = generate_op_key(rsc->id, task, interval);
     
     crm_info("Action %s on %s will be stopped: %s",
 	     key, active_node->details->uname, reason?reason:"unknown");
 
     cancel = custom_action(rsc, crm_strdup(key), RSC_CANCEL,
 			   active_node, FALSE, TRUE, data_set);
 
     crm_free(cancel->task);
     cancel->task = crm_strdup(RSC_CANCEL);
     
     add_hash_param(cancel->meta, XML_LRM_ATTR_TASK,     task);
     add_hash_param(cancel->meta, XML_LRM_ATTR_CALLID,   call_id);
     add_hash_param(cancel->meta, XML_LRM_ATTR_INTERVAL, interval_s);
     
     custom_action_order(rsc, stop_key(rsc), NULL,
 			rsc, NULL, cancel, pe_order_optional, data_set);
     crm_free(key); key = NULL;
 }
 
 static gboolean
 check_action_definition(resource_t *rsc, node_t *active_node, xmlNode *xml_op,
 			pe_working_set_t *data_set)
 {
 	char *key = NULL;
 	int interval = 0;
 	const char *interval_s = NULL;
 	
 	gboolean did_change = FALSE;
 	gboolean start_op = FALSE;
 
 	xmlNode *params_all = NULL;
 	xmlNode *params_restart = NULL;
 	GHashTable *local_rsc_params = NULL;
 	
 	char *digest_all_calc = NULL;
 	const char *digest_all = NULL;
 
 	const char *restart_list = NULL;
 	const char *digest_restart = NULL;
 	char *digest_restart_calc = NULL;
 
 	action_t *action = NULL;
 	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
 	const char *op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
 
 	CRM_CHECK(active_node != NULL, return FALSE);
 
 	interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
 	interval = crm_parse_int(interval_s, "0");
 	/* we need to reconstruct the key because of the way we used to construct resource IDs */
 	key = generate_op_key(rsc->id, task, interval);
 
 	if(interval > 0) {
 		xmlNode *op_match = NULL;
 
 		crm_debug_2("Checking parameters for %s", key);
 		op_match = find_rsc_op_entry(rsc, key);
 
 		if(op_match == NULL && is_set(data_set->flags, pe_flag_stop_action_orphans)) {
 			CancelXmlOp(rsc, xml_op, active_node, "orphan", data_set);
 			crm_free(key); key = NULL;
 			return TRUE;
 
 		} else if(op_match == NULL) {
 			crm_debug("Orphan action detected: %s on %s",
 				  key, active_node->details->uname);
 			crm_free(key); key = NULL;
 			return TRUE;
 		}
 	}
 
 	action = custom_action(rsc, key, task, active_node, TRUE, FALSE, data_set);
 	
 	local_rsc_params = g_hash_table_new_full(
 		g_str_hash, g_str_equal,
 		g_hash_destroy_str, g_hash_destroy_str);
 	
 	get_rsc_attributes(local_rsc_params, rsc, active_node, data_set);
 	
 	params_all = create_xml_node(NULL, XML_TAG_PARAMS);
 	g_hash_table_foreach(local_rsc_params, hash2field, params_all);
 	g_hash_table_foreach(action->extra, hash2field, params_all);
 	g_hash_table_foreach(rsc->parameters, hash2field, params_all);
 	g_hash_table_foreach(action->meta, hash2metafield, params_all);
 
 	filter_action_parameters(params_all, op_version);
 	digest_all_calc = calculate_xml_digest(params_all, TRUE, FALSE);
 	digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
 	digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);
 	restart_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_RESTART);
 
 	if(crm_str_eq(task, RSC_START, TRUE)) {
 		start_op = TRUE;
 	}
 	
 	if(start_op && digest_restart) {
 		params_restart = copy_xml(params_all);
 		if(restart_list) {
 			filter_reload_parameters(params_restart, restart_list);
 		}
 
 		digest_restart_calc = calculate_xml_digest(params_restart, TRUE, FALSE);
 		if(safe_str_neq(digest_restart_calc, digest_restart)) {
 			did_change = TRUE;
 			crm_log_xml_info(params_restart, "params:restart");
 			crm_warn("Parameters to %s on %s changed: recorded %s vs. %s (restart:%s) %s",
 				 key, active_node->details->uname,
 				 crm_str(digest_restart), digest_restart_calc,
 				 op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
 			
 			key = generate_op_key(rsc->id, task, interval);
 			custom_action(rsc, key, task, NULL, FALSE, TRUE, data_set);
 			goto cleanup;
 		}
 	}
 
 	if(safe_str_neq(digest_all_calc, digest_all)) {
 		action_t *op = NULL;
 		did_change = TRUE;
 		crm_log_xml_info(params_all, "params:all");
  		crm_warn("Parameters to %s on %s changed: recorded %s vs. %s (all:%s) %s",
 			 key, active_node->details->uname,
 			 crm_str(digest_all), digest_all_calc, op_version,
 			 crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
 
 		if(interval == 0 && safe_str_neq(task, RSC_STOP)) {
 		    /* Anything except stop actions should result in a restart,
 		     * never a re-probe
 		     */
 		    task = RSC_START;
 		}
 		
 		key = generate_op_key(rsc->id, task, interval);
 		op = custom_action(rsc, key, task, NULL, FALSE, TRUE, data_set);
 		if(start_op && digest_restart) {
 			op->allow_reload_conversion = TRUE;
 
 		} else if(interval > 0) {
 			custom_action_order(rsc, start_key(rsc), NULL,
 					    NULL, crm_strdup(op->task), op,
 					    pe_order_runnable_left, data_set);
 		}
 		
 	}
 
   cleanup:
 	free_xml(params_all);
 	free_xml(params_restart);
 	crm_free(digest_all_calc);
 	crm_free(digest_restart_calc);
 	g_hash_table_destroy(local_rsc_params);
 
 	pe_free_action(action);
 	
 	return did_change;
 }
 
 extern gboolean DeleteRsc(resource_t *rsc, node_t *node, gboolean optional, pe_working_set_t *data_set);
 
 static void
 check_actions_for(xmlNode *rsc_entry, resource_t *rsc, node_t *node, pe_working_set_t *data_set)
 {
 	int interval = 0;
 	int stop_index = 0;
 	int start_index = 0;
 
-	const char *id = NULL;
 	const char *task = NULL;
 	const char *interval_s = NULL;
 
 	GListPtr op_list = NULL;
 	GListPtr sorted_op_list = NULL;
 	gboolean is_probe = FALSE;
 
 	CRM_CHECK(node != NULL, return);
 
 	if(is_set(rsc->flags, pe_rsc_orphan)) {
 		crm_debug_2("Skipping param check for %s: orphan", rsc->id);
 		return;
 		
 	} else if(pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
 		crm_debug_2("Skipping param check for %s: no longer active on %s",
 			    rsc->id, node->details->uname);
 		return;
 	}
 	
 	crm_debug_3("Processing %s on %s", rsc->id, node->details->uname);
 	
 	if(check_rsc_parameters(rsc, node, rsc_entry, data_set)) {
 	    DeleteRsc(rsc, node, FALSE, data_set);
 	}
 	
 	xml_child_iter_filter(
 		rsc_entry, rsc_op, XML_LRM_TAG_RSC_OP,
 		op_list = g_list_append(op_list, rsc_op);
 		);
 
 	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
 
 	slist_iter(
 		rsc_op, xmlNode, sorted_op_list, lpc,
 
 		if(start_index < stop_index) {
 			/* stopped */
 			continue;
 		} else if(lpc < start_index) {
 			/* action occurred prior to a start */
 			continue;
 		}
 		
-		id   = ID(rsc_op);
 		is_probe = FALSE;
 		task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
 
 		interval_s = crm_element_value(rsc_op, XML_LRM_ATTR_INTERVAL);
 		interval = crm_parse_int(interval_s, "0");
 		
 		if(interval == 0 && safe_str_eq(task, RSC_STATUS)) {
 			is_probe = TRUE;
 		}
 
 		if(interval > 0 && is_set(data_set->flags, pe_flag_maintenance_mode)) {
 			CancelXmlOp(rsc, rsc_op, node, "maintenance mode", data_set);
 
 		} else if(is_probe || safe_str_eq(task, RSC_START) || interval > 0) {
 			check_action_definition(rsc, node, rsc_op, data_set);
 		}
 		);
 
 	g_list_free(sorted_op_list);
 	
 }
 
 static GListPtr
 find_rsc_list(
     GListPtr result, resource_t *rsc, const char *id, gboolean renamed_clones, gboolean partial,
     pe_working_set_t *data_set)
 {
     gboolean match = FALSE;
     
     if(id == NULL) {
 	return NULL;
 
     } else if(rsc == NULL && data_set) {
 	slist_iter(child, resource_t, data_set->resources, lpc, 
 		   result = find_rsc_list(result, child, id, renamed_clones, partial, NULL));
 	return result;
+
+    } else if(rsc == NULL) {
+	return NULL;
     }
 
     if(partial) {
 	if(strstr(rsc->id, id)) {
 	    match = TRUE;
 
 	} else if(rsc->long_name && strstr(rsc->long_name, id)) {
 	    match = TRUE;
 	    
 	} else if(renamed_clones && rsc->clone_name && strstr(rsc->clone_name, id)) {
 	    match = TRUE;
 	}
 	
     } else {
 	if(strcmp(rsc->id, id) == 0){
 	    match = TRUE;
 	    
 	} else if(rsc->long_name && strcmp(rsc->long_name, id) == 0) {
 	    match = TRUE;
 	    
 	} else if(renamed_clones && rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
 	    match = TRUE;
 	}	
     }
 
     if(match) {
 	    result = g_list_append(result, rsc);
     }
 
     if(rsc->children) {
 	slist_iter(child, resource_t, rsc->children, lpc,
 		   result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
 	    );
     }
     
     return result;
 }
 
 static void
 check_actions(pe_working_set_t *data_set)
 {
     const char *id = NULL;
     node_t *node = NULL;
     xmlNode *lrm_rscs = NULL;
     xmlNode *status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);
 
     xml_child_iter_filter(
 	status, node_state, XML_CIB_TAG_STATE,
 
 	id       = crm_element_value(node_state, XML_ATTR_ID);
 	lrm_rscs = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
 	lrm_rscs = find_xml_node(lrm_rscs, XML_LRM_TAG_RESOURCES, FALSE);
 
 	node = pe_find_node_id(data_set->nodes, id);
 
 	if(node == NULL) {
 	    continue;
 
 	} else if(can_run_resources(node) == FALSE) {
 	    crm_debug_2("Skipping param check for %s: cant run resources",
 			node->details->uname);
 	    continue;
 	}
 	
 	crm_debug_2("Processing node %s", node->details->uname);
 	if(node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) {
 	    xml_child_iter_filter(
 		lrm_rscs, rsc_entry, XML_LRM_TAG_RESOURCE,
 		if(xml_has_children(rsc_entry)) {
 		    GListPtr result = NULL;
 		    const char *rsc_id = ID(rsc_entry);
 		    CRM_CHECK(rsc_id != NULL, return);
 
 		    result = find_rsc_list(NULL, NULL, rsc_id, TRUE, FALSE, data_set);
 		    slist_iter(rsc, resource_t, result, lpc, 
 			       check_actions_for(rsc_entry, rsc, node, data_set));
 		    g_list_free(result);
 		}
 		);
 	}
 	);
 }
 
 static gboolean 
 apply_placement_constraints(pe_working_set_t *data_set)
 {
 	crm_debug_3("Applying constraints...");
 	slist_iter(
 		cons, rsc_to_node_t, data_set->placement_constraints, lpc,
 
 		cons->rsc_lh->cmds->rsc_location(cons->rsc_lh, cons);
 		);
 	
 	return TRUE;
 	
 }
 
 static void
 common_apply_stickiness(resource_t *rsc, node_t *node, pe_working_set_t *data_set) 
 {
 	int fail_count = 0;
 	resource_t *failed = rsc;
 
 	if(rsc->children) {
 	    slist_iter(
 		child_rsc, resource_t, rsc->children, lpc,
 		common_apply_stickiness(child_rsc, node, data_set);
 		);
 	    return;
 	}
 
 	if(is_set(rsc->flags, pe_rsc_managed)
 	   && rsc->stickiness != 0
 	   && g_list_length(rsc->running_on) == 1) {
 	    node_t *current = pe_find_node_id(rsc->running_on, node->details->id);
 	    node_t *match = pe_find_node_id(rsc->allowed_nodes, node->details->id);
 
 	    if(current == NULL) {
 		
 	    } else if(match != NULL || is_set(data_set->flags, pe_flag_symmetric_cluster)) {
 		resource_t *sticky_rsc = rsc;
 		
 		resource_location(sticky_rsc, node, rsc->stickiness, "stickiness", data_set);
 		crm_debug("Resource %s: preferring current location"
 			    " (node=%s, weight=%d)", sticky_rsc->id,
 			    node->details->uname, rsc->stickiness);
 	    } else {
 		crm_debug("Ignoring stickiness for %s: the cluster is asymmetric"
 			  " and node %s is not explicitly allowed",
 			  rsc->id, node->details->uname);
 		slist_iter(node, node_t, rsc->allowed_nodes, lpc,
 			   crm_err("%s[%s] = %d", rsc->id, node->details->uname, node->weight));
 	    }
 	}
 	
 	if(is_not_set(rsc->flags, pe_rsc_unique)) {
 	    failed = uber_parent(rsc);
 	}
 	    
 	fail_count = get_failcount(node, rsc, NULL, data_set);
 	if(fail_count > 0 && rsc->migration_threshold != 0) {
 	    if(rsc->migration_threshold <= fail_count) {
 		resource_location(failed, node, -INFINITY, "__fail_limit__", data_set);
 		crm_warn("Forcing %s away from %s after %d failures (max=%d)",
 			 failed->id, node->details->uname, fail_count, rsc->migration_threshold);
 	    } else {
 		crm_notice("%s can fail %d more times on %s before being forced off",
 			   failed->id, rsc->migration_threshold - fail_count, node->details->uname);
 	    }
 	}
 }
 
 static void complex_set_cmds(resource_t *rsc)
 {
     rsc->cmds = &resource_class_alloc_functions[rsc->variant];
     slist_iter(
 	child_rsc, resource_t, rsc->children, lpc,
 	complex_set_cmds(child_rsc);
 	);
 }
 
 void
 set_alloc_actions(pe_working_set_t *data_set) 
 {
 	slist_iter(
 		rsc, resource_t, data_set->resources, lpc,
 		complex_set_cmds(rsc);
 		);
 }
 
 static void
 calculate_system_health (gpointer gKey, gpointer gValue, gpointer user_data)
 {
 	const char    *key      = (const char *)gKey;
 	const char    *value    = (const char *)gValue;
 	int *system_health = (int *)user_data;
 
 	if (!gKey || !gValue || !user_data) {
 		return;
 	}
 
 	/* Does it start with #health? */
 	if (0 == strncmp (key, "#health", 7)) {
 		int score;
 
 		/* Convert the value into an integer */
 		score = char2score (value);
 
 		/* Add it to the running total */
 		*system_health = merge_weights (score, *system_health);
 	}
 }
 
 static gboolean
 apply_system_health(pe_working_set_t *data_set)
 {
     const char *health_strategy = pe_pref(data_set->config_hash, "node-health-strategy");
 
     if (health_strategy == NULL
 	|| safe_str_eq (health_strategy, "none")) {
 	/* Prevent any accidental health -> score translation */ 
 	node_score_red    = 0;
     	node_score_yellow = 0;
 	node_score_green  = 0;
 	return TRUE;
 
     } else if (safe_str_eq (health_strategy, "migrate-on-red")) {
 
 	/* Resources on nodes which have health values of red are
 	 * weighted away from that node.
 	 */
 	node_score_red    = -INFINITY;
 	node_score_yellow = 0;
 	node_score_green  = 0;
 
     } else if (safe_str_eq (health_strategy, "only-green")) {
 
 	/* Resources on nodes which have health values of red or yellow
 	 * are forced away from that node.
 	 */
 	node_score_red    = -INFINITY;
 	node_score_yellow = -INFINITY;
 	node_score_green  = 0;
 	    
     } else if (safe_str_eq (health_strategy, "progressive")) {
 	/* Same as the above, but use the r/y/g scores provided by the user
 	 * Defaults are provided by the pe_prefs table
 	 */
 	    
     } else if (safe_str_eq (health_strategy, "custom")) {
 
 	/* Requires the admin to configure the rsc_location constaints for
 	 * processing the stored health scores
 	 */
 	/* TODO: Check for the existance of appropriate node health constraints */ 
 	return TRUE;
 
     } else {
 	crm_err ("Unknown node health strategy: %s", health_strategy);
 	return FALSE;
     }
 
     crm_info ("Applying automated node health strategy: %s", health_strategy);	
     slist_iter(
 	node, node_t, data_set->nodes, lpc,
 
 	int system_health = 0;
 	
 	/* Search through the node hash table for system health entries. */
 	g_hash_table_foreach (
 	    node->details->attrs, calculate_system_health, &system_health);
 	
 	crm_info (" Node %s has an combined system health of %d",
 		  node->details->uname, system_health);
 	
 	/* If the health is non-zero, then create a new rsc2node so that the
 	 * weight will be added later on.
 	 */
 	if (system_health != 0) {
 	    slist_iter(
 		rsc, resource_t, data_set->resources, lpc,
 		
 		rsc2node_new (health_strategy, rsc, system_health, node, data_set);
 		);
 	}
 	
     );
 
     return TRUE;
 }
 
 gboolean
 stage0(pe_working_set_t *data_set)
 {
 	xmlNode * cib_constraints = get_object_root(
 		XML_CIB_TAG_CONSTRAINTS, data_set->input);
 
 	if(data_set->input == NULL) {
 		return FALSE;
 	}
 
 	cluster_status(data_set);
 	
 	set_alloc_actions(data_set);
 	apply_system_health(data_set);
 	unpack_constraints(cib_constraints, data_set);
 	
 	return TRUE;
 }
 
 /*
  * Check nodes for resources started outside of the LRM
  */
 gboolean
 probe_resources(pe_working_set_t *data_set)
 {
 	action_t *probe_complete = NULL;
 	action_t *probe_node_complete = NULL;
 
 	slist_iter(
 		node, node_t, data_set->nodes, lpc,
 		gboolean force_probe = FALSE;
 		const char *probed = g_hash_table_lookup(
 			node->details->attrs, CRM_OP_PROBED);
 
 		if(node->details->online == FALSE) {
 			continue;
 			
 		} else if(node->details->unclean) {
 			continue;
 
 		} else if(probe_complete == NULL) {
 			probe_complete = get_pseudo_op(CRM_OP_PROBED, data_set);
 		}
 
 		if(probed != NULL && crm_is_true(probed) == FALSE) {
 			force_probe = TRUE;
 		}
 		
 		probe_node_complete = custom_action(
 			NULL, crm_strdup(CRM_OP_PROBED),
 			CRM_OP_PROBED, node, FALSE, TRUE, data_set);
 		probe_node_complete->optional = crm_is_true(probed);
 		probe_node_complete->priority = INFINITY;
 		add_hash_param(probe_node_complete->meta,
 			       XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
 
 		if(node->details->pending) {
 		    probe_node_complete->runnable = FALSE;
 		    crm_info("Action %s on %s is unrunnable (pending)",
 			     probe_node_complete->uuid, probe_node_complete->node->details->uname);
 		}
 		
 		order_actions(probe_node_complete, probe_complete, pe_order_runnable_left);
 		
 		slist_iter(
 			rsc, resource_t, data_set->resources, lpc2,
 			
 			if(rsc->cmds->create_probe(
 				   rsc, node, probe_node_complete,
 				   force_probe, data_set)) {
 
 				probe_complete->optional = FALSE;
 				probe_node_complete->optional = FALSE;
 
 				custom_action_order(
 					NULL, NULL, probe_complete,
 					rsc, start_key(rsc), NULL,
 					pe_order_optional, data_set);
 
 				custom_action_order(
 					NULL, NULL, probe_complete,
 					rsc, stop_key(rsc), NULL,
 					pe_order_optional, data_set);
 			}
 			);
 		);
 
 	return TRUE;
 }
 
 
 /*
  * Count how many valid nodes we have (so we know the maximum number of
  *  colors we can resolve).
  *
  * Apply node constraints (ie. filter the "allowed_nodes" part of resources
  */
 gboolean
 stage2(pe_working_set_t *data_set)
 {
 	crm_debug_3("Applying placement constraints");	
 	
 	slist_iter(
 		node, node_t, data_set->nodes, lpc,
 		if(node == NULL) {
 			/* error */
 
 		} else if(node->weight >= 0.0 /* global weight */
 			  && node->details->online
 			  && node->details->type == node_member) {
 			data_set->max_valid_nodes++;
 		}
 		);
 
 	apply_placement_constraints(data_set);
 
 	slist_iter(node, node_t, data_set->nodes, lpc,
 		   slist_iter(
 		       rsc, resource_t, data_set->resources, lpc2,
 		       common_apply_stickiness(rsc, node, data_set);
 		       );
 	    );
 
 	return TRUE;
 }
 
 
 /*
  * Create internal resource constraints before allocation
  */
 gboolean
 stage3(pe_working_set_t *data_set)
 {
 	slist_iter(
 		rsc, resource_t, data_set->resources, lpc,
 		rsc->cmds->internal_constraints(rsc, data_set);
 		);
 	
 	return TRUE;
 }
 
 /*
  * Check for orphaned or redefined actions
  */
 gboolean
 stage4(pe_working_set_t *data_set)
 {
 	check_actions(data_set);
 	return TRUE;
 }
 
 gboolean
 stage5(pe_working_set_t *data_set)
 {
 	slist_iter(
 		node, node_t, data_set->nodes, lpc,
 		dump_node_capacity(show_utilization?0:utilization_log_level, "Original", node);
 		);
 
 	/* Take (next) highest resource, assign it and create its actions */
 	slist_iter(
 		rsc, resource_t, data_set->resources, lpc,
 		rsc->cmds->color(rsc, data_set);
 		);
 
 	slist_iter(
 		node, node_t, data_set->nodes, lpc,
 		dump_node_capacity(show_utilization?0:utilization_log_level, "Remaining", node);
 		);
 
 	probe_resources(data_set);
 	
 	slist_iter(
 		rsc, resource_t, data_set->resources, lpc,
 		rsc->cmds->create_actions(rsc, data_set);	
 		);
 
 	return TRUE;
 }
 
 static gboolean is_managed(const resource_t *rsc)
 {
     if(is_set(rsc->flags, pe_rsc_managed)) {
 	return TRUE;
     }
     
     slist_iter(
 	child_rsc, resource_t, rsc->children, lpc,
 	if(is_managed(child_rsc)) {
 	    return TRUE;
 	}
 	);
     
     return FALSE;
 }
 
 static gboolean any_managed_resouces(pe_working_set_t *data_set)
 {
     slist_iter(
 	rsc, resource_t, data_set->resources, lpc,
 	if(is_managed(rsc)) {
 	    return TRUE;
 	}
 	);
     return FALSE;
 }
 
 /*
  * Create dependancies for stonith and shutdown operations
  */
 gboolean
 stage6(pe_working_set_t *data_set)
 {
 	action_t *dc_down = NULL;
 	action_t *stonith_op = NULL;
 	action_t *last_stonith = NULL;
 	gboolean integrity_lost = FALSE;
 	action_t *ready = get_pseudo_op(STONITH_UP, data_set);
 	action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);
 	action_t *done = get_pseudo_op(STONITH_DONE, data_set);
 	gboolean need_stonith = FALSE;
 	
 	crm_debug_3("Processing fencing and shutdown cases");
 
 	if(is_set(data_set->flags, pe_flag_stonith_enabled)
 	   && (is_set(data_set->flags, pe_flag_have_quorum)
 	       || data_set->no_quorum_policy == no_quorum_ignore
 	       || data_set->no_quorum_policy == no_quorum_suicide)) {
 	    need_stonith = TRUE;
 	}
 	
 	if(need_stonith && any_managed_resouces(data_set) == FALSE) {
 	    crm_info("Delaying fencing operations until there are resources to manage");
 	    need_stonith = FALSE;
 	}
 	
 	slist_iter(
 		node, node_t, data_set->nodes, lpc,
 
 		stonith_op = NULL;
 		if(node->details->unclean && need_stonith) {
 			pe_warn("Scheduling Node %s for STONITH",
 				 node->details->uname);
 
 			stonith_op = custom_action(
 				NULL, crm_strdup(CRM_OP_FENCE),
 				CRM_OP_FENCE, node, FALSE, TRUE, data_set);
 
 			add_hash_param(
 				stonith_op->meta, XML_LRM_ATTR_TARGET,
 				node->details->uname);
 
 			add_hash_param(
 				stonith_op->meta, XML_LRM_ATTR_TARGET_UUID,
 				node->details->id);
 
 			add_hash_param(
 				stonith_op->meta, "stonith_action",
 				data_set->stonith_action);
 			
 			stonith_constraints(node, stonith_op, data_set);
 			order_actions(ready, stonith_op, pe_order_implies_left);
 			order_actions(stonith_op, all_stopped, pe_order_implies_right);
 
 			if(node->details->is_dc) {
 				dc_down = stonith_op;
 
 			} else {
 				if(last_stonith) {
 					order_actions(last_stonith, stonith_op, pe_order_implies_left);
 				}
 				last_stonith = stonith_op;			
 			}
 
 		} else if(node->details->online && node->details->shutdown) {			
 			action_t *down_op = NULL;	
 			crm_info("Scheduling Node %s for shutdown",
 				 node->details->uname);
 
 			down_op = custom_action(
 				NULL, crm_strdup(CRM_OP_SHUTDOWN),
 				CRM_OP_SHUTDOWN, node, FALSE, TRUE, data_set);
 
 			shutdown_constraints(node, down_op, data_set);
 			add_hash_param(down_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
 
 			if(node->details->is_dc) {
 				dc_down = down_op;
 			}
 		}
 
 		if(node->details->unclean && stonith_op == NULL) {
 			integrity_lost = TRUE;
 			pe_warn("Node %s is unclean!", node->details->uname);
 		}
 		);
 
 	if(integrity_lost) {
 	    if(is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
 		pe_warn("YOUR RESOURCES ARE NOW LIKELY COMPROMISED");
 		pe_err("ENABLE STONITH TO KEEP YOUR RESOURCES SAFE");
 
 	    } else if(is_set(data_set->flags, pe_flag_have_quorum) == FALSE) {
 		crm_notice("Cannot fence unclean nodes until quorum is"
 			   " attained (or no-quorum-policy is set to ignore)");
 	    }
 	}
 	
 	if(dc_down != NULL) {
 		GListPtr shutdown_matches = find_actions(
 			data_set->actions, CRM_OP_SHUTDOWN, NULL);
 		crm_debug_2("Ordering shutdowns before %s on %s (DC)",
 			dc_down->task, dc_down->node->details->uname);
 
 		add_hash_param(dc_down->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
 		
 		slist_iter(
 			node_stop, action_t, shutdown_matches, lpc,
 			if(node_stop->node->details->is_dc) {
 				continue;
 			}
 			crm_debug("Ordering shutdown on %s before %s on %s",
 				node_stop->node->details->uname,
 				dc_down->task, dc_down->node->details->uname);
 
 			order_actions(node_stop, dc_down, pe_order_implies_left);
 			);
 
 		if(last_stonith && dc_down != last_stonith) {
 			order_actions(last_stonith, dc_down, pe_order_implies_left);
 		}
 		g_list_free(shutdown_matches);
 	}
 
 	if(last_stonith) {
 	    order_actions(last_stonith, done, pe_order_implies_right);
 	}
 	order_actions(ready, done, pe_order_optional);
 	return TRUE;
 }
 
 /*
  * Determin the sets of independant actions and the correct order for the
  *  actions in each set.
  *
  * Mark dependencies of un-runnable actions un-runnable
  *
  */
 gboolean
 stage7(pe_working_set_t *data_set)
 {
 	crm_debug_4("Applying ordering constraints");
 
 	slist_iter(
 		order, order_constraint_t, data_set->ordering_constraints, lpc,
 
 		resource_t *rsc = order->lh_rsc;
 		crm_debug_3("Applying ordering constraint: %d", order->id);
 		
 		if(rsc != NULL) {
 			crm_debug_4("rsc_action-to-*");
 			rsc->cmds->rsc_order_lh(rsc, order, data_set);
 			continue;
 		}
 
 		rsc = order->rh_rsc;
 		if(rsc != NULL) {
 			crm_debug_4("action-to-rsc_action");
 			rsc->cmds->rsc_order_rh(order->lh_action, rsc, order);
 
 		} else {
 			crm_debug_4("action-to-action");
 			order_actions(
 				order->lh_action, order->rh_action, order->type);
 		}
 		);
 
 	/* This code may need optimization:
 	 *
 	 * ptest -x with 800 resources, 80 nodes, calling update_action_states() with:
 	 *   no probes					O(8s)
 	 *   full probe detection but no creation	O(2m)
 	 *   full probe detection and creation		O(Lifetime of the universe)
 	 *
 	 * Duplicate detection in order_actions() has resolved the issue for now,
 	 * the full detection with "creation" and "no creation" cases are now identical
 	 */
 	update_action_states(data_set->actions);
 
 	slist_iter(
 		rsc, resource_t, data_set->resources, lpc,
 
 		rsc->cmds->migrate_reload(rsc, data_set);
 		LogActions(rsc, data_set);
 		);
 
 	return TRUE;
 }
 
 static gint
 sort_notify_entries(gconstpointer a, gconstpointer b)
 {
 	int tmp;
 	const notify_entry_t *entry_a = a;
 	const notify_entry_t *entry_b = b;
 
 	if(entry_a == NULL && entry_b == NULL) { return 0; }
 	if(entry_a == NULL) { return 1; }
 	if(entry_b == NULL) { return -1; }
 
 	if(entry_a->rsc == NULL && entry_b->rsc == NULL) { return 0; }
 	if(entry_a->rsc == NULL) { return 1; }
 	if(entry_b->rsc == NULL) { return -1; }
 
 	tmp = strcmp(entry_a->rsc->id, entry_b->rsc->id);
 	if(tmp != 0) {
 		return tmp;
 	}
 
 	if(entry_a->node == NULL && entry_b->node == NULL) { return 0; }
 	if(entry_a->node == NULL) { return 1; }
 	if(entry_b->node == NULL) { return -1; }
 
 	return strcmp(entry_a->node->details->id, entry_b->node->details->id);
 }
 
 static void
 expand_list(GListPtr list, char **rsc_list, char **node_list)
 {
 	const char *uname = NULL;
 	const char *rsc_id = NULL;
 	const char *last_rsc_id = NULL;
 
 	if(list == NULL) {
 	    *rsc_list = crm_strdup(" ");
 	    if(node_list) {
 		*node_list = crm_strdup(" ");
 	    }
 	    return;
 	}
 	
 	*rsc_list = NULL;
 	if(node_list) {
 	    *node_list = NULL;
 	}
 	
 	slist_iter(entry, notify_entry_t, list, lpc,
 
 		   CRM_CHECK(entry != NULL, continue);
 		   CRM_CHECK(entry->rsc != NULL, continue);
 		   CRM_CHECK(node_list == NULL || entry->node != NULL, continue);
 
 		   uname = NULL;
 		   rsc_id = entry->rsc->id;
 		   CRM_ASSERT(rsc_id != NULL);
 
 		   /* filter dups */
 		   if(safe_str_eq(rsc_id, last_rsc_id)) {
 			   continue;
 		   }
 		   last_rsc_id = rsc_id;
 
 		   if(rsc_list != NULL) {
 			   int existing_len = 0;
 			   int len = 2 + strlen(rsc_id); /* +1 space, +1 EOS */
 			   if(rsc_list && *rsc_list) {
 				   existing_len = strlen(*rsc_list);
 			   }
 
 			   crm_debug_5("Adding %s (%dc) at offset %d",
 				       rsc_id, len-2, existing_len);
 			   crm_realloc(*rsc_list, len + existing_len);
 			   sprintf(*rsc_list + existing_len, "%s ", rsc_id);
 		   }
 
 		   if(entry->node != NULL) {
 		       uname = entry->node->details->uname;
 		   }
 		   
 		   if(node_list != NULL && uname) {
 			   int existing_len = 0;
 			   int len = 2 + strlen(uname);
 			   if(node_list && *node_list) {
 				   existing_len = strlen(*node_list);
 			   }
 			   
 			   crm_debug_5("Adding %s (%dc) at offset %d",
 				       uname, len-2, existing_len);
 			   crm_realloc(*node_list, len + existing_len);
 			   sprintf(*node_list + existing_len, "%s ", uname);
 		   }
 		   );
 }
 
 static void dup_attr(gpointer key, gpointer value, gpointer user_data)
 {
     g_hash_table_replace(user_data, crm_strdup(key), crm_strdup(value));
 }
 
 static action_t *
 pe_notify(resource_t *rsc, node_t *node, action_t *op, action_t *confirm,
 	  notify_data_t *n_data, pe_working_set_t *data_set)
 {
 	char *key = NULL;
 	action_t *trigger = NULL;
 	const char *value = NULL;
 	const char *task = NULL;
 	
 	if(op == NULL || confirm == NULL) {
 		crm_debug_2("Op=%p confirm=%p", op, confirm);
 		return NULL;
 	}
 
 	CRM_CHECK(node != NULL, return NULL);
 
 	if(node->details->online == FALSE) {
 		crm_debug_2("Skipping notification for %s: node offline", rsc->id);
 		return NULL;
 	} else if(op->runnable == FALSE) {
 		crm_debug_2("Skipping notification for %s: not runnable", op->uuid);
 		return NULL;
 	}
 	
 	value = g_hash_table_lookup(op->meta, "notify_type");
 	task = g_hash_table_lookup(op->meta, "notify_operation");
 
 	crm_debug_2("Creating notify actions for %s: %s (%s-%s)",
 		    op->uuid, rsc->id, value, task);
 	
 	key = generate_notify_key(rsc->id, value, task);
 	trigger = custom_action(rsc, key, op->task, node,
 				op->optional, TRUE, data_set);
 	g_hash_table_foreach(op->meta, dup_attr, trigger->meta);
 	g_hash_table_foreach(n_data->keys, dup_attr, trigger->meta);
 			
 	/* pseudo_notify before notify */
 	crm_debug_3("Ordering %s before %s (%d->%d)",
 		op->uuid, trigger->uuid, trigger->id, op->id);
 
 	order_actions(op, trigger, pe_order_implies_left);
 	order_actions(trigger, confirm, pe_order_implies_left);
 	return trigger;
 }
 
 static void
 pe_post_notify(resource_t *rsc, node_t *node, notify_data_t *n_data, pe_working_set_t *data_set)
 {
 	action_t *notify = NULL;
 
 	CRM_CHECK(rsc != NULL, return);
 
 	if(n_data->post == NULL) {
 	    return; /* Nothing to do */
 	}
 	
 	notify = pe_notify(rsc, node, n_data->post, n_data->post_done, n_data, data_set);
 
 	if(notify != NULL) {
 		notify->priority = INFINITY;
 	}
 
 	if(n_data->post_done) {
 		slist_iter(
 			mon, action_t, rsc->actions, lpc,
 
 			const char *interval = g_hash_table_lookup(mon->meta, "interval");
 			if(interval == NULL || safe_str_eq(interval, "0")) {
 				crm_debug_3("Skipping %s: interval", mon->uuid); 
 				continue;
 			} else if(safe_str_eq(mon->task, "cancel")) {
 				crm_debug_3("Skipping %s: cancel", mon->uuid); 
 				continue;
 			}
 
 			order_actions(n_data->post_done, mon, pe_order_optional);
 			);
 	}
 }
 
 notify_data_t *
 create_notification_boundaries(
     resource_t *rsc, const char *action, action_t *start, action_t *end, pe_working_set_t *data_set)
 {
     /* Create the pseudo ops that preceed and follow the actual notifications */
 
     /*
      * Creates two sequences (conditional on start and end being supplied):
      *   pre_notify -> pre_notify_complete -> start, and
      *   end -> post_notify -> post_notify_complete
      *
      * 'start' and 'end' may be the same event or ${X} and ${X}ed as per clones
      */
     char *key = NULL;
     notify_data_t *n_data = NULL;
 	
     if(is_not_set(rsc->flags, pe_rsc_notify)) {
 	return NULL;
     }
 
     crm_malloc0(n_data, sizeof(notify_data_t));
     n_data->action = action;
     n_data->keys = g_hash_table_new_full(
 	g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);
     
     if(start) {
 	/* create pre-event notification wrappers */
 	key = generate_notify_key(rsc->id, "pre", start->task);
 	n_data->pre = custom_action(
 	    rsc, key, RSC_NOTIFY, NULL, start->optional, TRUE, data_set);
 	
 	n_data->pre->pseudo = TRUE;
 	n_data->pre->runnable = TRUE;
 	add_hash_param(n_data->pre->meta, "notify_type", "pre");
 	add_hash_param(n_data->pre->meta, "notify_operation", n_data->action);
 
 	/* create pre_notify_complete */
 	key = generate_notify_key(rsc->id, "confirmed-pre", start->task);
 	n_data->pre_done = custom_action(
 	    rsc, key, RSC_NOTIFIED, NULL, start->optional, TRUE, data_set);
 
 	n_data->pre_done->pseudo = TRUE;
 	n_data->pre_done->runnable = TRUE;
 	add_hash_param(n_data->pre_done->meta, "notify_type", "pre");
 	add_hash_param(n_data->pre_done->meta, "notify_operation", n_data->action);
 
 	order_actions(n_data->pre_done, start, pe_order_optional);
 	order_actions(n_data->pre, n_data->pre_done, pe_order_optional);
     }
 
     if(end) {
 	/* create post-event notification wrappers */
 	key = generate_notify_key(rsc->id, "post", end->task);
 	n_data->post = custom_action(
 	    rsc, key, RSC_NOTIFY, NULL, end->optional, TRUE, data_set);
 
 	n_data->post->pseudo = TRUE;
 	n_data->post->runnable = TRUE;
 	n_data->post->priority = INFINITY;
 	n_data->post->runnable = end->runnable;
     
 	add_hash_param(n_data->post->meta, "notify_type", "post");
 	add_hash_param(n_data->post->meta, "notify_operation", n_data->action);
 	
 	/* create post_notify_complete */
 	key = generate_notify_key(rsc->id, "confirmed-post", end->task);
 	n_data->post_done = custom_action(
 	    rsc, key, RSC_NOTIFIED, NULL, end->optional, TRUE, data_set);
 
 	n_data->post_done->pseudo = TRUE;
 	n_data->post_done->runnable = TRUE;
 	n_data->post_done->priority = INFINITY;
 	n_data->post_done->runnable = end->runnable;
 
 	add_hash_param(n_data->post_done->meta, "notify_type", "pre");
 	add_hash_param(n_data->post_done->meta, "notify_operation", n_data->action);
 	
 	order_actions(end, n_data->post, pe_order_implies_right);
 	order_actions(n_data->post, n_data->post_done, pe_order_implies_right);
     }
 
     if(start && end) {
 	order_actions(n_data->pre_done, n_data->post, pe_order_optional);
     }
 
     if(safe_str_eq(action, RSC_STOP)) {
 	action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);
 	order_actions(n_data->post_done, all_stopped, pe_order_optional);	
     }    
 
     return n_data;
 }
 
 void
 collect_notification_data(resource_t *rsc, gboolean state, gboolean activity, notify_data_t *n_data)
 {
 
     if(rsc->children) {
 	slist_iter(child, resource_t, rsc->children, lpc,
 		   collect_notification_data(child, state, activity, n_data);
 	    );
 	return;
     }
     
     if(state) {
 	notify_entry_t *entry = NULL;
 
 	crm_malloc0(entry, sizeof(notify_entry_t));
 	entry->rsc = rsc;
 	if(rsc->running_on) {
 	    /* we only take the first one */
 	    entry->node = rsc->running_on->data;	    
 	}
 	
 	crm_debug_2("%s state: %s", rsc->id, role2text(rsc->role));
 
 	switch(rsc->role) {
 	    case RSC_ROLE_STOPPED:
 		n_data->inactive = g_list_append(n_data->inactive, entry);
 		break;
 	    case RSC_ROLE_STARTED:
 		n_data->active = g_list_append(n_data->active, entry);
 		break;
 	    case RSC_ROLE_SLAVE:
 		n_data->slave = g_list_append(n_data->slave, entry); 
 		break;
 	    case RSC_ROLE_MASTER:
 		n_data->master = g_list_append(n_data->master, entry);
 		break;
 	    default:
 		crm_err("Unsupported notify role");
 		crm_free(entry);
 		break;
 	}
     }
     
     if(activity) {
 	notify_entry_t *entry = NULL;
 	enum action_tasks task;
 	
 	slist_iter(
 	    op, action_t, rsc->actions, lpc,
 
 	    if(op->optional == FALSE && op->node != NULL) {
 		
 		crm_malloc0(entry, sizeof(notify_entry_t));
 		entry->node = op->node;
 		entry->rsc = rsc;
 
 		task = text2task(op->task);
 		switch(task) {
 		    case start_rsc:
 			n_data->start = g_list_append(n_data->start, entry);
 			break;
 		    case stop_rsc:
 			n_data->stop = g_list_append(n_data->stop, entry);
 			break;
 		    case action_promote:
 			n_data->promote = g_list_append(n_data->promote, entry);
 			break;
 		    case action_demote:
 			n_data->demote = g_list_append(n_data->demote, entry);
 			break;
 		    default:
 			crm_free(entry);
 			break;
 		}	
 	    }	
 	    );
     }
 }
 
 gboolean 
 expand_notification_data(notify_data_t *n_data)
 {
     /* Expand the notification entries into a key=value hashtable
      * This hashtable is later used in action2xml()
      */
     gboolean required = FALSE;
     char *rsc_list = NULL;
     char *node_list = NULL;
     
     if(n_data->stop) {
 	n_data->stop = g_list_sort(n_data->stop, sort_notify_entries);
     }
     expand_list(n_data->stop, &rsc_list, &node_list);
     if(rsc_list != NULL && safe_str_neq(" ", rsc_list)) {
 	if(safe_str_eq(n_data->action, RSC_STOP)) {
 	    required = TRUE;
 	}
     }
     g_hash_table_insert(n_data->keys, crm_strdup("notify_stop_resource"), rsc_list);
     g_hash_table_insert(n_data->keys, crm_strdup("notify_stop_uname"), node_list);
 	    
     if(n_data->start) {
 	n_data->start = g_list_sort(n_data->start, sort_notify_entries);
 	if(rsc_list && safe_str_eq(n_data->action, RSC_START)) {
 	    required = TRUE;
 	}
     }
     expand_list(n_data->start, &rsc_list, &node_list);
     g_hash_table_insert(n_data->keys, crm_strdup("notify_start_resource"), rsc_list);
     g_hash_table_insert(n_data->keys, crm_strdup("notify_start_uname"), node_list);
 	    
     if(n_data->demote) {
 	n_data->demote = g_list_sort(n_data->demote, sort_notify_entries);
 	if(safe_str_eq(n_data->action, RSC_DEMOTE)) {
 	    required = TRUE;
 	}
     }
 
     expand_list(n_data->demote, &rsc_list, &node_list);
     g_hash_table_insert(n_data->keys, crm_strdup("notify_demote_resource"), rsc_list);
     g_hash_table_insert(n_data->keys, crm_strdup("notify_demote_uname"), node_list);
 	    
     if(n_data->promote) {
 	n_data->promote = g_list_sort(n_data->promote, sort_notify_entries);
 	if(safe_str_eq(n_data->action, RSC_PROMOTE)) {
 	    required = TRUE;
 	}
     }
     expand_list(n_data->promote, &rsc_list, &node_list);
     g_hash_table_insert(n_data->keys, crm_strdup("notify_promote_resource"), rsc_list);
     g_hash_table_insert(n_data->keys, crm_strdup("notify_promote_uname"), node_list);
 	    
     if(n_data->active) {
 	n_data->active = g_list_sort(n_data->active, sort_notify_entries);
     }
     expand_list(n_data->active, &rsc_list, &node_list);
     g_hash_table_insert(n_data->keys, crm_strdup("notify_active_resource"), rsc_list);
     g_hash_table_insert(n_data->keys, crm_strdup("notify_active_uname"), node_list);
 	    
     if(n_data->slave) {
 	n_data->slave = g_list_sort(n_data->slave, sort_notify_entries);
     }
     expand_list(n_data->slave, &rsc_list, &node_list);
     g_hash_table_insert(n_data->keys, crm_strdup("notify_slave_resource"), rsc_list);
     g_hash_table_insert(n_data->keys, crm_strdup("notify_slave_uname"), node_list);
 	    
     if(n_data->master) {
 	n_data->master = g_list_sort(n_data->master, sort_notify_entries);
     }
     expand_list(n_data->master, &rsc_list, &node_list);
     g_hash_table_insert(n_data->keys, crm_strdup("notify_master_resource"), rsc_list);
     g_hash_table_insert(n_data->keys, crm_strdup("notify_master_uname"), node_list);
 	    
     if(n_data->inactive) {
 	n_data->inactive = g_list_sort(n_data->inactive, sort_notify_entries);
     }
     expand_list(n_data->inactive, &rsc_list, NULL);
     g_hash_table_insert(n_data->keys, crm_strdup("notify_inactive_resource"), rsc_list);
 
     if(required && n_data->pre) {
 	n_data->pre->optional = FALSE;
 	n_data->pre_done->optional = FALSE;
     }
     
     if(required && n_data->post) {
 	n_data->post->optional = FALSE;
 	n_data->post_done->optional = FALSE;
     }
     return required;
 }
 
 void
 create_notifications(resource_t *rsc, notify_data_t *n_data, pe_working_set_t *data_set)
 {
     action_t *stop = NULL;
     action_t *start = NULL;
     enum action_tasks task = text2task(n_data->action);
     
     if(rsc->children) {
 	slist_iter(
 	    child, resource_t, rsc->children, lpc,
 	    create_notifications(child, n_data, data_set);
 	    );
 	return;
     }
 
     /* Copy notification details into standard ops */
     slist_iter(
 	op, action_t, rsc->actions, lpc,
 	
 	if(op->optional == FALSE && op->node != NULL) {
 	    enum action_tasks t = text2task(op->task);
 	    switch(t) {
 		case start_rsc:
 		case stop_rsc:
 		case action_promote:
 		case action_demote:
 		    g_hash_table_foreach(n_data->keys, dup_attr, op->meta);
 		    break;
 		default:
 		    break;
 	    }
 	}
 	);
     
     crm_debug_2("Creating notificaitons for: %s.%s (%s->%s)",
 		n_data->action, rsc->id, role2text(rsc->role), role2text(rsc->next_role));
     
     stop = find_first_action(rsc->actions, NULL, RSC_STOP, NULL);
     start = find_first_action(rsc->actions, NULL, RSC_START, NULL);
 	
     /* stop / demote */
     if(rsc->role != RSC_ROLE_STOPPED) {
 	if(task == stop_rsc || task == action_demote) {
 	    slist_iter(current_node, node_t, rsc->running_on, lpc,
 		       pe_notify(rsc, current_node, n_data->pre, n_data->pre_done, n_data, data_set);
 		       if(task == action_demote || stop == NULL || stop->optional) {
 			   pe_post_notify(rsc, current_node, n_data, data_set);
 		       }
 		);
 	}
     }
 	
     /* start / promote */
     if(rsc->next_role != RSC_ROLE_STOPPED) {	
 	if(rsc->allocated_to == NULL) {
 	    pe_proc_err("Next role '%s' but %s is not allocated", role2text(rsc->next_role), rsc->id);
 			
 	} else if(task == start_rsc || task == action_promote) {
 	    if(task != start_rsc || start == NULL || start->optional) {
 		pe_notify(rsc, rsc->allocated_to, n_data->pre, n_data->pre_done, n_data, data_set);
 	    }
 	    pe_post_notify(rsc, rsc->allocated_to, n_data, data_set);
 	}
     }	
 }
 
 void free_notification_data(notify_data_t *n_data)
 {
     if(n_data == NULL) {
 	return;
     }
     
     pe_free_shallow(n_data->stop);
     pe_free_shallow(n_data->start);
     pe_free_shallow(n_data->demote);
     pe_free_shallow(n_data->promote);
     pe_free_shallow(n_data->master);
     pe_free_shallow(n_data->slave);
     pe_free_shallow(n_data->active);
     pe_free_shallow(n_data->inactive);
     g_hash_table_destroy(n_data->keys);
     crm_free(n_data);
 }
 
 int transition_id = -1;
 /*
  * Create a dependency graph to send to the transitioner (via the CRMd)
  */
 gboolean
 stage8(pe_working_set_t *data_set)
 {
 	const char *value = NULL;
 
 	transition_id++;
 	crm_debug_2("Creating transition graph %d.", transition_id);
 	
 	data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);
 
 	value = pe_pref(data_set->config_hash, "cluster-delay");
 	crm_xml_add(data_set->graph, "cluster-delay", value);
 
 	value = pe_pref(data_set->config_hash, "stonith-timeout");
 	crm_xml_add(data_set->graph, "stonith-timeout", value);
 
 	crm_xml_add(data_set->graph, "failed-stop-offset", "INFINITY");
 
 	if(is_set(data_set->flags, pe_flag_start_failure_fatal)) {
 	    crm_xml_add(data_set->graph, "failed-start-offset", "INFINITY");
 	} else {
 	    crm_xml_add(data_set->graph, "failed-start-offset", "1");
 	}
 	
 	value = pe_pref(data_set->config_hash, "batch-limit");
 	crm_xml_add(data_set->graph, "batch-limit", value);
 
 	crm_xml_add_int(data_set->graph, "transition_id", transition_id);
 	
 /* errors...
 	slist_iter(action, action_t, action_list, lpc,
 		   if(action->optional == FALSE && action->runnable == FALSE) {
 			   print_action("Ignoring", action, TRUE);
 		   }
 		);
 */
 	slist_iter(
 		rsc, resource_t, data_set->resources, lpc,
 
 		crm_debug_4("processing actions for rsc=%s", rsc->id);
 		rsc->cmds->expand(rsc, data_set);
 		);
 	crm_log_xml_debug_3(
 		data_set->graph, "created resource-driven action list");
 
 	/* catch any non-resource specific actions */
 	crm_debug_4("processing non-resource actions");
 	slist_iter(
 		action, action_t, data_set->actions, lpc,
 
 		graph_element_from_action(action, data_set);
 		);
 
 	crm_log_xml_debug_3(data_set->graph, "created generic action list");
 	crm_debug_2("Created transition graph %d.", transition_id);
 	
 	return TRUE;
 }
 
 void
 cleanup_alloc_calculations(pe_working_set_t *data_set)
 {
 	if(data_set == NULL) {
 		return;
 	}
 
 	crm_debug_3("deleting order cons: %p", data_set->ordering_constraints);
 	pe_free_ordering(data_set->ordering_constraints);
 	data_set->ordering_constraints = NULL;
 	
 	crm_debug_3("deleting node cons: %p", data_set->placement_constraints);
 	pe_free_rsc_to_node(data_set->placement_constraints);
 	data_set->placement_constraints = NULL;
 
 	crm_debug_3("deleting inter-resource cons: %p", data_set->colocation_constraints);
   	pe_free_shallow(data_set->colocation_constraints);
 	data_set->colocation_constraints = NULL;
 	
 	cleanup_calculations(data_set);
 }
diff --git a/pengine/group.c b/pengine/group.c
index c0bcd2aa82..bb2fd3e687 100644
--- a/pengine/group.c
+++ b/pengine/group.c
@@ -1,529 +1,525 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <pengine.h>
 #include <lib/pengine/utils.h>
 #include <crm/msg_xml.h>
 
 #include <allocate.h>
 #include <utils.h>
 
 #define VARIANT_GROUP 1
 #include <lib/pengine/variant.h>
 
 node_t *
 group_color(resource_t *rsc, pe_working_set_t *data_set)
 {
 	node_t *node = NULL;
 	node_t *group_node = NULL;
 	group_variant_data_t *group_data = NULL;
 	get_group_variant_data(group_data, rsc);
 
 	if(is_not_set(rsc->flags, pe_rsc_provisional)) {
 		return rsc->allocated_to;
 	}
 	crm_debug_2("Processing %s", rsc->id);
 	if(is_set(rsc->flags, pe_rsc_allocating)) {
 		crm_debug("Dependency loop detected involving %s", rsc->id);
 		return NULL;
 	}
 	
 	if(group_data->first_child == NULL) {
 	    /* nothign to allocate */
 	    clear_bit(rsc->flags, pe_rsc_provisional);
 	    return NULL;
 	}
 	
 	set_bit(rsc->flags, pe_rsc_allocating);
 	rsc->role = group_data->first_child->role;
 	
 	group_data->first_child->rsc_cons = g_list_concat(
 		group_data->first_child->rsc_cons, rsc->rsc_cons);
 	rsc->rsc_cons = NULL;
 
 	group_data->first_child->rsc_cons_lhs = g_list_concat(
 		group_data->first_child->rsc_cons_lhs, rsc->rsc_cons_lhs);
 	rsc->rsc_cons_lhs = NULL;
 	
 	dump_node_scores(show_scores?0:scores_log_level, rsc, __PRETTY_FUNCTION__, rsc->allowed_nodes);
 	
 	slist_iter(
 		child_rsc, resource_t, rsc->children, lpc,
 
 		node = child_rsc->cmds->color(child_rsc, data_set);
 		if(group_node == NULL) {
 		    group_node = node;
 		}
 		);
 
 	rsc->next_role = group_data->first_child->next_role;	
 	clear_bit(rsc->flags, pe_rsc_allocating);
 	clear_bit(rsc->flags, pe_rsc_provisional);
 
 	if(group_data->colocated) {
 		return group_node;
 	} 
 	return NULL;
 }
 
 void group_update_pseudo_status(resource_t *parent, resource_t *child);
 
 void group_create_actions(resource_t *rsc, pe_working_set_t *data_set)
 {
 	action_t *op = NULL;
 	const char *value = NULL;
 	group_variant_data_t *group_data = NULL;
 	get_group_variant_data(group_data, rsc);
 
 	crm_debug_2("Creating actions for %s", rsc->id);
 	
 	slist_iter(
 		child_rsc, resource_t, rsc->children, lpc,
 		child_rsc->cmds->create_actions(child_rsc, data_set);
 		group_update_pseudo_status(rsc, child_rsc);
 		);
 
 	op = start_action(rsc, NULL, TRUE/* !group_data->child_starting */);
 	op->pseudo = TRUE;
 	op->runnable = TRUE;
 
 	op = custom_action(rsc, started_key(rsc),
 			   RSC_STARTED, NULL,
 			   TRUE/* !group_data->child_starting */, TRUE, data_set);
 	op->pseudo = TRUE;
 	op->runnable = TRUE;
 
 	op = stop_action(rsc, NULL, TRUE/* !group_data->child_stopping */);
 	op->pseudo = TRUE;
 	op->runnable = TRUE;
 	
 	op = custom_action(rsc, stopped_key(rsc),
 			   RSC_STOPPED, NULL,
 			   TRUE/* !group_data->child_stopping */, TRUE, data_set);
 	op->pseudo = TRUE;
 	op->runnable = TRUE;
 
 	value = g_hash_table_lookup(rsc->meta, "stateful");
 	if(crm_is_true(value)) {
 	    op = custom_action(rsc, demote_key(rsc), RSC_DEMOTE, NULL, TRUE, TRUE, data_set);
 	    op->pseudo = TRUE; op->runnable = TRUE;
 	    op = custom_action(rsc, demoted_key(rsc), RSC_DEMOTED, NULL, TRUE, TRUE, data_set);
 	    op->pseudo = TRUE; op->runnable = TRUE;
 
 	    op = custom_action(rsc, promote_key(rsc), RSC_PROMOTE, NULL, TRUE, TRUE, data_set);
 	    op->pseudo = TRUE; op->runnable = TRUE;
 	    op = custom_action(rsc, promoted_key(rsc), RSC_PROMOTED, NULL, TRUE, TRUE, data_set);
 	    op->pseudo = TRUE; op->runnable = TRUE;
 	}
-
-	
-	rsc->actions = rsc->actions;
-/* 	rsc->actions = NULL; */
 }
 
 void
 group_update_pseudo_status(resource_t *parent, resource_t *child) 
 {
 	group_variant_data_t *group_data = NULL;
 	get_group_variant_data(group_data, parent);
 
 	if(group_data->ordered == FALSE) {
 	    /* If this group is not ordered, then leave the meta-actions as optional */ 
 	    return;
 	}
 	
 	if(group_data->child_stopping && group_data->child_starting) {
 		return;
 	}
 	slist_iter(
 		action, action_t, child->actions, lpc,
 
 		if(action->optional) {
 			continue;
 		}
 		if(safe_str_eq(RSC_STOP, action->task) && action->runnable) {
 			group_data->child_stopping = TRUE;
 			crm_debug_3("Based on %s the group is stopping", action->uuid);
 
 		} else if(safe_str_eq(RSC_START, action->task) && action->runnable) {
 			group_data->child_starting = TRUE;
 			crm_debug_3("Based on %s the group is starting", action->uuid);
 		}
 		
 		);
 }
 
 void group_internal_constraints(resource_t *rsc, pe_working_set_t *data_set)
 {
 	const char *value = NULL;
 	gboolean stateful = FALSE;
 	resource_t *last_rsc = NULL;
 	group_variant_data_t *group_data = NULL;
 	get_group_variant_data(group_data, rsc);
 
 	native_internal_constraints(rsc, data_set);
 
 	value = g_hash_table_lookup(rsc->meta, "stateful");
 	stateful = crm_is_true(value);
 	
 	new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_START,
 		      pe_order_optional, data_set);
 
 	new_rsc_order(rsc, RSC_STOP, rsc, RSC_STOPPED, 
 		      pe_order_runnable_left|pe_order_implies_right|pe_order_implies_left, data_set);
 
 	new_rsc_order(rsc, RSC_START, rsc, RSC_STARTED,
 		      pe_order_runnable_left, data_set);
 	
 	slist_iter(
 		child_rsc, resource_t, rsc->children, lpc,
 		int stop = pe_order_shutdown|pe_order_implies_right;
 		int stopped = pe_order_implies_right_printed;
 		int start = pe_order_implies_right|pe_order_runnable_left;
 		int started = pe_order_runnable_left|pe_order_implies_right|pe_order_implies_right_printed;
 		
 		child_rsc->cmds->internal_constraints(child_rsc, data_set);
 
 		if(last_rsc == NULL) {
 		    if(group_data->ordered) {
 			stop |= pe_order_implies_left;
 			stopped = pe_order_implies_right;
 		    }
 		    
 		} else if(group_data->colocated) {
 			rsc_colocation_new(
 				"group:internal_colocation", NULL, INFINITY,
 				child_rsc, last_rsc, NULL, NULL, data_set);
 		}
 
 		if(stateful) {
 		    new_rsc_order(rsc, RSC_DEMOTE, child_rsc, RSC_DEMOTE,
 				  stop|pe_order_implies_left_printed, data_set);
 
 		    new_rsc_order(child_rsc, RSC_DEMOTE, rsc, RSC_DEMOTED, stopped, data_set);
 
 		    new_rsc_order(child_rsc, RSC_PROMOTE, rsc, RSC_PROMOTED, started, data_set);
 
 		    new_rsc_order(rsc, RSC_PROMOTE, child_rsc, RSC_PROMOTE,
 				  pe_order_implies_left_printed, data_set);
 
 		}
 		
 		order_start_start(rsc, child_rsc, pe_order_implies_left_printed);
 		order_stop_stop(rsc, child_rsc, stop|pe_order_implies_left_printed);
 		
 		new_rsc_order(child_rsc, RSC_STOP, rsc, RSC_STOPPED, stopped, data_set);
 
 		new_rsc_order(child_rsc, RSC_START, rsc, RSC_STARTED, started, data_set);
 		
  		if(group_data->ordered == FALSE) {
 			order_start_start(rsc, child_rsc, start|pe_order_implies_left_printed);
 			if(stateful) {
 			    new_rsc_order(rsc, RSC_PROMOTE, child_rsc, RSC_PROMOTE,
 					  start|pe_order_implies_left_printed, data_set);
 			}
 
 		} else if(last_rsc != NULL) {
 			child_rsc->restart_type = pe_restart_restart;
 
 			order_start_start(last_rsc, child_rsc, start);
 			order_stop_stop(child_rsc, last_rsc, pe_order_implies_left);
 
 			if(stateful) {
 			    new_rsc_order(last_rsc, RSC_PROMOTE, child_rsc, RSC_PROMOTE, start, data_set);
 			    new_rsc_order(child_rsc, RSC_DEMOTE, last_rsc, RSC_DEMOTE, pe_order_implies_left, data_set);
 			}
 
 		} else {
 			/* If anyone in the group is starting, then
 			 *  pe_order_implies_right will cause _everyone_ in the group
 			 *  to be sent a start action
 			 * But this is safe since starting something that is already
 			 *  started is required to be "safe"
 			 */
 			int flags = pe_order_implies_left|pe_order_implies_right|pe_order_runnable_right|pe_order_runnable_left;
 		    
 			order_start_start(rsc, child_rsc, flags);
 			if(stateful) {
 			    new_rsc_order(rsc, RSC_PROMOTE, child_rsc, RSC_PROMOTE, flags, data_set);
 			}
 			
 		}
 		
 		last_rsc = child_rsc;
 		);
 
 	if(group_data->ordered && last_rsc != NULL) {
 		int stop_stop_flags = pe_order_implies_right;
 		int stop_stopped_flags = pe_order_implies_left;
 	    
 		order_stop_stop(rsc, last_rsc, stop_stop_flags);
 		new_rsc_order(last_rsc, RSC_STOP, rsc,  RSC_STOPPED, stop_stopped_flags, data_set);
 
 		if(stateful) {
 		    new_rsc_order(rsc, RSC_DEMOTE, last_rsc, RSC_DEMOTE, stop_stop_flags, data_set);
 		    new_rsc_order(last_rsc, RSC_DEMOTE, rsc, RSC_DEMOTED, stop_stopped_flags, data_set);
 		}
 	}
 }
 
 
 void group_rsc_colocation_lh(
 	resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint)
 {
 	group_variant_data_t *group_data = NULL;
 	
 	if(rsc_lh == NULL) {
 		pe_err("rsc_lh was NULL for %s", constraint->id);
 		return;
 
 	} else if(rsc_rh == NULL) {
 		pe_err("rsc_rh was NULL for %s", constraint->id);
 		return;
 	}
 		
 	crm_debug_4("Processing constraints from %s", rsc_lh->id);
 
 	get_group_variant_data(group_data, rsc_lh);
 
 	if(group_data->colocated) {
 		group_data->first_child->cmds->rsc_colocation_lh(
 			group_data->first_child, rsc_rh, constraint); 
 		return;
 
 	} else if(constraint->score >= INFINITY) {
 		crm_config_err("%s: Cannot perform manditory colocation"
 			       " between non-colocated group and %s",
 			       rsc_lh->id, rsc_rh->id);
 		return;
 	} 
 
 	slist_iter(
 		child_rsc, resource_t, rsc_lh->children, lpc,
 		child_rsc->cmds->rsc_colocation_lh(
 			child_rsc, rsc_rh, constraint); 
 		);
 }
 
 void group_rsc_colocation_rh(
 	resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint)
 {
 	group_variant_data_t *group_data = NULL;
 	get_group_variant_data(group_data, rsc_rh);
 	CRM_CHECK(rsc_lh->variant == pe_native, return);
 
 	crm_debug_3("Processing RH of constraint %s", constraint->id);
 	print_resource(LOG_DEBUG_3, "LHS", rsc_lh, TRUE);
 
 	if(is_set(rsc_rh->flags, pe_rsc_provisional)) {
 		return;
 	
 	} else if(group_data->colocated && group_data->first_child) {
 		group_data->first_child->cmds->rsc_colocation_rh(
 			rsc_lh, group_data->first_child, constraint); 
 		return;
 
 	} else if(constraint->score >= INFINITY) {
 		crm_config_err("%s: Cannot perform manditory colocation with"
 			       " non-colocated group: %s", rsc_lh->id, rsc_rh->id);
 		return;
 	} 
 
 	slist_iter(
 		child_rsc, resource_t, rsc_rh->children, lpc,
 		child_rsc->cmds->rsc_colocation_rh(
 			rsc_lh, child_rsc, constraint); 
 		);
 }
 
 void group_rsc_order_lh(resource_t *rsc, order_constraint_t *order, pe_working_set_t *data_set)
 {
 	group_variant_data_t *group_data = NULL;
 	get_group_variant_data(group_data, rsc);
 
 	crm_debug_4("%s->%s", order->lh_action_task, order->rh_action_task);
 
 	if(order->rh_rsc != NULL
 	   && (rsc == order->rh_rsc || rsc == order->rh_rsc->parent)) {
 		native_rsc_order_lh(rsc, order, data_set);
 		return;
 	}
 #if 0
 	if(order->type != pe_order_optional) {
 		native_rsc_order_lh(rsc, order, data_set);
 	}
 
 	if(order->type & pe_order_implies_left) {
  		native_rsc_order_lh(group_data->first_child, order, data_set);
 	}
 #endif
 
 	order->lh_action_task = convert_non_atomic_task(order->lh_action_task, rsc, TRUE, TRUE);
 	native_rsc_order_lh(rsc, order, data_set);
 }
 
 void group_rsc_order_rh(
 	action_t *lh_action, resource_t *rsc, order_constraint_t *order)
 {
 	enum pe_ordering type = order->type;
 	group_variant_data_t *group_data = NULL;
 	get_group_variant_data(group_data, rsc);
 
 	crm_debug_3("%s/%p: %s->%s", rsc->id, order, lh_action->uuid, order->rh_action_task);
 
 	if(rsc == NULL) {
 		return;
 	}
 
 	if(safe_str_eq(CRM_OP_PROBED, lh_action->uuid)) {
 	    slist_iter(
 		child_rsc, resource_t, rsc->children, lpc,
 		child_rsc->cmds->rsc_order_rh(lh_action, child_rsc, order);
 		);
 
 	    if(rsc->fns->state(rsc, TRUE) < RSC_ROLE_STARTED
 		&& rsc->fns->state(rsc, FALSE) > RSC_ROLE_STOPPED) {
 		order->type |= pe_order_implies_right;
 	    }
 	    
 	} else if(lh_action->rsc != NULL
 	   && lh_action->rsc != rsc
 	   && lh_action->rsc != rsc->parent
 	   && lh_action->rsc->parent != rsc) {
 	    char *tmp = NULL;
 	    char *task_s = NULL;
 	    int interval = 0;
 	    enum action_tasks task = 0;
 	    
 	    parse_op_key(order->lh_action_task, &tmp, &task_s, &interval);
 	    task = text2task(task_s);
 	    crm_free(task_s);
 	    crm_free(tmp);
 	    
 	    switch(task) {
 		case no_action:
 		case monitor_rsc:
 		case action_notify:
 		case action_notified:
 		case shutdown_crm:
 		case stonith_node:
 		    break;
 		case stop_rsc:
 		case stopped_rsc:
 		case action_demote:
 		case action_demoted:
 		    order->type |= pe_order_complex_left;
 		    break;
 		case start_rsc:
 		case started_rsc:
 		case action_promote:
 		case action_promoted:
 		    order->type |= pe_order_complex_right;
 		    break;
 	    }
 	    if(group_data->ordered == FALSE) {
 		/* Do for all children */
 		slist_iter(
 		    child_rsc, resource_t, rsc->children, lpc,
 		    child_rsc->cmds->rsc_order_rh(lh_action, child_rsc, order);
 		    );
 	    }
 	}
 	
 	native_rsc_order_rh(lh_action, rsc, order);
 	order->type = type;
 }
 
 void group_rsc_location(resource_t *rsc, rsc_to_node_t *constraint)
 {
 	GListPtr saved = constraint->node_list_rh;
 	GListPtr zero = node_list_dup(constraint->node_list_rh, TRUE, FALSE);
 	gboolean reset_scores = TRUE;
 	group_variant_data_t *group_data = NULL;
 	get_group_variant_data(group_data, rsc);
 
 	crm_debug("Processing rsc_location %s for %s",
 		  constraint->id, rsc->id);
 
 	native_rsc_location(rsc, constraint);
 	slist_iter(
 		child_rsc, resource_t, rsc->children, lpc,
 		child_rsc->cmds->rsc_location(child_rsc, constraint);
 		if(group_data->colocated && reset_scores) {
 			reset_scores = FALSE;
 			constraint->node_list_rh = zero;
 		}
 		);
 
 	constraint->node_list_rh = saved;
 	pe_free_shallow_adv(zero, TRUE);
 }
 
 void group_expand(resource_t *rsc, pe_working_set_t *data_set)
 {
 	group_variant_data_t *group_data = NULL;
 	get_group_variant_data(group_data, rsc);
 
 	crm_debug_3("Processing actions from %s", rsc->id);
 
 	CRM_CHECK(rsc != NULL, return);
 	native_expand(rsc, data_set);
 	
 	slist_iter(
 		child_rsc, resource_t, rsc->children, lpc,
 
 		child_rsc->cmds->expand(child_rsc, data_set);
 		);
 
 }
 
 GListPtr
 group_merge_weights(
     resource_t *rsc, const char *rhs, GListPtr nodes, const char *attr, int factor, gboolean allow_rollback)
 {
     group_variant_data_t *group_data = NULL;
     get_group_variant_data(group_data, rsc);
     
     if(is_set(rsc->flags, pe_rsc_merging)) {
 	crm_info("Breaking dependency loop with %s at %s", rsc->id, rhs);
 	return nodes;
     }
 
     set_bit(rsc->flags, pe_rsc_merging);
 
     nodes = group_data->first_child->cmds->merge_weights(
 	group_data->first_child, rhs, nodes, attr, factor, allow_rollback);
     
     slist_iter(
 	constraint, rsc_colocation_t, rsc->rsc_cons_lhs, lpc,
 	
 	nodes = native_merge_weights(
 	    constraint->rsc_lh, rsc->id, nodes,
 	    constraint->node_attribute,
 	    constraint->score/INFINITY, allow_rollback);
 	);
 
     clear_bit(rsc->flags, pe_rsc_merging);
     return nodes;
 }
 
 void group_append_meta(resource_t *rsc, xmlNode *xml)
 {
 }
diff --git a/pengine/master.c b/pengine/master.c
index 86717b35ae..2ada551c57 100644
--- a/pengine/master.c
+++ b/pengine/master.c
@@ -1,898 +1,898 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <crm/msg_xml.h>
 #include <allocate.h>
 #include <lib/pengine/utils.h>
 #include <utils.h>
 
 #define VARIANT_CLONE 1
 #include <lib/pengine/variant.h>
 
 extern gint sort_clone_instance(gconstpointer a, gconstpointer b, gpointer data_set);
 
 extern int master_score(resource_t *rsc, node_t *node, int not_set_value);
 
 static void
 child_promoting_constraints(
 	clone_variant_data_t *clone_data, enum pe_ordering type,
 	resource_t *rsc, resource_t *child, resource_t *last, pe_working_set_t *data_set)
 {
 	if(child == NULL) {
 		if(clone_data->ordered && last != NULL) {
 			crm_debug_4("Ordered version (last node)");
 			/* last child promote before promoted started */
 			new_rsc_order(last, RSC_PROMOTE, rsc, RSC_PROMOTED, type, data_set);
 		}
 		return;
 	}
 
 	/* child promote before global promoted */
 	new_rsc_order(child, RSC_PROMOTE, rsc, RSC_PROMOTED, type, data_set);
 	
 	/* global promote before child promote */
 	new_rsc_order(rsc, RSC_PROMOTE, child, RSC_PROMOTE, type, data_set);
 
 	if(clone_data->ordered) {
 		crm_debug_4("Ordered version");
 		if(last == NULL) {
 			/* global promote before first child promote */
 			last = rsc;
 
 		} /* else: child/child relative promote */
 
 		order_start_start(last, child, type);
 		new_rsc_order(last, RSC_PROMOTE, child, RSC_PROMOTE, type, data_set);
 
 	} else {
 		crm_debug_4("Un-ordered version");
 	}
 }
 
 static void
 child_demoting_constraints(
 	clone_variant_data_t *clone_data, enum pe_ordering type,
 	resource_t *rsc, resource_t *child, resource_t *last, pe_working_set_t *data_set)
 {
 	if(child == NULL) {
 		if(clone_data->ordered && last != NULL) {
 			crm_debug_4("Ordered version (last node)");
 			/* global demote before first child demote */
 			new_rsc_order(rsc, RSC_DEMOTE, last, RSC_DEMOTE,
 				      pe_order_implies_left, data_set);
 		}
 		return;
 	}
 	
 	/* child demote before global demoted */
 	new_rsc_order(child, RSC_DEMOTE, rsc, RSC_DEMOTED,
 		      pe_order_implies_right_printed, data_set);
 	
 	/* global demote before child demote */
 	new_rsc_order(rsc, RSC_DEMOTE, child, RSC_DEMOTE, 
 		      pe_order_implies_left_printed, data_set);
 	
 	if(clone_data->ordered && last != NULL) {
 		crm_debug_4("Ordered version");
 
 		/* child/child relative demote */
 		new_rsc_order(child, RSC_DEMOTE, last, RSC_DEMOTE, type, data_set);
 
 	} else if(clone_data->ordered) {
 		crm_debug_4("Ordered version (1st node)");
 		/* first child stop before global stopped */
 		new_rsc_order(child, RSC_DEMOTE, rsc, RSC_DEMOTED, type, data_set);
 
 	} else {
 		crm_debug_4("Un-ordered version");
 	}
 }
 
 static void
 master_update_pseudo_status(
 	resource_t *rsc, gboolean *demoting, gboolean *promoting) 
 {	
 	if(rsc->children) {
 	    slist_iter(child, resource_t, rsc->children, lpc,
 		       master_update_pseudo_status(child, demoting, promoting)
 		);
 	    return;
 	}
     
 	CRM_ASSERT(demoting != NULL);
 	CRM_ASSERT(promoting != NULL);
 
 	slist_iter(
 		action, action_t, rsc->actions, lpc,
 
 		if(*promoting && *demoting) {
 			return;
 
 		} else if(action->optional) {
 			continue;
 
 		} else if(safe_str_eq(RSC_DEMOTE, action->task)) {
 			*demoting = TRUE;
 
 		} else if(safe_str_eq(RSC_PROMOTE, action->task)) {
 			*promoting = TRUE;
 		}
 		);
 
 }
 
 #define apply_master_location(list)					\
 	slist_iter(							\
 		cons, rsc_to_node_t, list, lpc2,			\
 		cons_node = NULL;					\
 		if(cons->role_filter == RSC_ROLE_MASTER) {		\
 			crm_debug_2("Applying %s to %s",			\
 				  cons->id, child_rsc->id);		\
 			cons_node = pe_find_node_id(			\
 				cons->node_list_rh, chosen->details->id); \
 		}							\
 		if(cons_node != NULL) {					\
 			int new_priority = merge_weights(		\
 				child_rsc->priority, cons_node->weight); \
 			crm_debug_2("\t%s: %d->%d (%d)", child_rsc->id,	\
 				child_rsc->priority, new_priority, cons_node->weight);	\
 			child_rsc->priority = new_priority;		\
 		}							\
 		);
 
 static node_t *
 can_be_master(resource_t *rsc)
 {
 	node_t *node = NULL;
 	node_t *local_node = NULL;
 	resource_t *parent = uber_parent(rsc);
 	clone_variant_data_t *clone_data = NULL;
 	int level = LOG_DEBUG_2;
 #if 0
 	enum rsc_role_e role = RSC_ROLE_UNKNOWN;
 	role = rsc->fns->state(rsc, FALSE);
 	crm_info("%s role: %s", rsc->id, role2text(role));
 #endif
 	
 	if(rsc->children) {
 	    slist_iter(
 		child, resource_t, rsc->children, lpc,
 		if(can_be_master(child) == NULL) {
 		    do_crm_log_unlikely(level, "Child %s of %s can't be promoted", child->id, rsc->id);
 		    return NULL;
 		}
 		);
 	}
 
 	node = rsc->fns->location(rsc, NULL, FALSE);
 	if(node == NULL) {
 		do_crm_log_unlikely(level, "%s cannot be master: not allocated", rsc->id);
 		return NULL;
 	
 	} else if(is_not_set(rsc->flags, pe_rsc_managed)) {
 	    if(rsc->fns->state(rsc, TRUE) == RSC_ROLE_MASTER) {
 		crm_notice("Forcing unmanaged master %s to remain promoted on %s",
 			   rsc->id, node->details->uname);
 
 	    } else {
 		return NULL;
 	    }
 
 	} else if(rsc->priority < 0) {
 		do_crm_log_unlikely(level, "%s cannot be master: preference: %d",
 			   rsc->id, rsc->priority);
 		return NULL;
 
 	} else if(can_run_resources(node) == FALSE) {
 		do_crm_log_unlikely(level, "Node cant run any resources: %s",
 			    node->details->uname);
 		return NULL;
 	}
 	
 	get_clone_variant_data(clone_data, parent);
 	local_node = pe_find_node_id(
 		parent->allowed_nodes, node->details->id);
 
 	if(local_node == NULL) {
 		crm_err("%s cannot run on %s: node not allowed",
 			rsc->id, node->details->uname);
 		return NULL;
 
 	} else if(local_node->count < clone_data->master_node_max
 		  || is_not_set(rsc->flags, pe_rsc_managed)) {
 		return local_node;
 
 	} else {
 		do_crm_log_unlikely(level, "%s cannot be master on %s: node full",
 			    rsc->id, node->details->uname);
 	}
 
 	return NULL;
 }
 
 static gint sort_master_instance(gconstpointer a, gconstpointer b, gpointer data_set)
 {
 	int rc;
 	enum rsc_role_e role1 = RSC_ROLE_UNKNOWN;
 	enum rsc_role_e role2 = RSC_ROLE_UNKNOWN;
 
 	const resource_t *resource1 = (const resource_t*)a;
 	const resource_t *resource2 = (const resource_t*)b;
 
 	CRM_ASSERT(resource1 != NULL);
 	CRM_ASSERT(resource2 != NULL);
 
 	role1 = resource1->fns->state(resource1, TRUE);
 	role2 = resource2->fns->state(resource2, TRUE);
 	
 	rc = sort_rsc_index(a, b);
 	if( rc != 0 ) {
 		return rc;
 	}
 	
 	if(role1 > role2) {
 		return -1;
 
 	} else if(role1 < role2) {
 		return 1;
 	}
 	
 	return sort_clone_instance(a, b, data_set);
 }
 
 static void master_promotion_order(resource_t *rsc, pe_working_set_t *data_set) 
 {
     node_t *node = NULL;
     node_t *chosen = NULL;
     clone_variant_data_t *clone_data = NULL;
     get_clone_variant_data(clone_data, rsc);
 
     if(clone_data->merged_master_weights) {
 	return;
     }
     clone_data->merged_master_weights = TRUE;
     crm_debug_2("Merging weights for %s", rsc->id);
     slist_iter(
 	child, resource_t, rsc->children, lpc,
 	crm_debug_2("%s: %d", child->id, child->sort_index);
 	);
     dump_node_scores(LOG_DEBUG_3, rsc, "Before", rsc->allowed_nodes);
 
     slist_iter(
 	child, resource_t, rsc->children, lpc,
 
 	chosen = child->fns->location(child, NULL, FALSE);
 	if(chosen == NULL || child->sort_index < 0) {
 	    crm_debug_3("Skipping %s", child->id);
 	    continue;
 	}
 
 	node = (node_t*)pe_find_node_id(
 	    rsc->allowed_nodes, chosen->details->id);
 	CRM_ASSERT(node != NULL);
 	/* adds in master preferences and rsc_location.role=Master */
 	node->weight = merge_weights(child->sort_index, node->weight);
 	);
     
     dump_node_scores(LOG_DEBUG_3, rsc, "Middle", rsc->allowed_nodes);
 
     slist_iter(
 	constraint, rsc_colocation_t, rsc->rsc_cons, lpc,
 	/* (re-)adds location preferences of resources that the
 	 * master instance should/must be colocated with
 	 */
 	if(constraint->role_lh == RSC_ROLE_MASTER) {
 	    crm_debug_2("RHS: %s with %s: %d", constraint->rsc_lh->id, constraint->rsc_rh->id, constraint->score);
 	    rsc->allowed_nodes = constraint->rsc_rh->cmds->merge_weights(
 		constraint->rsc_rh, rsc->id, rsc->allowed_nodes,
 		constraint->node_attribute, constraint->score/INFINITY, constraint->score==INFINITY?FALSE:TRUE);
 	}
 	);
     
     slist_iter(
 	constraint, rsc_colocation_t, rsc->rsc_cons_lhs, lpc,
 	/* (re-)adds location preferences of resource that wish to be
 	 * colocated with the master instance
 	 */
 	if(constraint->role_rh == RSC_ROLE_MASTER) {
 	    crm_debug_2("LHS: %s with %s: %d", constraint->rsc_lh->id, constraint->rsc_rh->id, constraint->score);
 	    rsc->allowed_nodes = constraint->rsc_lh->cmds->merge_weights(
 		constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
 		constraint->node_attribute, constraint->score/INFINITY, TRUE);
 	}
 	);
 
     dump_node_scores(LOG_DEBUG_3, rsc, "After", rsc->allowed_nodes);
 
     /* write them back and sort */
     slist_iter(
 	child, resource_t, rsc->children, lpc,
 
 	chosen = child->fns->location(child, NULL, FALSE);
 
 	if(chosen == NULL || child->sort_index < 0) {
 	    crm_debug_2("%s: %d", child->id, child->sort_index);
 	    continue;
 	}
 
 	node = (node_t*)pe_find_node_id(
 	    rsc->allowed_nodes, chosen->details->id);
 	CRM_ASSERT(node != NULL);
 
 	child->sort_index = node->weight;
 	crm_debug_2("%s: %d", child->id, child->sort_index);
 	);
 
     rsc->children = g_list_sort_with_data(rsc->children, sort_master_instance, data_set);
 }
 
 int
 master_score(resource_t *rsc, node_t *node, int not_set_value)
 {
 	char *attr_name;
 	char *name = rsc->id;
 	const char *attr_value;
 	int score = not_set_value, len = 0;
 
 	if(rsc->children) {
 	    slist_iter(
 		child, resource_t, rsc->children, lpc,
 		int c_score = master_score(child, node, not_set_value);
 		if(score == not_set_value) {
 		    score = c_score;
 		} else {
 		    score += c_score;
 		}
 	    );
 	    return score;
 	}
 	
 	if(rsc->fns->state(rsc, TRUE) < RSC_ROLE_STARTED) {
 	    return score;
 	}
 
 	if(rsc->running_on) {
 	    node_t *match = pe_find_node_id(rsc->allowed_nodes, node->details->id);
-	    if(match->weight < 0) {
+	    if(match && match->weight < 0) {
 		crm_debug_2("%s on %s has score: %d - ignoring master pref",
 			    rsc->id, match->details->uname, match->weight);
 		return score;
 	    }
 	}
 	    
 	if(rsc->clone_name) {
 	    /* Use the name the lrm knows this resource as,
 	     * since that's what crm_master would have used too
 	     */
 	    name = rsc->clone_name;
 	}
 
 	len = 8 + strlen(name);
 	crm_malloc0(attr_name, len);
 	sprintf(attr_name, "master-%s", name);
 	
 	crm_debug_3("looking for %s on %s", attr_name,
 				node->details->uname);
 	attr_value = g_hash_table_lookup(
 		node->details->attrs, attr_name);
 	
 	if(attr_value == NULL) {
 		crm_free(attr_name);
 		len = 8 + strlen(rsc->long_name);
 		crm_malloc0(attr_name, len);
 		sprintf(attr_name, "master-%s", rsc->long_name);
 		crm_debug_3("looking for %s on %s", attr_name,
 					node->details->uname);
 		attr_value = g_hash_table_lookup(
 			node->details->attrs, attr_name);
 	}
 	
 	if(attr_value != NULL) {
 		crm_debug_2("%s[%s] = %s", attr_name,
 			    node->details->uname, crm_str(attr_value));
 		score = char2score(attr_value);
 	}
 
 	crm_free(attr_name);
 	return score;
 }
 
 #define max(a, b) a<b?b:a
 
 static void
 apply_master_prefs(resource_t *rsc) 
 {
     int score, new_score;
     clone_variant_data_t *clone_data = NULL;
     get_clone_variant_data(clone_data, rsc);
     
     if(clone_data->applied_master_prefs) {
 	/* Make sure we only do this once */
 	return;
     }
     
     clone_data->applied_master_prefs = TRUE;
     
     slist_iter(
 	child_rsc, resource_t, rsc->children, lpc,
 	slist_iter(
 	    node, node_t, child_rsc->allowed_nodes, lpc,
 
 	    if(can_run_resources(node) == FALSE) {
 		/* This node will never be promoted to master,
 		 *  so don't apply the master score as that may
 		 *  lead to clone shuffling
 		 */
 		continue;
 	    }
 	    
 	    score = master_score(child_rsc, node, 0);
 	    if(score > 0) {
 		new_score = merge_weights(node->weight, score);
 		if(new_score != node->weight) {
 		    crm_debug_2("\t%s: Updating preference for %s (%d->%d)",
 				child_rsc->id, node->details->uname, node->weight, new_score);
 		    node->weight = new_score;
 		}
 	    }
 	    
 	    new_score = max(child_rsc->priority, score);
 	    if(new_score != child_rsc->priority) {
 		crm_debug_2("\t%s: Updating priority (%d->%d)",
 			  child_rsc->id, child_rsc->priority, new_score);
 		child_rsc->priority = new_score;
 	    }
 	    );
 	);
 }
 
 static void set_role_slave(resource_t *rsc, gboolean current) 
 {
     if(current) {
 	if(rsc->role == RSC_ROLE_STARTED) {
 	    rsc->role = RSC_ROLE_SLAVE;
 	}
 	
     } else {
 	GListPtr allocated = NULL;
 	rsc->fns->location(rsc, &allocated, FALSE);
 	
 	if(allocated) {
 	    rsc->next_role = RSC_ROLE_SLAVE;
 	    
 	} else {
 	    rsc->next_role = RSC_ROLE_STOPPED;
 	}
 	g_list_free(allocated);	
     }
     
     slist_iter(
 	child_rsc, resource_t, rsc->children, lpc,
 	set_role_slave(child_rsc, current);
 	);
 }
 
 static void set_role_master(resource_t *rsc) 
 {
     if(rsc->next_role == RSC_ROLE_UNKNOWN) {
 	rsc->next_role = RSC_ROLE_MASTER;
     }
     
     slist_iter(
 	child_rsc, resource_t, rsc->children, lpc,
 	set_role_master(child_rsc);
 	);
 }
 
 node_t *
 master_color(resource_t *rsc, pe_working_set_t *data_set)
 {
 	int promoted = 0;
 	node_t *chosen = NULL;
 	node_t *cons_node = NULL;
 	enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;
 	
 	clone_variant_data_t *clone_data = NULL;
 	get_clone_variant_data(clone_data, rsc);
 
 	apply_master_prefs(rsc);
 
 	clone_color(rsc, data_set);
 	
 	/* count now tracks the number of masters allocated */
 	slist_iter(node, node_t, rsc->allowed_nodes, lpc,
 		   node->count = 0;
 		);
 
 	/*
 	 * assign priority
 	 */
 	slist_iter(
 		child_rsc, resource_t, rsc->children, lpc,
 
 		GListPtr list = NULL;
 		crm_debug_2("Assigning priority for %s: %s", child_rsc->id, role2text(child_rsc->next_role));
 
 		if(child_rsc->fns->state(child_rsc, TRUE) == RSC_ROLE_STARTED) {
 		    set_role_slave(child_rsc, TRUE);
 		}
 
 		chosen = child_rsc->fns->location(child_rsc, &list, FALSE);
 		if(g_list_length(list) > 1) {
 		    crm_config_err("Cannot promote non-colocated child %s", child_rsc->id);
 		}
 
 		g_list_free(list);
 		if(chosen == NULL) {
 			continue;
 		}
 		
 		next_role = child_rsc->fns->state(child_rsc, FALSE);
 		switch(next_role) {
 			case RSC_ROLE_STARTED:
 			case RSC_ROLE_UNKNOWN:
 				CRM_CHECK(chosen != NULL, break);
 				/*
 				 * Default to -1 if no value is set
 				 *
 				 * This allows master locations to be specified
 				 * based solely on rsc_location constraints,
 				 * but prevents anyone from being promoted if
 				 * neither a constraint nor a master-score is present
 				 */
 				child_rsc->priority = master_score(child_rsc, chosen, -1);
 				break;
 
 			case RSC_ROLE_SLAVE:
 			case RSC_ROLE_STOPPED:
 				child_rsc->priority = -INFINITY;
 				break;
 			case RSC_ROLE_MASTER:
 				/* We will arrive here if we're re-creating actions after a stonith
 				 *  OR target-role is set
 				 */
 				break;
 			default:
 				CRM_CHECK(FALSE/* unhandled */,
 					  crm_err("Unknown resource role: %d for %s",
 						  next_role, child_rsc->id));
 		}
 
 		apply_master_location(child_rsc->rsc_location);
 		apply_master_location(rsc->rsc_location);
 		slist_iter(
 		    cons, rsc_colocation_t, child_rsc->rsc_cons, lpc2,
 		    child_rsc->cmds->rsc_colocation_lh(child_rsc, cons->rsc_rh, cons);
 		    );
 		
 		child_rsc->sort_index = child_rsc->priority;
 		crm_debug_2("Assigning priority for %s: %d", child_rsc->id, child_rsc->priority);
 
 		if(next_role == RSC_ROLE_MASTER) {
 		    child_rsc->sort_index = INFINITY;
 		}
 
 	    );
 
 	master_promotion_order(rsc, data_set);
 
 	/* mark the first N as masters */
 	slist_iter(
 		child_rsc, resource_t, rsc->children, lpc,
 
 		char *score = score2char(child_rsc->sort_index);
 		chosen = child_rsc->fns->location(child_rsc, NULL, FALSE);
 		if(show_scores) {
 		    fprintf(stdout, "%s promotion score on %s: %s\n",
 			    child_rsc->id, chosen?chosen->details->uname:"none", score);
 		    
 		} else {
 		    do_crm_log_unlikely(scores_log_level, "%s promotion score on %s: %s",
 					child_rsc->id, chosen?chosen->details->uname:"none", score);
 		}
 		crm_free(score);
 
 		chosen = NULL; /* nuke 'chosen' so that we don't promote more than the
 				* required number of instances
 				*/
 		
 		if(child_rsc->sort_index < 0) {
 		    crm_debug_2("Not supposed to promote child: %s", child_rsc->id);
 		    
 		} else if(promoted < clone_data->master_max || is_not_set(rsc->flags, pe_rsc_managed)) {
 			chosen = can_be_master(child_rsc);
 		}
 
 		crm_debug("%s master score: %d", child_rsc->id, child_rsc->priority);
 		
 		if(chosen == NULL) {
 		    set_role_slave(child_rsc, FALSE);
 		    continue;
 		}
 
 		chosen->count++;
 		crm_info("Promoting %s (%s %s)",
 			 child_rsc->id, role2text(child_rsc->role), chosen->details->uname);
 		set_role_master(child_rsc);
 		promoted++;		
 		);
 	
 	clone_data->masters_allocated = promoted;
 	crm_info("%s: Promoted %d instances of a possible %d to master",
 		 rsc->id, promoted, clone_data->master_max);
 	return NULL;
 }
 
 void master_create_actions(resource_t *rsc, pe_working_set_t *data_set)
 {
 	action_t *action = NULL;
 	action_t *action_complete = NULL;
 	gboolean any_promoting = FALSE;
 	gboolean any_demoting = FALSE;
 	resource_t *last_promote_rsc = NULL;
 	resource_t *last_demote_rsc = NULL;
 
 	clone_variant_data_t *clone_data = NULL;
 	get_clone_variant_data(clone_data, rsc);
 	
 	crm_debug("Creating actions for %s", rsc->id);
 
 	/* create actions as normal */
 	clone_create_actions(rsc, data_set);
 
 	slist_iter(
 		child_rsc, resource_t, rsc->children, lpc,
 		gboolean child_promoting = FALSE;
 		gboolean child_demoting = FALSE;
 
 		crm_debug_2("Creating actions for %s", child_rsc->id);
 		child_rsc->cmds->create_actions(child_rsc, data_set);
 		master_update_pseudo_status(
 			child_rsc, &child_demoting, &child_promoting);
 
 		any_demoting = any_demoting || child_demoting;
 		any_promoting = any_promoting || child_promoting;
 		crm_debug_2("Created actions for %s: %d %d", child_rsc->id, child_promoting, child_demoting);
 		);
 	
 	/* promote */
 	action = promote_action(rsc, NULL, !any_promoting);
 	action_complete = custom_action(
 		rsc, promoted_key(rsc),
 		RSC_PROMOTED, NULL, !any_promoting, TRUE, data_set);
 
 	action->pseudo = TRUE;
 	action->runnable = FALSE;
 	action_complete->pseudo = TRUE;
 	action_complete->runnable = FALSE;
 	action_complete->priority = INFINITY;
 
 	if(clone_data->masters_allocated > 0) {
 	    action->runnable = TRUE;
 	    action_complete->runnable = TRUE;
 	}
 	
 	child_promoting_constraints(clone_data, pe_order_optional, 
 				    rsc, NULL, last_promote_rsc, data_set);
 
 	if(clone_data->promote_notify == NULL) {
 	    clone_data->promote_notify = create_notification_boundaries(
 		rsc, RSC_PROMOTE, action, action_complete, data_set);
 	}
 	
 	/* demote */
 	action = demote_action(rsc, NULL, !any_demoting);
 	action_complete = custom_action(
 		rsc, demoted_key(rsc),
 		RSC_DEMOTED, NULL, !any_demoting, TRUE, data_set);
 	action_complete->priority = INFINITY;
 
 	action->pseudo = TRUE;
 	action->runnable = TRUE;
 	action_complete->pseudo = TRUE;
 	action_complete->runnable = TRUE;
 	
 	child_demoting_constraints(clone_data, pe_order_optional,
 				   rsc, NULL, last_demote_rsc, data_set);
 
 	if(clone_data->demote_notify == NULL) {
 	    clone_data->demote_notify = create_notification_boundaries(
 		rsc, RSC_DEMOTE, action, action_complete, data_set);
 	
 	    if(clone_data->promote_notify) {
 		/* If we ever wanted groups to have notifications we'd need to move this to native_internal_constraints() one day
 		 * Requires exposing *_notify
 		 */
 		order_actions(clone_data->stop_notify->post_done,   clone_data->promote_notify->pre, pe_order_optional);
 		order_actions(clone_data->start_notify->post_done,  clone_data->promote_notify->pre, pe_order_optional);
 		order_actions(clone_data->demote_notify->post_done, clone_data->promote_notify->pre, pe_order_optional);
 		order_actions(clone_data->demote_notify->post_done, clone_data->start_notify->pre,   pe_order_optional);
 		order_actions(clone_data->demote_notify->post_done, clone_data->stop_notify->pre,    pe_order_optional);
 	    }
 	}
 
 	/* restore the correct priority */ 
 	slist_iter(
 		child_rsc, resource_t, rsc->children, lpc,
 		child_rsc->priority = rsc->priority;
 	    );
 }
 
 void
 master_internal_constraints(resource_t *rsc, pe_working_set_t *data_set)
 {
 	resource_t *last_rsc = NULL;	
 	clone_variant_data_t *clone_data = NULL;
 	get_clone_variant_data(clone_data, rsc);
 
 	clone_internal_constraints(rsc, data_set);
 	
 	/* global stopped before start */
 	new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_START, pe_order_optional, data_set);
 
 	/* global stopped before promote */
 	new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_PROMOTE, pe_order_optional, data_set);
 
 	/* global demoted before start */
 	new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_START, pe_order_optional, data_set);
 
 	/* global started before promote */
 	new_rsc_order(rsc, RSC_STARTED, rsc, RSC_PROMOTE, pe_order_optional, data_set);
 
 	/* global demoted before stop */
 	new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_STOP, pe_order_optional, data_set);
 
 	/* global demote before demoted */
 	new_rsc_order(rsc, RSC_DEMOTE, rsc, RSC_DEMOTED, pe_order_optional, data_set);
 
 	/* global demoted before promote */
 	new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_PROMOTE, pe_order_optional, data_set);
 
 	slist_iter(
 		child_rsc, resource_t, rsc->children, lpc,
 
 		/* child demote before promote */
 		new_rsc_order(child_rsc, RSC_DEMOTE, child_rsc, RSC_PROMOTE, pe_order_optional, data_set);
 		
 		child_promoting_constraints(clone_data, pe_order_optional,
 					    rsc, child_rsc, last_rsc, data_set);
 
 		child_demoting_constraints(clone_data, pe_order_optional,
 					   rsc, child_rsc, last_rsc, data_set);
 
 		last_rsc = child_rsc;
 		
 		);
 	
 }
 
 static void node_list_update_one(GListPtr list, node_t *other, const char *attr, int score)
 {
     const char *value = NULL;
     
     if(other == NULL) {
 	return;
 
     } else if(attr == NULL) {
 	attr = "#"XML_ATTR_UNAME;
     }
     
     value = g_hash_table_lookup(other->details->attrs, attr);
     slist_iter(node, node_t, list, lpc,
 	       const char *tmp = g_hash_table_lookup(node->details->attrs, attr);
 	       if(safe_str_eq(value, tmp)) {
 		   crm_debug_2("%s: %d + %d", node->details->uname, node->weight, other->weight);
 		   node->weight = merge_weights(node->weight, score);
 	       }
 	);
 }
 
 void master_rsc_colocation_rh(
     resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint)
 {
 	clone_variant_data_t *clone_data = NULL;
 	get_clone_variant_data(clone_data, rsc_rh);
 
 	
 	CRM_CHECK(rsc_rh != NULL, return);
 	if(is_set(rsc_rh->flags, pe_rsc_provisional)) {
 		return;
 
 	} else if(constraint->role_rh == RSC_ROLE_UNKNOWN) {
 		crm_debug_3("Handling %s as a clone colocation", constraint->id);
 		clone_rsc_colocation_rh(rsc_lh, rsc_rh, constraint);
 		return;
 	}
 	
 	CRM_CHECK(rsc_lh != NULL, return);
 	CRM_CHECK(rsc_lh->variant == pe_native, return);
 	crm_debug_2("Processing constraint %s: %d", constraint->id, constraint->score);
 
 	if(constraint->role_rh == RSC_ROLE_UNKNOWN) {
 		slist_iter(
 			child_rsc, resource_t, rsc_rh->children, lpc,
 			
 			child_rsc->cmds->rsc_colocation_rh(rsc_lh, child_rsc, constraint);
 			);
 
 	} else if(is_set(rsc_lh->flags, pe_rsc_provisional)) {
 		GListPtr lhs = NULL, rhs = NULL;
 		lhs = rsc_lh->allowed_nodes;
 		slist_iter(
 			child_rsc, resource_t, rsc_rh->children, lpc,
 			node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE);
 			enum rsc_role_e next_role = child_rsc->fns->state(child_rsc, FALSE);
 			crm_debug_3("Processing: %s", child_rsc->id);
 			if(chosen != NULL && next_role == constraint->role_rh) {
 			    crm_debug_3("Applying: %s %s %s %d", child_rsc->id,
 					role2text(next_role), chosen->details->uname, constraint->score);
 			    if(constraint->score < INFINITY) {
 				node_list_update_one(rsc_lh->allowed_nodes, chosen,
 						     constraint->node_attribute, constraint->score);
 			    }
 			    rhs = g_list_append(rhs, chosen);
 			}
 			);
 
 		/* Only do this if its not a master-master colocation
 		 * Doing this unconditionally would prevent the slaves from being started
 		 */
 		if(constraint->role_lh != RSC_ROLE_MASTER
 		   || constraint->role_rh != RSC_ROLE_MASTER) {
 		    if(constraint->score > 0) {
 			rsc_lh->allowed_nodes = node_list_exclude(lhs, rhs, TRUE);
 			pe_free_shallow(lhs);
 		    }
 		}
 		pe_free_shallow_adv(rhs, FALSE);
 
 	} else if(constraint->role_lh == RSC_ROLE_MASTER) {
 	    resource_t *rh_child = find_compatible_child(rsc_lh, rsc_rh, constraint->role_rh, FALSE);
 	    if(rh_child == NULL && constraint->score >= INFINITY) {
 		crm_debug_2("%s can't be promoted %s", rsc_lh->id, constraint->id);
 		rsc_lh->priority = -INFINITY;
 		
 	    } else if(rh_child != NULL) {
 		int new_priority = merge_weights(rsc_lh->priority, constraint->score);
 		crm_debug("Applying %s to %s", constraint->id, rsc_lh->id);
 		crm_debug("\t%s: %d->%d", rsc_lh->id, rsc_lh->priority, new_priority);
 		rsc_lh->priority = new_priority;
 	    }
 	}
 
 	return;
 }
 
 void master_append_meta(resource_t *rsc, xmlNode *xml)
 {
     char *name = NULL;
     clone_variant_data_t *clone_data = NULL;
     get_clone_variant_data(clone_data, rsc);
 
     clone_append_meta(rsc, xml);
 		
     name = crm_meta_name(XML_RSC_ATTR_MASTER_MAX);
     crm_xml_add_int(xml, name, clone_data->master_max);
     crm_free(name);
 
     name = crm_meta_name(XML_RSC_ATTR_MASTER_NODEMAX);
     crm_xml_add_int(xml, name, clone_data->master_node_max);
     crm_free(name);
 }
diff --git a/pengine/native.c b/pengine/native.c
index 19fca5a61b..a2d96320dd 100644
--- a/pengine/native.c
+++ b/pengine/native.c
@@ -1,2313 +1,2312 @@
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <pengine.h>
 #include <crm/pengine/rules.h>
 #include <lib/pengine/utils.h>
 #include <crm/msg_xml.h>
 #include <allocate.h>
 #include <utils.h>
 
 #define DELETE_THEN_REFRESH 1 /* The crmd will remove the resource from the CIB itself, making this redundant */
 
 #define VARIANT_NATIVE 1
 #include <lib/pengine/variant.h>
 
 void native_rsc_colocation_rh_must(resource_t *rsc_lh, gboolean update_lh,
 				   resource_t *rsc_rh, gboolean update_rh);
 
 void native_rsc_colocation_rh_mustnot(resource_t *rsc_lh, gboolean update_lh,
 				      resource_t *rsc_rh, gboolean update_rh);
 
 void Recurring(resource_t *rsc, action_t *start, node_t *node,
 			      pe_working_set_t *data_set);
 void RecurringOp(resource_t *rsc, action_t *start, node_t *node,
 		 xmlNode *operation, pe_working_set_t *data_set);
 void pe_post_notify(
 	resource_t *rsc, node_t *node, action_t *op, 
 	notify_data_t *n_data, pe_working_set_t *data_set);
 
 void NoRoleChange  (resource_t *rsc, node_t *current, node_t *next, pe_working_set_t *data_set);
 gboolean DeleteRsc (resource_t *rsc, node_t *node, gboolean optional, pe_working_set_t *data_set);
 gboolean StopRsc   (resource_t *rsc, node_t *next, gboolean optional, pe_working_set_t *data_set);
 gboolean StartRsc  (resource_t *rsc, node_t *next, gboolean optional, pe_working_set_t *data_set);
 gboolean DemoteRsc (resource_t *rsc, node_t *next, gboolean optional, pe_working_set_t *data_set);
 gboolean PromoteRsc(resource_t *rsc, node_t *next, gboolean optional, pe_working_set_t *data_set);
 gboolean RoleError (resource_t *rsc, node_t *next, gboolean optional, pe_working_set_t *data_set);
 gboolean NullOp    (resource_t *rsc, node_t *next, gboolean optional, pe_working_set_t *data_set);
 
 enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
 /* Current State */	
 /*    Next State:  Unknown 	    Stopped	      Started	        Slave	          Master */
 /* Unknown */	{ RSC_ROLE_UNKNOWN, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, },
 /* Stopped */	{ RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE,   RSC_ROLE_SLAVE, },
 /* Started */	{ RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE,   RSC_ROLE_MASTER, },
 /* Slave */	{ RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_UNKNOWN, RSC_ROLE_SLAVE,   RSC_ROLE_MASTER, },
 /* Master */	{ RSC_ROLE_STOPPED, RSC_ROLE_SLAVE,   RSC_ROLE_UNKNOWN, RSC_ROLE_SLAVE,   RSC_ROLE_MASTER, },
 };
 
 gboolean (*rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX])(resource_t*,node_t*,gboolean,pe_working_set_t*) = {
 /* Current State */	
 /*    Next State: Unknown	Stopped		Started		Slave		Master */
 /* Unknown */	{ RoleError,	StopRsc,	RoleError,	RoleError,	RoleError,  },
 /* Stopped */	{ RoleError,	NullOp,		StartRsc,	StartRsc,	RoleError,  },
 /* Started */	{ RoleError,	StopRsc,	NullOp,		NullOp,		PromoteRsc, },
 /* Slave */	{ RoleError,	StopRsc,	RoleError,	NullOp,		PromoteRsc, },
 /* Master */	{ RoleError,	RoleError,	RoleError,	DemoteRsc,	NullOp,     },
 };
 
 struct capacity_data
 {
 	node_t *node;
 	resource_t *rsc;
 	gboolean is_enough;
 };
 
 static void
 check_capacity(gpointer key, gpointer value, gpointer user_data)
 {
 	int required = 0;
 	int remaining = 0;
 	struct capacity_data *data = user_data;
 
 	required = crm_parse_int(value, "0");
 	remaining = crm_parse_int(g_hash_table_lookup(data->node->details->utilization, key), "0");
 		
 	if (required > remaining) {
 		crm_debug("Node %s has no enough %s for resource %s: required=%d remaining=%d",
 			data->node->details->uname, (char *)key, data->rsc->id, required, remaining);
 		data->is_enough = FALSE;
 	}
 }
 
 static gboolean
 have_enough_capacity(node_t *node, resource_t *rsc)
 {
 	struct capacity_data data;
 
 	data.node = node;
 	data.rsc = rsc;
 	data.is_enough = TRUE;
 
 	g_hash_table_foreach(rsc->utilization, check_capacity, &data);
 
 	return data.is_enough;
 }
 
 static gboolean
 native_choose_node(resource_t *rsc, pe_working_set_t *data_set)
 {
 	/*
 	  1. Sort by weight
 	  2. color.chosen_node = the node (of those with the highest wieght)
 				   with the fewest resources
 	  3. remove color.chosen_node from all other colors
 	*/
 	int alloc_details = scores_log_level+1;
 
 	GListPtr nodes = NULL;
 	node_t *chosen = NULL;
 
 	int lpc = 0;
 	int multiple = 0;
 	int length = 0;
 
 	if (safe_str_neq(data_set->placement_strategy, "default")) {
 		slist_iter(
 			node, node_t, data_set->nodes, lpc,
 			if (have_enough_capacity(node, rsc) == FALSE) {
 				crm_debug("Resource %s cannot be allocated to node %s: none of enough capacity",
 					rsc->id, node->details->uname);
 				resource_location(rsc, node, -INFINITY, "__limit_utilization_", data_set);
 			}
 	    	    );
 		dump_node_scores(alloc_details, rsc, "Post-utilization", rsc->allowed_nodes);
 	}
 	
 	length = g_list_length(rsc->allowed_nodes);
 
 	if(is_not_set(rsc->flags, pe_rsc_provisional)) {
 		return rsc->allocated_to?TRUE:FALSE;
 	}
 	
 	crm_debug_3("Choosing node for %s from %d candidates",
 		    rsc->id, length);
 
 	if(rsc->allowed_nodes) {
 	    rsc->allowed_nodes = g_list_sort_with_data(rsc->allowed_nodes, sort_node_weight, data_set);
 	    nodes = rsc->allowed_nodes;
 	    chosen = g_list_nth_data(nodes, 0);
 
 	    if(chosen
 	       && chosen->weight > 0
 	       && can_run_resources(chosen)) {
 		node_t *running = g_list_nth_data(rsc->running_on, 0);
 		if(can_run_resources(running) == FALSE) {
 		    running = NULL;
 		}
 		
 		for(lpc = 1; lpc < length; lpc++) {
 		    node_t *tmp = g_list_nth_data(nodes, lpc);
 		    if(tmp->weight == chosen->weight) {
 			multiple++;
 			if(running && tmp->details == running->details) {
 			    /* prefer the existing node if scores are equal */
 			    chosen = tmp;
 			}
 		    }
 		}
 	    }
 	}
 
 	if(multiple > 1) {
 		int log_level = LOG_INFO;
 		char *score = score2char(chosen->weight);
 		if(chosen->weight >= INFINITY) {
 			log_level = LOG_WARNING;
 		}
 		
 		do_crm_log(log_level, "%d nodes with equal score (%s) for"
 			   " running %s resources.  Chose %s.",
 			   multiple, score, rsc->id, chosen->details->uname);
 		crm_free(score);
 	}
 	
 	
 	return native_assign_node(rsc, nodes, chosen, FALSE);
 }
 
 int node_list_attr_score(GListPtr list, const char *attr, const char *value) 
 {
     int best_score = -INFINITY;
     const char *best_node = NULL;
 
     if(attr == NULL) {
 	attr = "#"XML_ATTR_UNAME;
     }
 
     slist_iter(node, node_t, list, lpc,
 	       int weight = node->weight;
 	       if(can_run_resources(node) == FALSE) {
 		   weight = -INFINITY;
 	       }
 	       if(weight > best_score || best_node == NULL) {
 		   const char *tmp = g_hash_table_lookup(node->details->attrs, attr);
 		   if(safe_str_eq(value, tmp)) {
 		       best_score = weight;
 		       best_node = node->details->uname;
 		   }
 	       }
 	);
 
     if(safe_str_neq(attr, "#"XML_ATTR_UNAME)) {
 	crm_info("Best score for %s=%s was %s with %d",
 		attr, value, best_node?best_node:"<none>", best_score);
     }
     
     return best_score;
 }
 
 
 static void
 node_list_update(GListPtr list1, GListPtr list2, const char *attr, int factor)
 {
     int score = 0;
     if(attr == NULL) {
 	attr = "#"XML_ATTR_UNAME;
     }
     
     slist_iter(
 	node, node_t, list1, lpc,
 	
 	CRM_CHECK(node != NULL, continue);
 	score = node_list_attr_score(list2, attr, g_hash_table_lookup(node->details->attrs, attr));
 	
 	if(factor < 0 && score < 0) {
 	    /* Negative preference for a node with a negative score
 	     * should not become a positive preference
 	     *
 	     * TODO: Decide if we want to filter only if weight == -INFINITY
 	     *
 	     */
 	    continue;
 	}
 	crm_debug_2("%s: %d + %d*%d",
 		    node->details->uname, node->weight, factor, score);
 	node->weight = merge_weights(factor*score, node->weight);
 	);
 }
 
 GListPtr
 native_merge_weights(
     resource_t *rsc, const char *rhs, GListPtr nodes, const char *attr, int factor, gboolean allow_rollback) 
 {
     GListPtr work = NULL;
     int multiplier = 1;
     if(factor < 0) {
 	multiplier = -1;
     }
 
     if(is_set(rsc->flags, pe_rsc_merging)) {
 	crm_info("%s: Breaking dependency loop at %s", rhs, rsc->id);
 	return nodes;
     }
 
     set_bit(rsc->flags, pe_rsc_merging);
     crm_debug_2("%s: Combining scores from %s", rhs, rsc->id);
 
     work = node_list_dup(nodes, FALSE, FALSE);
     node_list_update(work, rsc->allowed_nodes, attr, factor);
     
     if(allow_rollback && can_run_any(work) == FALSE) {
 	crm_info("%s: Rolling back scores from %s", rhs, rsc->id);
 	slist_destroy(node_t, n, work, crm_free(n));
 	clear_bit(rsc->flags, pe_rsc_merging);
 	return nodes;
     }
 
     if(can_run_any(work)) {
 	slist_iter(
 	    constraint, rsc_colocation_t, rsc->rsc_cons_lhs, lpc,
 
 	    crm_debug_2("Applying %s", constraint->id);
 	    work = constraint->rsc_lh->cmds->merge_weights(
 		constraint->rsc_lh, rhs, work,
 		constraint->node_attribute, 
 		multiplier*constraint->score/INFINITY, allow_rollback);
 	    );
     }
 
     slist_destroy(node_t, n, nodes, crm_free(n));
     clear_bit(rsc->flags, pe_rsc_merging);
     return work;
 }
 
 node_t *
 native_color(resource_t *rsc, pe_working_set_t *data_set)
 {
         int alloc_details = scores_log_level+1;
 	if(rsc->parent && is_not_set(rsc->parent->flags, pe_rsc_allocating)) {
 		/* never allocate children on their own */
 		crm_debug("Escalating allocation of %s to its parent: %s",
 			  rsc->id, rsc->parent->id);
 		rsc->parent->cmds->color(rsc->parent, data_set);
 	}
 	
 	if(is_not_set(rsc->flags, pe_rsc_provisional)) {
 		return rsc->allocated_to;
 	}
 
 	if(is_set(rsc->flags, pe_rsc_allocating)) {
 		crm_debug("Dependency loop detected involving %s", rsc->id);
 		return NULL;
 	}
 
 	set_bit(rsc->flags, pe_rsc_allocating);
 	print_resource(alloc_details, "Allocating: ", rsc, FALSE);
 	dump_node_scores(alloc_details, rsc, "Pre-allloc", rsc->allowed_nodes);
 
 	slist_iter(
 		constraint, rsc_colocation_t, rsc->rsc_cons, lpc,
 
 		GListPtr archive = NULL;
 		resource_t *rsc_rh = constraint->rsc_rh;
 		crm_debug_2("%s: Pre-Processing %s (%s, %d, %s)",
 			    rsc->id, constraint->id, rsc_rh->id,
 			    constraint->score, role2text(constraint->role_lh));
 		if(constraint->role_lh >= RSC_ROLE_MASTER
 		   || (constraint->score < 0 && constraint->score > -INFINITY)) {
 		    archive = node_list_dup(rsc->allowed_nodes, FALSE, FALSE);
 		}
 		rsc_rh->cmds->color(rsc_rh, data_set);
 		rsc->cmds->rsc_colocation_lh(rsc, rsc_rh, constraint);	
 		if(archive && can_run_any(rsc->allowed_nodes) == FALSE) {
 		    crm_info("%s: Rolling back scores from %s", rsc->id, rsc_rh->id);
 		    pe_free_shallow_adv(rsc->allowed_nodes, TRUE);
 		    rsc->allowed_nodes = archive;
 		    archive = NULL;
 		}
 		pe_free_shallow_adv(archive, TRUE);
 	    );	
 
 	dump_node_scores(alloc_details, rsc, "Post-coloc", rsc->allowed_nodes);
 
 	slist_iter(
 	    constraint, rsc_colocation_t, rsc->rsc_cons_lhs, lpc,
 	    
 	    rsc->allowed_nodes = constraint->rsc_lh->cmds->merge_weights(
 		constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
 		constraint->node_attribute, constraint->score/INFINITY, TRUE);
 	    );
 	
 	print_resource(LOG_DEBUG_2, "Allocating: ", rsc, FALSE);
 	if(rsc->next_role == RSC_ROLE_STOPPED) {
 		crm_debug_2("Making sure %s doesn't get allocated", rsc->id);
 		/* make sure it doesnt come up again */
 		resource_location(
 			rsc, NULL, -INFINITY, XML_RSC_ATTR_TARGET_ROLE, data_set);
 	}
 
 	dump_node_scores(show_scores?0:scores_log_level, rsc, __PRETTY_FUNCTION__, rsc->allowed_nodes);
 	if(is_set(data_set->flags, pe_flag_stonith_enabled)
 	       && is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {
 	    clear_bit(rsc->flags, pe_rsc_managed);
 	}
 
 	if(is_not_set(rsc->flags, pe_rsc_managed)) {
 	    const char *reason = NULL;
 	    node_t *assign_to = NULL;
 	    if(rsc->running_on == NULL) {
 		reason = "inactive";
 	    } else if(rsc->role == RSC_ROLE_MASTER) {
 		assign_to = rsc->running_on->data;
 		reason = "master";
 	    } else if(is_set(rsc->flags, pe_rsc_failed)) {
 		reason = "failed";		
 	    } else {
 		assign_to = rsc->running_on->data;
 		reason = "active";
 	    }
 	    crm_info("Unmanaged resource %s allocated to %s: %s", rsc->id,
 		     assign_to?assign_to->details->uname:"'nowhere'", reason);
 	    native_assign_node(rsc, NULL, assign_to, TRUE);
 
 	} else if(is_set(data_set->flags, pe_flag_stop_everything)) {
 	    crm_debug("Forcing %s to stop", rsc->id);
 	    native_assign_node(rsc, NULL, NULL, TRUE);
 
 	} else if(is_set(rsc->flags, pe_rsc_provisional)
 	   && native_choose_node(rsc, data_set) ) {
 		crm_debug_3("Allocated resource %s to %s",
 			    rsc->id, rsc->allocated_to->details->uname);
 
 	} else if(rsc->allocated_to == NULL) {
 		if(is_not_set(rsc->flags, pe_rsc_orphan)) {
 			crm_info("Resource %s cannot run anywhere", rsc->id);
 		} else if(rsc->running_on != NULL) {
 			crm_info("Stopping orphan resource %s", rsc->id);
 		}
 		
 	} else {
 		crm_debug("Pre-Allocated resource %s to %s",
 			  rsc->id, rsc->allocated_to->details->uname);
 	}
 	
 	clear_bit(rsc->flags, pe_rsc_allocating);
 	print_resource(LOG_DEBUG_3, "Allocated ", rsc, TRUE);
 
 	return rsc->allocated_to;
 }
 
 static gboolean is_op_dup(
     resource_t *rsc, const char *name, const char *interval) 
 {
     gboolean dup = FALSE;
     const char *id = NULL;
     const char *value = NULL;
     xml_child_iter_filter(
 	rsc->ops_xml, operation, "op",
 	value = crm_element_value(operation, "name");
 	if(safe_str_neq(value, name)) {
 	    continue;
 	}
 	
 	value = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
 	if(value == NULL) {
 	    value = "0";
 	}
 	
 	if(safe_str_neq(value, interval)) {
 	    continue;
 	}
 
 	if(id == NULL) {
 	    id = ID(operation);
 	    
 	} else {
 	    crm_config_err("Operation %s is a duplicate of %s", ID(operation), id);
 	    crm_config_err("Do not use the same (name, interval) combination more than once per resource");
 	    dup = TRUE;
 	}
 	);
     
     return dup;
 }
 
 void
 RecurringOp(resource_t *rsc, action_t *start, node_t *node,
 	    xmlNode *operation, pe_working_set_t *data_set) 
 {
 	char *key = NULL;
 	const char *name = NULL;
 	const char *value = NULL;
 	const char *interval = NULL;
 	const char *node_uname = NULL;
 
 	unsigned long long interval_ms = 0;
 	action_t *mon = NULL;
 	gboolean is_optional = TRUE;
 	GListPtr possible_matches = NULL;
 	
 	crm_debug_2("Creating recurring action %s for %s in role %s",
 		    ID(operation), rsc->id, role2text(rsc->next_role));
 	
 	if(node != NULL) {
 		node_uname = node->details->uname;
 	}
 
 	interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
 	interval_ms = crm_get_interval(interval);
 	
 	if(interval_ms == 0) {
 	    return;
 	}
 	
 	name = crm_element_value(operation, "name");
 	if(is_op_dup(rsc, name, interval)) {
 	    return;
 	}
 
 	key = generate_op_key(rsc->id, name, interval_ms);
 	if(find_rsc_op_entry(rsc, key) == NULL) {
 	    /* disabled */
 	    return;
 	}
 	
 	if(start != NULL) {
 		crm_debug_3("Marking %s %s due to %s",
 			    key, start->optional?"optional":"manditory",
 			    start->uuid);
 		is_optional = start->optional;
 	} else {
 		crm_debug_2("Marking %s optional", key);
 		is_optional = TRUE;
 	}
 
 	/* start a monitor for an already active resource */
 	possible_matches = find_actions_exact(rsc->actions, key, node);
 	if(possible_matches == NULL) {
 		is_optional = FALSE;
 		crm_debug_3("Marking %s manditory: not active", key);
 	} else {
 		g_list_free(possible_matches);
 	}
 	
 	value = crm_element_value(operation, "role");
 	if((rsc->next_role == RSC_ROLE_MASTER && value == NULL)
 	   || (value != NULL && text2role(value) != rsc->next_role)) {
 		int log_level = LOG_DEBUG_2;
 		const char *result = "Ignoring";
 		if(is_optional) {
 			char *local_key = crm_strdup(key);
 			log_level = LOG_INFO;
 			result = "Cancelling";
 			/* its running : cancel it */
 			
 			mon = custom_action(
 				rsc, local_key, RSC_CANCEL, node,
 				FALSE, TRUE, data_set);
 
 			crm_free(mon->task);
 			mon->task = crm_strdup(RSC_CANCEL);
 			add_hash_param(mon->meta, XML_LRM_ATTR_INTERVAL, interval);
 			add_hash_param(mon->meta, XML_LRM_ATTR_TASK, name);
 
 			local_key = NULL;
 
 			switch(rsc->role) {
 			    case RSC_ROLE_SLAVE:
 			    case RSC_ROLE_STARTED:
 				if(rsc->next_role == RSC_ROLE_MASTER) {
 				    local_key = promote_key(rsc);
 
 				} else if(rsc->next_role == RSC_ROLE_STOPPED) {
 				    local_key = stop_key(rsc);
 				}
 				
 				break;
 			    case RSC_ROLE_MASTER:
 				local_key = demote_key(rsc);
 				break;
 			    default:
 				break;
 			}
 
 			if(local_key) {
 			    custom_action_order(rsc, NULL, mon, rsc, local_key, NULL,
 						pe_order_runnable_left, data_set);
 			}
 			
 			mon = NULL;
 		}
 		
 		do_crm_log(log_level, "%s action %s (%s vs. %s)",
 			   result , key, value?value:role2text(RSC_ROLE_SLAVE),
 			   role2text(rsc->next_role));
 
 		crm_free(key);
 		key = NULL;
 		return;
 	}		
 		
 	mon = custom_action(rsc, key, name, node,
 			    is_optional, TRUE, data_set);
 	key = mon->uuid;
 	if(is_optional) {
 		crm_debug_2("%s\t   %s (optional)",
 			    crm_str(node_uname), mon->uuid);
 	}
 	
 	if(start == NULL || start->runnable == FALSE) {
 		crm_debug("%s\t   %s (cancelled : start un-runnable)",
 			  crm_str(node_uname), mon->uuid);
 		mon->runnable = FALSE;
 		
 	} else if(node == NULL
 		  || node->details->online == FALSE
 		  || node->details->unclean) {
 		crm_debug("%s\t   %s (cancelled : no node available)",
 			  crm_str(node_uname), mon->uuid);
 		mon->runnable = FALSE;
 		
 	} else if(mon->optional == FALSE) {
 	    crm_notice(" Start recurring %s (%llus) for %s on %s", mon->task, interval_ms/1000, rsc->id, crm_str(node_uname));
 	}
 
 	if(rsc->next_role == RSC_ROLE_MASTER) {
 	    char *running_master = crm_itoa(EXECRA_RUNNING_MASTER);
 	    add_hash_param(mon->meta, XML_ATTR_TE_TARGET_RC, running_master);
 	    crm_free(running_master);
 	}
 	    
 	if(node == NULL || is_set(rsc->flags, pe_rsc_managed)) {
 	    custom_action_order(rsc, start_key(rsc), NULL,
 			    NULL, crm_strdup(key), mon,
 			    pe_order_implies_right|pe_order_runnable_left, data_set);
 	
 	    if(rsc->next_role == RSC_ROLE_MASTER) {
 		custom_action_order(
 			rsc, promote_key(rsc), NULL,
 			rsc, NULL, mon,
 			pe_order_optional|pe_order_runnable_left, data_set);
 
 	    } else if(rsc->role == RSC_ROLE_MASTER) {
 		custom_action_order(
 			rsc, demote_key(rsc), NULL,
 			rsc, NULL, mon,
 			pe_order_optional|pe_order_runnable_left, data_set);
 	    }
 	}
 }
 
 void
 Recurring(resource_t *rsc, action_t *start, node_t *node,
 			 pe_working_set_t *data_set) 
 {
     if(is_not_set(data_set->flags, pe_flag_maintenance_mode)) {	
 	xml_child_iter_filter(
 		rsc->ops_xml, operation, "op",
 		RecurringOp(rsc, start, node, operation, data_set);		
 		);
     }
 }
 
 void native_create_actions(resource_t *rsc, pe_working_set_t *data_set)
 {
 	action_t *start = NULL;
 	node_t *chosen = NULL;
 	enum rsc_role_e role = RSC_ROLE_UNKNOWN;
 	enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;
 
 	crm_debug_2("Createing actions for %s: %s->%s", rsc->id,
 		    role2text(rsc->role), role2text(rsc->next_role));
 	
 	chosen = rsc->allocated_to;
 	if(chosen != NULL && rsc->next_role == RSC_ROLE_UNKNOWN) {
 	    rsc->next_role = RSC_ROLE_STARTED;
 
 	} else if(rsc->next_role == RSC_ROLE_UNKNOWN) {
 	    rsc->next_role = RSC_ROLE_STOPPED;
 	}
 
 	get_rsc_attributes(rsc->parameters, rsc, chosen, data_set);
 	
 	if(g_list_length(rsc->running_on) > 1) {
  		if(rsc->recovery_type == recovery_stop_start) {
 			pe_proc_warn("Attempting recovery of resource %s", rsc->id);
 			if(rsc->role == RSC_ROLE_MASTER) {
 			    DemoteRsc(rsc, NULL, FALSE, data_set);
 			}
 			StopRsc(rsc, NULL, FALSE, data_set);
 			rsc->role = RSC_ROLE_STOPPED;
 		}
 		
 	} else if(rsc->running_on != NULL) {
 		node_t *current = rsc->running_on->data;
 		NoRoleChange(rsc, current, chosen, data_set);
 
 	} else if(rsc->role == RSC_ROLE_STOPPED && rsc->next_role == RSC_ROLE_STOPPED) {
 		char *key = start_key(rsc);
 		GListPtr possible_matches = find_actions(rsc->actions, key, NULL);
 		slist_iter(
 			action, action_t, possible_matches, lpc,
 			action->optional = TRUE;
 /*			action->pseudo = TRUE; */
 			);
 		g_list_free(possible_matches);
 		crm_debug_2("Stopping a stopped resource");
 		crm_free(key);
 		goto do_recurring;
 		
 	} else if(rsc->role != RSC_ROLE_STOPPED) {
 	    /* A cheap trick to account for the fact that Master/Slave groups may not be
 	     * completely running when we set their role to Slave
 	     */
 	    crm_debug_2("Resetting %s.role = %s (was %s)",
 			rsc->id, role2text(RSC_ROLE_STOPPED), role2text(rsc->role));
 	    rsc->role = RSC_ROLE_STOPPED;
 	}
 
 	role = rsc->role;
 
 	while(role != rsc->next_role) {
 		next_role = rsc_state_matrix[role][rsc->next_role];
 		crm_debug_2("Executing: %s->%s (%s)",
 			  role2text(role), role2text(next_role), rsc->id);
 		if(rsc_action_matrix[role][next_role](
 			   rsc, chosen, FALSE, data_set) == FALSE) {
 			break;
 		}
 		role = next_role;
 	}
 
   do_recurring:
 	if(rsc->next_role != RSC_ROLE_STOPPED || is_set(rsc->flags, pe_rsc_managed) == FALSE) {
 		start = start_action(rsc, chosen, TRUE);
 		Recurring(rsc, start, chosen, data_set);
 	}
 }
 
 void native_internal_constraints(resource_t *rsc, pe_working_set_t *data_set)
 {
 	int type = pe_order_optional;
 	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 	action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);
 
 	if(rsc->variant == pe_native) {
 		type |= pe_order_implies_right;
 	}
 
 	if(rsc->parent == NULL || rsc->parent->variant == pe_group) {
 		type |= pe_order_restart;
 	}
 	
 	new_rsc_order(rsc, RSC_STOP,   rsc, RSC_START,   type, data_set);
 	new_rsc_order(rsc, RSC_DEMOTE, rsc, RSC_STOP,    pe_order_demote_stop, data_set);
 	new_rsc_order(rsc, RSC_START,  rsc, RSC_PROMOTE, pe_order_runnable_left, data_set);
 	new_rsc_order(rsc, RSC_DELETE, rsc, RSC_START,   pe_order_optional, data_set);	
 
 	if(is_not_set(rsc->flags, pe_rsc_managed)) {
 		crm_debug_3("Skipping fencing constraints for unmanaged resource: %s", rsc->id);
 		return;
 	} 
 
 	if(rsc->variant == pe_native && safe_str_neq(class, "stonith")) {
 	    custom_action_order(
 		rsc, stop_key(rsc), NULL,
 		NULL, crm_strdup(all_stopped->task), all_stopped,
 		pe_order_implies_right|pe_order_runnable_left, data_set);
 	}
 
 	if (safe_str_neq(data_set->placement_strategy, "default")
 			&& g_hash_table_size(rsc->utilization) > 0) {
 		
 		slist_iter(
 			current, node_t, rsc->running_on, lpc,
 
 			char *load_stopped_task = crm_concat(LOAD_STOPPED, current->details->uname, '_');
 			action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);
 			load_stopped->node = current;
 			load_stopped->optional = FALSE;
 
 	    		custom_action_order(
 				rsc, stop_key(rsc), NULL,
 				NULL, load_stopped_task, load_stopped,
 				pe_order_optional, data_set);
 			);
 
 		slist_iter(
 			next, node_t, rsc->allowed_nodes, lpc,
 
 			char *load_stopped_task = crm_concat(LOAD_STOPPED, next->details->uname, '_');
 			action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);
 			load_stopped->node = next;
 			load_stopped->optional = FALSE;
 
     			custom_action_order(
 				NULL, load_stopped_task, load_stopped,
 				rsc, start_key(rsc), NULL,
 				pe_order_optional, data_set);
 			);
 	}
 }
 
 void native_rsc_colocation_lh(
 	resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint)
 {
 	if(rsc_lh == NULL) {
 		pe_err("rsc_lh was NULL for %s", constraint->id);
 		return;
 
 	} else if(constraint->rsc_rh == NULL) {
 		pe_err("rsc_rh was NULL for %s", constraint->id);
 		return;
 	}
 	
 	crm_debug_2("Processing colocation constraint between %s and %s",
 		    rsc_lh->id, rsc_rh->id);
 	
 	rsc_rh->cmds->rsc_colocation_rh(rsc_lh, rsc_rh, constraint);		
 }
 
 static gboolean
 filter_colocation_constraint(
 	resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint)
 {
 	int level = LOG_DEBUG_4;
 	if(constraint->score == 0){
 		return FALSE;
 	}
 
 	if(constraint->score > 0
 	   && constraint->role_lh != RSC_ROLE_UNKNOWN
 	   && constraint->role_lh != rsc_lh->next_role) {
 		do_crm_log_unlikely(level, "LH: Skipping constraint: \"%s\" state filter",
 			    role2text(constraint->role_rh));
 		return FALSE;
 	}
 	
 	if(constraint->score > 0
 	   && constraint->role_rh != RSC_ROLE_UNKNOWN
 	   && constraint->role_rh != rsc_rh->next_role) {
 		do_crm_log_unlikely(level, "RH: Skipping constraint: \"%s\" state filter",
 			    role2text(constraint->role_rh));
 		return FALSE;
 	}
 
 	if(constraint->score < 0
 	   && constraint->role_lh != RSC_ROLE_UNKNOWN
 	   && constraint->role_lh == rsc_lh->next_role) {
 		do_crm_log_unlikely(level, "LH: Skipping -ve constraint: \"%s\" state filter",
 			    role2text(constraint->role_rh));
 		return FALSE;
 	}
 	
 	if(constraint->score < 0
 	   && constraint->role_rh != RSC_ROLE_UNKNOWN
 	   && constraint->role_rh == rsc_rh->next_role) {
 		do_crm_log_unlikely(level, "RH: Skipping -ve constraint: \"%s\" state filter",
 			    role2text(constraint->role_rh));
 		return FALSE;
 	}
 
 	return TRUE;
 }
 
 static void
 colocation_match(
 	resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint) 
 {
 	const char *tmp = NULL;
 	const char *value = NULL;
 	const char *attribute = "#id";
 
 	GListPtr work = NULL;
 	gboolean do_check = FALSE;
 
 	if(constraint->node_attribute != NULL) {
 		attribute = constraint->node_attribute;
 	}
 
 	if(rsc_rh->allocated_to) {
 		value = g_hash_table_lookup(
 			rsc_rh->allocated_to->details->attrs, attribute);
 		do_check = TRUE;
 
 	} else if(constraint->score < 0) {
 		/* nothing to do:
 		 *   anti-colocation with something thats not running
 		 */
 		return;
 	}
 
 	work = node_list_dup(rsc_lh->allowed_nodes, FALSE, FALSE);
 	
 	slist_iter(
 		node, node_t, work, lpc,
 		tmp = g_hash_table_lookup(node->details->attrs, attribute);
 		if(do_check && safe_str_eq(tmp, value)) {
 		    if(constraint->score < INFINITY) {
 			crm_debug_2("%s: %s.%s += %d", constraint->id, rsc_lh->id,
 				  node->details->uname, constraint->score);
 			node->weight = merge_weights(
 				constraint->score, node->weight);
 		    }
 
 		} else if(do_check == FALSE || constraint->score >= INFINITY) {
 			crm_debug_2("%s: %s.%s -= %d (%s)", constraint->id, rsc_lh->id,
 				    node->details->uname, constraint->score, do_check?"failed":"unallocated");
 			node->weight = merge_weights(-constraint->score, node->weight);
 		}
 		);
 
 	if(can_run_any(work)
 	   || constraint->score <= -INFINITY
 	   || constraint->score >= INFINITY) {
 		slist_destroy(node_t, node, rsc_lh->allowed_nodes, crm_free(node));
 		rsc_lh->allowed_nodes = work;
 		work = NULL;
 
 	} else {
 	    char *score = score2char(constraint->score);
 	    crm_info("%s: Rolling back scores from %s (%d, %s)",
 		     rsc_lh->id, rsc_rh->id, do_check, score);
 	    crm_free(score);
 	}
 
 	slist_destroy(node_t, node, work, crm_free(node));
 }
 
 void native_rsc_colocation_rh(
 	resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint)
 {
 	crm_debug_2("%sColocating %s with %s (%s, weight=%d)",
 		    constraint->score >= 0?"":"Anti-",
 		    rsc_lh->id, rsc_rh->id, constraint->id, constraint->score);
 	
 	if(filter_colocation_constraint(rsc_lh, rsc_rh, constraint) == FALSE) {
 		return;
 	}
 	
 	if(is_set(rsc_rh->flags, pe_rsc_provisional)) {
 		return;
 
 	} else if(is_not_set(rsc_lh->flags, pe_rsc_provisional)) {
 		/* error check */
 		struct node_shared_s *details_lh;
 		struct node_shared_s *details_rh;
 		if((constraint->score > -INFINITY) && (constraint->score < INFINITY)) {
 			return;
 		}
 
 		details_rh = rsc_rh->allocated_to?rsc_rh->allocated_to->details:NULL;
 		details_lh = rsc_lh->allocated_to?rsc_lh->allocated_to->details:NULL;
 		
 		if(constraint->score == INFINITY && details_lh != details_rh) {
 			crm_err("%s and %s are both allocated"
 				" but to different nodes: %s vs. %s",
 				rsc_lh->id, rsc_rh->id,
 				details_lh?details_lh->uname:"n/a",
 				details_rh?details_rh->uname:"n/a");
 
 		} else if(constraint->score == -INFINITY && details_lh == details_rh) {
 			crm_err("%s and %s are both allocated"
 				" but to the SAME node: %s",
 				rsc_lh->id, rsc_rh->id,
 				details_rh?details_rh->uname:"n/a");
 		}
 		
 		return;
 		
 	} else {
 		colocation_match(rsc_lh, rsc_rh, constraint);
 	}
 }
 
 static GListPtr find_actions_by_task(GListPtr actions, resource_t *rsc, const char *original_key)
 {
     GListPtr list = NULL;
 
     list = find_actions(actions, original_key, NULL);
     if(list == NULL) {
 	/* we're potentially searching a child of the original resource */
 	char *key = NULL;
 	char *tmp = NULL;
 	char *task = NULL;
 	int interval = 0;
 	if(parse_op_key(original_key, &tmp, &task, &interval)) {
 	    key = generate_op_key(rsc->id, task, interval);
 	    /* crm_err("looking up %s instead of %s", key, original_key); */
 	    /* slist_iter(action, action_t, actions, lpc, */
 	    /* 	       crm_err("  - %s", action->uuid)); */
 	    list = find_actions(actions, key, NULL);
 	    
 	} else {
 	    crm_err("search key: %s", original_key);
 	}	
 
 	crm_free(key);
 	crm_free(tmp);
 	crm_free(task);
     }
 
     return list;
 }
 
 void native_rsc_order_lh(resource_t *lh_rsc, order_constraint_t *order, pe_working_set_t *data_set)
 {
 	GListPtr lh_actions = NULL;
 	action_t *lh_action = order->lh_action;
 	resource_t *rh_rsc = order->rh_rsc;
 
 	crm_debug_3("Processing LH of ordering constraint %d", order->id);
 	CRM_ASSERT(lh_rsc != NULL);
 	
 	if(lh_action != NULL) {
 		lh_actions = g_list_append(NULL, lh_action);
 
 	} else if(lh_action == NULL) {
 		lh_actions = find_actions_by_task(
 		    lh_rsc->actions, lh_rsc, order->lh_action_task);
 	}
 
 	if(lh_actions == NULL && lh_rsc != rh_rsc) {
 		char *key = NULL;
 		char *rsc_id = NULL;
 		char *op_type = NULL;
 		int interval = 0;
 		
 		crm_debug_4("No LH-Side (%s/%s) found for constraint %d with %s - creating",
 			    lh_rsc->id, order->lh_action_task,
 			    order->id, order->rh_action_task);
 
 		parse_op_key(
 			order->lh_action_task, &rsc_id, &op_type, &interval);
 
 		key = generate_op_key(lh_rsc->id, op_type, interval);
 
 		lh_action = custom_action(lh_rsc, key, op_type,
 					  NULL, TRUE, TRUE, data_set);
 
 		if(lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_STOPPED
 		   && safe_str_eq(op_type, RSC_STOP)) {
 			lh_action->pseudo = TRUE;
 			lh_action->runnable = TRUE;
 		}
 		
 		lh_actions = g_list_append(NULL, lh_action);
 
 		crm_free(op_type);
 		crm_free(rsc_id);
 	}
 
 	slist_iter(
 		lh_action_iter, action_t, lh_actions, lpc,
 
 		if(rh_rsc == NULL && order->rh_action) {
 			rh_rsc = order->rh_action->rsc;
 		}
 		if(rh_rsc) {
 			rh_rsc->cmds->rsc_order_rh(
 				lh_action_iter, rh_rsc, order);
 
 		} else if(order->rh_action) {
 		    order_actions(
 				lh_action_iter, order->rh_action, order->type); 
 
 		}
 		);
 
 	pe_free_shallow_adv(lh_actions, FALSE);
 }
 
 void native_rsc_order_rh(
 	action_t *lh_action, resource_t *rsc, order_constraint_t *order)
 {
 	GListPtr rh_actions = NULL;
 	action_t *rh_action = NULL;
 
 	CRM_CHECK(rsc != NULL, return);
 	CRM_CHECK(order != NULL, return);
 
 	rh_action = order->rh_action;
 	crm_debug_3("Processing RH of ordering constraint %d", order->id);
 
 	if(rh_action != NULL) {
 		rh_actions = g_list_append(NULL, rh_action);
 
 	} else if(rsc != NULL) {
 		rh_actions = find_actions_by_task(
 		    rsc->actions, rsc, order->rh_action_task);
 	}
 
 	if(rh_actions == NULL) {
 		crm_debug_4("No RH-Side (%s/%s) found for constraint..."
 			    " ignoring", rsc->id,order->rh_action_task);
 		if(lh_action) {
 			crm_debug_4("LH-Side was: %s", lh_action->uuid);
 		}
 		return;
 	}
 	
 	slist_iter(
 		rh_action_iter, action_t, rh_actions, lpc,
 
 		if(lh_action) {
 			order_actions(lh_action, rh_action_iter, order->type); 
 			
 		} else if(order->type & pe_order_implies_right) {
 			rh_action_iter->runnable = FALSE;
 			crm_warn("Unrunnable %s 0x%.6x", rh_action_iter->uuid, order->type);
 		} else {
 			crm_warn("neither %s 0x%.6x", rh_action_iter->uuid, order->type);
 		}
 		
 		);
 
 	pe_free_shallow_adv(rh_actions, FALSE);
 }
 
 void native_rsc_location(resource_t *rsc, rsc_to_node_t *constraint)
 {
 	GListPtr or_list;
 
 	crm_debug_2("Applying %s (%s) to %s", constraint->id,
 		    role2text(constraint->role_filter), rsc->id);
 
 	/* take "lifetime" into account */
 	if(constraint == NULL) {
 		pe_err("Constraint is NULL");
 		return;
 
 	} else if(rsc == NULL) {
 		pe_err("LHS of rsc_to_node (%s) is NULL", constraint->id);
 		return;
 
 	} else if(constraint->role_filter > 0
 		  && constraint->role_filter != rsc->next_role) {
 		crm_debug("Constraint (%s) is not active (role : %s)",
 			  constraint->id, role2text(constraint->role_filter));
 		return;
 		
 	} else if(is_active(constraint) == FALSE) {
 		crm_debug_2("Constraint (%s) is not active", constraint->id);
 		return;
 	}
     
 	if(constraint->node_list_rh == NULL) {
 		crm_debug_2("RHS of constraint %s is NULL", constraint->id);
 		return;
 	}
 	or_list = node_list_or(
 		rsc->allowed_nodes, constraint->node_list_rh, FALSE);
 		
 	pe_free_shallow(rsc->allowed_nodes);
 	rsc->allowed_nodes = or_list;
 	slist_iter(node, node_t, or_list, lpc,
 		   crm_debug_3("%s + %s : %d", rsc->id, node->details->uname, node->weight);
 		);
 }
 
 void native_expand(resource_t *rsc, pe_working_set_t *data_set)
 {
 	crm_debug_3("Processing actions from %s", rsc->id);
 
 	slist_iter(
 		action, action_t, rsc->actions, lpc,
 		crm_debug_4("processing action %d for rsc=%s",
 			  action->id, rsc->id);
 		graph_element_from_action(action, data_set);
 		);
 
 	slist_iter(
 	    child_rsc, resource_t, rsc->children, lpc,
 	    
 	    child_rsc->cmds->expand(child_rsc, data_set);
 	    );
 }
 
 
 void
 LogActions(resource_t *rsc, pe_working_set_t *data_set)
 {
     node_t *next = NULL;
     node_t *current = NULL;
     gboolean moving = FALSE;
     
     if(rsc->children) {
 	slist_iter(
 	    child_rsc, resource_t, rsc->children, lpc,
 	    LogActions(child_rsc, data_set);
 	    );
 	return;
     }
     
     next = rsc->allocated_to;
     if(rsc->running_on) {
 	current = rsc->running_on->data;
 	if(rsc->role == RSC_ROLE_STOPPED) {
 	    /*
 	     * This can occur when resources are being recovered
 	     * We fiddle with the current role in native_create_actions()
 	     */
 	    rsc->role = RSC_ROLE_STARTED;
 	}
     }
 
     if(current == NULL && is_set(rsc->flags, pe_rsc_orphan)) {
 	/* Don't log stopped orphans */
 	return;
     }
     
     if(is_not_set(rsc->flags, pe_rsc_managed)
        || (current == NULL && next == NULL)) {
 	crm_notice("Leave resource %s\t(%s%s)",
 		   rsc->id, role2text(rsc->role), is_not_set(rsc->flags, pe_rsc_managed)?" unmanaged":"");
 	return;
     }
 
     if(current != NULL && next != NULL
        && safe_str_neq(current->details->id, next->details->id)) {
 	moving = TRUE;
     }
     
     if(rsc->role == rsc->next_role) {
 	action_t *start = NULL;
 	char *key = start_key(rsc);
 	GListPtr possible_matches = find_actions(rsc->actions, key, next);
 	crm_free(key);
 
 	if(possible_matches) {
 	    start = possible_matches->data;
 	    g_list_free(possible_matches);
 	}
 
 	key = generate_op_key(rsc->id, CRMD_ACTION_MIGRATED, 0);
 	possible_matches = find_actions(rsc->actions, key, next);
 	crm_free(key);
 	
 	CRM_CHECK(next != NULL,);
 	if(next == NULL) {
-	} else if(possible_matches) {
+	} else if(possible_matches && current) {
 	    crm_notice("Migrate resource %s\t(%s %s -> %s)",
 		       rsc->id, role2text(rsc->role), current->details->uname, next->details->uname);
 	    g_list_free(possible_matches);
 	    
 	} else if(start == NULL || start->optional) {
 	    crm_notice("Leave resource %s\t(%s %s)",
 		       rsc->id, role2text(rsc->role), next->details->uname);
 	    
 	} else if(moving && current) {
 	    crm_notice("Move resource %s\t(%s %s -> %s)",
 		       rsc->id, role2text(rsc->role), current->details->uname, next->details->uname);
 	    
 	} else if(is_set(rsc->flags, pe_rsc_failed)) {
 	    crm_notice("Recover resource %s\t(%s %s)",
 		       rsc->id, role2text(rsc->role), next->details->uname);
 	    
 	} else if(start && start->runnable == FALSE) {
 	    crm_notice("Stop resource %s\t(%s %s)",
 		       rsc->id, role2text(rsc->role), next->details->uname);
 
 	} else {
 	    crm_notice("Restart resource %s\t(%s %s)",
 		       rsc->id, role2text(rsc->role), next->details->uname);
 	}
 	
 	return;
     }
 
     if(rsc->role > RSC_ROLE_SLAVE && rsc->role > rsc->next_role) {
 	CRM_CHECK(current != NULL,);
 	if(current != NULL) {
 	    crm_notice("Demote %s\t(%s -> %s %s)", rsc->id,
 		       role2text(rsc->role), role2text(rsc->next_role),
 		       current->details->uname);
 	}
     }
 
     if(rsc->next_role == RSC_ROLE_STOPPED || moving) {
 	CRM_CHECK(current != NULL,);
 	slist_iter(node, node_t, rsc->running_on, lpc,
 		   crm_notice("Stop resource %s\t(%s)", rsc->id, node->details->uname));
     }
 
     if(rsc->role == RSC_ROLE_STOPPED || moving) {
 	CRM_CHECK(next != NULL,);
 	if(next != NULL) {
 	    crm_notice("Start %s\t(%s)", rsc->id, next->details->uname);
 	}
     }    
 
     if(rsc->next_role > RSC_ROLE_SLAVE && rsc->role < rsc->next_role) {
 	CRM_CHECK(next != NULL,);
 	crm_notice("Promote %s\t(%s -> %s %s)", rsc->id,
 		   role2text(rsc->role), role2text(rsc->next_role),
 		   next->details->uname);
     }
 }
 
 void
 NoRoleChange(resource_t *rsc, node_t *current, node_t *next,
 	     pe_working_set_t *data_set)
 {
 	action_t *stop = NULL;
 	action_t *start = NULL;		
 	GListPtr possible_matches = NULL;
 
 	crm_debug_2("Executing: %s (role=%s)", rsc->id, role2text(rsc->next_role));
 	
 	if(current == NULL || next == NULL) {
 	    return;
 	}
 	
 	if(is_set(rsc->flags, pe_rsc_failed)
 	   || safe_str_neq(current->details->id, next->details->id)) {
 
 		if(rsc->next_role > RSC_ROLE_STARTED) {
 		    gboolean optional = TRUE;
 		    if(rsc->role == RSC_ROLE_MASTER) {
 			optional = FALSE;
 		    }
 		    DemoteRsc(rsc, current, optional, data_set);
 		}
 		if(rsc->role == RSC_ROLE_MASTER) {
 			DemoteRsc(rsc, current, FALSE, data_set);
 		}
 		StopRsc(rsc, current, FALSE, data_set);
 		StartRsc(rsc, next, FALSE, data_set);
 		if(rsc->next_role == RSC_ROLE_MASTER) {
 		    PromoteRsc(rsc, next, FALSE, data_set);
 		}
 
 		possible_matches = find_recurring_actions(rsc->actions, next);
 		slist_iter(match, action_t, possible_matches, lpc,
 			   if(match->optional == FALSE) {
 				   crm_debug("Fixing recurring action: %s",
 					     match->uuid);
 				   match->optional = TRUE;
 			   }
 			);
 		g_list_free(possible_matches);
 		
 	} else if(is_set(rsc->flags, pe_rsc_start_pending)) {
 		start = start_action(rsc, next, TRUE);
 		if(start->runnable) {
 			/* wait for StartRsc() to be called */
 			rsc->role = RSC_ROLE_STOPPED;
 		} else {
 			/* wait for StopRsc() to be called */
 			rsc->next_role = RSC_ROLE_STOPPED;
 		}
 		
 	} else {
 	    
 		stop = stop_action(rsc, current, TRUE);
 		start = start_action(rsc, next, TRUE);
 		stop->optional = start->optional;
 		if(rsc->next_role > RSC_ROLE_STARTED) {
 		    DemoteRsc(rsc, current, start->optional, data_set);
 		}
 		StopRsc(rsc, current, start->optional, data_set);
 		StartRsc(rsc, current, start->optional, data_set);
 		if(rsc->next_role == RSC_ROLE_MASTER) {
 			PromoteRsc(rsc, next, start->optional, data_set);
 		}
 		
 		if(start->runnable == FALSE) {
 			rsc->next_role = RSC_ROLE_STOPPED;
 		}
 	}
 }
 
 
 gboolean
 StopRsc(resource_t *rsc, node_t *next, gboolean optional, pe_working_set_t *data_set)
 {
-	action_t *stop = NULL;
 	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 
 	crm_debug_2("Executing: %s", rsc->id);
 
 	if(rsc->next_role == RSC_ROLE_STOPPED
 	   && rsc->variant == pe_native
 	   && safe_str_eq(class, "stonith")) {
 	    action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);
 	    custom_action_order(
 		NULL, crm_strdup(all_stopped->task), all_stopped,
 		rsc, stop_key(rsc), NULL,
 		pe_order_implies_left|pe_order_stonith_stop, data_set);
 	}
 	
 	slist_iter(
 		current, node_t, rsc->running_on, lpc,
-		stop = stop_action(rsc, current, optional);
+		stop_action(rsc, current, optional);
 
 		if(is_set(data_set->flags, pe_flag_remove_after_stop)) {
 			DeleteRsc(rsc, current, optional, data_set);
 		}
 		);
 	
 	return TRUE;
 }
 
 
 gboolean
 StartRsc(resource_t *rsc, node_t *next, gboolean optional, pe_working_set_t *data_set)
 {
 	action_t *start = NULL;
 	
 	crm_debug_2("Executing: %s", rsc->id);
 	start = start_action(rsc, next, TRUE);
 	if(start->runnable && optional == FALSE) {
 		start->optional = FALSE;
 	}		
 	return TRUE;
 }
 
 gboolean
 PromoteRsc(resource_t *rsc, node_t *next, gboolean optional, pe_working_set_t *data_set)
 {
 	char *key = NULL;
 	gboolean runnable = TRUE;
 	GListPtr action_list = NULL;
 	crm_debug_2("Executing: %s", rsc->id);
 
 	CRM_CHECK(rsc->next_role == RSC_ROLE_MASTER,
 		  crm_err("Next role: %s", role2text(rsc->next_role));
 		  return FALSE);
 
 	CRM_CHECK(next != NULL, return FALSE);
 
 	key = start_key(rsc);
 	action_list = find_actions_exact(rsc->actions, key, next);
 	crm_free(key);
 
 	slist_iter(start, action_t, action_list, lpc,
 		   if(start->runnable == FALSE) {
 			   runnable = FALSE;
 		   }
 		);
 
 	g_list_free(action_list);
 
 	if(runnable) {
 		promote_action(rsc, next, optional);
 		return TRUE;
 	} 
 
 	crm_debug("%s\tPromote %s (canceled)", next->details->uname, rsc->id);
 
 	key = promote_key(rsc);
 	action_list = find_actions_exact(rsc->actions, key, next);
 	crm_free(key);
 
 	slist_iter(promote, action_t, action_list, lpc,
 		   promote->runnable = FALSE;
 		);
 	
 	g_list_free(action_list);
 	return TRUE;
 }
 
 gboolean
 DemoteRsc(resource_t *rsc, node_t *next, gboolean optional, pe_working_set_t *data_set)
 {
 	crm_debug_2("Executing: %s", rsc->id);
 
 /* 	CRM_CHECK(rsc->next_role == RSC_ROLE_SLAVE, return FALSE); */
 	slist_iter(
 		current, node_t, rsc->running_on, lpc,
 		demote_action(rsc, current, optional);
 		);
 	return TRUE;
 }
 
 gboolean
 RoleError(resource_t *rsc, node_t *next, gboolean optional, pe_working_set_t *data_set)
 {
 	crm_debug("Executing: %s", rsc->id);
 	CRM_CHECK(FALSE, return FALSE);
 	return FALSE;
 }
 
 gboolean
 NullOp(resource_t *rsc, node_t *next, gboolean optional, pe_working_set_t *data_set)
 {
 	crm_debug_2("Executing: %s", rsc->id);
 	return FALSE;
 }
 
 gboolean
 DeleteRsc(resource_t *rsc, node_t *node, gboolean optional, pe_working_set_t *data_set)
 {
 	action_t *delete = NULL;
 #if DELETE_THEN_REFRESH
  	action_t *refresh = NULL;
 #endif
 	if(is_set(rsc->flags, pe_rsc_failed)) {
 		crm_debug_2("Resource %s not deleted from %s: failed",
 			    rsc->id, node->details->uname);
 		return FALSE;
 		
 	} else if(node == NULL) {
 		crm_debug_2("Resource %s not deleted: NULL node", rsc->id);
 		return FALSE;
 		
 	} else if(node->details->unclean || node->details->online == FALSE) {
 		crm_debug_2("Resource %s not deleted from %s: unrunnable",
 			    rsc->id, node->details->uname);
 		return FALSE;
 	}
 	
 	crm_notice("Removing %s from %s",
 		 rsc->id, node->details->uname);
 	
 	delete = delete_action(rsc, node, optional);
 	
 	new_rsc_order(rsc, RSC_STOP, rsc, RSC_DELETE, 
 		      optional?pe_order_implies_right:pe_order_implies_left, data_set);
 	
 #if DELETE_THEN_REFRESH
 	refresh = custom_action(
 		NULL, crm_strdup(CRM_OP_LRM_REFRESH), CRM_OP_LRM_REFRESH,
 		node, FALSE, TRUE, data_set);
 
 	add_hash_param(refresh->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
 
 	order_actions(delete, refresh, pe_order_optional);
 #endif
 	
 	return TRUE;
 }
 
 #include <../lib/pengine/unpack.h>
 
 gboolean
 native_create_probe(resource_t *rsc, node_t *node, action_t *complete,
 		    gboolean force, pe_working_set_t *data_set) 
 {
 	char *key = NULL;
 	action_t *probe = NULL;
 	node_t *running = NULL;
 	resource_t *top = uber_parent(rsc);
 
 	static const char *rc_master = NULL;
 	static const char *rc_inactive = NULL;
 
 	if(rc_inactive == NULL) {
 	    rc_inactive = crm_itoa(EXECRA_NOT_RUNNING);
 	    rc_master = crm_itoa(EXECRA_RUNNING_MASTER);
 	}
 
 	CRM_CHECK(node != NULL, return FALSE);
 	if(force == FALSE && is_not_set(data_set->flags, pe_flag_startup_probes)) {
 	    crm_debug_2("Skipping active resource detection for %s", rsc->id);
 	    return FALSE;
 	}
 	
 	if(rsc->children) {
 	    gboolean any_created = FALSE;
 	    
 	    slist_iter(
 		child_rsc, resource_t, rsc->children, lpc,
 		
 		any_created = child_rsc->cmds->create_probe(
 		    child_rsc, node, complete, force, data_set) || any_created;
 		);
 
 	    return any_created;
 	}
 
 	if(is_set(rsc->flags, pe_rsc_orphan)) {
 		crm_debug_2("Skipping orphan: %s", rsc->id);
 		return FALSE;
 	}
 	
 	running = pe_find_node_id(rsc->known_on, node->details->id);
 	if(force == FALSE && running != NULL) {
 		/* we already know the status of the resource on this node */
 		crm_debug_3("Skipping active: %s", rsc->id);
 		return FALSE;
 	}
 	
 	if(running == NULL && is_set(top->flags, pe_rsc_unique) == FALSE) {
 	    /* Annoyingly we also need to check any other clone instances
 	     * Clumsy, but it will work.
 	     *
 	     * An alternative would be to update known_on for every peer
 	     * during process_rsc_state()
 	     *
 	     * This code desperately needs optimization
 	     * ptest -x with 100 nodes, 100 clones and clone-max=10:
 	     *   No probes				O(25s)
 	     *   Detection without clone loop		O(3m)
 	     *   Detection with clone loop     		O(8m)
 
 ptest[32211]: 2010/02/18_14:27:55 CRIT: stage5: Probing for unknown resources
 ptest[32211]: 2010/02/18_14:33:39 CRIT: stage5: Done
 ptest[32211]: 2010/02/18_14:35:05 CRIT: stage7: Updating action states
 ptest[32211]: 2010/02/18_14:35:05 CRIT: stage7: Done
 	     
 	     */
 	    char *clone_id = clone_zero(rsc->id);
 	    resource_t *peer = pe_find_resource(top->children, clone_id);
 
 	    while(peer && running == NULL) {
 		running = pe_find_node_id(peer->known_on, node->details->id);
 		if(force == FALSE && running != NULL) {
 		    /* we already know the status of the resource on this node */
 		    crm_debug_3("Skipping active clone: %s", rsc->id);
 		    crm_free(clone_id);
 		    return FALSE;
 		}
 		clone_id = increment_clone(clone_id);
 		peer = pe_find_resource(data_set->resources, clone_id);
 	    }
 	    
 	    crm_free(clone_id);
 	}
 
 	key = generate_op_key(rsc->id, RSC_STATUS, 0);
 	probe = custom_action(rsc, key, RSC_STATUS, node, FALSE, TRUE, data_set);
 	probe->optional = FALSE;
 	
 	if(running == NULL) {
 	    add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);
 
 	} else if(rsc->role == RSC_ROLE_MASTER) {
 	    add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_master);
 	}
 
 	crm_debug("Probing %s on %s (%s)", rsc->id, node->details->uname, role2text(rsc->role));
 	order_actions(probe, complete, pe_order_implies_right);
 	
 	return TRUE;
 }
 
 static void
 native_start_constraints(
 	resource_t *rsc,  action_t *stonith_op, gboolean is_stonith,
 	pe_working_set_t *data_set)
 {
 	node_t *target = stonith_op?stonith_op->node:NULL;
 
 	if(is_stonith) {
 		char *key = start_key(rsc);
 		action_t *ready = get_pseudo_op(STONITH_UP, data_set);
 
 		crm_debug_2("Ordering %s action before stonith events", key);
 		custom_action_order(
 		    rsc, key, NULL,
 		    NULL, crm_strdup(ready->task), ready,
 		    pe_order_optional, data_set);
 		
 	} else {
 		action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);
 		action_t *stonith_done = get_pseudo_op(STONITH_DONE, data_set);
 		slist_iter(action, action_t, rsc->actions, lpc2,
 			   if(action->needs == rsc_req_stonith) {
 			       order_actions(stonith_done, action, pe_order_implies_left);
 
 			   } else if(target != NULL
 			      && safe_str_eq(action->task, RSC_START)
 			      && NULL == pe_find_node_id(
 				      rsc->known_on, target->details->id)) {
 				   /* if known == NULL, then we dont know if
 				    *   the resource is active on the node
 				    *   we're about to shoot
 				    *
 				    * in this case, regardless of action->needs,
 				    *   the only safe option is to wait until
 				    *   the node is shot before doing anything
 				    *   to with the resource
 				    *
 				    * its analogous to waiting for all the probes
 				    *   for rscX to complete before starting rscX
 				    *
 				    * the most likely explaination is that the
 				    *   DC died and took its status with it
 				    */
 				   
 				   crm_debug("Ordering %s after %s recovery",
 					     action->uuid, target->details->uname);
 				   order_actions(all_stopped, action,
 						 pe_order_implies_left|pe_order_runnable_left);
 			   }
 			   
 			);
 	}
 }
 
 static void
 native_stop_constraints(
 	resource_t *rsc,  action_t *stonith_op, gboolean is_stonith,
 	pe_working_set_t *data_set)
 {
 	char *key = NULL;
 	GListPtr action_list = NULL;
 	resource_t *top = uber_parent(rsc);
 	
 	key = stop_key(rsc);
 	action_list = find_actions(rsc->actions, key, stonith_op->node);
 	crm_free(key);
 
 	/* add the stonith OP as a stop pre-req and the mark the stop
 	 * as a pseudo op - since its now redundant
 	 */
 	
 	slist_iter(
 	    action, action_t, action_list, lpc2,
 
 	    resource_t *parent = NULL;
 	    if(action->node->details->online
 	       && action->node->details->unclean == FALSE
 	       && is_set(rsc->flags, pe_rsc_failed)) {
 		continue;
 	    }
 
 	    if(is_set(rsc->flags, pe_rsc_failed)) {
 		crm_warn("Stop of failed resource %s is"
 			 " implicit after %s is fenced",
 			 rsc->id, action->node->details->uname);
 	    } else {
 		crm_info("%s is implicit after %s is fenced",
 			 action->uuid, action->node->details->uname);
 	    }
 
 	    /* the stop would never complete and is
 	     * now implied by the stonith operation
 	     */
 	    action->pseudo = TRUE;
 	    action->runnable = TRUE;
 	    action->implied_by_stonith = TRUE;
 	    
 	    if(is_stonith == FALSE) {
 		action_t *parent_stop = find_first_action(top->actions, NULL, RSC_STOP, NULL);
 		
 		order_actions(stonith_op, action, pe_order_optional);
 		order_actions(stonith_op, parent_stop, pe_order_optional);
 	    }
 
 	    if(is_set(rsc->flags, pe_rsc_notify)) {
 		/* Create a second notification that will be delivered
 		 *   immediately after the node is fenced
 		 *
 		 * Basic problem:
 		 * - C is a clone active on the node to be shot and stopping on another
 		 * - R is a resource that depends on C
 		 *
 		 * + C.stop depends on R.stop
 		 * + C.stopped depends on STONITH
 		 * + C.notify depends on C.stopped
 		 * + C.healthy depends on C.notify
 		 * + R.stop depends on C.healthy
 		 *
 		 * The extra notification here changes
 		 *  + C.healthy depends on C.notify
 		 * into: 
 		 *  + C.healthy depends on C.notify'
 		 *  + C.notify' depends on STONITH'
 		 * thus breaking the loop
 		 */
 		notify_data_t *n_data = create_notification_boundaries(rsc, RSC_STOP, NULL, stonith_op, data_set);
 		crm_info("Creating secondary notification for %s", action->uuid);
 
 		collect_notification_data(rsc, TRUE, FALSE, n_data);
 		g_hash_table_insert(n_data->keys, crm_strdup("notify_stop_resource"), crm_strdup(rsc->id));
 		g_hash_table_insert(n_data->keys, crm_strdup("notify_stop_uname"), crm_strdup(action->node->details->uname));
 		create_notifications(uber_parent(rsc), n_data, data_set);
 		free_notification_data(n_data);
 	    }
 	    
 	    /* find the top-most resource */
 	    parent = rsc->parent;
 	    while(parent != NULL && parent->parent != NULL) {
 		parent = parent->parent;
 	    }
 
 	    if(parent) {
 		crm_debug_2("Re-creating actions for %s", parent->id);
 		parent->cmds->create_actions(parent, data_set);
 		
 		/* make sure we dont mess anything up in create_actions */
 		CRM_CHECK(action->pseudo, action->pseudo = TRUE);
 		CRM_CHECK(action->runnable, action->runnable = TRUE);
 	    }
 /* From Bug #1601, successful fencing must be an input to a failed resources stop action.
 
    However given group(rA, rB) running on nodeX and B.stop has failed, 
    A := stop healthy resource (rA.stop)
    B := stop failed resource (pseudo operation B.stop)
    C := stonith nodeX
    A requires B, B requires C, C requires A
    This loop would prevent the cluster from making progress.
 
    This block creates the "C requires A" dependency and therefore must (at least
    for now) be disabled.
 
    Instead, run the block above and treat all resources on nodeX as B would be
    (marked as a pseudo op depending on the STONITH).
 
    TODO: Break the "A requires B" dependency in update_action() and re-enable this block
    
 		} else if(is_stonith == FALSE) {
 			crm_info("Moving healthy resource %s"
 				 " off %s before fencing",
 				 rsc->id, node->details->uname);
 			
 			 * stop healthy resources before the
 			 * stonith op
 			 *
 			custom_action_order(
 				rsc, stop_key(rsc), NULL,
 				NULL,crm_strdup(CRM_OP_FENCE),stonith_op,
 				pe_order_optional, data_set);
 */
 	    );
 	
 	g_list_free(action_list);
 
 	key = demote_key(rsc);
 	action_list = find_actions(rsc->actions, key, stonith_op->node);
 	crm_free(key);
 	
 	slist_iter(
 		action, action_t, action_list, lpc2,
 		if(action->node->details->online == FALSE || is_set(rsc->flags, pe_rsc_failed)) {
 			crm_info("Demote of failed resource %s is"
 				 " implict after %s is fenced",
 				 rsc->id, action->node->details->uname);
 			/* the stop would never complete and is
 			 * now implied by the stonith operation
 			 */
 			action->pseudo = TRUE;
 			action->runnable = TRUE;
 			if(is_stonith == FALSE) {
 			    order_actions(stonith_op, action, pe_order_optional);
 			}
 		}
 		);	
 	
 	g_list_free(action_list);
 }
 
 void
 complex_stonith_ordering(
 	resource_t *rsc,  action_t *stonith_op, pe_working_set_t *data_set)
 {
 	gboolean is_stonith = FALSE;
 	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 
 	if(rsc->children) {
 	    slist_iter(
 		child_rsc, resource_t, rsc->children, lpc,
 
 		child_rsc->cmds->stonith_ordering(
 		    child_rsc, stonith_op, data_set);
 		);
 	    return;
 	}
 	
 	if(is_not_set(rsc->flags, pe_rsc_managed)) {
 		crm_debug_3("Skipping fencing constraints for unmanaged resource: %s", rsc->id);
 		return;
 	} 
 
 	if(stonith_op != NULL && safe_str_eq(class, "stonith")) {
 		is_stonith = TRUE;
 	}
 	
 	/* Start constraints */
 	native_start_constraints(rsc,  stonith_op, is_stonith, data_set);
  
 	/* Stop constraints */
 	native_stop_constraints(rsc,  stonith_op, is_stonith, data_set);
 }
 
 enum stack_activity 
 {
     stack_stable   = 0,
     stack_starting = 1,
     stack_stopping = 2,
     stack_middle   = 4,
 };
 
 static enum stack_activity
 find_clone_activity_on(resource_t *rsc, resource_t *target, node_t *node, const char *type) 
 {
     int mode = stack_stable;
     action_t *active = NULL;
 
     if(target->children) {
 	slist_iter(
 	    child, resource_t, target->children, lpc,
 	    mode |= find_clone_activity_on(rsc, child, node, type);
 	    );
 	return mode;
     }
 
     active = find_first_action(target->actions, NULL, CRMD_ACTION_START, NULL);
     if(active && active->optional == FALSE && active->pseudo == FALSE) {
 	crm_debug("%s: found scheduled %s action (%s)", rsc->id, active->uuid, type);
 	mode |= stack_starting;
     }
 
     active = find_first_action(target->actions, NULL, CRMD_ACTION_STOP, node);
     if(active && active->optional == FALSE && active->pseudo == FALSE) {
 	crm_debug("%s: found scheduled %s action (%s)", rsc->id, active->uuid, type);
 	mode |= stack_stopping;
     }
 
     return mode;
 }
 
 static enum stack_activity
 check_stack_element(resource_t *rsc, resource_t *other_rsc, const char *type) 
 {
     resource_t *other_p = uber_parent(other_rsc);
     if(other_rsc == NULL || other_rsc == rsc) {
 	return stack_stable;
 
     } else if(other_p->variant == pe_native) {
 	crm_notice("Cannot migrate %s due to dependency on %s (%s)",
 		   rsc->id, other_rsc->id, type);
 	return stack_middle;
 
     } else if(other_rsc == rsc->parent) {
 	int mode = 0;
 	slist_iter(constraint, rsc_colocation_t, other_rsc->rsc_cons, lpc,
 		   if(constraint->score > 0) {
 		       mode |= check_stack_element(rsc, constraint->rsc_rh, type);
 		   }
 	    );
 	return mode;
 	    
     } else if(other_p->variant == pe_group) {
 	crm_notice("Cannot migrate %s due to dependency on group %s (%s)",
 		   rsc->id, other_rsc->id, type);
 	return stack_middle;
 
     } /* else: >= clone */
     
     /*
       
 ## Assumption
 A depends on clone(B)
 
 ## Resource Activity During Move
 
 	N1		N2		N3
 	---             ---		---
 t0	A.stop
 t1	B.stop				B.stop
 t2			B.start		B.start
 t3			A.start
 
 ## Resource Activity During Migration
 
        	N1		N2		N3
 	---             ---		---
 t0			B.start		B.start
 t1	A.stop (1)
 t2			A.start (2)
 t3	B.stop				B.stop
 
 Node 1: Rewritten to be a migrate-to operation
 Node 2: Rewritten to be a migrate-from operation
 
 # Constraints
 
 The following constraints already exist in the system.
 The 'ok' and 'fail' column refers to whether they still hold for migration.
 
 a) A.stop  -> A.start - ok
 b) B.stop  -> B.start - fail
 
 c) A.stop  -> B.stop  - ok
 d) B.start -> A.start - ok
 
 e) B.stop  -> A.start - fail
 f) A.stop  -> B.start - fail
 
 ## Scenarios
 B unchanged             - ok
 B stopping only         - fail - possible after fixing 'e'
 B starting only         - fail - possible after fixing 'f'
 B stoping and starting  - fail - constraint 'b' is unfixable
 B restarting only on N2 - fail - as-per previous only rarer
       
      */
     
     /* Only allow migration when the clone is either stable, only starting or only stopping */
     return find_clone_activity_on(rsc, other_rsc, NULL, type);
 }
 
 static gboolean
 at_stack_bottom(resource_t *rsc) 
 {
     char *key = NULL;
     action_t *start = NULL;
     action_t *other = NULL;
     int mode = stack_stable;
     GListPtr action_list = NULL;
 
     key = start_key(rsc);
     action_list = find_actions(rsc->actions, key, NULL);
     crm_free(key);
     
     crm_debug_3("%s: processing", rsc->id);
     CRM_CHECK(action_list != NULL, return FALSE);
     
     start = action_list->data;
     g_list_free(action_list);
 
     slist_iter(
 	constraint, rsc_colocation_t, rsc->rsc_cons, lpc,
 
 	resource_t *target = constraint->rsc_rh;
 	crm_debug_4("Checking %s: %s == %s (%d)", constraint->id, rsc->id, target->id, constraint->score);
 	if(constraint->score > 0) {
 	    mode |= check_stack_element(rsc, target, "coloc");
 	    if(mode & stack_middle) {
 		return FALSE;
 
 	    } else if((mode & stack_stopping) && (mode & stack_starting)) {
 		crm_notice("Cannot migrate %s due to colocation activity (last was %s)",
 			   rsc->id, target->id);
 		return FALSE;
 	    }
 	}
 	);
 
     slist_iter(
 	other_w, action_wrapper_t, start->actions_before, lpc,
 	other = other_w->action;
 
 	if(other_w->type & pe_order_serialize_only) {
 	    crm_debug_3("%s: depends on %s (serialize ordering)",
 			rsc->id, other->uuid);
 	    continue;
 	}	
 
 	crm_debug_2("%s: Checking %s ordering", rsc->id, other->uuid);
 
 	if(other->optional == FALSE) {
 	    mode |= check_stack_element(rsc, other->rsc, "order");
 	    if(mode & stack_middle) {
 		return FALSE;
 
 	    } else if((mode & stack_stopping) && (mode & stack_starting)) {
 		crm_notice("Cannot migrate %s due to ordering activity (last was %s)",
 			   rsc->id, other->rsc->id);
 		return FALSE;
 	    }
 	}
 	
 	);
 
     return TRUE;
 }
 
 void
 complex_migrate_reload(resource_t *rsc, pe_working_set_t *data_set)
 {
 	char *key = NULL;
 	int level = LOG_DEBUG;
 	GListPtr action_list = NULL;
 	
 	action_t *stop = NULL;
 	action_t *start = NULL;
 	action_t *other = NULL;
 	action_t *action = NULL;
 	const char *value = NULL;
 
 	if(rsc->children) {
 	    slist_iter(
 		child_rsc, resource_t, rsc->children, lpc,
 		
 		child_rsc->cmds->migrate_reload(child_rsc, data_set);
 		);
 	    other = NULL;
 	    return;
 	} else if(rsc->variant > pe_native) {
 	    return;
 	}
 
 	do_crm_log_unlikely(level+1, "Processing %s", rsc->id);
 	
 	if(is_not_set(rsc->flags, pe_rsc_managed)
 	   || is_set(rsc->flags, pe_rsc_failed)
 	   || is_set(rsc->flags, pe_rsc_start_pending)
 	   || rsc->next_role < RSC_ROLE_STARTED
 	   || g_list_length(rsc->running_on) != 1) {
 	    do_crm_log_unlikely(
 		level+1, "%s: general resource state: flags=0x%.16llx",
 		rsc->id, rsc->flags);
 	    return;
 	}
 
 	value = g_hash_table_lookup(rsc->meta, XML_OP_ATTR_ALLOW_MIGRATE);
 	if(crm_is_true(value)) {
 	    set_bit(rsc->flags, pe_rsc_can_migrate);	
 	}	
 
 	if(rsc->next_role > RSC_ROLE_SLAVE) {
 	    clear_bit(rsc->flags, pe_rsc_can_migrate);	
 	    do_crm_log_unlikely(
 		level+1, "%s: resource role: role=%s", rsc->id, role2text(rsc->next_role));
 	}
 	
 	key = start_key(rsc);
 	action_list = find_actions(rsc->actions, key, NULL);
 	crm_free(key);
 	
 	if(action_list == NULL) {
 		do_crm_log_unlikely(level, "%s: no start action", rsc->id);
 		return;
 	}
 	
 	start = action_list->data;
 	g_list_free(action_list);
 
 	if(is_not_set(rsc->flags, pe_rsc_can_migrate)
 	   && start->allow_reload_conversion == FALSE) {
 		do_crm_log_unlikely(level+1, "%s: no need to continue", rsc->id);
 		return;
 	}
 	
 	key = stop_key(rsc);
 	action_list = find_actions(rsc->actions, key, NULL);
 	crm_free(key);
 	
 	if(action_list == NULL) {
 		do_crm_log_unlikely(level, "%s: no stop action", rsc->id);
 		return;
 	}
 	
 	stop = action_list->data;
 	g_list_free(action_list);
 	
 	action = start;
 	if(action->pseudo
 	   || action->optional
 	   || action->node == NULL
 	   || action->runnable == FALSE) {
 		do_crm_log_unlikely(level, "%s: %s", rsc->id, action->task);
 		return;
 	}
 	
 	action = stop;
 	if(action->pseudo
 	   || action->optional
 	   || action->node == NULL
 	   || action->runnable == FALSE) {
 		do_crm_log_unlikely(level, "%s: %s", rsc->id, action->task);
 		return;
 	}
 	
 	if(is_set(rsc->flags, pe_rsc_can_migrate)) {
 	    if(start->node == NULL
 	       || stop->node == NULL
 	       || stop->node->details == start->node->details) {
 		clear_bit(rsc->flags, pe_rsc_can_migrate);
 
 	    } else if(at_stack_bottom(rsc) == FALSE) {
 		clear_bit(rsc->flags, pe_rsc_can_migrate);
 	    }
 	}
 
 	if(is_set(rsc->flags, pe_rsc_can_migrate)) {
 	    action_t *to = NULL;
 	    action_t *from = NULL;
 	    crm_info("Migrating %s from %s to %s", rsc->id,
 			 stop->node->details->uname,
 			 start->node->details->uname);
 		
 		crm_free(stop->uuid);
 		crm_free(stop->task);
 		stop->task = crm_strdup(RSC_MIGRATE);
 		stop->uuid = generate_op_key(rsc->id, stop->task, 0);
 		add_hash_param(stop->meta, "migrate_source",
 			       stop->node->details->uname);
 		add_hash_param(stop->meta, "migrate_target",
 			       start->node->details->uname);
 
 		/* Create the correct ordering ajustments based on find_clone_activity_on(); */
 		
 		slist_iter(
 		    constraint, rsc_colocation_t, rsc->rsc_cons, lpc,
 		    
 		    resource_t *target = constraint->rsc_rh;
 		    crm_info("Repairing %s: %s == %s (%d)", constraint->id, rsc->id, target->id, constraint->score);
 
 		    if(constraint->score > 0) {
 			int mode = check_stack_element(rsc, target, "coloc");
 			action_t *clone_stop = find_first_action(target->actions, NULL, RSC_STOP, NULL);
 			action_t *clone_start = find_first_action(target->actions, NULL, RSC_STARTED, NULL);
 
 			CRM_ASSERT(clone_stop != NULL);
 			CRM_ASSERT(clone_start != NULL);
 			CRM_ASSERT((mode & stack_middle) == 0);
 			CRM_ASSERT(((mode & stack_stopping) && (mode & stack_starting)) == 0);
 			
 			if(mode & stack_stopping) {
 			    action_t *clone_stop = find_first_action(target->actions, NULL, RSC_STOP, NULL);
 			    action_t *clone_start = find_first_action(target->actions, NULL, RSC_STARTED, NULL);
 			    crm_debug("Creating %s.start -> %s.stop ordering", rsc->id, target->id);
 
 			    order_actions(start, clone_stop, pe_order_optional);
 
 			    slist_iter(
 				other_w, action_wrapper_t, start->actions_before, lpc2,
 				/* Needed if the clone's started pseudo-action ever gets printed in the graph */ 
 				if(other_w->action == clone_start) {
 				    crm_debug("Breaking %s -> %s ordering", other_w->action->uuid, start->uuid);
 				    other_w->type = pe_order_none;
 				}
 				);
 			    
 			} else if(mode & stack_starting) {
 			    crm_debug("Creating %s.started -> %s.stop ordering", target->id, rsc->id);
 
 			    order_actions(clone_start, stop, pe_order_optional);
 			    
 			    slist_iter(
 				other_w, action_wrapper_t, clone_stop->actions_before, lpc2,
 				/* Needed if the clone's stop pseudo-action ever gets printed in the graph */ 
 				if(other_w->action == stop) {
 				    crm_debug("Breaking %s -> %s ordering", other_w->action->uuid, clone_stop->uuid);
 				    other_w->type = pe_order_none;
 				}
 				);
 			}
 		    }
 		    );
 
 		crm_free(start->uuid);
 		crm_free(start->task);
 		start->task = crm_strdup(RSC_MIGRATED);
 		start->uuid = generate_op_key(rsc->id, start->task, 0);
 		add_hash_param(start->meta, "migrate_source_uuid", stop->node->details->id);
 		add_hash_param(start->meta, "migrate_source", stop->node->details->uname);
 		add_hash_param(start->meta, "migrate_target", start->node->details->uname);
 
 		/* Anything that needed stop to complete, now also needs start to have completed */
  		slist_iter(
 			other_w, action_wrapper_t, stop->actions_after, lpc,
 			other = other_w->action;
 			if(other->optional || other->rsc != NULL) {
 			    continue;
 			}
 			crm_debug("Ordering %s before %s (stop)", start->uuid, other_w->action->uuid);
 			order_actions(start, other, other_w->type);
 		    );
 
 		/* Stop also needs anything that the start needed to have completed too */
  		slist_iter(
 			other_w, action_wrapper_t, start->actions_before, lpc,
 			other = other_w->action;
 			if(other->rsc == NULL) {
 			    /* nothing */
 			} else if(other->optional || other->rsc == rsc || other->rsc == rsc->parent) {
 			    continue;
 			}
 			crm_debug("Ordering %s before %s (start)", other_w->action->uuid, stop->uuid);
 			order_actions(other, stop, other_w->type);
 		    );
 
 		/* Overwrite any op-specific params with those from the migrate ops */
 		from = custom_action(
 		    rsc, crm_strdup(start->uuid), start->task, start->node,
 		    FALSE, FALSE, data_set);
 		g_hash_table_foreach(start->meta, append_hashtable, from->meta);
 		g_hash_table_destroy(start->meta);
 		start->meta = from->meta;
 		from->meta = NULL;
 		pe_free_action(from);
 
 		to = custom_action(
 		    rsc, crm_strdup(stop->uuid), stop->task, stop->node,
 		    FALSE, FALSE, data_set);
 		g_hash_table_foreach(stop->meta, append_hashtable, to->meta);
 		g_hash_table_destroy(stop->meta);
 		stop->meta = to->meta;
 		to->meta = NULL;
 		pe_free_action(to);
 		
 	} else if(start && stop
 		  && start->allow_reload_conversion
 		  && stop->node->details == start->node->details) {
 		action_t *rewrite = NULL;
 
 		start->pseudo = TRUE; /* easier than trying to delete it from the graph */
 
 		action = NULL;
 		key = promote_key(rsc);
 		action_list = find_actions(rsc->actions, key, NULL);
 		if(action_list) {
 		    action = action_list->data;
 		}
 		if(action && action->optional == FALSE) {
 		    action->pseudo = TRUE;
 		}
 		g_list_free(action_list);
 		crm_free(key);
 
 		action = NULL;
 		key = demote_key(rsc);
 		action_list = find_actions(rsc->actions, key, NULL);
 		if(action_list) {
 		    action = action_list->data;
 		}
 		g_list_free(action_list);
 		crm_free(key);
 
 		if(action && action->optional == FALSE) {
 		    rewrite = action;
 		    stop->pseudo = TRUE;
 		    
 		} else {
 		    rewrite = stop;
 		}
 		crm_info("Rewriting %s of %s on %s as a reload",
 			 rewrite->task, rsc->id, stop->node->details->uname);
 		
 		crm_free(rewrite->uuid);
 		crm_free(rewrite->task);
 		rewrite->task = crm_strdup("reload");
 		rewrite->uuid = generate_op_key(rsc->id, rewrite->task, 0);
 		
 	} else {
 		do_crm_log_unlikely(level+1, "%s nothing to do", rsc->id);
 	}
 }
 
 void native_append_meta(resource_t *rsc, xmlNode *xml)
 {
     char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION);
     if(value) {
 	char *name = NULL;
 	name = crm_meta_name(XML_RSC_ATTR_INCARNATION);
 	crm_xml_add(xml, name, value);
 	crm_free(name);
     }
 }
 
diff --git a/tools/cib_shadow.c b/tools/cib_shadow.c
index 848c7376c0..9a12bedfc7 100644
--- a/tools/cib_shadow.c
+++ b/tools/cib_shadow.c
@@ -1,561 +1,561 @@
 
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 
 
 
 
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 
 #include <crm/common/ipc.h>
 
 #include <crm/cib.h>
 
 int exit_code = cib_ok;
 GMainLoop *mainloop = NULL;
 IPC_Channel *crmd_channel = NULL;
 
 const char *host = NULL;
 void usage(const char *cmd, int exit_status);
 
 int command_options = cib_sync_call;
 const char *cib_action = NULL;
 
 cib_t *real_cib = NULL;
 
 int dump_data_element(
     int depth, char **buffer, int *max, int *offset, const char *prefix, xmlNode *data, gboolean formatted);
 
 void print_xml_diff(FILE *where, xmlNode *diff);
 
 static int force_flag = 0;
 static int batch_flag = 0;
 
 static char *get_shadow_prompt(const char *name)
 {
     int len = 16;
     char *prompt = NULL;
     CRM_ASSERT(name != NULL);
     
     len += strlen(name);
     crm_malloc0(prompt, len);
     
     snprintf(prompt, len, "shadow[%s] # ", name);
     return prompt;
 }
 
 
 static void shadow_setup(char *name, gboolean do_switch)
 {
     const char *prompt = getenv("PS1");
     const char *shell = getenv("SHELL");
     char *new_prompt = get_shadow_prompt(name);
     printf("Setting up shadow instance\n");
 
     if(safe_str_eq(new_prompt, prompt)) {
 	/* nothing to do */
 	goto done;
 	
     } else if(batch_flag == FALSE && shell != NULL) {
 	setenv("PS1", new_prompt, 1);
 	setenv("CIB_shadow", name, 1);
 	printf("Type Ctrl-D to exit the crm_shadow shell\n");
 	
 	execl(shell, "--norc", "--noprofile", NULL);
 	
     } else if (do_switch) {
 	printf("To switch to the named shadow instance, paste the following into your shell:\n");
 
     } else {
 	printf("A new shadow instance was created.  To begin using it paste the following into your shell:\n");
     }
     printf("  CIB_shadow=%s ; export CIB_shadow\n", name);
 
   done:
     crm_free(new_prompt);
 }
 
 static void shadow_teardown(char *name)
 {
     const char *prompt = getenv("PS1");
     char *our_prompt = get_shadow_prompt(name);
     
     if(prompt != NULL && strstr(prompt, our_prompt)) {
 	printf("Now type Ctrl-D to exit the crm_shadow shell\n");
 	
     } else {
 	printf("Please remember to unset the CIB_shadow variable by pasting the following into your shell:\n");
 	printf("  unset CIB_shadow\n");
     }
     crm_free(our_prompt);
 }
 
 static struct crm_option long_options[] = {
     /* Top-level Options */
     {"help",    0, 0, '?', "\t\tThis text"},
     {"version", 0, 0, '$', "\t\tVersion information"  },
     {"verbose", 0, 0, 'V', "\t\tIncrease debug output"},
     
     {"-spacer-",	1, 0, '-', "\nQueries:"},
     {"which",   no_argument,       NULL, 'w', "\t\tIndicate the active shadow copy"},
     {"display", no_argument,       NULL, 'p', "\t\tDisplay the contents of the active shadow copy"},
     {"edit",    no_argument,       NULL, 'E', "\t\tEdit the contents of the active shadow copy with your favorite $EDITOR"},
     {"diff",    no_argument,       NULL, 'd', "\t\tDisplay the changes in the active shadow copy\n"},
     {"file",    no_argument,       NULL, 'F', "\t\tDisplay the location of the active shadow copy file\n"},
 
     {"-spacer-",	1, 0, '-', "\nCommands:"},
     {"create",		required_argument, NULL, 'c', "\tCreate the named shadow copy of the active cluster configuration"},
     {"create-empty",	required_argument, NULL, 'e', "Create the named shadow copy with an empty cluster configuration"},
     {"commit",  required_argument, NULL, 'C', "\tUpload the contents of the named shadow copy to the cluster"},
     {"delete",  required_argument, NULL, 'D', "\tDelete the contents of the named shadow copy"},
     {"reset",   required_argument, NULL, 'r', "\tRecreate the named shadow copy from the active cluster configuration"},
     {"switch",  required_argument, NULL, 's', "\t(Advanced) Switch to the named shadow copy"},
     
     {"-spacer-",	1, 0, '-', "\nAdditional Options:"},
     {"force",	no_argument, NULL, 'f', "\t\t(Advanced) Force the action to be performed"},
     {"batch",   no_argument, NULL, 'b', "\t\t(Advanced) Don't spawn a new shell" },
 
     {"-spacer-",	1, 0, '-', "\nExamples:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', "Create a blank shadow configuration:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_shadow --create-empty myShadow", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Create a shadow configuration from the running cluster:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_shadow --create myShadow", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Display the current shadow configuration:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_shadow --display", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Discard the current shadow configuration (named myShadow):", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_shadow --delete myShadow", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Upload the current shadow configuration (named myShadow) to the running cluster:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_shadow --commit myShadow", pcmk_option_example},
     
     {0, 0, 0, 0}
 };
 
 int
 main(int argc, char **argv)
 {
     int rc = 0;
     int flag;
     int argerr = 0;
     static int command = '?';
     char *shadow = NULL;
     char *shadow_file = NULL;
     gboolean dangerous_cmd = FALSE;
     struct stat buf;
     int option_index = 0;
 
     crm_log_init("crm_shadow", LOG_CRIT, FALSE, FALSE, argc, argv);
     crm_set_options("V$?bfwc:dr:C:D:ps:Ee:F", "(query|command) [modifiers]", long_options,
 		    "Perform configuration changes in a sandbox before updating the live cluster."
 		    "\n\nSets up an environment in which configuration tools (cibadmin, crm_resource, etc) work"
 		    " offline instead of against a live cluster, allowing changes to be previewed and tested"
 		    " for side-effects.\n");
     
     if(argc < 2) {
 	crm_help('?', LSB_EXIT_EINVAL);
     }
 
     while (1) {
 	flag = crm_get_option(argc, argv, &option_index);
 	if (flag == -1 || flag == 0)
 	    break;
 
 	switch(flag) {
 	    case 'd':
 	    case 'E':
 	    case 'p':
 	    case 'w':
 	    case 'F':
 		command = flag;
 		shadow = crm_strdup(getenv("CIB_shadow"));
 		break;
 	    case 'e':
 	    case 'c':
 	    case 's':
 	    case 'r':
 		command = flag;
 		shadow = crm_strdup(optarg);
 		break;
 	    case 'C':
 	    case 'D':
 		command = flag;
 		dangerous_cmd = TRUE;
 		shadow = crm_strdup(optarg);
 		break;
 	    case 'V':
 		command_options = command_options | cib_verbose;
 		cl_log_enable_stderr(TRUE);
 		alter_debug(DEBUG_INC);
 		break;
 	    case '$':
 	    case '?':
 		crm_help(flag, LSB_EXIT_OK);
 		break;
 	    case 'f':
 		command_options |= cib_quorum_override;
 		force_flag = 1;
 		break;
 	    case 'b':
 		batch_flag = 1;
 		break;
 	    default:
 		printf("Argument code 0%o (%c)"
 		       " is not (?yet?) supported\n",
 		       flag, flag);
 		++argerr;
 		break;
 	}
     }
 
     if (optind < argc) {
 	printf("non-option ARGV-elements: ");
 	while (optind < argc)
 	    printf("%s ", argv[optind++]);
 	printf("\n");
 	crm_help('?', LSB_EXIT_EINVAL);
     }
 
     if (optind > argc) {
 	++argerr;
     }
 	
     if (argerr) {
 	crm_help('?', LSB_EXIT_GENERIC);
     }
 
     if(command == 'w') {
 	/* which shadow instance is active? */
 	const char *local = getenv("CIB_shadow");
 	if(local == NULL) {
 	    fprintf(stderr, "No shadow instance provided\n");
 	    return cib_NOTEXISTS;
 	}
 	fprintf(stdout, "%s\n", local);
 	return 0;
     }
     
     if(shadow == NULL) {
 	fprintf(stderr, "No shadow instance provided\n");
 	fflush(stderr);
 	return CIBRES_MISSING_FIELD;
 
     } else if(command != 's' && command != 'c') {
 	const char *local = getenv("CIB_shadow");
 	if(local != NULL && safe_str_neq(local, shadow) && force_flag == FALSE) {
 	    fprintf(stderr, "The supplied shadow instance (%s) is not the same as the active one (%s).\n"
 		    "  To prevent accidental destruction of the cluster,"
 		    " the --force flag is required in order to proceed.\n", shadow, local);
 	    fflush(stderr);
 	    exit(LSB_EXIT_GENERIC);
 	}
     }
 
     if(dangerous_cmd && force_flag == FALSE) {
 	fprintf(stderr, "The supplied command is considered dangerous."
 		"  To prevent accidental destruction of the cluster,"
 		" the --force flag is required in order to proceed.\n");
 	fflush(stderr);
 	exit(LSB_EXIT_GENERIC);
     }
 
     shadow_file = get_shadow_file(shadow);
     if(command == 'D') {
 	/* delete the file */
 	rc = stat(shadow_file, &buf);
 	if(rc == 0) {
 	    rc = unlink(shadow_file);
 	    if(rc != 0) {
 		fprintf(stderr, "Could not remove shadow instance '%s': %s\n", shadow, strerror(errno));
 		return rc;
 	    }
 	}
 
 	shadow_teardown(shadow);
 	return rc;
 
     } else if(command == 'F') {
 	printf("%s\n", shadow_file);
 	return 0;
     }
 
     if(command == 'd' || command == 'r' || command == 'c' || command == 'C') {
 	real_cib = cib_new_no_shadow();
 	rc = real_cib->cmds->signon(real_cib, crm_system_name, cib_command);
 	if(rc != cib_ok) {
 	    fprintf(stderr, "Signon to CIB failed: %s\n", cib_error2string(rc));
 	    return rc;
 	}
     }
     
     rc = stat(shadow_file, &buf);
 
     if(command == 'e' || command == 'c') {
 	if (rc == 0 && force_flag == FALSE) {
 	    fprintf(stderr, "A shadow instance '%s' already exists.\n"
 		   "  To prevent accidental destruction of the cluster,"
 		   " the --force flag is required in order to proceed.\n", shadow);
 	    return cib_EXISTS;
 	}
 
     } else if(rc != 0) {
 	fprintf(stderr, "Could not access shadow instance '%s': %s\n", shadow, strerror(errno));
 	return cib_NOTEXISTS;
     }
 
     rc = cib_ok;
     if(command == 'c' || command == 'e') {
 	xmlNode *output = NULL;
 
 	/* create a shadow instance based on the current cluster config */
 	if(command == 'c') {
 	    rc = real_cib->cmds->query(real_cib, NULL, &output, command_options);
 	    if(rc != cib_ok) {
 		fprintf(stderr, "Could not connect to the CIB: %s\n", cib_error2string(rc));
 		return rc;
 	    }
 
 	} else {
 	    output = createEmptyCib();
 	    crm_xml_add(output, XML_ATTR_GENERATION, "0");
 	    crm_xml_add(output, XML_ATTR_NUMUPDATES, "0");
 	    crm_xml_add(output, XML_ATTR_GENERATION_ADMIN, "0");
 	    crm_xml_add(output, XML_ATTR_VALIDATION, LATEST_SCHEMA_VERSION);
 	}
 	
 	rc = write_xml_file(output, shadow_file, FALSE);
 	if(rc < 0) {
 	    fprintf(stderr, "Could not create the shadow instance '%s': %s\n",
 		    shadow, strerror(errno));
 	    return rc;
 	}
 	shadow_setup(shadow, FALSE);
 	rc = cib_ok;
 	
     } else if(command == 'E') {
 	const char *err = NULL;
 	char *editor = getenv("EDITOR");
 	if(editor == NULL) {
 	    fprintf(stderr, "No value for $EDITOR defined\n");
 	    return cib_missing;
 	}
 
 	execlp(editor, "--", shadow_file, NULL);
 	err = strerror(errno);
-	fprintf(stderr, "Could not invoke $EDITOR (%s %s)\n", editor, shadow_file);
+	fprintf(stderr, "Could not invoke $EDITOR (%s %s): %s\n", editor, shadow_file, err);
 	return cib_missing;
 	
     } else if(command == 's') {
 	shadow_setup(shadow, TRUE);
 	return 0;
     
     } else if(command == 'P') {
 	/* display the current contents */
 	char *output_s = NULL;
 	xmlNode *output = filename2xml(shadow_file);
 	
 	output_s = dump_xml_formatted(output);
 	printf("%s", output_s);
 	
 	crm_free(output_s);
 	free_xml(output);
 	
     } else if(command == 'd') {
 	/* diff against cluster */
 	xmlNode *diff = NULL;
 	xmlNode *old_config = NULL;
 	xmlNode *new_config = filename2xml(shadow_file);
 	
 	rc = real_cib->cmds->query(real_cib, NULL, &old_config, command_options);
 	
 	if(rc != cib_ok) {
 	    fprintf(stderr, "Could not query the CIB: %s\n", cib_error2string(rc));
 	    return rc;
 	}
 
 	diff = diff_xml_object(old_config, new_config, FALSE);
 	if(diff != NULL) {
 	    print_xml_diff(stdout, diff);
 	    return 1;
 	}
 	return 0;	
 	
     } else if(command == 'C') {
 	/* commit to the cluster */
 	xmlNode *input = filename2xml(shadow_file);
 	rc = real_cib->cmds->replace(real_cib, NULL, input, command_options);
 	if(rc != cib_ok) {
 	    fprintf(stderr, "Could not commit shadow instance '%s' to the CIB: %s\n",
 		    shadow, cib_error2string(rc));
 	    return rc;
 	}	
 	shadow_teardown(shadow);
     }
 
     return rc;
 }
 
 #define bhead(buffer, offset) ((*buffer) + (*offset)) 
 #define bremain(max, offset) ((*max) - (*offset)) 
 #define update_buffer_head(len) do {					\
 	int total = (*offset) + len + 1;				\
 	if(total >= (*max)) { /* too late */				\
 	    (*buffer) = EOS; return -1;					\
 	} else if(((*max) - total) < 256) {				\
 	    (*max) *= 10;						\
 	    crm_realloc(*buffer, (*max));				\
 	}								\
 	(*offset) += len;						\
     } while(0)
 
 extern int print_spaces(char *buffer, int depth, int max);
 
 int
 dump_data_element(
     int depth, char **buffer, int *max, int *offset, const char *prefix, xmlNode *data, gboolean formatted) 
 {
     int printed = 0;
     int has_children = 0;
     const char *name = NULL;
     
     CRM_CHECK(data != NULL, return 0);
     
     name = crm_element_name(data);
     
     CRM_CHECK(name != NULL, return 0);
     CRM_CHECK(buffer != NULL && *buffer != NULL, return 0);
     
     crm_debug_5("Dumping %s...", name);
 
     if(prefix) {
 	printed = snprintf(bhead(buffer, offset), bremain(max, offset), "%s", prefix);
 	update_buffer_head(printed);
     }
 
     if(formatted) {
 	printed = print_spaces(bhead(buffer, offset), depth, bremain(max, offset));
 	update_buffer_head(printed);
     }
     
     printed = snprintf(bhead(buffer, offset), bremain(max, offset), "<%s", name);
     update_buffer_head(printed);
     
     xml_prop_iter(data, prop_name, prop_value,
 		  crm_debug_5("Dumping <%s %s=\"%s\"...",
 			      name, prop_name, prop_value);
 		  printed = snprintf(bhead(buffer, offset), bremain(max, offset),  " %s=\"%s\"", prop_name, prop_value);
 		  update_buffer_head(printed);
 	);
     
     has_children = xml_has_children(data);
     printed = snprintf(bhead(buffer, offset), bremain(max, offset),  "%s>%s",
 		       has_children==0?"/":"", formatted?"\n":"");
     update_buffer_head(printed);
     
     if(has_children == 0) {
 	return 0;
     }
     
     xml_child_iter(data, child, 
 		   if(dump_data_element(depth+1, buffer, max, offset, prefix, child, formatted) < 0) {
 		       return -1;
 		   }
 	);
     
     if(prefix) {
 	printed = snprintf(bhead(buffer, offset), bremain(max, offset), "%s", prefix);
 	update_buffer_head(printed);
     }
 
     if(formatted) {
 	printed = print_spaces(bhead(buffer, offset), depth, bremain(max, offset));
 	update_buffer_head(printed);
     }
     
     printed = snprintf(bhead(buffer, offset), bremain(max, offset),  "</%s>%s", name, formatted?"\n":"");
     update_buffer_head(printed);
     crm_debug_5("Dumped %s...", name);
     
     return has_children;
 }
 
 void
 print_xml_diff(FILE *where, xmlNode *diff)
 {
 	char *buffer = NULL;
 	int max = 1024, len = 0;
 	gboolean is_first = TRUE;
 	xmlNode *added = find_xml_node(diff, "diff-added", FALSE);
 	xmlNode *removed = find_xml_node(diff, "diff-removed", FALSE);
 
 	is_first = TRUE;
 	xml_child_iter(
 		removed, child, 
 
 		len = 0;
 		max = 1024;
 		crm_free(buffer);
 		crm_malloc0(buffer, max);
 
 		if(is_first) {
 		    is_first = FALSE;
 		} else {
 		    fprintf(where, " --- \n");
 		}
 
 		CRM_CHECK(dump_data_element(
 			      0, &buffer, &max, &len, "-", child, TRUE) >= 0,
 			  continue);
 		fprintf(where, "%s", buffer);
 		);
 
 	is_first = TRUE;
 	xml_child_iter(
 		added, child, 
 
 		len = 0;
 		max = 1024;
 		crm_free(buffer);
 		crm_malloc0(buffer, max);
 
 		if(is_first) {
 		    is_first = FALSE;
 		} else {
 		    fprintf(where, " +++ \n");
 		}
 
 		CRM_CHECK(dump_data_element(
 			      0, &buffer, &max, &len, "+", child, TRUE) >= 0,
 			  continue);
 		fprintf(where, "%s", buffer);
 		);
 }
 
diff --git a/tools/crm_inject.c b/tools/crm_inject.c
index 3fc5cf33c8..2e55ae0b77 100644
--- a/tools/crm_inject.c
+++ b/tools/crm_inject.c
@@ -1,1150 +1,1149 @@
 /* 
  * Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <stdio.h>
 #include <unistd.h>
 #include <stdlib.h>
 
 #include <sys/param.h>
 #include <sys/types.h>
 
 #include <crm/crm.h>
 #include <crm/cib.h>
 #include <crm/common/util.h>
 #include <crm/transition.h>
 #include <crm/common/iso8601.h>
 #include <crm/pengine/status.h>
 #include <allocate.h>
 
 cib_t *global_cib = NULL;
 GListPtr op_fail = NULL;
 gboolean quiet = FALSE;
  
 #define new_node_template "//"XML_CIB_TAG_NODE"[@uname='%s']"
 #define node_template "//"XML_CIB_TAG_STATE"[@uname='%s']"
 #define rsc_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']"
 #define op_template  "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s']"
 /* #define op_template  "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s' and @"XML_LRM_ATTR_CALLID"='%d']" */
 
 #define FAKE_TE_ID	"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
 
 #define quiet_log(fmt, args...) do {	\
 	if(quiet == FALSE) {		\
 	    printf(fmt , ##args);	\
 	}				\
     } while(0)
 
 extern void cleanup_alloc_calculations(pe_working_set_t *data_set);
 
 extern xmlNode * do_calculations(
     pe_working_set_t *data_set, xmlNode *xml_input, ha_time_t *now);
 
 static xmlNode *find_resource(xmlNode *cib_node, const char *resource)
 {
     char *xpath = NULL;
     xmlNode *match = NULL;
     const char *node = crm_element_value(cib_node, XML_ATTR_UNAME);
     int max = strlen(rsc_template) + strlen(resource) + strlen(node) + 1;
     crm_malloc0(xpath, max);
     
     snprintf(xpath, max, rsc_template, node, resource);
     match = get_xpath_object(xpath, cib_node, LOG_DEBUG_2);
     
     crm_free(xpath);
     return match;
 }
 
 static void create_node_entry(cib_t *cib_conn, char *node)
 {
     int rc = cib_ok;
     int max = strlen(new_node_template) + strlen(node) + 1;
     char *xpath = NULL;
     crm_malloc0(xpath, max);
 
     snprintf(xpath, max, new_node_template, node);
     rc = cib_conn->cmds->query(cib_conn, xpath, NULL, cib_xpath|cib_sync_call|cib_scope_local);
 
     if (rc == cib_NOTEXISTS) {
 	xmlNode *cib_object = create_xml_node(NULL, XML_CIB_TAG_NODE);
 
 	/* Using node uname as uuid ala corosync/openais */
 	crm_xml_add(cib_object, XML_ATTR_ID,    node);
 	crm_xml_add(cib_object, XML_ATTR_UNAME, node);
 	crm_xml_add(cib_object, XML_ATTR_TYPE,  NORMALNODE);
 	cib_conn->cmds->create(cib_conn, XML_CIB_TAG_NODES, cib_object, cib_sync_call|cib_scope_local);
 	/* Not bothering with subsequent query to see if it exists,
 	   we'll bomb out later in the call to determine_host... */
 
 	free_xml(cib_object);
     }
 
     crm_free(xpath);
 }
 
 static xmlNode *inject_node_state(cib_t *cib_conn, char *node)
 {
     int rc = cib_ok;
     int max = strlen(rsc_template) + strlen(node) + 1;
     char *xpath = NULL;
     xmlNode *cib_object = NULL;
     crm_malloc0(xpath, max);
 
     create_node_entry(cib_conn, node);
     
     snprintf(xpath, max, node_template, node);
     rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object, cib_xpath|cib_sync_call|cib_scope_local);
 
     if(rc == cib_NOTEXISTS) {
 	char *uuid = NULL;
 
 	cib_object = create_xml_node(NULL, XML_CIB_TAG_STATE);	
 	determine_host(cib_conn, &node, &uuid);
 	crm_xml_add(cib_object, XML_ATTR_UUID,  uuid);
 	crm_xml_add(cib_object, XML_ATTR_UNAME, node);
 	cib_conn->cmds->create(cib_conn, XML_CIB_TAG_STATUS, cib_object, cib_sync_call|cib_scope_local);
 	free_xml(cib_object);
 	crm_free(uuid);
 	
 	rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object, cib_xpath|cib_sync_call|cib_scope_local);
     }
     
     crm_free(xpath);
     CRM_ASSERT(rc == cib_ok);
     return cib_object;
 }
 
 static xmlNode *modify_node(cib_t *cib_conn, char *node, gboolean up) 
 {
     xmlNode *cib_node = inject_node_state(cib_conn, node);
     if(up) {
 	crm_xml_add(cib_node, XML_CIB_ATTR_HASTATE,   ACTIVESTATUS);
 	crm_xml_add(cib_node, XML_CIB_ATTR_INCCM,     XML_BOOLEAN_YES);
 	crm_xml_add(cib_node, XML_CIB_ATTR_CRMDSTATE, ONLINESTATUS);
 	crm_xml_add(cib_node, XML_CIB_ATTR_JOINSTATE, CRMD_JOINSTATE_MEMBER);
 	crm_xml_add(cib_node, XML_CIB_ATTR_EXPSTATE,  CRMD_JOINSTATE_MEMBER);
 
     } else {
 	crm_xml_add(cib_node, XML_CIB_ATTR_HASTATE,   DEADSTATUS);
 	crm_xml_add(cib_node, XML_CIB_ATTR_INCCM,     XML_BOOLEAN_NO);
 	crm_xml_add(cib_node, XML_CIB_ATTR_CRMDSTATE, OFFLINESTATUS);
 	crm_xml_add(cib_node, XML_CIB_ATTR_JOINSTATE, CRMD_JOINSTATE_DOWN);
 	crm_xml_add(cib_node, XML_CIB_ATTR_EXPSTATE,  CRMD_JOINSTATE_DOWN);
     }
     
     crm_xml_add(cib_node, XML_ATTR_ORIGIN, crm_system_name);
     return cib_node;
 }
 
 static void inject_transient_attr(xmlNode *cib_node, const char *name, const char *value)
 {
     xmlNode *attrs = NULL;
     xmlNode *container = NULL;
     xmlNode *nvp = NULL;
     const char *node_uuid = ID(cib_node);
     char *nvp_id = crm_concat(name, node_uuid, '-');
 
     crm_info("Injecting attribute %s=%s into %s '%s'", name, value, xmlGetNodePath(cib_node), ID(cib_node));
     
     attrs = first_named_child(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
     if(attrs == NULL) {
 	attrs = create_xml_node(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
 	crm_xml_add(attrs, XML_ATTR_ID, node_uuid);
     }
     
     container = first_named_child(attrs, XML_TAG_ATTR_SETS);
     if(container == NULL) {
 	container = create_xml_node(attrs, XML_TAG_ATTR_SETS);
 	crm_xml_add(container, XML_ATTR_ID, node_uuid);
     }
 
     nvp = create_xml_node(container, XML_CIB_TAG_NVPAIR);
     crm_xml_add(nvp, XML_ATTR_ID, nvp_id);
     crm_xml_add(nvp, XML_NVPAIR_ATTR_NAME, name);
     crm_xml_add(nvp, XML_NVPAIR_ATTR_VALUE, value);
 
     crm_free(nvp_id);
 }
 
 static xmlNode *inject_resource(xmlNode *cib_node, const char *resource, const char *rclass, const char *rtype, const char *rprovider)
 {
     xmlNode *lrm = NULL;
     xmlNode *container = NULL;
     xmlNode *cib_resource = NULL;
 
     cib_resource = find_resource(cib_node, resource);
     if(cib_resource != NULL) {
 	return cib_resource;
     }
     
     /* One day, add query for class, provider, type */
     
     if(rclass == NULL || rtype == NULL) {
 	fprintf(stderr, "Resource %s not found in the status section of %s."
 		"  Please supply the class and type to continue\n", resource, ID(cib_node));
 	return NULL;
 	
     } else if(safe_str_neq(rclass, "ocf") 
 	      && safe_str_neq(rclass, "stonith")
 	      && safe_str_neq(rclass, "heartbeat")
 	      && safe_str_neq(rclass, "lsb")) {
 	fprintf(stderr, "Invalid class for %s: %s\n", resource, rclass);
 	return NULL;
 	
     } else if(safe_str_eq(rclass, "ocf") && rprovider == NULL) {
 	fprintf(stderr, "Please specify the provider for resource %s\n", resource);
 	return NULL;
     }
 
     crm_info("Injecting new resource %s into %s '%s'", resource, xmlGetNodePath(cib_node), ID(cib_node));
     
     lrm = first_named_child(cib_node, XML_CIB_TAG_LRM);
     if(lrm == NULL) {
 	const char *node_uuid = ID(cib_node);
 	lrm = create_xml_node(cib_node, XML_CIB_TAG_LRM);
 	crm_xml_add(lrm, XML_ATTR_ID, node_uuid);
     }
     
     container = first_named_child(lrm, XML_LRM_TAG_RESOURCES);
     if(container == NULL) {
 	container = create_xml_node(lrm, XML_LRM_TAG_RESOURCES);
     }
     
     cib_resource = create_xml_node(container, XML_LRM_TAG_RESOURCE);
     crm_xml_add(cib_resource, XML_ATTR_ID, resource);
     
     crm_xml_add(cib_resource, XML_AGENT_ATTR_CLASS, rclass);
     crm_xml_add(cib_resource, XML_AGENT_ATTR_PROVIDER, rprovider);
     crm_xml_add(cib_resource, XML_ATTR_TYPE, rtype);
     
     return cib_resource;
 }
 
 static lrm_op_t *create_op(
     xmlNode *cib_resource, const char *task, int interval, int outcome)
 {
     lrm_op_t *op = NULL;
     crm_malloc0(op, sizeof(lrm_op_t));
     
     op->app_name = crm_strdup(crm_system_name);
 
     op->rsc_id = crm_strdup(ID(cib_resource));
     op->interval = interval;
     op->op_type = crm_strdup(task);
 
     op->rc = outcome;
     op->op_status = 0;
     op->params = NULL; /* TODO: Fill me in */
 
     op->call_id = 0;
     xml_child_iter(cib_resource, xop,
 		   int tmp = 0;
 		   crm_element_value_int(xop, XML_LRM_ATTR_CALLID, &tmp);
 		   if(tmp > op->call_id) {
 		       op->call_id = tmp;
 		   }
 	);
     op->call_id++;
     
     return op;
 }
 
 static xmlNode *inject_op(xmlNode *cib_resource, lrm_op_t *op, int target_rc)
 {
     return create_operation_update(cib_resource, op, CRM_FEATURE_SET, target_rc, crm_system_name, LOG_DEBUG_2);
 }
 
 static void update_failcounts(xmlNode *cib_node, const char *resource, int interval, int rc) 
 {
     if(rc == 0) {
 	return;
 
     } else if(rc == 7 && interval == 0) {
 	return;
 
     } else {
 	char *name = NULL;
 	char *now = crm_itoa(time(NULL));
 	
 	name = crm_concat("fail-count", resource, '-');
 	inject_transient_attr(cib_node, name, "value++");
 	
 	name = crm_concat("last-failure", resource, '-');
 	inject_transient_attr(cib_node, name, now);
 	
 	crm_free(name);
 	crm_free(now);
     }
 }
 
 static gboolean exec_pseudo_action(crm_graph_t *graph, crm_action_t *action) 
 {
     action->confirmed = TRUE;
     update_graph(graph, action);
     return TRUE;
 }
 
 static gboolean exec_rsc_action(crm_graph_t *graph, crm_action_t *action) 
 {
     int rc = 0;
     lrm_op_t *op = NULL;
     int target_outcome = 0;
 
     const char *rtype = NULL;
     const char *rclass = NULL;
     const char *resource = NULL;
     const char *rprovider = NULL;
     const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);
     
-    xmlNode *cib_op = NULL;
     xmlNode *cib_node = NULL;
     xmlNode *cib_resource = NULL;
     xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
     
     char *node = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
 
     if(safe_str_eq(crm_element_value(action->xml, "operation"), "probe_complete")) {
 	crm_info("Skipping %s op for %s\n", crm_element_value(action->xml, "operation"), node);
 	goto done;
     }
     
     if(action_rsc == NULL) {
 	crm_log_xml_err(action->xml, "Bad");
 	crm_free(node);
 	return FALSE;
     }
     
     resource = ID(action_rsc);
     rclass = crm_element_value(action_rsc, XML_AGENT_ATTR_CLASS);
     rtype = crm_element_value(action_rsc, XML_ATTR_TYPE);
     rprovider = crm_element_value(action_rsc, XML_AGENT_ATTR_PROVIDER);
 
     if(target_rc_s != NULL) {
 	target_outcome = crm_parse_int(target_rc_s, "0");
     }
 
     CRM_ASSERT(global_cib->cmds->query(global_cib, NULL, NULL, cib_sync_call|cib_scope_local) == cib_ok);
 
     cib_node = inject_node_state(global_cib, node);
     CRM_ASSERT(cib_node != NULL);
 
     cib_resource = inject_resource(cib_node, resource, rclass, rtype, rprovider);
     CRM_ASSERT(cib_resource != NULL);
 
     op = convert_graph_action(cib_resource, action, 0, target_outcome);
     quiet_log(" * Executing action %d: %s_%s_%d on %s\n", action->id, resource, op->op_type, op->interval, node);
 
     slist_iter(spec, char, op_fail, lpc,
 	       
 	       char *key = NULL;	       
 	       crm_malloc0(key, strlen(spec));
 	       snprintf(key, strlen(spec), "%s_%s_%d@%s=", resource, op->op_type, op->interval, node);
 
 	       if(strncasecmp(key, spec, strlen(key)) == 0) {
 		   rc = sscanf(spec, "%*[^=]=%d", &op->rc);
 		   
 		   action->failed = TRUE;
 		   graph->abort_priority = INFINITY;
 		   printf("\tPretending action %d failed with rc=%d\n", action->id, op->rc);
 		   update_failcounts(cib_node, resource, op->interval, op->rc);
 		   break;
 	       }
 	);
 	
-    cib_op = inject_op(cib_resource, op, target_outcome);
+    inject_op(cib_resource, op, target_outcome);
     crm_free(op->user_data);
     crm_free(op->output);
     crm_free(op->rsc_id);
     crm_free(op->op_type);
     crm_free(op->app_name);
     crm_free(op);	
     
     rc = global_cib->cmds->modify(global_cib, XML_CIB_TAG_STATUS, cib_node, cib_sync_call|cib_scope_local);
     CRM_ASSERT(rc == cib_ok);
 
   done:
     crm_free(node);
     free_xml(cib_node);
     action->confirmed = TRUE;
     update_graph(graph, action);
     return TRUE;
 }
 
 static gboolean exec_crmd_action(crm_graph_t *graph, crm_action_t *action)
 {
     action->confirmed = TRUE;
     update_graph(graph, action);
     return TRUE;
 }
 
 #define STATUS_PATH_MAX 512
 static gboolean exec_stonith_action(crm_graph_t *graph, crm_action_t *action)
 {
     int rc = 0;
     char xpath[STATUS_PATH_MAX];
     char *target = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
     xmlNode *cib_node = modify_node(global_cib, target, FALSE);	       
     crm_xml_add(cib_node, XML_ATTR_ORIGIN, __FUNCTION__);
     CRM_ASSERT(cib_node != NULL);
 
     quiet_log(" * Fencing %s\n", target);
     rc = global_cib->cmds->replace(global_cib, XML_CIB_TAG_STATUS, cib_node, cib_sync_call|cib_scope_local);
     CRM_ASSERT(rc == cib_ok);
 
     snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target, XML_CIB_TAG_LRM);
     rc = global_cib->cmds->delete(global_cib, xpath, NULL, cib_xpath|cib_sync_call|cib_scope_local);
 
     snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target, XML_TAG_TRANSIENT_NODEATTRS);
     rc = global_cib->cmds->delete(global_cib, xpath, NULL, cib_xpath|cib_sync_call|cib_scope_local);
     
     action->confirmed = TRUE;
     update_graph(graph, action);
     free_xml(cib_node);
     crm_free(target);
     return TRUE;
 }
 
 static char *
 add_list_element(char *list, const char *value) 
 {
     int len = 0;
     int last = 0;
 
     if(value == NULL) {
 	return list;
     }
     if(list) {
 	last = strlen(list);
     }
     len = last + 2;  /* +1 space, +1 EOS */
     len += strlen(value);
     crm_realloc(list, len);
     sprintf(list + last, " %s", value);
     return list;
 }
 
 static void print_cluster_status(pe_working_set_t *data_set) 
 {
     char *online_nodes = NULL;
     char *offline_nodes = NULL;
 
     slist_iter(node, node_t, data_set->nodes, lpc2,
 	       const char *node_mode = NULL;
 	       
 	       if(node->details->unclean) {
 		   if(node->details->online && node->details->unclean) {
 		       node_mode = "UNCLEAN (online)";
 		       
 		   } else if(node->details->pending) {
 		       node_mode = "UNCLEAN (pending)";
 
 		   } else {
 		       node_mode = "UNCLEAN (offline)";
 		   }
 
 	       } else if(node->details->pending) {
 		   node_mode = "pending";
 
 	       } else if(node->details->standby_onfail && node->details->online) {
 		   node_mode = "standby (on-fail)";
 
 	       } else if(node->details->standby) {
 		   if(node->details->online) {
 		       node_mode = "standby";
 		   } else {
 		       node_mode = "OFFLINE (standby)";
 		   }
 		   
 	       } else if(node->details->online) {
 		   node_mode = "online";
 		   online_nodes = add_list_element(online_nodes, node->details->uname);
 		   continue;
 
 	       } else {
 		   node_mode = "OFFLINE";
 		   offline_nodes = add_list_element(offline_nodes, node->details->uname);
 		   continue;
 	       }
 	       
 	       if(safe_str_eq(node->details->uname, node->details->id)) {
 		   printf("Node %s: %s\n",
 			    node->details->uname, node_mode);
 	       } else {
 		   printf("Node %s (%s): %s\n",
 			node->details->uname, node->details->id,
 			node_mode);
 	       }
 	);
 
     if(online_nodes) {
 	printf("Online: [%s ]\n", online_nodes);
 	crm_free(online_nodes);
     }
     if(offline_nodes) {
 	printf("OFFLINE: [%s ]\n", offline_nodes);
 	crm_free(offline_nodes);
     }
     
     fprintf(stdout, "\n");
     slist_iter(rsc, resource_t, data_set->resources, lpc,
 	       if(is_set(rsc->flags, pe_rsc_orphan)
 		  && rsc->role == RSC_ROLE_STOPPED) {
 		   continue;
 	       }
 	       rsc->fns->print(rsc, NULL, pe_print_printf, stdout);
 	);
     fprintf(stdout, "\n");
 }
 
 static int
 run_simulation(pe_working_set_t *data_set)
 {	
     crm_graph_t *transition = NULL;
     enum transition_status graph_rc = -1;
 
     crm_graph_functions_t exec_fns = 
 	{
 	    exec_pseudo_action,
 	    exec_rsc_action,
 	    exec_crmd_action,
 	    exec_stonith_action,
 	};
 
     set_graph_functions(&exec_fns);
 
     quiet_log("\nExecuting cluster transition:\n");
     transition = unpack_graph(data_set->graph, crm_system_name);
     print_graph(LOG_DEBUG, transition);
 	
     do {
 	graph_rc = run_graph(transition);
 	    
     } while(graph_rc == transition_active);
 	
     if(graph_rc != transition_complete) {
 	fprintf(stderr, "Transition failed: %s\n", transition_status(graph_rc));
 	print_graph(LOG_ERR, transition);
     }
     destroy_graph(transition);
     CRM_CHECK(graph_rc == transition_complete, fprintf(stderr, "An invalid transition was produced"));
 
     if(quiet == FALSE) {
 	xmlNode *cib_object = NULL;
 	ha_time_t *a_date = data_set->now;
 	int rc = global_cib->cmds->query(global_cib, NULL, &cib_object, cib_sync_call|cib_scope_local);
 	CRM_ASSERT(rc == cib_ok);
 
 	quiet_log("\nRevised cluster status:\n");
 	set_working_set_defaults(data_set);
 	data_set->input = cib_object;
 	data_set->now = a_date;
 	
 	cluster_status(data_set);
 	print_cluster_status(data_set);
     }
     
     if(graph_rc != transition_complete) {
 	return graph_rc;
     }
     return 0;
 }
 
 static char *
 create_action_name(action_t *action) 
 {
 	char *action_name = NULL;
 	const char *action_host = NULL;
 	if(action->node) {
 		action_host = action->node->details->uname;
 		action_name = crm_concat(action->uuid, action_host, ' ');
 
 	} else if(action->pseudo) {
 		action_name = crm_strdup(action->uuid);
 		
 	} else {
 		action_host = "<none>";
 		action_name = crm_concat(action->uuid, action_host, ' ');
 	}
 	if(safe_str_eq(action->task, RSC_CANCEL)) {
 	    char *tmp_action_name = action_name;
 	    action_name = crm_concat("Cancel", tmp_action_name, ' ');
 	    crm_free(tmp_action_name);
 	}
 	
 	return action_name;
 }
 
 static void
 create_dotfile(pe_working_set_t *data_set, const char *dot_file, gboolean all_actions)
 {
     FILE *dot_strm = fopen(dot_file, "w");
     if(dot_strm == NULL) {
 	crm_perror(LOG_ERR,"Could not open %s for writing", dot_file);
 	return;
     }
 
     fprintf(dot_strm, " digraph \"g\" {\n");
     slist_iter(
 	action, action_t, data_set->actions, lpc,
 
 	const char *style = "filled";
 	const char *font  = "black";
 	const char *color = "black";
 	const char *fill  = NULL;
 	char *action_name = create_action_name(action);
 	crm_debug_3("Action %d: %p", action->id, action);
 
 	if(action->pseudo) {
 	    font = "orange";
 	}
 		
 	style = "dashed";
 	if(action->dumped) {
 	    style = "bold";
 	    color = "green";
 			
 	} else if(action->rsc != NULL
 		  && is_not_set(action->rsc->flags, pe_rsc_managed)) {
 	    color = "purple";
 	    if(all_actions == FALSE) {
 		goto dont_write;
 	    }			
 			
 	} else if(action->optional) {
 	    color = "blue";
 	    if(all_actions == FALSE) {
 		goto dont_write;
 	    }			
 				
 	} else {
 	    color = "red";
 	    CRM_CHECK(action->runnable == FALSE, ;);	
 	}
 
 	action->dumped = TRUE;
 	fprintf(dot_strm, "\"%s\" [ style=%s color=\"%s\" fontcolor=\"%s\"  %s%s]\n",
 		  action_name, style, color, font, fill?"fillcolor=":"", fill?fill:"");
       dont_write:
 	crm_free(action_name);
 	);
 
 
     slist_iter(
 	action, action_t, data_set->actions, lpc,
 	slist_iter(
 	    before, action_wrapper_t, action->actions_before, lpc2,
 	    char *before_name = NULL;
 	    char *after_name = NULL;
 	    const char *style = "dashed";
 	    gboolean optional = TRUE;
 	    if(before->state == pe_link_dumped) {
 		optional = FALSE;
 		style = "bold";
 	    } else if(action->pseudo
 		      && (before->type & pe_order_stonith_stop)) {
 		continue;
 	    } else if(before->state == pe_link_dup) {
 		continue;
 	    } else if(before->type == pe_order_none) {
 		continue;
 	    } else if(action->dumped && before->action->dumped) {
 		optional = FALSE;
 	    }
 
 	    if(all_actions || optional == FALSE) {
 		before_name = create_action_name(before->action);
 		after_name = create_action_name(action);
 		fprintf(dot_strm, "\"%s\" -> \"%s\" [ style = %s]\n",
 			before_name, after_name, style);
 		crm_free(before_name);
 		crm_free(after_name);
 	    }
 	    );
 	);
     fprintf(dot_strm, "}\n");
     if(dot_strm != NULL) {
 	fflush(dot_strm);
 	fclose(dot_strm);
     }
 }
 
 static void modify_configuration(
     pe_working_set_t *data_set,
     const char *quorum, GListPtr node_up, GListPtr node_down, GListPtr node_fail, GListPtr op_inject)
 {
     int rc = cib_ok;
 
     xmlNode *cib_op = NULL;
     xmlNode *cib_node = NULL;
     xmlNode *cib_resource = NULL;
 
     lrm_op_t *op = NULL;
 
     if(quorum) {
 	xmlNode *top = create_xml_node(NULL, XML_TAG_CIB);
 	quiet_log(" + Setting quorum: %s\n", quorum);
 	/* crm_xml_add(top, XML_ATTR_DC_UUID, dc_uuid);	     */
 	crm_xml_add(top, XML_ATTR_HAVE_QUORUM, quorum);
 
 	rc = global_cib->cmds->modify(global_cib, NULL, top, cib_sync_call|cib_scope_local);
 	CRM_ASSERT(rc == cib_ok);
     }
     
     slist_iter(node, char, node_up, lpc,
 	       quiet_log(" + Bringing node %s online\n", node);
 	       cib_node = modify_node(global_cib, node, TRUE);	       
 	       CRM_ASSERT(cib_node != NULL);
 
 	       rc = global_cib->cmds->modify(global_cib, XML_CIB_TAG_STATUS, cib_node, cib_sync_call|cib_scope_local);
 	       CRM_ASSERT(rc == cib_ok);
 	);
 
     slist_iter(node, char, node_down, lpc,
 	       quiet_log(" + Taking node %s offline\n", node);
 	       cib_node = modify_node(global_cib, node, FALSE);	       
 	       CRM_ASSERT(cib_node != NULL);
 
 	       rc = global_cib->cmds->modify(global_cib, XML_CIB_TAG_STATUS, cib_node, cib_sync_call|cib_scope_local);
 	       CRM_ASSERT(rc == cib_ok);
 	);
 
     slist_iter(node, char, node_fail, lpc,
 	       quiet_log(" + Failing node %s\n", node);
 	       cib_node = modify_node(global_cib, node, TRUE);	       
 	       crm_xml_add(cib_node, XML_CIB_ATTR_INCCM, XML_BOOLEAN_NO);
 	       CRM_ASSERT(cib_node != NULL);
 
 	       rc = global_cib->cmds->modify(global_cib, XML_CIB_TAG_STATUS, cib_node, cib_sync_call|cib_scope_local);
 	       CRM_ASSERT(rc == cib_ok);
 	);
 
 
     slist_iter(spec, char, op_inject, lpc,
 
 	       int rc = 0;
 	       int outcome = 0;
 	       int interval = 0;
 
 	       char *key = NULL;
 	       char *node = NULL;
 	       char *task = NULL;
 	       char *resource = NULL;
 
 	       const char *rtype = NULL;
 	       const char *rclass = NULL;
 	       const char *rprovider = NULL;
 
 	       resource_t *rsc = NULL;
 	       quiet_log(" + Injecting %s into the configuration\n", spec);
 	       
-	       crm_malloc0(key, strlen(spec));
-	       crm_malloc0(node, strlen(spec));
+	       crm_malloc0(key, strlen(spec)+1);
+	       crm_malloc0(node, strlen(spec)+1);
 	       rc = sscanf(spec, "%[^@]@%[^=]=%d", key, node, &outcome);
 	       CRM_CHECK(rc == 3, fprintf(stderr, "Invalid operation spec: %s.  Only found %d fields\n", spec, rc); continue);
 	       
 	       parse_op_key(key, &resource, &task, &interval);
 
 	       rsc = pe_find_resource(data_set->resources, resource);
 	       CRM_CHECK(rsc != NULL, fprintf(stderr, "Invalid resource name: %s\n", resource); continue);
 
 	       rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 	       rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
 	       rprovider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
 	       
 	       cib_node = inject_node_state(global_cib, node);
 	       CRM_ASSERT(cib_node != NULL);
 
 	       update_failcounts(cib_node, resource, interval, rc);
 	       
 	       cib_resource = inject_resource(cib_node, resource, rclass, rtype, rprovider);
 	       CRM_ASSERT(cib_resource != NULL);
 	       
 	       op = create_op(cib_resource, task, interval, outcome);
 	       CRM_ASSERT(op != NULL);
 	       
 	       cib_op = inject_op(cib_resource, op, 0);
 	       CRM_ASSERT(cib_op != NULL);
 	       
 	       rc = global_cib->cmds->modify(global_cib, XML_CIB_TAG_STATUS, cib_node, cib_sync_call|cib_scope_local);
 	       CRM_ASSERT(rc == cib_ok);
 	);
 }
     
 static void
 setup_input(const char *input, const char *output) 
 {
     int rc = cib_ok;
     cib_t *cib_conn = NULL;
     xmlNode *cib_object = NULL;
     char *local_output = NULL;
     
     if(input == NULL) {
 	/* Use live CIB */
 	cib_conn = cib_new();
 	rc = cib_conn->cmds->signon(cib_conn, crm_system_name, cib_command);
 	
 	if(rc == cib_ok) {
 	    cib_object = get_cib_copy(cib_conn);
 	}
 
 	cib_conn->cmds->signoff(cib_conn);
 	cib_delete(cib_conn);
 	cib_conn = NULL;
 
 	if(cib_object == NULL) {
 	    fprintf(stderr, "Live CIB query failed: empty result\n");
 	    exit(3);
 	}
 
     } else if(safe_str_eq(input, "-")) {
 	cib_object = filename2xml(NULL);
 	
     } else {
 	cib_object = filename2xml(input);
     }
 
     if(cli_config_update(&cib_object, NULL, FALSE) == FALSE) {
 	free_xml(cib_object);
 	exit(cib_STALE);
     }
     
     if(validate_xml(cib_object, NULL, FALSE) != TRUE) {
 	free_xml(cib_object);
 	exit(cib_dtd_validation);
     }
     
     if(output == NULL) {
 	char *pid = crm_itoa(getpid());
 	local_output = get_shadow_file(pid);
 	output = local_output;
 	crm_free(pid);
     }
 
     rc = write_xml_file(cib_object, output, FALSE);
     free_xml(cib_object);
     cib_object = NULL;
     
     if(rc < 0) {
 	fprintf(stderr, "Could not create '%s': %s\n", output, strerror(errno));
 	exit(rc);
     }
     setenv("CIB_file", output, 1);
     crm_free(local_output);
 }
 
 
 static struct crm_option long_options[] = {
     /* Top-level Options */
     {"help",    0, 0, '?', "\tThis text"},
     {"version", 0, 0, '$', "\tVersion information"  },
     {"quiet",   0, 0, 'Q', "\tDisplay only essentialoutput"},
     {"verbose", 0, 0, 'V', "\tIncrease debug output"},
 
     {"-spacer-",      0, 0, '-', "\nOperations:"},
     {"run",           0, 0, 'R', "\tDetermine the cluster's response to the given configuration and status"},
     {"simulate",      0, 0, 'S', "Simulate the transition's execution and display the resulting cluster status"},
     {"in-place",      0, 0, 'X', "Simulate the transition's execution and store the result back to the input file"},
     {"show-scores",   0, 0, 's', "Show allocation scores"},
     {"show-utilization",   0, 0, 'U', "Show utilization information"},
 
     {"-spacer-",     0, 0, '-', "\nSynthetic Cluster Events:"},
     {"node-up",      1, 0, 'u', "\tBring a node online"},
     {"node-down",    1, 0, 'd', "\tTake a node offline"},
     {"node-fail",    1, 0, 'f', "\tMark a node as failed"},
     {"op-inject",    1, 0, 'i', "\t$rsc_$task_$interval@$node=$rc - Inject the specified task before running the simulation"},
     {"op-fail",      1, 0, 'F', "\t$rsc_$task_$interval@$node=$rc - Fail the specified task while running the simulation"},
     {"set-datetime", 1, 0, 't', "Set date/time"},
     {"quorum",       1, 0, 'q', "\tSpecify a value for quorum"},
 
     {"-spacer-",     0, 0, '-', "\nOutput Options:"},
     
     {"save-input",   1, 0, 'I', "\tSave the input configuration to the named file"},
     {"save-output",  1, 0, 'O', "Save the output configuration to the named file"},
     {"save-graph",   1, 0, 'G', "\tSave the transition graph (XML format) to the named file"},
     {"save-dotfile", 1, 0, 'D', "Save the transition graph (DOT format) to the named file"},
     {"all-actions",  0, 0, 'a', "\tDisplay all possible actions in the DOT graph - even ones not part of the transition"},
     
     {"-spacer-",    0, 0, '-', "\nData Source:"},
     {"live-check",  0, 0, 'L', "\tConnect to the CIB and use the current contents as input"},
     {"xml-file",    1, 0, 'x', "\tRetrieve XML from the named file"},
     {"xml-pipe",    0, 0, 'p', "\tRetrieve XML from stdin"},
     
     {0, 0, 0, 0}
 };
 
 int
 main(int argc, char ** argv)
 {
     int rc = 0;
     guint modified = 0;
 
     gboolean store = FALSE;
     gboolean process = FALSE;
     gboolean verbose = FALSE;
     gboolean simulate = FALSE;
     gboolean all_actions = FALSE;
 
     pe_working_set_t data_set;
     ha_time_t *a_date = NULL;
 
     const char *xml_file = "-";
     const char *quorum = NULL;
     const char *dot_file = NULL;
     const char *graph_file = NULL;
     const char *input_file = NULL;
     const char *output_file = NULL;
     
     int flag = 0;
     int index = 0;
     int argerr = 0;
     char *use_date = NULL;
 
     GListPtr node_up = NULL;
     GListPtr node_down = NULL;
     GListPtr node_fail = NULL;
     GListPtr op_inject = NULL;
     
     xmlNode *input = NULL;
 
     crm_log_init("crm_simulate", LOG_ERR, FALSE, FALSE, argc, argv);
     crm_set_options("?$VQx:Lpu:d:f:i:RSXD:G:I:O:sUaF:t:q:", "datasource operation [additional options]",
 		    long_options, "Tool for simulating the cluster's response to events");
 
     if(argc < 2) {
 	crm_help('?', LSB_EXIT_EINVAL);
     }
 
     while (1) {
 	flag = crm_get_option(argc, argv,  &index);
 	if (flag == -1)
 	    break;
 
 	switch(flag) {
 	    case 'V':
 		verbose = TRUE;
 		alter_debug(DEBUG_INC);
 		cl_log_enable_stderr(TRUE);
 		break;
 	    case '?':
 	    case '$':
 		crm_help(flag, LSB_EXIT_OK);
 		break;
 	    case 'p':
 		xml_file = "-";
 		break;
 	    case 'Q':
 		quiet = TRUE;
 		break;
 	    case 'L':
 		xml_file = NULL;
 		break;
 	    case 'x':
 		xml_file = optarg;
 		break;
 	    case 'u':
 		modified++;
 		node_up = g_list_append(node_up, optarg);
 		break;
 	    case 'd':
 		modified++;
 		node_down = g_list_append(node_down, optarg);
 		break;
 	    case 'f':
 		modified++;
 		node_fail = g_list_append(node_fail, optarg);
 		break;
 	    case 't':
 		use_date = crm_strdup(optarg);
 		break;
 	    case 'i':
 		modified++;
 		op_inject = g_list_append(op_inject, optarg);
 		break;
 	    case 'F':
 		process = TRUE;
 		simulate = TRUE;
 		op_fail = g_list_append(op_fail, optarg);
 		break;
 	    case 'q':
 		modified++;
 		quorum = optarg;
 		break;	
 	    case 'a':
 		all_actions = TRUE;
 		break;
 	    case 's':
 		process = TRUE;
 		show_scores = TRUE;
 		break;
 	    case 'U':
 		process = TRUE;
 		show_utilization = TRUE;
 		break;
 	    case 'S':
 		process = TRUE;
 		simulate = TRUE;
 		break;
 	    case 'X':
 		store = TRUE;
 		process = TRUE;
 		simulate = TRUE;
 		break;
 	    case 'R':
 		process = TRUE;
 		break;
 	    case 'D':
 		process = TRUE;
 		dot_file = optarg;
 		break;
 	    case 'G':
 		process = TRUE;
 		graph_file = optarg;
 		break;
 	    case 'I':
 		input_file = optarg;
 		break;
 	    case 'O':
 		simulate = TRUE;
 		output_file = optarg;
 		break;
 	    default:
 		++argerr;
 		break;
 	}
     }
 
     if (optind > argc) {
 	++argerr;
     }
 
     if (argerr) {
 	crm_help('?', LSB_EXIT_GENERIC);
     }
 
     setup_input(xml_file, store?xml_file:output_file);
     
     global_cib = cib_new();
     global_cib->cmds->signon(global_cib, crm_system_name, cib_command);
 
     if(use_date != NULL) {
 	char *date_m = use_date;
 	a_date = parse_date(&date_m);
 	quiet_log(" + Setting effective cluster time: %s", use_date);
 	log_date(LOG_WARNING, "Set fake 'now' to", a_date, ha_log_date|ha_log_time);
 	crm_free(use_date);
 	use_date = NULL;
     }
     
+    set_working_set_defaults(&data_set);
     if(quiet == FALSE) {
 	xmlNode *cib_object = NULL;
 	rc = global_cib->cmds->query(global_cib, NULL, &cib_object, cib_sync_call|cib_scope_local);
 	CRM_ASSERT(rc == cib_ok);
 	
-	set_working_set_defaults(&data_set);
 	data_set.input = cib_object;
 	data_set.now = a_date;
 	
 	cluster_status(&data_set);
 	quiet_log("\nCurrent cluster status:\n");
 	print_cluster_status(&data_set);
 	if(process == FALSE && modified == FALSE) {
 	    rc = 0;
 	    goto done;
 	}	
     }    
 
     if(modified) {
 	quiet_log("Performing requested modifications\n");
 	modify_configuration(&data_set, quorum, node_up, node_down, node_fail, op_inject);
     }	    
 
     rc = global_cib->cmds->query(global_cib, NULL, &input, cib_sync_call);
     if(rc != cib_ok) {
 	fprintf(stderr, "Could not connect to the CIB for input: %s\n", cib_error2string(rc));
 	goto done;
     }
     
     if(input_file != NULL) {	
 	rc = write_xml_file(input, input_file, FALSE);
 	if(rc < 0) {
 	    fprintf(stderr, "Could not create '%s': %s\n", input_file, strerror(errno));
 	    goto done;
 	}
 	free_xml(input);
     }
 
     rc = 0;
     if(process || simulate) {
 	if(show_scores && show_utilization) {
 	    printf("Allocation scores and utilization information:\n");
 	} else if(show_scores) {
 	    printf("Allocation scores:\n");
 	} else if(show_utilization) {
 	    printf("Utilization information:\n");
 	}
 	
 	do_calculations(&data_set, input, a_date);
 	
 	if(graph_file != NULL) {
 	    char *msg_buffer = dump_xml_formatted(data_set.graph);
 	    FILE *graph_strm = fopen(graph_file, "w");
 	    if(graph_strm == NULL) {
 		crm_perror(LOG_ERR,"Could not open %s for writing", graph_file);
 
 	    } else {
 		if(fprintf(graph_strm, "%s\n\n", msg_buffer) < 0) {
 		    crm_perror(LOG_ERR,"Write to %s failed", graph_file);
 		}
 		fflush(graph_strm);
 		fclose(graph_strm);
 	    }
 	    crm_free(msg_buffer);
 	}
 
 	if(dot_file != NULL) {
 	    create_dotfile(&data_set, dot_file, all_actions);
 	}
 
 	if(quiet == FALSE && verbose == FALSE) {
 	    quiet_log("%sTransition Summary:\n", show_scores||show_utilization||modified?"\n":"");
 	    fflush(stdout);
 
 	    crm_log_level = LOG_NOTICE;
 	    cl_log_enable_stderr(TRUE);
 	    slist_iter(
 		rsc, resource_t, data_set.resources, lpc,
 		LogActions(rsc, &data_set);
 		);
 
 	    cl_log_enable_stderr(FALSE);
 	}
     }
     
     if(simulate) {
 	rc = run_simulation(&data_set);	
     }
 
   done:
     cleanup_alloc_calculations(&data_set);
     
     global_cib->cmds->signoff(global_cib);
     cib_delete(global_cib);
     fflush(stderr);
     return rc;
 }
diff --git a/tools/crm_resource.c b/tools/crm_resource.c
index 7daa65256e..da93667dd3 100644
--- a/tools/crm_resource.c
+++ b/tools/crm_resource.c
@@ -1,1519 +1,1519 @@
 
 /* 
  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
  * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  * 
  * This software is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * General Public License for more details.
  * 
  * You should have received a copy of the GNU General Public
  * License along with this library; if not, write to the Free Software
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <crm_internal.h>
 
 #include <sys/param.h>
 
 #include <crm/crm.h>
 
 #include <stdio.h>
 #include <sys/types.h>
 #include <unistd.h>
 
 #include <stdlib.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <libgen.h>
 
 #include <crm/msg_xml.h>
 #include <crm/common/xml.h>
 #include <crm/common/ipc.h>
 
 #include <crm/cib.h>
 #include <crm/pengine/rules.h>
 #include <crm/pengine/status.h>
 
 gboolean do_force = FALSE;
 gboolean BE_QUIET = FALSE;
 const char *attr_set_type = XML_TAG_ATTR_SETS;
 char *host_id = NULL;
 const char *rsc_id = NULL;
 const char *host_uname = NULL;
 const char *prop_name = NULL;
 const char *prop_value = NULL;
 const char *rsc_type = NULL;
 const char *prop_id = NULL;
 const char *prop_set = NULL;
 char *move_lifetime = NULL;
 char rsc_cmd = 'L';
 char *our_pid = NULL;
 IPC_Channel *crmd_channel = NULL;
 char *xml_file = NULL;
 int cib_options = cib_sync_call;
 
 #define CMD_ERR(fmt, args...) do {		\
 	crm_warn(fmt, ##args);			\
 	fprintf(stderr, fmt, ##args);		\
     } while(0)
 
 static int
 do_find_resource(const char *rsc, pe_working_set_t *data_set)
 {
 	int found = 0;
 	resource_t *the_rsc = pe_find_resource(data_set->resources, rsc);
 
 	if(the_rsc == NULL) {
 		return cib_NOTEXISTS;
 	}
 
 	slist_iter(node, node_t, the_rsc->running_on, lpc,
 		   crm_debug_3("resource %s is running on: %s",
 			       rsc, node->details->uname);
 		   if(BE_QUIET) {
 			   fprintf(stdout, "%s\n", node->details->uname);
 		   } else {
 			   fprintf(stdout, "resource %s is running on: %s\n",
 				   rsc, node->details->uname);
 		   }
 		   
 		   found++;
 		);
 	
 	if(BE_QUIET == FALSE && found == 0) {
 		fprintf(stderr, "resource %s is NOT running\n", rsc);
 	}
 	
 	return 0;
 }
 
 #define cons_string(x) x?x:"NA"
 static void
 print_cts_constraints(pe_working_set_t *data_set) 
 {
     xmlNode *lifetime = NULL;
     xmlNode * cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);
     xml_child_iter(cib_constraints, xml_obj, 
 
 		   const char *id = crm_element_value(xml_obj, XML_ATTR_ID);
 		   if(id == NULL) {
 		       continue;
 		   }
 		   
 		   lifetime = first_named_child(xml_obj, "lifetime");
 		   
 		   if(test_ruleset(lifetime, NULL, data_set->now) == FALSE) {
 		       continue;
 		   }
 		   
 		   if(safe_str_eq(XML_CONS_TAG_RSC_DEPEND, crm_element_name(xml_obj))) {
 		       printf("Constraint %s %s %s %s %s %s %s\n",
 			      crm_element_name(xml_obj),
 			      cons_string(crm_element_value(xml_obj, XML_ATTR_ID)),
 			      cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE)),
 			      cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET)),
 			      cons_string(crm_element_value(xml_obj, XML_RULE_ATTR_SCORE)),
 			      cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_SOURCE_ROLE)),
 			      cons_string(crm_element_value(xml_obj, XML_COLOC_ATTR_TARGET_ROLE)));
 		       
 		   } else if(safe_str_eq(XML_CONS_TAG_RSC_LOCATION, crm_element_name(xml_obj))) {
 		       /* unpack_rsc_location(xml_obj, data_set); */
 		   }
 	);
 }
 
 static void
 print_cts_rsc(resource_t *rsc) 
 {
     const char *host = NULL;
     gboolean needs_quorum = TRUE;
     const char *rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
     const char *rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
     const char *rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 
     if(safe_str_eq(rclass, "stonith")) {
 	needs_quorum = FALSE;
 
     } else {
 	xml_child_iter_filter(rsc->ops_xml, op, "op",
 			  const char *name = crm_element_value(op, "name");
 			  if(safe_str_neq(name, CRMD_ACTION_START)) {
 			      const char *value = crm_element_value(op, "requires");
 			      if(safe_str_eq(value, "nothing")) {
 				  needs_quorum = FALSE;
 			      }
 			      break;
 			  }
 	);
     }
 
     if(rsc->running_on != NULL && g_list_length(rsc->running_on) == 1) {
 	node_t *tmp = rsc->running_on->data;
 	host = tmp->details->uname;
     }
 
     printf("Resource: %s %s %s %s %s %s %s %s %d %lld 0x%.16llx\n",
 	   crm_element_name(rsc->xml), rsc->id,
 	   rsc->clone_name?rsc->clone_name:rsc->id, rsc->parent?rsc->parent->id:"NA",
 	   rprov?rprov:"NA", rclass, rtype, host?host:"NA", needs_quorum, rsc->flags, rsc->flags);
 
     slist_iter(child, resource_t, rsc->children, lpc,
 	       print_cts_rsc(child);
 	);
 }
 
 
 static void
 print_raw_rsc(resource_t *rsc) 
 {
 	GListPtr children = rsc->children;
 
 	if(children == NULL) {
 	    printf("%s\n", rsc->id);
 	}
 	
 	slist_iter(child, resource_t, children, lpc,
 		   print_raw_rsc(child);
 		);
 }
 
 
 static int
 do_find_resource_list(pe_working_set_t *data_set, gboolean raw)
 {
 	int found = 0;
 	
 	slist_iter(
 		rsc, resource_t, data_set->resources, lpc,
 		if(is_set(rsc->flags, pe_rsc_orphan)
 		   && rsc->fns->active(rsc, TRUE) == FALSE) {
 			continue;
 		}
 		rsc->fns->print(
 			rsc, NULL, pe_print_printf|pe_print_rsconly, stdout);
 		found++;
 		);
 
 	if(found == 0) {
 		printf("NO resources configured\n");
 		return cib_NOTEXISTS;
 	}
 
 	return 0;
 }
 
 
 static resource_t *find_rsc_or_clone(const char *rsc, pe_working_set_t *data_set) 
 {
     resource_t *the_rsc = pe_find_resource(data_set->resources, rsc);
     if(the_rsc == NULL) {
 	char *as_clone = crm_concat(rsc, "0", ':');
 	the_rsc = pe_find_resource(data_set->resources, as_clone);
 	crm_free(as_clone);
     }
     return the_rsc;
 }
 
 static int
 dump_resource(const char *rsc, pe_working_set_t *data_set)
 {
 	char *rsc_xml = NULL;
 	resource_t *the_rsc = find_rsc_or_clone(rsc, data_set);
 
 	if(the_rsc == NULL) {
 		return cib_NOTEXISTS;
 	}
 	the_rsc->fns->print(the_rsc, NULL, pe_print_printf, stdout);
 
 	rsc_xml = dump_xml_formatted(the_rsc->xml);
 
 	fprintf(stdout, "raw xml:\n%s\n", rsc_xml);
 	
 	crm_free(rsc_xml);
 	
 	return 0;
 }
 
 static int
 dump_resource_attr(
 	const char *rsc, const char *attr, pe_working_set_t *data_set)
 {
 	int rc = cib_NOTEXISTS;
 	node_t *current = NULL;
 	GHashTable *params = NULL;
 	resource_t *the_rsc = find_rsc_or_clone(rsc, data_set);
 	const char *value = NULL;
 
 	if(the_rsc == NULL) {
 		return cib_NOTEXISTS;
 	}
 
 	if(g_list_length(the_rsc->running_on) == 1) {
 		current = the_rsc->running_on->data;
 
 	} else if(g_list_length(the_rsc->running_on) > 1) {
 		CMD_ERR("%s is active on more than one node,"
 			" returning the default value for %s\n",
 			the_rsc->id, crm_str(value));
 	} 
 
 	params = g_hash_table_new_full(
 		g_str_hash, g_str_equal,
 		g_hash_destroy_str, g_hash_destroy_str);
 
 	if(safe_str_eq(attr_set_type, XML_TAG_ATTR_SETS)) {
 	    get_rsc_attributes(params, the_rsc, current, data_set);
 	} else if(safe_str_eq(attr_set_type, XML_TAG_META_SETS)) {
 	    get_meta_attributes(params, the_rsc, current, data_set);
 	} else {
 	    unpack_instance_attributes(data_set->input, the_rsc->xml, XML_TAG_UTILIZATION, NULL,
 		    params, NULL, FALSE, data_set->now);
 	}
 	
 	crm_debug("Looking up %s in %s", attr, the_rsc->id);
 	value = g_hash_table_lookup(params, attr);
 	if(value != NULL) {
 		fprintf(stdout, "%s\n", value);
 		rc = 0;
 	}
 
 	g_hash_table_destroy(params);
 	return rc;
 }
 
 static int find_resource_attr(
     cib_t *the_cib, const char *attr, const char *rsc, const char *set_type, const char *set_name,
     const char *attr_id, const char *attr_name, char **value)
 {
     int offset = 0;
     static int xpath_max = 1024;
     enum cib_errors rc = cib_ok;
     xmlNode *xml_search = NULL;
 
     char *xpath_string = NULL;
 
     CRM_ASSERT(value != NULL);
     *value = NULL;
     
     crm_malloc0(xpath_string, xpath_max);
     offset += snprintf(xpath_string + offset, xpath_max - offset, "%s", get_object_path("resources"));
 
     offset += snprintf(xpath_string + offset, xpath_max - offset, "//*[@id=\"%s\"]", rsc);
 
     if(set_type) {
 	offset += snprintf(xpath_string + offset, xpath_max - offset, "//%s", set_type);
 	if(set_name) {
 	    offset += snprintf(xpath_string + offset, xpath_max - offset, "[@id=\"%s\"]", set_name);
 	}
     }
     
     offset += snprintf(xpath_string + offset, xpath_max - offset, "//nvpair[");
     if(attr_id) {
 	offset += snprintf(xpath_string + offset, xpath_max - offset, "@id=\"%s\"", attr_id);
     }
     
     if(attr_name) {
 	if(attr_id) {
 	    offset += snprintf(xpath_string + offset, xpath_max - offset, " and ");
 	}
 	offset += snprintf(xpath_string + offset, xpath_max - offset, "@name=\"%s\"", attr_name);
     }   
     offset += snprintf(xpath_string + offset, xpath_max - offset, "]");
 
     rc = the_cib->cmds->query(
 	the_cib, xpath_string, &xml_search, cib_sync_call|cib_scope_local|cib_xpath);
 	
     if(rc != cib_ok) {
 	return rc;
     }
 
     crm_log_xml_debug(xml_search, "Match");
     if(xml_has_children(xml_search)) {
 	rc = cib_missing_data;
 	printf("Multiple attributes match name=%s\n", attr_name);
 	
 	xml_child_iter(xml_search, child,
 		       printf("  Value: %s \t(id=%s)\n", 
 			      crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
 	    );
 
     } else {
 	const char *tmp = crm_element_value(xml_search, attr);
 	if(tmp) {
 	    *value = crm_strdup(tmp);
 	}
     }
 
     free_xml(xml_search);
     return rc;
 }
 
 static int
 set_resource_attr(const char *rsc_id, const char *attr_set, const char *attr_id,
 		  const char *attr_name, const char *attr_value,
 		  cib_t *cib, pe_working_set_t *data_set)
 {
 	int rc = cib_ok;
 	
 	char *local_attr_id = NULL;
 	char *local_attr_set = NULL;
 	
 	xmlNode *xml_top = NULL;
 	xmlNode *xml_obj = NULL;
 
 	gboolean use_attributes_tag = FALSE;
 	resource_t *rsc = find_rsc_or_clone(rsc_id, data_set);
 
 	if(rsc == NULL) {
 		return cib_NOTEXISTS;
 	}
 
 	if(safe_str_eq(attr_set_type, XML_TAG_ATTR_SETS)) {
 	    rc = find_resource_attr(
 		cib, XML_ATTR_ID, rsc_id, XML_TAG_META_SETS, attr_set, attr_id, attr_name, &local_attr_id);
 	    if(rc == cib_ok) {
 		printf("WARNING: There is already a meta attribute called %s (id=%s)\n", attr_name, local_attr_id);
 	    }
 	}
  	rc = find_resource_attr(
 	    cib, XML_ATTR_ID, rsc_id, attr_set_type, attr_set, attr_id, attr_name, &local_attr_id);
 
 	if(rc == cib_ok) {
 	    crm_debug("Found a match for name=%s: id=%s", attr_name, local_attr_id);
 	    attr_id = local_attr_id;
 
 	} else if(rc != cib_NOTEXISTS) {
 	    return rc;
 
 	} else {
 	    const char *value = NULL;
 	    xmlNode *cib_top = NULL;
 	    const char *tag = crm_element_name(rsc->xml);
 
 	    rc = cib->cmds->query(cib, "/cib", &cib_top, cib_sync_call|cib_scope_local|cib_xpath|cib_no_children);
 	    value = crm_element_value(cib_top, "ignore_dtd");
 	    if(value != NULL) {
 		use_attributes_tag = TRUE;
 		
 	    } else {
 		value = crm_element_value(cib_top, XML_ATTR_VALIDATION);
 		if(value && strstr(value, "-0.6")) {
 		    use_attributes_tag = TRUE;
 		}
 	    }
 	    free_xml(cib_top);
 
 	    if(attr_set == NULL) {
 		local_attr_set = crm_concat(rsc_id, attr_set_type, '-');
 		attr_set = local_attr_set;
 	    }
 	    if(attr_id == NULL) {
 		local_attr_id = crm_concat(attr_set, attr_name, '-');
 		attr_id = local_attr_id;
 	    }
 
 	    if(use_attributes_tag && safe_str_eq(tag, XML_CIB_TAG_MASTER)) {
 		tag = "master_slave"; /* use the old name */
 	    }
 	    
 	    xml_top = create_xml_node(NULL, tag);
 	    crm_xml_add(xml_top, XML_ATTR_ID, rsc_id);
 	    
 	    xml_obj = create_xml_node(xml_top, attr_set_type);
 	    crm_xml_add(xml_obj, XML_ATTR_ID, attr_set);
 
 	    if(use_attributes_tag) {
 		xml_obj = create_xml_node(xml_obj, XML_TAG_ATTRS);
 	    }
 	}
 		
 	xml_obj = create_xml_node(xml_obj, XML_CIB_TAG_NVPAIR);
 	if(xml_top == NULL) {
 	    xml_top = xml_obj;
 	}
 	
 	crm_xml_add(xml_obj, XML_ATTR_ID, attr_id);
 	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, attr_name);
 	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_VALUE, attr_value);
 	
 	crm_log_xml_debug(xml_top, "Update");
 	
 	rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top, cib_options);
 	free_xml(xml_top);
 	crm_free(local_attr_id);
 	crm_free(local_attr_set);
 	return rc;
 }
 
 static int
 delete_resource_attr(
 	const char *rsc_id, const char *attr_set, const char *attr_id,
 	const char *attr_name, cib_t *cib, pe_working_set_t *data_set)
 {
 	xmlNode *xml_obj = NULL;
 
 	int rc = cib_ok;
 	char *local_attr_id = NULL;
 	resource_t *rsc = find_rsc_or_clone(rsc_id, data_set);
 
 	if(rsc == NULL) {
 		return cib_NOTEXISTS;
 	}
 
  	rc = find_resource_attr(
 	    cib, XML_ATTR_ID, rsc_id, attr_set_type, attr_set, attr_id, attr_name, &local_attr_id);
 
 	if(rc == cib_NOTEXISTS) {
 	    return cib_ok;
 
 	} else if(rc != cib_ok) {
 	    return rc;
 	}
 	
 	if(attr_id == NULL) {
 		attr_id = local_attr_id;
 	}
 
 	xml_obj = create_xml_node(NULL, XML_CIB_TAG_NVPAIR);
 	crm_xml_add(xml_obj, XML_ATTR_ID, attr_id);
 	crm_xml_add(xml_obj, XML_NVPAIR_ATTR_NAME, attr_name);
 	
 	crm_log_xml_debug(xml_obj, "Delete");
 	
 	rc = cib->cmds->delete(cib, XML_CIB_TAG_RESOURCES, xml_obj, cib_options);
 
 	if(rc == cib_ok) {
 	    printf("Deleted %s option: id=%s%s%s%s%s\n", rsc_id, local_attr_id,
 		   attr_set?" set=":"", attr_set?attr_set:"",
 		   attr_name?" name=":"", attr_name?attr_name:"");
 	}
 
 	free_xml(xml_obj);
 	crm_free(local_attr_id);
 	return rc;
 }
 
 static int
 dump_resource_prop(
 	const char *rsc, const char *attr, pe_working_set_t *data_set)
 {
 	const char *value = NULL;
 	resource_t *the_rsc = pe_find_resource(data_set->resources, rsc);
 
 	if(the_rsc == NULL) {
 		return cib_NOTEXISTS;
 	}
 
 	value = crm_element_value(the_rsc->xml, attr);
 
 	if(value != NULL) {
 		fprintf(stdout, "%s\n", value);
 		return 0;
 	}
 	return cib_NOTEXISTS;
 }
 
 static void
 resource_ipc_connection_destroy(gpointer user_data)
 {
 	crm_info("Connection to CRMd was terminated");
 	exit(1);
 }
 
 static gboolean
 crmd_msg_callback(IPC_Channel * server, void *private_data)
 {
 	int lpc = 0;
 	IPC_Message *msg = NULL;
 	gboolean hack_return_good = TRUE;
 
 	while (server->ch_status != IPC_DISCONNECT
 	       && server->ops->is_message_pending(server) == TRUE) {
 		if (server->ops->recv(server, &msg) != IPC_OK) {
 			perror("Receive failure:");
 			return !hack_return_good;
 		}
 
 		if (msg == NULL) {
 			crm_debug_4("No message this time");
 			continue;
 		}
 
 		lpc++;
 		msg->msg_done(msg);		
 	}
 
 	if (server->ch_status == IPC_DISCONNECT) {
 		crm_debug_2("admin_msg_callback: received HUP");
 		return !hack_return_good;
 	}
 
 	return hack_return_good;
 }
 
 static int
 send_lrm_rsc_op(IPC_Channel *crmd_channel, const char *op,
 		const char *host_uname, const char *rsc_id,
 		gboolean only_failed, pe_working_set_t *data_set)
 {
 	char *key = NULL;
 	int rc = cib_send_failed;
 	xmlNode *cmd = NULL;
 	xmlNode *xml_rsc = NULL;
 	const char *value = NULL;
 	xmlNode *params = NULL;
 	xmlNode *msg_data = NULL;
 	resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
 
 	if(rsc == NULL) {
 		CMD_ERR("Resource %s not found\n", rsc_id);
 		return cib_NOTEXISTS;
 
 	} else if(rsc->variant != pe_native) {
 		CMD_ERR("We can only process primitive resources, not %s\n", rsc_id);
 		return cib_invalid_argument;
 
 	} else if(host_uname == NULL) {
 		CMD_ERR("Please supply a hostname with -H\n");
 		return cib_invalid_argument;
 	}
 	
 	key = crm_concat("0:0:crm-resource", our_pid, '-');
 	
 	msg_data = create_xml_node(NULL, XML_GRAPH_TAG_RSC_OP);
 	crm_xml_add(msg_data, XML_ATTR_TRANSITION_KEY, key);
 	crm_free(key);
 	
 	xml_rsc = create_xml_node(msg_data, XML_CIB_TAG_RESOURCE);
 	if(rsc->clone_name) {
 	    crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->clone_name);
 	    crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc->id);
 	    
 	} else {
 	    crm_xml_add(xml_rsc, XML_ATTR_ID, rsc->id);
 	    crm_xml_add(xml_rsc, XML_ATTR_ID_LONG, rsc->long_name);
 	}
 	
 	value = crm_element_value(rsc->xml, XML_ATTR_TYPE);
 	crm_xml_add(xml_rsc, XML_ATTR_TYPE, value);
 	if(value == NULL) {
 		CMD_ERR("%s has no type!  Aborting...\n", rsc_id);
 		return cib_NOTEXISTS;
 	}
 
 	value = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 	crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, value);
 	if(value == NULL) {
 		CMD_ERR("%s has no class!  Aborting...\n", rsc_id);
 		return cib_NOTEXISTS;
 	}
 
 	value = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
 	crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, value);
 
 	params = create_xml_node(msg_data, XML_TAG_ATTRS);
 	crm_xml_add(params, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
 
 	key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
 	crm_xml_add(params, key, "60000"); /* 1 minute */
 	crm_free(key);
 	
 	cmd = create_request(op, msg_data, host_uname,
 			     CRM_SYSTEM_CRMD, crm_system_name, our_pid);
 
 /* 	crm_log_xml_warn(cmd, "send_lrm_rsc_op"); */	
 	free_xml(msg_data);
 
 	if(send_ipc_message(crmd_channel, cmd)) {
 	    rc = 0;
 	    sleep(1); /* dont exit striaght away, give the crmd time
 		       * to process our request
 		       */
 	} else {
 	    CMD_ERR("Could not send %s op to the crmd", op);
 	}
 	
 	free_xml(cmd);
 	return rc;
 }
 
 
 static int
 delete_lrm_rsc(IPC_Channel *crmd_channel, const char *host_uname,
 	       resource_t *rsc, pe_working_set_t *data_set)
 {
     int rc = cib_ok;
     
     if(rsc == NULL) {
 	return cib_NOTEXISTS;
 
     } else if(rsc->children) {
 	slist_iter(child, resource_t, rsc->children, lpc,
 		   delete_lrm_rsc(crmd_channel, host_uname, child, data_set));
 	return cib_ok;
 
     } else if(host_uname == NULL) {
 	slist_iter(node, node_t, data_set->nodes, lpc,
 		   delete_lrm_rsc(crmd_channel, node->details->uname, rsc, data_set));
 	return cib_ok;	
     }
 
     printf("Cleaning up %s on %s\n", rsc->id, host_uname);
     rc = send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_DELETE, host_uname, rsc->id, TRUE, data_set);
     if(rc == cib_ok) {
 	char *attr_name = NULL;
 	const char *id = rsc->id;
 
 	if(rsc->clone_name) {
 	    id = rsc->clone_name;
 	}
 	
 	attr_name = crm_concat("fail-count", id, '-');
 	attrd_lazy_update('D', host_uname, attr_name, NULL, XML_CIB_TAG_STATUS, NULL, NULL);
 	crm_free(attr_name);
     }
     return rc;
 }
 
 static int
 fail_lrm_rsc(IPC_Channel *crmd_channel, const char *host_uname,
 	     const char *rsc_id, pe_working_set_t *data_set)
 {
     crm_warn("Failing: %s", rsc_id);
     return send_lrm_rsc_op(crmd_channel, CRM_OP_LRM_FAIL, host_uname, rsc_id, FALSE, data_set); 
 }
 
 static int
 refresh_lrm(IPC_Channel *crmd_channel, const char *host_uname)  
 {
 	xmlNode *cmd = NULL;
 	int rc = cib_send_failed;
 	
 	cmd = create_request(CRM_OP_LRM_REFRESH, NULL, host_uname,
 			     CRM_SYSTEM_CRMD, crm_system_name, our_pid);
 	
 	if(send_ipc_message(crmd_channel, cmd)) {
 		rc = 0;
 	}
 	free_xml(cmd);
 	return rc;
 }
 
 static int
 move_resource(
 	const char *rsc_id,
 	const char *existing_node, const char *preferred_node,
 	cib_t *	cib_conn) 
 {
 	char *later_s = NULL;
 	enum cib_errors rc = cib_ok;
 	char *id = NULL;
 	xmlNode *rule = NULL;
 	xmlNode *expr = NULL;
 	xmlNode *constraints = NULL;
 	xmlNode *fragment = NULL;
 	
 	xmlNode *can_run = NULL;
 	xmlNode *dont_run = NULL;
 
 	fragment = create_xml_node(NULL, XML_CIB_TAG_CONSTRAINTS);
 	constraints = fragment;
 
 	id = crm_concat("cli-prefer", rsc_id, '-');
 	can_run = create_xml_node(NULL, XML_CONS_TAG_RSC_LOCATION);
 	crm_xml_add(can_run, XML_ATTR_ID, id);
 	crm_free(id);
 
 	id = crm_concat("cli-standby", rsc_id, '-');
 	dont_run = create_xml_node(NULL, XML_CONS_TAG_RSC_LOCATION);
 	crm_xml_add(dont_run, XML_ATTR_ID, id);
 	crm_free(id);
 
 	if(move_lifetime) {
 		char *life = crm_strdup(move_lifetime);
 		char *life_mutable = life;
 		
 		ha_time_t *now = NULL;
 		ha_time_t *later = NULL;
 		ha_time_t *duration = parse_time_duration(&life_mutable);
 		
 		if(duration == NULL) {
 			CMD_ERR("Invalid duration specified: %s\n",
 				move_lifetime);
 			CMD_ERR("Please refer to"
 				" http://en.wikipedia.org/wiki/ISO_8601#Duration"
 				" for examples of valid durations\n");
 			crm_free(life);
 			return cib_invalid_argument;
 		}
 		now = new_ha_date(TRUE);
 		later = add_time(now, duration);
 		log_date(LOG_INFO, "now     ", now, ha_log_date|ha_log_time);
 		log_date(LOG_INFO, "later   ", later, ha_log_date|ha_log_time);
 		log_date(LOG_INFO, "duration", duration, ha_log_date|ha_log_time|ha_log_local);
 		later_s = date_to_string(later, ha_log_date|ha_log_time);
 		printf("Migration will take effect until: %s\n", later_s);
 
 		free_ha_date(duration);
 		free_ha_date(later);
 		free_ha_date(now);
 		crm_free(life);
 	}
 
 	if(existing_node == NULL) {
 		crm_log_xml_notice(can_run, "Deleting");
 		rc = cib_conn->cmds->delete(
 		    cib_conn, XML_CIB_TAG_CONSTRAINTS, dont_run, cib_options);
 		if(rc == cib_NOTEXISTS) {
 			rc = cib_ok;
 
 		} else if(rc != cib_ok) {
 			goto bail;
 		}
 
 	} else {
 		if(BE_QUIET == FALSE) {
 			fprintf(stderr,
 				"WARNING: Creating rsc_location constraint '%s'"
 				" with a score of -INFINITY for resource %s"
 				" on %s.\n",
 				ID(dont_run), rsc_id, existing_node);
 			CMD_ERR("\tThis will prevent %s from running"
 				" on %s until the constraint is removed using"
 				" the 'crm_resource -U' command or manually"
 				" with cibadmin\n", rsc_id, existing_node);
 			CMD_ERR("\tThis will be the case even if %s is"
 				" the last node in the cluster\n", existing_node);
 			CMD_ERR("\tThis message can be disabled with -Q\n");
 		}
 		
 		crm_xml_add(dont_run, "rsc", rsc_id);
 		
 		rule = create_xml_node(dont_run, XML_TAG_RULE);
 		expr = create_xml_node(rule, XML_TAG_EXPRESSION);
 		id = crm_concat("cli-standby-rule", rsc_id, '-');
 		crm_xml_add(rule, XML_ATTR_ID, id);
 		crm_free(id);
 		
 		crm_xml_add(rule, XML_RULE_ATTR_SCORE, MINUS_INFINITY_S);
 		crm_xml_add(rule, XML_RULE_ATTR_BOOLEAN_OP, "and");
 		
 		id = crm_concat("cli-standby-expr", rsc_id, '-');
 		crm_xml_add(expr, XML_ATTR_ID, id);
 		crm_free(id);
 		
 		crm_xml_add(expr, XML_EXPR_ATTR_ATTRIBUTE, "#uname");
 		crm_xml_add(expr, XML_EXPR_ATTR_OPERATION, "eq");
 		crm_xml_add(expr, XML_EXPR_ATTR_VALUE, existing_node);
 		crm_xml_add(expr, XML_EXPR_ATTR_TYPE, "string");
 
 		if(later_s) {
 		    expr = create_xml_node(rule, "date_expression");
 		    id = crm_concat("cli-standby-lifetime-end",rsc_id,'-');
 		    crm_xml_add(expr, XML_ATTR_ID, id);
 		    crm_free(id);			
 		    
 		    crm_xml_add(expr, "operation", "lt");
 		    crm_xml_add(expr, "end", later_s);
 		}
 		
 		add_node_copy(constraints, dont_run);
 	}
 
 	if(preferred_node == NULL) {
 		crm_log_xml_notice(can_run, "Deleting");
 		rc = cib_conn->cmds->delete(
 		    cib_conn, XML_CIB_TAG_CONSTRAINTS, can_run, cib_options);
 		if(rc == cib_NOTEXISTS) {
 			rc = cib_ok;
 
 		} else if(rc != cib_ok) {
 			goto bail;
 		}
 
 	} else {
 		crm_xml_add(can_run, "rsc", rsc_id);
 
 		rule = create_xml_node(can_run, XML_TAG_RULE);
 		expr = create_xml_node(rule, XML_TAG_EXPRESSION);
 		id = crm_concat("cli-prefer-rule", rsc_id, '-');
 		crm_xml_add(rule, XML_ATTR_ID, id);
 		crm_free(id);
 
 		crm_xml_add(rule, XML_RULE_ATTR_SCORE, INFINITY_S);
 		crm_xml_add(rule, XML_RULE_ATTR_BOOLEAN_OP, "and");
 	
 		id = crm_concat("cli-prefer-expr", rsc_id, '-');
 		crm_xml_add(expr, XML_ATTR_ID, id);
 		crm_free(id);
 
 		crm_xml_add(expr, XML_EXPR_ATTR_ATTRIBUTE, "#uname");
 		crm_xml_add(expr, XML_EXPR_ATTR_OPERATION, "eq");
 		crm_xml_add(expr, XML_EXPR_ATTR_VALUE, preferred_node);
 		crm_xml_add(expr, XML_EXPR_ATTR_TYPE, "string");
 
 		if(later_s) {
 		    expr = create_xml_node(rule, "date_expression");
 		    id = crm_concat("cli-prefer-lifetime-end", rsc_id, '-');
 		    crm_xml_add(expr, XML_ATTR_ID, id);
 		    crm_free(id);			
 		    
 		    crm_xml_add(expr, "operation", "lt");
 		    crm_xml_add(expr, "end", later_s);
 		}
 		
 		add_node_copy(constraints, can_run);
 	}
 
 	if(preferred_node != NULL || existing_node != NULL) {
 		crm_log_xml_notice(fragment, "CLI Update");
 		rc = cib_conn->cmds->update(
 		    cib_conn, XML_CIB_TAG_CONSTRAINTS, fragment, cib_options);
 	}
 
   bail:
 	free_xml(fragment);
 	free_xml(dont_run);
 	free_xml(can_run);
 	crm_free(later_s);
 	return rc;
 }
 
 static int
 list_resource_operations(
     const char *rsc_id, const char *host_uname, gboolean active, pe_working_set_t *data_set) 
 {
     resource_t *rsc = NULL;
     int opts = pe_print_printf|pe_print_rsconly|pe_print_suppres_nl;
     GListPtr ops = find_operations(rsc_id, host_uname, active, data_set);
     slist_iter(xml_op, xmlNode, ops, lpc,
 	       const char *op_rsc = crm_element_value(xml_op, "resource");
 	       const char *last = crm_element_value(xml_op, "last_run");
 	       const char *status_s = crm_element_value(xml_op, XML_LRM_ATTR_OPSTATUS);
 	       int status = crm_parse_int(status_s, "0");
 
 	       rsc = pe_find_resource(data_set->resources, op_rsc);
 	       rsc->fns->print(rsc, "", opts, stdout);
 	       
 	       fprintf(stdout, ": %s (node=%s, call=%s, rc=%s",
 		       ID(xml_op),
 		       crm_element_value(xml_op, XML_ATTR_UNAME),
 		       crm_element_value(xml_op, XML_LRM_ATTR_CALLID),
 		       crm_element_value(xml_op, XML_LRM_ATTR_RC));
 	       if(last) {
 		   time_t run_at = crm_parse_int(last, "0");
 		   fprintf(stdout, ", last-run=%s, exec=%sms\n",
 			    ctime(&run_at), crm_element_value(xml_op, "exec_time"));
 	       }
 	       fprintf(stdout, "): %s\n", op_status2text(status));
 	);
     return cib_ok;
 }
 
 #include "../pengine/pengine.h"
 
 static void show_location(resource_t *rsc) 
 {
     GListPtr list = rsc->rsc_location;
 
     slist_iter(cons, rsc_to_node_t, list, lpc,
 	       slist_iter(node, node_t, cons->node_list_rh, lpc2,
 			  fprintf(stdout, "+ '%s': %s = %s \n",
 				  cons->id, node->details->uname, score2char(node->weight));
 		   );
 	);
 }
 
 
 static void show_colocation(resource_t *rsc, gboolean dependants, gboolean raw) 
 {
     const char *prefix = "    ";
     GListPtr list = rsc->rsc_cons;
 
     if(dependants) {
 	 prefix = "   ";
 	 list = rsc->rsc_cons_lhs;
     }
     
     if(is_set(rsc->flags, pe_rsc_allocating)) {
 	/* Break colocation loops */
 	return;
     }
 	       
     set_bit(rsc->flags, pe_rsc_allocating);
     slist_iter(cons, rsc_colocation_t, list, lpc,
 	       resource_t *peer = cons->rsc_rh;
 
 	       if(dependants) {
 		   peer = cons->rsc_lh;
 	       }
 	       
 	       if(raw) {
 		   fprintf(stdout, "%s '%s': %s = %s\n", prefix, cons->id, peer->id, score2char(cons->score));
 		   continue;
 	       }
 	       
 	       if(dependants) {
 		   if(is_set(peer->flags, pe_rsc_allocating)) {
 		       continue;
 		   }
 		   show_colocation(peer, dependants, raw);
 	       }
 	       fprintf(stdout, "%s%s%s\n", prefix, peer->id, is_set(peer->flags, pe_rsc_allocating)?" (loop) ":"");
 	       if(!dependants) {
 		   show_colocation(peer, dependants, raw);
 	       }
 	);
     clear_bit(rsc->flags, pe_rsc_allocating);
 }	
 
 static struct crm_option long_options[] = {
     /* Top-level Options */
     {"help",    0, 0, '?', "\t\tThis text"},
     {"version", 0, 0, '$', "\t\tVersion information"  },
     {"verbose", 0, 0, 'V', "\t\tIncrease debug output"},
     {"quiet",   0, 0, 'Q', "\t\tPrint only the value on stdout\n"},
 
     {"resource",   1, 0, 'r', "\tResource ID" },
 
     {"-spacer-",1, 0, '-', "\nQueries:"},
     {"list",       0, 0, 'L', "\t\tList all resources"},
     {"list-raw",   0, 0, 'l', "\tList the IDs of all instansiated resources (no groups/clones/...)"},
     {"list-cts",   0, 0, 'c', NULL, 1},
     {"list-operations", 0, 0, 'O', "\tList active resource operations.  Optionally filtered by resource (-r) and/or node (-N)"},
     {"list-all-operations", 0, 0, 'o', "List all resource operations.  Optionally filtered by resource (-r) and/or node (-N)\n"},    
     {"query-xml",  0, 0, 'q', "\tQuery the definition of a resource"},
     {"locate",     0, 0, 'W', "\t\tDisplay the current location(s) of a resource"},
     {"stack",      0, 0, 'A', "\t\tDisplay the pre-requisits and depandants of a resource"},
     {"constraints",0, 0, 'a', "\tDisplay the (co)location constraints that apply to a resource"},
 
     {"-spacer-",	1, 0, '-', "\nCommands:"},
     {"set-parameter",   1, 0, 'p', "Set the named parameter for a resource. See also -m, --meta"},
     {"get-parameter",   1, 0, 'g', "Display the named parameter for a resource. See also -m, --meta"},
     {"delete-parameter",1, 0, 'd', "Delete the named parameter for a resource. See also -m, --meta"},
     {"get-property",    1, 0, 'G', "Display the 'class', 'type' or 'provider' of a resource", 1},
     {"set-property",    1, 0, 'S', "(Advanced) Set the class, type or provider of a resource", 1},
     {"move",    0, 0, 'M',
      "\t\tMove a resource from its current location, optionally specifying a destination (-N) and/or a period for which it should take effect (-u)"
      "\n\t\t\t\tIf -N is not specified, the cluster will force the resource to move by creating a rule for the current location and a score of -INFINITY"
      "\n\t\t\t\tNOTE: This will prevent the resource from running on this node until the constraint is removed with -U"},
     {"un-move", 0, 0, 'U', "\tRemove all constraints created by a move command"},
     
     {"-spacer-",	1, 0, '-', "\nAdvanced Commands:"},
     {"delete",     0, 0, 'D', "\t\tDelete a resource from the CIB"},
     {"fail",       0, 0, 'F', "\t\tTell the cluster this resource has failed"},
     {"refresh",    0, 0, 'R', "\t\t(Advanced) Refresh the CIB from the LRM"},
     {"cleanup",    0, 0, 'C', "\t\t(Advanced) Delete a resource from the LRM"},
     {"reprobe",    0, 0, 'P', "\t\t(Advanced) Re-check for resources started outside of the CRM\n"},    
     
     {"-spacer-",	1, 0, '-', "\nAdditional Options:"},
     {"node",		1, 0, 'N', "\tHost uname"},
     {"resource-type",	1, 0, 't', "Resource type (primitive, clone, group, ...)"},
     {"parameter-value", 1, 0, 'v', "Value to use with -p, -g or -d"},
     {"lifetime",	1, 0, 'u', "\tLifespan of migration constraints\n"},
     {"meta",		0, 0, 'm', "\t\tModify a resource's configuration option rather than one which is passed to the resource agent script. For use with -p, -g, -d"},
     {"utilization",	0, 0, 'z', "\tModify a resource's utilization attribute. For use with -p, -g, -d"},
     {"set-name",        1, 0, 's', "\t(Advanced) ID of the instance_attributes object to change"},
     {"nvpair",          1, 0, 'i', "\t(Advanced) ID of the nvpair object to change/delete"},    
     {"force",		0, 0, 'f', "\n" /* Is this actually true anymore? 
      "\t\tForce the resource to move by creating a rule for the current location and a score of -INFINITY"
      "\n\t\tThis should be used if the resource's stickiness and constraint scores total more than INFINITY (Currently 100,000)"
      "\n\t\tNOTE: This will prevent the resource from running on this node until the constraint is removed with -U or the --lifetime duration expires\n"*/ },
     
     {"xml-file", 1, 0, 'x', NULL, 1},\
 
      /* legacy options */
     {"host-uname", 1, 0, 'H', NULL, 1},
     {"migrate",    0, 0, 'M', NULL, 1},
     {"un-migrate", 0, 0, 'U', NULL, 1},
 
     {"-spacer-",	1, 0, '-', "\nExamples:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', "List the configured resources:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --list", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Display the current location of 'myResource':", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --locate", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Move 'myResource' to another machine:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --move", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Move 'myResource' to a specific machine:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --move --node altNode", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Allow (but not force) 'myResource' to move back to its original location:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --un-move", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Tell the cluster that 'myResource' failed:", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --fail", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Stop a 'myResource' (and anything that depends on it):", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --set-parameter target-role --meta --parameter-value Stopped", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Tell the cluster not to manage 'myResource':", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', "The cluster will not attempt to start or stop the resource under any circumstances."},
     {"-spacer-",	1, 0, '-', "Useful when performing maintenance tasks on a resource.", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --set-parameter is-managed --meta --parameter-value false", pcmk_option_example},
     {"-spacer-",	1, 0, '-', "Erase the operation history of 'myResource' on 'aNode':", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', "The cluster will 'forget' the existing resource state (including any errors) and attempt to recover the resource."},
     {"-spacer-",	1, 0, '-', "Useful when a resource had failed permanently and has been repaired by an administrator.", pcmk_option_paragraph},
     {"-spacer-",	1, 0, '-', " crm_resource --resource myResource --cleanup --node aNode", pcmk_option_example},
     
     {0, 0, 0, 0}
 };
 
 int
 main(int argc, char **argv)
 {
 	pe_working_set_t data_set;
 	xmlNode *cib_xml_copy = NULL;
 
 	cib_t *	cib_conn = NULL;
 	enum cib_errors rc = cib_ok;
 
 	gboolean need_cib = TRUE;
 	int option_index = 0;
 	int argerr = 0;
 	int flag;
 
 	crm_log_init(basename(argv[0]), LOG_ERR, FALSE, FALSE, argc, argv);
 	crm_set_options("V?$LRQxDCPp:WMUr:H:h:v:t:p:g:d:i:s:G:S:fx:lmzu:FOocqN:aA", "(query|command) [options]", long_options,
 			"Perform tasks related to cluster resources.\n  Allows resources to be queried (definition and location), modified, and moved around the cluster.\n");
 
 	if(argc < 2) {
 	    crm_help('?', LSB_EXIT_EINVAL);
 	}
 
 	while (1) {
 		flag = crm_get_option(argc, argv, &option_index);
 		if (flag == -1)
 			break;
 			    
 		switch(flag) {
 			case 'V':
 				cl_log_enable_stderr(TRUE);
 				alter_debug(DEBUG_INC);
 				break;
 			case '$':
 			case '?':
 				crm_help(flag, LSB_EXIT_OK);
 				break;
 			case 'x':
 				xml_file = crm_strdup(optarg);
 				break;
 			case 'Q':
 				BE_QUIET = TRUE;
 				break;
 			case 'm':
 				attr_set_type = XML_TAG_META_SETS;
 				break;
 			case 'z':
 				attr_set_type = XML_TAG_UTILIZATION;
 				break;
 			case 'u':
 				move_lifetime = crm_strdup(optarg);
 				break;
 			case 'f':
 				do_force = TRUE;
 				break;
 			case 'i':
 				prop_id = optarg;
 				break;
 			case 's':
 				prop_set = optarg;
 				break;
 			case 'r':
 				rsc_id = optarg;
 				break;
 			case 'v':
 				prop_value = optarg;
 				break;
 			case 't':
 				rsc_type = optarg;
 				break;
 			case 'R':
 			case 'P':
 				need_cib = FALSE;
 				rsc_cmd = flag;
 				break;		
 			case 'L':
 			case 'c':
 			case 'l':
 			case 'q':
 			case 'D':
 			case 'F':
 			case 'C':
 			case 'W':
 			case 'M':
 			case 'U':
 			case 'O':
 			case 'o':
 			case 'A':
 			case 'a':
 				rsc_cmd = flag;
 				break;	
 			case 'p':
 			case 'g':
 			case 'd':
 			case 'S':
 			case 'G':
 				prop_name = optarg;
 				rsc_cmd = flag;
 				break;
 			case 'h':
 			case 'H':
 			case 'N':
 				crm_debug_2("Option %c => %s", flag, optarg);
 				host_uname = optarg;
 				break;
 				
 			default:
 				CMD_ERR("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
 				++argerr;
 				break;
 		}
 	}
 
 	if (optind < argc && argv[optind] != NULL) {
 		CMD_ERR("non-option ARGV-elements: ");
 		while (optind < argc && argv[optind] != NULL) {
 			CMD_ERR("%s ", argv[optind++]);
 			++argerr;
 		}
 		CMD_ERR("\n");
 	}
 
 	if (optind > argc) {
 		++argerr;
 	}
 
 	if (argerr) {
 		crm_help('?', LSB_EXIT_GENERIC);
 	}
 
 	crm_malloc0(our_pid, 11);
 	if(our_pid != NULL) {
 		snprintf(our_pid, 10, "%d", getpid());
 		our_pid[10] = '\0';
 	}
 
 	if(do_force) {
 		crm_debug("Forcing...");
 		cib_options |= cib_scope_local|cib_quorum_override;
 	}
 
+	set_working_set_defaults(&data_set);
 	if(need_cib) {
 		resource_t *rsc = NULL;
 		if(xml_file != NULL) {
 		    cib_xml_copy = filename2xml(xml_file);
 
 		} else {
 			cib_conn = cib_new();
 			rc = cib_conn->cmds->signon(
 				cib_conn, crm_system_name, cib_command);
 			if(rc != cib_ok) {
 				CMD_ERR("Error signing on to the CIB service: %s\n",
 					cib_error2string(rc));
 				return rc;
 			}
 
 			cib_xml_copy = get_cib_copy(cib_conn);
 		}
 		
-		set_working_set_defaults(&data_set);
 		if(cli_config_update(&cib_xml_copy, NULL, FALSE) == FALSE) {
 		    return cib_STALE;
 		}
 
 		data_set.input = cib_xml_copy;
 		data_set.now = new_ha_date(TRUE);
 
 		cluster_status(&data_set);
 		if(rsc_id) {
 		    rsc = find_rsc_or_clone(rsc_id, &data_set);
 		}
 		if(rsc == NULL) {
 		    rc = cib_NOTEXISTS;
 		}
 	}
 
 	if(rsc_cmd == 'R'
 	   || rsc_cmd == 'C'
 	   || rsc_cmd == 'F'
 	   || rsc_cmd == 'P') {
 		GCHSource *src = NULL;
 		src = init_client_ipc_comms(CRM_SYSTEM_CRMD, crmd_msg_callback,
 				      NULL, &crmd_channel);
 
 		if(src == NULL) {
 			CMD_ERR("Error signing on to the CRMd service\n");
 			return 1;
 		}
 		
 		send_hello_message(
 			crmd_channel, our_pid, crm_system_name, "0", "1");
 
 		set_IPC_Channel_dnotify(src, resource_ipc_connection_destroy);
 	}
 
 	if(rsc_cmd == 'L') {
 		rc = cib_ok;
 		do_find_resource_list(&data_set, FALSE);
 		
 	} else if(rsc_cmd == 'l') {
 	    int found = 0;
 	    rc = cib_ok;
 	    slist_iter(
 		rsc, resource_t, data_set.resources, lpc,
 		found++;
 		print_raw_rsc(rsc);
 		);
 	    
 	    if(found == 0) {
 		printf("NO resources configured\n");
 		return cib_NOTEXISTS;
 	    }
 		
 	} else if(rsc_cmd == 'A') {
 	    resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
 	    xmlNode * cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set.input);
 	    if(rsc == NULL) {
 		CMD_ERR("Must supply a resource id with -r\n");
 		return cib_NOTEXISTS;
 	    }
 
 	    unpack_constraints(cib_constraints, &data_set);
 
 	    show_colocation(rsc, TRUE, FALSE);
 	    fprintf(stdout, "* %s\n", rsc->id);	       
 	    show_colocation(rsc, FALSE, FALSE);
 	    
 	} else if(rsc_cmd == 'a') {
 	    resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
 	    xmlNode * cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set.input);
 	    if(rsc == NULL) {
 		CMD_ERR("Must supply a resource id with -r\n");
 		return cib_NOTEXISTS;
 	    }
 	    unpack_constraints(cib_constraints, &data_set);
 
 	    show_colocation(rsc, TRUE, TRUE);
 	    fprintf(stdout, "* %s\n", rsc->id);	       
 	    show_colocation(rsc, FALSE, TRUE);
 
 	    show_location(rsc);
 	    
 	    
 	} else if(rsc_cmd == 'c') {
 	    int found = 0;
 	    rc = cib_ok;
 	    slist_iter(
 		rsc, resource_t, data_set.resources, lpc,
 		found++;
 		print_cts_rsc(rsc);
 		);
 	    print_cts_constraints(&data_set);
 		
 	} else if(rsc_cmd == 'C') {
 	    resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
 	    delete_lrm_rsc(crmd_channel, host_uname, rsc, &data_set);
 		
 	} else if(rsc_cmd == 'F') {
 		rc = fail_lrm_rsc(crmd_channel, host_uname, rsc_id, &data_set);
 		
 	} else if(rsc_cmd == 'O') {
 	    rc = list_resource_operations(rsc_id, host_uname, TRUE, &data_set);
 	    
 	} else if(rsc_cmd == 'o') {
 	    rc = list_resource_operations(rsc_id, host_uname, FALSE, &data_set);
 	    
 	} else if(rc == cib_NOTEXISTS) {
 		CMD_ERR("Resource %s not found: %s\n",
 			crm_str(rsc_id), cib_error2string(rc));
 
 	} else if(rsc_cmd == 'W') {
 		if(rsc_id == NULL) {
 			CMD_ERR("Must supply a resource id with -r\n");
 			return cib_NOTEXISTS;
 		} 
 		rc = do_find_resource(rsc_id, &data_set);
 		
 	} else if(rsc_cmd == 'q') {
 		if(rsc_id == NULL) {
 			CMD_ERR("Must supply a resource id with -r\n");
 			return cib_NOTEXISTS;
 		} 
 		rc = dump_resource(rsc_id, &data_set);
 
 	} else if(rsc_cmd == 'U') {
 		if(rsc_id == NULL) {
 			CMD_ERR("Must supply a resource id with -r\n");
 			return cib_NOTEXISTS;
 		} 
 		rc = move_resource(rsc_id, NULL, NULL, cib_conn);
 
 	} else if(rsc_cmd == 'M') {
 		node_t *dest = NULL;
 		node_t *current = NULL;
 		const char *current_uname = NULL;
 		resource_t *rsc = pe_find_resource(data_set.resources, rsc_id);
 		if(rsc != NULL && rsc->running_on != NULL) {
 			current = rsc->running_on->data;
 			if(current != NULL) {
 				current_uname = current->details->uname;
 			}
 		}
 
 		if(host_uname != NULL) {
 			dest = pe_find_node(data_set.nodes, host_uname);
 		}
 		
 		if(rsc == NULL) {
 			CMD_ERR("Resource %s not moved:"
 				" not found\n", rsc_id);
 
 		} else if(rsc->variant == pe_native
 			  && g_list_length(rsc->running_on) > 1) {
 			CMD_ERR("Resource %s not moved:"
 				" active on multiple nodes\n", rsc_id);
 			
 		} else if(host_uname != NULL && dest == NULL) {
 			CMD_ERR("Error performing operation: "
 				"%s is not a known node\n", host_uname);
 			rc = cib_NOTEXISTS;
 
 		} else if(host_uname != NULL
 			  && safe_str_eq(current_uname, host_uname)) {
 			CMD_ERR("Error performing operation: "
 				"%s is already active on %s\n",
 				rsc_id, host_uname);
 
 		} else if(current_uname != NULL
 			  && (do_force || host_uname == NULL)) {
 			rc = move_resource(rsc_id, current_uname,
 					      host_uname, cib_conn);
 
 			
 		} else if(host_uname != NULL) {
 			rc = move_resource(
 				rsc_id, NULL, host_uname, cib_conn);
 
 		} else {
 			CMD_ERR("Resource %s not moved: "
 				"not-active and no preferred location"
 				" specified.\n", rsc_id);
 			rc = cib_missing;
 		}
 		
 	} else if(rsc_cmd == 'G') {
 		if(rsc_id == NULL) {
 			CMD_ERR("Must supply a resource id with -r\n");
 			return cib_NOTEXISTS;
 		} 
 		rc = dump_resource_prop(rsc_id, prop_name, &data_set);
 
 	} else if(rsc_cmd == 'S') {
 		xmlNode *msg_data = NULL;
 		if(prop_value == NULL || strlen(prop_value) == 0) {
 			CMD_ERR("You need to supply a value with the -v option\n");
 			return CIBRES_MISSING_FIELD;
 
 		} else if(cib_conn == NULL) {
 			return cib_connection;
 		}
 
 		if(rsc_id == NULL) {
 			CMD_ERR("Must supply a resource id with -r\n");
 			return cib_NOTEXISTS;
 		} 
 		CRM_DEV_ASSERT(rsc_type != NULL);
 		CRM_DEV_ASSERT(prop_name != NULL);
 		CRM_DEV_ASSERT(prop_value != NULL);
 
 		msg_data = create_xml_node(NULL, rsc_type);
 		crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);
 		crm_xml_add(msg_data, prop_name, prop_value);
 		
 		rc = cib_conn->cmds->modify(
 		    cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
 		free_xml(msg_data);
 
 	} else if(rsc_cmd == 'g') {
 		if(rsc_id == NULL) {
 			CMD_ERR("Must supply a resource id with -r\n");
 			return cib_NOTEXISTS;
 		} 
 		rc = dump_resource_attr(rsc_id, prop_name, &data_set);
 
 	} else if(rsc_cmd == 'p') {
 		if(rsc_id == NULL) {
 			CMD_ERR("Must supply a resource id with -r\n");
 			return cib_NOTEXISTS;
 		} 
 		if(prop_value == NULL || strlen(prop_value) == 0) {
 			CMD_ERR("You need to supply a value with the -v option\n");
 			return CIBRES_MISSING_FIELD;
 		}
 		rc = set_resource_attr(rsc_id, prop_set, prop_id, prop_name,
 				       prop_value, cib_conn, &data_set);
 
 	} else if(rsc_cmd == 'd') {
 		if(rsc_id == NULL) {
 			CMD_ERR("Must supply a resource id with -r\n");
 			return cib_NOTEXISTS;
 		} 
 		rc = delete_resource_attr(rsc_id, prop_set, prop_id, prop_name,
 					  cib_conn, &data_set);
 
 	} else if(rsc_cmd == 'P') {
 		xmlNode *cmd = NULL;
 		
 		cmd = create_request(CRM_OP_REPROBE, NULL, host_uname,
 				     CRM_SYSTEM_CRMD, crm_system_name, our_pid);
 		send_ipc_message(crmd_channel, cmd);
 		free_xml(cmd);
 
 	} else if(rsc_cmd == 'R') {
 		refresh_lrm(crmd_channel, host_uname);
 
 	} else if(rsc_cmd == 'D') {
 		xmlNode *msg_data = NULL;
 		
 		if(rsc_id == NULL) {
 			CMD_ERR("Must supply a resource id with -r\n");
 			return cib_NOTEXISTS;
 		} 
 		if(rsc_type == NULL) {
 			CMD_ERR("You need to specify a resource type with -t");
 			return cib_NOTEXISTS;
 
 		} else if(cib_conn == NULL) {
 			return cib_connection;
 		}
 
 		msg_data = create_xml_node(NULL, rsc_type);
 		crm_xml_add(msg_data, XML_ATTR_ID, rsc_id);
 
 		rc = cib_conn->cmds->delete(
 		    cib_conn, XML_CIB_TAG_RESOURCES, msg_data, cib_options);
 		free_xml(msg_data);
 
 	} else {
 		CMD_ERR("Unknown command: %c\n", rsc_cmd);
 	}
 
 	if(cib_conn != NULL) {
 		cleanup_calculations(&data_set);
 		cib_conn->cmds->signoff(cib_conn);
 	}
 	if(rc == cib_no_quorum) {
 		CMD_ERR("Error performing operation: %s\n",
 			cib_error2string(rc));
 		CMD_ERR("Try using -f\n");
 
 	} else if(rc != cib_ok) {
 		CMD_ERR("Error performing operation: %s\n",
 			cib_error2string(rc));
 	}
 	
 	return rc;
 }