No OneTemporary
Actions

Size

1000 KB

Referenced Files

None

Subscribers

None

View Options

This file is larger than 256 KB, so syntax highlighting was skipped.

	diff --git a/cib/io.c b/cib/io.c
	index 4e2b24affe..068275f165 100644
	--- a/cib/io.c
	+++ b/cib/io.c
	@@ -1,490 +1,490 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <string.h>
	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <dirent.h>

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#include <sys/stat.h>

	#include <crm/crm.h>

	#include <cibio.h>
	#include <crm/cib.h>
	#include <crm/common/util.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/cib/internal.h>
	#include <crm/cluster.h>

	extern const char *cib_root;

	crm_trigger_t *cib_writer = NULL;
	gboolean initialized = FALSE;

	extern int cib_status;

	int write_cib_contents(gpointer p);

	static void
	cib_rename(const char *old)
	{
	int new_fd;
	char *new = crm_strdup_printf("%s/cib.auto.XXXXXX", cib_root);

	crm_err("Archiving unusable file %s as %s", old, new);
	umask(S_IWGRP \| S_IWOTH \| S_IROTH);
	if ((new_fd = mkstemp(new) < 0) \|\| (rename(old, new) < 0)) {
	crm_perror(LOG_ERR, "Couldn't rename %s as %s", old, new);
	crm_err("Disabling disk writes and continuing");
	cib_writes_enabled = FALSE;
	}
	if (new_fd > 0) {
	close(new_fd);
	}
	free(new);
	}

	/*
	* It is the callers responsibility to free the output of this function
	*/

	static xmlNode *
	retrieveCib(const char filename, const char sigfile)
	{
	xmlNode *root = NULL;

	crm_info("Reading cluster configuration file %s (digest: %s)",
	filename, sigfile);
	switch (cib_file_read_and_verify(filename, sigfile, &root)) {
	case -pcmk_err_cib_corrupt:
	crm_warn("Continuing but %s will NOT be used.", filename);
	break;

	case -pcmk_err_cib_modified:
	/* Archive the original files so the contents are not lost */
	crm_warn("Continuing but %s will NOT be used.", filename);
	cib_rename(filename);
	cib_rename(sigfile);
	break;
	}
	return root;
	}

	/*
	* for OSs without support for direntry->d_type, like Solaris
	*/
	#ifndef DT_UNKNOWN
	# define DT_UNKNOWN 0
	# define DT_FIFO 1
	# define DT_CHR 2
	# define DT_DIR 4
	# define DT_BLK 6
	# define DT_REG 8
	# define DT_LNK 10
	# define DT_SOCK 12
	# define DT_WHT 14
	#endif /DT_UNKNOWN/

	static int cib_archive_filter(const struct dirent * a)
	{
	int rc = 0;
	/* Looking for regular files (d_type = 8) starting with 'cib-' and not ending in .sig */
	struct stat s;
	char *a_path = crm_strdup_printf("%s/%s", cib_root, a->d_name);

	if(stat(a_path, &s) != 0) {
	rc = errno;
	crm_trace("%s - stat failed: %s (%d)", a->d_name, pcmk_strerror(rc), rc);
	rc = 0;

	} else if ((s.st_mode & S_IFREG) != S_IFREG) {
	unsigned char dtype;
	#ifdef HAVE_STRUCT_DIRENT_D_TYPE
	dtype = a->d_type;
	#else
	switch (s.st_mode & S_IFMT) {
	case S_IFREG: dtype = DT_REG; break;
	case S_IFDIR: dtype = DT_DIR; break;
	case S_IFCHR: dtype = DT_CHR; break;
	case S_IFBLK: dtype = DT_BLK; break;
	case S_IFLNK: dtype = DT_LNK; break;
	case S_IFIFO: dtype = DT_FIFO; break;
	case S_IFSOCK: dtype = DT_SOCK; break;
	default: dtype = DT_UNKNOWN; break;
	}
	#endif
	crm_trace("%s - wrong type (%d)", a->d_name, dtype);

	} else if(strstr(a->d_name, "cib-") != a->d_name) {
	crm_trace("%s - wrong prefix", a->d_name);

	} else if(strstr(a->d_name, ".sig") != NULL) {
	crm_trace("%s - wrong suffix", a->d_name);

	} else {
	crm_debug("%s - candidate", a->d_name);
	rc = 1;
	}

	free(a_path);
	return rc;
	}

	static int cib_archive_sort(const struct dirent a, const struct dirent b)
	{
	/* Order by creation date - most recently created file first */
	int rc = 0;
	struct stat buf;

	time_t a_age = 0;
	time_t b_age = 0;

	char *a_path = crm_strdup_printf("%s/%s", cib_root, a[0]->d_name);
	char *b_path = crm_strdup_printf("%s/%s", cib_root, b[0]->d_name);

	if(stat(a_path, &buf) == 0) {
	a_age = buf.st_ctime;
	}
	if(stat(b_path, &buf) == 0) {
	b_age = buf.st_ctime;
	}

	free(a_path);
	free(b_path);

	if(a_age > b_age) {
	rc = 1;
	} else if(a_age < b_age) {
	rc = -1;
	}

	crm_trace("%s (%u) vs. %s (%u) : %d", a[0]->d_name, a_age, b[0]->d_name, b_age, rc);
	return rc;
	}

	xmlNode *
	readCibXmlFile(const char dir, const char file, gboolean discard_status)
	{
	struct dirent **namelist = NULL;

	int lpc = 0;
	char *sigfile = NULL;
	char *filename = NULL;
	const char *name = NULL;
	const char *value = NULL;
	const char *validation = NULL;
	const char *use_valgrind = getenv("PCMK_valgrind_enabled");

	xmlNode *root = NULL;
	xmlNode *status = NULL;

	if (!crm_is_writable(dir, file, CRM_DAEMON_USER, NULL, FALSE)) {
	cib_status = -EACCES;
	return NULL;
	}

	filename = crm_concat(dir, file, '/');
	sigfile = crm_concat(filename, "sig", '.');

	cib_status = pcmk_ok;
	root = retrieveCib(filename, sigfile);
	free(filename);
	free(sigfile);

	if (root == NULL) {
	crm_warn("Primary configuration corrupt or unusable, trying backups in %s", cib_root);
	lpc = scandir(cib_root, &namelist, cib_archive_filter, cib_archive_sort);
	if (lpc < 0) {
	crm_perror(LOG_NOTICE, "scandir(%s) failed", cib_root);
	}
	}

	while (root == NULL && lpc > 1) {
	crm_debug("Testing %d candidates", lpc);

	lpc--;

	filename = crm_strdup_printf("%s/%s", cib_root, namelist[lpc]->d_name);
	sigfile = crm_concat(filename, "sig", '.');

	crm_info("Reading cluster configuration file %s (digest: %s)",
	filename, sigfile);
	if (cib_file_read_and_verify(filename, sigfile, &root) < 0) {
	crm_warn("Continuing but %s will NOT be used.", filename);
	} else {
	crm_notice("Continuing with last valid configuration archive: %s", filename);
	}

	free(namelist[lpc]);
	free(filename);
	free(sigfile);
	}
	free(namelist);

	if (root == NULL) {
	root = createEmptyCib(0);
	crm_warn("Continuing with an empty configuration.");
	}

	if (cib_writes_enabled && use_valgrind) {
	if (crm_is_true(use_valgrind) \|\| strstr(use_valgrind, "cib")) {
	cib_writes_enabled = FALSE;
	crm_err("* Disabling disk writes to avoid confusing Valgrind *");
	}
	}

	status = find_xml_node(root, XML_CIB_TAG_STATUS, FALSE);
	if (discard_status && status != NULL) {
	/* strip out the status section if there is one */
	free_xml(status);
	status = NULL;
	}
	if (status == NULL) {
	create_xml_node(root, XML_CIB_TAG_STATUS);
	}

	/* Do this before DTD validation happens */

	/* fill in some defaults */
	name = XML_ATTR_GENERATION_ADMIN;
	value = crm_element_value(root, name);
	if (value == NULL) {
	crm_warn("No value for %s was specified in the configuration.", name);
	- crm_warn("The reccomended course of action is to shutdown,"
	+ crm_warn("The recommended course of action is to shutdown,"
	" run crm_verify and fix any errors it reports.");
	crm_warn("We will default to zero and continue but may get"
	" confused about which configuration to use if"
	" multiple nodes are powered up at the same time.");
	crm_xml_add_int(root, name, 0);
	}

	name = XML_ATTR_GENERATION;
	value = crm_element_value(root, name);
	if (value == NULL) {
	crm_xml_add_int(root, name, 0);
	}

	name = XML_ATTR_NUMUPDATES;
	value = crm_element_value(root, name);
	if (value == NULL) {
	crm_xml_add_int(root, name, 0);
	}

	/* unset these and require the DC/CCM to update as needed */
	xml_remove_prop(root, XML_ATTR_DC_UUID);

	if (discard_status) {
	crm_log_xml_trace(root, "[on-disk]");
	}

	validation = crm_element_value(root, XML_ATTR_VALIDATION);
	if (validate_xml(root, NULL, TRUE) == FALSE) {
	crm_err("CIB does not validate with %s", crm_str(validation));
	cib_status = -pcmk_err_schema_validation;

	} else if (validation == NULL) {
	int version = 0;

	update_validation(&root, &version, 0, FALSE, FALSE);
	if (version > 0) {
	crm_notice("Enabling %s validation on"
	" the existing (sane) configuration", get_schema_name(version));
	} else {
	crm_err("CIB does not validate with any known DTD or schema");
	cib_status = -pcmk_err_schema_validation;
	}
	}

	return root;
	}

	/*
	* The caller should never free the return value
	*/
	xmlNode *
	get_the_CIB(void)
	{
	return the_cib;
	}

	gboolean
	uninitializeCib(void)
	{
	xmlNode *tmp_cib = the_cib;

	if (tmp_cib == NULL) {
	crm_debug("The CIB has already been deallocated.");
	return FALSE;
	}

	initialized = FALSE;
	the_cib = NULL;

	crm_debug("Deallocating the CIB.");

	free_xml(tmp_cib);

	crm_debug("The CIB has been deallocated.");

	return TRUE;
	}

	/*
	* This method will not free the old CIB pointer or the new one.
	* We rely on the caller to have saved a pointer to the old CIB
	* and to free the old/bad one depending on what is appropriate.
	*/
	gboolean
	initializeCib(xmlNode * new_cib)
	{
	if (new_cib == NULL) {
	return FALSE;
	}

	the_cib = new_cib;
	initialized = TRUE;
	return TRUE;
	}

	/*
	* This method will free the old CIB pointer on success and the new one
	* on failure.
	*/
	int
	activateCibXml(xmlNode * new_cib, gboolean to_disk, const char *op)
	{
	xmlNode *saved_cib = the_cib;

	CRM_ASSERT(new_cib != saved_cib);
	if (initializeCib(new_cib) == FALSE) {
	free_xml(new_cib);
	crm_err("Ignoring invalid or NULL CIB");

	if (saved_cib != NULL) {
	crm_warn("Reverting to last known CIB");
	if (initializeCib(saved_cib) == FALSE) {
	/* oh we are so dead */
	crm_crit("Couldn't re-initialize the old CIB!");
	exit(1);
	}

	} else {
	crm_crit("Could not write out new CIB and no saved" " version to revert to");
	}
	return -ENODATA;
	}

	free_xml(saved_cib);
	if (cib_writes_enabled && cib_status == pcmk_ok && to_disk) {
	crm_debug("Triggering CIB write for %s op", op);
	mainloop_set_trigger(cib_writer);
	}

	return pcmk_ok;
	}

	static void
	cib_diskwrite_complete(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode)
	{
	if (signo) {
	crm_notice("Disk write process terminated with signal %d (pid=%d, core=%d)", signo, pid,
	core);

	} else {
	do_crm_log(exitcode == 0 ? LOG_TRACE : LOG_ERR, "Disk write process exited (pid=%d, rc=%d)",
	pid, exitcode);
	}

	if (exitcode != 0 && cib_writes_enabled) {
	crm_err("Disabling disk writes after write failure");
	cib_writes_enabled = FALSE;
	}

	mainloop_trigger_complete(cib_writer);
	}

	int
	write_cib_contents(gpointer p)
	{
	int exit_rc = pcmk_ok;
	xmlNode *cib_local = NULL;

	/* Make a copy of the CIB to write (possibly in a forked child) */
	if (p) {
	/* Synchronous write out */
	cib_local = copy_xml(p);

	} else {
	int pid = 0;
	int bb_state = qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_STATE_GET, 0);

	/* Turn it off before the fork() to avoid:
	* - 2 processes writing to the same shared mem
	* - the child needing to disable it
	* (which would close it from underneath the parent)
	* This way, the shared mem files are already closed
	*/
	qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_FALSE);

	pid = fork();
	if (pid < 0) {
	crm_perror(LOG_ERR, "Disabling disk writes after fork failure");
	cib_writes_enabled = FALSE;
	return FALSE;
	}

	if (pid) {
	/* Parent */
	mainloop_child_add(pid, 0, "disk-writer", NULL, cib_diskwrite_complete);
	if (bb_state == QB_LOG_STATE_ENABLED) {
	/* Re-enable now that it it safe */
	qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_ENABLED, QB_TRUE);
	}

	return -1; /* -1 means 'still work to do' */
	}

	/* A-synchronous write out after a fork() */

	/* In theory we can scribble on "the_cib" here and not affect the parent
	* But lets be safe anyway
	*/
	cib_local = copy_xml(the_cib);
	}

	/* Write the CIB */
	exit_rc = cib_file_write_with_digest(cib_local, cib_root, "cib.xml");

	/* A nonzero exit code will cause further writes to be disabled */
	free_xml(cib_local);
	if (p == NULL) {
	/* Use _exit() because exit() could affect the parent adversely */
	_exit(exit_rc);
	}
	return exit_rc;
	}
	diff --git a/cib/messages.c b/cib/messages.c
	index ff197cd298..ff149f78d5 100644
	--- a/cib/messages.c
	+++ b/cib/messages.c
	@@ -1,577 +1,577 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <time.h>

	#include <sys/param.h>
	#include <sys/types.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>

	#include <crm/common/xml.h>
	#include <crm/common/ipcs.h>
	#include <crm/cluster/internal.h>

	#include <cibio.h>
	#include <cibmessages.h>
	#include <callbacks.h>

	#define MAX_DIFF_RETRY 5

	#ifdef CIBPIPE
	gboolean cib_is_master = TRUE;
	#else
	gboolean cib_is_master = FALSE;
	#endif

	xmlNode *the_cib = NULL;
	gboolean syncd_once = FALSE;
	extern const char *cib_our_uname;
	int revision_check(xmlNode * cib_update, xmlNode * cib_copy, int flags);
	int get_revision(xmlNode * xml_obj, int cur_revision);

	int updateList(xmlNode * local_cib, xmlNode * update_command, xmlNode * failed,
	int operation, const char *section);

	gboolean check_generation(xmlNode * newCib, xmlNode * oldCib);

	gboolean update_results(xmlNode * failed, xmlNode * target, const char *operation, int return_code);

	int cib_update_counter(xmlNode * xml_obj, const char *field, gboolean reset);

	int sync_our_cib(xmlNode * request, gboolean all);

	extern xmlNode cib_msg_copy(const xmlNode msg, gboolean with_data);
	extern gboolean cib_shutdown_flag;

	int
	cib_process_shutdown_req(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer)
	{
	#ifdef CIBPIPE
	return -EINVAL;
	#else
	int result = pcmk_ok;
	const char *host = crm_element_value(req, F_ORIG);

	*answer = NULL;

	if (crm_element_value(req, F_CIB_ISREPLY) == NULL) {
	crm_info("Shutdown REQ from %s", host);
	return pcmk_ok;

	} else if (cib_shutdown_flag) {
	crm_info("Shutdown ACK from %s", host);
	terminate_cib(__FUNCTION__, 0);
	return pcmk_ok;

	} else {
	crm_err("Shutdown ACK from %s - not shutting down", host);
	result = -EINVAL;
	}

	return result;
	#endif
	}

	int
	cib_process_default(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer)
	{
	int result = pcmk_ok;

	crm_trace("Processing \"%s\" event", op);
	*answer = NULL;

	if (op == NULL) {
	result = -EINVAL;
	crm_err("No operation specified");

	} else if (strcasecmp(CRM_OP_NOOP, op) == 0) {
	;

	} else {
	result = -EPROTONOSUPPORT;
	crm_err("Action [%s] is not supported by the CIB", op);
	}
	return result;
	}

	int
	cib_process_quit(const char op, int options, const char section, xmlNode * req, xmlNode * input,
	xmlNode * existing_cib, xmlNode result_cib, xmlNode answer)
	{
	int result = pcmk_ok;

	crm_trace("Processing \"%s\" event", op);

	crm_warn("The CRMd has asked us to exit... complying");
	crm_exit(pcmk_ok);
	return result;
	}

	int
	cib_process_readwrite(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer)
	{
	#ifdef CIBPIPE
	return -EINVAL;
	#else
	int result = pcmk_ok;

	crm_trace("Processing \"%s\" event", op);

	if (safe_str_eq(op, CIB_OP_ISMASTER)) {
	if (cib_is_master == TRUE) {
	result = pcmk_ok;
	} else {
	result = -EPERM;
	}
	return result;
	}

	if (safe_str_eq(op, CIB_OP_MASTER)) {
	if (cib_is_master == FALSE) {
	crm_info("We are now in R/W mode");
	cib_is_master = TRUE;
	syncd_once = TRUE;

	} else {
	crm_debug("We are still in R/W mode");
	}

	} else if (cib_is_master) {
	crm_info("We are now in R/O mode");
	cib_is_master = FALSE;
	}

	return result;
	#endif
	}

	int sync_in_progress = 0;
	void
	send_sync_request(const char *host)
	{
	xmlNode *sync_me = create_xml_node(NULL, "sync-me");

	crm_info("Requesting re-sync from peer");
	sync_in_progress++;

	crm_xml_add(sync_me, F_TYPE, "cib");
	crm_xml_add(sync_me, F_CIB_OPERATION, CIB_OP_SYNC_ONE);
	crm_xml_add(sync_me, F_CIB_DELEGATED, cib_our_uname);

	send_cluster_message(host ? crm_get_peer(0, host) : NULL, crm_msg_cib, sync_me, FALSE);
	free_xml(sync_me);
	}

	int
	cib_process_ping(const char op, int options, const char section, xmlNode * req, xmlNode * input,
	xmlNode * existing_cib, xmlNode result_cib, xmlNode answer)
	{
	#ifdef CIBPIPE
	return -EINVAL;
	#else
	const char *host = crm_element_value(req, F_ORIG);
	const char *seq = crm_element_value(req, F_CIB_PING_ID);
	char *digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, CRM_FEATURE_SET);

	static struct qb_log_callsite *cs = NULL;

	crm_trace("Processing \"%s\" event %s from %s", op, seq, host);
	*answer = create_xml_node(NULL, XML_CRM_TAG_PING);

	crm_xml_add(*answer, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	crm_xml_add(*answer, XML_ATTR_DIGEST, digest);
	crm_xml_add(*answer, F_CIB_PING_ID, seq);

	if (cs == NULL) {
	cs = qb_log_callsite_get(__func__, __FILE__, __FUNCTION__, LOG_TRACE, __LINE__, crm_trace_nonlog);
	}
	if (cs && cs->targets) {
	/* Append additional detail so the reciever can log the differences */
	add_message_xml(*answer, F_CIB_CALLDATA, the_cib);

	} else {
	/* Always include at least the version details */
	const char *tag = TYPE(the_cib);
	xmlNode *shallow = create_xml_node(NULL, tag);

	copy_in_properties(shallow, the_cib);
	add_message_xml(*answer, F_CIB_CALLDATA, shallow);
	free_xml(shallow);
	}

	crm_info("Reporting our current digest to %s: %s for %s.%s.%s (%p %d)",
	host, digest,
	crm_element_value(existing_cib, XML_ATTR_GENERATION_ADMIN),
	crm_element_value(existing_cib, XML_ATTR_GENERATION),
	crm_element_value(existing_cib, XML_ATTR_NUMUPDATES),
	existing_cib,
	cs && cs->targets);

	free(digest);

	return pcmk_ok;
	#endif
	}

	int
	cib_process_sync(const char op, int options, const char section, xmlNode * req, xmlNode * input,
	xmlNode * existing_cib, xmlNode result_cib, xmlNode answer)
	{
	#ifdef CIBPIPE
	return -EINVAL;
	#else
	return sync_our_cib(req, TRUE);
	#endif
	}

	int
	cib_process_upgrade_server(const char op, int options, const char section, xmlNode * req, xmlNode * input,
	xmlNode * existing_cib, xmlNode result_cib, xmlNode answer)
	{
	#ifdef CIBPIPE
	return -EINVAL;
	#else
	int rc = pcmk_ok;

	*answer = NULL;

	if(crm_element_value(req, F_CIB_SCHEMA_MAX)) {
	return cib_process_upgrade(
	op, options, section, req, input, existing_cib, result_cib, answer);

	} else {
	int new_version = 0;
	int current_version = 0;
	xmlNode *scratch = copy_xml(existing_cib);
	const char *host = crm_element_value(req, F_ORIG);
	const char *value = crm_element_value(existing_cib, XML_ATTR_VALIDATION);

	crm_trace("Processing \"%s\" event", op);
	if (value != NULL) {
	current_version = get_schema_version(value);
	}

	rc = update_validation(&scratch, &new_version, 0, TRUE, TRUE);
	if (new_version > current_version) {
	xmlNode *up = create_xml_node(NULL, __FUNCTION__);

	rc = pcmk_ok;
	crm_notice("Upgrade request from %s verified", host);

	crm_xml_add(up, F_TYPE, "cib");
	crm_xml_add(up, F_CIB_OPERATION, CIB_OP_UPGRADE);
	crm_xml_add(up, F_CIB_SCHEMA_MAX, get_schema_name(new_version));
	crm_xml_add(up, F_CIB_DELEGATED, host);
	crm_xml_add(up, F_CIB_CLIENTID, crm_element_value(req, F_CIB_CLIENTID));
	crm_xml_add(up, F_CIB_CALLOPTS, crm_element_value(req, F_CIB_CALLOPTS));
	crm_xml_add(up, F_CIB_CALLID, crm_element_value(req, F_CIB_CALLID));

	if (cib_legacy_mode() && cib_is_master) {
	rc = cib_process_upgrade(
	op, options, section, up, input, existing_cib, result_cib, answer);

	} else {
	send_cluster_message(NULL, crm_msg_cib, up, FALSE);
	}

	free_xml(up);

	} else if(rc == pcmk_ok) {
	rc = -pcmk_err_schema_unchanged;
	}

	free_xml(scratch);
	}
	return rc;
	#endif
	}

	int
	cib_process_sync_one(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer)
	{
	#ifdef CIBPIPE
	return -EINVAL;
	#else
	return sync_our_cib(req, FALSE);
	#endif
	}

	int
	cib_server_process_diff(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer)
	{
	int rc = pcmk_ok;

	if (cib_is_master) {
	/* the master is never waiting for a resync */
	sync_in_progress = 0;
	}

	if (sync_in_progress > MAX_DIFF_RETRY) {
	/* request another full-sync,
	* the last request may have been lost
	*/
	sync_in_progress = 0;
	}

	if (sync_in_progress) {
	int diff_add_updates = 0;
	int diff_add_epoch = 0;
	int diff_add_admin_epoch = 0;

	int diff_del_updates = 0;
	int diff_del_epoch = 0;
	int diff_del_admin_epoch = 0;

	cib_diff_version_details(input,
	&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
	&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);

	sync_in_progress++;
	crm_notice("Not applying diff %d.%d.%d -> %d.%d.%d (sync in progress)",
	diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
	diff_add_admin_epoch, diff_add_epoch, diff_add_updates);
	return -pcmk_err_diff_resync;
	}

	rc = cib_process_diff(op, options, section, req, input, existing_cib, result_cib, answer);
	crm_trace("result: %s (%d), %s", pcmk_strerror(rc), rc, cib_is_master?"master":"slave");

	if (rc == -pcmk_err_diff_resync && cib_is_master == FALSE) {
	free_xml(*result_cib);
	*result_cib = NULL;
	send_sync_request(NULL);

	} else if (rc == -pcmk_err_diff_resync) {
	rc = -pcmk_err_diff_failed;
	if (options & cib_force_diff) {
	crm_warn("Not requesting full refresh in R/W mode");
	}

	} else if(rc != pcmk_ok && cib_legacy_mode()) {
	crm_warn("Something went wrong in compatibility mode, requesting full refresh");
	xml_log_patchset(LOG_INFO, __FUNCTION__, input);
	free_xml(*result_cib);
	*result_cib = NULL;
	send_sync_request(NULL);
	}

	return rc;
	}

	int
	cib_process_replace_svr(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer)
	{
	const char *tag = crm_element_name(input);
	int rc =
	cib_process_replace(op, options, section, req, input, existing_cib, result_cib, answer);
	if (rc == pcmk_ok && safe_str_eq(tag, XML_TAG_CIB)) {
	sync_in_progress = 0;
	}
	return rc;
	}

	static int
	delete_cib_object(xmlNode * parent, xmlNode * delete_spec)
	{
	const char *object_name = NULL;
	const char *object_id = NULL;
	xmlNode *equiv_node = NULL;
	int result = pcmk_ok;

	if (delete_spec != NULL) {
	object_name = crm_element_name(delete_spec);
	}
	object_id = crm_element_value(delete_spec, XML_ATTR_ID);

	crm_trace("Processing: <%s id=%s>", crm_str(object_name), crm_str(object_id));

	if (delete_spec == NULL) {
	result = -EINVAL;

	} else if (parent == NULL) {
	result = -EINVAL;

	} else if (object_id == NULL) {
	/* placeholder object */
	equiv_node = find_xml_node(parent, object_name, FALSE);

	} else {
	equiv_node = find_entity(parent, object_name, object_id);
	}

	if (result != pcmk_ok) {
	; /* nothing */

	} else if (equiv_node == NULL) {
	result = pcmk_ok;

	} else if (xml_has_children(delete_spec) == FALSE) {
	/* only leaves are deleted */
	crm_debug("Removing leaf: <%s id=%s>", crm_str(object_name), crm_str(object_id));
	free_xml(equiv_node);
	equiv_node = NULL;

	} else {
	xmlNode *child = NULL;

	for (child = __xml_first_child(delete_spec); child != NULL; child = __xml_next(child)) {
	int tmp_result = delete_cib_object(equiv_node, child);

	/* only the first error is likely to be interesting */
	if (tmp_result != pcmk_ok && result == pcmk_ok) {
	result = tmp_result;
	}
	}
	}

	return result;
	}

	int
	cib_process_delete_absolute(const char op, int options, const char section, xmlNode * req,
	xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
	xmlNode ** answer)
	{
	xmlNode *failed = NULL;
	int result = pcmk_ok;
	xmlNode *update_section = NULL;

	crm_trace("Processing \"%s\" event for section=%s", op, crm_str(section));
	if (safe_str_eq(XML_CIB_TAG_SECTION_ALL, section)) {
	section = NULL;

	} else if (safe_str_eq(XML_TAG_CIB, section)) {
	section = NULL;

	} else if (safe_str_eq(crm_element_name(input), XML_TAG_CIB)) {
	section = NULL;
	}

	CRM_CHECK(strcasecmp(CIB_OP_DELETE, op) == 0, return -EINVAL);

	if (input == NULL) {
	crm_err("Cannot perform modification with no data");
	return -EINVAL;
	}

	failed = create_xml_node(NULL, XML_TAG_FAILED);

	update_section = get_object_root(section, *result_cib);
	result = delete_cib_object(update_section, input);
	update_results(failed, input, op, result);

	if (xml_has_children(failed)) {
	CRM_CHECK(result != pcmk_ok, result = -EINVAL);
	}

	if (result != pcmk_ok) {
	crm_log_xml_err(failed, "CIB Update failures");
	*answer = failed;

	} else {
	free_xml(failed);
	}

	return result;
	}

	gboolean
	check_generation(xmlNode * newCib, xmlNode * oldCib)
	{
	if (cib_compare_generation(newCib, oldCib) >= 0) {
	return TRUE;
	}

	crm_warn("Generation from update is older than the existing one");
	return FALSE;
	}

	#ifndef CIBPIPE
	int
	sync_our_cib(xmlNode * request, gboolean all)
	{
	int result = pcmk_ok;
	char *digest = NULL;
	const char *host = crm_element_value(request, F_ORIG);
	const char *op = crm_element_value(request, F_CIB_OPERATION);

	xmlNode *replace_request = cib_msg_copy(request, FALSE);

	CRM_CHECK(the_cib != NULL,;);
	CRM_CHECK(replace_request != NULL,;);

	crm_debug("Syncing CIB to %s", all ? "all peers" : host);
	if (all == FALSE && host == NULL) {
	crm_log_xml_err(request, "bad sync");
	}

	/* remove the "all == FALSE" condition
	*
	* sync_from was failing, the local client wasnt being notified
	- * because it didnt know it was a reply
	+ * because it didn't know it was a reply
	* setting this does not prevent the other nodes from applying it
	* if all == TRUE
	*/
	if (host != NULL) {
	crm_xml_add(replace_request, F_CIB_ISREPLY, host);
	}
	if (all) {
	xml_remove_prop(replace_request, F_CIB_HOST);
	}

	crm_xml_add(replace_request, F_CIB_OPERATION, CIB_OP_REPLACE);
	crm_xml_add(replace_request, "original_" F_CIB_OPERATION, op);
	crm_xml_add(replace_request, F_CIB_GLOBAL_UPDATE, XML_BOOLEAN_TRUE);

	crm_xml_add(replace_request, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, CRM_FEATURE_SET);
	crm_xml_add(replace_request, XML_ATTR_DIGEST, digest);

	add_message_xml(replace_request, F_CIB_CALLDATA, the_cib);

	if (send_cluster_message
	(all ? NULL : crm_get_peer(0, host), crm_msg_cib, replace_request, FALSE) == FALSE) {
	result = -ENOTCONN;
	}
	free_xml(replace_request);
	free(digest);
	return result;
	}
	#endif
	diff --git a/crmd/control.c b/crmd/control.c
	index a3dd5ddefb..56a5f4114e 100644
	--- a/crmd/control.c
	+++ b/crmd/control.c
	@@ -1,1153 +1,1153 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>

	#include <crm/msg_xml.h>

	#include <crm/pengine/rules.h>
	#include <crm/cluster/internal.h>
	#include <crm/cluster/election.h>
	#include <crm/common/ipcs.h>

	#include <crmd.h>
	#include <crmd_fsa.h>
	#include <fsa_proto.h>
	#include <crmd_messages.h>
	#include <crmd_callbacks.h>
	#include <crmd_lrm.h>
	#include <tengine.h>
	#include <throttle.h>

	#include <sys/types.h>
	#include <sys/stat.h>

	qb_ipcs_service_t *ipcs = NULL;

	extern gboolean crm_connect_corosync(crm_cluster_t * cluster);
	extern void crmd_ha_connection_destroy(gpointer user_data);

	void crm_shutdown(int nsig);
	gboolean crm_read_options(gpointer user_data);

	gboolean fsa_has_quorum = FALSE;
	crm_trigger_t *fsa_source = NULL;
	crm_trigger_t *config_read = NULL;
	bool no_quorum_suicide_escalation = FALSE;

	static gboolean
	election_timeout_popped(gpointer data)
	{
	/* Not everyone voted */
	crm_info("Election failed: Declaring ourselves the winner");
	register_fsa_input(C_TIMER_POPPED, I_ELECTION_DC, NULL);
	return FALSE;
	}

	/* A_HA_CONNECT */
	void
	do_ha_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	gboolean registered = FALSE;
	static crm_cluster_t *cluster = NULL;

	if (cluster == NULL) {
	cluster = calloc(1, sizeof(crm_cluster_t));
	}

	if (action & A_HA_DISCONNECT) {
	crm_cluster_disconnect(cluster);
	crm_info("Disconnected from the cluster");

	set_bit(fsa_input_register, R_HA_DISCONNECTED);
	}

	if (action & A_HA_CONNECT) {
	crm_set_status_callback(&peer_update_callback);
	crm_set_autoreap(FALSE);

	if (is_openais_cluster()) {
	#if SUPPORT_COROSYNC
	registered = crm_connect_corosync(cluster);
	#endif
	} else if (is_heartbeat_cluster()) {
	#if SUPPORT_HEARTBEAT
	cluster->destroy = crmd_ha_connection_destroy;
	cluster->hb_dispatch = crmd_ha_msg_callback;

	registered = crm_cluster_connect(cluster);
	fsa_cluster_conn = cluster->hb_conn;

	crm_trace("Be informed of Node Status changes");
	if (registered &&
	fsa_cluster_conn->llc_ops->set_nstatus_callback(fsa_cluster_conn,
	crmd_ha_status_callback,
	fsa_cluster_conn) != HA_OK) {

	crm_err("Cannot set nstatus callback: %s",
	fsa_cluster_conn->llc_ops->errmsg(fsa_cluster_conn));
	registered = FALSE;
	}

	crm_trace("Be informed of CRM Client Status changes");
	if (registered &&
	fsa_cluster_conn->llc_ops->set_cstatus_callback(fsa_cluster_conn,
	crmd_client_status_callback,
	fsa_cluster_conn) != HA_OK) {

	crm_err("Cannot set cstatus callback: %s",
	fsa_cluster_conn->llc_ops->errmsg(fsa_cluster_conn));
	registered = FALSE;
	}

	if (registered) {
	crm_trace("Requesting an initial dump of CRMD client_status");
	fsa_cluster_conn->llc_ops->client_status(fsa_cluster_conn, NULL, CRM_SYSTEM_CRMD,
	-1);
	}
	#endif
	}
	fsa_election = election_init(NULL, cluster->uname, 60000/60s/, election_timeout_popped);
	fsa_our_uname = cluster->uname;
	fsa_our_uuid = cluster->uuid;
	if(cluster->uuid == NULL) {
	crm_err("Could not obtain local uuid");
	registered = FALSE;
	}

	if (registered == FALSE) {
	set_bit(fsa_input_register, R_HA_DISCONNECTED);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	return;
	}

	populate_cib_nodes(node_update_none, __FUNCTION__);
	clear_bit(fsa_input_register, R_HA_DISCONNECTED);
	crm_info("Connected to the cluster");
	}

	if (action & ~(A_HA_CONNECT \| A_HA_DISCONNECT)) {
	crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__);
	}
	}

	static bool
	need_spawn_pengine_from_crmd(void)
	{
	static int result = -1;

	if (result != -1)
	return result;
	if (!is_heartbeat_cluster()) {
	result = 0;
	return result;
	}

	/* NULL, or "strange" value: rather spawn from here. */
	result = TRUE;
	crm_str_to_boolean(daemon_option("crmd_spawns_pengine"), &result);
	return result;
	}

	/* A_SHUTDOWN */
	void
	do_shutdown(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	/* just in case */
	set_bit(fsa_input_register, R_SHUTDOWN);

	if (need_spawn_pengine_from_crmd()) {
	if (is_set(fsa_input_register, pe_subsystem->flag_connected)) {
	crm_info("Terminating the %s", pe_subsystem->name);
	if (stop_subsystem(pe_subsystem, TRUE) == FALSE) {
	/* its gone... */
	crm_err("Faking %s exit", pe_subsystem->name);
	clear_bit(fsa_input_register, pe_subsystem->flag_connected);
	} else {
	crm_info("Waiting for subsystems to exit");
	crmd_fsa_stall(FALSE);
	}
	}
	crm_info("All subsystems stopped, continuing");
	}

	if (stonith_api) {
	- /* Prevent it from comming up again */
	+ /* Prevent it from coming up again */
	clear_bit(fsa_input_register, R_ST_REQUIRED);

	crm_info("Disconnecting STONITH...");
	stonith_api->cmds->disconnect(stonith_api);
	}
	}

	/* A_SHUTDOWN_REQ */
	void
	do_shutdown_req(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	xmlNode *msg = NULL;

	crm_info("Sending shutdown request to %s", crm_str(fsa_our_dc));
	msg = create_request(CRM_OP_SHUTDOWN_REQ, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);

	/* set_bit(fsa_input_register, R_STAYDOWN); */
	if (send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) {
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	free_xml(msg);
	}

	extern crm_ipc_t *attrd_ipc;
	extern char *max_generation_from;
	extern xmlNode *max_generation_xml;
	extern GHashTable *resource_history;
	extern GHashTable *voted;
	extern GHashTable *metadata_hash;
	extern char *te_client_id;

	void log_connected_client(gpointer key, gpointer value, gpointer user_data);

	void
	log_connected_client(gpointer key, gpointer value, gpointer user_data)
	{
	crm_client_t *client = value;

	crm_err("%s is still connected at exit", crm_client_name(client));
	}

	int
	crmd_fast_exit(int rc)
	{
	if (is_set(fsa_input_register, R_STAYDOWN)) {
	crm_warn("Inhibiting respawn: %d -> %d", rc, 100);
	rc = 100;
	}

	if (rc == pcmk_ok && is_set(fsa_input_register, R_IN_RECOVERY)) {
	crm_err("Could not recover from internal error");
	rc = pcmk_err_generic;
	}
	return crm_exit(rc);
	}

	int
	crmd_exit(int rc)
	{
	GListPtr gIter = NULL;
	GMainLoop *mloop = crmd_mainloop;

	static bool in_progress = FALSE;

	if(in_progress && rc == 0) {
	crm_debug("Exit is already in progress");
	return rc;

	} else if(in_progress) {
	crm_notice("Error during shutdown process, terminating now: %s (%d)", pcmk_strerror(rc), rc);
	crm_write_blackbox(SIGTRAP, NULL);
	crmd_fast_exit(rc);
	}

	in_progress = TRUE;
	crm_trace("Preparing to exit: %d", rc);

	/* Suppress secondary errors resulting from us disconnecting everything */
	set_bit(fsa_input_register, R_HA_DISCONNECTED);

	/* Close all IPC servers and clients to ensure any and all shared memory files are cleaned up */

	if(ipcs) {
	crm_trace("Closing IPC server");
	mainloop_del_ipc_server(ipcs);
	ipcs = NULL;
	}

	if (attrd_ipc) {
	crm_trace("Closing attrd connection");
	crm_ipc_close(attrd_ipc);
	crm_ipc_destroy(attrd_ipc);
	attrd_ipc = NULL;
	}

	if (pe_subsystem && pe_subsystem->client && pe_subsystem->client->ipcs) {
	crm_trace("Disconnecting Policy Engine");
	qb_ipcs_disconnect(pe_subsystem->client->ipcs);
	}

	if(stonith_api) {
	crm_trace("Disconnecting fencing API");
	clear_bit(fsa_input_register, R_ST_REQUIRED);
	stonith_api->cmds->free(stonith_api); stonith_api = NULL;
	}

	if (rc == pcmk_ok && crmd_mainloop == NULL) {
	crm_debug("No mainloop detected");
	rc = EPROTO;
	}

	/* On an error, just get out.
	*
	* Otherwise, make the effort to have mainloop exit gracefully so
	* that it (mostly) cleans up after itself and valgrind has less
	* to report on - allowing real errors stand out
	*/
	if(rc != pcmk_ok) {
	crm_notice("Forcing immediate exit: %s (%d)", pcmk_strerror(rc), rc);
	crm_write_blackbox(SIGTRAP, NULL);
	return crmd_fast_exit(rc);
	}

	/* Clean up as much memory as possible for valgrind */

	for (gIter = fsa_message_queue; gIter != NULL; gIter = gIter->next) {
	fsa_data_t *fsa_data = gIter->data;

	crm_info("Dropping %s: [ state=%s cause=%s origin=%s ]",
	fsa_input2string(fsa_data->fsa_input),
	fsa_state2string(fsa_state),
	fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
	delete_fsa_input(fsa_data);
	}

	clear_bit(fsa_input_register, R_MEMBERSHIP);
	g_list_free(fsa_message_queue); fsa_message_queue = NULL;

	free(pe_subsystem); pe_subsystem = NULL;
	free(te_subsystem); te_subsystem = NULL;
	free(cib_subsystem); cib_subsystem = NULL;

	if (metadata_hash) {
	crm_trace("Destroying reload cache with %d members", g_hash_table_size(metadata_hash));
	g_hash_table_destroy(metadata_hash); metadata_hash = NULL;
	}

	election_fini(fsa_election);
	fsa_election = NULL;

	cib_delete(fsa_cib_conn);
	fsa_cib_conn = NULL;

	verify_stopped(fsa_state, LOG_WARNING);
	clear_bit(fsa_input_register, R_LRM_CONNECTED);
	lrm_state_destroy_all();

	/* This basically will not work, since mainloop has a reference to it */
	mainloop_destroy_trigger(fsa_source); fsa_source = NULL;

	mainloop_destroy_trigger(config_read); config_read = NULL;
	mainloop_destroy_trigger(stonith_reconnect); stonith_reconnect = NULL;
	mainloop_destroy_trigger(transition_trigger); transition_trigger = NULL;

	crm_client_cleanup();
	crm_peer_destroy();

	crm_timer_stop(transition_timer);
	crm_timer_stop(integration_timer);
	crm_timer_stop(finalization_timer);
	crm_timer_stop(election_trigger);
	election_timeout_stop(fsa_election);
	crm_timer_stop(shutdown_escalation_timer);
	crm_timer_stop(wait_timer);
	crm_timer_stop(recheck_timer);

	free(transition_timer); transition_timer = NULL;
	free(integration_timer); integration_timer = NULL;
	free(finalization_timer); finalization_timer = NULL;
	free(election_trigger); election_trigger = NULL;
	election_fini(fsa_election);
	free(shutdown_escalation_timer); shutdown_escalation_timer = NULL;
	free(wait_timer); wait_timer = NULL;
	free(recheck_timer); recheck_timer = NULL;

	free(fsa_our_dc_version); fsa_our_dc_version = NULL;
	free(fsa_our_uname); fsa_our_uname = NULL;
	free(fsa_our_uuid); fsa_our_uuid = NULL;
	free(fsa_our_dc); fsa_our_dc = NULL;

	free(fsa_cluster_name); fsa_cluster_name = NULL;

	free(te_uuid); te_uuid = NULL;
	free(te_client_id); te_client_id = NULL;
	free(fsa_pe_ref); fsa_pe_ref = NULL;
	free(failed_stop_offset); failed_stop_offset = NULL;
	free(failed_start_offset); failed_start_offset = NULL;

	free(max_generation_from); max_generation_from = NULL;
	free_xml(max_generation_xml); max_generation_xml = NULL;

	mainloop_destroy_signal(SIGPIPE);
	mainloop_destroy_signal(SIGUSR1);
	mainloop_destroy_signal(SIGTERM);
	mainloop_destroy_signal(SIGTRAP);
	mainloop_destroy_signal(SIGCHLD);

	if (mloop) {
	int lpc = 0;
	GMainContext *ctx = g_main_loop_get_context(crmd_mainloop);

	/* Don't re-enter this block */
	crmd_mainloop = NULL;

	crm_trace("Draining mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));

	while(g_main_context_pending(ctx) && lpc < 10) {
	lpc++;
	crm_trace("Iteration %d", lpc);
	g_main_context_dispatch(ctx);
	}

	crm_trace("Closing mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx));
	g_main_loop_quit(mloop);

	#if SUPPORT_HEARTBEAT
	/* Do this only after g_main_loop_quit().
	*
	* This interface was broken (incomplete) since it was introduced.
	* ->delete() does cleanup and free most of it, but it does not
	* actually remove and destroy the corresponding GSource, so the next
	* prepare/check iteratioin would find a corrupt (because partially
	* freed) GSource, and segfault.
	*
	* Apparently one was supposed to store the GSource as returned by
	* G_main_add_ll_cluster(), and g_source_destroy() that "by hand".
	*
	* But no-one ever did this, not even in the old hb code when this was
	* introduced.
	*
	* Note that fsa_cluster_conn was set as an "alias" to cluster->hb_conn
	* in do_ha_control() right after crm_cluster_connect(), and only
	* happens to still point at that object, because do_ha_control() does
	* not reset it to NULL after crm_cluster_disconnect() above does
	* reset cluster->hb_conn to NULL.
	* Not sure if that's something to cleanup, too.
	*
	* I'll try to fix this up in heartbeat proper, so ->delete
	* will actually remove, and destroy, and unref, and free this thing.
	* Doing so after g_main_loop_quit() is valid with both old,
	* and eventually fixed heartbeat.
	*
	* If we introduce the "by hand" destroy/remove/unref,
	* this may break again once heartbeat is fixed :-(
	*
	* -- Lars Ellenberg
	*/
	if (fsa_cluster_conn) {
	crm_trace("Deleting heartbeat api object");
	fsa_cluster_conn->llc_ops->delete(fsa_cluster_conn);
	fsa_cluster_conn = NULL;
	}
	#endif

	/* Won't do anything yet, since we're inside it now */
	g_main_loop_unref(mloop);

	crm_trace("Done %d", rc);
	}

	/* Graceful */
	return rc;
	}

	/* A_EXIT_0, A_EXIT_1 */
	void
	do_exit(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	int exit_code = pcmk_ok;
	int log_level = LOG_INFO;
	const char *exit_type = "gracefully";

	if (action & A_EXIT_1) {
	/* exit_code = pcmk_err_generic; */
	log_level = LOG_ERR;
	exit_type = "forcefully";
	exit_code = pcmk_err_generic;
	}

	verify_stopped(cur_state, LOG_ERR);
	do_crm_log(log_level, "Performing %s - %s exiting the CRMd",
	fsa_action2string(action), exit_type);

	crm_info("[%s] stopped (%d)", crm_system_name, exit_code);
	crmd_exit(exit_code);
	}

	static void sigpipe_ignore(int nsig) { return; }

	/* A_STARTUP */
	void
	do_startup(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	int was_error = 0;

	crm_debug("Registering Signal Handlers");
	mainloop_add_signal(SIGTERM, crm_shutdown);
	mainloop_add_signal(SIGPIPE, sigpipe_ignore);

	fsa_source = mainloop_add_trigger(G_PRIORITY_HIGH, crm_fsa_trigger, NULL);
	config_read = mainloop_add_trigger(G_PRIORITY_HIGH, crm_read_options, NULL);
	transition_trigger = mainloop_add_trigger(G_PRIORITY_LOW, te_graph_trigger, NULL);

	crm_debug("Creating CIB and LRM objects");
	fsa_cib_conn = cib_new();

	lrm_state_init_local();

	/* set up the timers */
	transition_timer = calloc(1, sizeof(fsa_timer_t));
	integration_timer = calloc(1, sizeof(fsa_timer_t));
	finalization_timer = calloc(1, sizeof(fsa_timer_t));
	election_trigger = calloc(1, sizeof(fsa_timer_t));
	shutdown_escalation_timer = calloc(1, sizeof(fsa_timer_t));
	wait_timer = calloc(1, sizeof(fsa_timer_t));
	recheck_timer = calloc(1, sizeof(fsa_timer_t));

	if (election_trigger != NULL) {
	election_trigger->source_id = 0;
	election_trigger->period_ms = -1;
	election_trigger->fsa_input = I_DC_TIMEOUT;
	election_trigger->callback = crm_timer_popped;
	election_trigger->repeat = FALSE;
	} else {
	was_error = TRUE;
	}

	if (transition_timer != NULL) {
	transition_timer->source_id = 0;
	transition_timer->period_ms = -1;
	transition_timer->fsa_input = I_PE_CALC;
	transition_timer->callback = crm_timer_popped;
	transition_timer->repeat = FALSE;
	} else {
	was_error = TRUE;
	}

	if (integration_timer != NULL) {
	integration_timer->source_id = 0;
	integration_timer->period_ms = -1;
	integration_timer->fsa_input = I_INTEGRATED;
	integration_timer->callback = crm_timer_popped;
	integration_timer->repeat = FALSE;
	} else {
	was_error = TRUE;
	}

	if (finalization_timer != NULL) {
	finalization_timer->source_id = 0;
	finalization_timer->period_ms = -1;
	finalization_timer->fsa_input = I_FINALIZED;
	finalization_timer->callback = crm_timer_popped;
	finalization_timer->repeat = FALSE;
	/* for possible enabling... a bug in the join protocol left
	* a slave in S_PENDING while we think its in S_NOT_DC
	*
	* raising I_FINALIZED put us into a transition loop which is
	* never resolved.
	* in this loop we continually send probes which the node
	* NACK's because its in S_PENDING
	*
	* if we have nodes where heartbeat is active but the
	* CRM is not... then this will be handled in the
	* integration phase
	*/
	finalization_timer->fsa_input = I_ELECTION;

	} else {
	was_error = TRUE;
	}

	if (shutdown_escalation_timer != NULL) {
	shutdown_escalation_timer->source_id = 0;
	shutdown_escalation_timer->period_ms = -1;
	shutdown_escalation_timer->fsa_input = I_STOP;
	shutdown_escalation_timer->callback = crm_timer_popped;
	shutdown_escalation_timer->repeat = FALSE;
	} else {
	was_error = TRUE;
	}

	if (wait_timer != NULL) {
	wait_timer->source_id = 0;
	wait_timer->period_ms = 2000;
	wait_timer->fsa_input = I_NULL;
	wait_timer->callback = crm_timer_popped;
	wait_timer->repeat = FALSE;
	} else {
	was_error = TRUE;
	}

	if (recheck_timer != NULL) {
	recheck_timer->source_id = 0;
	recheck_timer->period_ms = -1;
	recheck_timer->fsa_input = I_PE_CALC;
	recheck_timer->callback = crm_timer_popped;
	recheck_timer->repeat = FALSE;
	} else {
	was_error = TRUE;
	}

	/* set up the sub systems */
	cib_subsystem = calloc(1, sizeof(struct crm_subsystem_s));
	te_subsystem = calloc(1, sizeof(struct crm_subsystem_s));
	pe_subsystem = calloc(1, sizeof(struct crm_subsystem_s));

	if (cib_subsystem != NULL) {
	cib_subsystem->pid = -1;
	cib_subsystem->name = CRM_SYSTEM_CIB;
	cib_subsystem->flag_connected = R_CIB_CONNECTED;
	cib_subsystem->flag_required = R_CIB_REQUIRED;

	} else {
	was_error = TRUE;
	}

	if (te_subsystem != NULL) {
	te_subsystem->pid = -1;
	te_subsystem->name = CRM_SYSTEM_TENGINE;
	te_subsystem->flag_connected = R_TE_CONNECTED;
	te_subsystem->flag_required = R_TE_REQUIRED;

	} else {
	was_error = TRUE;
	}

	if (pe_subsystem != NULL) {
	pe_subsystem->pid = -1;
	pe_subsystem->path = CRM_DAEMON_DIR;
	pe_subsystem->name = CRM_SYSTEM_PENGINE;
	pe_subsystem->command = CRM_DAEMON_DIR "/" CRM_SYSTEM_PENGINE;
	pe_subsystem->args = NULL;
	pe_subsystem->flag_connected = R_PE_CONNECTED;
	pe_subsystem->flag_required = R_PE_REQUIRED;

	} else {
	was_error = TRUE;
	}

	if (was_error == FALSE && need_spawn_pengine_from_crmd()) {
	if (start_subsystem(pe_subsystem) == FALSE) {
	was_error = TRUE;
	}
	}

	if (was_error) {
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}

	}

	static int32_t
	crmd_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	crm_trace("Connection %p", c);
	if (crm_client_new(c, uid, gid) == NULL) {
	return -EIO;
	}
	return 0;
	}

	static void
	crmd_ipc_created(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p", c);
	}

	static int32_t
	crmd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	crm_client_t *client = crm_client_get(c);

	xmlNode *msg = crm_ipcs_recv(client, data, size, &id, &flags);

	crm_trace("Invoked: %s", crm_client_name(client));
	crm_ipcs_send_ack(client, id, flags, "ack", __FUNCTION__, __LINE__);

	if (msg == NULL) {
	return 0;
	}

	#if ENABLE_ACL
	CRM_ASSERT(client->user != NULL);
	crm_acl_get_set_user(msg, F_CRM_USER, client->user);
	#endif

	crm_trace("Processing msg from %s", crm_client_name(client));
	crm_log_xml_trace(msg, "CRMd[inbound]");

	crm_xml_add(msg, F_CRM_SYS_FROM, client->id);
	if (crmd_authorize_message(msg, client, NULL)) {
	route_message(C_IPC_MESSAGE, msg);
	}

	trigger_fsa(fsa_source);
	free_xml(msg);
	return 0;
	}

	static int32_t
	crmd_ipc_closed(qb_ipcs_connection_t * c)
	{
	crm_client_t *client = crm_client_get(c);
	struct crm_subsystem_s *the_subsystem = NULL;

	if (client == NULL) {
	return 0;
	}

	crm_trace("Connection %p", c);

	if (client->userdata == NULL) {
	crm_trace("Client hadn't registered with us yet");

	} else if (strcasecmp(CRM_SYSTEM_PENGINE, client->userdata) == 0) {
	the_subsystem = pe_subsystem;

	} else if (strcasecmp(CRM_SYSTEM_TENGINE, client->userdata) == 0) {
	the_subsystem = te_subsystem;

	} else if (strcasecmp(CRM_SYSTEM_CIB, client->userdata) == 0) {
	the_subsystem = cib_subsystem;
	}

	if (the_subsystem != NULL) {
	the_subsystem->source = NULL;
	the_subsystem->client = NULL;
	crm_info("Received HUP from %s:[%d]", the_subsystem->name, the_subsystem->pid);

	} else {
	/* else that was a transient client */
	crm_trace("Received HUP from transient client");
	}

	crm_trace("Disconnecting client %s (%p)", crm_client_name(client), client);
	free(client->userdata);
	crm_client_destroy(client);

	trigger_fsa(fsa_source);
	return 0;
	}

	static void
	crmd_ipc_destroy(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p", c);
	crmd_ipc_closed(c);
	}

	/* A_STOP */
	void
	do_stop(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	crm_trace("Closing IPC server");
	mainloop_del_ipc_server(ipcs); ipcs = NULL;
	register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
	}

	/* A_STARTED */
	void
	do_started(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	static struct qb_ipcs_service_handlers crmd_callbacks = {
	.connection_accept = crmd_ipc_accept,
	.connection_created = crmd_ipc_created,
	.msg_process = crmd_ipc_dispatch,
	.connection_closed = crmd_ipc_closed,
	.connection_destroyed = crmd_ipc_destroy
	};

	if (cur_state != S_STARTING) {
	crm_err("Start cancelled... %s", fsa_state2string(cur_state));
	return;

	} else if (is_set(fsa_input_register, R_MEMBERSHIP) == FALSE) {
	crm_info("Delaying start, no membership data (%.16llx)", R_MEMBERSHIP);

	crmd_fsa_stall(TRUE);
	return;

	} else if (is_set(fsa_input_register, R_LRM_CONNECTED) == FALSE) {
	crm_info("Delaying start, LRM not connected (%.16llx)", R_LRM_CONNECTED);

	crmd_fsa_stall(TRUE);
	return;

	} else if (is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) {
	crm_info("Delaying start, CIB not connected (%.16llx)", R_CIB_CONNECTED);

	crmd_fsa_stall(TRUE);
	return;

	} else if (is_set(fsa_input_register, R_READ_CONFIG) == FALSE) {
	crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG);

	crmd_fsa_stall(TRUE);
	return;

	} else if (is_set(fsa_input_register, R_PEER_DATA) == FALSE) {

	/* try reading from HA */
	crm_info("Delaying start, No peer data (%.16llx)", R_PEER_DATA);

	#if SUPPORT_HEARTBEAT
	if (is_heartbeat_cluster()) {
	HA_Message *msg = NULL;

	crm_trace("Looking for a HA message");
	msg = fsa_cluster_conn->llc_ops->readmsg(fsa_cluster_conn, 0);
	if (msg != NULL) {
	crm_trace("There was a HA message");
	ha_msg_del(msg);
	}
	}
	#endif
	crmd_fsa_stall(TRUE);
	return;
	}

	crm_debug("Init server comms");
	ipcs = crmd_ipc_server_init(&crmd_callbacks);
	if (ipcs == NULL) {
	crm_err("Failed to create IPC server: shutting down and inhibiting respawn");
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}

	if (stonith_reconnect == NULL) {
	int dummy;

	stonith_reconnect = mainloop_add_trigger(G_PRIORITY_LOW, te_connect_stonith, &dummy);
	}
	set_bit(fsa_input_register, R_ST_REQUIRED);
	mainloop_set_trigger(stonith_reconnect);

	crm_notice("The local CRM is operational");
	clear_bit(fsa_input_register, R_STARTING);
	register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL);
	}

	/* A_RECOVER */
	void
	do_recover(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	set_bit(fsa_input_register, R_IN_RECOVERY);
	crm_warn("Fast-tracking shutdown in response to errors");

	register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL);
	}

	/* INDENT-OFF */
	pe_cluster_option crmd_opts[] = {
	/* name, old-name, validate, values, default, short description, long description */
	{ "dc-version", NULL, "string", NULL, "none", NULL,
	"Version of Pacemaker on the cluster's DC.",
	"Includes the hash which identifies the exact changeset it was built from. Used for diagnostic purposes."
	},
	{ "cluster-infrastructure", NULL, "string", NULL, "heartbeat", NULL,
	"The messaging stack on which Pacemaker is currently running.",
	"Used for informational and diagnostic purposes." },
	{ XML_CONFIG_ATTR_DC_DEADTIME, "dc_deadtime", "time", NULL, "20s", &check_time,
	"How long to wait for a response from other nodes during startup.",
	"The \"correct\" value will depend on the speed/load of your network and the type of switches used."
	},
	{ XML_CONFIG_ATTR_RECHECK, "cluster_recheck_interval", "time",
	"Zero disables polling. Positive values are an interval in seconds (unless other SI units are specified. eg. 5min)",
	"15min", &check_timer,
	"Polling interval for time based changes to options, resource parameters and constraints.",
	"The Cluster is primarily event driven, however the configuration can have elements that change based on time."
	" To ensure these changes take effect, we can optionally poll the cluster's status for changes."
	},

	#ifdef RHEL7_COMPAT
	/* this interface is expected to change but was released in RHEL 7 */
	{ "notification-agent", NULL, "string", NULL, "/dev/null", &check_script,
	"Notification script or tool to be called after significant cluster events",
	"Full path to a script or binary that will be invoked when resources start/stop/fail, fencing occurs or nodes join/leave the cluster.\n"
	"Must exist on all nodes in the cluster."
	},
	{ "notification-recipient", NULL, "string", NULL, "", NULL,
	"Destination for notifications (Optional)",
	"Where should the supplied script send notifications to. Useful to avoid hard-coding this in the script."
	},
	#endif

	{ "load-threshold", NULL, "percentage", NULL, "80%", &check_utilization,
	"The maximum amount of system resources that should be used by nodes in the cluster",
	"The cluster will slow down its recovery process when the amount of system resources used"
	" (currently CPU) approaches this limit",
	},
	{ "node-action-limit", NULL, "integer", NULL, "0", &check_number,
	"The maximum number of jobs that can be scheduled per node. Defaults to 2x cores"},
	{ XML_CONFIG_ATTR_ELECTION_FAIL, "election_timeout", "time", NULL, "2min", &check_timer,
	"* Advanced Use Only *.", "If need to adjust this value, it probably indicates the presence of a bug."
	},
	{ XML_CONFIG_ATTR_FORCE_QUIT, "shutdown_escalation", "time", NULL, "20min", &check_timer,
	"* Advanced Use Only *.", "If need to adjust this value, it probably indicates the presence of a bug."
	},
	{ "crmd-integration-timeout", NULL, "time", NULL, "3min", &check_timer,
	"* Advanced Use Only *.", "If need to adjust this value, it probably indicates the presence of a bug."
	},
	{ "crmd-finalization-timeout", NULL, "time", NULL, "30min", &check_timer,
	"* Advanced Use Only *.", "If you need to adjust this value, it probably indicates the presence of a bug."
	},
	{ "crmd-transition-delay", NULL, "time", NULL, "0s", &check_timer,
	"* Advanced Use Only *\n"
	"Enabling this option will slow down cluster recovery under all conditions",
	"Delay cluster recovery for the configured interval to allow for additional/related events to occur.\n"
	"Useful if your configuration is sensitive to the order in which ping updates arrive."
	},
	{ "stonith-watchdog-timeout", NULL, "time", NULL, NULL, &check_timer,
	"How long to wait before we can assume nodes are safely down", NULL
	},
	{ "no-quorum-policy", "no_quorum_policy", "enum", "stop, freeze, ignore, suicide", "stop", &check_quorum, NULL, NULL },

	#if SUPPORT_PLUGIN
	{ XML_ATTR_EXPECTED_VOTES, NULL, "integer", NULL, "2", &check_number, "The number of nodes expected to be in the cluster", "Used to calculate quorum in openais based clusters." },
	#endif
	};
	/* INDENT-ON */

	void
	crmd_metadata(void)
	{
	config_metadata("CRM Daemon", "1.0",
	"CRM Daemon Options",
	"This is a fake resource that details the options that can be configured for the CRM Daemon.",
	crmd_opts, DIMOF(crmd_opts));
	}

	static void
	verify_crmd_options(GHashTable * options)
	{
	verify_all_options(options, crmd_opts, DIMOF(crmd_opts));
	}

	static const char *
	crmd_pref(GHashTable * options, const char *name)
	{
	return get_cluster_pref(options, crmd_opts, DIMOF(crmd_opts), name);
	}

	static void
	config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	#ifdef RHEL7_COMPAT
	const char *script = NULL;
	#endif
	const char *value = NULL;
	GHashTable *config_hash = NULL;
	crm_time_t *now = crm_time_new(NULL);
	long st_timeout = 0;
	long sbd_timeout = 0;

	if (rc != pcmk_ok) {
	fsa_data_t *msg_data = NULL;

	crm_err("Local CIB query resulted in an error: %s", pcmk_strerror(rc));
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);

	if (rc == -EACCES \|\| rc == -pcmk_err_schema_validation) {
	crm_err("The cluster is mis-configured - shutting down and staying down");
	set_bit(fsa_input_register, R_STAYDOWN);
	}
	goto bail;
	}

	crm_debug("Call %d : Parsing CIB options", call_id);
	config_hash =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	unpack_instance_attributes(output, output, XML_CIB_TAG_PROPSET, NULL, config_hash,
	CIB_OPTIONS_FIRST, FALSE, now);

	verify_crmd_options(config_hash);

	#ifdef RHEL7_COMPAT
	script = crmd_pref(config_hash, "notification-agent");
	value = crmd_pref(config_hash, "notification-recipient");
	crmd_enable_notifications(script, value);
	#endif

	value = crmd_pref(config_hash, XML_CONFIG_ATTR_DC_DEADTIME);
	election_trigger->period_ms = crm_get_msec(value);

	value = crmd_pref(config_hash, "node-action-limit"); /* Also checks migration-limit */
	throttle_update_job_max(value);

	value = crmd_pref(config_hash, "load-threshold");
	if(value) {
	throttle_load_target = strtof(value, NULL) / 100;
	}

	value = getenv("SBD_WATCHDOG_TIMEOUT");
	sbd_timeout = crm_get_msec(value);

	value = crmd_pref(config_hash, "stonith-watchdog-timeout");
	st_timeout = crm_get_msec(value);

	if(st_timeout > 0 && !daemon_option_enabled(crm_system_name, "watchdog")) {
	do_crm_log_always(LOG_EMERG, "Shutting down pacemaker, no watchdog device configured");
	crmd_exit(DAEMON_RESPAWN_STOP);

	} else if(!daemon_option_enabled(crm_system_name, "watchdog")) {
	crm_trace("Watchdog disabled");

	} else if(value == NULL && sbd_timeout > 0) {
	char *timeout = NULL;

	st_timeout = 2 * sbd_timeout / 1000;
	timeout = crm_strdup_printf("%lds", st_timeout);
	crm_notice("Setting stonith-watchdog-timeout=%s", timeout);

	update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	"stonith-watchdog-timeout", timeout, FALSE, NULL, NULL);
	free(timeout);

	} else if(st_timeout <= 0) {
	crm_notice("Watchdog enabled but stonith-watchdog-timeout is disabled");

	} else if(st_timeout < sbd_timeout) {
	do_crm_log_always(LOG_EMERG, "Shutting down pacemaker, stonith-watchdog-timeout (%ldms) is too short (must be greater than %ldms)",
	st_timeout, sbd_timeout);
	crmd_exit(DAEMON_RESPAWN_STOP);
	}

	value = crmd_pref(config_hash, "no-quorum-policy");
	if (safe_str_eq(value, "suicide") && daemon_option_enabled(crm_system_name, "watchdog")) {
	no_quorum_suicide_escalation = TRUE;
	}

	value = crmd_pref(config_hash, XML_CONFIG_ATTR_FORCE_QUIT);
	shutdown_escalation_timer->period_ms = crm_get_msec(value);
	/* How long to declare an election over - even if not everyone voted */
	crm_debug("Shutdown escalation occurs after: %dms", shutdown_escalation_timer->period_ms);

	value = crmd_pref(config_hash, XML_CONFIG_ATTR_ELECTION_FAIL);
	election_timeout_set_period(fsa_election, crm_get_msec(value));

	value = crmd_pref(config_hash, XML_CONFIG_ATTR_RECHECK);
	recheck_timer->period_ms = crm_get_msec(value);
	crm_debug("Checking for expired actions every %dms", recheck_timer->period_ms);

	value = crmd_pref(config_hash, "crmd-transition-delay");
	transition_timer->period_ms = crm_get_msec(value);

	value = crmd_pref(config_hash, "crmd-integration-timeout");
	integration_timer->period_ms = crm_get_msec(value);

	value = crmd_pref(config_hash, "crmd-finalization-timeout");
	finalization_timer->period_ms = crm_get_msec(value);

	#if SUPPORT_COROSYNC
	if (is_classic_ais_cluster()) {
	value = crmd_pref(config_hash, XML_ATTR_EXPECTED_VOTES);
	crm_debug("Sending expected-votes=%s to corosync", value);
	send_cluster_text(crm_class_quorum, value, TRUE, NULL, crm_msg_ais);
	}
	#endif

	free(fsa_cluster_name);
	fsa_cluster_name = NULL;

	value = g_hash_table_lookup(config_hash, "cluster-name");
	if (value) {
	fsa_cluster_name = strdup(value);
	}

	set_bit(fsa_input_register, R_READ_CONFIG);
	crm_trace("Triggering FSA: %s", __FUNCTION__);
	mainloop_set_trigger(fsa_source);

	g_hash_table_destroy(config_hash);
	bail:
	crm_time_free(now);
	}

	gboolean
	crm_read_options(gpointer user_data)
	{
	int call_id =
	fsa_cib_conn->cmds->query(fsa_cib_conn, XML_CIB_TAG_CRMCONFIG, NULL, cib_scope_local);

	fsa_register_cib_callback(call_id, FALSE, NULL, config_query_callback);
	crm_trace("Querying the CIB... call %d", call_id);
	return TRUE;
	}

	/* A_READCONFIG */
	void
	do_read_config(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	throttle_init();
	mainloop_set_trigger(config_read);
	}

	void
	crm_shutdown(int nsig)
	{
	if (crmd_mainloop != NULL && g_main_is_running(crmd_mainloop)) {
	if (is_set(fsa_input_register, R_SHUTDOWN)) {
	crm_err("Escalating the shutdown");
	register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL);

	} else {
	set_bit(fsa_input_register, R_SHUTDOWN);
	register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);

	if (shutdown_escalation_timer->period_ms < 1) {
	const char *value = crmd_pref(NULL, XML_CONFIG_ATTR_FORCE_QUIT);
	int msec = crm_get_msec(value);

	crm_debug("Using default shutdown escalation: %dms", msec);
	shutdown_escalation_timer->period_ms = msec;
	}

	- /* cant rely on this... */
	+ /* can't rely on this... */
	crm_notice("Requesting shutdown, upper limit is %dms",
	shutdown_escalation_timer->period_ms);
	crm_timer_start(shutdown_escalation_timer);
	}

	} else {
	crm_info("exit from shutdown");
	crmd_exit(pcmk_ok);
	}
	}
	diff --git a/crmd/election.c b/crmd/election.c
	index dee475069b..cfa68ab84d 100644
	--- a/crmd/election.c
	+++ b/crmd/election.c
	@@ -1,267 +1,267 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/
	#include <crm_internal.h>

	#include <sys/time.h>
	#include <sys/resource.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <crm/cluster/internal.h>
	#include <crm/cluster/election.h>
	#include <crm/crm.h>
	#include <crmd_fsa.h>
	#include <crmd_messages.h>
	#include <crmd_callbacks.h>
	#include <tengine.h>

	#define STORM_INTERVAL 2 /* in seconds */
	#define STORM_MULTIPLIER 5 /* multiplied by the number of nodes */

	/* A_ELECTION_VOTE */
	void
	do_election_vote(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	gboolean not_voting = FALSE;

	/* don't vote if we're in one of these states or wanting to shut down */
	switch (cur_state) {
	case S_STARTING:
	case S_RECOVERY:
	case S_STOPPING:
	case S_TERMINATE:
	crm_warn("Not voting in election, we're in state %s", fsa_state2string(cur_state));
	not_voting = TRUE;
	break;
	case S_ELECTION:
	case S_INTEGRATION:
	case S_RELEASE_DC:
	break;
	default:
	crm_err("Broken? Voting in state %s", fsa_state2string(cur_state));
	break;
	}

	if (not_voting == FALSE) {
	if (is_set(fsa_input_register, R_STARTING)) {
	not_voting = TRUE;
	}
	}

	if (not_voting) {
	if (AM_I_DC) {
	register_fsa_input(C_FSA_INTERNAL, I_RELEASE_DC, NULL);

	} else {
	register_fsa_input(C_FSA_INTERNAL, I_PENDING, NULL);
	}
	return;
	}

	election_vote(fsa_election);
	return;
	}

	void
	do_election_check(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	if (fsa_state != S_ELECTION) {
	crm_debug("Ignore election check: we not in an election");

	} else if(election_check(fsa_election)) {
	register_fsa_input(C_FSA_INTERNAL, I_ELECTION_DC, NULL);
	}

	return;
	}

	#define loss_dampen 2 /* in seconds */

	/* A_ELECTION_COUNT */
	void
	do_election_count_vote(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	enum election_result rc = 0;
	ha_msg_input_t *vote = fsa_typed_data(fsa_dt_ha_msg);

	if(crm_peer_cache == NULL) {
	if(is_not_set(fsa_input_register, R_SHUTDOWN)) {
	crm_err("Internal error, no peer cache");
	}
	return;
	}

	rc = election_count_vote(fsa_election, vote->msg, cur_state != S_STARTING);
	switch(rc) {
	case election_start:
	election_reset(fsa_election);
	register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
	break;

	case election_lost:
	update_dc(NULL);

	if (fsa_input_register & R_THE_DC) {
	register_fsa_input(C_FSA_INTERNAL, I_RELEASE_DC, NULL);
	fsa_cib_conn->cmds->set_slave(fsa_cib_conn, cib_scope_local);

	} else if (cur_state != S_STARTING) {
	register_fsa_input(C_FSA_INTERNAL, I_PENDING, NULL);
	}
	break;

	case election_in_progress:
	break;
	default:
	crm_err("Unhandled election result: %d", rc);
	}
	}

	/* A_ELECT_TIMER_START, A_ELECTION_TIMEOUT */
	/* we won */
	void
	do_election_timer_ctrl(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	}

	static void
	feature_update_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	if (rc != pcmk_ok) {
	fsa_data_t *msg_data = NULL;

	crm_notice("Update failed: %s (%d)", pcmk_strerror(rc), rc);
	register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
	}
	}

	/* A_DC_TAKEOVER */
	void
	do_dc_takeover(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	int rc = pcmk_ok;
	xmlNode *cib = NULL;
	const char *cluster_type = name_for_cluster_type(get_cluster_type());
	const char *watchdog = NULL;

	crm_info("Taking over DC status for this partition");
	set_bit(fsa_input_register, R_THE_DC);
	execute_stonith_cleanup();

	#if SUPPORT_COROSYNC
	if (is_classic_ais_cluster()) {
	send_cluster_text(crm_class_quorum, NULL, TRUE, NULL, crm_msg_ais);
	}
	#endif

	election_reset(fsa_election);
	set_bit(fsa_input_register, R_JOIN_OK);
	set_bit(fsa_input_register, R_INVOKE_PE);

	fsa_cib_conn->cmds->set_master(fsa_cib_conn, cib_scope_local);

	cib = create_xml_node(NULL, XML_TAG_CIB);
	crm_xml_add(cib, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	fsa_cib_update(XML_TAG_CIB, cib, cib_quorum_override, rc, NULL);
	fsa_register_cib_callback(rc, FALSE, NULL, feature_update_callback);

	watchdog = daemon_option("watchdog");
	if (watchdog) {
	update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	XML_ATTR_HAVE_WATCHDOG, watchdog, FALSE, NULL, NULL);
	}

	update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	"dc-version", PACEMAKER_VERSION "-" BUILD_VERSION, FALSE, NULL, NULL);

	update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	"cluster-infrastructure", cluster_type, FALSE, NULL, NULL);

	#if SUPPORT_COROSYNC
	# if !SUPPORT_PLUGIN
	if (fsa_cluster_name == NULL && is_corosync_cluster()) {
	char *cluster_name = corosync_cluster_name();

	if (cluster_name) {
	update_attr_delegate(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
	"cluster-name", cluster_name, FALSE, NULL, NULL);
	}
	free(cluster_name);
	}
	# endif
	#endif

	mainloop_set_trigger(config_read);
	free_xml(cib);
	}

	/* A_DC_RELEASE */
	void
	do_dc_release(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	if (action & A_DC_RELEASE) {
	crm_debug("Releasing the role of DC");
	clear_bit(fsa_input_register, R_THE_DC);

	} else if (action & A_DC_RELEASED) {
	crm_info("DC role released");
	#if 0
	if (are there errors) {
	- /* we cant stay up if not healthy */
	+ /* we can't stay up if not healthy */
	/* or perhaps I_ERROR and go to S_RECOVER? */
	result = I_SHUTDOWN;
	}
	#endif
	if (is_set(fsa_input_register, R_SHUTDOWN)) {
	xmlNode *update = NULL;
	crm_node_t *node = crm_get_peer(0, fsa_our_uname);

	crm_update_peer_expected(__FUNCTION__, node, CRMD_JOINSTATE_DOWN);
	update = do_update_node_cib(node, node_update_expected, NULL, __FUNCTION__);
	fsa_cib_anon_update(XML_CIB_TAG_STATUS, update,
	cib_scope_local \| cib_quorum_override \| cib_can_create);
	free_xml(update);
	}
	register_fsa_input(C_FSA_INTERNAL, I_RELEASE_SUCCESS, NULL);

	} else {
	crm_err("Unknown action %s", fsa_action2string(action));
	}

	crm_trace("Am I still the DC? %s", AM_I_DC ? XML_BOOLEAN_YES : XML_BOOLEAN_NO);

	}
	diff --git a/crmd/fsa_defines.h b/crmd/fsa_defines.h
	index 43567e78c2..4ff35d6404 100644
	--- a/crmd/fsa_defines.h
	+++ b/crmd/fsa_defines.h
	@@ -1,499 +1,498 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/
	#ifndef FSA_DEFINES__H
	# define FSA_DEFINES__H

	/*======================================
	* States the DC/CRMd can be in
	======================================/
	enum crmd_fsa_state {
	S_IDLE = 0, /* Nothing happening */

	S_ELECTION, /* Take part in the election algorithm as
	* described below
	*/
	S_INTEGRATION, /* integrate that status of new nodes (which is
	* all of them if we have just been elected DC)
	* to form a complete and up-to-date picture of
	* the CIB
	*/
	S_FINALIZE_JOIN, /* integrate that status of new nodes (which is
	* all of them if we have just been elected DC)
	* to form a complete and up-to-date picture of
	* the CIB
	*/
	S_NOT_DC, /* we are in crmd/slave mode */
	- S_POLICY_ENGINE, /* Determin the next stable state of the cluster
	- */
	+ S_POLICY_ENGINE, /* Determine next stable state of the cluster */
	S_RECOVERY, /* Something bad happened, check everything is ok
	* before continuing and attempt to recover if
	* required
	*/
	S_RELEASE_DC, /* we were the DC, but now we arent anymore,
	* possibly by our own request, and we should
	* release all unnecessary sub-systems, finish
	* any pending actions, do general cleanup and
	* unset anything that makes us think we are
	* special :)
	*/
	S_STARTING, /* we are just starting out */
	S_PENDING, /* we are not a full/active member yet */
	S_STOPPING, /* We are in the final stages of shutting down */
	S_TERMINATE, /* We are going to shutdown, this is the equiv of
	* "Sending TERM signal to all processes" in Linux
	* and in worst case scenarios could be considered
	* a self STONITH
	*/
	S_TRANSITION_ENGINE, /* Attempt to make the calculated next stable
	* state of the cluster a reality
	*/

	S_HALT, /* Freeze - dont do anything
	* Something ad happened that needs the admin to fix
	* Wait for I_ELECTION
	*/

	/* ----------- Last input found in table is above ---------- */
	S_ILLEGAL /* This is an illegal FSA state */
	/* (must be last) */
	};

	# define MAXSTATE S_ILLEGAL
	/*
	A state diagram can be constructed from the dc_fsa.dot with the
	following command:

	dot -Tpng crmd_fsa.dot > crmd_fsa.png

	Description:

	Once we start and do some basic sanity checks, we go into the
	S_NOT_DC state and await instructions from the DC or input from
	the CCM which indicates the election algorithm needs to run.

	If the election algorithm is triggered we enter the S_ELECTION state
	from where we can either go back to the S_NOT_DC state or progress
	to the S_INTEGRATION state (or S_RELEASE_DC if we used to be the DC
	but arent anymore).

	The election algorithm has been adapted from
	http://www.cs.indiana.edu/cgi-bin/techreports/TRNNN.cgi?trnum=TR521

	Loosly known as the Bully Algorithm, its major points are:
	- Election is initiated by any node (N) notices that the controller
	is no longer responding
	- Concurrent multiple elections are possible
	- Algorithm
	+ N sends ELECTION messages to all nodes that occur earlier in
	the CCM's membership list.
	+ If no one responds, N wins and becomes controller
	+ N sends out CONTROLLER messages to all other nodes in the
	partition
	+ If one of higher-ups answers, it takes over. N is done.

	Once the election is complete, if we are the DC, we enter the
	S_INTEGRATION state which is a DC-in-waiting style state. We are
	- the DC, but we shouldnt do anything yet because we may not have an
	+ the DC, but we shouldn't do anything yet because we may not have an
	up-to-date picture of the cluster. There may of course be times
	when this fails, so we should go back to the S_RECOVERY stage and
	check everything is ok. We may also end up here if a new node came
	online, since each node is authorative on itself and we would want
	to incorporate its information into the CIB.

	Once we have the latest CIB, we then enter the S_POLICY_ENGINE state
	where invoke the Policy Engine. It is possible that between
	invoking the Policy Engine and receiving an answer, that we receive
	more input. In this case we would discard the orginal result and
	invoke it again.

	Once we are satisfied with the output from the Policy Engine we
	enter S_TRANSITION_ENGINE and feed the Policy Engine's output to the
	Transition Engine who attempts to make the Policy Engine's
	calculation a reality. If the transition completes successfully,
	we enter S_IDLE, otherwise we go back to S_POLICY_ENGINE with the
	current unstable state and try again.

	Of course we may be asked to shutdown at any time, however we must
	progress to S_NOT_DC before doing so. Once we have handed over DC
	duties to another node, we can then shut down like everyone else,
	that is by asking the DC for permission and waiting it to take all
	our resources away.

	The case where we are the DC and the only node in the cluster is a
	special case and handled as an escalation which takes us to
	S_SHUTDOWN. Similarly if any other point in the shutdown
	fails or stalls, this is escalated and we end up in S_TERMINATE.

	At any point, the CRMd/DC can relay messages for its sub-systems,
	but outbound messages (from sub-systems) should probably be blocked
	until S_INTEGRATION (for the DC case) or the join protocol has
	completed (for the CRMd case)

	*/

	/*======================================
	*
	* Inputs/Events/Stimuli to be given to the finite state machine
	*
	* Some of these a true events, and others a synthesised based on
	* the "register" (see below) and the contents or source of messages.
	*
	* At this point, my plan is to have a loop of some sort that keeps
	* going until receiving I_NULL
	*
	======================================/
	enum crmd_fsa_input {
	/* 0 */
	I_NULL, /* Nothing happened */
	/* 1 */

	I_CIB_OP, /* An update to the CIB occurred */
	I_CIB_UPDATE, /* An update to the CIB occurred */
	I_DC_TIMEOUT, /* We have lost communication with the DC */
	I_ELECTION, /* Someone started an election */
	I_PE_CALC, /* The Policy Engine needs to be invoked */
	I_RELEASE_DC, /* The election completed and we were not
	* elected, but we were the DC beforehand
	*/
	I_ELECTION_DC, /* The election completed and we were (re-)elected
	* DC
	*/
	I_ERROR, /* Something bad happened (more serious than
	* I_FAIL) and may not have been due to the action
	* being performed. For example, we may have lost
	* our connection to the CIB.
	*/
	/* 9 */
	I_FAIL, /* The action failed to complete successfully */
	I_INTEGRATED,
	I_FINALIZED,
	I_NODE_JOIN, /* A node has entered the cluster */
	I_NOT_DC, /* We are not and were not the DC before or after
	* the current operation or state
	*/
	I_RECOVERED, /* The recovery process completed successfully */
	I_RELEASE_FAIL, /* We could not give up DC status for some reason
	*/
	I_RELEASE_SUCCESS, /* We are no longer the DC */
	I_RESTART, /* The current set of actions needs to be
	* restarted
	*/
	I_TE_SUCCESS, /* Some non-resource, non-ccm action is required
	* of us, eg. ping
	*/
	/* 20 */
	I_ROUTER, /* Do our job as router and forward this to the
	* right place
	*/
	I_SHUTDOWN, /* We are asking to shutdown */
	I_STOP, /* We have been told to shutdown */
	I_TERMINATE, /* Actually exit */
	I_STARTUP,
	I_PE_SUCCESS, /* The action completed successfully */

	I_JOIN_OFFER, /* The DC is offering membership */
	I_JOIN_REQUEST, /* The client is requesting membership */
	I_JOIN_RESULT, /* If not the DC: The result of a join request
	* Else: A client is responding with its local state info
	*/

	I_WAIT_FOR_EVENT, /* we may be waiting for an async task to "happen"
	- * and until it does, we cant do anything else
	+ * and until it does, we can't do anything else
	*/

	I_DC_HEARTBEAT, /* The DC is telling us that it is alive and well */

	I_LRM_EVENT,

	/* 30 */
	I_PENDING,
	I_HALT,

	/* ------------ Last input found in table is above ----------- */
	I_ILLEGAL /* This is an illegal value for an FSA input */
	/* (must be last) */
	};

	# define MAXINPUT I_ILLEGAL

	# define I_MESSAGE I_ROUTER

	/*======================================
	*
	* actions
	*
	* Some of the actions below will always occur together for now, but I can
	* forsee that this may not always be the case. So I've spilt them up so
	* that if they ever do need to be called independantly in the future, it
	- * wont be a problem.
	+ * won't be a problem.
	*
	- * For example, separating A_LRM_CONNECT from A_STARTUP might be useful
	+ * For example, separating A_LRM_CONNECT from A_STARTUP might be useful
	* if we ever try to recover from a faulty or disconnected LRM.
	*
	======================================/

	/* Don't do anything */
	# define A_NOTHING 0x0000000000000000ULL

	/* -- Startup actions -- */
	/* Hook to perform any actions (other than starting the CIB,
	* connecting to HA or the CCM) that might be needed as part
	* of the startup.
	*/
	# define A_STARTUP 0x0000000000000001ULL
	/* Hook to perform any actions that might be needed as part
	* after startup is successful.
	*/
	# define A_STARTED 0x0000000000000002ULL
	/* Connect to Heartbeat */
	# define A_HA_CONNECT 0x0000000000000004ULL
	# define A_HA_DISCONNECT 0x0000000000000008ULL

	# define A_INTEGRATE_TIMER_START 0x0000000000000010ULL
	# define A_INTEGRATE_TIMER_STOP 0x0000000000000020ULL
	# define A_FINALIZE_TIMER_START 0x0000000000000040ULL
	# define A_FINALIZE_TIMER_STOP 0x0000000000000080ULL

	/* -- Election actions -- */
	# define A_DC_TIMER_START 0x0000000000000100ULL
	# define A_DC_TIMER_STOP 0x0000000000000200ULL
	# define A_ELECTION_COUNT 0x0000000000000400ULL
	# define A_ELECTION_VOTE 0x0000000000000800ULL

	# define A_ELECTION_START 0x0000000000001000ULL

	/* -- Message processing -- */
	/* Process the queue of requests */
	# define A_MSG_PROCESS 0x0000000000002000ULL
	/* Send the message to the correct recipient */
	# define A_MSG_ROUTE 0x0000000000004000ULL

	/* Send a welcome message to new node(s) */
	# define A_DC_JOIN_OFFER_ONE 0x0000000000008000ULL

	/* -- Server Join protocol actions -- */
	/* Send a welcome message to all nodes */
	# define A_DC_JOIN_OFFER_ALL 0x0000000000010000ULL
	/* Process the remote node's ack of our join message */
	# define A_DC_JOIN_PROCESS_REQ 0x0000000000020000ULL
	/* Send out the reults of the Join phase */
	# define A_DC_JOIN_FINALIZE 0x0000000000040000ULL
	/* Send out the reults of the Join phase */
	# define A_DC_JOIN_PROCESS_ACK 0x0000000000080000ULL

	/* -- Client Join protocol actions -- */
	# define A_CL_JOIN_QUERY 0x0000000000100000ULL
	# define A_CL_JOIN_ANNOUNCE 0x0000000000200000ULL
	/* Request membership to the DC list */
	# define A_CL_JOIN_REQUEST 0x0000000000400000ULL
	/* Did the DC accept or reject the request */
	# define A_CL_JOIN_RESULT 0x0000000000800000ULL

	/* -- Recovery, DC start/stop -- */
	/* Something bad happened, try to recover */
	# define A_RECOVER 0x0000000001000000ULL
	/* Hook to perform any actions (apart from starting, the TE, PE
	* and gathering the latest CIB) that might be necessary before
	* giving up the responsibilities of being the DC.
	*/
	# define A_DC_RELEASE 0x0000000002000000ULL
	/* */
	# define A_DC_RELEASED 0x0000000004000000ULL
	/* Hook to perform any actions (apart from starting, the TE, PE
	* and gathering the latest CIB) that might be necessary before
	* taking over the responsibilities of being the DC.
	*/
	# define A_DC_TAKEOVER 0x0000000008000000ULL

	/* -- Shutdown actions -- */
	# define A_SHUTDOWN 0x0000000010000000ULL
	# define A_STOP 0x0000000020000000ULL
	# define A_EXIT_0 0x0000000040000000ULL
	# define A_EXIT_1 0x0000000080000000ULL

	# define A_SHUTDOWN_REQ 0x0000000100000000ULL
	# define A_ELECTION_CHECK 0x0000000200000000ULL
	# define A_DC_JOIN_FINAL 0x0000000400000000ULL

	/* -- CCM actions -- */
	# define A_CCM_CONNECT 0x0000001000000000ULL
	# define A_CCM_DISCONNECT 0x0000002000000000ULL

	/* -- CIB actions -- */
	# define A_CIB_START 0x0000020000000000ULL
	# define A_CIB_STOP 0x0000040000000000ULL

	/* -- Transition Engine actions -- */
	- /* Attempt to reach the newly calculated cluster state. This is
	+ /* Attempt to reach the newly calculated cluster state. This is
	* only called once per transition (except if it is asked to
	* stop the transition or start a new one).
	- * Once given a cluster state to reach, the TE will determin
	+ * Once given a cluster state to reach, the TE will determine
	* tasks that can be performed in parallel, execute them, wait
	- * for replies and then determin the next set until the new
	+ * for replies and then determine the next set until the new
	* state is reached or no further tasks can be taken.
	*/
	# define A_TE_INVOKE 0x0000100000000000ULL
	# define A_TE_START 0x0000200000000000ULL
	# define A_TE_STOP 0x0000400000000000ULL
	# define A_TE_CANCEL 0x0000800000000000ULL
	# define A_TE_HALT 0x0001000000000000ULL

	/* -- Policy Engine actions -- */
	/* Calculate the next state for the cluster. This is only
	* invoked once per needed calculation.
	*/
	# define A_PE_INVOKE 0x0002000000000000ULL
	# define A_PE_START 0x0004000000000000ULL
	# define A_PE_STOP 0x0008000000000000ULL
	/* -- Misc actions -- */
	/* Add a system generate "block" so that resources arent moved
	* to or are activly moved away from the affected node. This
	* way we can return quickly even if busy with other things.
	*/
	# define A_NODE_BLOCK 0x0010000000000000ULL
	/* Update our information in the local CIB */
	# define A_UPDATE_NODESTATUS 0x0020000000000000ULL
	# define A_CIB_BUMPGEN 0x0040000000000000ULL
	# define A_READCONFIG 0x0080000000000000ULL

	/* -- LRM Actions -- */
	/* Connect to the Local Resource Manager */
	# define A_LRM_CONNECT 0x0100000000000000ULL
	/* Disconnect from the Local Resource Manager */
	# define A_LRM_DISCONNECT 0x0200000000000000ULL
	# define A_LRM_INVOKE 0x0400000000000000ULL
	# define A_LRM_EVENT 0x0800000000000000ULL

	/* -- Logging actions -- */
	# define A_LOG 0x1000000000000000ULL
	# define A_ERROR 0x2000000000000000ULL
	# define A_WARN 0x4000000000000000ULL

	# define O_EXIT (A_SHUTDOWN\|A_STOP\|A_CCM_DISCONNECT\|A_LRM_DISCONNECT\|A_HA_DISCONNECT\|A_EXIT_0\|A_CIB_STOP)
	# define O_RELEASE (A_DC_TIMER_STOP\|A_DC_RELEASE\|A_PE_STOP\|A_TE_STOP\|A_DC_RELEASED)
	# define O_PE_RESTART (A_PE_START\|A_PE_STOP)
	# define O_TE_RESTART (A_TE_START\|A_TE_STOP)
	# define O_CIB_RESTART (A_CIB_START\|A_CIB_STOP)
	# define O_LRM_RECONNECT (A_LRM_CONNECT\|A_LRM_DISCONNECT)
	# define O_DC_TIMER_RESTART (A_DC_TIMER_STOP\|A_DC_TIMER_START)
	/*======================================
	*
	* "register" contents
	*
	* Things we may want to remember regardless of which state we are in.
	*
	* These also count as inputs for synthesizing I_*
	*
	======================================/
	# define R_THE_DC 0x00000001ULL
	/* Are we the DC? */
	# define R_STARTING 0x00000002ULL
	/* Are we starting up? */
	# define R_SHUTDOWN 0x00000004ULL
	/* Are we trying to shut down? */
	# define R_STAYDOWN 0x00000008ULL
	/* Should we restart? */

	# define R_JOIN_OK 0x00000010ULL /* Have we completed the join process */
	# define R_READ_CONFIG 0x00000040ULL
	# define R_INVOKE_PE 0x00000080ULL
	/* Does the PE needed to be invoked at
	the next appropriate point? */

	# define R_CIB_CONNECTED 0x00000100ULL
	/* Is the CIB connected? */
	# define R_PE_CONNECTED 0x00000200ULL
	/* Is the Policy Engine connected? */
	# define R_TE_CONNECTED 0x00000400ULL
	/* Is the Transition Engine connected? */
	# define R_LRM_CONNECTED 0x00000800ULL
	/* Is the Local Resource Manager
	connected? */

	# define R_CIB_REQUIRED 0x00001000ULL
	/* Is the CIB required? */
	# define R_PE_REQUIRED 0x00002000ULL
	/* Is the Policy Engine required? */
	# define R_TE_REQUIRED 0x00004000ULL
	/* Is the Transition Engine required? */
	# define R_ST_REQUIRED 0x00008000ULL
	/* Is the Stonith daemon required? */

	# define R_CIB_DONE 0x00010000ULL
	/* Have we calculated the CIB? */
	# define R_HAVE_CIB 0x00020000ULL /* Do we have an up-to-date CIB */
	# define R_CIB_ASKED 0x00040000ULL /* Have we asked for an up-to-date CIB */

	# define R_MEMBERSHIP 0x00100000ULL /* Have we got CCM data yet */
	# define R_PEER_DATA 0x00200000ULL /* Have we got T_CL_STATUS data yet */

	# define R_HA_DISCONNECTED 0x00400000ULL /* did we sign out of our own accord */
	# define R_CCM_DISCONNECTED 0x00800000ULL /* did we sign out of our own accord */

	# define R_REQ_PEND 0x01000000ULL
	/* Are there Requests waiting for
	processing? */
	# define R_PE_PEND 0x02000000ULL
	/* Has the PE been invoked and we're
	awaiting a reply? */
	# define R_TE_PEND 0x04000000ULL
	/* Has the TE been invoked and we're
	awaiting completion? */
	# define R_RESP_PEND 0x08000000ULL
	/* Do we have clients waiting on a
	- response? if so perhaps we shouldnt
	+ response? if so perhaps we shouldn't
	stop yet */

	# define R_IN_TRANSITION 0x10000000ULL
	/* */
	# define R_SENT_RSC_STOP 0x20000000ULL /* Have we sent a stop action to all
	* resources in preparation for
	* shutting down */

	# define R_IN_RECOVERY 0x80000000ULL

	/*
	* Magic RC used within CRMd to indicate direct nacks
	* (operation is invalid in current state)
	*/
	#define CRM_DIRECT_NACK_RC (99)

	enum crmd_fsa_cause {
	C_UNKNOWN = 0,
	C_STARTUP,
	C_IPC_MESSAGE,
	C_HA_MESSAGE,
	C_CCM_CALLBACK,
	C_CRMD_STATUS_CALLBACK,
	C_LRM_OP_CALLBACK,
	C_LRM_MONITOR_CALLBACK,
	C_TIMER_POPPED,
	C_SHUTDOWN,
	C_HEARTBEAT_FAILED,
	C_SUBSYSTEM_CONNECT,
	C_HA_DISCONNECT,
	C_FSA_INTERNAL,
	C_ILLEGAL
	};

	extern const char *fsa_input2string(enum crmd_fsa_input input);
	extern const char *fsa_state2string(enum crmd_fsa_state state);
	extern const char *fsa_cause2string(enum crmd_fsa_cause cause);
	extern const char *fsa_action2string(long long action);

	#endif
	diff --git a/crmd/messages.c b/crmd/messages.c
	index a6e8376aad..5114519da7 100644
	--- a/crmd/messages.c
	+++ b/crmd/messages.c
	@@ -1,982 +1,982 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <crm/crm.h>
	#include <string.h>
	#include <time.h>
	#include <crmd_fsa.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <crm/cluster/internal.h>
	#include <crm/cib.h>
	#include <crm/common/ipcs.h>

	#include <crmd.h>
	#include <crmd_messages.h>
	#include <crmd_lrm.h>
	#include <throttle.h>

	GListPtr fsa_message_queue = NULL;
	extern void crm_shutdown(int nsig);

	extern crm_ipc_t *attrd_ipc;
	void handle_response(xmlNode * stored_msg);
	enum crmd_fsa_input handle_request(xmlNode * stored_msg, enum crmd_fsa_cause cause);
	enum crmd_fsa_input handle_shutdown_request(xmlNode * stored_msg);

	#define ROUTER_RESULT(x) crm_trace("Router result: %s", x)

	/* debug only, can wrap all it likes */
	int last_data_id = 0;

	void
	register_fsa_error_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
	fsa_data_t * cur_data, void new_data, const char raised_from)
	{
	/* save the current actions if any */
	if (fsa_actions != A_NOTHING) {
	register_fsa_input_adv(cur_data ? cur_data->fsa_cause : C_FSA_INTERNAL,
	I_NULL, cur_data ? cur_data->data : NULL,
	fsa_actions, TRUE, __FUNCTION__);
	}

	/* reset the action list */
	crm_info("Resetting the current action list");
	fsa_dump_actions(fsa_actions, "Drop");
	fsa_actions = A_NOTHING;

	/* register the error */
	register_fsa_input_adv(cause, input, new_data, A_NOTHING, TRUE, raised_from);
	}

	int
	register_fsa_input_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
	void *data, long long with_actions,
	gboolean prepend, const char *raised_from)
	{
	unsigned old_len = g_list_length(fsa_message_queue);
	fsa_data_t *fsa_data = NULL;

	CRM_CHECK(raised_from != NULL, raised_from = "<unknown>");

	if (input == I_NULL && with_actions == A_NOTHING /* && data == NULL */ ) {
	/* no point doing anything */
	crm_err("Cannot add entry to queue: no input and no action");
	return 0;
	}

	if (input == I_WAIT_FOR_EVENT) {
	do_fsa_stall = TRUE;
	crm_debug("Stalling the FSA pending further input: source=%s cause=%s data=%p queue=%d",
	raised_from, fsa_cause2string(cause), data, old_len);

	if (old_len > 0) {
	fsa_dump_queue(LOG_TRACE);
	prepend = FALSE;
	}

	if (data == NULL) {
	fsa_actions \|= with_actions;
	fsa_dump_actions(with_actions, "Restored");
	return 0;
	}

	/* Store everything in the new event and reset fsa_actions */
	with_actions \|= fsa_actions;
	fsa_actions = A_NOTHING;
	}

	last_data_id++;
	crm_trace("%s %s FSA input %d (%s) (cause=%s) %s data",
	raised_from, prepend ? "prepended" : "appended", last_data_id,
	fsa_input2string(input), fsa_cause2string(cause), data ? "with" : "without");

	fsa_data = calloc(1, sizeof(fsa_data_t));
	fsa_data->id = last_data_id;
	fsa_data->fsa_input = input;
	fsa_data->fsa_cause = cause;
	fsa_data->origin = raised_from;
	fsa_data->data = NULL;
	fsa_data->data_type = fsa_dt_none;
	fsa_data->actions = with_actions;

	if (with_actions != A_NOTHING) {
	crm_trace("Adding actions %.16llx to input", with_actions);
	}

	if (data != NULL) {
	switch (cause) {
	case C_FSA_INTERNAL:
	case C_CRMD_STATUS_CALLBACK:
	case C_IPC_MESSAGE:
	case C_HA_MESSAGE:
	crm_trace("Copying %s data from %s as a HA msg",
	fsa_cause2string(cause), raised_from);
	CRM_CHECK(((ha_msg_input_t *) data)->msg != NULL,
	crm_err("Bogus data from %s", raised_from));
	fsa_data->data = copy_ha_msg_input(data);
	fsa_data->data_type = fsa_dt_ha_msg;
	break;

	case C_LRM_OP_CALLBACK:
	crm_trace("Copying %s data from %s as lrmd_event_data_t",
	fsa_cause2string(cause), raised_from);
	fsa_data->data = lrmd_copy_event((lrmd_event_data_t *) data);
	fsa_data->data_type = fsa_dt_lrm;
	break;

	case C_CCM_CALLBACK:
	case C_SUBSYSTEM_CONNECT:
	case C_LRM_MONITOR_CALLBACK:
	case C_TIMER_POPPED:
	case C_SHUTDOWN:
	case C_HEARTBEAT_FAILED:
	case C_HA_DISCONNECT:
	case C_ILLEGAL:
	case C_UNKNOWN:
	case C_STARTUP:
	crm_err("Copying %s data (from %s)"
	" not yet implemented", fsa_cause2string(cause), raised_from);
	crmd_exit(pcmk_err_generic);
	break;
	}
	crm_trace("%s data copied", fsa_cause2string(fsa_data->fsa_cause));
	}

	/* make sure to free it properly later */
	if (prepend) {
	crm_trace("Prepending input");
	fsa_message_queue = g_list_prepend(fsa_message_queue, fsa_data);
	} else {
	fsa_message_queue = g_list_append(fsa_message_queue, fsa_data);
	}

	crm_trace("Queue len: %d", g_list_length(fsa_message_queue));

	/* fsa_dump_queue(LOG_DEBUG_2); */

	if (old_len == g_list_length(fsa_message_queue)) {
	- crm_err("Couldnt add message to the queue");
	+ crm_err("Couldn't add message to the queue");
	}

	if (fsa_source && input != I_WAIT_FOR_EVENT) {
	crm_trace("Triggering FSA: %s", __FUNCTION__);
	mainloop_set_trigger(fsa_source);
	}
	return last_data_id;
	}

	void
	fsa_dump_queue(int log_level)
	{
	int offset = 0;
	GListPtr lpc = NULL;

	for (lpc = fsa_message_queue; lpc != NULL; lpc = lpc->next) {
	fsa_data_t data = (fsa_data_t ) lpc->data;

	do_crm_log_unlikely(log_level,
	"queue[%d.%d]: input %s raised by %s(%p.%d)\t(cause=%s)",
	offset++, data->id, fsa_input2string(data->fsa_input),
	data->origin, data->data, data->data_type,
	fsa_cause2string(data->fsa_cause));
	}
	}

	ha_msg_input_t *
	copy_ha_msg_input(ha_msg_input_t * orig)
	{
	ha_msg_input_t *copy = NULL;
	xmlNodePtr data = NULL;

	if (orig != NULL) {
	crm_trace("Copy msg");
	data = copy_xml(orig->msg);

	} else {
	crm_trace("No message to copy");
	}
	copy = new_ha_msg_input(data);
	if (orig && orig->msg != NULL) {
	CRM_CHECK(copy->msg != NULL, crm_err("copy failed"));
	}
	return copy;
	}

	void
	delete_fsa_input(fsa_data_t * fsa_data)
	{
	lrmd_event_data_t *op = NULL;
	xmlNode *foo = NULL;

	if (fsa_data == NULL) {
	return;
	}
	crm_trace("About to free %s data", fsa_cause2string(fsa_data->fsa_cause));

	if (fsa_data->data != NULL) {
	switch (fsa_data->data_type) {
	case fsa_dt_ha_msg:
	delete_ha_msg_input(fsa_data->data);
	break;

	case fsa_dt_xml:
	foo = fsa_data->data;
	free_xml(foo);
	break;

	case fsa_dt_lrm:
	op = (lrmd_event_data_t *) fsa_data->data;
	lrmd_free_event(op);
	break;

	case fsa_dt_none:
	if (fsa_data->data != NULL) {
	crm_err("Don't know how to free %s data from %s",
	fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
	crmd_exit(pcmk_err_generic);
	}
	break;
	}
	crm_trace("%s data freed", fsa_cause2string(fsa_data->fsa_cause));
	}

	free(fsa_data);
	}

	/* returns the next message */
	fsa_data_t *
	get_message(void)
	{
	fsa_data_t *message = g_list_nth_data(fsa_message_queue, 0);

	fsa_message_queue = g_list_remove(fsa_message_queue, message);
	crm_trace("Processing input %d", message->id);
	return message;
	}

	/* returns the current head of the FIFO queue */
	gboolean
	is_message(void)
	{
	return (g_list_length(fsa_message_queue) > 0);
	}

	void *
	fsa_typed_data_adv(fsa_data_t * fsa_data, enum fsa_data_type a_type, const char *caller)
	{
	void *ret_val = NULL;

	if (fsa_data == NULL) {
	crm_err("%s: No FSA data available", caller);

	} else if (fsa_data->data == NULL) {
	crm_err("%s: No message data available. Origin: %s", caller, fsa_data->origin);

	} else if (fsa_data->data_type != a_type) {
	crm_crit("%s: Message data was the wrong type! %d vs. requested=%d. Origin: %s",
	caller, fsa_data->data_type, a_type, fsa_data->origin);
	CRM_ASSERT(fsa_data->data_type == a_type);
	} else {
	ret_val = fsa_data->data;
	}

	return ret_val;
	}

	/* A_MSG_ROUTE */
	void
	do_msg_route(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input, fsa_data_t * msg_data)
	{
	ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);

	route_message(msg_data->fsa_cause, input->msg);
	}

	void
	route_message(enum crmd_fsa_cause cause, xmlNode * input)
	{
	ha_msg_input_t fsa_input;
	enum crmd_fsa_input result = I_NULL;

	fsa_input.msg = input;
	CRM_CHECK(cause == C_IPC_MESSAGE \|\| cause == C_HA_MESSAGE, return);

	/* try passing the buck first */
	if (relay_message(input, cause == C_IPC_MESSAGE)) {
	return;
	}

	/* handle locally */
	result = handle_message(input, cause);

	/* done or process later? */
	switch (result) {
	case I_NULL:
	case I_CIB_OP:
	case I_ROUTER:
	case I_NODE_JOIN:
	case I_JOIN_REQUEST:
	case I_JOIN_RESULT:
	break;
	default:
	/* Defering local processing of message */
	register_fsa_input_later(cause, result, &fsa_input);
	return;
	}

	if (result != I_NULL) {
	/* add to the front of the queue */
	register_fsa_input(cause, result, &fsa_input);
	}
	}

	gboolean
	relay_message(xmlNode * msg, gboolean originated_locally)
	{
	int dest = 1;
	int is_for_dc = 0;
	int is_for_dcib = 0;
	int is_for_te = 0;
	int is_for_crm = 0;
	int is_for_cib = 0;
	int is_local = 0;
	gboolean processing_complete = FALSE;
	const char *host_to = crm_element_value(msg, F_CRM_HOST_TO);
	const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
	const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
	const char *type = crm_element_value(msg, F_TYPE);
	const char *msg_error = NULL;

	crm_trace("Routing message %s", crm_element_value(msg, XML_ATTR_REFERENCE));

	if (msg == NULL) {
	msg_error = "Cannot route empty message";

	} else if (safe_str_eq(CRM_OP_HELLO, crm_element_value(msg, F_CRM_TASK))) {
	/* quietly ignore */
	processing_complete = TRUE;

	} else if (safe_str_neq(type, T_CRM)) {
	msg_error = "Bad message type";

	} else if (sys_to == NULL) {
	msg_error = "Bad message destination: no subsystem";
	}

	if (msg_error != NULL) {
	processing_complete = TRUE;
	crm_err("%s", msg_error);
	crm_log_xml_warn(msg, "bad msg");
	}

	if (processing_complete) {
	return TRUE;
	}

	processing_complete = TRUE;

	is_for_dc = (strcasecmp(CRM_SYSTEM_DC, sys_to) == 0);
	is_for_dcib = (strcasecmp(CRM_SYSTEM_DCIB, sys_to) == 0);
	is_for_te = (strcasecmp(CRM_SYSTEM_TENGINE, sys_to) == 0);
	is_for_cib = (strcasecmp(CRM_SYSTEM_CIB, sys_to) == 0);
	is_for_crm = (strcasecmp(CRM_SYSTEM_CRMD, sys_to) == 0);

	is_local = 0;
	if (host_to == NULL \|\| strlen(host_to) == 0) {
	if (is_for_dc \|\| is_for_te) {
	is_local = 0;

	} else if (is_for_crm && originated_locally) {
	is_local = 0;

	} else {
	is_local = 1;
	}

	} else if (safe_str_eq(fsa_our_uname, host_to)) {
	is_local = 1;
	}

	if (is_for_dc \|\| is_for_dcib \|\| is_for_te) {
	if (AM_I_DC && is_for_te) {
	ROUTER_RESULT("Message result: Local relay");
	send_msg_via_ipc(msg, sys_to);

	} else if (AM_I_DC) {
	ROUTER_RESULT("Message result: DC/CRMd process");
	processing_complete = FALSE; /* more to be done by caller */
	} else if (originated_locally && safe_str_neq(sys_from, CRM_SYSTEM_PENGINE)
	&& safe_str_neq(sys_from, CRM_SYSTEM_TENGINE)) {

	/* Neither the TE or PE should be sending messages
	* to DC's on other nodes
	*
	* By definition, if we are no longer the DC, then
	* the PE or TE's data should be discarded
	*/

	#if SUPPORT_COROSYNC
	if (is_openais_cluster()) {
	dest = text2msg_type(sys_to);
	}
	#endif
	ROUTER_RESULT("Message result: External relay to DC");
	send_cluster_message(host_to ? crm_get_peer(0, host_to) : NULL, dest, msg, TRUE);

	} else {
	/* discard */
	ROUTER_RESULT("Message result: Discard, not DC");
	}

	} else if (is_local && (is_for_crm \|\| is_for_cib)) {
	ROUTER_RESULT("Message result: CRMd process");
	processing_complete = FALSE; /* more to be done by caller */

	} else if (is_local) {
	ROUTER_RESULT("Message result: Local relay");
	send_msg_via_ipc(msg, sys_to);

	} else {
	crm_node_t *node_to = NULL;

	#if SUPPORT_COROSYNC
	if (is_openais_cluster()) {
	dest = text2msg_type(sys_to);

	if (dest == crm_msg_none \|\| dest > crm_msg_stonith_ng) {
	dest = crm_msg_crmd;
	}
	}
	#endif

	if (host_to) {
	node_to = crm_find_peer(0, host_to);
	if (node_to == NULL) {
	crm_err("Cannot route message to unknown node %s", host_to);
	return TRUE;
	}
	}

	ROUTER_RESULT("Message result: External relay");
	send_cluster_message(host_to ? node_to : NULL, dest, msg, TRUE);
	}

	return processing_complete;
	}

	static gboolean
	process_hello_message(xmlNode * hello,
	char client_name, char major_version, char **minor_version)
	{
	const char *local_client_name;
	const char *local_major_version;
	const char *local_minor_version;

	*client_name = NULL;
	*major_version = NULL;
	*minor_version = NULL;

	if (hello == NULL) {
	return FALSE;
	}

	local_client_name = crm_element_value(hello, "client_name");
	local_major_version = crm_element_value(hello, "major_version");
	local_minor_version = crm_element_value(hello, "minor_version");

	if (local_client_name == NULL \|\| strlen(local_client_name) == 0) {
	crm_err("Hello message was not valid (field %s not found)", "client name");
	return FALSE;

	} else if (local_major_version == NULL \|\| strlen(local_major_version) == 0) {
	crm_err("Hello message was not valid (field %s not found)", "major version");
	return FALSE;

	} else if (local_minor_version == NULL \|\| strlen(local_minor_version) == 0) {
	crm_err("Hello message was not valid (field %s not found)", "minor version");
	return FALSE;
	}

	*client_name = strdup(local_client_name);
	*major_version = strdup(local_major_version);
	*minor_version = strdup(local_minor_version);

	crm_trace("Hello message ok");
	return TRUE;
	}

	gboolean
	crmd_authorize_message(xmlNode * client_msg, crm_client_t * curr_client, const char *proxy_session)
	{
	char *client_name = NULL;
	char *major_version = NULL;
	char *minor_version = NULL;
	gboolean auth_result = FALSE;

	xmlNode *xml = NULL;
	const char *op = crm_element_value(client_msg, F_CRM_TASK);
	const char *uuid = curr_client ? curr_client->id : proxy_session;

	if (uuid == NULL) {
	crm_warn("Message [%s] not authorized", crm_element_value(client_msg, XML_ATTR_REFERENCE));
	return FALSE;

	} else if (safe_str_neq(CRM_OP_HELLO, op)) {
	return TRUE;
	}

	xml = get_message_xml(client_msg, F_CRM_DATA);
	auth_result = process_hello_message(xml, &client_name, &major_version, &minor_version);

	if (auth_result == TRUE) {
	if (client_name == NULL) {
	crm_err("Bad client details (client_name=%s, uuid=%s)",
	crm_str(client_name), uuid);
	auth_result = FALSE;
	}
	}

	if (auth_result == TRUE) {
	/* check version */
	int mav = atoi(major_version);
	int miv = atoi(minor_version);

	crm_trace("Checking client version number");
	if (mav < 0 \|\| miv < 0) {
	crm_err("Client version (%d:%d) is not acceptable", mav, miv);
	auth_result = FALSE;
	}
	}

	if (auth_result == TRUE) {
	crm_trace("Accepted client %s", client_name);
	if (curr_client) {
	curr_client->userdata = strdup(client_name);
	}

	crm_trace("Triggering FSA: %s", __FUNCTION__);
	mainloop_set_trigger(fsa_source);

	} else {
	crm_warn("Rejected client logon request");
	if (curr_client) {
	qb_ipcs_disconnect(curr_client->ipcs);
	}
	}

	free(minor_version);
	free(major_version);
	free(client_name);

	/* hello messages should never be processed further */
	return FALSE;
	}

	enum crmd_fsa_input
	handle_message(xmlNode * msg, enum crmd_fsa_cause cause)
	{
	const char *type = NULL;

	CRM_CHECK(msg != NULL, return I_NULL);

	type = crm_element_value(msg, F_CRM_MSG_TYPE);
	if (crm_str_eq(type, XML_ATTR_REQUEST, TRUE)) {
	return handle_request(msg, cause);

	} else if (crm_str_eq(type, XML_ATTR_RESPONSE, TRUE)) {
	handle_response(msg);
	return I_NULL;
	}

	crm_err("Unknown message type: %s", type);
	return I_NULL;
	}

	static enum crmd_fsa_input
	handle_failcount_op(xmlNode * stored_msg)
	{
	const char *rsc = NULL;
	const char *uname = NULL;
	gboolean is_remote_node = FALSE;
	xmlNode *xml_rsc = get_xpath_object("//" XML_CIB_TAG_RESOURCE, stored_msg, LOG_ERR);

	if (xml_rsc) {
	rsc = ID(xml_rsc);
	}

	uname = crm_element_value(stored_msg, XML_LRM_ATTR_TARGET);
	if (crm_element_value(stored_msg, XML_LRM_ATTR_ROUTER_NODE)) {
	is_remote_node = TRUE;
	}

	if (rsc) {
	char *attr = NULL;

	crm_info("Removing failcount for %s", rsc);

	attr = crm_concat("fail-count", rsc, '-');
	update_attrd(uname, attr, NULL, NULL, is_remote_node);
	free(attr);

	attr = crm_concat("last-failure", rsc, '-');
	update_attrd(uname, attr, NULL, NULL, is_remote_node);
	free(attr);

	lrm_clear_last_failure(rsc, uname);
	} else {
	crm_log_xml_warn(stored_msg, "invalid failcount op");
	}

	return I_NULL;
	}

	enum crmd_fsa_input
	handle_request(xmlNode * stored_msg, enum crmd_fsa_cause cause)
	{
	xmlNode *msg = NULL;
	const char *op = crm_element_value(stored_msg, F_CRM_TASK);

	/* Optimize this for the DC - it has the most to do */

	if (op == NULL) {
	crm_log_xml_err(stored_msg, "Bad message");
	return I_NULL;
	}

	if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
	const char *from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
	crm_node_t *node = crm_find_peer(0, from);

	crm_update_peer_expected(__FUNCTION__, node, CRMD_JOINSTATE_DOWN);
	if(AM_I_DC == FALSE) {
	return I_NULL; /* Done */
	}
	}

	/========== DC-Only Actions ==========/
	if (AM_I_DC) {
	if (strcmp(op, CRM_OP_JOIN_ANNOUNCE) == 0) {
	return I_NODE_JOIN;

	} else if (strcmp(op, CRM_OP_JOIN_REQUEST) == 0) {
	return I_JOIN_REQUEST;

	} else if (strcmp(op, CRM_OP_JOIN_CONFIRM) == 0) {
	return I_JOIN_RESULT;

	} else if (strcmp(op, CRM_OP_SHUTDOWN) == 0) {
	const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
	gboolean dc_match = safe_str_eq(host_from, fsa_our_dc);

	if (is_set(fsa_input_register, R_SHUTDOWN)) {
	crm_info("Shutting ourselves down (DC)");
	return I_STOP;

	} else if (dc_match) {
	- crm_err("We didnt ask to be shut down, yet our"
	- " TE is telling us too." " Better get out now!");
	+ crm_err("We didn't ask to be shut down, yet our"
	+ " TE is telling us to. Better get out now!");
	return I_TERMINATE;

	} else if (fsa_state != S_STOPPING) {
	crm_err("Another node is asking us to shutdown" " but we think we're ok.");
	return I_ELECTION;
	}

	} else if (strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {
	/* a slave wants to shut down */
	/* create cib fragment and add to message */
	return handle_shutdown_request(stored_msg);
	}
	}

	/========== common actions ==========/
	if (strcmp(op, CRM_OP_NOVOTE) == 0) {
	ha_msg_input_t fsa_input;

	fsa_input.msg = stored_msg;
	register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
	A_ELECTION_COUNT \| A_ELECTION_CHECK, FALSE, __FUNCTION__);

	} else if (strcmp(op, CRM_OP_THROTTLE) == 0) {
	throttle_update(stored_msg);
	return I_NULL;

	} else if (strcmp(op, CRM_OP_CLEAR_FAILCOUNT) == 0) {
	return handle_failcount_op(stored_msg);

	} else if (strcmp(op, CRM_OP_VOTE) == 0) {
	/* count the vote and decide what to do after that */
	ha_msg_input_t fsa_input;

	fsa_input.msg = stored_msg;
	register_fsa_input_adv(C_HA_MESSAGE, I_NULL, &fsa_input,
	A_ELECTION_COUNT \| A_ELECTION_CHECK, FALSE, __FUNCTION__);

	/* Sometimes we _must_ go into S_ELECTION */
	if (fsa_state == S_HALT) {
	crm_debug("Forcing an election from S_HALT");
	return I_ELECTION;
	#if 0
	} else if (AM_I_DC) {
	/* This is the old way of doing things but what is gained? */
	return I_ELECTION;
	#endif
	}

	} else if (strcmp(op, CRM_OP_JOIN_OFFER) == 0) {
	crm_debug("Raising I_JOIN_OFFER: join-%s", crm_element_value(stored_msg, F_CRM_JOIN_ID));
	return I_JOIN_OFFER;

	} else if (strcmp(op, CRM_OP_JOIN_ACKNAK) == 0) {
	crm_debug("Raising I_JOIN_RESULT: join-%s", crm_element_value(stored_msg, F_CRM_JOIN_ID));
	return I_JOIN_RESULT;

	} else if (strcmp(op, CRM_OP_LRM_DELETE) == 0
	\|\| strcmp(op, CRM_OP_LRM_FAIL) == 0
	\|\| strcmp(op, CRM_OP_LRM_REFRESH) == 0 \|\| strcmp(op, CRM_OP_REPROBE) == 0) {

	crm_xml_add(stored_msg, F_CRM_SYS_TO, CRM_SYSTEM_LRMD);
	return I_ROUTER;

	} else if (strcmp(op, CRM_OP_NOOP) == 0) {
	return I_NULL;

	} else if (strcmp(op, CRM_OP_LOCAL_SHUTDOWN) == 0) {

	crm_shutdown(SIGTERM);
	/return I_SHUTDOWN; /
	return I_NULL;

	/========== (NOT_DC)-Only Actions ==========/
	} else if (AM_I_DC == FALSE && strcmp(op, CRM_OP_SHUTDOWN) == 0) {

	const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
	gboolean dc_match = safe_str_eq(host_from, fsa_our_dc);

	if (dc_match \|\| fsa_our_dc == NULL) {
	if (is_set(fsa_input_register, R_SHUTDOWN) == FALSE) {
	- crm_err("We didn't ask to be shut down, yet our" " DC is telling us too.");
	+ crm_err("We didn't ask to be shut down, yet our DC is telling us to.");
	set_bit(fsa_input_register, R_STAYDOWN);
	return I_STOP;
	}
	crm_info("Shutting down");
	return I_STOP;

	} else {
	crm_warn("Discarding %s op from %s", op, host_from);
	}

	} else if (strcmp(op, CRM_OP_PING) == 0) {
	/* eventually do some stuff to figure out
	* if we /are/ ok
	*/
	const char *sys_to = crm_element_value(stored_msg, F_CRM_SYS_TO);
	xmlNode *ping = create_xml_node(NULL, XML_CRM_TAG_PING);

	crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok");
	crm_xml_add(ping, XML_PING_ATTR_SYSFROM, sys_to);
	crm_xml_add(ping, "crmd_state", fsa_state2string(fsa_state));

	/* Ok, so technically not so interesting, but CTS needs to see this */
	crm_notice("Current ping state: %s", fsa_state2string(fsa_state));

	msg = create_reply(stored_msg, ping);
	if (msg) {
	(void)relay_message(msg, TRUE);
	}

	free_xml(ping);
	free_xml(msg);

	} else if (strcmp(op, CRM_OP_RM_NODE_CACHE) == 0) {
	int id = 0;
	const char *name = NULL;

	crm_element_value_int(stored_msg, XML_ATTR_ID, &id);
	name = crm_element_value(stored_msg, XML_ATTR_UNAME);

	if(cause == C_IPC_MESSAGE) {
	msg = create_request(CRM_OP_RM_NODE_CACHE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
	if (send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) {
	crm_err("Could not instruct peers to remove references to node %s/%u", name, id);
	} else {
	crm_notice("Instructing peers to remove references to node %s/%u", name, id);
	}
	free_xml(msg);

	} else {
	reap_crm_member(id, name);
	}

	} else {
	crm_err("Unexpected request (%s) sent to %s", op, AM_I_DC ? "the DC" : "non-DC node");
	crm_log_xml_err(stored_msg, "Unexpected");
	}

	return I_NULL;
	}

	void
	handle_response(xmlNode * stored_msg)
	{
	const char *op = crm_element_value(stored_msg, F_CRM_TASK);

	if (op == NULL) {
	crm_log_xml_err(stored_msg, "Bad message");

	} else if (AM_I_DC && strcmp(op, CRM_OP_PECALC) == 0) {
	/* Check if the PE answer been superseded by a subsequent request? */
	const char *msg_ref = crm_element_value(stored_msg, XML_ATTR_REFERENCE);

	if (msg_ref == NULL) {
	crm_err("%s - Ignoring calculation with no reference", op);

	} else if (safe_str_eq(msg_ref, fsa_pe_ref)) {
	ha_msg_input_t fsa_input;

	fsa_input.msg = stored_msg;
	register_fsa_input_later(C_IPC_MESSAGE, I_PE_SUCCESS, &fsa_input);
	crm_trace("Completed: %s...", fsa_pe_ref);

	} else {
	crm_info("%s calculation %s is obsolete", op, msg_ref);
	}

	} else if (strcmp(op, CRM_OP_VOTE) == 0
	\|\| strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0 \|\| strcmp(op, CRM_OP_SHUTDOWN) == 0) {

	} else {
	const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);

	crm_err("Unexpected response (op=%s, src=%s) sent to the %s",
	op, host_from, AM_I_DC ? "DC" : "CRMd");
	}
	}

	enum crmd_fsa_input
	handle_shutdown_request(xmlNode * stored_msg)
	{
	/* handle here to avoid potential version issues
	- * where the shutdown message/proceedure may have
	+ * where the shutdown message/procedure may have
	* been changed in later versions.
	*
	* This way the DC is always in control of the shutdown
	*/

	char *now_s = NULL;
	time_t now = time(NULL);
	const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);

	if (host_from == NULL) {
	/* we're shutting down and the DC */
	host_from = fsa_our_uname;
	}

	crm_info("Creating shutdown request for %s (state=%s)", host_from, fsa_state2string(fsa_state));
	crm_log_xml_trace(stored_msg, "message");

	now_s = crm_itoa(now);
	update_attrd(host_from, XML_CIB_ATTR_SHUTDOWN, now_s, NULL, FALSE);
	free(now_s);

	/* will be picked up by the TE as long as its running */
	return I_NULL;
	}

	/* msg is deleted by the time this returns */
	extern gboolean process_te_message(xmlNode * msg, xmlNode * xml_data);

	gboolean
	send_msg_via_ipc(xmlNode * msg, const char *sys)
	{
	gboolean send_ok = TRUE;
	crm_client_t *client_channel = crm_client_get_by_id(sys);

	if (crm_element_value(msg, F_CRM_HOST_FROM) == NULL) {
	crm_xml_add(msg, F_CRM_HOST_FROM, fsa_our_uname);
	}

	if (client_channel != NULL) {
	/* Transient clients such as crmadmin */
	send_ok = crm_ipcs_send(client_channel, 0, msg, crm_ipc_server_event);

	} else if (sys != NULL && strcmp(sys, CRM_SYSTEM_TENGINE) == 0) {
	xmlNode *data = get_message_xml(msg, F_CRM_DATA);

	process_te_message(msg, data);

	} else if (sys != NULL && strcmp(sys, CRM_SYSTEM_LRMD) == 0) {
	fsa_data_t fsa_data;
	ha_msg_input_t fsa_input;

	fsa_input.msg = msg;
	fsa_input.xml = get_message_xml(msg, F_CRM_DATA);

	fsa_data.id = 0;
	fsa_data.actions = 0;
	fsa_data.data = &fsa_input;
	fsa_data.fsa_input = I_MESSAGE;
	fsa_data.fsa_cause = C_IPC_MESSAGE;
	fsa_data.origin = __FUNCTION__;
	fsa_data.data_type = fsa_dt_ha_msg;

	#ifdef FSA_TRACE
	crm_trace("Invoking action A_LRM_INVOKE (%.16llx)", A_LRM_INVOKE);
	#endif
	do_lrm_invoke(A_LRM_INVOKE, C_IPC_MESSAGE, fsa_state, I_MESSAGE, &fsa_data);

	} else if (sys != NULL && crmd_is_proxy_session(sys)) {
	crmd_proxy_send(sys, msg);

	} else {
	crm_debug("Unknown Sub-system (%s)... discarding message.", crm_str(sys));
	send_ok = FALSE;
	}

	return send_ok;
	}

	ha_msg_input_t *
	new_ha_msg_input(xmlNode * orig)
	{
	ha_msg_input_t *input_copy = NULL;

	input_copy = calloc(1, sizeof(ha_msg_input_t));
	input_copy->msg = orig;
	input_copy->xml = get_message_xml(input_copy->msg, F_CRM_DATA);
	return input_copy;
	}

	void
	delete_ha_msg_input(ha_msg_input_t * orig)
	{
	if (orig == NULL) {
	return;
	}
	free_xml(orig->msg);
	free(orig);
	}
	diff --git a/crmd/te_actions.c b/crmd/te_actions.c
	index b3e6078428..039083d6aa 100644
	--- a/crmd/te_actions.c
	+++ b/crmd/te_actions.c
	@@ -1,745 +1,745 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>

	#include <crm/common/xml.h>
	#include <tengine.h>

	#include <crmd_fsa.h>
	#include <crmd_messages.h>
	#include <crm/cluster.h>
	#include <throttle.h>

	char *te_uuid = NULL;
	GHashTable *te_targets = NULL;
	void send_rsc_command(crm_action_t * action);
	static void te_update_job_count(crm_action_t * action, int offset);

	static void
	te_start_action_timer(crm_graph_t * graph, crm_action_t * action)
	{
	action->timer = calloc(1, sizeof(crm_action_timer_t));
	action->timer->timeout = action->timeout;
	action->timer->reason = timeout_action;
	action->timer->action = action;
	action->timer->source_id = g_timeout_add(action->timer->timeout + graph->network_delay,
	action_timer_callback, (void *)action->timer);

	CRM_ASSERT(action->timer->source_id != 0);
	}

	static gboolean
	te_pseudo_action(crm_graph_t * graph, crm_action_t * pseudo)
	{
	crm_debug("Pseudo action %d fired and confirmed", pseudo->id);
	te_action_confirmed(pseudo);
	update_graph(graph, pseudo);
	trigger_graph();
	return TRUE;
	}

	void
	send_stonith_update(crm_action_t * action, const char target, const char uuid)
	{
	int rc = pcmk_ok;
	crm_node_t *peer = NULL;

	/* zero out the node-status & remove all LRM status info */
	xmlNode *node_state = NULL;

	CRM_CHECK(target != NULL, return);
	CRM_CHECK(uuid != NULL, return);

	/* Make sure the membership and join caches are accurate */
	peer = crm_get_peer_full(0, target, CRM_GET_PEER_ANY);

	CRM_CHECK(peer != NULL, return);

	if (peer->uuid == NULL) {
	crm_info("Recording uuid '%s' for node '%s'", uuid, target);
	peer->uuid = strdup(uuid);
	}

	crmd_peer_down(peer, TRUE);

	/* Generate a node state update for the CIB.
	* We rely on the membership layer to do node_update_cluster,
	* and the peer status callback to do node_update_peer,
	* because the node might rejoin before we get the stonith result.
	*/
	node_state = do_update_node_cib(peer, node_update_join\|node_update_expected,
	NULL, __FUNCTION__);

	/* we have to mark whether or not remote nodes have already been fenced */
	if (peer->flags & crm_remote_node) {
	time_t now = time(NULL);
	char *now_s = crm_itoa(now);
	crm_xml_add(node_state, XML_NODE_IS_FENCED, now_s);
	free(now_s);
	}

	/* Force our known ID */
	crm_xml_add(node_state, XML_ATTR_UUID, uuid);

	rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, node_state,
	cib_quorum_override \| cib_scope_local \| cib_can_create);

	/* Delay processing the trigger until the update completes */
	crm_debug("Sending fencing update %d for %s", rc, target);
	fsa_register_cib_callback(rc, FALSE, strdup(target), cib_fencing_updated);

	/* Make sure it sticks */
	/* fsa_cib_conn->cmds->bump_epoch(fsa_cib_conn, cib_quorum_override\|cib_scope_local); */

	erase_status_tag(peer->uname, XML_CIB_TAG_LRM, cib_scope_local);
	erase_status_tag(peer->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);

	free_xml(node_state);
	return;
	}

	static gboolean
	te_fence_node(crm_graph_t * graph, crm_action_t * action)
	{
	int rc = 0;
	const char *id = NULL;
	const char *uuid = NULL;
	const char *target = NULL;
	const char *type = NULL;
	gboolean invalid_action = FALSE;
	enum stonith_call_options options = st_opt_none;

	id = ID(action->xml);
	target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
	type = crm_meta_value(action->params, "stonith_action");

	CRM_CHECK(id != NULL, invalid_action = TRUE);
	CRM_CHECK(uuid != NULL, invalid_action = TRUE);
	CRM_CHECK(type != NULL, invalid_action = TRUE);
	CRM_CHECK(target != NULL, invalid_action = TRUE);

	if (invalid_action) {
	crm_log_xml_warn(action->xml, "BadAction");
	return FALSE;
	}

	crm_notice("Executing %s fencing operation (%s) on %s (timeout=%d)",
	type, id, target, transition_graph->stonith_timeout);

	/* Passing NULL means block until we can connect... */
	te_connect_stonith(NULL);

	if (crmd_join_phase_count(crm_join_confirmed) == 1) {
	options \|= st_opt_allow_suicide;
	}

	rc = stonith_api->cmds->fence(stonith_api, options, target, type,
	transition_graph->stonith_timeout / 1000, 0);

	stonith_api->cmds->register_callback(stonith_api, rc, transition_graph->stonith_timeout / 1000,
	st_opt_timeout_updates,
	generate_transition_key(transition_graph->id, action->id,
	0, te_uuid),
	"tengine_stonith_callback", tengine_stonith_callback);

	return TRUE;
	}

	static int
	get_target_rc(crm_action_t * action)
	{
	const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);

	if (target_rc_s != NULL) {
	return crm_parse_int(target_rc_s, "0");
	}
	return 0;
	}

	static gboolean
	te_crm_command(crm_graph_t * graph, crm_action_t * action)
	{
	char *counter = NULL;
	xmlNode *cmd = NULL;
	gboolean is_local = FALSE;

	const char *id = NULL;
	const char *task = NULL;
	const char *value = NULL;
	const char *on_node = NULL;
	const char *router_node = NULL;

	gboolean rc = TRUE;
	gboolean no_wait = FALSE;

	id = ID(action->xml);
	task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);

	if (!router_node) {
	router_node = on_node;
	}

	CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
	crm_err("Corrupted command (id=%s) %s: no node", crm_str(id), crm_str(task));
	return FALSE);

	crm_info("Executing crm-event (%s): %s on %s%s%s",
	crm_str(id), crm_str(task), on_node,
	is_local ? " (local)" : "", no_wait ? " - no waiting" : "");

	if (safe_str_eq(router_node, fsa_our_uname)) {
	is_local = TRUE;
	}

	value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
	if (crm_is_true(value)) {
	no_wait = TRUE;
	}

	if (is_local && safe_str_eq(task, CRM_OP_SHUTDOWN)) {
	/* defer until everything else completes */
	crm_info("crm-event (%s) is a local shutdown", crm_str(id));
	graph->completion_action = tg_shutdown;
	graph->abort_reason = "local shutdown";
	te_action_confirmed(action);
	update_graph(graph, action);
	trigger_graph();
	return TRUE;

	} else if (safe_str_eq(task, CRM_OP_SHUTDOWN)) {
	crm_node_t *peer = crm_get_peer(0, router_node);
	crm_update_peer_expected(__FUNCTION__, peer, CRMD_JOINSTATE_DOWN);
	}

	cmd = create_request(task, action->xml, router_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);

	counter =
	generate_transition_key(transition_graph->id, action->id, get_target_rc(action), te_uuid);
	crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter);

	rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_crmd, cmd, TRUE);
	free(counter);
	free_xml(cmd);

	if (rc == FALSE) {
	crm_err("Action %d failed: send", action->id);
	return FALSE;

	} else if (no_wait) {
	te_action_confirmed(action);
	update_graph(graph, action);
	trigger_graph();

	} else {
	if (action->timeout <= 0) {
	crm_err("Action %d: %s on %s had an invalid timeout (%dms). Using %dms instead",
	action->id, task, on_node, action->timeout, graph->network_delay);
	action->timeout = graph->network_delay;
	}
	te_start_action_timer(graph, action);
	}

	return TRUE;
	}

	gboolean
	cib_action_update(crm_action_t * action, int status, int op_rc)
	{
	lrmd_event_data_t *op = NULL;
	xmlNode *state = NULL;
	xmlNode *rsc = NULL;
	xmlNode *xml_op = NULL;
	xmlNode *action_rsc = NULL;

	int rc = pcmk_ok;

	const char *name = NULL;
	const char *value = NULL;
	const char *rsc_id = NULL;
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	const char *task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
	const char *target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);

	int call_options = cib_quorum_override \| cib_scope_local;
	int target_rc = get_target_rc(action);

	if (status == PCMK_LRM_OP_PENDING) {
	crm_debug("%s %d: Recording pending operation %s on %s",
	crm_element_name(action->xml), action->id, task_uuid, target);
	} else {
	crm_warn("%s %d: %s on %s timed out",
	crm_element_name(action->xml), action->id, task_uuid, target);
	}

	action_rsc = find_xml_node(action->xml, XML_CIB_TAG_RESOURCE, TRUE);
	if (action_rsc == NULL) {
	return FALSE;
	}

	rsc_id = ID(action_rsc);
	CRM_CHECK(rsc_id != NULL, crm_log_xml_err(action->xml, "Bad:action");
	return FALSE);

	/*
	update the CIB

	<node_state id="hadev">
	<lrm>
	<lrm_resources>
	<lrm_resource id="rsc2" last_op="start" op_code="0" target="hadev"/>
	*/

	state = create_xml_node(NULL, XML_CIB_TAG_STATE);

	crm_xml_add(state, XML_ATTR_UUID, target_uuid);
	crm_xml_add(state, XML_ATTR_UNAME, target);

	rsc = create_xml_node(state, XML_CIB_TAG_LRM);
	crm_xml_add(rsc, XML_ATTR_ID, target_uuid);

	rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCES);
	rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCE);
	crm_xml_add(rsc, XML_ATTR_ID, rsc_id);

	name = XML_ATTR_TYPE;
	value = crm_element_value(action_rsc, name);
	crm_xml_add(rsc, name, value);
	name = XML_AGENT_ATTR_CLASS;
	value = crm_element_value(action_rsc, name);
	crm_xml_add(rsc, name, value);
	name = XML_AGENT_ATTR_PROVIDER;
	value = crm_element_value(action_rsc, name);
	crm_xml_add(rsc, name, value);

	op = convert_graph_action(NULL, action, status, op_rc);
	op->call_id = -1;
	op->user_data = generate_transition_key(transition_graph->id, action->id, target_rc, te_uuid);

	xml_op = create_operation_update(rsc, op, CRM_FEATURE_SET, target_rc, target, __FUNCTION__, LOG_INFO);
	lrmd_free_event(op);

	crm_trace("Updating CIB with \"%s\" (%s): %s %s on %s",
	status < 0 ? "new action" : XML_ATTR_TIMEOUT,
	crm_element_name(action->xml), crm_str(task), rsc_id, target);
	crm_log_xml_trace(xml_op, "Op");

	rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, state, call_options);

	crm_trace("Updating CIB with %s action %d: %s on %s (call_id=%d)",
	services_lrm_status_str(status), action->id, task_uuid, target, rc);

	fsa_register_cib_callback(rc, FALSE, NULL, cib_action_updated);
	free_xml(state);

	action->sent_update = TRUE;

	if (rc < pcmk_ok) {
	return FALSE;
	}

	return TRUE;
	}

	static gboolean
	te_rsc_command(crm_graph_t * graph, crm_action_t * action)
	{
	/* never overwrite stop actions in the CIB with
	* anything other than completed results
	*
	* Writing pending stops makes it look like the
	* resource is running again
	*/
	xmlNode *cmd = NULL;
	xmlNode *rsc_op = NULL;

	gboolean rc = TRUE;
	gboolean no_wait = FALSE;
	gboolean is_local = FALSE;

	char *counter = NULL;
	const char *task = NULL;
	const char *value = NULL;
	const char *on_node = NULL;
	const char *router_node = NULL;
	const char *task_uuid = NULL;

	CRM_ASSERT(action != NULL);
	CRM_ASSERT(action->xml != NULL);

	action->executed = FALSE;
	on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);

	CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
	crm_err("Corrupted command(id=%s) %s: no node", ID(action->xml), crm_str(task));
	return FALSE);

	rsc_op = action->xml;
	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
	router_node = crm_element_value(rsc_op, XML_LRM_ATTR_ROUTER_NODE);

	if (!router_node) {
	router_node = on_node;
	}

	counter =
	generate_transition_key(transition_graph->id, action->id, get_target_rc(action), te_uuid);
	crm_xml_add(rsc_op, XML_ATTR_TRANSITION_KEY, counter);

	if (safe_str_eq(router_node, fsa_our_uname)) {
	is_local = TRUE;
	}

	value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
	if (crm_is_true(value)) {
	no_wait = TRUE;
	}

	crm_notice("Initiating action %d: %s %s on %s%s%s",
	action->id, task, task_uuid, on_node,
	is_local ? " (local)" : "", no_wait ? " - no waiting" : "");

	cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, router_node,
	CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);

	if (is_local) {
	/* shortcut local resource commands */
	ha_msg_input_t data = {
	.msg = cmd,
	.xml = rsc_op,
	};

	fsa_data_t msg = {
	.id = 0,
	.data = &data,
	.data_type = fsa_dt_ha_msg,
	.fsa_input = I_NULL,
	.fsa_cause = C_FSA_INTERNAL,
	.actions = A_LRM_INVOKE,
	.origin = __FUNCTION__,
	};

	do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, fsa_state, I_NULL, &msg);

	} else {
	rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_lrmd, cmd, TRUE);
	}

	free(counter);
	free_xml(cmd);

	action->executed = TRUE;

	if (rc == FALSE) {
	crm_err("Action %d failed: send", action->id);
	return FALSE;

	} else if (no_wait) {
	crm_info("Action %d confirmed - no wait", action->id);
	action->confirmed = TRUE; /* Just mark confirmed.
	* Don't bump the job count only to immediately decrement it
	*/
	update_graph(transition_graph, action);
	trigger_graph();

	} else if (action->confirmed == TRUE) {
	crm_debug("Action %d: %s %s on %s(timeout %dms) was already confirmed.",
	action->id, task, task_uuid, on_node, action->timeout);
	} else {
	if (action->timeout <= 0) {
	crm_err("Action %d: %s %s on %s had an invalid timeout (%dms). Using %dms instead",
	action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
	action->timeout = graph->network_delay;
	}
	te_update_job_count(action, 1);
	te_start_action_timer(graph, action);
	}

	value = crm_meta_value(action->params, XML_OP_ATTR_PENDING);
	if (crm_is_true(value)
	&& safe_str_neq(task, CRMD_ACTION_CANCEL)
	&& safe_str_neq(task, CRMD_ACTION_DELETE)) {
	/* write a "pending" entry to the CIB, inhibit notification */
	crm_debug("Recording pending op %s in the CIB", task_uuid);
	cib_action_update(action, PCMK_LRM_OP_PENDING, PCMK_OCF_UNKNOWN);
	}

	return TRUE;
	}

	struct te_peer_s
	{
	char *name;
	int jobs;
	int migrate_jobs;
	};

	static void te_peer_free(gpointer p)
	{
	struct te_peer_s *peer = p;

	free(peer->name);
	free(peer);
	}

	void te_reset_job_counts(void)
	{
	GHashTableIter iter;
	struct te_peer_s *peer = NULL;

	if(te_targets == NULL) {
	te_targets = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, te_peer_free);
	}

	g_hash_table_iter_init(&iter, te_targets);
	while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & peer)) {
	peer->jobs = 0;
	peer->migrate_jobs = 0;
	}
	}

	static void
	te_update_job_count_on(const char *target, int offset, bool migrate)
	{
	struct te_peer_s *r = NULL;

	if(target == NULL \|\| te_targets == NULL) {
	return;
	}

	r = g_hash_table_lookup(te_targets, target);
	if(r == NULL) {
	r = calloc(1, sizeof(struct te_peer_s));
	r->name = strdup(target);
	g_hash_table_insert(te_targets, r->name, r);
	}

	r->jobs += offset;
	if(migrate) {
	r->migrate_jobs += offset;
	}
	crm_trace("jobs[%s] = %d", target, r->jobs);
	}

	static void
	te_update_job_count(crm_action_t * action, int offset)
	{
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);

	if (action->type != action_type_rsc \|\| target == NULL) {
	/* No limit on these */
	return;
	}

	/* if we have a router node, this means the action is performing
	* on a remote node. For now, we count all action occuring on a
	* remote node against the job list on the cluster node hosting
	* the connection resources */
	target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);

	if ((target == NULL) &&
	(safe_str_eq(task, CRMD_ACTION_MIGRATE) \|\| safe_str_eq(task, CRMD_ACTION_MIGRATED))) {

	const char *t1 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
	const char *t2 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);

	te_update_job_count_on(t1, offset, TRUE);
	te_update_job_count_on(t2, offset, TRUE);
	return;
	} else if (target == NULL) {
	target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	}

	te_update_job_count_on(target, offset, FALSE);
	}

	static gboolean
	te_should_perform_action_on(crm_graph_t * graph, crm_action_t * action, const char *target)
	{
	int limit = 0;
	struct te_peer_s *r = NULL;
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
	const char *id = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);

	if(target == NULL) {
	/* No limit on these */
	return TRUE;

	} else if(te_targets == NULL) {
	return FALSE;
	}

	r = g_hash_table_lookup(te_targets, target);
	limit = throttle_get_job_limit(target);

	if(r == NULL) {
	r = calloc(1, sizeof(struct te_peer_s));
	r->name = strdup(target);
	g_hash_table_insert(te_targets, r->name, r);
	}

	if(limit <= r->jobs) {
	crm_trace("Peer %s is over their job limit of %d (%d): deferring %s",
	target, limit, r->jobs, id);
	return FALSE;

	} else if(graph->migration_limit > 0 && r->migrate_jobs >= graph->migration_limit) {
	if (safe_str_eq(task, CRMD_ACTION_MIGRATE) \|\| safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
	crm_trace("Peer %s is over their migration job limit of %d (%d): deferring %s",
	target, graph->migration_limit, r->migrate_jobs, id);
	return FALSE;
	}
	}

	crm_trace("Peer %s has not hit their limit yet. current jobs = %d limit= %d limit", target, r->jobs, limit);

	return TRUE;
	}

	static gboolean
	te_should_perform_action(crm_graph_t * graph, crm_action_t * action)
	{
	const char *target = NULL;
	const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);

	if (action->type != action_type_rsc) {
	/* No limit on these */
	return TRUE;
	}

	/* if we have a router node, this means the action is performing
	* on a remote node. For now, we count all action occuring on a
	* remote node against the job list on the cluster node hosting
	* the connection resources */
	target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);

	if ((target == NULL) &&
	(safe_str_eq(task, CRMD_ACTION_MIGRATE) \|\| safe_str_eq(task, CRMD_ACTION_MIGRATED))) {

	target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
	if(te_should_perform_action_on(graph, action, target) == FALSE) {
	return FALSE;
	}

	target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);

	} else if (target == NULL) {
	target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
	}

	return te_should_perform_action_on(graph, action, target);
	}

	void
	te_action_confirmed(crm_action_t * action)
	{
	const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);

	if (action->confirmed == FALSE && action->type == action_type_rsc && target != NULL) {
	te_update_job_count(action, -1);
	}
	action->confirmed = TRUE;
	}


	crm_graph_functions_t te_graph_fns = {
	te_pseudo_action,
	te_rsc_command,
	te_crm_command,
	te_fence_node,
	te_should_perform_action,
	};

	void
	notify_crmd(crm_graph_t * graph)
	{
	const char *type = "unknown";
	enum crmd_fsa_input event = I_NULL;

	crm_debug("Processing transition completion in state %s", fsa_state2string(fsa_state));

	CRM_CHECK(graph->complete, graph->complete = TRUE);

	switch (graph->completion_action) {
	case tg_stop:
	type = "stop";
	if (fsa_state == S_TRANSITION_ENGINE) {
	event = I_TE_SUCCESS;
	}
	break;
	case tg_done:
	type = "done";
	if (fsa_state == S_TRANSITION_ENGINE) {
	event = I_TE_SUCCESS;
	}
	break;

	case tg_restart:
	type = "restart";
	if (fsa_state == S_TRANSITION_ENGINE) {
	if (too_many_st_failures() == FALSE) {
	if (transition_timer->period_ms > 0) {
	crm_timer_stop(transition_timer);
	crm_timer_start(transition_timer);
	} else {
	event = I_PE_CALC;
	}
	} else {
	event = I_TE_SUCCESS;
	}

	} else if (fsa_state == S_POLICY_ENGINE) {
	register_fsa_action(A_PE_INVOKE);
	}
	break;

	case tg_shutdown:
	type = "shutdown";
	if (is_set(fsa_input_register, R_SHUTDOWN)) {
	event = I_STOP;

	} else {
	- crm_err("We didn't ask to be shut down, yet our" " PE is telling us too.");
	+ crm_err("We didn't ask to be shut down, yet our PE is telling us to.");
	event = I_TERMINATE;
	}
	}

	crm_debug("Transition %d status: %s - %s", graph->id, type, crm_str(graph->abort_reason));

	graph->abort_reason = NULL;
	graph->completion_action = tg_done;
	clear_bit(fsa_input_register, R_IN_TRANSITION);

	if (event != I_NULL) {
	register_fsa_input(C_FSA_INTERNAL, event, NULL);

	} else if (fsa_source) {
	mainloop_set_trigger(fsa_source);
	}
	}
	diff --git a/doc/Pacemaker_Remote/en-US/Ch-Intro.txt b/doc/Pacemaker_Remote/en-US/Ch-Intro.txt
	index 16934907f8..9edf054a69 100644
	--- a/doc/Pacemaker_Remote/en-US/Ch-Intro.txt
	+++ b/doc/Pacemaker_Remote/en-US/Ch-Intro.txt
	@@ -1,198 +1,198 @@
	= Scaling a Pacemaker Cluster =

	== Overview ==

	In a basic Pacemaker high-availability
	cluster,footnote:[See the http://www.clusterlabs.org/doc/[Pacemaker
	documentation], especially 'Clusters From Scratch' and 'Pacemaker Explained',
	for basic information about high-availability using Pacemaker]
	each node runs the full cluster stack of corosync and all Pacemaker components.
	This allows great flexibility but limits scalability to around 16 nodes.

	To allow for scalability to dozens or even hundreds of nodes, Pacemaker
	allows nodes not running the full cluster stack to integrate into the cluster
	and have the cluster manage their resources as if they were a cluster node.

	== Terms ==

	cluster node::
	A node running the full high-availability stack of corosync and all
	Pacemaker components. Cluster nodes may run cluster resources, run
	all Pacemaker command-line tools (`crm_mon`, `crm_resource` and so on),
	execute fencing actions, count toward cluster quorum, and serve as the
	cluster's Designated Controller (DC).
	(((cluster node)))
	(((node,cluster node)))

	pacemaker_remote::
	A small service daemon that allows a host to be used as a Pacemaker node
	without running the full cluster stack. Nodes running pacemaker_remote
	may run cluster resources and most command-line tools, but cannot perform
	other functions of full cluster nodes such as fencing execution, quorum
	voting or DC eligibility. The pacemaker_remote daemon is an enhanced
	version of Pacemaker's local resource management daemon (LRMD).
	(((pacemaker_remote)))

	remote node::
	A physical host running pacemaker_remote. Remote nodes have a special
	resource that manages communication with the cluster. This is sometimes
	referred to as the 'baremetal' case.
	(((remote node)))
	(((node,remote node)))

	guest node::
	A virtual host running pacemaker_remote. Guest nodes differ from remote
	nodes mainly in that the guest node is itself a resource that the cluster
	manages.
	(((guest node)))
	(((node,guest node)))

	[NOTE]
	======
	'Remote' in this document refers to the node not being a part of the underlying
	corosync cluster. It has nothing to do with physical proximity. Remote nodes
	and guest nodes are subject to the same latency requirements as cluster nodes,
	which means they are typically in the same data center.
	======

	[NOTE]
	======
	It is important to distinguish the various roles a virtual machine can serve
	in Pacemaker clusters:

	* A virtual machine can run the full cluster stack, in which case it is a
	cluster node and is not itself managed by the cluster.
	* A virtual machine can be managed by the cluster as a resource, without the
	cluster having any awareness of the services running inside the virtual
	machine. The virtual machine is 'opaque' to the cluster.
	* A virtual machine can be a cluster resource, and run pacemaker_remote
	- to make it a a guest node, allowing the cluster to manage services
	+ to make it a guest node, allowing the cluster to manage services
	inside it. The virtual machine is 'transparent' to the cluster.
	======

	== Support in Pacemaker Versions ==

	It is recommended to run Pacemaker 1.1.12 or later when using pacemaker_remote
	due to important bug fixes. An overview of changes in pacemaker_remote
	capability by version:

	.1.1.14
	* Resources that create guest nodes can be included in groups
	* reconnect_interval option for remote nodes
	* Bug fixes, including a memory leak

	.1.1.13
	* Support for maintenance mode
	* Remote nodes can recover without being fenced when the cluster node
	hosting their connection fails
	* Running pacemaker_remote within LXC environments is deprecated due to
	newly added Pacemaker support for isolated resources
	* Bug fixes

	.1.1.12
	* Support for permanent node attributes
	* Support for migration
	* Bug fixes

	.1.1.11
	* Support for IPv6
	* Support for remote nodes
	* Support for transient node attributes
	* Support for clusters with mixed endian architectures
	* Bug fixes

	.1.1.10
	* Bug fixes

	.1.1.9
	* Initial version to include pacemaker_remote
	* Limited to guest nodes in KVM/LXC environments using only IPv4;
	all nodes' architectures must have same endianness

	== Guest Nodes ==
	(((guest node)))
	(((node,guest node)))

	*"I want a Pacemaker cluster to manage virtual machine resources, but I also
	want Pacemaker to be able to manage the resources that live within those
	virtual machines."*

	Without pacemaker_remote, the possibilities for implementing the above use case
	have significant limitations:

	* The cluster stack could be run on the physical hosts only, which loses the
	ability to monitor resources within the guests.
	* A separate cluster could be on the virtual guests, which quickly hits
	scalability issues.
	* The cluster stack could be run on the guests using the same cluster as the
	physical hosts, which also hits scalability issues and complicates fencing.

	With pacemaker_remote:

	* The physical hosts are cluster nodes (running the full cluster stack).
	* The virtual machines are guest nodes (running the pacemaker_remote service).
	Nearly zero configuration is required on the virtual machine.
	* The cluster stack on the cluster nodes launches the virtual machines and
	immediately connects to the pacemaker_remote service on them, allowing the
	virtual machines to integrate into the cluster.

	The key difference here between the guest nodes and the cluster nodes is that
	the guest nodes do not run the cluster stack. This means they will never become
	the DC, initiate fencing actions or participate in quorum voting.

	On the other hand, this also means that they are not bound to the scalability
	limits associated with the cluster stack (no 16-node corosync member limits to
	deal with). That isn't to say that guest nodes can scale indefinitely, but it
	is known that guest nodes scale horizontally much further than cluster nodes.

	Other than the quorum limitation, these guest nodes behave just like cluster
	nodes with respect to resource management. The cluster is fully capable of
	managing and monitoring resources on each guest node. You can build constraints
	against guest nodes, put them in standby, or do whatever else you'd expect to
	be able to do with cluster nodes. They even show up in `crm_mon` output as
	nodes.

	To solidify the concept, below is an example that is very similar to an actual
	deployment we test in our developer environment to verify guest node scalability:

	* 16 cluster nodes running the full corosync + pacemaker stack
	* 64 Pacemaker-managed virtual machine resources running pacemaker_remote configured as guest nodes
	* 64 Pacemaker-managed webserver and database resources configured to run on the 64 guest nodes

	With this deployment, you would have 64 webservers and databases running on 64
	virtual machines on 16 hardware nodes, all of which are managed and monitored by
	the same Pacemaker deployment. It is known that pacemaker_remote can scale to
	these lengths and possibly much further depending on the specific scenario.


	== Remote Nodes ==
	(((remote node)))
	(((node,remote node)))

	*"I want my traditional high-availability cluster to scale beyond the limits
	imposed by the corosync messaging layer."*

	Ultimately, the primary advantage of remote nodes over cluster nodes is
	scalability. There are likely some other use cases related to geographically
	distributed HA clusters that remote nodes may serve a purpose in, but those use
	cases are not well understood at this point.

	Like guest nodes, remote nodes will never become the DC, initiate
	fencing actions or participate in quorum voting.

	That is not to say, however, that fencing of a remote node works any
	differently than that of a cluster node. The Pacemaker policy engine
	understands how to fence remote nodes. As long as a fencing device exists, the
	cluster is capable of ensuring remote nodes are fenced in the exact same way as
	cluster nodes.

	== Expanding the Cluster Stack ==

	With pacemaker_remote, the traditional view of the high-availability stack can
	be expanded to include a new layer:

	.Traditional HA Stack
	image::images/pcmk-ha-cluster-stack.png["Traditional Pacemaker+Corosync Stack",width="17cm",height="9cm",align="center"]

	.HA Stack With Guest Nodes
	image::images/pcmk-ha-remote-stack.png["Pacemaker+Corosync Stack With pacemaker_remote",width="20cm",height="10cm",align="center"]
	diff --git a/fencing/main.c b/fencing/main.c
	index 0dc449240c..c3edae9cf7 100644
	--- a/fencing/main.c
	+++ b/fencing/main.c
	@@ -1,1521 +1,1521 @@
	/*
	* Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <sys/utsname.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/ipc.h>
	#include <crm/common/ipcs.h>
	#include <crm/cluster/internal.h>

	#include <crm/stonith-ng.h>
	#include <crm/fencing/internal.h>
	#include <crm/common/xml.h>

	#include <crm/common/mainloop.h>

	#include <crm/cib/internal.h>
	#include <crm/pengine/status.h>
	#include <allocate.h>

	#include <internal.h>

	#include <standalone_config.h>

	char *stonith_our_uname = NULL;
	char *stonith_our_uuid = NULL;
	long stonith_watchdog_timeout_ms = 0;

	GMainLoop *mainloop = NULL;

	gboolean stand_alone = FALSE;
	gboolean no_cib_connect = FALSE;
	gboolean stonith_shutdown_flag = FALSE;

	qb_ipcs_service_t *ipcs = NULL;
	xmlNode *local_cib = NULL;

	static cib_t *cib_api = NULL;
	static void *cib_library = NULL;

	static void stonith_shutdown(int nsig);
	static void stonith_cleanup(void);

	static int32_t
	st_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
	{
	if (stonith_shutdown_flag) {
	crm_info("Ignoring new client [%d] during shutdown", crm_ipcs_client_pid(c));
	return -EPERM;
	}

	if (crm_client_new(c, uid, gid) == NULL) {
	return -EIO;
	}
	return 0;
	}

	static void
	st_ipc_created(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection created for %p", c);
	}

	/* Exit code means? */
	static int32_t
	st_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
	{
	uint32_t id = 0;
	uint32_t flags = 0;
	int call_options = 0;
	xmlNode *request = NULL;
	crm_client_t *c = crm_client_get(qbc);
	const char *op = NULL;

	if (c == NULL) {
	crm_info("Invalid client: %p", qbc);
	return 0;
	}

	request = crm_ipcs_recv(c, data, size, &id, &flags);
	if (request == NULL) {
	crm_ipcs_send_ack(c, id, flags, "nack", __FUNCTION__, __LINE__);
	return 0;
	}


	op = crm_element_value(request, F_CRM_TASK);
	if(safe_str_eq(op, CRM_OP_RM_NODE_CACHE)) {
	crm_xml_add(request, F_TYPE, T_STONITH_NG);
	crm_xml_add(request, F_STONITH_OPERATION, op);
	crm_xml_add(request, F_STONITH_CLIENTID, c->id);
	crm_xml_add(request, F_STONITH_CLIENTNAME, crm_client_name(c));
	crm_xml_add(request, F_STONITH_CLIENTNODE, stonith_our_uname);

	send_cluster_message(NULL, crm_msg_stonith_ng, request, FALSE);
	free_xml(request);
	return 0;
	}

	if (c->name == NULL) {
	const char *value = crm_element_value(request, F_STONITH_CLIENTNAME);

	if (value == NULL) {
	value = "unknown";
	}
	c->name = crm_strdup_printf("%s.%u", value, c->pid);
	}

	crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
	crm_trace("Flags %u/%u for command %u from %s", flags, call_options, id, crm_client_name(c));

	if (is_set(call_options, st_opt_sync_call)) {
	CRM_ASSERT(flags & crm_ipc_client_response);
	CRM_LOG_ASSERT(c->request_id == 0); /* This means the client has two synchronous events in-flight */
	c->request_id = id; /* Reply only to the last one */
	}

	crm_xml_add(request, F_STONITH_CLIENTID, c->id);
	crm_xml_add(request, F_STONITH_CLIENTNAME, crm_client_name(c));
	crm_xml_add(request, F_STONITH_CLIENTNODE, stonith_our_uname);

	crm_log_xml_trace(request, "Client[inbound]");
	stonith_command(c, id, flags, request, NULL);

	free_xml(request);
	return 0;
	}

	/* Error code means? */
	static int32_t
	st_ipc_closed(qb_ipcs_connection_t * c)
	{
	crm_client_t *client = crm_client_get(c);

	if (client == NULL) {
	return 0;
	}

	crm_trace("Connection %p closed", c);
	crm_client_destroy(client);

	/* 0 means: yes, go ahead and destroy the connection */
	return 0;
	}

	static void
	st_ipc_destroy(qb_ipcs_connection_t * c)
	{
	crm_trace("Connection %p destroyed", c);
	st_ipc_closed(c);
	}

	static void
	stonith_peer_callback(xmlNode * msg, void *private_data)
	{
	const char *remote_peer = crm_element_value(msg, F_ORIG);
	const char *op = crm_element_value(msg, F_STONITH_OPERATION);

	if (crm_str_eq(op, "poke", TRUE)) {
	return;
	}

	crm_log_xml_trace(msg, "Peer[inbound]");
	stonith_command(NULL, 0, 0, msg, remote_peer);
	}

	#if SUPPORT_HEARTBEAT
	static void
	stonith_peer_hb_callback(HA_Message * msg, void *private_data)
	{
	xmlNode *xml = convert_ha_message(NULL, msg, __FUNCTION__);

	stonith_peer_callback(xml, private_data);
	free_xml(xml);
	}

	static void
	stonith_peer_hb_destroy(gpointer user_data)
	{
	if (stonith_shutdown_flag) {
	crm_info("Heartbeat disconnection complete... exiting");
	} else {
	crm_err("Heartbeat connection lost! Exiting.");
	}
	stonith_shutdown(0);
	}
	#endif

	#if SUPPORT_COROSYNC
	static void
	stonith_peer_ais_callback(cpg_handle_t handle,
	const struct cpg_name *groupName,
	uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len)
	{
	uint32_t kind = 0;
	xmlNode *xml = NULL;
	const char *from = NULL;
	char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from);

	if(data == NULL) {
	return;
	}
	if (kind == crm_class_cluster) {
	xml = string2xml(data);
	if (xml == NULL) {
	crm_err("Invalid XML: '%.120s'", data);
	free(data);
	return;
	}
	crm_xml_add(xml, F_ORIG, from);
	/* crm_xml_add_int(xml, F_SEQ, wrapper->id); */
	stonith_peer_callback(xml, NULL);
	}

	free_xml(xml);
	free(data);
	return;
	}

	static void
	stonith_peer_cs_destroy(gpointer user_data)
	{
	crm_err("Corosync connection terminated");
	stonith_shutdown(0);
	}
	#endif

	void
	do_local_reply(xmlNode * notify_src, const char *client_id, gboolean sync_reply, gboolean from_peer)
	{
	/* send callback to originating child */
	crm_client_t *client_obj = NULL;
	int local_rc = pcmk_ok;

	crm_trace("Sending response");
	client_obj = crm_client_get_by_id(client_id);

	crm_trace("Sending callback to request originator");
	if (client_obj == NULL) {
	local_rc = -1;
	crm_trace("No client to sent the response to. F_STONITH_CLIENTID not set.");

	} else {
	int rid = 0;

	if (sync_reply) {
	CRM_LOG_ASSERT(client_obj->request_id);

	rid = client_obj->request_id;
	client_obj->request_id = 0;

	crm_trace("Sending response %d to %s %s",
	rid, client_obj->name, from_peer ? "(originator of delegated request)" : "");

	} else {
	crm_trace("Sending an event to %s %s",
	client_obj->name, from_peer ? "(originator of delegated request)" : "");
	}

	local_rc = crm_ipcs_send(client_obj, rid, notify_src, sync_reply?crm_ipc_flags_none:crm_ipc_server_event);
	}

	if (local_rc < pcmk_ok && client_obj != NULL) {
	crm_warn("%sSync reply to %s failed: %s",
	sync_reply ? "" : "A-",
	client_obj ? client_obj->name : "<unknown>", pcmk_strerror(local_rc));
	}
	}

	long long
	get_stonith_flag(const char *name)
	{
	if (safe_str_eq(name, T_STONITH_NOTIFY_FENCE)) {
	return 0x01;

	} else if (safe_str_eq(name, STONITH_OP_DEVICE_ADD)) {
	return 0x04;

	} else if (safe_str_eq(name, STONITH_OP_DEVICE_DEL)) {
	return 0x10;
	}
	return 0;
	}

	static void
	stonith_notify_client(gpointer key, gpointer value, gpointer user_data)
	{

	xmlNode *update_msg = user_data;
	crm_client_t *client = value;
	const char *type = NULL;

	CRM_CHECK(client != NULL, return);
	CRM_CHECK(update_msg != NULL, return);

	type = crm_element_value(update_msg, F_SUBTYPE);
	CRM_CHECK(type != NULL, crm_log_xml_err(update_msg, "notify"); return);

	if (client->ipcs == NULL) {
	crm_trace("Skipping client with NULL channel");
	return;
	}

	if (client->options & get_stonith_flag(type)) {
	int rc = crm_ipcs_send(client, 0, update_msg, crm_ipc_server_event \| crm_ipc_server_error);

	if (rc <= 0) {
	crm_warn("%s notification of client %s.%.6s failed: %s (%d)",
	type, crm_client_name(client), client->id, pcmk_strerror(rc), rc);
	} else {
	crm_trace("Sent %s notification to client %s.%.6s", type, crm_client_name(client),
	client->id);
	}
	}
	}

	void
	do_stonith_async_timeout_update(const char client_id, const char call_id, int timeout)
	{
	crm_client_t *client = NULL;
	xmlNode *notify_data = NULL;

	if (!timeout \|\| !call_id \|\| !client_id) {
	return;
	}

	client = crm_client_get_by_id(client_id);
	if (!client) {
	return;
	}

	notify_data = create_xml_node(NULL, T_STONITH_TIMEOUT_VALUE);
	crm_xml_add(notify_data, F_TYPE, T_STONITH_TIMEOUT_VALUE);
	crm_xml_add(notify_data, F_STONITH_CALLID, call_id);
	crm_xml_add_int(notify_data, F_STONITH_TIMEOUT, timeout);

	crm_trace("timeout update is %d for client %s and call id %s", timeout, client_id, call_id);

	if (client) {
	crm_ipcs_send(client, 0, notify_data, crm_ipc_server_event);
	}

	free_xml(notify_data);
	}

	void
	do_stonith_notify(int options, const char type, int result, xmlNode data)
	{
	/* TODO: Standardize the contents of data */
	xmlNode *update_msg = create_xml_node(NULL, "notify");

	CRM_CHECK(type != NULL,;);

	crm_xml_add(update_msg, F_TYPE, T_STONITH_NOTIFY);
	crm_xml_add(update_msg, F_SUBTYPE, type);
	crm_xml_add(update_msg, F_STONITH_OPERATION, type);
	crm_xml_add_int(update_msg, F_STONITH_RC, result);

	if (data != NULL) {
	add_message_xml(update_msg, F_STONITH_CALLDATA, data);
	}

	crm_trace("Notifying clients");
	g_hash_table_foreach(client_connections, stonith_notify_client, update_msg);
	free_xml(update_msg);
	crm_trace("Notify complete");
	}

	static void
	do_stonith_notify_config(int options, const char *op, int rc,
	const char *desc, int active)
	{
	xmlNode *notify_data = create_xml_node(NULL, op);

	CRM_CHECK(notify_data != NULL, return);

	crm_xml_add(notify_data, F_STONITH_DEVICE, desc);
	crm_xml_add_int(notify_data, F_STONITH_ACTIVE, active);

	do_stonith_notify(options, op, rc, notify_data);
	free_xml(notify_data);
	}

	void
	do_stonith_notify_device(int options, const char op, int rc, const char desc)
	{
	do_stonith_notify_config(options, op, rc, desc, g_hash_table_size(device_list));
	}

	void
	do_stonith_notify_level(int options, const char op, int rc, const char desc)
	{
	do_stonith_notify_config(options, op, rc, desc, g_hash_table_size(topology));
	}

	static void
	topology_remove_helper(const char *node, int level)
	{
	int rc;
	char *desc = NULL;
	xmlNode *data = create_xml_node(NULL, XML_TAG_FENCING_LEVEL);

	crm_xml_add(data, F_STONITH_ORIGIN, __FUNCTION__);
	crm_xml_add_int(data, XML_ATTR_STONITH_INDEX, level);
	crm_xml_add(data, XML_ATTR_STONITH_TARGET, node);

	rc = stonith_level_remove(data, &desc);
	do_stonith_notify_level(0, STONITH_OP_LEVEL_DEL, rc, desc);

	free_xml(data);
	free(desc);
	}

	static void
	remove_cib_device(xmlXPathObjectPtr xpathObj)
	{
	int max = numXpathResults(xpathObj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	const char *rsc_id = NULL;
	const char *standard = NULL;
	xmlNode *match = getXpathResult(xpathObj, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if(match != NULL) {
	standard = crm_element_value(match, XML_AGENT_ATTR_CLASS);
	}

	if (safe_str_neq(standard, "stonith")) {
	continue;
	}

	rsc_id = crm_element_value(match, XML_ATTR_ID);

	stonith_device_remove(rsc_id, TRUE);
	}
	}

	static void
	handle_topology_change(xmlNode *match, bool remove)
	{
	int rc;
	char *desc = NULL;

	CRM_CHECK(match != NULL, return);
	crm_trace("Updating %s", ID(match));

	if(remove) {
	int index = 0;
	char *key = stonith_level_key(match, -1);

	crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index);
	topology_remove_helper(key, index);
	free(key);
	}

	rc = stonith_level_register(match, &desc);
	do_stonith_notify_level(0, STONITH_OP_LEVEL_ADD, rc, desc);

	free(desc);
	}

	static void
	remove_fencing_topology(xmlXPathObjectPtr xpathObj)
	{
	int max = numXpathResults(xpathObj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);

	CRM_LOG_ASSERT(match != NULL);
	if (match && crm_element_value(match, XML_DIFF_MARKER)) {
	/* Deletion */
	int index = 0;
	char *target = stonith_level_key(match, -1);

	crm_element_value_int(match, XML_ATTR_STONITH_INDEX, &index);
	if (target == NULL) {
	crm_err("Invalid fencing target in element %s", ID(match));

	} else if (index <= 0) {
	crm_err("Invalid level for %s in element %s", target, ID(match));

	} else {
	topology_remove_helper(target, index);
	}
	/* } else { Deal with modifications during the 'addition' stage */
	}
	}
	}

	static void
	register_fencing_topology(xmlXPathObjectPtr xpathObj)
	{
	int max = numXpathResults(xpathObj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);

	handle_topology_change(match, TRUE);
	}
	}

	/* Fencing
	<diff crm_feature_set="3.0.6">
	<diff-removed>
	<fencing-topology>
	<fencing-level id="f-p1.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="removed:top"/>
	<fencing-level id="f-p1.2" target="pcmk-1" index="2" devices="power" __crm_diff_marker__="removed:top"/>
	<fencing-level devices="disk,network" id="f-p2.1"/>
	</fencing-topology>
	</diff-removed>
	<diff-added>
	<fencing-topology>
	<fencing-level id="f-p.1" target="pcmk-1" index="1" devices="poison-pill" __crm_diff_marker__="added:top"/>
	<fencing-level id="f-p2.1" target="pcmk-2" index="1" devices="disk,something"/>
	<fencing-level id="f-p3.1" target="pcmk-2" index="2" devices="power" __crm_diff_marker__="added:top"/>
	</fencing-topology>
	</diff-added>
	</diff>
	*/

	static void
	fencing_topology_init()
	{
	xmlXPathObjectPtr xpathObj = NULL;
	const char *xpath = "//" XML_TAG_FENCING_LEVEL;

	crm_trace("Full topology refresh");

	if(topology) {
	g_hash_table_destroy(topology);
	topology = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_topology_entry);
	}

	/* Grab everything */
	xpathObj = xpath_search(local_cib, xpath);
	register_fencing_topology(xpathObj);

	freeXpathObject(xpathObj);
	}

	#define rsc_name(x) x->clone_name?x->clone_name:x->id

	/*!
	* \internal
	* \brief Check whether our uname is in a resource's allowed node list
	*
	* \param[in] rsc Resource to check
	*
	* \return Pointer to node object if found, NULL otherwise
	*/
	static node_t *
	our_node_allowed_for(resource_t *rsc)
	{
	GHashTableIter iter;
	node_t *node = NULL;

	if (rsc && stonith_our_uname) {
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (node && strcmp(node->details->uname, stonith_our_uname) == 0) {
	break;
	}
	node = NULL;
	}
	}
	return node;
	}

	/*!
	* \internal
	* \brief If a resource or any of its children are STONITH devices, update their
	* definitions given a cluster working set.
	*
	* \param[in] rsc Resource to check
	* \param[in] data_set Cluster working set with device information
	*/
	static void cib_device_update(resource_t rsc, pe_working_set_t data_set)
	{
	node_t *node = NULL;
	const char *value = NULL;
	const char *rclass = NULL;
	node_t *parent = NULL;
	gboolean remove = TRUE;

	/* If this is a complex resource, check children rather than this resource itself.
	* TODO: Mark each installed device and remove if untouched when this process finishes.
	*/
	if(rsc->children) {
	GListPtr gIter = NULL;
	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	cib_device_update(gIter->data, data_set);
	if(rsc->variant == pe_clone \|\| rsc->variant == pe_master) {
	crm_trace("Only processing one copy of the clone %s", rsc->id);
	break;
	}
	}
	return;
	}

	/* We only care about STONITH resources. */
	rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	if(safe_str_neq(rclass, "stonith")) {
	return;
	}

	/* If this STONITH resource is disabled, just remove it. */
	value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	if (safe_str_eq(value, RSC_STOPPED)) {
	crm_info("Device %s has been disabled", rsc->id);
	goto update_done;
	}

	/* Check whether our node is allowed for this resource (and its parent if in a group) */
	node = our_node_allowed_for(rsc);
	if (rsc->parent && (rsc->parent->variant == pe_group)) {
	parent = our_node_allowed_for(rsc->parent);
	}

	if(node == NULL) {
	/* Our node is disallowed, so remove the device */
	GHashTableIter iter;

	crm_info("Device %s has been disabled on %s: unknown", rsc->id, stonith_our_uname);
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	crm_trace("Available: %s = %d", node->details->uname, node->weight);
	}

	goto update_done;

	} else if(node->weight < 0 \|\| (parent && parent->weight < 0)) {
	/* Our node (or its group) is disallowed by score, so remove the device */
	char *score = score2char((node->weight < 0) ? node->weight : parent->weight);

	crm_info("Device %s has been disabled on %s: score=%s", rsc->id, stonith_our_uname, score);
	free(score);

	goto update_done;

	} else {
	/* Our node is allowed, so update the device information */
	xmlNode *data;
	GHashTableIter gIter;
	stonith_key_value_t *params = NULL;

	const char *name = NULL;
	const char *agent = crm_element_value(rsc->xml, XML_EXPR_ATTR_TYPE);
	const char *provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	const char *rsc_provides = NULL;

	crm_debug("Device %s is allowed on %s: score=%d", rsc->id, stonith_our_uname, node->weight);
	get_rsc_attributes(rsc->parameters, rsc, node, data_set);
	get_meta_attributes(rsc->meta, rsc, node, data_set);

	rsc_provides = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_PROVIDES);

	g_hash_table_iter_init(&gIter, rsc->parameters);
	while (g_hash_table_iter_next(&gIter, (gpointer ) & name, (gpointer ) & value)) {
	if (!name \|\| !value) {
	continue;
	}
	params = stonith_key_value_add(params, name, value);
	crm_trace(" %s=%s", name, value);
	}

	remove = FALSE;
	data = create_device_registration_xml(rsc_name(rsc), provider, agent, params, rsc_provides);
	stonith_device_register(data, NULL, TRUE);

	stonith_key_value_freeall(params, 1, 1);
	free_xml(data);
	}

	update_done:

	if(remove && g_hash_table_lookup(device_list, rsc_name(rsc))) {
	stonith_device_remove(rsc_name(rsc), TRUE);
	}
	}

	extern xmlNode do_calculations(pe_working_set_t data_set, xmlNode * xml_input, crm_time_t * now);
	extern node_t create_node(const char id, const char uname, const char type, const char score, pe_working_set_t data_set);

	/*!
	* \internal
	* \brief Update all STONITH device definitions based on current CIB
	*/
	static void
	cib_devices_update(void)
	{
	GListPtr gIter = NULL;
	pe_working_set_t data_set;

	crm_info("Updating devices to version %s.%s.%s",
	crm_element_value(local_cib, XML_ATTR_GENERATION_ADMIN),
	crm_element_value(local_cib, XML_ATTR_GENERATION),
	crm_element_value(local_cib, XML_ATTR_NUMUPDATES));

	set_working_set_defaults(&data_set);
	data_set.input = local_cib;
	data_set.now = crm_time_new(NULL);
	data_set.flags \|= pe_flag_quick_location;
	data_set.localhost = stonith_our_uname;

	cluster_status(&data_set);
	do_calculations(&data_set, NULL, NULL);

	for (gIter = data_set.resources; gIter != NULL; gIter = gIter->next) {
	cib_device_update(gIter->data, &data_set);
	}
	data_set.input = NULL; /* Wasn't a copy */
	cleanup_alloc_calculations(&data_set);
	}

	static void
	update_cib_stonith_devices_v2(const char event, xmlNode msg)
	{
	xmlNode *change = NULL;
	char *reason = NULL;
	bool needs_update = FALSE;
	xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	for (change = __xml_first_child(patchset); change != NULL; change = __xml_next(change)) {
	const char *op = crm_element_value(change, XML_DIFF_OP);
	const char *xpath = crm_element_value(change, XML_DIFF_PATH);
	const char *shortpath = NULL;

	if(op == NULL \|\| strcmp(op, "move") == 0) {
	continue;

	} else if(safe_str_eq(op, "delete") && strstr(xpath, XML_CIB_TAG_RESOURCE)) {
	const char *rsc_id = NULL;
	char *search = NULL;
	char *mutable = NULL;

	if (strstr(xpath, XML_TAG_ATTR_SETS)) {
	needs_update = TRUE;
	break;
	}
	mutable = strdup(xpath);
	rsc_id = strstr(mutable, "primitive[@id=\'");
	if (rsc_id != NULL) {
	rsc_id += strlen("primitive[@id=\'");
	search = strchr(rsc_id, '\'');
	}
	if (search != NULL) {
	*search = 0;
	stonith_device_remove(rsc_id, TRUE);
	} else {
	crm_warn("Ignoring malformed CIB update (resource deletion)");
	}
	free(mutable);

	} else if(strstr(xpath, "/"XML_CIB_TAG_RESOURCES)) {
	shortpath = strrchr(xpath, '/'); CRM_ASSERT(shortpath);
	reason = crm_strdup_printf("%s %s", op, shortpath+1);
	needs_update = TRUE;
	break;

	} else if(strstr(xpath, XML_CONS_TAG_RSC_LOCATION)) {
	shortpath = strrchr(xpath, '/'); CRM_ASSERT(shortpath);
	reason = crm_strdup_printf("%s %s", op, shortpath+1);
	needs_update = TRUE;
	break;
	}
	}

	if(needs_update) {
	crm_info("Updating device list from the cib: %s", reason);
	cib_devices_update();
	}
	free(reason);
	}


	static void
	update_cib_stonith_devices_v1(const char event, xmlNode msg)
	{
	const char *reason = "none";
	gboolean needs_update = FALSE;
	xmlXPathObjectPtr xpath_obj = NULL;

	/* process new constraints */
	xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_CONS_TAG_RSC_LOCATION);
	if (numXpathResults(xpath_obj) > 0) {
	int max = numXpathResults(xpath_obj), lpc = 0;

	/* Safest and simplest to always recompute */
	needs_update = TRUE;
	reason = "new location constraint";

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpath_obj, lpc);

	crm_log_xml_trace(match, "new constraint");
	}
	}
	freeXpathObject(xpath_obj);

	/* process deletions */
	xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_RESOURCE);
	if (numXpathResults(xpath_obj) > 0) {
	remove_cib_device(xpath_obj);
	}
	freeXpathObject(xpath_obj);

	/* process additions */
	xpath_obj = xpath_search(msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_RESOURCE);
	if (numXpathResults(xpath_obj) > 0) {
	int max = numXpathResults(xpath_obj), lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	const char *rsc_id = NULL;
	const char *standard = NULL;
	xmlNode *match = getXpathResult(xpath_obj, lpc);

	rsc_id = crm_element_value(match, XML_ATTR_ID);
	standard = crm_element_value(match, XML_AGENT_ATTR_CLASS);

	if (safe_str_neq(standard, "stonith")) {
	continue;
	}

	crm_trace("Fencing resource %s was added or modified", rsc_id);
	reason = "new resource";
	needs_update = TRUE;
	}
	}
	freeXpathObject(xpath_obj);

	if(needs_update) {
	crm_info("Updating device list from the cib: %s", reason);
	cib_devices_update();
	}
	}

	static void
	update_cib_stonith_devices(const char event, xmlNode msg)
	{
	int format = 1;
	xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	CRM_ASSERT(patchset);
	crm_element_value_int(patchset, "format", &format);
	switch(format) {
	case 1:
	update_cib_stonith_devices_v1(event, msg);
	break;
	case 2:
	update_cib_stonith_devices_v2(event, msg);
	break;
	default:
	crm_warn("Unknown patch format: %d", format);
	}
	}

	/* Needs to hold node name + attribute name + attribute value + 75 */
	#define XPATH_MAX 512

	/*
	* \internal
	* \brief Check whether a node has a specific attribute name/value
	*
	* \param[in] node Name of node to check
	* \param[in] name Name of an attribute to look for
	* \param[in] value The value the named attribute needs to be set to in order to be considered a match
	*
	* \return TRUE if the locally cached CIB has the specified node attribute
	*/
	gboolean
	node_has_attr(const char node, const char name, const char *value)
	{
	char xpath[XPATH_MAX];
	xmlNode *match;
	int n;

	CRM_CHECK(local_cib != NULL, return FALSE);

	/* Search for the node's attributes in the CIB. While the schema allows
	* multiple sets of instance attributes, and allows instance attributes to
	* use id-ref to reference values elsewhere, that is intended for resources,
	* so we ignore that here.
	*/
	n = snprintf(xpath, XPATH_MAX, "//" XML_CIB_TAG_NODES
	"/" XML_CIB_TAG_NODE "[@uname='%s']/" XML_TAG_ATTR_SETS
	"/" XML_CIB_TAG_NVPAIR "[@name='%s' and @value='%s']",
	node, name, value);
	match = get_xpath_object(xpath, local_cib, LOG_TRACE);

	CRM_CHECK(n < XPATH_MAX, return FALSE);
	return (match != NULL);
	}

	static void
	update_fencing_topology(const char event, xmlNode msg)
	{
	int format = 1;
	const char *xpath;
	xmlXPathObjectPtr xpathObj = NULL;
	xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);

	CRM_ASSERT(patchset);
	crm_element_value_int(patchset, "format", &format);

	if(format == 1) {
	/* Process deletions (only) */
	xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_TAG_FENCING_LEVEL;
	xpathObj = xpath_search(msg, xpath);

	remove_fencing_topology(xpathObj);
	freeXpathObject(xpathObj);

	/* Process additions and changes */
	xpath = "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_TAG_FENCING_LEVEL;
	xpathObj = xpath_search(msg, xpath);

	register_fencing_topology(xpathObj);
	freeXpathObject(xpathObj);

	} else if(format == 2) {
	xmlNode *change = NULL;
	int add[] = { 0, 0, 0 };
	int del[] = { 0, 0, 0 };

	xml_patch_versions(patchset, add, del);

	for (change = __xml_first_child(patchset); change != NULL; change = __xml_next(change)) {
	const char *op = crm_element_value(change, XML_DIFF_OP);
	const char *xpath = crm_element_value(change, XML_DIFF_PATH);

	if(op == NULL) {
	continue;

	} else if(strstr(xpath, "/" XML_TAG_FENCING_LEVEL) != NULL) {
	/* Change to a specific entry */

	crm_trace("Handling %s operation %d.%d.%d for %s", op, add[0], add[1], add[2], xpath);
	if(strcmp(op, "move") == 0) {
	continue;

	} else if(strcmp(op, "create") == 0) {
	handle_topology_change(change->children, FALSE);

	} else if(strcmp(op, "modify") == 0) {
	xmlNode *match = first_named_child(change, XML_DIFF_RESULT);

	if(match) {
	handle_topology_change(match->children, TRUE);
	}

	} else if(strcmp(op, "delete") == 0) {
	/* Nuclear option, all we have is the path and an id... not enough to remove a specific entry */
	crm_info("Re-initializing fencing topology after %s operation %d.%d.%d for %s",
	op, add[0], add[1], add[2], xpath);
	fencing_topology_init();
	return;
	}

	} else if (strstr(xpath, "/" XML_TAG_FENCING_TOPOLOGY) != NULL) {
	/* Change to the topology in general */
	crm_info("Re-initializing fencing topology after top-level %s operation %d.%d.%d for %s",
	op, add[0], add[1], add[2], xpath);
	fencing_topology_init();
	return;

	} else if (strstr(xpath, "/" XML_CIB_TAG_CONFIGURATION)) {
	/* Changes to the whole config section, possibly including the topology as a whild */
	if(first_named_child(change, XML_TAG_FENCING_TOPOLOGY) == NULL) {
	crm_trace("Nothing for us in %s operation %d.%d.%d for %s.",
	op, add[0], add[1], add[2], xpath);

	} else if(strcmp(op, "delete") == 0 \|\| strcmp(op, "create") == 0) {
	crm_info("Re-initializing fencing topology after top-level %s operation %d.%d.%d for %s.",
	op, add[0], add[1], add[2], xpath);
	fencing_topology_init();
	return;
	}

	} else {
	crm_trace("Nothing for us in %s operation %d.%d.%d for %s",
	op, add[0], add[1], add[2], xpath);
	}
	}

	} else {
	crm_warn("Unknown patch format: %d", format);
	}
	}
	static bool have_cib_devices = FALSE;

	static void
	update_cib_cache_cb(const char event, xmlNode msg)
	{
	int rc = pcmk_ok;
	xmlNode *stonith_enabled_xml = NULL;
	xmlNode *stonith_watchdog_xml = NULL;
	const char *stonith_enabled_s = NULL;
	static gboolean stonith_enabled_saved = TRUE;

	if(!have_cib_devices) {
	crm_trace("Skipping updates until we get a full dump");
	return;

	} else if(msg == NULL) {
	crm_trace("Missing %s update", event);
	return;
	}

	/* Maintain a local copy of the CIB so that we have full access
	* to device definitions, location constraints, and node attributes
	*/
	if (local_cib != NULL) {
	int rc = pcmk_ok;
	xmlNode *patchset = NULL;

	crm_element_value_int(msg, F_CIB_RC, &rc);
	if (rc != pcmk_ok) {
	return;
	}

	patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT);
	xml_log_patchset(LOG_TRACE, "Config update", patchset);
	rc = xml_apply_patchset(local_cib, patchset, TRUE);
	switch (rc) {
	case pcmk_ok:
	case -pcmk_err_old_data:
	break;
	case -pcmk_err_diff_resync:
	case -pcmk_err_diff_failed:
	crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
	free_xml(local_cib);
	local_cib = NULL;
	break;
	default:
	crm_warn("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
	free_xml(local_cib);
	local_cib = NULL;
	}
	}

	if (local_cib == NULL) {
	crm_trace("Re-requesting the full cib");
	rc = cib_api->cmds->query(cib_api, NULL, &local_cib, cib_scope_local \| cib_sync_call);
	if(rc != pcmk_ok) {
	- crm_err("Couldnt retrieve the CIB: %s (%d)", pcmk_strerror(rc), rc);
	+ crm_err("Couldn't retrieve the CIB: %s (%d)", pcmk_strerror(rc), rc);
	return;
	}
	CRM_ASSERT(local_cib != NULL);
	stonith_enabled_saved = FALSE; /* Trigger a full refresh below */
	}

	stonith_enabled_xml = get_xpath_object("//nvpair[@name='stonith-enabled']", local_cib, LOG_TRACE);
	if (stonith_enabled_xml) {
	stonith_enabled_s = crm_element_value(stonith_enabled_xml, XML_NVPAIR_ATTR_VALUE);
	}

	if(daemon_option_enabled(crm_system_name, "watchdog")) {
	const char *value = NULL;
	long timeout_ms = 0;

	if(value == NULL) {
	stonith_watchdog_xml = get_xpath_object("//nvpair[@name='stonith-watchdog-timeout']", local_cib, LOG_TRACE);
	if (stonith_watchdog_xml) {
	value = crm_element_value(stonith_watchdog_xml, XML_NVPAIR_ATTR_VALUE);
	}
	}

	if(value) {
	timeout_ms = crm_get_msec(value);
	}

	if(timeout_ms != stonith_watchdog_timeout_ms) {
	crm_notice("New watchdog timeout %lds (was %lds)", timeout_ms/1000, stonith_watchdog_timeout_ms/1000);
	stonith_watchdog_timeout_ms = timeout_ms;
	}
	}

	if (stonith_enabled_s && crm_is_true(stonith_enabled_s) == FALSE) {
	crm_trace("Ignoring cib updates while stonith is disabled");
	stonith_enabled_saved = FALSE;
	return;

	} else if (stonith_enabled_saved == FALSE) {
	crm_info("Updating stonith device and topology lists now that stonith is enabled");
	stonith_enabled_saved = TRUE;
	fencing_topology_init();
	cib_devices_update();

	} else {
	update_fencing_topology(event, msg);
	update_cib_stonith_devices(event, msg);
	}
	}

	static void
	init_cib_cache_cb(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
	{
	crm_info("Updating device list from the cib: init");
	have_cib_devices = TRUE;
	local_cib = copy_xml(output);

	fencing_topology_init();
	cib_devices_update();
	}

	static void
	stonith_shutdown(int nsig)
	{
	stonith_shutdown_flag = TRUE;
	crm_info("Terminating with %d clients", crm_hash_table_size(client_connections));
	if (mainloop != NULL && g_main_is_running(mainloop)) {
	g_main_quit(mainloop);
	} else {
	stonith_cleanup();
	crm_exit(pcmk_ok);
	}
	}

	static void
	cib_connection_destroy(gpointer user_data)
	{
	if (stonith_shutdown_flag) {
	crm_info("Connection to the CIB closed.");
	return;
	} else {
	crm_notice("Connection to the CIB terminated. Shutting down.");
	}
	if (cib_api) {
	cib_api->cmds->signoff(cib_api);
	}
	stonith_shutdown(0);
	}

	static void
	stonith_cleanup(void)
	{
	if (cib_api) {
	cib_api->cmds->signoff(cib_api);
	}

	if (ipcs) {
	qb_ipcs_destroy(ipcs);
	}
	crm_peer_destroy();
	crm_client_cleanup();
	free(stonith_our_uname);
	free_xml(local_cib);
	}

	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	{"stand-alone", 0, 0, 's'},
	{"stand-alone-w-cpg", 0, 0, 'c'},
	{"logfile", 1, 0, 'l'},
	{"verbose", 0, 0, 'V'},
	{"version", 0, 0, '$'},
	{"help", 0, 0, '?'},

	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	static void
	setup_cib(void)
	{
	int rc, retries = 0;
	static cib_t (cib_new_fn) (void) = NULL;

	if (cib_new_fn == NULL) {
	cib_new_fn = find_library_function(&cib_library, CIB_LIBRARY, "cib_new", TRUE);
	}

	if (cib_new_fn != NULL) {
	cib_api = (*cib_new_fn) ();
	}

	if (cib_api == NULL) {
	crm_err("No connection to the CIB");
	return;
	}

	do {
	sleep(retries);
	rc = cib_api->cmds->signon(cib_api, CRM_SYSTEM_CRMD, cib_command);
	} while (rc == -ENOTCONN && ++retries < 5);

	if (rc != pcmk_ok) {
	crm_err("Could not connect to the CIB service: %s (%d)", pcmk_strerror(rc), rc);

	} else if (pcmk_ok !=
	cib_api->cmds->add_notify_callback(cib_api, T_CIB_DIFF_NOTIFY, update_cib_cache_cb)) {
	crm_err("Could not set CIB notification callback");

	} else {
	rc = cib_api->cmds->query(cib_api, NULL, NULL, cib_scope_local);
	cib_api->cmds->register_callback(cib_api, rc, 120, FALSE, NULL, "init_cib_cache_cb",
	init_cib_cache_cb);
	cib_api->cmds->set_connection_dnotify(cib_api, cib_connection_destroy);
	crm_notice("Watching for stonith topology changes");
	}
	}

	struct qb_ipcs_service_handlers ipc_callbacks = {
	.connection_accept = st_ipc_accept,
	.connection_created = st_ipc_created,
	.msg_process = st_ipc_dispatch,
	.connection_closed = st_ipc_closed,
	.connection_destroyed = st_ipc_destroy
	};

	/*!
	* \internal
	* \brief Callback for peer status changes
	*
	* \param[in] type What changed
	* \param[in] node What peer had the change
	* \param[in] data Previous value of what changed
	*/
	static void
	st_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
	{
	if ((type != crm_status_processes) && !is_set(node->flags, crm_remote_node)) {
	/*
	* This is a hack until we can send to a nodeid and/or we fix node name lookups
	* These messages are ignored in stonith_peer_callback()
	*/
	xmlNode *query = create_xml_node(NULL, "stonith_command");

	crm_xml_add(query, F_XML_TAGNAME, "stonith_command");
	crm_xml_add(query, F_TYPE, T_STONITH_NG);
	crm_xml_add(query, F_STONITH_OPERATION, "poke");

	crm_debug("Broadcasting our uname because of node %u", node->id);
	send_cluster_message(NULL, crm_msg_stonith_ng, query, FALSE);

	free_xml(query);
	}
	}

	int
	main(int argc, char **argv)
	{
	int flag;
	int rc = 0;
	int lpc = 0;
	int argerr = 0;
	int option_index = 0;
	crm_cluster_t cluster;
	const char *actions[] = { "reboot", "off", "list", "monitor", "status" };

	crm_log_preinit("stonith-ng", argc, argv);
	crm_set_options(NULL, "mode [options]", long_options,
	"Provides a summary of cluster's current state."
	"\n\nOutputs varying levels of detail in a number of different formats.\n");

	while (1) {
	flag = crm_get_option(argc, argv, &option_index);
	if (flag == -1) {
	break;
	}

	switch (flag) {
	case 'V':
	crm_bump_log_level(argc, argv);
	break;
	case 'l':
	crm_add_logfile(optarg);
	break;
	case 's':
	stand_alone = TRUE;
	break;
	case 'c':
	stand_alone = FALSE;
	no_cib_connect = TRUE;
	break;
	case '$':
	case '?':
	crm_help(flag, EX_OK);
	break;
	default:
	++argerr;
	break;
	}
	}

	if (argc - optind == 1 && safe_str_eq("metadata", argv[optind])) {
	printf("<?xml version=\"1.0\"?><!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n");
	printf("<resource-agent name=\"stonithd\">\n");
	printf(" <version>1.0</version>\n");
	printf
	(" <longdesc lang=\"en\">This is a fake resource that details the instance attributes handled by stonithd.</longdesc>\n");
	printf(" <shortdesc lang=\"en\">Options available for all stonith resources</shortdesc>\n");
	printf(" <parameters>\n");

	printf(" <parameter name=\"priority\" unique=\"0\">\n");
	printf
	(" <shortdesc lang=\"en\">The priority of the stonith resource. Devices are tried in order of highest priority to lowest.</shortdesc>\n");
	printf(" <content type=\"integer\" default=\"0\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTARG);
	printf
	(" <shortdesc lang=\"en\">Advanced use only: An alternate parameter to supply instead of 'port'</shortdesc>\n");
	printf
	(" <longdesc lang=\"en\">Some devices do not support the standard 'port' parameter or may provide additional ones.\n"
	"Use this to specify an alternate, device-specific, parameter that should indicate the machine to be fenced.\n"
	"A value of 'none' can be used to tell the cluster not to supply any additional parameters.\n"
	" </longdesc>\n");
	printf(" <content type=\"string\" default=\"port\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTMAP);
	printf
	(" <shortdesc lang=\"en\">A mapping of host names to ports numbers for devices that do not support host names.</shortdesc>\n");
	printf
	(" <longdesc lang=\"en\">Eg. node1:1;node2:2,3 would tell the cluster to use port 1 for node1 and ports 2 and 3 for node2</longdesc>\n");
	printf(" <content type=\"string\" default=\"\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTLIST);
	printf
	(" <shortdesc lang=\"en\">A list of machines controlled by this device (Optional unless %s=static-list).</shortdesc>\n",
	STONITH_ATTR_HOSTCHECK);
	printf(" <content type=\"string\" default=\"\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_HOSTCHECK);
	printf
	(" <shortdesc lang=\"en\">How to determine which machines are controlled by the device.</shortdesc>\n");
	printf
	(" <longdesc lang=\"en\">Allowed values: dynamic-list (query the device), static-list (check the %s attribute), none (assume every device can fence every machine)</longdesc>\n",
	STONITH_ATTR_HOSTLIST);
	printf(" <content type=\"string\" default=\"dynamic-list\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_DELAY_MAX);
	printf
	(" <shortdesc lang=\"en\">Enable random delay for stonith actions and specify the maximum of random delay</shortdesc>\n");
	printf
	(" <longdesc lang=\"en\">This prevents double fencing when using slow devices such as sbd.\n"
	"Use this to enable random delay for stonith actions and specify the maximum of random delay.</longdesc>\n");
	printf(" <content type=\"time\" default=\"0s\"/>\n");
	printf(" </parameter>\n");

	for (lpc = 0; lpc < DIMOF(actions); lpc++) {
	printf(" <parameter name=\"pcmk_%s_action\" unique=\"0\">\n", actions[lpc]);
	printf
	(" <shortdesc lang=\"en\">Advanced use only: An alternate command to run instead of '%s'</shortdesc>\n",
	actions[lpc]);
	printf
	(" <longdesc lang=\"en\">Some devices do not support the standard commands or may provide additional ones.\n"
	"Use this to specify an alternate, device-specific, command that implements the '%s' action.</longdesc>\n",
	actions[lpc]);
	printf(" <content type=\"string\" default=\"%s\"/>\n", actions[lpc]);
	printf(" </parameter>\n");

	printf(" <parameter name=\"pcmk_%s_timeout\" unique=\"0\">\n", actions[lpc]);
	printf
	(" <shortdesc lang=\"en\">Advanced use only: Specify an alternate timeout to use for %s actions instead of stonith-timeout</shortdesc>\n",
	actions[lpc]);
	printf
	(" <longdesc lang=\"en\">Some devices need much more/less time to complete than normal.\n"
	"Use this to specify an alternate, device-specific, timeout for '%s' actions.</longdesc>\n",
	actions[lpc]);
	printf(" <content type=\"time\" default=\"60s\"/>\n");
	printf(" </parameter>\n");

	printf(" <parameter name=\"pcmk_%s_retries\" unique=\"0\">\n", actions[lpc]);
	printf
	(" <shortdesc lang=\"en\">Advanced use only: The maximum number of times to retry the '%s' command within the timeout period</shortdesc>\n",
	actions[lpc]);
	printf(" <longdesc lang=\"en\">Some devices do not support multiple connections."
	" Operations may 'fail' if the device is busy with another task so Pacemaker will automatically retry the operation, if there is time remaining."
	" Use this option to alter the number of times Pacemaker retries '%s' actions before giving up."
	"</longdesc>\n", actions[lpc]);
	printf(" <content type=\"integer\" default=\"2\"/>\n");
	printf(" </parameter>\n");
	}

	printf(" </parameters>\n");
	printf("</resource-agent>\n");
	return 0;
	}

	if (optind != argc) {
	++argerr;
	}

	if (argerr) {
	crm_help('?', EX_USAGE);
	}

	crm_log_init("stonith-ng", LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
	mainloop_add_signal(SIGTERM, stonith_shutdown);

	crm_peer_init();

	if (stand_alone == FALSE) {
	#if SUPPORT_HEARTBEAT
	cluster.hb_conn = NULL;
	cluster.hb_dispatch = stonith_peer_hb_callback;
	cluster.destroy = stonith_peer_hb_destroy;
	#endif

	if (is_openais_cluster()) {
	#if SUPPORT_COROSYNC
	cluster.destroy = stonith_peer_cs_destroy;
	cluster.cpg.cpg_deliver_fn = stonith_peer_ais_callback;
	cluster.cpg.cpg_confchg_fn = pcmk_cpg_membership;
	#endif
	}

	if (crm_cluster_connect(&cluster) == FALSE) {
	crm_crit("Cannot sign in to the cluster... terminating");
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	stonith_our_uname = cluster.uname;
	stonith_our_uuid = cluster.uuid;

	#if SUPPORT_HEARTBEAT
	if (is_heartbeat_cluster()) {
	/* crm_cluster_connect() registered us for crm_system_name, which
	* usually is the only F_TYPE used by the respective sub system.
	* Stonith needs to register two additional F_TYPE callbacks,
	* because it can :-/ */
	if (HA_OK !=
	cluster.hb_conn->llc_ops->set_msg_callback(cluster.hb_conn, T_STONITH_NOTIFY,
	cluster.hb_dispatch, cluster.hb_conn)) {
	crm_crit("Cannot set msg callback %s: %s", T_STONITH_NOTIFY, cluster.hb_conn->llc_ops->errmsg(cluster.hb_conn));
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	if (HA_OK !=
	cluster.hb_conn->llc_ops->set_msg_callback(cluster.hb_conn, T_STONITH_TIMEOUT_VALUE,
	cluster.hb_dispatch, cluster.hb_conn)) {
	crm_crit("Cannot set msg callback %s: %s", T_STONITH_TIMEOUT_VALUE, cluster.hb_conn->llc_ops->errmsg(cluster.hb_conn));
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	}
	#endif

	if (no_cib_connect == FALSE) {
	setup_cib();
	}

	} else {
	stonith_our_uname = strdup("localhost");
	}

	crm_set_status_callback(&st_peer_update_callback);

	device_list = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_device);

	topology = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_topology_entry);

	if(daemon_option_enabled(crm_system_name, "watchdog")) {
	xmlNode *xml;
	stonith_key_value_t *params = NULL;

	params = stonith_key_value_add(params, STONITH_ATTR_HOSTLIST, stonith_our_uname);

	xml = create_device_registration_xml("watchdog", "internal", STONITH_WATCHDOG_AGENT, params, NULL);
	stonith_device_register(xml, NULL, FALSE);

	stonith_key_value_freeall(params, 1, 1);
	free_xml(xml);
	}

	stonith_ipc_server_init(&ipcs, &ipc_callbacks);

	#if SUPPORT_STONITH_CONFIG
	if (((stand_alone == TRUE)) && !(standalone_cfg_read_file(STONITH_NG_CONF_FILE))) {
	standalone_cfg_commit();
	}
	#endif

	/* Create the mainloop and run it... */
	mainloop = g_main_new(FALSE);
	crm_info("Starting %s mainloop", crm_system_name);

	g_main_run(mainloop);
	stonith_cleanup();

	#if SUPPORT_HEARTBEAT
	if (cluster.hb_conn) {
	cluster.hb_conn->llc_ops->delete(cluster.hb_conn);
	}
	#endif

	crm_info("Done");

	return crm_exit(rc);
	}
	diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c
	index 4dc65aad21..2817e5a56a 100644
	--- a/lib/cib/cib_utils.c
	+++ b/lib/cib/cib_utils.c
	@@ -1,834 +1,834 @@
	/*
	* Copyright (c) 2004 International Business Machines
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*
	*/
	#include <crm_internal.h>
	#include <unistd.h>
	#include <stdlib.h>
	#include <stdio.h>
	#include <stdarg.h>
	#include <string.h>
	#include <sys/utsname.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/cib/internal.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/pengine/rules.h>

	struct config_root_s {
	const char *name;
	const char *parent;
	const char *path;
	};

	/*
	* "//crm_config" will also work in place of "/cib/configuration/crm_config"
	* The / prefix means find starting from the root, whereas the // prefix means
	* find anywhere and risks multiple matches
	*/
	/* INDENT-OFF */
	struct config_root_s known_paths[] = {
	{ NULL, NULL, "//cib" },
	{ XML_TAG_CIB, NULL, "//cib" },
	{ XML_CIB_TAG_STATUS, "/cib", "//cib/status" },
	{ XML_CIB_TAG_CONFIGURATION,"/cib", "//cib/configuration" },
	{ XML_CIB_TAG_CRMCONFIG, "/cib/configuration", "//cib/configuration/crm_config" },
	{ XML_CIB_TAG_NODES, "/cib/configuration", "//cib/configuration/nodes" },
	{ XML_CIB_TAG_DOMAINS, "/cib/configuration", "//cib/configuration/domains" },
	{ XML_CIB_TAG_RESOURCES, "/cib/configuration", "//cib/configuration/resources" },
	{ XML_CIB_TAG_CONSTRAINTS, "/cib/configuration", "//cib/configuration/constraints" },
	{ XML_CIB_TAG_OPCONFIG, "/cib/configuration", "//cib/configuration/op_defaults" },
	{ XML_CIB_TAG_RSCCONFIG, "/cib/configuration", "//cib/configuration/rsc_defaults" },
	{ XML_CIB_TAG_ACLS, "/cib/configuration", "//cib/configuration/acls" },
	{ XML_TAG_FENCING_TOPOLOGY, "/cib/configuration", "//cib/configuration/fencing-topology" },
	{ XML_CIB_TAG_SECTION_ALL, NULL, "//cib" },
	};
	/* INDENT-ON */

	int
	cib_compare_generation(xmlNode * left, xmlNode * right)
	{
	int lpc = 0;

	const char *attributes[] = {
	XML_ATTR_GENERATION_ADMIN,
	XML_ATTR_GENERATION,
	XML_ATTR_NUMUPDATES,
	};

	crm_log_xml_trace(left, "left");
	crm_log_xml_trace(right, "right");

	for (lpc = 0; lpc < DIMOF(attributes); lpc++) {
	int int_elem_l = -1;
	int int_elem_r = -1;
	const char *elem_r = NULL;
	const char *elem_l = crm_element_value(left, attributes[lpc]);

	if (right != NULL) {
	elem_r = crm_element_value(right, attributes[lpc]);
	}

	if (elem_l != NULL) {
	int_elem_l = crm_parse_int(elem_l, NULL);
	}
	if (elem_r != NULL) {
	int_elem_r = crm_parse_int(elem_r, NULL);
	}

	if (int_elem_l < int_elem_r) {
	crm_trace("%s (%s < %s)", attributes[lpc], crm_str(elem_l), crm_str(elem_r));
	return -1;

	} else if (int_elem_l > int_elem_r) {
	crm_trace("%s (%s > %s)", attributes[lpc], crm_str(elem_l), crm_str(elem_r));
	return 1;
	}
	}

	return 0;
	}

	/* Deprecated - doesn't expose -EACCES */
	xmlNode *
	get_cib_copy(cib_t * cib)
	{
	xmlNode *xml_cib;
	int options = cib_scope_local \| cib_sync_call;
	int rc = pcmk_ok;

	if (cib->state == cib_disconnected) {
	return NULL;
	}

	rc = cib->cmds->query(cib, NULL, &xml_cib, options);
	if (rc == -EACCES) {
	return NULL;

	} else if (rc != pcmk_ok) {
	- crm_err("Couldnt retrieve the CIB");
	+ crm_err("Couldn't retrieve the CIB");
	free_xml(xml_cib);
	return NULL;

	} else if (xml_cib == NULL) {
	crm_err("The CIB result was empty");
	free_xml(xml_cib);
	return NULL;
	}

	if (safe_str_eq(crm_element_name(xml_cib), XML_TAG_CIB)) {
	return xml_cib;
	}
	free_xml(xml_cib);
	return NULL;
	}

	xmlNode *
	cib_get_generation(cib_t * cib)
	{
	xmlNode *the_cib = NULL;
	xmlNode *generation = create_xml_node(NULL, XML_CIB_TAG_GENERATION_TUPPLE);

	cib->cmds->query(cib, NULL, &the_cib, cib_scope_local \| cib_sync_call);
	if (the_cib != NULL) {
	copy_in_properties(generation, the_cib);
	free_xml(the_cib);
	}

	return generation;
	}

	gboolean
	cib_version_details(xmlNode * cib, int admin_epoch, int epoch, int *updates)
	{
	*epoch = -1;
	*updates = -1;
	*admin_epoch = -1;

	if (cib == NULL) {
	return FALSE;

	} else {
	crm_element_value_int(cib, XML_ATTR_GENERATION, epoch);
	crm_element_value_int(cib, XML_ATTR_NUMUPDATES, updates);
	crm_element_value_int(cib, XML_ATTR_GENERATION_ADMIN, admin_epoch);
	}
	return TRUE;
	}

	gboolean
	cib_diff_version_details(xmlNode * diff, int admin_epoch, int epoch, int *updates,
	int _admin_epoch, int _epoch, int *_updates)
	{
	int add[] = { 0, 0, 0 };
	int del[] = { 0, 0, 0 };

	xml_patch_versions(diff, add, del);

	*admin_epoch = add[0];
	*epoch = add[1];
	*updates = add[2];

	*_admin_epoch = del[0];
	*_epoch = del[1];
	*_updates = del[2];

	return TRUE;
	}

	/*
	* The caller should never free the return value
	*/

	const char *
	get_object_path(const char *object_type)
	{
	int lpc = 0;
	int max = DIMOF(known_paths);

	for (; lpc < max; lpc++) {
	if ((object_type == NULL && known_paths[lpc].name == NULL)
	\|\| safe_str_eq(object_type, known_paths[lpc].name)) {
	return known_paths[lpc].path;
	}
	}
	return NULL;
	}

	const char *
	get_object_parent(const char *object_type)
	{
	int lpc = 0;
	int max = DIMOF(known_paths);

	for (; lpc < max; lpc++) {
	if (safe_str_eq(object_type, known_paths[lpc].name)) {
	return known_paths[lpc].parent;
	}
	}
	return NULL;
	}

	xmlNode *
	get_object_root(const char object_type, xmlNode the_root)
	{
	const char *xpath = get_object_path(object_type);

	if (xpath == NULL) {
	return the_root; /* or return NULL? */
	}

	return get_xpath_object(xpath, the_root, LOG_DEBUG_4);
	}

	/*
	* It is the callers responsibility to free both the new CIB (output)
	* and the new CIB (input)
	*/
	xmlNode *
	createEmptyCib(int admin_epoch)
	{
	xmlNode cib_root = NULL, config = NULL;

	cib_root = create_xml_node(NULL, XML_TAG_CIB);
	crm_xml_add(cib_root, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
	crm_xml_add(cib_root, XML_ATTR_VALIDATION, xml_latest_schema());

	crm_xml_add_int(cib_root, XML_ATTR_GENERATION, admin_epoch);
	crm_xml_add_int(cib_root, XML_ATTR_NUMUPDATES, 0);
	crm_xml_add_int(cib_root, XML_ATTR_GENERATION_ADMIN, 0);

	config = create_xml_node(cib_root, XML_CIB_TAG_CONFIGURATION);
	create_xml_node(cib_root, XML_CIB_TAG_STATUS);

	create_xml_node(config, XML_CIB_TAG_CRMCONFIG);
	create_xml_node(config, XML_CIB_TAG_NODES);
	create_xml_node(config, XML_CIB_TAG_RESOURCES);
	create_xml_node(config, XML_CIB_TAG_CONSTRAINTS);

	return cib_root;
	}

	static bool
	cib_acl_enabled(xmlNode xml, const char user)
	{
	bool rc = FALSE;

	#if ENABLE_ACL
	if(pcmk_acl_required(user)) {
	const char *value = NULL;
	GHashTable *options = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	cib_read_config(options, xml);
	value = cib_pref(options, "enable-acl");
	rc = crm_is_true(value);
	g_hash_table_destroy(options);
	}

	crm_debug("CIB ACL is %s", rc ? "enabled" : "disabled");
	#endif
	return rc;
	}

	int
	cib_perform_op(const char op, int call_options, cib_op_t fn, gboolean is_query,
	const char section, xmlNode req, xmlNode * input,
	gboolean manage_counters, gboolean * config_changed,
	xmlNode * current_cib, xmlNode result_cib, xmlNode diff, xmlNode ** output)
	{
	int rc = pcmk_ok;
	gboolean check_dtd = TRUE;
	xmlNode *top = NULL;
	xmlNode *scratch = NULL;
	xmlNode *local_diff = NULL;

	const char *new_version = NULL;
	static struct qb_log_callsite *diff_cs = NULL;
	const char *user = crm_element_value(req, F_CIB_USER);
	bool with_digest = FALSE;

	crm_trace("Begin %s%s op", is_query ? "read-only " : "", op);

	CRM_CHECK(output != NULL, return -ENOMSG);
	CRM_CHECK(result_cib != NULL, return -ENOMSG);
	CRM_CHECK(config_changed != NULL, return -ENOMSG);

	if(output) {
	*output = NULL;
	}

	*result_cib = NULL;
	*config_changed = FALSE;

	if (fn == NULL) {
	return -EINVAL;
	}

	if (is_query) {
	xmlNode *cib_ro = current_cib;
	xmlNode *cib_filtered = NULL;

	if(cib_acl_enabled(cib_ro, user)) {
	if(xml_acl_filtered_copy(user, current_cib, current_cib, &cib_filtered)) {
	if (cib_filtered == NULL) {
	crm_debug("Pre-filtered the entire cib");
	return -EACCES;
	}
	cib_ro = cib_filtered;
	crm_log_xml_trace(cib_ro, "filtered");
	}
	}

	rc = (*fn) (op, call_options, section, req, input, cib_ro, result_cib, output);

	if(output == NULL \|\| *output == NULL) {
	/* nothing */

	} else if(cib_filtered == *output) {
	cib_filtered = NULL; /* Let them have this copy */

	} else if(*output == current_cib) {
	/* They already know not to free it */

	} else if(cib_filtered && (*output)->doc == cib_filtered->doc) {
	/* We're about to free the document of which output is a part /
	output = copy_xml(output);

	} else if((*output)->doc == current_cib->doc) {
	/* Give them a copy they can free */
	output = copy_xml(output);
	}

	free_xml(cib_filtered);
	return rc;
	}


	if (is_set(call_options, cib_zero_copy)) {
	/* Conditional on v2 patch style */

	scratch = current_cib;

	/* Create a shallow copy of current_cib for the version details */
	current_cib = create_xml_node(NULL, (const char *)scratch->name);
	copy_in_properties(current_cib, scratch);
	top = current_cib;

	xml_track_changes(scratch, user, NULL, cib_acl_enabled(scratch, user));
	rc = (*fn) (op, call_options, section, req, input, scratch, &scratch, output);

	} else {
	scratch = copy_xml(current_cib);
	xml_track_changes(scratch, user, NULL, cib_acl_enabled(scratch, user));
	rc = (*fn) (op, call_options, section, req, input, current_cib, &scratch, output);

	if(scratch && xml_tracking_changes(scratch) == FALSE) {
	crm_trace("Inferring changes after %s op", op);
	xml_track_changes(scratch, user, current_cib, cib_acl_enabled(current_cib, user));
	xml_calculate_changes(current_cib, scratch);
	}
	CRM_CHECK(current_cib != scratch, return -EINVAL);
	}

	xml_acl_disable(scratch); /* Allow the system to make any additional changes */

	if (rc == pcmk_ok && scratch == NULL) {
	rc = -EINVAL;
	goto done;

	} else if(rc == pcmk_ok && xml_acl_denied(scratch)) {
	crm_trace("ACL rejected part or all of the proposed changes");
	rc = -EACCES;
	goto done;

	} else if (rc != pcmk_ok) {
	goto done;
	}

	if (scratch) {
	new_version = crm_element_value(scratch, XML_ATTR_CRM_VERSION);

	if (new_version && compare_version(new_version, CRM_FEATURE_SET) > 0) {
	crm_err("Discarding update with feature set '%s' greater than our own '%s'",
	new_version, CRM_FEATURE_SET);
	rc = -EPROTONOSUPPORT;
	goto done;
	}
	}

	if (current_cib) {
	int old = 0;
	int new = 0;

	crm_element_value_int(scratch, XML_ATTR_GENERATION_ADMIN, &new);
	crm_element_value_int(current_cib, XML_ATTR_GENERATION_ADMIN, &old);

	if (old > new) {
	crm_err("%s went backwards: %d -> %d (Opts: 0x%x)",
	XML_ATTR_GENERATION_ADMIN, old, new, call_options);
	crm_log_xml_warn(req, "Bad Op");
	crm_log_xml_warn(input, "Bad Data");
	rc = -pcmk_err_old_data;

	} else if (old == new) {
	crm_element_value_int(scratch, XML_ATTR_GENERATION, &new);
	crm_element_value_int(current_cib, XML_ATTR_GENERATION, &old);
	if (old > new) {
	crm_err("%s went backwards: %d -> %d (Opts: 0x%x)",
	XML_ATTR_GENERATION, old, new, call_options);
	crm_log_xml_warn(req, "Bad Op");
	crm_log_xml_warn(input, "Bad Data");
	rc = -pcmk_err_old_data;
	}
	}
	}

	crm_trace("Massaging CIB contents");
	strip_text_nodes(scratch);
	fix_plus_plus_recursive(scratch);

	if (is_set(call_options, cib_zero_copy)) {
	/* At this point, current_cib is just the 'cib' tag and its properties,
	*
	* The v1 format would barf on this, but we know the v2 patch
	* format only needs it for the top-level version fields
	*/
	local_diff = xml_create_patchset(2, current_cib, scratch, (bool*)config_changed, manage_counters);

	} else {
	static time_t expires = 0;
	time_t tm_now = time(NULL);

	if (expires < tm_now) {
	expires = tm_now + 60; /* Validate clients are correctly applying v2-style diffs at most once a minute */
	with_digest = TRUE;
	}

	local_diff = xml_create_patchset(0, current_cib, scratch, (bool*)config_changed, manage_counters);
	}

	xml_log_changes(LOG_TRACE, __FUNCTION__, scratch);
	xml_accept_changes(scratch);

	if (diff_cs == NULL) {
	diff_cs = qb_log_callsite_get(__PRETTY_FUNCTION__, __FILE__, "diff-validation", LOG_DEBUG, __LINE__, crm_trace_nonlog);
	}

	if(local_diff) {
	patchset_process_digest(local_diff, current_cib, scratch, with_digest);

	xml_log_patchset(LOG_INFO, __FUNCTION__, local_diff);
	crm_log_xml_trace(local_diff, "raw patch");
	}

	if (is_not_set(call_options, cib_zero_copy) /* The original to compare against doesn't exist */
	&& local_diff
	&& crm_is_callsite_active(diff_cs, LOG_TRACE, 0)) {

	/* Validate the calculated patch set */
	int test_rc, format = 1;
	xmlNode * c = copy_xml(current_cib);

	crm_element_value_int(local_diff, "format", &format);
	test_rc = xml_apply_patchset(c, local_diff, manage_counters);

	if(test_rc != pcmk_ok) {
	save_xml_to_file(c, "PatchApply:calculated", NULL);
	save_xml_to_file(current_cib, "PatchApply:input", NULL);
	save_xml_to_file(scratch, "PatchApply:actual", NULL);
	save_xml_to_file(local_diff, "PatchApply:diff", NULL);
	crm_err("v%d patchset error, patch failed to apply: %s (%d)", format, pcmk_strerror(test_rc), test_rc);
	}
	free_xml(c);
	}

	if (safe_str_eq(section, XML_CIB_TAG_STATUS)) {
	/* Throttle the amount of costly validation we perform due to status updates
	* a) we don't really care whats in the status section
	* b) we don't validate any of it's contents at the moment anyway
	*/
	check_dtd = FALSE;
	}

	/* === scratch must not be modified after this point ===
	* Exceptions, anything in:

	static filter_t filter[] = {
	{ 0, XML_ATTR_ORIGIN },
	{ 0, XML_CIB_ATTR_WRITTEN },
	{ 0, XML_ATTR_UPDATE_ORIG },
	{ 0, XML_ATTR_UPDATE_CLIENT },
	{ 0, XML_ATTR_UPDATE_USER },
	};
	*/

	if (*config_changed && is_not_set(call_options, cib_no_mtime)) {
	char *now_str = NULL;
	time_t now = time(NULL);
	const char *schema = crm_element_value(scratch, XML_ATTR_VALIDATION);

	now_str = ctime(&now);
	now_str[24] = EOS; /* replace the newline */
	crm_xml_replace(scratch, XML_CIB_ATTR_WRITTEN, now_str);

	if (schema) {
	static int minimum_schema = 0;
	int current_schema = get_schema_version(schema);

	if (minimum_schema == 0) {
	minimum_schema = get_schema_version("pacemaker-1.2");
	}

	/* Does the CIB support the "update-" attributes... /
	if (current_schema >= minimum_schema) {
	const char *origin = crm_element_value(req, F_ORIG);

	CRM_LOG_ASSERT(origin != NULL);
	crm_xml_replace(scratch, XML_ATTR_UPDATE_ORIG, origin);
	crm_xml_replace(scratch, XML_ATTR_UPDATE_CLIENT,
	crm_element_value(req, F_CIB_CLIENTNAME));
	#if ENABLE_ACL
	crm_xml_replace(scratch, XML_ATTR_UPDATE_USER, crm_element_value(req, F_CIB_USER));
	#endif
	}
	}
	}

	crm_trace("Perform validation: %s", check_dtd ? "true" : "false");
	if (rc == pcmk_ok && check_dtd && validate_xml(scratch, NULL, TRUE) == FALSE) {
	const char *current_dtd = crm_element_value(scratch, XML_ATTR_VALIDATION);

	crm_warn("Updated CIB does not validate against %s schema/dtd", crm_str(current_dtd));
	rc = -pcmk_err_schema_validation;
	}

	done:

	*result_cib = scratch;
	#if ENABLE_ACL
	if(rc != pcmk_ok && cib_acl_enabled(current_cib, user)) {
	if(xml_acl_filtered_copy(user, current_cib, scratch, result_cib)) {
	if (*result_cib == NULL) {
	crm_debug("Pre-filtered the entire cib result");
	}
	free_xml(scratch);
	}
	}
	#endif

	if(diff) {
	*diff = local_diff;
	} else {
	free_xml(local_diff);
	}

	free_xml(top);
	crm_trace("Done");
	return rc;
	}

	xmlNode *
	cib_create_op(int call_id, const char token, const char op, const char host, const char section,
	xmlNode * data, int call_options, const char *user_name)
	{
	xmlNode *op_msg = create_xml_node(NULL, "cib_command");

	CRM_CHECK(op_msg != NULL, return NULL);
	CRM_CHECK(token != NULL, return NULL);

	crm_xml_add(op_msg, F_XML_TAGNAME, "cib_command");

	crm_xml_add(op_msg, F_TYPE, T_CIB);
	crm_xml_add(op_msg, F_CIB_CALLBACK_TOKEN, token);
	crm_xml_add(op_msg, F_CIB_OPERATION, op);
	crm_xml_add(op_msg, F_CIB_HOST, host);
	crm_xml_add(op_msg, F_CIB_SECTION, section);
	crm_xml_add_int(op_msg, F_CIB_CALLID, call_id);
	#if ENABLE_ACL
	if (user_name) {
	crm_xml_add(op_msg, F_CIB_USER, user_name);
	}
	#endif
	crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options);
	crm_xml_add_int(op_msg, F_CIB_CALLOPTS, call_options);

	if (data != NULL) {
	add_message_xml(op_msg, F_CIB_CALLDATA, data);
	}

	if (call_options & cib_inhibit_bcast) {
	CRM_CHECK((call_options & cib_scope_local), return NULL);
	}
	return op_msg;
	}

	void
	cib_native_callback(cib_t * cib, xmlNode * msg, int call_id, int rc)
	{
	xmlNode *output = NULL;
	cib_callback_client_t *blob = NULL;

	if (msg != NULL) {
	crm_element_value_int(msg, F_CIB_RC, &rc);
	crm_element_value_int(msg, F_CIB_CALLID, &call_id);
	output = get_message_xml(msg, F_CIB_CALLDATA);
	}

	blob = g_hash_table_lookup(cib_op_callback_table, GINT_TO_POINTER(call_id));
	if (blob == NULL) {
	crm_trace("No callback found for call %d", call_id);
	}

	if (cib == NULL) {
	crm_debug("No cib object supplied");
	}

	if (rc == -pcmk_err_diff_resync) {
	/* This is an internal value that clients do not and should not care about */
	rc = pcmk_ok;
	}

	if (blob && blob->callback && (rc == pcmk_ok \|\| blob->only_success == FALSE)) {
	crm_trace("Invoking callback %s for call %d", crm_str(blob->id), call_id);
	blob->callback(msg, call_id, rc, output, blob->user_data);

	} else if (cib && cib->op_callback == NULL && rc != pcmk_ok) {
	crm_warn("CIB command failed: %s", pcmk_strerror(rc));
	crm_log_xml_debug(msg, "Failed CIB Update");
	}

	/* This may free user_data, so do it after the callback */
	if (blob) {
	remove_cib_op_callback(call_id, FALSE);
	}

	if (cib && cib->op_callback != NULL) {
	crm_trace("Invoking global callback for call %d", call_id);
	cib->op_callback(msg, call_id, rc, output);
	}
	crm_trace("OP callback activated for %d", call_id);
	}

	void
	cib_native_notify(gpointer data, gpointer user_data)
	{
	xmlNode *msg = user_data;
	cib_notify_client_t *entry = data;
	const char *event = NULL;

	if (msg == NULL) {
	crm_warn("Skipping callback - NULL message");
	return;
	}

	event = crm_element_value(msg, F_SUBTYPE);

	if (entry == NULL) {
	crm_warn("Skipping callback - NULL callback client");
	return;

	} else if (entry->callback == NULL) {
	crm_warn("Skipping callback - NULL callback");
	return;

	} else if (safe_str_neq(entry->event, event)) {
	crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event);
	return;
	}

	crm_trace("Invoking callback for %p/%s event...", entry, event);
	entry->callback(event, msg);
	crm_trace("Callback invoked...");
	}

	pe_cluster_option cib_opts[] = {
	/* name, old-name, validate, default, description */
	{"enable-acl", NULL, "boolean", NULL, "false", &check_boolean,
	"Enable CIB ACL", NULL}
	,
	};

	void
	cib_metadata(void)
	{
	config_metadata("Cluster Information Base", "1.0",
	"Cluster Information Base Options",
	"This is a fake resource that details the options that can be configured for the Cluster Information Base.",
	cib_opts, DIMOF(cib_opts));
	}

	void
	verify_cib_options(GHashTable * options)
	{
	verify_all_options(options, cib_opts, DIMOF(cib_opts));
	}

	const char *
	cib_pref(GHashTable * options, const char *name)
	{
	return get_cluster_pref(options, cib_opts, DIMOF(cib_opts), name);
	}

	gboolean
	cib_read_config(GHashTable * options, xmlNode * current_cib)
	{
	xmlNode *config = NULL;
	crm_time_t *now = NULL;

	if (options == NULL \|\| current_cib == NULL) {
	return FALSE;
	}

	now = crm_time_new(NULL);

	g_hash_table_remove_all(options);

	config = get_object_root(XML_CIB_TAG_CRMCONFIG, current_cib);
	if (config) {
	unpack_instance_attributes(current_cib, config, XML_CIB_TAG_PROPSET, NULL, options,
	CIB_OPTIONS_FIRST, FALSE, now);
	}

	verify_cib_options(options);

	crm_time_free(now);

	return TRUE;
	}

	int
	cib_apply_patch_event(xmlNode * event, xmlNode * input, xmlNode ** output, int level)
	{
	int rc = pcmk_err_generic;

	xmlNode *diff = NULL;

	CRM_ASSERT(event);
	CRM_ASSERT(input);
	CRM_ASSERT(output);

	crm_element_value_int(event, F_CIB_RC, &rc);
	diff = get_message_xml(event, F_CIB_UPDATE_RESULT);

	if (rc < pcmk_ok \|\| diff == NULL) {
	return rc;
	}

	if (level > LOG_CRIT) {
	xml_log_patchset(level, "Config update", diff);
	}

	if (input != NULL) {
	rc = cib_process_diff(NULL, cib_none, NULL, event, diff, input, output, NULL);

	if (rc != pcmk_ok) {
	crm_debug("Update didn't apply: %s (%d) %p", pcmk_strerror(rc), rc, *output);

	if (rc == -pcmk_err_old_data) {
	crm_trace("Masking error, we already have the supplied update");
	return pcmk_ok;
	}
	free_xml(output); output = NULL;

	return rc;
	}
	}

	return rc;
	}

	gboolean
	cib_internal_config_changed(xmlNode * diff)
	{
	gboolean changed = FALSE;
	xmlXPathObject *xpathObj = NULL;

	if (diff == NULL) {
	return FALSE;
	}

	xpathObj = xpath_search(diff, "//" XML_CIB_TAG_CRMCONFIG);
	if (numXpathResults(xpathObj) > 0) {
	changed = TRUE;
	}

	freeXpathObject(xpathObj);

	return changed;
	}

	int
	cib_internal_op(cib_t * cib, const char op, const char host,
	const char section, xmlNode data,
	xmlNode ** output_data, int call_options, const char *user_name)
	{
	int (delegate) (cib_t cib, const char op, const char host,
	const char section, xmlNode data,
	xmlNode ** output_data, int call_options, const char *user_name) =
	cib->delegate_fn;

	#if ENABLE_ACL
	if(user_name == NULL) {
	user_name = getenv("CIB_user");
	}
	#endif

	return delegate(cib, op, host, section, data, output_data, call_options, user_name);
	}
	diff --git a/lib/cluster/election.c b/lib/cluster/election.c
	index f164dd693b..c2c5f516a3 100644
	--- a/lib/cluster/election.c
	+++ b/lib/cluster/election.c
	@@ -1,527 +1,527 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/
	#include <crm_internal.h>

	#include <sys/time.h>
	#include <sys/resource.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <crm/common/mainloop.h>
	#include <crm/cluster/internal.h>
	#include <crm/cluster/election.h>
	#include <crm/crm.h>

	#define STORM_INTERVAL 2 /* in seconds */
	#define STORM_MULTIPLIER 5 /* multiplied by the number of nodes */

	struct election_s
	{
	enum election_result state;
	guint count;
	char *name;
	char *uname;
	GSourceFunc cb;
	GHashTable *voted;
	mainloop_timer_t timeout; / When to stop if not everyone casts a vote */
	};

	static void election_complete(election_t *e)
	{
	crm_info("Election %s complete", e->name);
	e->state = election_won;

	if(e->cb) {
	e->cb(e);
	}

	election_reset(e);
	}

	static gboolean election_timer_cb(gpointer user_data)
	{
	election_t *e = user_data;

	crm_info("Election %s %p timed out", e->name, e);
	election_complete(e);
	return FALSE;
	}

	enum election_result
	election_state(election_t *e)
	{
	if(e) {
	return e->state;
	}
	return election_error;
	}

	election_t *
	election_init(const char name, const char uname, guint period_ms, GSourceFunc cb)
	{
	static guint count = 0;
	election_t *e = calloc(1, sizeof(election_t));

	if(e != NULL) {
	if(name) {
	e->name = crm_strdup_printf("election-%s", name);
	} else {
	e->name = crm_strdup_printf("election-%u", count++);
	}

	e->cb = cb;
	e->uname = strdup(uname);
	e->timeout = mainloop_timer_add(e->name, period_ms, FALSE, election_timer_cb, e);
	crm_trace("Created %s %p", e->name, e);
	}
	return e;
	}

	void
	election_remove(election_t e, const char uname)
	{
	if(e && uname && e->voted) {
	g_hash_table_remove(e->voted, uname);
	}
	}

	void
	election_reset(election_t *e)
	{
	crm_trace("Resetting election %s", e->name);
	if(e) {
	mainloop_timer_stop(e->timeout);
	}
	if (e && e->voted) {
	crm_trace("Destroying voted cache with %d members", g_hash_table_size(e->voted));
	g_hash_table_destroy(e->voted);
	e->voted = NULL;
	}
	}

	void
	election_fini(election_t *e)
	{
	if(e) {
	election_reset(e);
	crm_trace("Destroying %s", e->name);
	mainloop_timer_del(e->timeout);
	free(e->uname);
	free(e->name);
	free(e);
	}
	}

	static void
	election_timeout_start(election_t *e)
	{
	if(e) {
	mainloop_timer_start(e->timeout);
	}
	}

	void
	election_timeout_stop(election_t *e)
	{
	if(e) {
	mainloop_timer_stop(e->timeout);
	}
	}

	void
	election_timeout_set_period(election_t *e, guint period)
	{
	if(e) {
	mainloop_timer_set_period(e->timeout, period);
	} else {
	crm_err("No election defined");
	}
	}

	static int
	crm_uptime(struct timeval *output)
	{
	static time_t expires = 0;
	static struct rusage info;

	time_t tm_now = time(NULL);

	if (expires < tm_now) {
	int rc = 0;

	info.ru_utime.tv_sec = 0;
	info.ru_utime.tv_usec = 0;
	rc = getrusage(RUSAGE_SELF, &info);

	output->tv_sec = 0;
	output->tv_usec = 0;

	if (rc < 0) {
	crm_perror(LOG_ERR, "Could not calculate the current uptime");
	expires = 0;
	return -1;
	}

	crm_debug("Current CPU usage is: %lds, %ldus", (long)info.ru_utime.tv_sec,
	(long)info.ru_utime.tv_usec);
	}

	expires = tm_now + STORM_INTERVAL; /* N seconds after the last _access_ */
	output->tv_sec = info.ru_utime.tv_sec;
	output->tv_usec = info.ru_utime.tv_usec;

	return 1;
	}

	static int
	crm_compare_age(struct timeval your_age)
	{
	struct timeval our_age;

	crm_uptime(&our_age); /* If an error occurred, our_age will be compared as {0,0} */

	if (our_age.tv_sec > your_age.tv_sec) {
	crm_debug("Win: %ld vs %ld (seconds)", (long)our_age.tv_sec, (long)your_age.tv_sec);
	return 1;
	} else if (our_age.tv_sec < your_age.tv_sec) {
	crm_debug("Loose: %ld vs %ld (seconds)", (long)our_age.tv_sec, (long)your_age.tv_sec);
	return -1;
	} else if (our_age.tv_usec > your_age.tv_usec) {
	crm_debug("Win: %ld.%ld vs %ld.%ld (usec)",
	(long)our_age.tv_sec, (long)our_age.tv_usec, (long)your_age.tv_sec, (long)your_age.tv_usec);
	return 1;
	} else if (our_age.tv_usec < your_age.tv_usec) {
	crm_debug("Loose: %ld.%ld vs %ld.%ld (usec)",
	(long)our_age.tv_sec, (long)our_age.tv_usec, (long)your_age.tv_sec, (long)your_age.tv_usec);
	return -1;
	}

	return 0;
	}

	void
	election_vote(election_t *e)
	{
	struct timeval age;
	xmlNode *vote = NULL;
	crm_node_t *our_node;

	if(e == NULL) {
	crm_trace("Not voting in election: not initialized");
	return;
	}

	our_node = crm_get_peer(0, e->uname);
	if (our_node == NULL \|\| crm_is_peer_active(our_node) == FALSE) {
	crm_trace("Cannot vote yet: %p", our_node);
	return;
	}

	e->state = election_in_progress;
	vote = create_request(CRM_OP_VOTE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);

	e->count++;
	crm_xml_add(vote, F_CRM_ELECTION_OWNER, our_node->uuid);
	crm_xml_add_int(vote, F_CRM_ELECTION_ID, e->count);

	crm_uptime(&age);
	crm_xml_add_int(vote, F_CRM_ELECTION_AGE_S, age.tv_sec);
	crm_xml_add_int(vote, F_CRM_ELECTION_AGE_US, age.tv_usec);

	send_cluster_message(NULL, crm_msg_crmd, vote, TRUE);
	free_xml(vote);

	crm_debug("Started election %d", e->count);
	if (e->voted) {
	g_hash_table_destroy(e->voted);
	e->voted = NULL;
	}

	election_timeout_start(e);
	return;
	}

	bool
	election_check(election_t *e)
	{
	int voted_size = 0;
	int num_members = crm_active_peers();

	if(e == NULL) {
	crm_trace("not initialized");
	return FALSE;
	}

	if (e->voted) {
	voted_size = g_hash_table_size(e->voted);
	}
	/* in the case of #voted > #members, it is better to
	* wait for the timeout and give the cluster time to
	* stabilize
	*/
	if (voted_size >= num_members) {
	/* we won and everyone has voted */
	election_timeout_stop(e);
	if (voted_size > num_members) {
	GHashTableIter gIter;
	const crm_node_t *node;
	char *key = NULL;

	g_hash_table_iter_init(&gIter, crm_peer_cache);
	while (g_hash_table_iter_next(&gIter, NULL, (gpointer *) & node)) {
	if (crm_is_peer_active(node)) {
	crm_err("member: %s proc=%.32x", node->uname, node->processes);
	}
	}

	g_hash_table_iter_init(&gIter, e->voted);
	while (g_hash_table_iter_next(&gIter, (gpointer *) & key, NULL)) {
	crm_err("voted: %s", key);
	}

	}

	election_complete(e);
	return TRUE;

	} else {
	crm_debug("Still waiting on %d non-votes (%d total)",
	num_members - voted_size, num_members);
	}

	return FALSE;
	}

	#define loss_dampen 2 /* in seconds */

	/* A_ELECTION_COUNT */
	enum election_result
	election_count_vote(election_t e, xmlNode vote, bool can_win)
	{
	int age = 0;
	int election_id = -1;
	int log_level = LOG_INFO;
	gboolean use_born_on = FALSE;
	gboolean done = FALSE;
	gboolean we_loose = FALSE;
	const char *op = NULL;
	const char *from = NULL;
	const char *reason = "unknown";
	const char *election_owner = NULL;
	crm_node_t our_node = NULL, your_node = NULL;

	static int election_wins = 0;

	xmlNode *novote = NULL;
	time_t tm_now = time(NULL);
	static time_t expires = 0;
	static time_t last_election_loss = 0;

	- /* if the membership copy is NULL we REALLY shouldnt be voting
	+ /* if the membership copy is NULL we REALLY shouldn't be voting
	* the question is how we managed to get here.
	*/

	CRM_CHECK(vote != NULL, return election_error);

	if(e == NULL) {
	crm_info("Not voting in election: not initialized");
	return election_lost;

	} else if(crm_peer_cache == NULL) {
	crm_info("Not voting in election: no peer cache");
	return election_lost;
	}

	op = crm_element_value(vote, F_CRM_TASK);
	from = crm_element_value(vote, F_CRM_HOST_FROM);
	election_owner = crm_element_value(vote, F_CRM_ELECTION_OWNER);
	crm_element_value_int(vote, F_CRM_ELECTION_ID, &election_id);

	your_node = crm_get_peer(0, from);
	our_node = crm_get_peer(0, e->uname);

	if (e->voted == NULL) {
	crm_debug("Created voted hash");
	e->voted = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);
	}

	if (is_heartbeat_cluster()) {
	use_born_on = TRUE;
	} else if (is_classic_ais_cluster()) {
	use_born_on = TRUE;
	}

	if(can_win == FALSE) {
	reason = "Not eligible";
	we_loose = TRUE;

	} else if (our_node == NULL \|\| crm_is_peer_active(our_node) == FALSE) {
	reason = "We are not part of the cluster";
	log_level = LOG_ERR;
	we_loose = TRUE;

	} else if (election_id != e->count && crm_str_eq(our_node->uuid, election_owner, TRUE)) {
	log_level = LOG_TRACE;
	reason = "Superseded";
	done = TRUE;

	} else if (your_node == NULL \|\| crm_is_peer_active(your_node) == FALSE) {
	/* Possibly we cached the message in the FSA queue at a point that it wasn't */
	reason = "Peer is not part of our cluster";
	log_level = LOG_WARNING;
	done = TRUE;

	} else if (crm_str_eq(op, CRM_OP_NOVOTE, TRUE)) {
	char *op_copy = strdup(op);
	char *uname_copy = strdup(from);

	CRM_ASSERT(crm_str_eq(our_node->uuid, election_owner, TRUE));

	/* update the list of nodes that have voted */
	g_hash_table_replace(e->voted, uname_copy, op_copy);
	reason = "Recorded";
	done = TRUE;

	} else {
	struct timeval your_age;
	const char *your_version = crm_element_value(vote, F_CRM_VERSION);
	int tv_sec = 0;
	int tv_usec = 0;

	crm_element_value_int(vote, F_CRM_ELECTION_AGE_S, &tv_sec);
	crm_element_value_int(vote, F_CRM_ELECTION_AGE_US, &tv_usec);

	your_age.tv_sec = tv_sec;
	your_age.tv_usec = tv_usec;

	age = crm_compare_age(your_age);
	if (crm_str_eq(from, e->uname, TRUE)) {
	char *op_copy = strdup(op);
	char *uname_copy = strdup(from);

	CRM_ASSERT(crm_str_eq(our_node->uuid, election_owner, TRUE));

	/* update ourselves in the list of nodes that have voted */
	g_hash_table_replace(e->voted, uname_copy, op_copy);
	reason = "Recorded";
	done = TRUE;

	} else if (compare_version(your_version, CRM_FEATURE_SET) < 0) {
	reason = "Version";
	we_loose = TRUE;

	} else if (compare_version(your_version, CRM_FEATURE_SET) > 0) {
	reason = "Version";

	} else if (age < 0) {
	reason = "Uptime";
	we_loose = TRUE;

	} else if (age > 0) {
	reason = "Uptime";

	/* TODO: Check for y(our) born < 0 */
	} else if (use_born_on && your_node->born < our_node->born) {
	reason = "Born";
	we_loose = TRUE;

	} else if (use_born_on && your_node->born > our_node->born) {
	reason = "Born";

	} else if (e->uname == NULL) {
	reason = "Unknown host name";
	we_loose = TRUE;

	} else if (strcasecmp(e->uname, from) > 0) {
	reason = "Host name";
	we_loose = TRUE;

	} else {
	reason = "Host name";
	CRM_ASSERT(strcasecmp(e->uname, from) < 0);
	-/* cant happen...
	+/* can't happen...
	* } else if(strcasecmp(e->uname, from) == 0) {
	*
	*/
	}
	}

	if (expires < tm_now) {
	election_wins = 0;
	expires = tm_now + STORM_INTERVAL;

	} else if (done == FALSE && we_loose == FALSE) {
	int peers = 1 + g_hash_table_size(crm_peer_cache);

	/* If every node has to vote down every other node, thats N*(N-1) total elections
	* Allow some leway before _really_ complaining
	*/
	election_wins++;
	if (election_wins > (peers * peers)) {
	crm_warn("Election storm detected: %d elections in %d seconds", election_wins,
	STORM_INTERVAL);
	election_wins = 0;
	expires = tm_now + STORM_INTERVAL;
	crm_write_blackbox(0, NULL);
	}
	}

	if (done) {
	do_crm_log(log_level + 1, "Election %d (current: %d, owner: %s): Processed %s from %s (%s)",
	election_id, e->count, election_owner, op, from, reason);
	return e->state;

	} else if(we_loose == FALSE) {
	do_crm_log(log_level, "Election %d (owner: %s) pass: %s from %s (%s)",
	election_id, election_owner, op, from, reason);

	if (last_election_loss == 0
	\|\| tm_now - last_election_loss > (time_t) loss_dampen) {

	last_election_loss = 0;
	election_timeout_stop(e);

	/* Start a new election by voting down this, and other, peers */
	e->state = election_start;
	return e->state;
	}

	crm_info("Election %d ignore: We already lost an election less than %ds ago (%s)",
	election_id, loss_dampen, ctime(&last_election_loss));
	}

	novote = create_request(CRM_OP_NOVOTE, NULL, from,
	CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);

	do_crm_log(log_level, "Election %d (owner: %s) lost: %s from %s (%s)",
	election_id, election_owner, op, from, reason);

	election_timeout_stop(e);

	crm_xml_add(novote, F_CRM_ELECTION_OWNER, election_owner);
	crm_xml_add_int(novote, F_CRM_ELECTION_ID, election_id);

	send_cluster_message(your_node, crm_msg_crmd, novote, TRUE);
	free_xml(novote);

	last_election_loss = tm_now;
	e->state = election_lost;
	return e->state;
	}
	diff --git a/lib/cluster/heartbeat.c b/lib/cluster/heartbeat.c
	index e55eeecd1b..47702b1c3c 100644
	--- a/lib/cluster/heartbeat.c
	+++ b/lib/cluster/heartbeat.c
	@@ -1,660 +1,660 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>
	#include <dlfcn.h>

	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>
	#include <string.h>
	#include <stdlib.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>

	#include <crm/common/ipc.h>
	#include <crm/cluster/internal.h>

	#if HAVE_BZLIB_H
	# include <bzlib.h>
	#endif

	#if SUPPORT_HEARTBEAT
	ll_cluster_t *heartbeat_cluster = NULL;

	static void
	convert_ha_field(xmlNode * parent, void *msg_v, int lpc)
	{
	int type = 0;
	const char *name = NULL;
	const char *value = NULL;
	xmlNode *xml = NULL;
	HA_Message *msg = msg_v;

	int rc = BZ_OK;
	size_t orig_len = 0;
	unsigned int used = 0;
	char *uncompressed = NULL;
	char *compressed = NULL;
	int size = orig_len * 10;

	CRM_CHECK(parent != NULL, return);
	CRM_CHECK(msg != NULL, return);

	name = msg->names[lpc];
	type = cl_get_type(msg, name);

	switch (type) {
	case FT_STRUCT:
	convert_ha_message(parent, msg->values[lpc], name);
	break;
	case FT_COMPRESS:
	case FT_UNCOMPRESS:
	convert_ha_message(parent, cl_get_struct(msg, name), name);
	break;
	case FT_STRING:
	value = msg->values[lpc];
	CRM_CHECK(value != NULL, return);
	crm_trace("Converting %s/%d/%s", name, type, value[0] == '<' ? "xml" : "field");

	if (value[0] != '<') {
	crm_xml_add(parent, name, value);
	break;
	}

	/* unpack xml string */
	xml = string2xml(value);
	if (xml == NULL) {
	crm_err("Conversion of field '%s' failed", name);
	return;
	}

	add_node_nocopy(parent, NULL, xml);
	break;

	case FT_BINARY:
	value = cl_get_binary(msg, name, &orig_len);
	size = orig_len * 10 + 1; /* +1 because an exact 10x compression factor happens occasionally */

	if (orig_len < 3 \|\| value[0] != 'B' \|\| value[1] != 'Z' \|\| value[2] != 'h') {
	if (strstr(name, "uuid") == NULL) {
	crm_err("Skipping non-bzip binary field: %s", name);
	}
	return;
	}

	compressed = calloc(1, orig_len);
	memcpy(compressed, value, orig_len);

	crm_trace("Trying to decompress %d bytes", (int)orig_len);
	retry:
	uncompressed = realloc_safe(uncompressed, size);
	memset(uncompressed, 0, size);
	used = size - 1; /* always leave room for a trailing '\0'
	- * BZ2_bzBuffToBuffDecompress wont say anything if
	+ * BZ2_bzBuffToBuffDecompress won't say anything if
	* the uncompressed data is exactly 'size' bytes
	*/

	rc = BZ2_bzBuffToBuffDecompress(uncompressed, &used, compressed, orig_len, 1, 0);

	if (rc == BZ_OUTBUFF_FULL) {
	size = size * 2;
	/* dont try to allocate more memory than we have */
	if (size > 0) {
	goto retry;
	}
	}

	if (rc != BZ_OK) {
	crm_err("Decompression of %s (%d bytes) into %d failed: %d",
	name, (int)orig_len, size, rc);

	} else if (used >= size) {
	CRM_ASSERT(used < size);

	} else {
	CRM_LOG_ASSERT(uncompressed[used] == 0);
	uncompressed[used] = 0;
	xml = string2xml(uncompressed);
	}

	if (xml != NULL) {
	add_node_copy(parent, xml);
	free_xml(xml);
	}

	free(uncompressed);
	free(compressed);
	break;
	}
	}

	xmlNode *
	convert_ha_message(xmlNode * parent, HA_Message * msg, const char *field)
	{
	int lpc = 0;
	xmlNode *child = NULL;
	const char *tag = NULL;

	CRM_CHECK(msg != NULL, crm_err("Empty message for %s", field);
	return parent);

	tag = cl_get_string(msg, F_XML_TAGNAME);
	if (tag == NULL) {
	tag = field;

	} else if (parent && safe_str_neq(field, tag)) {
	/* For compatability with 0.6.x */
	crm_debug("Creating intermediate parent %s between %s and %s", field,
	crm_element_name(parent), tag);
	parent = create_xml_node(parent, field);
	}

	if (parent == NULL) {
	parent = create_xml_node(NULL, tag);
	child = parent;

	} else {
	child = create_xml_node(parent, tag);
	}

	for (lpc = 0; lpc < msg->nfields; lpc++) {
	convert_ha_field(child, msg, lpc);
	}

	return parent;
	}

	static void
	add_ha_nocopy(HA_Message * parent, HA_Message * child, const char *field)
	{
	int next = parent->nfields;

	if (parent->nfields >= parent->nalloc && ha_msg_expand(parent) != HA_OK) {
	crm_err("Parent expansion failed");
	return;
	}

	parent->names[next] = strdup(field);
	parent->nlens[next] = strlen(field);
	parent->values[next] = child;
	parent->vlens[next] = sizeof(HA_Message);
	parent->types[next] = FT_UNCOMPRESS;
	parent->nfields++;
	}

	static HA_Message *
	convert_xml_message_struct(HA_Message * parent, xmlNode * src_node, const char *field)
	{
	xmlNode *child = NULL;
	xmlNode *__crm_xml_iter = src_node->children;
	xmlAttrPtr prop_iter = src_node->properties;
	const char *name = NULL;
	const char *value = NULL;

	HA_Message *result = ha_msg_new(3);

	ha_msg_add(result, F_XML_TAGNAME, (const char *)src_node->name);

	while (prop_iter != NULL) {
	name = (const char *)prop_iter->name;
	value = (const char *)xmlGetProp(src_node, prop_iter->name);
	prop_iter = prop_iter->next;
	ha_msg_add(result, name, value);
	}

	while (__crm_xml_iter != NULL) {
	child = __crm_xml_iter;
	__crm_xml_iter = __crm_xml_iter->next;
	convert_xml_message_struct(result, child, NULL);
	}

	if (parent == NULL) {
	return result;
	}

	if (field) {
	HA_Message *holder = ha_msg_new(3);

	CRM_ASSERT(holder != NULL);

	ha_msg_add(holder, F_XML_TAGNAME, field);
	add_ha_nocopy(holder, result, (const char *)src_node->name);

	ha_msg_addstruct_compress(parent, field, holder);
	ha_msg_del(holder);

	} else {
	add_ha_nocopy(parent, result, (const char *)src_node->name);
	}
	return result;
	}

	static void
	convert_xml_child(HA_Message * msg, xmlNode * xml)
	{
	int orig = 0;
	int rc = BZ_OK;
	unsigned int len = 0;

	char *buffer = NULL;
	char *compressed = NULL;
	const char *name = NULL;

	name = (const char *)xml->name;
	buffer = dump_xml_unformatted(xml);
	orig = strlen(buffer);
	if (orig < CRM_BZ2_THRESHOLD) {
	ha_msg_add(msg, name, buffer);
	goto done;
	}

	len = (orig * 1.1) + 600; /* recomended size */

	compressed = malloc(len);
	rc = BZ2_bzBuffToBuffCompress(compressed, &len, buffer, orig, CRM_BZ2_BLOCKS, 0, CRM_BZ2_WORK);

	if (rc != BZ_OK) {
	crm_err("Compression failed: %d", rc);
	free(compressed);
	convert_xml_message_struct(msg, xml, name);
	goto done;
	}

	free(buffer);
	buffer = compressed;
	crm_trace("Compression details: %d -> %d", orig, len);
	ha_msg_addbin(msg, name, buffer, len);
	done:
	free(buffer);

	# if 0
	{
	unsigned int used = orig;
	char *uncompressed = NULL;

	crm_debug("Trying to decompress %d bytes", len);
	uncompressed = calloc(1, orig);
	rc = BZ2_bzBuffToBuffDecompress(uncompressed, &used, compressed, len, 1, 0);
	CRM_CHECK(rc == BZ_OK,;
	);
	CRM_CHECK(used == orig,;
	);
	crm_debug("rc=%d, used=%d", rc, used);
	if (rc != BZ_OK) {
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	crm_debug("Original %s, decompressed %s", buffer, uncompressed);
	free(uncompressed);
	}
	# endif
	}

	static HA_Message *
	convert_xml_message(xmlNode * xml)
	{
	xmlNode *child = NULL;
	xmlAttrPtr pIter = NULL;
	HA_Message *result = NULL;

	result = ha_msg_new(3);
	ha_msg_add(result, F_XML_TAGNAME, (const char *)xml->name);

	for (pIter = xml->properties; pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;

	if (pIter->children) {
	const char p_value = (const char )pIter->children->content;

	ha_msg_add(result, p_name, p_value);
	}
	}
	for (child = __xml_first_child(xml); child != NULL; child = __xml_next(child)) {
	convert_xml_child(result, child);
	}

	return result;
	}

	gboolean
	crm_is_heartbeat_peer_active(const crm_node_t * node)
	{
	enum crm_proc_flag proc = text2proc(crm_system_name);

	if (node == NULL) {
	crm_trace("NULL");
	return FALSE;

	} else if (safe_str_neq(node->state, CRM_NODE_MEMBER)) {
	crm_trace("%s: state=%s", node->uname, node->state);
	return FALSE;

	} else if ((node->processes & crm_proc_heartbeat) == 0) {
	crm_trace("%s: processes=%.16x", node->uname, node->processes);
	return FALSE;

	} else if (proc == crm_proc_none) {
	return TRUE;

	} else if ((node->processes & proc) == 0) {
	crm_trace("%s: proc %.16x not in %.16x", node->uname, proc, node->processes);
	return FALSE;
	}
	return TRUE;
	}

	crm_node_t *
	crm_update_ccm_node(const oc_ev_membership_t * oc, int offset, const char *state, uint64_t seq)
	{
	enum crm_proc_flag this_proc = text2proc(crm_system_name);
	crm_node_t *peer = NULL;
	const char *uuid = NULL;

	CRM_CHECK(oc->m_array[offset].node_uname != NULL, return NULL);

	peer = crm_get_peer(0, oc->m_array[offset].node_uname);
	uuid = crm_peer_uuid(peer);

	peer = crm_update_peer(__FUNCTION__, oc->m_array[offset].node_id,
	oc->m_array[offset].node_born_on, seq, -1, 0,
	uuid, oc->m_array[offset].node_uname, NULL, state);
	if (peer == NULL) {
	return NULL;
	}

	if (safe_str_eq(CRM_NODE_MEMBER, state)) {
	/* Heartbeat doesn't send status notifications for nodes that were already part of the cluster.
	* Nor does it send status notifications for processes that were already active.
	* Do not optimistically assume the peer client process to be online as well.
	* We ask for cluster wide updated client status for crm_system_name
	* directly in the ccm status callback, which will then tell us.
	* For ourselves, we know. */
	enum crm_proc_flag flags = crm_proc_heartbeat;
	const char *const_uname = heartbeat_cluster->llc_ops->get_mynodeid(heartbeat_cluster);
	if (safe_str_eq(const_uname, peer->uname)) {
	flags \|= this_proc;
	}
	peer = crm_update_peer_proc(__FUNCTION__, peer, flags, ONLINESTATUS);
	} else {
	/* crm_update_peer_proc(__FUNCTION__, peer, crm_proc_heartbeat, OFFLINESTATUS); */
	/* heartbeat may well be still alive. peer client process apparently vanished, though ... */
	peer = crm_update_peer_proc(__FUNCTION__, peer, this_proc, OFFLINESTATUS);
	}
	return peer;
	}

	gboolean
	send_ha_message(ll_cluster_t * hb_conn, xmlNode * xml, const char *node, gboolean force_ordered)
	{
	gboolean all_is_good = TRUE;
	HA_Message *msg = convert_xml_message(xml);

	if (msg == NULL) {
	- crm_err("cant send NULL message");
	+ crm_err("can't send NULL message");
	all_is_good = FALSE;

	} else if (hb_conn == NULL) {
	crm_err("No heartbeat connection specified");
	all_is_good = FALSE;

	} else if (hb_conn->llc_ops->chan_is_connected(hb_conn) == FALSE) {
	crm_err("Not connected to Heartbeat");
	all_is_good = FALSE;

	} else if (node != NULL) {
	char *host_lowercase = g_ascii_strdown(node, -1);

	if (hb_conn->llc_ops->send_ordered_nodemsg(hb_conn, msg, host_lowercase) != HA_OK) {
	all_is_good = FALSE;
	crm_err("Send failed");
	}
	free(host_lowercase);

	} else if (force_ordered) {
	if (hb_conn->llc_ops->send_ordered_clustermsg(hb_conn, msg) != HA_OK) {
	all_is_good = FALSE;
	crm_err("Broadcast Send failed");
	}

	} else {
	if (hb_conn->llc_ops->sendclustermsg(hb_conn, msg) != HA_OK) {
	all_is_good = FALSE;
	crm_err("Broadcast Send failed");
	}
	}

	if (all_is_good == FALSE && hb_conn != NULL) {
	IPC_Channel *ipc = NULL;
	IPC_Queue *send_q = NULL;

	if (hb_conn->llc_ops->chan_is_connected(hb_conn) != HA_OK) {
	ipc = hb_conn->llc_ops->ipcchan(hb_conn);
	}
	if (ipc != NULL) {
	/* ipc->ops->resume_io(ipc); */
	send_q = ipc->send_queue;
	}
	if (send_q != NULL) {
	CRM_CHECK(send_q->current_qlen < send_q->max_qlen,;
	);
	}
	}

	if (all_is_good) {
	crm_log_xml_trace(xml, "outbound");
	} else {
	crm_log_xml_warn(xml, "outbound");
	}

	if (msg != NULL) {
	ha_msg_del(msg);
	}
	return all_is_good;
	}

	gboolean
	ha_msg_dispatch(ll_cluster_t * cluster_conn, gpointer user_data)
	{
	IPC_Channel *channel = NULL;

	crm_trace("Invoked");

	if (cluster_conn != NULL) {
	channel = cluster_conn->llc_ops->ipcchan(cluster_conn);
	}

	CRM_CHECK(cluster_conn != NULL, return FALSE);
	CRM_CHECK(channel != NULL, return FALSE);

	if (channel != NULL && IPC_ISRCONN(channel)) {
	struct ha_msg *msg;
	if (cluster_conn->llc_ops->msgready(cluster_conn) == 0) {
	crm_trace("no message ready yet");
	}
	/* invoke the callbacks but dont block.
	* cluster_conn->llc_ops->rcvmsg(cluster_conn, 0); */
	msg = cluster_conn->llc_ops->readmsg(cluster_conn, 0);
	if (msg) {
	/* Message core refuses to pass on messages with F_TYPE not set.
	* Messages with no specific F_TOID are notifications delivered to all.
	*/
	const char *msg_type = ha_msg_value(msg, F_TYPE) ?: "[type not set]";
	const char *msg_to_id = ha_msg_value(msg, F_TOID);
	if (safe_str_eq(msg_to_id, crm_system_name)) {
	crm_err("Ignored incoming message. Please set_msg_callback on %s", msg_type);
	} else if (msg_to_id) {
	/* Message core will not deliver messages addressed to someone else to us.
	* Are we not registered as crm_system_name? */
	crm_notice("Ignored incoming message %s=%s %s=%s, please set_msg_callback",
	F_TOID, msg_to_id, F_TYPE, msg_type);
	} else {
	crm_debug("Ignored incoming message %s=%s", F_TYPE, msg_type);
	}
	ha_msg_del(msg);
	}
	}

	if (channel == NULL \|\| channel->ch_status != IPC_CONNECT) {
	crm_info("Lost connection to heartbeat service.");
	return FALSE;
	}

	return TRUE;
	}

	gboolean
	register_heartbeat_conn(crm_cluster_t * cluster)
	{
	crm_node_t *peer = NULL;
	const char *const_uuid = NULL;
	const char *const_uname = NULL;

	crm_debug("Signing in with Heartbeat");
	if (cluster->hb_conn->llc_ops->signon(cluster->hb_conn, crm_system_name) != HA_OK) {
	crm_err("Cannot sign on with heartbeat: %s",
	cluster->hb_conn->llc_ops->errmsg(cluster->hb_conn));
	return FALSE;
	}

	if (HA_OK !=
	cluster->hb_conn->llc_ops->set_msg_callback(cluster->hb_conn, crm_system_name,
	cluster->hb_dispatch, cluster->hb_conn)) {

	crm_err("Cannot set msg callback: %s", cluster->hb_conn->llc_ops->errmsg(cluster->hb_conn));
	return FALSE;

	} else {
	void *handle = NULL;
	GLLclusterSource (g_main_add_cluster) (int priority, ll_cluster_t * api,
	gboolean can_recurse,
	gboolean(dispatch) (ll_cluster_t source_data,
	gpointer user_data),
	gpointer userdata, GDestroyNotify notify) =
	find_library_function(&handle, HEARTBEAT_LIBRARY, "G_main_add_ll_cluster", 1);

	(*g_main_add_cluster) (G_PRIORITY_HIGH, cluster->hb_conn,
	FALSE, ha_msg_dispatch, cluster->hb_conn, cluster->destroy);
	dlclose(handle);
	}

	const_uname = cluster->hb_conn->llc_ops->get_mynodeid(cluster->hb_conn);
	CRM_CHECK(const_uname != NULL, return FALSE);

	peer = crm_get_peer(0, const_uname);
	const_uuid = crm_peer_uuid(peer);

	CRM_CHECK(const_uuid != NULL, return FALSE);

	crm_info("Hostname: %s", const_uname);
	crm_info("UUID: %s", const_uuid);

	cluster->uname = strdup(const_uname);
	cluster->uuid = strdup(const_uuid);

	return TRUE;
	}

	gboolean
	ccm_have_quorum(oc_ed_t event)
	{
	if (event == OC_EV_MS_NEW_MEMBERSHIP \|\| event == OC_EV_MS_PRIMARY_RESTORED) {
	return TRUE;
	}
	return FALSE;
	}

	const char *
	ccm_event_name(oc_ed_t event)
	{

	if (event == OC_EV_MS_NEW_MEMBERSHIP) {
	return "NEW MEMBERSHIP";

	} else if (event == OC_EV_MS_NOT_PRIMARY) {
	return "NOT PRIMARY";

	} else if (event == OC_EV_MS_PRIMARY_RESTORED) {
	return "PRIMARY RESTORED";

	} else if (event == OC_EV_MS_EVICTED) {
	return "EVICTED";

	} else if (event == OC_EV_MS_INVALID) {
	return "INVALID";
	}

	return "NO QUORUM MEMBERSHIP";

	}

	gboolean
	heartbeat_initialize_nodelist(void cluster, gboolean force_member, xmlNode xml_parent)
	{
	const char *ha_node = NULL;
	ll_cluster_t *conn = cluster;

	if (conn == NULL) {
	crm_debug("Not connected");
	return FALSE;
	}

	/* Async get client status information in the cluster */
	crm_info("Requesting the list of configured nodes");
	conn->llc_ops->init_nodewalk(conn);

	do {
	xmlNode *node = NULL;
	crm_node_t *peer = NULL;
	const char *ha_node_type = NULL;
	const char *ha_node_uuid = NULL;

	ha_node = conn->llc_ops->nextnode(conn);
	if (ha_node == NULL) {
	continue;
	}

	ha_node_type = conn->llc_ops->node_type(conn, ha_node);
	if (safe_str_neq(NORMALNODE, ha_node_type)) {
	crm_debug("Node %s: skipping '%s'", ha_node, ha_node_type);
	continue;
	}

	peer = crm_get_peer(0, ha_node);
	ha_node_uuid = crm_peer_uuid(peer);

	if (ha_node_uuid == NULL) {
	crm_warn("Node %s: no uuid found", ha_node);
	continue;
	}

	crm_debug("Node: %s (uuid: %s)", ha_node, ha_node_uuid);
	node = create_xml_node(xml_parent, XML_CIB_TAG_NODE);
	crm_xml_add(node, XML_ATTR_ID, ha_node_uuid);
	crm_xml_add(node, XML_ATTR_UNAME, ha_node);
	crm_xml_add(node, XML_ATTR_TYPE, ha_node_type);

	} while (ha_node != NULL);

	conn->llc_ops->end_nodewalk(conn);
	return TRUE;
	}

	#endif
	diff --git a/lib/common/digest.c b/lib/common/digest.c
	index f5a7cc486e..64dc1fa782 100644
	--- a/lib/common/digest.c
	+++ b/lib/common/digest.c
	@@ -1,242 +1,242 @@
	/*
	* Copyright (C) 2015 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <string.h>
	#include <stdlib.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#define BEST_EFFORT_STATUS 0

	/*!
	* \brief Dump XML in a format used with v1 digests
	*
	* \param[in] an_xml_node Root of XML to dump
	*
	* \return Newly allocated buffer containing dumped XML
	*/
	static char *
	dump_xml_for_digest(xmlNode * an_xml_node)
	{
	char *buffer = NULL;
	int offset = 0, max = 0;

	/* for compatability with the old result which is used for v1 digests */
	crm_buffer_add_char(&buffer, &offset, &max, ' ');
	crm_xml_dump(an_xml_node, 0, &buffer, &offset, &max, 0);
	crm_buffer_add_char(&buffer, &offset, &max, '\n');

	return buffer;
	}

	/*!
	* \brief Calculate and return v1 digest of XML tree
	*
	* \param[in] input Root of XML to digest
	* \param[in] sort Whether to sort the XML before calculating digest
	* \param[in] ignored Not used
	*
	* \return Newly allocated string containing digest
	* \note Example return value: "c048eae664dba840e1d2060f00299e9d"
	*/
	static char *
	calculate_xml_digest_v1(xmlNode * input, gboolean sort, gboolean ignored)
	{
	char *digest = NULL;
	char *buffer = NULL;
	xmlNode *copy = NULL;

	if (sort) {
	crm_trace("Sorting xml...");
	copy = sorted_xml(input, NULL, TRUE);
	crm_trace("Done");
	input = copy;
	}

	buffer = dump_xml_for_digest(input);
	CRM_CHECK(buffer != NULL && strlen(buffer) > 0, free_xml(copy);
	free(buffer);
	return NULL);

	digest = crm_md5sum(buffer);
	crm_log_xml_trace(input, "digest:source");

	free(buffer);
	free_xml(copy);
	return digest;
	}

	/*!
	* \brief Calculate and return v2 digest of XML tree
	*
	* \param[in] source Root of XML to digest
	* \param[in] do_filter Whether to filter certain XML attributes
	*
	* \return Newly allocated string containing digest
	*/
	static char *
	calculate_xml_digest_v2(xmlNode * source, gboolean do_filter)
	{
	char *digest = NULL;
	char *buffer = NULL;
	int offset, max;

	static struct qb_log_callsite *digest_cs = NULL;

	crm_trace("Begin digest %s", do_filter?"filtered":"");
	if (do_filter && BEST_EFFORT_STATUS) {
	/* Exclude the status calculation from the digest
	*
	- * This doesn't mean it wont be sync'd, we just wont be paranoid
	+ * This doesn't mean it won't be sync'd, we just won't be paranoid
	* about it being an _exact_ copy
	*
	* We don't need it to be exact, since we throw it away and regenerate
	* from our peers whenever a new DC is elected anyway
	*
	* Importantly, this reduces the amount of XML to copy+export as
	* well as the amount of data for MD5 needs to operate on
	*/

	} else {
	crm_xml_dump(source, do_filter ? xml_log_option_filtered : 0, &buffer, &offset, &max, 0);
	}

	CRM_ASSERT(buffer != NULL);
	digest = crm_md5sum(buffer);

	if (digest_cs == NULL) {
	digest_cs = qb_log_callsite_get(__func__, __FILE__, "cib-digest", LOG_TRACE, __LINE__,
	crm_trace_nonlog);
	}
	if (digest_cs && digest_cs->targets) {
	char *trace_file = crm_concat("/tmp/digest", digest, '-');

	crm_trace("Saving %s.%s.%s to %s",
	crm_element_value(source, XML_ATTR_GENERATION_ADMIN),
	crm_element_value(source, XML_ATTR_GENERATION),
	crm_element_value(source, XML_ATTR_NUMUPDATES), trace_file);
	save_xml_to_file(source, "digest input", trace_file);
	free(trace_file);
	}

	free(buffer);
	crm_trace("End digest");
	return digest;
	}

	/*!
	* \brief Calculate and return digest of XML tree, suitable for storing on disk
	*
	* \param[in] input Root of XML to digest
	*
	* \return Newly allocated string containing digest
	*/
	char *
	calculate_on_disk_digest(xmlNode * input)
	{
	/* Always use the v1 format for on-disk digests
	* a) its a compatability nightmare
	* b) we only use this once at startup, all other
	* invocations are in a separate child process
	*/
	return calculate_xml_digest_v1(input, FALSE, FALSE);
	}

	/*!
	* \brief Calculate and return digest of XML operation
	*
	* \param[in] input Root of XML to digest
	* \param[in] version Not used
	*
	* \return Newly allocated string containing digest
	*/
	char *
	calculate_operation_digest(xmlNode input, const char version)
	{
	/* We still need the sorting for operation digests */
	return calculate_xml_digest_v1(input, TRUE, FALSE);
	}

	/*!
	* \brief Calculate and return digest of XML tree
	*
	* \param[in] input Root of XML to digest
	* \param[in] sort Whether to sort XML before calculating digest
	* \param[in] do_filter Whether to filter certain XML attributes
	* \param[in] version CRM feature set version (used to select v1/v2 digest)
	*
	* \return Newly allocated string containing digest
	*/
	char *
	calculate_xml_versioned_digest(xmlNode * input, gboolean sort, gboolean do_filter,
	const char *version)
	{
	/*
	* The sorting associated with v1 digest creation accounted for 23% of
	* the CIB's CPU usage on the server. v2 drops this.
	*
	* The filtering accounts for an additional 2.5% and we may want to
	* remove it in future.
	*
	* v2 also uses the xmlBuffer contents directly to avoid additional copying
	*/
	if (version == NULL \|\| compare_version("3.0.5", version) > 0) {
	crm_trace("Using v1 digest algorithm for %s", crm_str(version));
	return calculate_xml_digest_v1(input, sort, do_filter);
	}
	crm_trace("Using v2 digest algorithm for %s", crm_str(version));
	return calculate_xml_digest_v2(input, do_filter);
	}

	/*!
	* \brief Return whether calculated digest of XML tree matches expected digest
	*
	* \param[in] input Root of XML to digest
	* \param[in] expected Expected digest in on-disk format
	*
	* \return TRUE if digests match, FALSE otherwise or on error
	*/
	gboolean
	crm_digest_verify(xmlNode input, const char expected)
	{
	char *calculated = NULL;
	gboolean passed;

	if (input != NULL) {
	calculated = calculate_on_disk_digest(input);
	if (calculated == NULL) {
	crm_perror(LOG_ERR, "Could not calculate digest for comparison");
	return FALSE;
	}
	}
	passed = safe_str_eq(expected, calculated);
	if (passed) {
	crm_trace("Digest comparison passed: %s", calculated);
	} else {
	crm_err("Digest comparison failed: expected %s, calculated %s",
	expected, calculated);
	}
	free(calculated);
	return passed;
	}
	diff --git a/lib/common/mainloop.c b/lib/common/mainloop.c
	index be24271883..51d129d50a 100644
	--- a/lib/common/mainloop.c
	+++ b/lib/common/mainloop.c
	@@ -1,1247 +1,1247 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <stdlib.h>
	#include <signal.h>
	#include <errno.h>

	#include <sys/wait.h>

	#include <crm/crm.h>
	#include <crm/common/xml.h>
	#include <crm/common/mainloop.h>
	#include <crm/common/ipcs.h>

	struct mainloop_child_s {
	pid_t pid;
	char *desc;
	unsigned timerid;
	unsigned watchid;
	gboolean timeout;
	void *privatedata;

	enum mainloop_child_flags flags;

	/* Called when a process dies */
	void (callback) (mainloop_child_t p, pid_t pid, int core, int signo, int exitcode);
	};

	struct trigger_s {
	GSource source;
	gboolean running;
	gboolean trigger;
	void *user_data;
	guint id;

	};

	static gboolean
	crm_trigger_prepare(GSource * source, gint * timeout)
	{
	crm_trigger_t trig = (crm_trigger_t ) source;

	/* cluster-glue's FD and IPC related sources make use of
	* g_source_add_poll() but do not set a timeout in their prepare
	* functions
	*
	* This means mainloop's poll() will block until an event for one
	* of these sources occurs - any /other/ type of source, such as
	* this one or g_idle_*, that doesn't use g_source_add_poll() is
	- * S-O-L and wont be processed until there is something fd-based
	+ * S-O-L and won't be processed until there is something fd-based
	* happens.
	*
	* Luckily the timeout we can set here affects all sources and
	* puts an upper limit on how long poll() can take.
	*
	* So unconditionally set a small-ish timeout, not too small that
	* we're in constant motion, which will act as an upper bound on
	* how long the signal handling might be delayed for.
	*/
	timeout = 500; / Timeout in ms */

	return trig->trigger;
	}

	static gboolean
	crm_trigger_check(GSource * source)
	{
	crm_trigger_t trig = (crm_trigger_t ) source;

	return trig->trigger;
	}

	static gboolean
	crm_trigger_dispatch(GSource * source, GSourceFunc callback, gpointer userdata)
	{
	int rc = TRUE;
	crm_trigger_t trig = (crm_trigger_t ) source;

	if (trig->running) {
	/* Wait until the existing job is complete before starting the next one */
	return TRUE;
	}
	trig->trigger = FALSE;

	if (callback) {
	rc = callback(trig->user_data);
	if (rc < 0) {
	crm_trace("Trigger handler %p not yet complete", trig);
	trig->running = TRUE;
	rc = TRUE;
	}
	}
	return rc;
	}

	static void
	crm_trigger_finalize(GSource * source)
	{
	crm_trace("Trigger %p destroyed", source);
	}

	#if 0
	struct _GSourceCopy
	{
	gpointer callback_data;
	GSourceCallbackFuncs *callback_funcs;

	const GSourceFuncs *source_funcs;
	guint ref_count;

	GMainContext *context;

	gint priority;
	guint flags;
	guint source_id;

	GSList *poll_fds;

	GSource *prev;
	GSource *next;

	char *name;

	void *priv;
	};

	static int
	g_source_refcount(GSource * source)
	{
	/* Duplicating the contents of private header files is a necessary evil */
	if (source) {
	struct _GSourceCopy evil = (struct _GSourceCopy)source;
	return evil->ref_count;
	}
	return 0;
	}
	#else
	static int g_source_refcount(GSource * source)
	{
	return 0;
	}
	#endif

	static GSourceFuncs crm_trigger_funcs = {
	crm_trigger_prepare,
	crm_trigger_check,
	crm_trigger_dispatch,
	crm_trigger_finalize,
	};

	static crm_trigger_t *
	mainloop_setup_trigger(GSource * source, int priority, int (*dispatch) (gpointer user_data),
	gpointer userdata)
	{
	crm_trigger_t *trigger = NULL;

	trigger = (crm_trigger_t *) source;

	trigger->id = 0;
	trigger->trigger = FALSE;
	trigger->user_data = userdata;

	if (dispatch) {
	g_source_set_callback(source, dispatch, trigger, NULL);
	}

	g_source_set_priority(source, priority);
	g_source_set_can_recurse(source, FALSE);

	crm_trace("Setup %p with ref-count=%u", source, g_source_refcount(source));
	trigger->id = g_source_attach(source, NULL);
	crm_trace("Attached %p with ref-count=%u", source, g_source_refcount(source));

	return trigger;
	}

	void
	mainloop_trigger_complete(crm_trigger_t * trig)
	{
	crm_trace("Trigger handler %p complete", trig);
	trig->running = FALSE;
	}

	/* If dispatch returns:
	* -1: Job running but not complete
	* 0: Remove the trigger from mainloop
	* 1: Leave the trigger in mainloop
	*/
	crm_trigger_t *
	mainloop_add_trigger(int priority, int (*dispatch) (gpointer user_data), gpointer userdata)
	{
	GSource *source = NULL;

	CRM_ASSERT(sizeof(crm_trigger_t) > sizeof(GSource));
	source = g_source_new(&crm_trigger_funcs, sizeof(crm_trigger_t));
	CRM_ASSERT(source != NULL);

	return mainloop_setup_trigger(source, priority, dispatch, userdata);
	}

	void
	mainloop_set_trigger(crm_trigger_t * source)
	{
	if(source) {
	source->trigger = TRUE;
	}
	}

	gboolean
	mainloop_destroy_trigger(crm_trigger_t * source)
	{
	GSource *gs = NULL;

	if(source == NULL) {
	return TRUE;
	}

	gs = (GSource *)source;

	if(g_source_refcount(gs) > 2) {
	crm_info("Trigger %p is still referenced %u times", gs, g_source_refcount(gs));
	}

	g_source_destroy(gs); /* Remove from mainloop, ref_count-- */
	g_source_unref(gs); /* The caller no longer carries a reference to source
	*
	* At this point the source should be free'd,
	* unless we're currently processing said
	* source, in which case mainloop holds an
	* additional reference and it will be free'd
	* once our processing completes
	*/
	return TRUE;
	}

	typedef struct signal_s {
	crm_trigger_t trigger; /* must be first */
	void (*handler) (int sig);
	int signal;

	} crm_signal_t;

	static crm_signal_t *crm_signals[NSIG];

	static gboolean
	crm_signal_dispatch(GSource * source, GSourceFunc callback, gpointer userdata)
	{
	crm_signal_t sig = (crm_signal_t ) source;

	if(sig->signal != SIGCHLD) {
	crm_notice("Invoking handler for signal %d: %s", sig->signal, strsignal(sig->signal));
	}

	sig->trigger.trigger = FALSE;
	if (sig->handler) {
	sig->handler(sig->signal);
	}
	return TRUE;
	}

	static void
	mainloop_signal_handler(int sig)
	{
	if (sig > 0 && sig < NSIG && crm_signals[sig] != NULL) {
	mainloop_set_trigger((crm_trigger_t *) crm_signals[sig]);
	}
	}

	static GSourceFuncs crm_signal_funcs = {
	crm_trigger_prepare,
	crm_trigger_check,
	crm_signal_dispatch,
	crm_trigger_finalize,
	};

	gboolean
	crm_signal(int sig, void (*dispatch) (int sig))
	{
	sigset_t mask;
	struct sigaction sa;
	struct sigaction old;

	if (sigemptyset(&mask) < 0) {
	crm_perror(LOG_ERR, "Call to sigemptyset failed");
	return FALSE;
	}

	memset(&sa, 0, sizeof(struct sigaction));
	sa.sa_handler = dispatch;
	sa.sa_flags = SA_RESTART;
	sa.sa_mask = mask;

	if (sigaction(sig, &sa, &old) < 0) {
	crm_perror(LOG_ERR, "Could not install signal handler for signal %d", sig);
	return FALSE;
	}

	return TRUE;
	}

	gboolean
	mainloop_add_signal(int sig, void (*dispatch) (int sig))
	{
	GSource *source = NULL;
	int priority = G_PRIORITY_HIGH - 1;

	if (sig == SIGTERM) {
	/* TERM is higher priority than other signals,
	* signals are higher priority than other ipc.
	* Yes, minus: smaller is "higher"
	*/
	priority--;
	}

	if (sig >= NSIG \|\| sig < 0) {
	crm_err("Signal %d is out of range", sig);
	return FALSE;

	} else if (crm_signals[sig] != NULL && crm_signals[sig]->handler == dispatch) {
	crm_trace("Signal handler for %d is already installed", sig);
	return TRUE;

	} else if (crm_signals[sig] != NULL) {
	crm_err("Different signal handler for %d is already installed", sig);
	return FALSE;
	}

	CRM_ASSERT(sizeof(crm_signal_t) > sizeof(GSource));
	source = g_source_new(&crm_signal_funcs, sizeof(crm_signal_t));

	crm_signals[sig] = (crm_signal_t *) mainloop_setup_trigger(source, priority, NULL, NULL);
	CRM_ASSERT(crm_signals[sig] != NULL);

	crm_signals[sig]->handler = dispatch;
	crm_signals[sig]->signal = sig;

	if (crm_signal(sig, mainloop_signal_handler) == FALSE) {
	crm_signal_t *tmp = crm_signals[sig];

	crm_signals[sig] = NULL;

	mainloop_destroy_trigger((crm_trigger_t *) tmp);
	return FALSE;
	}
	#if 0
	/* If we want signals to interrupt mainloop's poll(), instead of waiting for
	* the timeout, then we should call siginterrupt() below
	*
	* For now, just enforce a low timeout
	*/
	if (siginterrupt(sig, 1) < 0) {
	crm_perror(LOG_INFO, "Could not enable system call interruptions for signal %d", sig);
	}
	#endif

	return TRUE;
	}

	gboolean
	mainloop_destroy_signal(int sig)
	{
	crm_signal_t *tmp = NULL;

	if (sig >= NSIG \|\| sig < 0) {
	crm_err("Signal %d is out of range", sig);
	return FALSE;

	} else if (crm_signal(sig, NULL) == FALSE) {
	crm_perror(LOG_ERR, "Could not uninstall signal handler for signal %d", sig);
	return FALSE;

	} else if (crm_signals[sig] == NULL) {
	return TRUE;
	}

	crm_trace("Destroying signal %d", sig);
	tmp = crm_signals[sig];
	crm_signals[sig] = NULL;
	mainloop_destroy_trigger((crm_trigger_t *) tmp);
	return TRUE;
	}

	static qb_array_t *gio_map = NULL;

	void
	mainloop_cleanup(void)
	{
	if(gio_map) {
	qb_array_free(gio_map);
	}
	}

	/*
	* libqb...
	*/
	struct gio_to_qb_poll {
	int32_t is_used;
	guint source;
	int32_t events;
	void *data;
	qb_ipcs_dispatch_fn_t fn;
	enum qb_loop_priority p;
	};

	static gboolean
	gio_read_socket(GIOChannel * gio, GIOCondition condition, gpointer data)
	{
	struct gio_to_qb_poll adaptor = (struct gio_to_qb_poll )data;
	gint fd = g_io_channel_unix_get_fd(gio);

	crm_trace("%p.%d %d", data, fd, condition);

	/* if this assert get's hit, then there is a race condition between
	* when we destroy a fd and when mainloop actually gives it up */
	CRM_ASSERT(adaptor->is_used > 0);

	return (adaptor->fn(fd, condition, adaptor->data) == 0);
	}

	static void
	gio_poll_destroy(gpointer data)
	{
	struct gio_to_qb_poll adaptor = (struct gio_to_qb_poll )data;

	adaptor->is_used--;
	CRM_ASSERT(adaptor->is_used >= 0);

	if (adaptor->is_used == 0) {
	crm_trace("Marking adaptor %p unused", adaptor);
	adaptor->source = 0;
	}
	}

	static int32_t
	gio_poll_dispatch_update(enum qb_loop_priority p, int32_t fd, int32_t evts,
	void *data, qb_ipcs_dispatch_fn_t fn, int32_t add)
	{
	struct gio_to_qb_poll *adaptor;
	GIOChannel *channel;
	int32_t res = 0;

	res = qb_array_index(gio_map, fd, (void **)&adaptor);
	if (res < 0) {
	crm_err("Array lookup failed for fd=%d: %d", fd, res);
	return res;
	}

	crm_trace("Adding fd=%d to mainloop as adaptor %p", fd, adaptor);

	if (add && adaptor->source) {
	crm_err("Adaptor for descriptor %d is still in-use", fd);
	return -EEXIST;
	}
	if (!add && !adaptor->is_used) {
	crm_err("Adaptor for descriptor %d is not in-use", fd);
	return -ENOENT;
	}

	/* channel is created with ref_count = 1 */
	channel = g_io_channel_unix_new(fd);
	if (!channel) {
	crm_err("No memory left to add fd=%d", fd);
	return -ENOMEM;
	}

	if (adaptor->source) {
	g_source_remove(adaptor->source);
	adaptor->source = 0;
	}

	/* Because unlike the poll() API, glib doesn't tell us about HUPs by default */
	evts \|= (G_IO_HUP \| G_IO_NVAL \| G_IO_ERR);

	adaptor->fn = fn;
	adaptor->events = evts;
	adaptor->data = data;
	adaptor->p = p;
	adaptor->is_used++;
	adaptor->source =
	g_io_add_watch_full(channel, G_PRIORITY_DEFAULT, evts, gio_read_socket, adaptor,
	gio_poll_destroy);

	/* Now that mainloop now holds a reference to channel,
	* thanks to g_io_add_watch_full(), drop ours from g_io_channel_unix_new().
	*
	* This means that channel will be free'd by:
	* g_main_context_dispatch()
	* -> g_source_destroy_internal()
	* -> g_source_callback_unref()
	* shortly after gio_poll_destroy() completes
	*/
	g_io_channel_unref(channel);

	crm_trace("Added to mainloop with gsource id=%d", adaptor->source);
	if (adaptor->source > 0) {
	return 0;
	}

	return -EINVAL;
	}

	static int32_t
	gio_poll_dispatch_add(enum qb_loop_priority p, int32_t fd, int32_t evts,
	void *data, qb_ipcs_dispatch_fn_t fn)
	{
	return gio_poll_dispatch_update(p, fd, evts, data, fn, QB_TRUE);
	}

	static int32_t
	gio_poll_dispatch_mod(enum qb_loop_priority p, int32_t fd, int32_t evts,
	void *data, qb_ipcs_dispatch_fn_t fn)
	{
	return gio_poll_dispatch_update(p, fd, evts, data, fn, QB_FALSE);
	}

	static int32_t
	gio_poll_dispatch_del(int32_t fd)
	{
	struct gio_to_qb_poll *adaptor;

	crm_trace("Looking for fd=%d", fd);
	if (qb_array_index(gio_map, fd, (void **)&adaptor) == 0) {
	if (adaptor->source) {
	g_source_remove(adaptor->source);
	adaptor->source = 0;
	}
	}
	return 0;
	}

	struct qb_ipcs_poll_handlers gio_poll_funcs = {
	.job_add = NULL,
	.dispatch_add = gio_poll_dispatch_add,
	.dispatch_mod = gio_poll_dispatch_mod,
	.dispatch_del = gio_poll_dispatch_del,
	};

	static enum qb_ipc_type
	pick_ipc_type(enum qb_ipc_type requested)
	{
	const char *env = getenv("PCMK_ipc_type");

	if (env && strcmp("shared-mem", env) == 0) {
	return QB_IPC_SHM;
	} else if (env && strcmp("socket", env) == 0) {
	return QB_IPC_SOCKET;
	} else if (env && strcmp("posix", env) == 0) {
	return QB_IPC_POSIX_MQ;
	} else if (env && strcmp("sysv", env) == 0) {
	return QB_IPC_SYSV_MQ;
	} else if (requested == QB_IPC_NATIVE) {
	/* We prefer shared memory because the server never blocks on
	* send. If part of a message fits into the socket, libqb
	* needs to block until the remainder can be sent also.
	* Otherwise the client will wait forever for the remaining
	* bytes.
	*/
	return QB_IPC_SHM;
	}
	return requested;
	}

	qb_ipcs_service_t *
	mainloop_add_ipc_server(const char *name, enum qb_ipc_type type,
	struct qb_ipcs_service_handlers * callbacks)
	{
	int rc = 0;
	qb_ipcs_service_t *server = NULL;

	if (gio_map == NULL) {
	gio_map = qb_array_create_2(64, sizeof(struct gio_to_qb_poll), 1);
	}

	crm_client_init();
	server = qb_ipcs_create(name, 0, pick_ipc_type(type), callbacks);

	#ifdef HAVE_IPCS_GET_BUFFER_SIZE
	/* All clients should use at least ipc_buffer_max as their buffer size */
	qb_ipcs_enforce_buffer_size(server, crm_ipc_default_buffer_size());
	#endif

	qb_ipcs_poll_handlers_set(server, &gio_poll_funcs);

	rc = qb_ipcs_run(server);
	if (rc < 0) {
	crm_err("Could not start %s IPC server: %s (%d)", name, pcmk_strerror(rc), rc);
	return NULL;
	}

	return server;
	}

	void
	mainloop_del_ipc_server(qb_ipcs_service_t * server)
	{
	if (server) {
	qb_ipcs_destroy(server);
	}
	}

	struct mainloop_io_s {
	char *name;
	void *userdata;

	int fd;
	guint source;
	crm_ipc_t *ipc;
	GIOChannel *channel;

	int (dispatch_fn_ipc) (const char buffer, ssize_t length, gpointer userdata);
	int (*dispatch_fn_io) (gpointer userdata);
	void (*destroy_fn) (gpointer userdata);

	};

	static gboolean
	mainloop_gio_callback(GIOChannel * gio, GIOCondition condition, gpointer data)
	{
	gboolean keep = TRUE;
	mainloop_io_t *client = data;

	CRM_ASSERT(client->fd == g_io_channel_unix_get_fd(gio));

	if (condition & G_IO_IN) {
	if (client->ipc) {
	long rc = 0;
	int max = 10;

	do {
	rc = crm_ipc_read(client->ipc);
	if (rc <= 0) {
	crm_trace("Message acquisition from %s[%p] failed: %s (%ld)",
	client->name, client, pcmk_strerror(rc), rc);

	} else if (client->dispatch_fn_ipc) {
	const char *buffer = crm_ipc_buffer(client->ipc);

	crm_trace("New message from %s[%p] = %d", client->name, client, rc, condition);
	if (client->dispatch_fn_ipc(buffer, rc, client->userdata) < 0) {
	crm_trace("Connection to %s no longer required", client->name);
	keep = FALSE;
	}
	}

	} while (keep && rc > 0 && --max > 0);

	} else {
	crm_trace("New message from %s[%p] %u", client->name, client, condition);
	if (client->dispatch_fn_io) {
	if (client->dispatch_fn_io(client->userdata) < 0) {
	crm_trace("Connection to %s no longer required", client->name);
	keep = FALSE;
	}
	}
	}
	}

	if (client->ipc && crm_ipc_connected(client->ipc) == FALSE) {
	crm_err("Connection to %s[%p] closed (I/O condition=%d)", client->name, client, condition);
	keep = FALSE;

	} else if (condition & (G_IO_HUP \| G_IO_NVAL \| G_IO_ERR)) {
	crm_trace("The connection %s[%p] has been closed (I/O condition=%d)",
	client->name, client, condition);
	keep = FALSE;

	} else if ((condition & G_IO_IN) == 0) {
	/*
	#define GLIB_SYSDEF_POLLIN =1
	#define GLIB_SYSDEF_POLLPRI =2
	#define GLIB_SYSDEF_POLLOUT =4
	#define GLIB_SYSDEF_POLLERR =8
	#define GLIB_SYSDEF_POLLHUP =16
	#define GLIB_SYSDEF_POLLNVAL =32

	typedef enum
	{
	G_IO_IN GLIB_SYSDEF_POLLIN,
	G_IO_OUT GLIB_SYSDEF_POLLOUT,
	G_IO_PRI GLIB_SYSDEF_POLLPRI,
	G_IO_ERR GLIB_SYSDEF_POLLERR,
	G_IO_HUP GLIB_SYSDEF_POLLHUP,
	G_IO_NVAL GLIB_SYSDEF_POLLNVAL
	} GIOCondition;

	A bitwise combination representing a condition to watch for on an event source.

	G_IO_IN There is data to read.
	G_IO_OUT Data can be written (without blocking).
	G_IO_PRI There is urgent data to read.
	G_IO_ERR Error condition.
	G_IO_HUP Hung up (the connection has been broken, usually for pipes and sockets).
	G_IO_NVAL Invalid request. The file descriptor is not open.
	*/
	crm_err("Strange condition: %d", condition);
	}

	/* keep == FALSE results in mainloop_gio_destroy() being called
	* just before the source is removed from mainloop
	*/
	return keep;
	}

	static void
	mainloop_gio_destroy(gpointer c)
	{
	mainloop_io_t *client = c;
	char *c_name = strdup(client->name);

	/* client->source is valid but about to be destroyed (ref_count == 0) in gmain.c
	* client->channel will still have ref_count > 0... should be == 1
	*/
	crm_trace("Destroying client %s[%p]", c_name, c);

	if (client->ipc) {
	crm_ipc_close(client->ipc);
	}

	if (client->destroy_fn) {
	void (*destroy_fn) (gpointer userdata) = client->destroy_fn;

	client->destroy_fn = NULL;
	destroy_fn(client->userdata);
	}

	if (client->ipc) {
	crm_ipc_t *ipc = client->ipc;

	client->ipc = NULL;
	crm_ipc_destroy(ipc);
	}

	crm_trace("Destroyed client %s[%p]", c_name, c);

	free(client->name); client->name = NULL;
	free(client);

	free(c_name);
	}

	mainloop_io_t *
	mainloop_add_ipc_client(const char name, int priority, size_t max_size, void userdata,
	struct ipc_client_callbacks *callbacks)
	{
	mainloop_io_t *client = NULL;
	crm_ipc_t *conn = crm_ipc_new(name, max_size);

	if (conn && crm_ipc_connect(conn)) {
	int32_t fd = crm_ipc_get_fd(conn);

	client = mainloop_add_fd(name, priority, fd, userdata, NULL);
	}

	if (client == NULL) {
	crm_perror(LOG_TRACE, "Connection to %s failed", name);
	if (conn) {
	crm_ipc_close(conn);
	crm_ipc_destroy(conn);
	}
	return NULL;
	}

	client->ipc = conn;
	client->destroy_fn = callbacks->destroy;
	client->dispatch_fn_ipc = callbacks->dispatch;
	return client;
	}

	void
	mainloop_del_ipc_client(mainloop_io_t * client)
	{
	mainloop_del_fd(client);
	}

	crm_ipc_t *
	mainloop_get_ipc_client(mainloop_io_t * client)
	{
	if (client) {
	return client->ipc;
	}
	return NULL;
	}

	mainloop_io_t *
	mainloop_add_fd(const char name, int priority, int fd, void userdata,
	struct mainloop_fd_callbacks * callbacks)
	{
	mainloop_io_t *client = NULL;

	if (fd >= 0) {
	client = calloc(1, sizeof(mainloop_io_t));
	if (client == NULL) {
	return NULL;
	}
	client->name = strdup(name);
	client->userdata = userdata;

	if (callbacks) {
	client->destroy_fn = callbacks->destroy;
	client->dispatch_fn_io = callbacks->dispatch;
	}

	client->fd = fd;
	client->channel = g_io_channel_unix_new(fd);
	client->source =
	g_io_add_watch_full(client->channel, priority,
	(G_IO_IN \| G_IO_HUP \| G_IO_NVAL \| G_IO_ERR), mainloop_gio_callback,
	client, mainloop_gio_destroy);

	/* Now that mainloop now holds a reference to channel,
	* thanks to g_io_add_watch_full(), drop ours from g_io_channel_unix_new().
	*
	* This means that channel will be free'd by:
	* g_main_context_dispatch() or g_source_remove()
	* -> g_source_destroy_internal()
	* -> g_source_callback_unref()
	* shortly after mainloop_gio_destroy() completes
	*/
	g_io_channel_unref(client->channel);
	crm_trace("Added connection %d for %s[%p].%d", client->source, client->name, client, fd);
	} else {
	errno = EINVAL;
	}

	return client;
	}

	void
	mainloop_del_fd(mainloop_io_t * client)
	{
	if (client != NULL) {
	crm_trace("Removing client %s[%p]", client->name, client);
	if (client->source) {
	/* Results in mainloop_gio_destroy() being called just
	* before the source is removed from mainloop
	*/
	g_source_remove(client->source);
	}
	}
	}

	static GListPtr child_list = NULL;

	pid_t
	mainloop_child_pid(mainloop_child_t * child)
	{
	return child->pid;
	}

	const char *
	mainloop_child_name(mainloop_child_t * child)
	{
	return child->desc;
	}

	int
	mainloop_child_timeout(mainloop_child_t * child)
	{
	return child->timeout;
	}

	void *
	mainloop_child_userdata(mainloop_child_t * child)
	{
	return child->privatedata;
	}

	void
	mainloop_clear_child_userdata(mainloop_child_t * child)
	{
	child->privatedata = NULL;
	}

	/* good function name */
	static void
	child_free(mainloop_child_t *child)
	{
	if (child->timerid != 0) {
	crm_trace("Removing timer %d", child->timerid);
	g_source_remove(child->timerid);
	child->timerid = 0;
	}
	free(child->desc);
	free(child);
	}

	/* terrible function name */
	static int
	child_kill_helper(mainloop_child_t *child)
	{
	int rc;
	if (child->flags & mainloop_leave_pid_group) {
	crm_debug("Kill pid %d only. leave group intact.", child->pid);
	rc = kill(child->pid, SIGKILL);
	} else {
	crm_debug("Kill pid %d's group", child->pid);
	rc = kill(-child->pid, SIGKILL);
	}

	if (rc < 0) {
	if (errno != ESRCH) {
	crm_perror(LOG_ERR, "kill(%d, KILL) failed", child->pid);
	}
	return -errno;
	}
	return 0;
	}

	static gboolean
	child_timeout_callback(gpointer p)
	{
	mainloop_child_t *child = p;
	int rc = 0;

	child->timerid = 0;
	if (child->timeout) {
	crm_crit("%s process (PID %d) will not die!", child->desc, (int)child->pid);
	return FALSE;
	}

	rc = child_kill_helper(child);
	if (rc == ESRCH) {
	/* Nothing left to do. pid doesn't exist */
	return FALSE;
	}

	child->timeout = TRUE;
	crm_warn("%s process (PID %d) timed out", child->desc, (int)child->pid);

	child->timerid = g_timeout_add(5000, child_timeout_callback, child);
	return FALSE;
	}

	static gboolean
	child_waitpid(mainloop_child_t *child, int flags)
	{
	int rc = 0;
	int core = 0;
	int signo = 0;
	int status = 0;
	int exitcode = 0;

	rc = waitpid(child->pid, &status, flags);
	if(rc == 0) {
	crm_perror(LOG_DEBUG, "wait(%d) = %d", child->pid, rc);
	return FALSE;

	} else if(rc != child->pid) {
	signo = SIGCHLD;
	exitcode = 1;
	status = 1;
	crm_perror(LOG_ERR, "Call to waitpid(%d) failed", child->pid);

	} else {
	crm_trace("Managed process %d exited: %p", child->pid, child);

	if (WIFEXITED(status)) {
	exitcode = WEXITSTATUS(status);
	crm_trace("Managed process %d (%s) exited with rc=%d", child->pid, child->desc, exitcode);

	} else if (WIFSIGNALED(status)) {
	signo = WTERMSIG(status);
	crm_trace("Managed process %d (%s) exited with signal=%d", child->pid, child->desc, signo);
	}
	#ifdef WCOREDUMP
	if (WCOREDUMP(status)) {
	core = 1;
	crm_err("Managed process %d (%s) dumped core", child->pid, child->desc);
	}
	#endif
	}

	if (child->callback) {
	child->callback(child, child->pid, core, signo, exitcode);
	}
	return TRUE;
	}

	static void
	child_death_dispatch(int signal)
	{
	GListPtr iter = child_list;
	gboolean exited;

	while(iter) {
	GListPtr saved = NULL;
	mainloop_child_t *child = iter->data;
	exited = child_waitpid(child, WNOHANG);

	saved = iter;
	iter = iter->next;

	if (exited == FALSE) {
	continue;
	}
	crm_trace("Removing process entry %p for %d", child, child->pid);

	child_list = g_list_remove_link(child_list, saved);
	g_list_free(saved);
	child_free(child);
	}
	}

	static gboolean
	child_signal_init(gpointer p)
	{
	crm_trace("Installed SIGCHLD handler");
	/* Do NOT use g_child_watch_add() and friends, they rely on pthreads */
	mainloop_add_signal(SIGCHLD, child_death_dispatch);

	/* In case they terminated before the signal handler was installed */
	child_death_dispatch(SIGCHLD);
	return FALSE;
	}

	int
	mainloop_child_kill(pid_t pid)
	{
	GListPtr iter;
	mainloop_child_t *child = NULL;
	mainloop_child_t *match = NULL;
	/* It is impossible to block SIGKILL, this allows us to
	* call waitpid without WNOHANG flag.*/
	int waitflags = 0, rc = 0;

	for (iter = child_list; iter != NULL && match == NULL; iter = iter->next) {
	child = iter->data;
	if (pid == child->pid) {
	match = child;
	}
	}

	if (match == NULL) {
	return FALSE;
	}

	rc = child_kill_helper(match);
	if(rc == -ESRCH) {
	/* Its gone, but hasn't shown up in waitpid() yet
	*
	* Wait until we get SIGCHLD and let child_death_dispatch()
	* clean it up as normal (so we get the correct return
	* code/status)
	*
	* The blocking alternative would be to call:
	* child_waitpid(match, 0);
	*/
	crm_trace("Waiting for child %d to be reaped by child_death_dispatch()", match->pid);
	return TRUE;

	} else if(rc != 0) {
	/* If KILL for some other reason set the WNOHANG flag since we
	* can't be certain what happened.
	*/
	waitflags = WNOHANG;
	}

	if (child_waitpid(match, waitflags) == FALSE) {
	/* not much we can do if this occurs */
	return FALSE;
	}

	child_list = g_list_remove(child_list, match);
	child_free(match);
	return TRUE;
	}

	/* Create/Log a new tracked process
	* To track a process group, use -pid
	*/
	void
	mainloop_child_add_with_flags(pid_t pid, int timeout, const char desc, void privatedata, enum mainloop_child_flags flags,
	void (callback) (mainloop_child_t p, pid_t pid, int core, int signo, int exitcode))
	{
	static bool need_init = TRUE;
	mainloop_child_t *child = g_new(mainloop_child_t, 1);

	child->pid = pid;
	child->timerid = 0;
	child->timeout = FALSE;
	child->privatedata = privatedata;
	child->callback = callback;
	child->flags = flags;

	if(desc) {
	child->desc = strdup(desc);
	}

	if (timeout) {
	child->timerid = g_timeout_add(timeout, child_timeout_callback, child);
	}

	child_list = g_list_append(child_list, child);

	if(need_init) {
	need_init = FALSE;
	/* SIGCHLD processing has to be invoked from mainloop.
	* We do not want it to be possible to both add a child pid
	* to mainloop, and have the pid's exit callback invoked within
	* the same callstack. */
	g_timeout_add(1, child_signal_init, NULL);
	}
	}

	void
	mainloop_child_add(pid_t pid, int timeout, const char desc, void privatedata,
	void (callback) (mainloop_child_t p, pid_t pid, int core, int signo, int exitcode))
	{
	mainloop_child_add_with_flags(pid, timeout, desc, privatedata, 0, callback);
	}

	struct mainloop_timer_s {
	guint id;
	guint period_ms;
	bool repeat;
	char *name;
	GSourceFunc cb;
	void *userdata;
	};

	struct mainloop_timer_s mainloop;

	static gboolean mainloop_timer_cb(gpointer user_data)
	{
	int id = 0;
	bool repeat = FALSE;
	struct mainloop_timer_s *t = user_data;

	CRM_ASSERT(t != NULL);

	id = t->id;
	t->id = 0; /* Ensure its unset during callbacks so that
	* mainloop_timer_running() works as expected
	*/

	if(t->cb) {
	crm_trace("Invoking callbacks for timer %s", t->name);
	repeat = t->repeat;
	if(t->cb(t->userdata) == FALSE) {
	crm_trace("Timer %s complete", t->name);
	repeat = FALSE;
	}
	}

	if(repeat) {
	/* Restore if repeating */
	t->id = id;
	}

	return repeat;
	}

	bool mainloop_timer_running(mainloop_timer_t *t)
	{
	if(t && t->id != 0) {
	return TRUE;
	}
	return FALSE;
	}

	void mainloop_timer_start(mainloop_timer_t *t)
	{
	mainloop_timer_stop(t);
	if(t && t->period_ms > 0) {
	crm_trace("Starting timer %s", t->name);
	t->id = g_timeout_add(t->period_ms, mainloop_timer_cb, t);
	}
	}

	void mainloop_timer_stop(mainloop_timer_t *t)
	{
	if(t && t->id != 0) {
	crm_trace("Stopping timer %s", t->name);
	g_source_remove(t->id);
	t->id = 0;
	}
	}

	guint mainloop_timer_set_period(mainloop_timer_t *t, guint period_ms)
	{
	guint last = 0;

	if(t) {
	last = t->period_ms;
	t->period_ms = period_ms;
	}

	if(t && t->id != 0 && last != t->period_ms) {
	mainloop_timer_start(t);
	}
	return last;
	}


	mainloop_timer_t *
	mainloop_timer_add(const char name, guint period_ms, bool repeat, GSourceFunc cb, void userdata)
	{
	mainloop_timer_t *t = calloc(1, sizeof(mainloop_timer_t));

	if(t) {
	if(name) {
	t->name = crm_strdup_printf("%s-%u-%d", name, period_ms, repeat);
	} else {
	t->name = crm_strdup_printf("%p-%u-%d", t, period_ms, repeat);
	}
	t->id = 0;
	t->period_ms = period_ms;
	t->repeat = repeat;
	t->cb = cb;
	t->userdata = userdata;
	crm_trace("Created timer %s with %p %p", t->name, userdata, t->userdata);
	}
	return t;
	}

	void
	mainloop_timer_del(mainloop_timer_t *t)
	{
	if(t) {
	crm_trace("Destroying timer %s", t->name);
	mainloop_timer_stop(t);
	free(t->name);
	free(t);
	}
	}

	diff --git a/lib/common/utils.c b/lib/common/utils.c
	index e08dbafa3b..264e171cad 100644
	--- a/lib/common/utils.c
	+++ b/lib/common/utils.c
	@@ -1,2413 +1,2413 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>
	#include <dlfcn.h>

	#ifndef _GNU_SOURCE
	# define _GNU_SOURCE
	#endif

	#include <sys/param.h>
	#include <sys/types.h>
	#include <sys/wait.h>
	#include <sys/stat.h>
	#include <sys/utsname.h>

	#include <stdio.h>
	#include <unistd.h>
	#include <string.h>
	#include <stdlib.h>
	#include <limits.h>
	#include <ctype.h>
	#include <pwd.h>
	#include <time.h>
	#include <libgen.h>
	#include <signal.h>

	#include <qb/qbdefs.h>

	#include <crm/crm.h>
	#include <crm/lrmd.h>
	#include <crm/services.h>
	#include <crm/msg_xml.h>
	#include <crm/cib/internal.h>
	#include <crm/common/xml.h>
	#include <crm/common/util.h>
	#include <crm/common/ipc.h>
	#include <crm/common/iso8601.h>
	#include <crm/common/mainloop.h>
	#include <crm/attrd.h>
	#include <libxml2/libxml/relaxng.h>

	#ifndef MAXLINE
	# define MAXLINE 512
	#endif

	#ifdef HAVE_GETOPT_H
	# include <getopt.h>
	#endif

	#ifndef PW_BUFFER_LEN
	# define PW_BUFFER_LEN 500
	#endif

	CRM_TRACE_INIT_DATA(common);

	gboolean crm_config_error = FALSE;
	gboolean crm_config_warning = FALSE;
	char *crm_system_name = NULL;

	int node_score_red = 0;
	int node_score_green = 0;
	int node_score_yellow = 0;
	int node_score_infinity = INFINITY;

	static struct crm_option *crm_long_options = NULL;
	static const char *crm_app_description = NULL;
	static char *crm_short_options = NULL;
	static const char *crm_app_usage = NULL;

	int
	crm_exit(int rc)
	{
	mainloop_cleanup();

	#if HAVE_LIBXML2
	crm_trace("cleaning up libxml");
	crm_xml_cleanup();
	#endif

	crm_trace("exit %d", rc);
	qb_log_fini();

	free(crm_short_options);
	free(crm_system_name);

	exit(ABS(rc)); /* Always exit with a positive value so that it can be passed to crm_error
	*
	* Otherwise the system wraps it around and people
	* have to jump through hoops figuring out what the
	* error was
	*/
	return rc; /* Can never happen, but allows return crm_exit(rc)
	* where "return rc" was used previously - which
	* keeps compilers happy.
	*/
	}

	gboolean
	check_time(const char *value)
	{
	if (crm_get_msec(value) < 5000) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	check_timer(const char *value)
	{
	if (crm_get_msec(value) < 0) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	check_boolean(const char *value)
	{
	int tmp = FALSE;

	if (crm_str_to_boolean(value, &tmp) != 1) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	check_number(const char *value)
	{
	errno = 0;
	if (value == NULL) {
	return FALSE;

	} else if (safe_str_eq(value, MINUS_INFINITY_S)) {

	} else if (safe_str_eq(value, INFINITY_S)) {

	} else {
	crm_int_helper(value, NULL);
	}

	if (errno != 0) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	check_quorum(const char *value)
	{
	if (safe_str_eq(value, "stop")) {
	return TRUE;

	} else if (safe_str_eq(value, "freeze")) {
	return TRUE;

	} else if (safe_str_eq(value, "ignore")) {
	return TRUE;

	} else if (safe_str_eq(value, "suicide")) {
	return TRUE;
	}
	return FALSE;
	}

	gboolean
	check_script(const char *value)
	{
	struct stat st;

	if(safe_str_eq(value, "/dev/null")) {
	return TRUE;
	}

	if(stat(value, &st) != 0) {
	crm_err("Script %s does not exist", value);
	return FALSE;
	}

	if(S_ISREG(st.st_mode) == 0) {
	crm_err("Script %s is not a regular file", value);
	return FALSE;
	}

	if( (st.st_mode & (S_IXUSR \| S_IXGRP )) == 0) {
	crm_err("Script %s is not executable", value);
	return FALSE;
	}

	return TRUE;
	}

	gboolean
	check_utilization(const char *value)
	{
	char *end = NULL;
	long number = strtol(value, &end, 10);

	if(end && end[0] != '%') {
	return FALSE;
	} else if(number < 0) {
	return FALSE;
	}

	return TRUE;
	}

	int
	char2score(const char *score)
	{
	int score_f = 0;

	if (score == NULL) {

	} else if (safe_str_eq(score, MINUS_INFINITY_S)) {
	score_f = -node_score_infinity;

	} else if (safe_str_eq(score, INFINITY_S)) {
	score_f = node_score_infinity;

	} else if (safe_str_eq(score, "+" INFINITY_S)) {
	score_f = node_score_infinity;

	} else if (safe_str_eq(score, "red")) {
	score_f = node_score_red;

	} else if (safe_str_eq(score, "yellow")) {
	score_f = node_score_yellow;

	} else if (safe_str_eq(score, "green")) {
	score_f = node_score_green;

	} else {
	score_f = crm_parse_int(score, NULL);
	if (score_f > 0 && score_f > node_score_infinity) {
	score_f = node_score_infinity;

	} else if (score_f < 0 && score_f < -node_score_infinity) {
	score_f = -node_score_infinity;
	}
	}

	return score_f;
	}

	char *
	score2char_stack(int score, char *buf, size_t len)
	{
	if (score >= node_score_infinity) {
	strncpy(buf, INFINITY_S, 9);
	} else if (score <= -node_score_infinity) {
	strncpy(buf, MINUS_INFINITY_S , 10);
	} else {
	return crm_itoa_stack(score, buf, len);
	}

	return buf;
	}

	char *
	score2char(int score)
	{
	if (score >= node_score_infinity) {
	return strdup(INFINITY_S);

	} else if (score <= -node_score_infinity) {
	return strdup("-" INFINITY_S);
	}
	return crm_itoa(score);
	}

	const char *
	cluster_option(GHashTable * options, gboolean(validate) (const char ),
	const char name, const char old_name, const char *def_value)
	{
	const char *value = NULL;

	CRM_ASSERT(name != NULL);

	if (options != NULL) {
	value = g_hash_table_lookup(options, name);
	}

	if (value == NULL && old_name && options != NULL) {
	value = g_hash_table_lookup(options, old_name);
	if (value != NULL) {
	crm_config_warn("Using deprecated name '%s' for"
	" cluster option '%s'", old_name, name);
	g_hash_table_insert(options, strdup(name), strdup(value));
	value = g_hash_table_lookup(options, old_name);
	}
	}

	if (value == NULL) {
	crm_trace("Using default value '%s' for cluster option '%s'", def_value, name);

	if (options == NULL) {
	return def_value;

	} else if(def_value == NULL) {
	return def_value;
	}

	g_hash_table_insert(options, strdup(name), strdup(def_value));
	value = g_hash_table_lookup(options, name);
	}

	if (validate && validate(value) == FALSE) {
	crm_config_err("Value '%s' for cluster option '%s' is invalid."
	" Defaulting to %s", value, name, def_value);
	g_hash_table_replace(options, strdup(name), strdup(def_value));
	value = g_hash_table_lookup(options, name);
	}

	return value;
	}

	const char *
	get_cluster_pref(GHashTable * options, pe_cluster_option * option_list, int len, const char *name)
	{
	int lpc = 0;
	const char *value = NULL;
	gboolean found = FALSE;

	for (lpc = 0; lpc < len; lpc++) {
	if (safe_str_eq(name, option_list[lpc].name)) {
	found = TRUE;
	value = cluster_option(options,
	option_list[lpc].is_valid,
	option_list[lpc].name,
	option_list[lpc].alt_name, option_list[lpc].default_value);
	}
	}
	CRM_CHECK(found, crm_err("No option named: %s", name));
	return value;
	}

	void
	config_metadata(const char name, const char version, const char *desc_short,
	const char desc_long, pe_cluster_option option_list, int len)
	{
	int lpc = 0;

	fprintf(stdout, "<?xml version=\"1.0\"?>"
	"<!DOCTYPE resource-agent SYSTEM \"ra-api-1.dtd\">\n"
	"<resource-agent name=\"%s\">\n"
	" <version>%s</version>\n"
	" <longdesc lang=\"en\">%s</longdesc>\n"
	" <shortdesc lang=\"en\">%s</shortdesc>\n"
	" <parameters>\n", name, version, desc_long, desc_short);

	for (lpc = 0; lpc < len; lpc++) {
	if (option_list[lpc].description_long == NULL && option_list[lpc].description_short == NULL) {
	continue;
	}
	fprintf(stdout, " <parameter name=\"%s\" unique=\"0\">\n"
	" <shortdesc lang=\"en\">%s</shortdesc>\n"
	" <content type=\"%s\" default=\"%s\"/>\n"
	" <longdesc lang=\"en\">%s%s%s</longdesc>\n"
	" </parameter>\n",
	option_list[lpc].name,
	option_list[lpc].description_short,
	option_list[lpc].type,
	option_list[lpc].default_value,
	option_list[lpc].description_long ? option_list[lpc].
	description_long : option_list[lpc].description_short,
	option_list[lpc].values ? " Allowed values: " : "",
	option_list[lpc].values ? option_list[lpc].values : "");
	}
	fprintf(stdout, " </parameters>\n</resource-agent>\n");
	}

	void
	verify_all_options(GHashTable * options, pe_cluster_option * option_list, int len)
	{
	int lpc = 0;

	for (lpc = 0; lpc < len; lpc++) {
	cluster_option(options,
	option_list[lpc].is_valid,
	option_list[lpc].name,
	option_list[lpc].alt_name, option_list[lpc].default_value);
	}
	}

	char *
	crm_concat(const char prefix, const char suffix, char join)
	{
	int len = 0;
	char *new_str = NULL;

	CRM_ASSERT(prefix != NULL);
	CRM_ASSERT(suffix != NULL);
	len = strlen(prefix) + strlen(suffix) + 2;

	new_str = malloc(len);
	if(new_str) {
	sprintf(new_str, "%s%c%s", prefix, join, suffix);
	new_str[len - 1] = 0;
	}
	return new_str;
	}

	char *
	generate_hash_key(const char crm_msg_reference, const char sys)
	{
	char *hash_key = crm_concat(sys ? sys : "none", crm_msg_reference, '_');

	crm_trace("created hash key: (%s)", hash_key);
	return hash_key;
	}


	char *
	crm_itoa_stack(int an_int, char *buffer, size_t len)
	{
	if (buffer != NULL) {
	snprintf(buffer, len, "%d", an_int);
	}

	return buffer;
	}

	char *
	crm_itoa(int an_int)
	{
	int len = 32;
	char *buffer = NULL;

	buffer = malloc(len + 1);
	if (buffer != NULL) {
	snprintf(buffer, len, "%d", an_int);
	}

	return buffer;
	}

	int
	crm_user_lookup(const char name, uid_t uid, gid_t * gid)
	{
	int rc = -1;
	char *buffer = NULL;
	struct passwd pwd;
	struct passwd *pwentry = NULL;

	buffer = calloc(1, PW_BUFFER_LEN);
	getpwnam_r(name, &pwd, buffer, PW_BUFFER_LEN, &pwentry);
	if (pwentry) {
	rc = 0;
	if (uid) {
	*uid = pwentry->pw_uid;
	}
	if (gid) {
	*gid = pwentry->pw_gid;
	}
	crm_trace("Cluster user %s has uid=%d gid=%d", name, pwentry->pw_uid, pwentry->pw_gid);

	} else {
	crm_err("Cluster user %s does not exist", name);
	}

	free(buffer);
	return rc;
	}

	static int
	crm_version_helper(const char text, char *end_text)
	{
	int atoi_result = -1;

	CRM_ASSERT(end_text != NULL);

	errno = 0;

	if (text != NULL && text[0] != 0) {
	atoi_result = (int)strtol(text, end_text, 10);

	if (errno == EINVAL) {
	crm_err("Conversion of '%s' %c failed", text, text[0]);
	atoi_result = -1;
	}
	}
	return atoi_result;
	}

	/*
	* version1 < version2 : -1
	* version1 = version2 : 0
	* version1 > version2 : 1
	*/
	int
	compare_version(const char version1, const char version2)
	{
	int rc = 0;
	int lpc = 0;
	char ver1_copy = NULL, ver2_copy = NULL;
	char rest1 = NULL, rest2 = NULL;

	if (version1 == NULL && version2 == NULL) {
	return 0;
	} else if (version1 == NULL) {
	return -1;
	} else if (version2 == NULL) {
	return 1;
	}

	ver1_copy = strdup(version1);
	ver2_copy = strdup(version2);
	rest1 = ver1_copy;
	rest2 = ver2_copy;

	while (1) {
	int digit1 = 0;
	int digit2 = 0;

	lpc++;

	if (rest1 == rest2) {
	break;
	}

	if (rest1 != NULL) {
	digit1 = crm_version_helper(rest1, &rest1);
	}

	if (rest2 != NULL) {
	digit2 = crm_version_helper(rest2, &rest2);
	}

	if (digit1 < digit2) {
	rc = -1;
	break;

	} else if (digit1 > digit2) {
	rc = 1;
	break;
	}

	if (rest1 != NULL && rest1[0] == '.') {
	rest1++;
	}
	if (rest1 != NULL && rest1[0] == 0) {
	rest1 = NULL;
	}

	if (rest2 != NULL && rest2[0] == '.') {
	rest2++;
	}
	if (rest2 != NULL && rest2[0] == 0) {
	rest2 = NULL;
	}
	}

	free(ver1_copy);
	free(ver2_copy);

	if (rc == 0) {
	crm_trace("%s == %s (%d)", version1, version2, lpc);
	} else if (rc < 0) {
	crm_trace("%s < %s (%d)", version1, version2, lpc);
	} else if (rc > 0) {
	crm_trace("%s > %s (%d)", version1, version2, lpc);
	}

	return rc;
	}

	gboolean do_stderr = FALSE;

	void
	g_hash_destroy_str(gpointer data)
	{
	free(data);
	}

	#include <sys/types.h>
	/* #include <stdlib.h> */
	/* #include <limits.h> */

	long long
	crm_int_helper(const char text, char *end_text)
	{
	long long result = -1;
	char *local_end_text = NULL;
	int saved_errno = 0;

	errno = 0;

	if (text != NULL) {
	#ifdef ANSI_ONLY
	if (end_text != NULL) {
	result = strtol(text, end_text, 10);
	} else {
	result = strtol(text, &local_end_text, 10);
	}
	#else
	if (end_text != NULL) {
	result = strtoll(text, end_text, 10);
	} else {
	result = strtoll(text, &local_end_text, 10);
	}
	#endif

	saved_errno = errno;
	/* CRM_CHECK(errno != EINVAL); */
	if (errno == EINVAL) {
	crm_err("Conversion of %s failed", text);
	result = -1;

	} else if (errno == ERANGE) {
	crm_err("Conversion of %s was clipped: %lld", text, result);

	} else if (errno != 0) {
	crm_perror(LOG_ERR, "Conversion of %s failed:", text);
	}

	if (local_end_text != NULL && local_end_text[0] != '\0') {
	crm_err("Characters left over after parsing '%s': '%s'", text, local_end_text);
	}

	errno = saved_errno;
	}
	return result;
	}

	int
	crm_parse_int(const char text, const char default_text)
	{
	int atoi_result = -1;

	if (text != NULL) {
	atoi_result = crm_int_helper(text, NULL);
	if (errno == 0) {
	return atoi_result;
	}
	}

	if (default_text != NULL) {
	atoi_result = crm_int_helper(default_text, NULL);
	if (errno == 0) {
	return atoi_result;
	}

	} else {
	crm_err("No default conversion value supplied");
	}

	return -1;
	}

	gboolean
	safe_str_neq(const char a, const char b)
	{
	if (a == b) {
	return FALSE;

	} else if (a == NULL \|\| b == NULL) {
	return TRUE;

	} else if (strcasecmp(a, b) == 0) {
	return FALSE;
	}
	return TRUE;
	}

	gboolean
	crm_is_true(const char *s)
	{
	gboolean ret = FALSE;

	if (s != NULL) {
	crm_str_to_boolean(s, &ret);
	}
	return ret;
	}

	int
	crm_str_to_boolean(const char s, int ret)
	{
	if (s == NULL) {
	return -1;

	} else if (strcasecmp(s, "true") == 0
	\|\| strcasecmp(s, "on") == 0
	\|\| strcasecmp(s, "yes") == 0 \|\| strcasecmp(s, "y") == 0 \|\| strcasecmp(s, "1") == 0) {
	*ret = TRUE;
	return 1;

	} else if (strcasecmp(s, "false") == 0
	\|\| strcasecmp(s, "off") == 0
	\|\| strcasecmp(s, "no") == 0 \|\| strcasecmp(s, "n") == 0 \|\| strcasecmp(s, "0") == 0) {
	*ret = FALSE;
	return 1;
	}
	return -1;
	}

	#ifndef NUMCHARS
	# define NUMCHARS "0123456789."
	#endif

	#ifndef WHITESPACE
	# define WHITESPACE " \t\n\r\f"
	#endif

	unsigned long long
	crm_get_interval(const char *input)
	{
	unsigned long long msec = 0;

	if (input == NULL) {
	return msec;

	} else if (input[0] != 'P') {
	long long tmp = crm_get_msec(input);

	if(tmp > 0) {
	msec = tmp;
	}

	} else {
	crm_time_t *interval = crm_time_parse_duration(input);

	msec = 1000 * crm_time_get_seconds(interval);
	crm_time_free(interval);
	}

	return msec;
	}

	long long
	crm_get_msec(const char *input)
	{
	const char *cp = input;
	const char *units;
	long long multiplier = 1000;
	long long divisor = 1;
	long long msec = -1;
	char *end_text = NULL;

	/* double dret; */

	if (input == NULL) {
	return msec;
	}

	cp += strspn(cp, WHITESPACE);
	units = cp + strspn(cp, NUMCHARS);
	units += strspn(units, WHITESPACE);

	if (strchr(NUMCHARS, *cp) == NULL) {
	return msec;
	}

	if (strncasecmp(units, "ms", 2) == 0 \|\| strncasecmp(units, "msec", 4) == 0) {
	multiplier = 1;
	divisor = 1;
	} else if (strncasecmp(units, "us", 2) == 0 \|\| strncasecmp(units, "usec", 4) == 0) {
	multiplier = 1;
	divisor = 1000;
	} else if (strncasecmp(units, "s", 1) == 0 \|\| strncasecmp(units, "sec", 3) == 0) {
	multiplier = 1000;
	divisor = 1;
	} else if (strncasecmp(units, "m", 1) == 0 \|\| strncasecmp(units, "min", 3) == 0) {
	multiplier = 60 * 1000;
	divisor = 1;
	} else if (strncasecmp(units, "h", 1) == 0 \|\| strncasecmp(units, "hr", 2) == 0) {
	multiplier = 60 * 60 * 1000;
	divisor = 1;
	} else if (units != EOS && units != '\n' && *units != '\r') {
	return msec;
	}

	msec = crm_int_helper(cp, &end_text);
	if (msec > LLONG_MAX/multiplier) {
	/* arithmetics overflow while multiplier/divisor mutually exclusive */
	return LLONG_MAX;
	}
	msec *= multiplier;
	msec /= divisor;
	/* dret += 0.5; */
	/* msec = (long long)dret; */
	return msec;
	}

	char *
	generate_op_key(const char rsc_id, const char op_type, int interval)
	{
	int len = 35;
	char *op_id = NULL;

	CRM_CHECK(rsc_id != NULL, return NULL);
	CRM_CHECK(op_type != NULL, return NULL);

	len += strlen(op_type);
	len += strlen(rsc_id);
	op_id = malloc(len);
	CRM_CHECK(op_id != NULL, return NULL);
	sprintf(op_id, "%s_%s_%d", rsc_id, op_type, interval);
	return op_id;
	}

	gboolean
	parse_op_key(const char key, char rsc_id, char op_type, int interval)
	{
	char *notify = NULL;
	char *mutable_key = NULL;
	char *mutable_key_ptr = NULL;
	int len = 0, offset = 0, ch = 0;

	CRM_CHECK(key != NULL, return FALSE);

	*interval = 0;
	len = strlen(key);
	offset = len - 1;

	crm_trace("Source: %s", key);

	while (offset > 0 && isdigit(key[offset])) {
	int digits = len - offset;

	ch = key[offset] - '0';
	CRM_CHECK(ch < 10, return FALSE);
	CRM_CHECK(ch >= 0, return FALSE);
	while (digits > 1) {
	digits--;
	ch = ch * 10;
	}
	*interval += ch;
	offset--;
	}

	crm_trace(" Interval: %d", *interval);
	CRM_CHECK(key[offset] == '_', return FALSE);

	mutable_key = strdup(key);
	mutable_key[offset] = 0;
	offset--;

	while (offset > 0 && key[offset] != '_') {
	offset--;
	}

	CRM_CHECK(key[offset] == '_', free(mutable_key);
	return FALSE);

	mutable_key_ptr = mutable_key + offset + 1;

	crm_trace(" Action: %s", mutable_key_ptr);

	*op_type = strdup(mutable_key_ptr);

	mutable_key[offset] = 0;
	offset--;

	CRM_CHECK(mutable_key != mutable_key_ptr, free(mutable_key);
	return FALSE);

	notify = strstr(mutable_key, "_post_notify");
	if (notify && safe_str_eq(notify, "_post_notify")) {
	notify[0] = 0;
	}

	notify = strstr(mutable_key, "_pre_notify");
	if (notify && safe_str_eq(notify, "_pre_notify")) {
	notify[0] = 0;
	}

	crm_trace(" Resource: %s", mutable_key);
	*rsc_id = mutable_key;

	return TRUE;
	}

	char *
	generate_notify_key(const char rsc_id, const char notify_type, const char *op_type)
	{
	int len = 12;
	char *op_id = NULL;

	CRM_CHECK(rsc_id != NULL, return NULL);
	CRM_CHECK(op_type != NULL, return NULL);
	CRM_CHECK(notify_type != NULL, return NULL);

	len += strlen(op_type);
	len += strlen(rsc_id);
	len += strlen(notify_type);
	if(len > 0) {
	op_id = malloc(len);
	}
	if (op_id != NULL) {
	sprintf(op_id, "%s_%s_notify_%s_0", rsc_id, notify_type, op_type);
	}
	return op_id;
	}

	char *
	generate_transition_magic_v202(const char *transition_key, int op_status)
	{
	int len = 80;
	char *fail_state = NULL;

	CRM_CHECK(transition_key != NULL, return NULL);

	len += strlen(transition_key);

	fail_state = malloc(len);
	if (fail_state != NULL) {
	snprintf(fail_state, len, "%d:%s", op_status, transition_key);
	}
	return fail_state;
	}

	char *
	generate_transition_magic(const char *transition_key, int op_status, int op_rc)
	{
	int len = 80;
	char *fail_state = NULL;

	CRM_CHECK(transition_key != NULL, return NULL);

	len += strlen(transition_key);

	fail_state = malloc(len);
	if (fail_state != NULL) {
	snprintf(fail_state, len, "%d:%d;%s", op_status, op_rc, transition_key);
	}
	return fail_state;
	}

	gboolean
	decode_transition_magic(const char magic, char uuid, int transition_id, int *action_id,
	int op_status, int op_rc, int *target_rc)
	{
	int res = 0;
	char *key = NULL;
	gboolean result = TRUE;

	CRM_CHECK(magic != NULL, return FALSE);
	CRM_CHECK(op_rc != NULL, return FALSE);
	CRM_CHECK(op_status != NULL, return FALSE);

	key = calloc(1, strlen(magic) + 1);
	res = sscanf(magic, "%d:%d;%s", op_status, op_rc, key);
	if (res != 3) {
	crm_warn("Only found %d items in: '%s'", res, magic);
	free(key);
	return FALSE;
	}

	CRM_CHECK(decode_transition_key(key, uuid, transition_id, action_id, target_rc), result = FALSE);

	free(key);
	return result;
	}

	char *
	generate_transition_key(int transition_id, int action_id, int target_rc, const char *node)
	{
	int len = 40;
	char *fail_state = NULL;

	CRM_CHECK(node != NULL, return NULL);

	len += strlen(node);

	fail_state = malloc(len);
	if (fail_state != NULL) {
	snprintf(fail_state, len, "%d:%d:%d:%-*s", action_id, transition_id, target_rc, 36, node);
	}
	return fail_state;
	}

	gboolean
	decode_transition_key(const char key, char uuid, int transition_id, int *action_id,
	int *target_rc)
	{
	int res = 0;
	gboolean done = FALSE;

	CRM_CHECK(uuid != NULL, return FALSE);
	CRM_CHECK(target_rc != NULL, return FALSE);
	CRM_CHECK(action_id != NULL, return FALSE);
	CRM_CHECK(transition_id != NULL, return FALSE);

	*uuid = calloc(1, 37);
	res = sscanf(key, "%d:%d:%d:%36s", action_id, transition_id, target_rc, *uuid);
	switch (res) {
	case 4:
	/* Post Pacemaker 0.6 */
	done = TRUE;
	break;
	case 3:
	case 2:
	/* this can be tricky - the UUID might start with an integer */

	/* Until Pacemaker 0.6 */
	done = TRUE;
	*target_rc = -1;
	res = sscanf(key, "%d:%d:%36s", action_id, transition_id, *uuid);
	if (res == 2) {
	*action_id = -1;
	res = sscanf(key, "%d:%36s", transition_id, *uuid);
	CRM_CHECK(res == 2, done = FALSE);

	} else if (res != 3) {
	CRM_CHECK(res == 3, done = FALSE);
	}
	break;

	case 1:
	/* Prior to Heartbeat 2.0.8 */
	done = TRUE;
	*action_id = -1;
	*target_rc = -1;
	res = sscanf(key, "%d:%36s", transition_id, *uuid);
	CRM_CHECK(res == 2, done = FALSE);
	break;
	default:
	crm_crit("Unhandled sscanf result (%d) for %s", res, key);
	}

	if (strlen(*uuid) != 36) {
	crm_warn("Bad UUID (%s) in sscanf result (%d) for %s", *uuid, res, key);
	}

	if (done == FALSE) {
	crm_err("Cannot decode '%s' rc=%d", key, res);

	free(*uuid);
	*uuid = NULL;
	*target_rc = -1;
	*action_id = -1;
	*transition_id = -1;
	}

	return done;
	}

	void
	filter_action_parameters(xmlNode * param_set, const char *version)
	{
	char *key = NULL;
	char *timeout = NULL;
	char *interval = NULL;

	const char *attr_filter[] = {
	XML_ATTR_ID,
	XML_ATTR_CRM_VERSION,
	XML_LRM_ATTR_OP_DIGEST,
	};

	gboolean do_delete = FALSE;
	int lpc = 0;
	static int meta_len = 0;

	if (meta_len == 0) {
	meta_len = strlen(CRM_META);
	}

	if (param_set == NULL) {
	return;
	}

	for (lpc = 0; lpc < DIMOF(attr_filter); lpc++) {
	xml_remove_prop(param_set, attr_filter[lpc]);
	}

	key = crm_meta_name(XML_LRM_ATTR_INTERVAL);
	interval = crm_element_value_copy(param_set, key);
	free(key);

	key = crm_meta_name(XML_ATTR_TIMEOUT);
	timeout = crm_element_value_copy(param_set, key);

	if (param_set) {
	xmlAttrPtr xIter = param_set->properties;

	while (xIter) {
	const char prop_name = (const char )xIter->name;

	xIter = xIter->next;
	do_delete = FALSE;
	if (strncasecmp(prop_name, CRM_META, meta_len) == 0) {
	do_delete = TRUE;
	}

	if (do_delete) {
	xml_remove_prop(param_set, prop_name);
	}
	}
	}

	if (crm_get_msec(interval) > 0 && compare_version(version, "1.0.8") > 0) {
	/* Re-instate the operation's timeout value */
	if (timeout != NULL) {
	crm_xml_add(param_set, key, timeout);
	}
	}

	free(interval);
	free(timeout);
	free(key);
	}

	extern bool crm_is_daemon;

	/* coverity[+kill] */
	void
	crm_abort(const char file, const char function, int line,
	const char *assert_condition, gboolean do_core, gboolean do_fork)
	{
	int rc = 0;
	int pid = 0;
	int status = 0;

	/* Implied by the parent's error logging below */
	/* crm_write_blackbox(0); */

	if(crm_is_daemon == FALSE) {
	/* This is a command line tool - do not fork */

	/* crm_add_logfile(NULL); * Record it to a file? */
	crm_enable_stderr(TRUE); /* Make sure stderr is enabled so we can tell the caller */
	do_fork = FALSE; /* Just crash if needed */
	}

	if (do_core == FALSE) {
	crm_err("%s: Triggered assert at %s:%d : %s", function, file, line, assert_condition);
	return;

	} else if (do_fork) {
	pid = fork();

	} else {
	crm_err("%s: Triggered fatal assert at %s:%d : %s", function, file, line, assert_condition);
	}

	if (pid == -1) {
	crm_crit("%s: Cannot create core for non-fatal assert at %s:%d : %s",
	function, file, line, assert_condition);
	return;

	} else if(pid == 0) {
	/* Child process */
	abort();
	return;
	}

	/* Parent process */
	crm_err("%s: Forked child %d to record non-fatal assert at %s:%d : %s",
	function, pid, file, line, assert_condition);
	crm_write_blackbox(SIGTRAP, NULL);

	do {
	rc = waitpid(pid, &status, 0);
	if(rc == pid) {
	return; /* Job done */
	}

	} while(errno == EINTR);

	if (errno == ECHILD) {
	/* crm_mon does this */
	crm_trace("Cannot wait on forked child %d - SIGCHLD is probably set to SIG_IGN", pid);
	return;
	}
	crm_perror(LOG_ERR, "Cannot wait on forked child %d", pid);
	}

	int
	crm_pid_active(long pid, const char *daemon)
	{
	static int have_proc_pid = 0;

	if(have_proc_pid == 0) {
	char proc_path[PATH_MAX], exe_path[PATH_MAX];

	/* check to make sure pid hasn't been reused by another process */
	snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", (long unsigned int)getpid());

	have_proc_pid = 1;
	if(readlink(proc_path, exe_path, PATH_MAX - 1) < 0) {
	have_proc_pid = -1;
	}
	}

	if (pid <= 0) {
	return -1;

	} else if (kill(pid, 0) < 0 && errno == ESRCH) {
	return 0;

	} else if(daemon == NULL \|\| have_proc_pid == -1) {
	return 1;

	} else {
	int rc = 0;
	char proc_path[PATH_MAX], exe_path[PATH_MAX], myexe_path[PATH_MAX];

	/* check to make sure pid hasn't been reused by another process */
	snprintf(proc_path, sizeof(proc_path), "/proc/%lu/exe", pid);

	rc = readlink(proc_path, exe_path, PATH_MAX - 1);
	if (rc < 0) {
	crm_perror(LOG_ERR, "Could not read from %s", proc_path);
	return 0;
	}

	exe_path[rc] = 0;

	if(daemon[0] != '/') {
	rc = snprintf(myexe_path, sizeof(proc_path), CRM_DAEMON_DIR"/%s", daemon);
	myexe_path[rc] = 0;
	} else {
	rc = snprintf(myexe_path, sizeof(proc_path), "%s", daemon);
	myexe_path[rc] = 0;
	}

	if (strcmp(exe_path, myexe_path) == 0) {
	return 1;
	}
	}

	return 0;
	}

	#define LOCKSTRLEN 11

	int
	crm_read_pidfile(const char *filename)
	{
	int fd;
	long pid = -1;
	char buf[LOCKSTRLEN + 1];

	if ((fd = open(filename, O_RDONLY)) < 0) {
	goto bail;
	}

	if (read(fd, buf, sizeof(buf)) < 1) {
	goto bail;
	}

	if (sscanf(buf, "%lu", &pid) > 0) {
	if (pid <= 0) {
	pid = -ESRCH;
	}
	}

	bail:
	if (fd >= 0) {
	close(fd);
	}
	return pid;
	}

	int
	crm_pidfile_inuse(const char filename, long mypid, const char daemon)
	{
	long pid = 0;
	struct stat sbuf;
	char buf[LOCKSTRLEN + 1];
	int rc = -ENOENT, fd = 0;

	if ((fd = open(filename, O_RDONLY)) >= 0) {
	if (fstat(fd, &sbuf) >= 0 && sbuf.st_size < LOCKSTRLEN) {
	sleep(2); /* if someone was about to create one,
	* give'm a sec to do so
	*/
	}
	if (read(fd, buf, sizeof(buf)) > 0) {
	if (sscanf(buf, "%lu", &pid) > 0) {
	crm_trace("Got pid %lu from %s\n", pid, filename);
	if (pid <= 1) {
	/* Invalid pid */
	rc = -ENOENT;
	unlink(filename);

	} else if (mypid && pid == mypid) {
	/* In use by us */
	rc = pcmk_ok;

	} else if (crm_pid_active(pid, daemon) == FALSE) {
	/* Contains a stale value */
	unlink(filename);
	rc = -ENOENT;

	} else if (mypid && pid != mypid) {
	/* locked by existing process - give up */
	rc = -EEXIST;
	}
	}
	}
	close(fd);
	}
	return rc;
	}

	static int
	crm_lock_pidfile(const char filename, const char name)
	{
	long mypid = 0;
	int fd = 0, rc = 0;
	char buf[LOCKSTRLEN + 1];

	mypid = (unsigned long)getpid();

	rc = crm_pidfile_inuse(filename, 0, name);
	if (rc == -ENOENT) {
	/* exists but the process is not active */

	} else if (rc != pcmk_ok) {
	/* locked by existing process - give up */
	return rc;
	}

	if ((fd = open(filename, O_CREAT \| O_WRONLY \| O_EXCL, 0644)) < 0) {
	/* Hmmh, why did we fail? Anyway, nothing we can do about it */
	return -errno;
	}

	snprintf(buf, sizeof(buf), "%*lu\n", LOCKSTRLEN - 1, mypid);
	rc = write(fd, buf, LOCKSTRLEN);
	close(fd);

	if (rc != LOCKSTRLEN) {
	crm_perror(LOG_ERR, "Incomplete write to %s", filename);
	return -errno;
	}

	return crm_pidfile_inuse(filename, mypid, name);
	}

	void
	crm_make_daemon(const char name, gboolean daemonize, const char pidfile)
	{
	int rc;
	long pid;
	const char *devnull = "/dev/null";

	if (daemonize == FALSE) {
	return;
	}

	/* Check before we even try... */
	rc = crm_pidfile_inuse(pidfile, 1, name);
	if(rc < pcmk_ok && rc != -ENOENT) {
	pid = crm_read_pidfile(pidfile);
	crm_err("%s: already running [pid %ld in %s]", name, pid, pidfile);
	printf("%s: already running [pid %ld in %s]\n", name, pid, pidfile);
	crm_exit(rc);
	}

	pid = fork();
	if (pid < 0) {
	fprintf(stderr, "%s: could not start daemon\n", name);
	crm_perror(LOG_ERR, "fork");
	crm_exit(EINVAL);

	} else if (pid > 0) {
	crm_exit(pcmk_ok);
	}

	rc = crm_lock_pidfile(pidfile, name);
	if(rc < pcmk_ok) {
	crm_err("Could not lock '%s' for %s: %s (%d)", pidfile, name, pcmk_strerror(rc), rc);
	printf("Could not lock '%s' for %s: %s (%d)\n", pidfile, name, pcmk_strerror(rc), rc);
	crm_exit(rc);
	}

	umask(S_IWGRP \| S_IWOTH \| S_IROTH);

	close(STDIN_FILENO);
	(void)open(devnull, O_RDONLY); /* Stdin: fd 0 */
	close(STDOUT_FILENO);
	(void)open(devnull, O_WRONLY); /* Stdout: fd 1 */
	close(STDERR_FILENO);
	(void)open(devnull, O_WRONLY); /* Stderr: fd 2 */
	}

	char *
	crm_strip_trailing_newline(char *str)
	{
	int len;

	if (str == NULL) {
	return str;
	}

	for (len = strlen(str) - 1; len >= 0 && str[len] == '\n'; len--) {
	str[len] = '\0';
	}

	return str;
	}

	gboolean
	crm_str_eq(const char a, const char b, gboolean use_case)
	{
	if (use_case) {
	return g_strcmp0(a, b) == 0;

	/* TODO - Figure out which calls, if any, really need to be case independant */
	} else if (a == b) {
	return TRUE;

	} else if (a == NULL \|\| b == NULL) {
	/* shouldn't be comparing NULLs */
	return FALSE;

	} else if (strcasecmp(a, b) == 0) {
	return TRUE;
	}
	return FALSE;
	}

	char *
	crm_meta_name(const char *field)
	{
	int lpc = 0;
	int max = 0;
	char *crm_name = NULL;

	CRM_CHECK(field != NULL, return NULL);
	crm_name = crm_concat(CRM_META, field, '_');

	/* Massage the names so they can be used as shell variables */
	max = strlen(crm_name);
	for (; lpc < max; lpc++) {
	switch (crm_name[lpc]) {
	case '-':
	crm_name[lpc] = '_';
	break;
	}
	}
	return crm_name;
	}

	const char *
	crm_meta_value(GHashTable * hash, const char *field)
	{
	char *key = NULL;
	const char *value = NULL;

	key = crm_meta_name(field);
	if (key) {
	value = g_hash_table_lookup(hash, key);
	free(key);
	}

	return value;
	}

	static struct option *
	crm_create_long_opts(struct crm_option *long_options)
	{
	struct option *long_opts = NULL;

	#ifdef HAVE_GETOPT_H
	int index = 0, lpc = 0;

	/*
	* A previous, possibly poor, choice of '?' as the short form of --help
	* means that getopt_long() returns '?' for both --help and for "unknown option"
	*
	* This dummy entry allows us to differentiate between the two in crm_get_option()
	* and exit with the correct error code
	*/
	long_opts = realloc_safe(long_opts, (index + 1) * sizeof(struct option));
	long_opts[index].name = "__dummmy__";
	long_opts[index].has_arg = 0;
	long_opts[index].flag = 0;
	long_opts[index].val = '_';
	index++;

	for (lpc = 0; long_options[lpc].name != NULL; lpc++) {
	if (long_options[lpc].name[0] == '-') {
	continue;
	}

	long_opts = realloc_safe(long_opts, (index + 1) * sizeof(struct option));
	/*fprintf(stderr, "Creating %d %s = %c\n", index,
	* long_options[lpc].name, long_options[lpc].val); */
	long_opts[index].name = long_options[lpc].name;
	long_opts[index].has_arg = long_options[lpc].has_arg;
	long_opts[index].flag = long_options[lpc].flag;
	long_opts[index].val = long_options[lpc].val;
	index++;
	}

	/* Now create the list terminator */
	long_opts = realloc_safe(long_opts, (index + 1) * sizeof(struct option));
	long_opts[index].name = NULL;
	long_opts[index].has_arg = 0;
	long_opts[index].flag = 0;
	long_opts[index].val = 0;
	#endif

	return long_opts;
	}

	void
	crm_set_options(const char short_options, const char app_usage, struct crm_option *long_options,
	const char *app_desc)
	{
	if (short_options) {
	crm_short_options = strdup(short_options);

	} else if (long_options) {
	int lpc = 0;
	int opt_string_len = 0;
	char *local_short_options = NULL;

	for (lpc = 0; long_options[lpc].name != NULL; lpc++) {
	if (long_options[lpc].val && long_options[lpc].val != '-' && long_options[lpc].val < UCHAR_MAX) {
	local_short_options = realloc_safe(local_short_options, opt_string_len + 4);
	local_short_options[opt_string_len++] = long_options[lpc].val;
	/* getopt(3) says: Two colons mean an option takes an optional arg; */
	if (long_options[lpc].has_arg == optional_argument) {
	local_short_options[opt_string_len++] = ':';
	}
	if (long_options[lpc].has_arg >= required_argument) {
	local_short_options[opt_string_len++] = ':';
	}
	local_short_options[opt_string_len] = 0;
	}
	}
	crm_short_options = local_short_options;
	crm_trace("Generated short option string: '%s'", local_short_options);
	}

	if (long_options) {
	crm_long_options = long_options;
	}
	if (app_desc) {
	crm_app_description = app_desc;
	}
	if (app_usage) {
	crm_app_usage = app_usage;
	}
	}

	int
	crm_get_option(int argc, char *argv, int index)
	{
	return crm_get_option_long(argc, argv, index, NULL);
	}

	int
	crm_get_option_long(int argc, char *argv, int index, const char **longname)
	{
	#ifdef HAVE_GETOPT_H
	static struct option *long_opts = NULL;

	if (long_opts == NULL && crm_long_options) {
	long_opts = crm_create_long_opts(crm_long_options);
	}

	*index = 0;
	if (long_opts) {
	int flag = getopt_long(argc, argv, crm_short_options, long_opts, index);

	switch (flag) {
	case 0:
	if (long_opts[*index].val) {
	return long_opts[*index].val;
	} else if (longname) {
	longname = long_opts[index].name;
	} else {
	crm_notice("Unhandled option --%s", long_opts[*index].name);
	return flag;
	}
	case -1: /* End of option processing */
	break;
	case ':':
	crm_trace("Missing argument");
	crm_help('?', 1);
	break;
	case '?':
	crm_help('?', *index ? 0 : 1);
	break;
	}
	return flag;
	}
	#endif

	if (crm_short_options) {
	return getopt(argc, argv, crm_short_options);
	}

	return -1;
	}

	int
	crm_help(char cmd, int exit_code)
	{
	int i = 0;
	FILE *stream = (exit_code ? stderr : stdout);

	if (cmd == 'v' \|\| cmd == '$') {
	fprintf(stream, "Pacemaker %s\n", PACEMAKER_VERSION);
	fprintf(stream, "Written by Andrew Beekhof\n");
	goto out;
	}

	if (cmd == '!') {
	fprintf(stream, "Pacemaker %s (Build: %s): %s\n", PACEMAKER_VERSION, BUILD_VERSION, CRM_FEATURES);
	goto out;
	}

	fprintf(stream, "%s - %s\n", crm_system_name, crm_app_description);

	if (crm_app_usage) {
	fprintf(stream, "Usage: %s %s\n", crm_system_name, crm_app_usage);
	}

	if (crm_long_options) {
	fprintf(stream, "Options:\n");
	for (i = 0; crm_long_options[i].name != NULL; i++) {
	if (crm_long_options[i].flags & pcmk_option_hidden) {

	} else if (crm_long_options[i].flags & pcmk_option_paragraph) {
	fprintf(stream, "%s\n\n", crm_long_options[i].desc);

	} else if (crm_long_options[i].flags & pcmk_option_example) {
	fprintf(stream, "\t#%s\n\n", crm_long_options[i].desc);

	} else if (crm_long_options[i].val == '-' && crm_long_options[i].desc) {
	fprintf(stream, "%s\n", crm_long_options[i].desc);

	} else {
	/* is val printable as char ? */
	if (crm_long_options[i].val && crm_long_options[i].val <= UCHAR_MAX) {
	fprintf(stream, " -%c,", crm_long_options[i].val);
	} else {
	fputs(" ", stream);
	}
	fprintf(stream, " --%s%s\t%s\n", crm_long_options[i].name,
	crm_long_options[i].has_arg == optional_argument ? "[=value]" :
	crm_long_options[i].has_arg == required_argument ? "=value" : "",
	crm_long_options[i].desc ? crm_long_options[i].desc : "");
	}
	}

	} else if (crm_short_options) {
	fprintf(stream, "Usage: %s - %s\n", crm_system_name, crm_app_description);
	for (i = 0; crm_short_options[i] != 0; i++) {
	int has_arg = no_argument /* 0 */;

	if (crm_short_options[i + 1] == ':') {
	if (crm_short_options[i + 2] == ':')
	has_arg = optional_argument /* 2 */;
	else
	has_arg = required_argument /* 1 */;
	}

	fprintf(stream, " -%c %s\n", crm_short_options[i],
	has_arg == optional_argument ? "[value]" :
	has_arg == required_argument ? "{value}" : "");
	i += has_arg;
	}
	}

	fprintf(stream, "\nReport bugs to %s\n", PACKAGE_BUGREPORT);

	out:
	return crm_exit(exit_code);
	}

	void cib_ipc_servers_init(qb_ipcs_service_t **ipcs_ro,
	qb_ipcs_service_t **ipcs_rw,
	qb_ipcs_service_t **ipcs_shm,
	struct qb_ipcs_service_handlers *ro_cb,
	struct qb_ipcs_service_handlers *rw_cb)
	{
	*ipcs_ro = mainloop_add_ipc_server(cib_channel_ro, QB_IPC_NATIVE, ro_cb);
	*ipcs_rw = mainloop_add_ipc_server(cib_channel_rw, QB_IPC_NATIVE, rw_cb);
	*ipcs_shm = mainloop_add_ipc_server(cib_channel_shm, QB_IPC_SHM, rw_cb);

	if (ipcs_ro == NULL \|\| ipcs_rw == NULL \|\| *ipcs_shm == NULL) {
	crm_err("Failed to create cib servers: exiting and inhibiting respawn.");
	crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	}

	void cib_ipc_servers_destroy(qb_ipcs_service_t *ipcs_ro,
	qb_ipcs_service_t *ipcs_rw,
	qb_ipcs_service_t *ipcs_shm)
	{
	qb_ipcs_destroy(ipcs_ro);
	qb_ipcs_destroy(ipcs_rw);
	qb_ipcs_destroy(ipcs_shm);
	}

	qb_ipcs_service_t *
	crmd_ipc_server_init(struct qb_ipcs_service_handlers *cb)
	{
	return mainloop_add_ipc_server(CRM_SYSTEM_CRMD, QB_IPC_NATIVE, cb);
	}

	void
	attrd_ipc_server_init(qb_ipcs_service_t *ipcs, struct qb_ipcs_service_handlers cb)
	{
	*ipcs = mainloop_add_ipc_server(T_ATTRD, QB_IPC_NATIVE, cb);

	if (*ipcs == NULL) {
	crm_err("Failed to create attrd servers: exiting and inhibiting respawn.");
	crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	}

	void
	stonith_ipc_server_init(qb_ipcs_service_t *ipcs, struct qb_ipcs_service_handlers cb)
	{
	*ipcs = mainloop_add_ipc_server("stonith-ng", QB_IPC_NATIVE, cb);

	if (*ipcs == NULL) {
	crm_err("Failed to create stonith-ng servers: exiting and inhibiting respawn.");
	crm_warn("Verify pacemaker and pacemaker_remote are not both enabled.");
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	}

	int
	attrd_update_delegate(crm_ipc_t * ipc, char command, const char host, const char name,
	const char value, const char section, const char set, const char dampen,
	const char *user_name, int options)
	{
	int rc = -ENOTCONN;
	int max = 5;
	xmlNode *update = create_xml_node(NULL, __FUNCTION__);

	static gboolean connected = TRUE;
	static crm_ipc_t *local_ipc = NULL;
	static enum crm_ipc_flags flags = crm_ipc_flags_none;

	if (ipc == NULL && local_ipc == NULL) {
	local_ipc = crm_ipc_new(T_ATTRD, 0);
	flags \|= crm_ipc_client_response;
	connected = FALSE;
	}

	if (ipc == NULL) {
	ipc = local_ipc;
	}

	/* remap common aliases */
	if (safe_str_eq(section, "reboot")) {
	section = XML_CIB_TAG_STATUS;

	} else if (safe_str_eq(section, "forever")) {
	section = XML_CIB_TAG_NODES;
	}

	crm_xml_add(update, F_TYPE, T_ATTRD);
	crm_xml_add(update, F_ORIG, crm_system_name);

	if (name == NULL && command == 'U') {
	command = 'R';
	}

	switch (command) {
	case 'u':
	crm_xml_add(update, F_ATTRD_TASK, ATTRD_OP_UPDATE);
	crm_xml_add(update, F_ATTRD_REGEX, name);
	break;
	case 'D':
	case 'U':
	case 'v':
	crm_xml_add(update, F_ATTRD_TASK, ATTRD_OP_UPDATE);
	crm_xml_add(update, F_ATTRD_ATTRIBUTE, name);
	break;
	case 'R':
	crm_xml_add(update, F_ATTRD_TASK, ATTRD_OP_REFRESH);
	break;
	case 'Q':
	crm_xml_add(update, F_ATTRD_TASK, ATTRD_OP_QUERY);
	crm_xml_add(update, F_ATTRD_ATTRIBUTE, name);
	break;
	case 'C':
	crm_xml_add(update, F_ATTRD_TASK, ATTRD_OP_PEER_REMOVE);
	break;
	}

	crm_xml_add(update, F_ATTRD_VALUE, value);
	crm_xml_add(update, F_ATTRD_DAMPEN, dampen);
	crm_xml_add(update, F_ATTRD_SECTION, section);
	crm_xml_add(update, F_ATTRD_HOST, host);
	crm_xml_add(update, F_ATTRD_SET, set);
	crm_xml_add_int(update, F_ATTRD_IS_REMOTE, is_set(options, attrd_opt_remote));
	crm_xml_add_int(update, F_ATTRD_IS_PRIVATE, is_set(options, attrd_opt_private));
	#if ENABLE_ACL
	if (user_name) {
	crm_xml_add(update, F_ATTRD_USER, user_name);
	}
	#endif

	while (max > 0) {
	if (connected == FALSE) {
	crm_info("Connecting to cluster... %d retries remaining", max);
	connected = crm_ipc_connect(ipc);
	}

	if (connected) {
	rc = crm_ipc_send(ipc, update, flags, 0, NULL);
	} else {
	crm_perror(LOG_INFO, "Connection to cluster attribute manager failed");
	}

	if (ipc != local_ipc) {
	break;

	} else if (rc > 0) {
	break;

	} else if (rc == -EAGAIN \|\| rc == -EALREADY) {
	sleep(5 - max);
	max--;

	} else {
	crm_ipc_close(ipc);
	connected = FALSE;
	sleep(5 - max);
	max--;
	}
	}

	free_xml(update);
	if (rc > 0) {
	crm_debug("Sent update: %s=%s for %s", name, value, host ? host : "localhost");
	rc = pcmk_ok;

	} else {
	crm_debug("Could not send update %s=%s for %s: %s (%d)", name, value,
	host ? host : "localhost", pcmk_strerror(rc), rc);
	}
	return rc;
	}

	#define FAKE_TE_ID "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
	static void
	append_digest(lrmd_event_data_t * op, xmlNode * update, const char version, const char magic,
	int level)
	{
	/* this will enable us to later determine that the
	* resource's parameters have changed and we should force
	* a restart
	*/
	char *digest = NULL;
	xmlNode *args_xml = NULL;

	if (op->params == NULL) {
	return;
	}

	args_xml = create_xml_node(NULL, XML_TAG_PARAMS);
	g_hash_table_foreach(op->params, hash2field, args_xml);
	filter_action_parameters(args_xml, version);
	digest = calculate_operation_digest(args_xml, version);

	#if 0
	if (level < get_crm_log_level()
	&& op->interval == 0 && crm_str_eq(op->op_type, CRMD_ACTION_START, TRUE)) {
	char *digest_source = dump_xml_unformatted(args_xml);

	do_crm_log(level, "Calculated digest %s for %s (%s). Source: %s\n",
	digest, ID(update), magic, digest_source);
	free(digest_source);
	}
	#endif
	crm_xml_add(update, XML_LRM_ATTR_OP_DIGEST, digest);

	free_xml(args_xml);
	free(digest);
	}

	int
	rsc_op_expected_rc(lrmd_event_data_t * op)
	{
	int rc = 0;

	if (op && op->user_data) {
	int dummy = 0;
	char *uuid = NULL;

	decode_transition_key(op->user_data, &uuid, &dummy, &dummy, &rc);
	free(uuid);
	}
	return rc;
	}

	gboolean
	did_rsc_op_fail(lrmd_event_data_t * op, int target_rc)
	{
	switch (op->op_status) {
	case PCMK_LRM_OP_CANCELLED:
	case PCMK_LRM_OP_PENDING:
	return FALSE;
	break;

	case PCMK_LRM_OP_NOTSUPPORTED:
	case PCMK_LRM_OP_TIMEOUT:
	case PCMK_LRM_OP_ERROR:
	return TRUE;
	break;

	default:
	if (target_rc != op->rc) {
	return TRUE;
	}
	}

	return FALSE;
	}

	xmlNode *
	create_operation_update(xmlNode * parent, lrmd_event_data_t * op, const char * caller_version,
	int target_rc, const char * node, const char * origin, int level)
	{
	char *key = NULL;
	char *magic = NULL;
	char *op_id = NULL;
	char *op_id_additional = NULL;
	char *local_user_data = NULL;
	const char *exit_reason = NULL;

	xmlNode *xml_op = NULL;
	const char *task = NULL;
	gboolean dc_munges_migrate_ops = (compare_version(caller_version, "3.0.3") < 0);
	gboolean dc_needs_unique_ops = (compare_version(caller_version, "3.0.6") < 0);

	CRM_CHECK(op != NULL, return NULL);
	do_crm_log(level, "%s: Updating resource %s after %s op %s (interval=%d)",
	origin, op->rsc_id, op->op_type, services_lrm_status_str(op->op_status),
	op->interval);

	crm_trace("DC version: %s", caller_version);

	task = op->op_type;
	/* remap the task name under various scenarios
	- * this makes life easier for the PE when its trying determin the current state
	+ * this makes life easier for the PE when trying determine the current state
	*/
	if (crm_str_eq(task, "reload", TRUE)) {
	if (op->op_status == PCMK_LRM_OP_DONE) {
	task = CRMD_ACTION_START;
	} else {
	task = CRMD_ACTION_STATUS;
	}

	} else if (dc_munges_migrate_ops && crm_str_eq(task, CRMD_ACTION_MIGRATE, TRUE)) {
	/* if the migrate_from fails it will have enough info to do the right thing */
	if (op->op_status == PCMK_LRM_OP_DONE) {
	task = CRMD_ACTION_STOP;
	} else {
	task = CRMD_ACTION_STATUS;
	}

	} else if (dc_munges_migrate_ops
	&& op->op_status == PCMK_LRM_OP_DONE
	&& crm_str_eq(task, CRMD_ACTION_MIGRATED, TRUE)) {
	task = CRMD_ACTION_START;
	}

	key = generate_op_key(op->rsc_id, task, op->interval);
	if (dc_needs_unique_ops && op->interval > 0) {
	op_id = strdup(key);

	} else if (crm_str_eq(task, CRMD_ACTION_NOTIFY, TRUE)) {
	const char *n_type = crm_meta_value(op->params, "notify_type");
	const char *n_task = crm_meta_value(op->params, "notify_operation");

	CRM_LOG_ASSERT(n_type != NULL);
	CRM_LOG_ASSERT(n_task != NULL);
	op_id = generate_notify_key(op->rsc_id, n_type, n_task);

	/* these are not yet allowed to fail */
	op->op_status = PCMK_LRM_OP_DONE;
	op->rc = 0;

	} else if (did_rsc_op_fail(op, target_rc)) {
	op_id = generate_op_key(op->rsc_id, "last_failure", 0);
	if (op->interval == 0) {
	/* Ensure 'last' gets updated too in case recording-pending="true" */
	op_id_additional = generate_op_key(op->rsc_id, "last", 0);
	}
	exit_reason = op->exit_reason;

	} else if (op->interval > 0) {
	op_id = strdup(key);

	} else {
	op_id = generate_op_key(op->rsc_id, "last", 0);
	}

	again:
	xml_op = find_entity(parent, XML_LRM_TAG_RSC_OP, op_id);
	if (xml_op == NULL) {
	xml_op = create_xml_node(parent, XML_LRM_TAG_RSC_OP);
	}

	if (op->user_data == NULL) {
	crm_debug("Generating fake transition key for:"
	" %s_%s_%d %d from %s",
	op->rsc_id, op->op_type, op->interval, op->call_id, origin);
	local_user_data = generate_transition_key(-1, op->call_id, target_rc, FAKE_TE_ID);
	op->user_data = local_user_data;
	}

	if(magic == NULL) {
	magic = generate_transition_magic(op->user_data, op->op_status, op->rc);
	}

	crm_xml_add(xml_op, XML_ATTR_ID, op_id);
	crm_xml_add(xml_op, XML_LRM_ATTR_TASK_KEY, key);
	crm_xml_add(xml_op, XML_LRM_ATTR_TASK, task);
	crm_xml_add(xml_op, XML_ATTR_ORIGIN, origin);
	crm_xml_add(xml_op, XML_ATTR_CRM_VERSION, caller_version);
	crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY, op->user_data);
	crm_xml_add(xml_op, XML_ATTR_TRANSITION_MAGIC, magic);
	crm_xml_add(xml_op, XML_LRM_ATTR_EXIT_REASON, exit_reason);
	crm_xml_add(xml_op, XML_LRM_ATTR_TARGET, node); /* For context during triage */

	crm_xml_add_int(xml_op, XML_LRM_ATTR_CALLID, op->call_id);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_RC, op->rc);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS, op->op_status);
	crm_xml_add_int(xml_op, XML_LRM_ATTR_INTERVAL, op->interval);

	if (compare_version("2.1", caller_version) <= 0) {
	if (op->t_run \|\| op->t_rcchange \|\| op->exec_time \|\| op->queue_time) {
	crm_trace("Timing data (%s_%s_%d): last=%lu change=%lu exec=%lu queue=%lu",
	op->rsc_id, op->op_type, op->interval,
	op->t_run, op->t_rcchange, op->exec_time, op->queue_time);

	if (op->interval == 0) {
	/* The values are the same for non-recurring ops */
	crm_xml_add_int(xml_op, XML_RSC_OP_LAST_RUN, op->t_run);
	crm_xml_add_int(xml_op, XML_RSC_OP_LAST_CHANGE, op->t_run);

	} else if(op->t_rcchange) {
	/* last-run is not accurate for recurring ops */
	crm_xml_add_int(xml_op, XML_RSC_OP_LAST_CHANGE, op->t_rcchange);

	} else {
	/* ...but is better than nothing otherwise */
	crm_xml_add_int(xml_op, XML_RSC_OP_LAST_CHANGE, op->t_run);
	}

	crm_xml_add_int(xml_op, XML_RSC_OP_T_EXEC, op->exec_time);
	crm_xml_add_int(xml_op, XML_RSC_OP_T_QUEUE, op->queue_time);
	}
	}

	if (crm_str_eq(op->op_type, CRMD_ACTION_MIGRATE, TRUE)
	\|\| crm_str_eq(op->op_type, CRMD_ACTION_MIGRATED, TRUE)) {
	/*
	* Record migrate_source and migrate_target always for migrate ops.
	*/
	const char *name = XML_LRM_ATTR_MIGRATE_SOURCE;

	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));

	name = XML_LRM_ATTR_MIGRATE_TARGET;
	crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
	}

	append_digest(op, xml_op, caller_version, magic, LOG_DEBUG);

	if (op_id_additional) {
	free(op_id);
	op_id = op_id_additional;
	op_id_additional = NULL;
	goto again;
	}

	if (local_user_data) {
	free(local_user_data);
	op->user_data = NULL;
	}
	free(magic);
	free(op_id);
	free(key);
	return xml_op;
	}

	bool
	pcmk_acl_required(const char *user)
	{
	#if ENABLE_ACL
	if(user == NULL \|\| strlen(user) == 0) {
	crm_trace("no user set");
	return FALSE;

	} else if (strcmp(user, CRM_DAEMON_USER) == 0) {
	return FALSE;

	} else if (strcmp(user, "root") == 0) {
	return FALSE;
	}
	crm_trace("acls required for %s", user);
	return TRUE;
	#else
	crm_trace("acls not supported");
	return FALSE;
	#endif
	}

	#if ENABLE_ACL
	char *
	uid2username(uid_t uid)
	{
	struct passwd *pwent = getpwuid(uid);

	if (pwent == NULL) {
	crm_perror(LOG_ERR, "Cannot get password entry of uid: %d", uid);
	return NULL;

	} else {
	return strdup(pwent->pw_name);
	}
	}

	const char *
	crm_acl_get_set_user(xmlNode * request, const char field, const char peer_user)
	{
	/* field is only checked for backwards compatibility */
	static const char *effective_user = NULL;
	const char *requested_user = NULL;
	const char *user = NULL;

	if(effective_user == NULL) {
	effective_user = uid2username(geteuid());
	}

	requested_user = crm_element_value(request, XML_ACL_TAG_USER);
	if(requested_user == NULL) {
	requested_user = crm_element_value(request, field);
	}

	if (is_privileged(effective_user) == FALSE) {
	/* We're not running as a privileged user, set or overwrite any existing value for $XML_ACL_TAG_USER */
	user = effective_user;

	} else if(peer_user == NULL && requested_user == NULL) {
	/* No user known or requested, use 'effective_user' and make sure one is set for the request */
	user = effective_user;

	} else if(peer_user == NULL) {
	/* No user known, trusting 'requested_user' */
	user = requested_user;

	} else if (is_privileged(peer_user) == FALSE) {
	/* The peer is not a privileged user, set or overwrite any existing value for $XML_ACL_TAG_USER */
	user = peer_user;

	} else if (requested_user == NULL) {
	/* Even if we're privileged, make sure there is always a value set */
	user = peer_user;

	} else {
	/* Legal delegation to 'requested_user' */
	user = requested_user;
	}

	/* Yes, pointer comparision */
	if(user != crm_element_value(request, XML_ACL_TAG_USER)) {
	crm_xml_add(request, XML_ACL_TAG_USER, user);
	}

	if(field != NULL && user != crm_element_value(request, field)) {
	crm_xml_add(request, field, user);
	}

	return requested_user;
	}

	void
	determine_request_user(const char user, xmlNode request, const char *field)
	{
	/* Get our internal validation out of the way first */
	CRM_CHECK(user != NULL && request != NULL && field != NULL, return);

	/* If our peer is a privileged user, we might be doing something on behalf of someone else */
	if (is_privileged(user) == FALSE) {
	/* We're not a privileged user, set or overwrite any existing value for $field */
	crm_xml_replace(request, field, user);

	} else if (crm_element_value(request, field) == NULL) {
	/* Even if we're privileged, make sure there is always a value set */
	crm_xml_replace(request, field, user);

	/* } else { Legal delegation */
	}

	crm_trace("Processing msg as user '%s'", crm_element_value(request, field));
	}
	#endif

	/*
	* This re-implements g_str_hash as it was prior to glib2-2.28:
	*
	* http://git.gnome.org/browse/glib/commit/?id=354d655ba8a54b754cb5a3efb42767327775696c
	*
	* Note that the new g_str_hash is presumably a better hash (it's actually
	* a correct implementation of DJB's hash), but we need to preserve existing
	* behaviour, because the hash key ultimately determines the "sort" order
	* when iterating through GHashTables, which affects allocation of scores to
	* clone instances when iterating through rsc->allowed_nodes. It (somehow)
	* also appears to have some minor impact on the ordering of a few
	* pseudo_event IDs in the transition graph.
	*/
	guint
	g_str_hash_traditional(gconstpointer v)
	{
	const signed char *p;
	guint32 h = 0;

	for (p = v; *p != '\0'; p++)
	h = (h << 5) - h + *p;

	return h;
	}

	guint
	crm_strcase_hash(gconstpointer v)
	{
	const signed char *p;
	guint32 h = 0;

	for (p = v; *p != '\0'; p++)
	h = (h << 5) - h + g_ascii_tolower(*p);

	return h;
	}

	void *
	find_library_function(void *handle, const char lib, const char *fn, gboolean fatal)
	{
	char *error;
	void *a_function;

	if (*handle == NULL) {
	*handle = dlopen(lib, RTLD_LAZY);
	}

	if (!(*handle)) {
	crm_err("%sCould not open %s: %s", fatal ? "Fatal: " : "", lib, dlerror());
	if (fatal) {
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	return NULL;
	}

	a_function = dlsym(*handle, fn);
	if ((error = dlerror()) != NULL) {
	crm_err("%sCould not find %s in %s: %s", fatal ? "Fatal: " : "", fn, lib, error);
	if (fatal) {
	crm_exit(DAEMON_RESPAWN_STOP);
	}
	}

	return a_function;
	}

	char *
	add_list_element(char list, const char value)
	{
	int len = 0;
	int last = 0;

	if (value == NULL) {
	return list;
	}
	if (list) {
	last = strlen(list);
	}
	len = last + 2; /* +1 space, +1 EOS */
	len += strlen(value);
	list = realloc_safe(list, len);
	sprintf(list + last, " %s", value);
	return list;
	}

	void *
	convert_const_pointer(const void *ptr)
	{
	/* Worst function ever */
	return (void *)ptr;
	}

	#ifdef HAVE_UUID_UUID_H
	# include <uuid/uuid.h>
	#endif

	char *
	crm_generate_uuid(void)
	{
	unsigned char uuid[16];
	char buffer = malloc(37); / Including NUL byte */

	uuid_generate(uuid);
	uuid_unparse(uuid, buffer);
	return buffer;
	}

	#include <md5.h>

	char *
	crm_md5sum(const char *buffer)
	{
	int lpc = 0, len = 0;
	char *digest = NULL;
	unsigned char raw_digest[MD5_DIGEST_SIZE];

	if (buffer == NULL) {
	buffer = "";
	}
	len = strlen(buffer);

	crm_trace("Beginning digest of %d bytes", len);
	digest = malloc(2 * MD5_DIGEST_SIZE + 1);
	if(digest) {
	md5_buffer(buffer, len, raw_digest);
	for (lpc = 0; lpc < MD5_DIGEST_SIZE; lpc++) {
	sprintf(digest + (2 * lpc), "%02x", raw_digest[lpc]);
	}
	digest[(2 * MD5_DIGEST_SIZE)] = 0;
	crm_trace("Digest %s.", digest);

	} else {
	crm_err("Could not create digest");
	}
	return digest;
	}

	#include <time.h>
	#include <bzlib.h>

	bool
	crm_compress_string(const char data, int length, int max, char result, unsigned int result_len)
	{
	int rc;
	char *compressed = NULL;
	char *uncompressed = strdup(data);
	struct timespec after_t;
	struct timespec before_t;

	if(max == 0) {
	max = (length * 1.1) + 600; /* recomended size */
	}

	#ifdef CLOCK_MONOTONIC
	clock_gettime(CLOCK_MONOTONIC, &before_t);
	#endif

	/* coverity[returned_null] Ignore */
	compressed = malloc(max);

	*result_len = max;
	rc = BZ2_bzBuffToBuffCompress(compressed, result_len, uncompressed, length, CRM_BZ2_BLOCKS, 0,
	CRM_BZ2_WORK);

	free(uncompressed);

	if (rc != BZ_OK) {
	crm_err("Compression of %d bytes failed: %s (%d)", length, bz2_strerror(rc), rc);
	free(compressed);
	return FALSE;
	}

	#ifdef CLOCK_MONOTONIC
	clock_gettime(CLOCK_MONOTONIC, &after_t);

	crm_info("Compressed %d bytes into %d (ratio %d:1) in %dms",
	length, result_len, length / (result_len),
	(after_t.tv_sec - before_t.tv_sec) * 1000 + (after_t.tv_nsec -
	before_t.tv_nsec) / 1000000);
	#else
	crm_info("Compressed %d bytes into %d (ratio %d:1)",
	length, result_len, length / (result_len));
	#endif

	*result = compressed;
	return TRUE;
	}

	#ifdef HAVE_GNUTLS_GNUTLS_H
	void
	crm_gnutls_global_init(void)
	{
	signal(SIGPIPE, SIG_IGN);
	gnutls_global_init();
	}
	#endif

	diff --git a/lib/common/xml.c b/lib/common/xml.c
	index 048a1293ab..bebfcef20b 100644
	--- a/lib/common/xml.c
	+++ b/lib/common/xml.c
	@@ -1,5859 +1,5859 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>
	#include <sys/param.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <sys/stat.h>
	#include <unistd.h>
	#include <time.h>
	#include <string.h>
	#include <dirent.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <ctype.h>
	#include <math.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <libxml/xmlreader.h>

	#if HAVE_BZLIB_H
	# include <bzlib.h>
	#endif

	#if HAVE_LIBXML2
	# include <libxml/parser.h>
	# include <libxml/tree.h>
	# include <libxml/relaxng.h>
	#endif

	#if HAVE_LIBXSLT
	# include <libxslt/xslt.h>
	# include <libxslt/transform.h>
	#endif

	#define XML_BUFFER_SIZE 4096
	#define XML_PARSER_DEBUG 0

	void
	xml_log(int priority, const char *fmt, ...)
	G_GNUC_PRINTF(2, 3);
	static inline int
	__get_prefix(const char prefix, xmlNode xml, char *buffer, int offset);

	void
	xml_log(int priority, const char *fmt, ...)
	{
	va_list ap;

	va_start(ap, fmt);
	qb_log_from_external_source_va(__FUNCTION__, __FILE__, fmt, priority, __LINE__, 0, ap);
	va_end(ap);
	}

	typedef struct {
	xmlRelaxNGPtr rng;
	xmlRelaxNGValidCtxtPtr valid;
	xmlRelaxNGParserCtxtPtr parser;
	} relaxng_ctx_cache_t;

	struct schema_s {
	int type;
	float version;
	char *name;
	char *location;
	char *transform;
	int after_transform;
	void *cache;
	};

	typedef struct {
	int found;
	const char *string;
	} filter_t;

	enum xml_private_flags {
	xpf_none = 0x0000,
	xpf_dirty = 0x0001,
	xpf_deleted = 0x0002,
	xpf_created = 0x0004,
	xpf_modified = 0x0008,

	xpf_tracking = 0x0010,
	xpf_processed = 0x0020,
	xpf_skip = 0x0040,
	xpf_moved = 0x0080,

	xpf_acl_enabled = 0x0100,
	xpf_acl_read = 0x0200,
	xpf_acl_write = 0x0400,
	xpf_acl_deny = 0x0800,

	xpf_acl_create = 0x1000,
	xpf_acl_denied = 0x2000,
	};

	typedef struct xml_private_s
	{
	long check;
	uint32_t flags;
	char *user;
	GListPtr acls;
	GListPtr deleted_paths;
	} xml_private_t;

	typedef struct xml_acl_s {
	enum xml_private_flags mode;
	char *xpath;
	} xml_acl_t;

	/* INDENT-OFF */

	static filter_t filter[] = {
	{ 0, XML_ATTR_ORIGIN },
	{ 0, XML_CIB_ATTR_WRITTEN },
	{ 0, XML_ATTR_UPDATE_ORIG },
	{ 0, XML_ATTR_UPDATE_CLIENT },
	{ 0, XML_ATTR_UPDATE_USER },
	};
	/* INDENT-ON */

	static struct schema_s *known_schemas = NULL;
	static int xml_schema_max = 0;

	static xmlNode subtract_xml_comment(xmlNode parent, xmlNode * left, xmlNode * right, gboolean * changed);
	static xmlNode find_xml_comment(xmlNode root, xmlNode * search_comment);
	static int add_xml_comment(xmlNode * parent, xmlNode * target, xmlNode * update);
	static bool __xml_acl_check(xmlNode xml, const char name, enum xml_private_flags mode);
	const char *__xml_acl_to_text(enum xml_private_flags flags);

	static int
	xml_latest_schema_index(void)
	{
	return xml_schema_max - 4;
	}

	static int
	xml_minimum_schema_index(void)
	{
	static int best = 0;
	if(best == 0) {
	int lpc = 0;
	float target = 0.0;

	best = xml_latest_schema_index();
	target = floor(known_schemas[best].version);

	for(lpc = best; lpc > 0; lpc--) {
	if(known_schemas[lpc].version < target) {
	return best;
	} else {
	best = lpc;
	}
	}
	best = xml_latest_schema_index();
	}
	return best;
	}

	const char *
	xml_latest_schema(void)
	{
	return get_schema_name(xml_latest_schema_index());
	}

	#define CHUNK_SIZE 1024
	static inline bool TRACKING_CHANGES(xmlNode *xml)
	{
	if(xml == NULL \|\| xml->doc == NULL \|\| xml->doc->_private == NULL) {
	return FALSE;
	} else if(is_set(((xml_private_t *)xml->doc->_private)->flags, xpf_tracking)) {
	return TRUE;
	}
	return FALSE;
	}

	#define buffer_print(buffer, max, offset, fmt, args...) do { \
	int rc = (max); \
	if(buffer) { \
	rc = snprintf((buffer) + (offset), (max) - (offset), fmt, ##args); \
	} \
	if(buffer && rc < 0) { \
	crm_perror(LOG_ERR, "snprintf failed at offset %d", offset); \
	(buffer)[(offset)] = 0; \
	} else if(rc >= ((max) - (offset))) { \
	char *tmp = NULL; \
	(max) = QB_MAX(CHUNK_SIZE, (max) * 2); \
	tmp = realloc_safe((buffer), (max) + 1); \
	CRM_ASSERT(tmp); \
	(buffer) = tmp; \
	} else { \
	offset += rc; \
	break; \
	} \
	} while(1);

	static void
	insert_prefix(int options, char *buffer, int offset, int *max, int depth)
	{
	if (options & xml_log_option_formatted) {
	size_t spaces = 2 * depth;

	if ((buffer) == NULL \|\| spaces >= ((max) - (*offset))) {
	(max) = QB_MAX(CHUNK_SIZE, (max) * 2);
	(buffer) = realloc_safe((buffer), (*max) + 1);
	}
	memset((buffer) + (offset), ' ', spaces);
	(*offset) += spaces;
	}
	}

	static const char *
	get_schema_root(void)
	{
	static const char *base = NULL;

	if (base == NULL) {
	base = getenv("PCMK_schema_directory");
	}
	if (base == NULL \|\| strlen(base) == 0) {
	base = CRM_DTD_DIRECTORY;
	}
	return base;
	}

	static char *
	get_schema_path(const char name, const char file)
	{
	const char *base = get_schema_root();

	if(file) {
	return crm_strdup_printf("%s/%s", base, file);
	}
	return crm_strdup_printf("%s/%s.rng", base, name);
	}

	static int schema_filter(const struct dirent * a)
	{
	int rc = 0;
	float version = 0;

	if(strstr(a->d_name, "pacemaker-") != a->d_name) {
	/* crm_trace("%s - wrong prefix", a->d_name); */

	} else if(strstr(a->d_name, ".rng") == NULL) {
	/* crm_trace("%s - wrong suffix", a->d_name); */

	} else if(sscanf(a->d_name, "pacemaker-%f.rng", &version) == 0) {
	/* crm_trace("%s - wrong format", a->d_name); */

	} else if(strcmp(a->d_name, "pacemaker-1.1.rng") == 0) {
	/* crm_trace("%s - hack", a->d_name); */

	} else {
	/* crm_debug("%s - candidate", a->d_name); */
	rc = 1;
	}

	return rc;
	}

	static int schema_sort(const struct dirent a, const struct dirent b)
	{
	int rc = 0;
	float a_version = 0.0;
	float b_version = 0.0;

	sscanf(a[0]->d_name, "pacemaker-%f.rng", &a_version);
	sscanf(b[0]->d_name, "pacemaker-%f.rng", &b_version);

	if(a_version > b_version) {
	rc = 1;
	} else if(a_version < b_version) {
	rc = -1;
	}

	/* crm_trace("%s (%f) vs. %s (%f) : %d", a[0]->d_name, a_version, b[0]->d_name, b_version, rc); */
	return rc;
	}

	static void __xml_schema_add(
	int type, float version, const char name, const char location, const char *transform, int after_transform)
	{
	int last = xml_schema_max;

	xml_schema_max++;
	known_schemas = realloc_safe(known_schemas, xml_schema_max*sizeof(struct schema_s));
	CRM_ASSERT(known_schemas != NULL);
	memset(known_schemas+last, 0, sizeof(struct schema_s));
	known_schemas[last].type = type;
	known_schemas[last].after_transform = after_transform;

	if(version > 0.0) {
	known_schemas[last].version = version;
	known_schemas[last].name = crm_strdup_printf("pacemaker-%.1f", version);
	known_schemas[last].location = crm_strdup_printf("%s.rng", known_schemas[last].name);

	} else {
	char dummy[1024];
	CRM_ASSERT(name);
	CRM_ASSERT(location);
	sscanf(name, "%[^-]-%f", dummy, &version);
	known_schemas[last].version = version;
	known_schemas[last].name = strdup(name);
	known_schemas[last].location = strdup(location);
	}

	if(transform) {
	known_schemas[last].transform = strdup(transform);
	}
	if(after_transform == 0) {
	after_transform = xml_schema_max;
	}
	known_schemas[last].after_transform = after_transform;

	if(known_schemas[last].after_transform < 0) {
	crm_debug("Added supported schema %d: %s (%s)",
	last, known_schemas[last].name, known_schemas[last].location);

	} else if(known_schemas[last].transform) {
	crm_debug("Added supported schema %d: %s (%s upgrades to %d with %s)",
	last, known_schemas[last].name, known_schemas[last].location,
	known_schemas[last].after_transform,
	known_schemas[last].transform);

	} else {
	crm_debug("Added supported schema %d: %s (%s upgrades to %d)",
	last, known_schemas[last].name, known_schemas[last].location,
	known_schemas[last].after_transform);
	}
	}


	static int __xml_build_schema_list(void)
	{
	int lpc, max;
	const char *base = get_schema_root();
	struct dirent **namelist = NULL;

	max = scandir(base, &namelist, schema_filter, schema_sort);
	__xml_schema_add(1, 0.0, "pacemaker-0.6", "crm.dtd", "upgrade06.xsl", 3);
	__xml_schema_add(1, 0.0, "transitional-0.6", "crm-transitional.dtd", "upgrade06.xsl", 3);
	__xml_schema_add(2, 0.0, "pacemaker-0.7", "pacemaker-1.0.rng", NULL, 0);

	if (max < 0) {
	crm_notice("scandir(%s) failed: %s (%d)", base, strerror(errno), errno);

	} else {
	for (lpc = 0; lpc < max; lpc++) {
	int next = 0;
	float version = 0.0;
	char *transform = NULL;

	sscanf(namelist[lpc]->d_name, "pacemaker-%f.rng", &version);
	if((lpc + 1) < max) {
	float next_version = 0.0;

	sscanf(namelist[lpc+1]->d_name, "pacemaker-%f.rng", &next_version);

	if(floor(version) < floor(next_version)) {
	struct stat s;
	char *xslt = NULL;

	transform = crm_strdup_printf("upgrade-%.1f.xsl", version);
	xslt = get_schema_path(NULL, transform);
	if(stat(xslt, &s) != 0) {
	crm_err("Transform %s not found", xslt);
	free(xslt);
	__xml_schema_add(2, version, NULL, NULL, NULL, -1);
	break;
	} else {
	free(xslt);
	}
	}

	} else {
	next = -1;
	}
	__xml_schema_add(2, version, NULL, NULL, transform, next);
	free(namelist[lpc]);
	free(transform);
	}
	}

	/* 1.1 was the old name for -next */
	__xml_schema_add(2, 0.0, "pacemaker-1.1", "pacemaker-next.rng", NULL, 0);
	__xml_schema_add(2, 0.0, "pacemaker-next", "pacemaker-next.rng", NULL, -1);
	__xml_schema_add(0, 0.0, "none", "N/A", NULL, -1);
	free(namelist);
	return TRUE;
	}

	static void
	set_parent_flag(xmlNode *xml, long flag)
	{

	for(; xml; xml = xml->parent) {
	xml_private_t *p = xml->_private;

	if(p == NULL) {
	/* During calls to xmlDocCopyNode(), _private will be unset for parent nodes */
	} else {
	p->flags \|= flag;
	/* crm_trace("Setting flag %x due to %s[@id=%s]", flag, xml->name, ID(xml)); */
	}
	}
	}

	static void
	set_doc_flag(xmlNode *xml, long flag)
	{

	if(xml && xml->doc && xml->doc->_private){
	/* During calls to xmlDocCopyNode(), xml->doc may be unset */
	xml_private_t *p = xml->doc->_private;

	p->flags \|= flag;
	/* crm_trace("Setting flag %x due to %s[@id=%s]", flag, xml->name, ID(xml)); */
	}
	}

	static void
	__xml_node_dirty(xmlNode *xml)
	{
	set_doc_flag(xml, xpf_dirty);
	set_parent_flag(xml, xpf_dirty);
	}

	static void
	__xml_node_clean(xmlNode *xml)
	{
	xmlNode *cIter = NULL;
	xml_private_t *p = xml->_private;

	if(p) {
	p->flags = 0;
	}

	for (cIter = __xml_first_child(xml); cIter != NULL; cIter = __xml_next(cIter)) {
	__xml_node_clean(cIter);
	}
	}

	static void
	crm_node_created(xmlNode *xml)
	{
	xmlNode *cIter = NULL;
	xml_private_t *p = xml->_private;

	if(p && TRACKING_CHANGES(xml)) {
	if(is_not_set(p->flags, xpf_created)) {
	p->flags \|= xpf_created;
	__xml_node_dirty(xml);
	}

	for (cIter = __xml_first_child(xml); cIter != NULL; cIter = __xml_next(cIter)) {
	crm_node_created(cIter);
	}
	}
	}

	static void
	crm_attr_dirty(xmlAttr *a)
	{
	xmlNode *parent = a->parent;
	xml_private_t *p = NULL;

	p = a->_private;
	p->flags \|= (xpf_dirty\|xpf_modified);
	p->flags = (p->flags & ~xpf_deleted);
	/* crm_trace("Setting flag %x due to %s[@id=%s, @%s=%s]", */
	/* xpf_dirty, parent?parent->name:NULL, ID(parent), a->name, a->children->content); */

	__xml_node_dirty(parent);
	}

	int get_tag_name(const char *input, size_t offset, size_t max);
	int get_attr_name(const char *input, size_t offset, size_t max);
	int get_attr_value(const char *input, size_t offset, size_t max);
	gboolean can_prune_leaf(xmlNode * xml_node);

	void diff_filter_context(int context, int upper_bound, int lower_bound,
	xmlNode * xml_node, xmlNode * parent);
	int in_upper_context(int depth, int context, xmlNode * xml_node);
	int add_xml_object(xmlNode * parent, xmlNode * target, xmlNode * update, gboolean as_diff);

	static inline const char *
	crm_attr_value(xmlAttr * attr)
	{
	if (attr == NULL \|\| attr->children == NULL) {
	return NULL;
	}
	return (const char *)attr->children->content;
	}

	static inline xmlAttr *
	crm_first_attr(xmlNode * xml)
	{
	if (xml == NULL) {
	return NULL;
	}
	return xml->properties;
	}

	#define XML_PRIVATE_MAGIC (long) 0x81726354

	static void
	__xml_acl_free(void *data)
	{
	if(data) {
	xml_acl_t *acl = data;

	free(acl->xpath);
	free(acl);
	}
	}

	static void
	__xml_private_clean(xml_private_t *p)
	{
	if(p) {
	CRM_ASSERT(p->check == XML_PRIVATE_MAGIC);

	free(p->user);
	p->user = NULL;

	if(p->acls) {
	g_list_free_full(p->acls, __xml_acl_free);
	p->acls = NULL;
	}

	if(p->deleted_paths) {
	g_list_free_full(p->deleted_paths, free);
	p->deleted_paths = NULL;
	}
	}
	}


	static void
	__xml_private_free(xml_private_t *p)
	{
	__xml_private_clean(p);
	free(p);
	}

	static void
	pcmkDeregisterNode(xmlNodePtr node)
	{
	__xml_private_free(node->_private);
	}

	static void
	pcmkRegisterNode(xmlNodePtr node)
	{
	xml_private_t *p = NULL;

	switch(node->type) {
	case XML_ELEMENT_NODE:
	case XML_DOCUMENT_NODE:
	case XML_ATTRIBUTE_NODE:
	case XML_COMMENT_NODE:
	p = calloc(1, sizeof(xml_private_t));
	p->check = XML_PRIVATE_MAGIC;
	/* Flags will be reset if necessary when tracking is enabled */
	p->flags \|= (xpf_dirty\|xpf_created);
	node->_private = p;
	break;
	case XML_TEXT_NODE:
	case XML_DTD_NODE:
	case XML_CDATA_SECTION_NODE:
	break;
	default:
	/* Ignore */
	crm_trace("Ignoring %p %d", node, node->type);
	CRM_LOG_ASSERT(node->type == XML_ELEMENT_NODE);
	break;
	}

	if(p && TRACKING_CHANGES(node)) {
	/* XML_ELEMENT_NODE doesn't get picked up here, node->doc is
	* not hooked up at the point we are called
	*/
	set_doc_flag(node, xpf_dirty);
	__xml_node_dirty(node);
	}
	}

	static xml_acl_t *
	__xml_acl_create(xmlNode * xml, xmlNode *target, enum xml_private_flags mode)
	{
	xml_acl_t *acl = NULL;

	xml_private_t *p = NULL;
	const char *tag = crm_element_value(xml, XML_ACL_ATTR_TAG);
	const char *ref = crm_element_value(xml, XML_ACL_ATTR_REF);
	const char *xpath = crm_element_value(xml, XML_ACL_ATTR_XPATH);

	if(tag == NULL) {
	/* Compatability handling for pacemaker < 1.1.12 */
	tag = crm_element_value(xml, XML_ACL_ATTR_TAGv1);
	}
	if(ref == NULL) {
	/* Compatability handling for pacemaker < 1.1.12 */
	ref = crm_element_value(xml, XML_ACL_ATTR_REFv1);
	}

	if(target == NULL \|\| target->doc == NULL \|\| target->doc->_private == NULL){
	CRM_ASSERT(target);
	CRM_ASSERT(target->doc);
	CRM_ASSERT(target->doc->_private);
	return NULL;

	} else if (tag == NULL && ref == NULL && xpath == NULL) {
	crm_trace("No criteria %p", xml);
	return NULL;
	}

	p = target->doc->_private;
	acl = calloc(1, sizeof(xml_acl_t));
	if (acl) {
	const char *attr = crm_element_value(xml, XML_ACL_ATTR_ATTRIBUTE);

	acl->mode = mode;
	if(xpath) {
	acl->xpath = strdup(xpath);
	crm_trace("Using xpath: %s", acl->xpath);

	} else {
	int offset = 0;
	char buffer[XML_BUFFER_SIZE];

	if(tag) {
	offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "//%s", tag);
	} else {
	offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "//*");
	}

	if(ref \|\| attr) {
	offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "[");
	}

	if(ref) {
	offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "@id='%s'", ref);
	}

	if(ref && attr) {
	offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, " and ");
	}

	if(attr) {
	offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "@%s", attr);
	}

	if(ref \|\| attr) {
	offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "]");
	}

	CRM_LOG_ASSERT(offset > 0);
	acl->xpath = strdup(buffer);
	crm_trace("Built xpath: %s", acl->xpath);
	}

	p->acls = g_list_append(p->acls, acl);
	}
	return acl;
	}

	static gboolean
	__xml_acl_parse_entry(xmlNode * acl_top, xmlNode * acl_entry, xmlNode *target)
	{
	xmlNode *child = NULL;

	for (child = __xml_first_child(acl_entry); child; child = __xml_next(child)) {
	const char *tag = crm_element_name(child);
	const char *kind = crm_element_value(child, XML_ACL_ATTR_KIND);

	if (strcmp(XML_ACL_TAG_PERMISSION, tag) == 0){
	tag = kind;
	}

	crm_trace("Processing %s %p", tag, child);
	if(tag == NULL) {
	CRM_ASSERT(tag != NULL);

	} else if (strcmp(XML_ACL_TAG_ROLE_REF, tag) == 0
	\|\| strcmp(XML_ACL_TAG_ROLE_REFv1, tag) == 0) {
	const char *ref_role = crm_element_value(child, XML_ATTR_ID);

	if (ref_role) {
	xmlNode *role = NULL;

	for (role = __xml_first_child(acl_top); role; role = __xml_next(role)) {
	if (strcmp(XML_ACL_TAG_ROLE, (const char *)role->name) == 0) {
	const char *role_id = crm_element_value(role, XML_ATTR_ID);

	if (role_id && strcmp(ref_role, role_id) == 0) {
	crm_debug("Unpacking referenced role: %s", role_id);
	__xml_acl_parse_entry(acl_top, role, target);
	break;
	}
	}
	}
	}

	} else if (strcmp(XML_ACL_TAG_READ, tag) == 0) {
	__xml_acl_create(child, target, xpf_acl_read);

	} else if (strcmp(XML_ACL_TAG_WRITE, tag) == 0) {
	__xml_acl_create(child, target, xpf_acl_write);

	} else if (strcmp(XML_ACL_TAG_DENY, tag) == 0) {
	__xml_acl_create(child, target, xpf_acl_deny);

	} else {
	crm_warn("Unknown ACL entry: %s/%s", tag, kind);
	}
	}

	return TRUE;
	}

	/*
	<acls>
	<acl_target id="l33t-haxor"><role id="auto-l33t-haxor"/></acl_target>
	<acl_role id="auto-l33t-haxor">
	<acl_permission id="crook-nothing" kind="deny" xpath="/cib"/>
	</acl_role>
	<acl_target id="niceguy">
	<role id="observer"/>
	</acl_target>
	<acl_role id="observer">
	<acl_permission id="observer-read-1" kind="read" xpath="/cib"/>
	<acl_permission id="observer-write-1" kind="write" xpath="//nvpair[@name='stonith-enabled']"/>
	<acl_permission id="observer-write-2" kind="write" xpath="//nvpair[@name='target-role']"/>
	</acl_role>
	<acl_target id="badidea"><role id="auto-badidea"/></acl_target>
	<acl_role id="auto-badidea">
	<acl_permission id="badidea-resources" kind="read" xpath="//meta_attributes"/>
	<acl_permission id="badidea-resources-2" kind="deny" reference="dummy-meta_attributes"/>
	</acl_role>
	</acls>
	*/

	const char *
	__xml_acl_to_text(enum xml_private_flags flags)
	{
	if(is_set(flags, xpf_acl_deny)) {
	return "deny";
	}
	if(is_set(flags, xpf_acl_write)) {
	return "read/write";
	}
	if(is_set(flags, xpf_acl_read)) {
	return "read";
	}

	return "none";
	}

	static void
	__xml_acl_apply(xmlNode *xml)
	{
	GListPtr aIter = NULL;
	xml_private_t *p = NULL;
	xmlXPathObjectPtr xpathObj = NULL;

	if(xml_acl_enabled(xml) == FALSE) {
	p = xml->doc->_private;
	crm_trace("Not applying ACLs for %s", p->user);
	return;
	}

	p = xml->doc->_private;
	for(aIter = p->acls; aIter != NULL; aIter = aIter->next) {
	int max = 0, lpc = 0;
	xml_acl_t *acl = aIter->data;

	xpathObj = xpath_search(xml, acl->xpath);
	max = numXpathResults(xpathObj);

	for(lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);
	char *path = xml_get_path(match);

	p = match->_private;
	crm_trace("Applying %x to %s for %s", acl->mode, path, acl->xpath);

	#ifdef SUSE_ACL_COMPAT
	if(is_not_set(p->flags, acl->mode)) {
	if(is_set(p->flags, xpf_acl_read)
	\|\| is_set(p->flags, xpf_acl_write)
	\|\| is_set(p->flags, xpf_acl_deny)) {
	crm_config_warn("Configuration element %s is matched by multiple ACL rules, only the first applies ('%s' wins over '%s')",
	path, __xml_acl_to_text(p->flags), __xml_acl_to_text(acl->mode));
	free(path);
	continue;
	}
	}
	#endif

	p->flags \|= acl->mode;
	free(path);
	}
	crm_trace("Now enforcing ACL: %s (%d matches)", acl->xpath, max);
	freeXpathObject(xpathObj);
	}

	p = xml->_private;
	if(is_not_set(p->flags, xpf_acl_read) && is_not_set(p->flags, xpf_acl_write)) {
	p->flags \|= xpf_acl_deny;
	p = xml->doc->_private;
	crm_info("Enforcing default ACL for %s to %s", p->user, crm_element_name(xml));
	}

	}

	static void
	__xml_acl_unpack(xmlNode source, xmlNode target, const char *user)
	{
	#if ENABLE_ACL
	xml_private_t *p = NULL;

	if(target == NULL \|\| target->doc == NULL \|\| target->doc->_private == NULL) {
	return;
	}

	p = target->doc->_private;
	if(pcmk_acl_required(user) == FALSE) {
	crm_trace("no acls needed for '%s'", user);

	} else if(p->acls == NULL) {
	xmlNode *acls = get_xpath_object("//"XML_CIB_TAG_ACLS, source, LOG_TRACE);

	free(p->user);
	p->user = strdup(user);

	if(acls) {
	xmlNode *child = NULL;

	for (child = __xml_first_child(acls); child; child = __xml_next(child)) {
	const char *tag = crm_element_name(child);

	if (strcmp(tag, XML_ACL_TAG_USER) == 0 \|\| strcmp(tag, XML_ACL_TAG_USERv1) == 0) {
	const char *id = crm_element_value(child, XML_ATTR_ID);

	if(id && strcmp(id, user) == 0) {
	crm_debug("Unpacking ACLs for %s", id);
	__xml_acl_parse_entry(acls, child, target);
	}
	}
	}
	}
	}
	#endif
	}

	static inline bool
	__xml_acl_mode_test(enum xml_private_flags allowed, enum xml_private_flags requested)
	{
	if(is_set(allowed, xpf_acl_deny)) {
	return FALSE;

	} else if(is_set(allowed, requested)) {
	return TRUE;

	} else if(is_set(requested, xpf_acl_read) && is_set(allowed, xpf_acl_write)) {
	return TRUE;

	} else if(is_set(requested, xpf_acl_create) && is_set(allowed, xpf_acl_write)) {
	return TRUE;

	} else if(is_set(requested, xpf_acl_create) && is_set(allowed, xpf_created)) {
	return TRUE;
	}
	return FALSE;
	}

	/* rc = TRUE if orig_cib has been filtered
	* That means '*result' rather than 'xml' should be exploited afterwards
	*/
	static bool
	__xml_purge_attributes(xmlNode *xml)
	{
	xmlNode *child = NULL;
	xmlAttr *xIter = NULL;
	bool readable_children = FALSE;
	xml_private_t *p = xml->_private;

	if(__xml_acl_mode_test(p->flags, xpf_acl_read)) {
	crm_trace("%s is readable", crm_element_name(xml), ID(xml));
	return TRUE;
	}

	xIter = crm_first_attr(xml);
	while(xIter != NULL) {
	xmlAttr *tmp = xIter;
	const char prop_name = (const char )xIter->name;

	xIter = xIter->next;
	if (strcmp(prop_name, XML_ATTR_ID) == 0) {
	continue;
	}

	xmlUnsetProp(xml, tmp->name);
	}

	child = __xml_first_child(xml);
	while ( child != NULL ) {
	xmlNode *tmp = child;

	child = __xml_next(child);
	readable_children \|= __xml_purge_attributes(tmp);
	}

	if(readable_children == FALSE) {
	free_xml(xml); /* Nothing readable under here, purge completely */
	}
	return readable_children;
	}

	bool
	xml_acl_filtered_copy(const char user, xmlNode acl_source, xmlNode xml, xmlNode * result)
	{
	GListPtr aIter = NULL;
	xmlNode *target = NULL;
	xml_private_t *p = NULL;
	xml_private_t *doc = NULL;

	*result = NULL;
	if(xml == NULL \|\| pcmk_acl_required(user) == FALSE) {
	crm_trace("no acls needed for '%s'", user);
	return FALSE;
	}

	crm_trace("filtering copy of %p for '%s'", xml, user);
	target = copy_xml(xml);
	if(target == NULL) {
	return TRUE;
	}

	__xml_acl_unpack(acl_source, target, user);
	set_doc_flag(target, xpf_acl_enabled);
	__xml_acl_apply(target);

	doc = target->doc->_private;
	for(aIter = doc->acls; aIter != NULL && target; aIter = aIter->next) {
	int max = 0;
	xml_acl_t *acl = aIter->data;

	if(acl->mode != xpf_acl_deny) {
	/* Nothing to do */

	} else if(acl->xpath) {
	int lpc = 0;
	xmlXPathObjectPtr xpathObj = xpath_search(target, acl->xpath);

	max = numXpathResults(xpathObj);
	for(lpc = 0; lpc < max; lpc++) {
	xmlNode *match = getXpathResult(xpathObj, lpc);

	crm_trace("Purging attributes from %s", acl->xpath);
	if(__xml_purge_attributes(match) == FALSE && match == target) {
	crm_trace("No access to the entire document for %s", user);
	freeXpathObject(xpathObj);
	return TRUE;
	}
	}
	crm_trace("Enforced ACL %s (%d matches)", acl->xpath, max);
	freeXpathObject(xpathObj);
	}
	}

	p = target->_private;
	if(is_set(p->flags, xpf_acl_deny) && __xml_purge_attributes(target) == FALSE) {
	crm_trace("No access to the entire document for %s", user);
	return TRUE;
	}

	if(doc->acls) {
	g_list_free_full(doc->acls, __xml_acl_free);
	doc->acls = NULL;

	} else {
	crm_trace("Ordinary user '%s' cannot access the CIB without any defined ACLs", doc->user);
	free_xml(target);
	target = NULL;
	}

	if(target) {
	*result = target;
	}

	return TRUE;
	}

	static void
	__xml_acl_post_process(xmlNode * xml)
	{
	xmlNode *cIter = __xml_first_child(xml);
	xml_private_t *p = xml->_private;

	if(is_set(p->flags, xpf_created)) {
	xmlAttr *xIter = NULL;
	char *path = xml_get_path(xml);

	/* Always allow new scaffolding, ie. node with no attributes or only an 'id'
	* Except in the ACLs section
	*/

	for (xIter = crm_first_attr(xml); xIter != NULL; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;

	if (strcmp(prop_name, XML_ATTR_ID) == 0 && strstr(path, "/"XML_CIB_TAG_ACLS"/") == NULL) {
	/* Delay the acl check */
	continue;

	} else if(__xml_acl_check(xml, NULL, xpf_acl_write)) {
	crm_trace("Creation of %s=%s is allowed", crm_element_name(xml), ID(xml));
	break;

	} else {
	crm_trace("Cannot add new node %s at %s", crm_element_name(xml), path);

	if(xml != xmlDocGetRootElement(xml->doc)) {
	xmlUnlinkNode(xml);
	xmlFreeNode(xml);
	}
	free(path);
	return;
	}
	}
	free(path);
	}

	while (cIter != NULL) {
	xmlNode *child = cIter;
	cIter = __xml_next(cIter); /* In case it is free'd */
	__xml_acl_post_process(child);
	}
	}

	bool
	xml_acl_denied(xmlNode *xml)
	{
	if(xml && xml->doc && xml->doc->_private){
	xml_private_t *p = xml->doc->_private;

	return is_set(p->flags, xpf_acl_denied);
	}
	return FALSE;
	}

	void
	xml_acl_disable(xmlNode *xml)
	{
	if(xml_acl_enabled(xml)) {
	xml_private_t *p = xml->doc->_private;

	/* Catch anything that was created but shouldn't have been */
	__xml_acl_apply(xml);
	__xml_acl_post_process(xml);
	clear_bit(p->flags, xpf_acl_enabled);
	}
	}

	bool
	xml_acl_enabled(xmlNode *xml)
	{
	if(xml && xml->doc && xml->doc->_private){
	xml_private_t *p = xml->doc->_private;

	return is_set(p->flags, xpf_acl_enabled);
	}
	return FALSE;
	}

	void
	xml_track_changes(xmlNode * xml, const char user, xmlNode acl_source, bool enforce_acls)
	{
	xml_accept_changes(xml);
	crm_trace("Tracking changes%s to %p", enforce_acls?" with ACLs":"", xml);
	set_doc_flag(xml, xpf_tracking);
	if(enforce_acls) {
	if(acl_source == NULL) {
	acl_source = xml;
	}
	set_doc_flag(xml, xpf_acl_enabled);
	__xml_acl_unpack(acl_source, xml, user);
	__xml_acl_apply(xml);
	}
	}

	bool xml_tracking_changes(xmlNode * xml)
	{
	if(xml == NULL) {
	return FALSE;

	} else if(is_set(((xml_private_t *)xml->doc->_private)->flags, xpf_tracking)) {
	return TRUE;
	}
	return FALSE;
	}

	bool xml_document_dirty(xmlNode *xml)
	{
	if(xml != NULL && xml->doc && xml->doc->_private) {
	xml_private_t *doc = xml->doc->_private;

	return is_set(doc->flags, xpf_dirty);
	}
	return FALSE;
	}

	/*
	<diff format="2.0">
	<version>
	<source admin_epoch="1" epoch="2" num_updates="3"/>
	<target admin_epoch="1" epoch="3" num_updates="0"/>
	</version>
	<change operation="add" xpath="/cib/configuration/nodes">
	<node id="node2" uname="node2" description="foo"/>
	</change>
	<change operation="add" xpath="/cib/configuration/nodes/node[node2]">
	<instance_attributes id="nodes-node"><!-- NOTE: can be a full tree -->
	<nvpair id="nodes-node2-ram" name="ram" value="1024M"/>
	</instance_attributes>
	</change>
	<change operation="update" xpath="/cib/configuration/nodes[@id='node2']">
	<change-list>
	<change-attr operation="set" name="type" value="member"/>
	<change-attr operation="unset" name="description"/>
	</change-list>
	<change-result>
	<node id="node2" uname="node2" type="member"/><!-- NOTE: not recursive -->
	</change-result>
	</change>
	<change operation="delete" xpath="/cib/configuration/nodes/node[@id='node3'] /">
	<change operation="update" xpath="/cib/configuration/resources/group[@id='g1']">
	<change-list>
	<change-attr operation="set" name="description" value="some grabage here"/>
	</change-list>
	<change-result>
	<group id="g1" description="some grabage here"/><!-- NOTE: not recursive -->
	</change-result>
	</change>
	<change operation="update" xpath="/cib/status/node_state[@id='node2]/lrm[@id='node2']/lrm_resources/lrm_resource[@id='Fence']">
	<change-list>
	<change-attr operation="set" name="oper" value="member"/>
	<change-attr operation="set" name="operation_key" value="Fence_start_0"/>
	<change-attr operation="set" name="operation" value="start"/>
	<change-attr operation="set" name="transition-key" value="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"/>
	<change-attr operation="set" name="transition-magic" value="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"/>
	<change-attr operation="set" name="call-id" value="2"/>
	<change-attr operation="set" name="rc-code" value="0"/>
	</change-list>
	<change-result>
	<lrm_rsc_op id="Fence_last_0" operation_key="Fence_start_0" operation="start" crm-debug-origin="crm_simulate" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
	</change-result>
	</change>
	</diff>
	*/
	static int __xml_offset(xmlNode *xml)
	{
	int position = 0;
	xmlNode *cIter = NULL;

	for(cIter = xml; cIter->prev; cIter = cIter->prev) {
	xml_private_t p = ((xmlNode)cIter->prev)->_private;

	if(is_not_set(p->flags, xpf_skip)) {
	position++;
	}
	}

	return position;
	}

	static int __xml_offset_no_deletions(xmlNode *xml)
	{
	int position = 0;
	xmlNode *cIter = NULL;

	for(cIter = xml; cIter->prev; cIter = cIter->prev) {
	xml_private_t p = ((xmlNode)cIter->prev)->_private;

	if(is_not_set(p->flags, xpf_deleted)) {
	position++;
	}
	}

	return position;
	}

	static void
	__xml_build_changes(xmlNode * xml, xmlNode *patchset)
	{
	xmlNode *cIter = NULL;
	xmlAttr *pIter = NULL;
	xmlNode *change = NULL;
	xml_private_t *p = xml->_private;

	if(patchset && is_set(p->flags, xpf_created)) {
	int offset = 0;
	char buffer[XML_BUFFER_SIZE];

	if(__get_prefix(NULL, xml->parent, buffer, offset) > 0) {
	int position = __xml_offset_no_deletions(xml);

	change = create_xml_node(patchset, XML_DIFF_CHANGE);

	crm_xml_add(change, XML_DIFF_OP, "create");
	crm_xml_add(change, XML_DIFF_PATH, buffer);
	crm_xml_add_int(change, XML_DIFF_POSITION, position);
	add_node_copy(change, xml);
	}

	return;
	}

	for (pIter = crm_first_attr(xml); pIter != NULL; pIter = pIter->next) {
	xmlNode *attr = NULL;

	p = pIter->_private;
	if(is_not_set(p->flags, xpf_deleted) && is_not_set(p->flags, xpf_dirty)) {
	continue;
	}

	if(change == NULL) {
	int offset = 0;
	char buffer[XML_BUFFER_SIZE];

	if(__get_prefix(NULL, xml, buffer, offset) > 0) {
	change = create_xml_node(patchset, XML_DIFF_CHANGE);

	crm_xml_add(change, XML_DIFF_OP, "modify");
	crm_xml_add(change, XML_DIFF_PATH, buffer);

	change = create_xml_node(change, XML_DIFF_LIST);
	}
	}

	attr = create_xml_node(change, XML_DIFF_ATTR);

	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, (const char *)pIter->name);
	if(p->flags & xpf_deleted) {
	crm_xml_add(attr, XML_DIFF_OP, "unset");

	} else {
	const char value = crm_element_value(xml, (const char )pIter->name);

	crm_xml_add(attr, XML_DIFF_OP, "set");
	crm_xml_add(attr, XML_NVPAIR_ATTR_VALUE, value);
	}
	}

	if(change) {
	xmlNode *result = NULL;

	change = create_xml_node(change->parent, XML_DIFF_RESULT);
	result = create_xml_node(change, (const char *)xml->name);

	for (pIter = crm_first_attr(xml); pIter != NULL; pIter = pIter->next) {
	const char value = crm_element_value(xml, (const char )pIter->name);

	p = pIter->_private;
	if (is_not_set(p->flags, xpf_deleted)) {
	crm_xml_add(result, (const char *)pIter->name, value);
	}
	}
	}

	for (cIter = __xml_first_child(xml); cIter != NULL; cIter = __xml_next(cIter)) {
	__xml_build_changes(cIter, patchset);
	}

	p = xml->_private;
	if(patchset && is_set(p->flags, xpf_moved)) {
	int offset = 0;
	char buffer[XML_BUFFER_SIZE];

	crm_trace("%s.%s moved to position %d", xml->name, ID(xml), __xml_offset(xml));
	if(__get_prefix(NULL, xml, buffer, offset) > 0) {
	change = create_xml_node(patchset, XML_DIFF_CHANGE);

	crm_xml_add(change, XML_DIFF_OP, "move");
	crm_xml_add(change, XML_DIFF_PATH, buffer);
	crm_xml_add_int(change, XML_DIFF_POSITION, __xml_offset_no_deletions(xml));
	}
	}
	}

	static void
	__xml_accept_changes(xmlNode * xml)
	{
	xmlNode *cIter = NULL;
	xmlAttr *pIter = NULL;
	xml_private_t *p = xml->_private;

	p->flags = xpf_none;
	pIter = crm_first_attr(xml);

	while (pIter != NULL) {
	const xmlChar *name = pIter->name;

	p = pIter->_private;
	pIter = pIter->next;

	if(p->flags & xpf_deleted) {
	xml_remove_prop(xml, (const char *)name);

	} else {
	p->flags = xpf_none;
	}
	}

	for (cIter = __xml_first_child(xml); cIter != NULL; cIter = __xml_next(cIter)) {
	__xml_accept_changes(cIter);
	}
	}

	static bool
	is_config_change(xmlNode *xml)
	{
	GListPtr gIter = NULL;
	xml_private_t *p = NULL;
	xmlNode *config = first_named_child(xml, XML_CIB_TAG_CONFIGURATION);

	if(config) {
	p = config->_private;
	}
	if(p && is_set(p->flags, xpf_dirty)) {
	return TRUE;
	}

	if(xml->doc && xml->doc->_private) {
	p = xml->doc->_private;
	for(gIter = p->deleted_paths; gIter; gIter = gIter->next) {
	char *path = gIter->data;

	if(strstr(path, "/"XML_TAG_CIB"/"XML_CIB_TAG_CONFIGURATION) != NULL) {
	return TRUE;
	}
	}
	}

	return FALSE;
	}

	static void
	xml_repair_v1_diff(xmlNode * last, xmlNode * next, xmlNode * local_diff, gboolean changed)
	{
	int lpc = 0;
	xmlNode *cib = NULL;
	xmlNode *diff_child = NULL;

	const char *tag = NULL;

	const char *vfields[] = {
	XML_ATTR_GENERATION_ADMIN,
	XML_ATTR_GENERATION,
	XML_ATTR_NUMUPDATES,
	};

	if (local_diff == NULL) {
	crm_trace("Nothing to do");
	return;
	}

	tag = "diff-removed";
	diff_child = find_xml_node(local_diff, tag, FALSE);
	if (diff_child == NULL) {
	diff_child = create_xml_node(local_diff, tag);
	}

	tag = XML_TAG_CIB;
	cib = find_xml_node(diff_child, tag, FALSE);
	if (cib == NULL) {
	cib = create_xml_node(diff_child, tag);
	}

	for(lpc = 0; last && lpc < DIMOF(vfields); lpc++){
	const char *value = crm_element_value(last, vfields[lpc]);

	crm_xml_add(diff_child, vfields[lpc], value);
	if(changed \|\| lpc == 2) {
	crm_xml_add(cib, vfields[lpc], value);
	}
	}

	tag = "diff-added";
	diff_child = find_xml_node(local_diff, tag, FALSE);
	if (diff_child == NULL) {
	diff_child = create_xml_node(local_diff, tag);
	}

	tag = XML_TAG_CIB;
	cib = find_xml_node(diff_child, tag, FALSE);
	if (cib == NULL) {
	cib = create_xml_node(diff_child, tag);
	}

	for(lpc = 0; next && lpc < DIMOF(vfields); lpc++){
	const char *value = crm_element_value(next, vfields[lpc]);

	crm_xml_add(diff_child, vfields[lpc], value);
	}

	if (next) {
	xmlAttrPtr xIter = NULL;

	for (xIter = next->properties; xIter; xIter = xIter->next) {
	const char p_name = (const char )xIter->name;
	const char *p_value = crm_element_value(next, p_name);

	xmlSetProp(cib, (const xmlChar )p_name, (const xmlChar )p_value);
	}
	}

	crm_log_xml_explicit(local_diff, "Repaired-diff");
	}

	static xmlNode *
	xml_create_patchset_v1(xmlNode source, xmlNode target, bool config, bool suppress)
	{
	xmlNode *patchset = diff_xml_object(source, target, suppress);

	if(patchset) {
	CRM_LOG_ASSERT(xml_document_dirty(target));
	xml_repair_v1_diff(source, target, patchset, config);
	crm_xml_add(patchset, "format", "1");
	}
	return patchset;
	}

	static xmlNode *
	xml_create_patchset_v2(xmlNode source, xmlNode target)
	{
	int lpc = 0;
	GListPtr gIter = NULL;
	xml_private_t *doc = NULL;

	xmlNode *v = NULL;
	xmlNode *version = NULL;
	xmlNode *patchset = NULL;
	const char *vfields[] = {
	XML_ATTR_GENERATION_ADMIN,
	XML_ATTR_GENERATION,
	XML_ATTR_NUMUPDATES,
	};

	CRM_ASSERT(target);
	if(xml_document_dirty(target) == FALSE) {
	return NULL;
	}

	CRM_ASSERT(target->doc);
	doc = target->doc->_private;

	patchset = create_xml_node(NULL, XML_TAG_DIFF);
	crm_xml_add_int(patchset, "format", 2);

	version = create_xml_node(patchset, XML_DIFF_VERSION);

	v = create_xml_node(version, XML_DIFF_VSOURCE);
	for(lpc = 0; lpc < DIMOF(vfields); lpc++){
	const char *value = crm_element_value(source, vfields[lpc]);

	if(value == NULL) {
	value = "1";
	}
	crm_xml_add(v, vfields[lpc], value);
	}

	v = create_xml_node(version, XML_DIFF_VTARGET);
	for(lpc = 0; lpc < DIMOF(vfields); lpc++){
	const char *value = crm_element_value(target, vfields[lpc]);

	if(value == NULL) {
	value = "1";
	}
	crm_xml_add(v, vfields[lpc], value);
	}

	for(gIter = doc->deleted_paths; gIter; gIter = gIter->next) {
	xmlNode *change = create_xml_node(patchset, XML_DIFF_CHANGE);

	crm_xml_add(change, XML_DIFF_OP, "delete");
	crm_xml_add(change, XML_DIFF_PATH, gIter->data);
	}

	__xml_build_changes(target, patchset);
	return patchset;
	}

	static gboolean patch_legacy_mode(void)
	{
	static gboolean init = TRUE;
	static gboolean legacy = FALSE;

	if(init) {
	init = FALSE;
	legacy = daemon_option_enabled("cib", "legacy");
	if(legacy) {
	crm_notice("Enabled legacy mode");
	}
	}
	return legacy;
	}

	xmlNode *
	xml_create_patchset(int format, xmlNode source, xmlNode target, bool *config_changed, bool manage_version)
	{
	int counter = 0;
	bool config = FALSE;
	xmlNode *patch = NULL;
	const char *version = crm_element_value(source, XML_ATTR_CRM_VERSION);

	xml_acl_disable(target);
	if(xml_document_dirty(target) == FALSE) {
	crm_trace("No change %d", format);
	return NULL; /* No change */
	}

	config = is_config_change(target);
	if(config_changed) {
	*config_changed = config;
	}

	if(manage_version && config) {
	crm_trace("Config changed %d", format);
	crm_xml_add(target, XML_ATTR_NUMUPDATES, "0");

	crm_element_value_int(target, XML_ATTR_GENERATION, &counter);
	crm_xml_add_int(target, XML_ATTR_GENERATION, counter+1);

	} else if(manage_version) {
	crm_trace("Status changed %d", format);
	crm_element_value_int(target, XML_ATTR_NUMUPDATES, &counter);
	crm_xml_add_int(target, XML_ATTR_NUMUPDATES, counter+1);
	}

	if(format == 0) {
	if(patch_legacy_mode()) {
	format = 1;

	} else if(compare_version("3.0.8", version) < 0) {
	format = 2;

	} else {
	format = 1;
	}
	crm_trace("Using patch format %d for version: %s", format, version);
	}

	switch(format) {
	case 1:
	patch = xml_create_patchset_v1(source, target, config, FALSE);
	break;
	case 2:
	patch = xml_create_patchset_v2(source, target);
	break;
	default:
	crm_err("Unknown patch format: %d", format);
	return NULL;
	}

	return patch;
	}

	void
	patchset_process_digest(xmlNode patch, xmlNode source, xmlNode *target, bool with_digest)
	{
	int format = 1;
	const char *version = NULL;
	char *digest = NULL;

	if (patch == NULL \|\| source == NULL \|\| target == NULL) {
	return;
	}

	/* NOTE: We should always call xml_accept_changes() before calculating digest. */
	/* Otherwise, with an on-tracking dirty target, we could get a wrong digest. */
	CRM_LOG_ASSERT(xml_document_dirty(target) == FALSE);

	crm_element_value_int(patch, "format", &format);
	if (format > 1 && with_digest == FALSE) {
	return;
	}

	version = crm_element_value(source, XML_ATTR_CRM_VERSION);
	digest = calculate_xml_versioned_digest(target, FALSE, TRUE, version);

	crm_xml_add(patch, XML_ATTR_DIGEST, digest);
	free(digest);

	return;
	}

	static void
	__xml_log_element(int log_level, const char file, const char function, int line,
	const char prefix, xmlNode data, int depth, int options);

	void
	xml_log_patchset(uint8_t log_level, const char function, xmlNode patchset)
	{
	int format = 1;
	xmlNode *child = NULL;
	xmlNode *added = NULL;
	xmlNode *removed = NULL;
	gboolean is_first = TRUE;

	int add[] = { 0, 0, 0 };
	int del[] = { 0, 0, 0 };

	const char *fmt = NULL;
	const char *digest = NULL;
	int options = xml_log_option_formatted;

	static struct qb_log_callsite *patchset_cs = NULL;

	if (patchset_cs == NULL) {
	patchset_cs = qb_log_callsite_get(function, __FILE__, "xml-patchset", log_level, __LINE__, 0);
	}

	if (patchset == NULL) {
	crm_trace("Empty patch");
	return;

	} else if (log_level == 0) {
	/* Log to stdout */
	} else if (crm_is_callsite_active(patchset_cs, log_level, 0) == FALSE) {
	return;
	}

	xml_patch_versions(patchset, add, del);
	fmt = crm_element_value(patchset, "format");
	digest = crm_element_value(patchset, XML_ATTR_DIGEST);

	if (add[2] != del[2] \|\| add[1] != del[1] \|\| add[0] != del[0]) {
	do_crm_log_alias(log_level, __FILE__, function, __LINE__,
	"Diff: --- %d.%d.%d %s", del[0], del[1], del[2], fmt);
	do_crm_log_alias(log_level, __FILE__, function, __LINE__,
	"Diff: +++ %d.%d.%d %s", add[0], add[1], add[2], digest);

	} else if (patchset != NULL && (add[0] \|\| add[1] \|\| add[2])) {
	do_crm_log_alias(log_level, __FILE__, function, __LINE__,
	"%s: Local-only Change: %d.%d.%d", function ? function : "",
	add[0], add[1], add[2]);
	}

	crm_element_value_int(patchset, "format", &format);
	if(format == 2) {
	xmlNode *change = NULL;

	for (change = __xml_first_child(patchset); change != NULL; change = __xml_next(change)) {
	const char *op = crm_element_value(change, XML_DIFF_OP);
	const char *xpath = crm_element_value(change, XML_DIFF_PATH);

	if(op == NULL) {
	} else if(strcmp(op, "create") == 0) {
	int lpc = 0, max = 0;
	char *prefix = crm_strdup_printf("++ %s: ", xpath);

	max = strlen(prefix);
	__xml_log_element(log_level, __FILE__, function, __LINE__, prefix, change->children,
	0, xml_log_option_formatted\|xml_log_option_open);

	for(lpc = 2; lpc < max; lpc++) {
	prefix[lpc] = ' ';
	}

	__xml_log_element(log_level, __FILE__, function, __LINE__, prefix, change->children,
	0, xml_log_option_formatted\|xml_log_option_close\|xml_log_option_children);
	free(prefix);

	} else if(strcmp(op, "move") == 0) {
	do_crm_log_alias(log_level, __FILE__, function, __LINE__, "+~ %s moved to offset %s", xpath, crm_element_value(change, XML_DIFF_POSITION));

	} else if(strcmp(op, "modify") == 0) {
	xmlNode *clist = first_named_child(change, XML_DIFF_LIST);
	char buffer_set[XML_BUFFER_SIZE];
	char buffer_unset[XML_BUFFER_SIZE];
	int o_set = 0;
	int o_unset = 0;

	buffer_set[0] = 0;
	buffer_unset[0] = 0;
	for (child = __xml_first_child(clist); child != NULL; child = __xml_next(child)) {
	const char *name = crm_element_value(child, "name");

	op = crm_element_value(child, XML_DIFF_OP);
	if(op == NULL) {
	} else if(strcmp(op, "set") == 0) {
	const char *value = crm_element_value(child, "value");

	if(o_set > 0) {
	o_set += snprintf(buffer_set + o_set, XML_BUFFER_SIZE - o_set, ", ");
	}
	o_set += snprintf(buffer_set + o_set, XML_BUFFER_SIZE - o_set, "@%s=%s", name, value);

	} else if(strcmp(op, "unset") == 0) {
	if(o_unset > 0) {
	o_unset += snprintf(buffer_unset + o_unset, XML_BUFFER_SIZE - o_unset, ", ");
	}
	o_unset += snprintf(buffer_unset + o_unset, XML_BUFFER_SIZE - o_unset, "@%s", name);
	}
	}
	if(o_set) {
	do_crm_log_alias(log_level, __FILE__, function, __LINE__, "+ %s: %s", xpath, buffer_set);
	}
	if(o_unset) {
	do_crm_log_alias(log_level, __FILE__, function, __LINE__, "-- %s: %s", xpath, buffer_unset);
	}

	} else if(strcmp(op, "delete") == 0) {
	do_crm_log_alias(log_level, __FILE__, function, __LINE__, "-- %s", xpath);
	}
	}
	return;
	}

	if (log_level < LOG_DEBUG \|\| function == NULL) {
	options \|= xml_log_option_diff_short;
	}

	removed = find_xml_node(patchset, "diff-removed", FALSE);
	for (child = __xml_first_child(removed); child != NULL; child = __xml_next(child)) {
	log_data_element(log_level, __FILE__, function, __LINE__, "- ", child, 0,
	options \| xml_log_option_diff_minus);
	if (is_first) {
	is_first = FALSE;
	} else {
	do_crm_log_alias(log_level, __FILE__, function, __LINE__, " --- ");
	}
	}

	is_first = TRUE;
	added = find_xml_node(patchset, "diff-added", FALSE);
	for (child = __xml_first_child(added); child != NULL; child = __xml_next(child)) {
	log_data_element(log_level, __FILE__, function, __LINE__, "+ ", child, 0,
	options \| xml_log_option_diff_plus);
	if (is_first) {
	is_first = FALSE;
	} else {
	do_crm_log_alias(log_level, __FILE__, function, __LINE__, " +++ ");
	}
	}
	}

	void
	xml_log_changes(uint8_t log_level, const char function, xmlNode xml)
	{
	GListPtr gIter = NULL;
	xml_private_t *doc = NULL;

	CRM_ASSERT(xml);
	CRM_ASSERT(xml->doc);

	doc = xml->doc->_private;
	if(is_not_set(doc->flags, xpf_dirty)) {
	return;
	}

	for(gIter = doc->deleted_paths; gIter; gIter = gIter->next) {
	do_crm_log_alias(log_level, __FILE__, function, __LINE__, "-- %s", (char*)gIter->data);
	}

	log_data_element(log_level, __FILE__, function, __LINE__, "+ ", xml, 0,
	xml_log_option_formatted\|xml_log_option_dirty_add);
	}

	void
	xml_accept_changes(xmlNode * xml)
	{
	xmlNode *top = NULL;
	xml_private_t *doc = NULL;

	if(xml == NULL) {
	return;
	}

	crm_trace("Accepting changes to %p", xml);
	doc = xml->doc->_private;
	top = xmlDocGetRootElement(xml->doc);

	__xml_private_clean(xml->doc->_private);

	if(is_not_set(doc->flags, xpf_dirty)) {
	doc->flags = xpf_none;
	return;
	}

	doc->flags = xpf_none;
	__xml_accept_changes(top);
	}

	/* Simplified version for applying v1-style XML patches */
	static void
	__subtract_xml_object(xmlNode * target, xmlNode * patch)
	{
	xmlNode *patch_child = NULL;
	xmlNode *cIter = NULL;
	xmlAttrPtr xIter = NULL;

	char *id = NULL;
	const char *name = NULL;
	const char *value = NULL;

	if (target == NULL \|\| patch == NULL) {
	return;
	}

	if (target->type == XML_COMMENT_NODE) {
	gboolean dummy;

	subtract_xml_comment(target->parent, target, patch, &dummy);
	}

	name = crm_element_name(target);
	CRM_CHECK(name != NULL, return);
	CRM_CHECK(safe_str_eq(crm_element_name(target), crm_element_name(patch)), return);
	CRM_CHECK(safe_str_eq(ID(target), ID(patch)), return);

	/* check for XML_DIFF_MARKER in a child */
	id = crm_element_value_copy(target, XML_ATTR_ID);
	value = crm_element_value(patch, XML_DIFF_MARKER);
	if (value != NULL && strcmp(value, "removed:top") == 0) {
	crm_trace("We are the root of the deletion: %s.id=%s", name, id);
	free_xml(target);
	free(id);
	return;
	}

	for (xIter = crm_first_attr(patch); xIter != NULL; xIter = xIter->next) {
	const char p_name = (const char )xIter->name;

	/* Removing and then restoring the id field would change the ordering of properties */
	if (safe_str_neq(p_name, XML_ATTR_ID)) {
	xml_remove_prop(target, p_name);
	}
	}

	/* changes to child objects */
	cIter = __xml_first_child(target);
	while (cIter) {
	xmlNode *target_child = cIter;

	cIter = __xml_next(cIter);

	if (target_child->type == XML_COMMENT_NODE) {
	patch_child = find_xml_comment(patch, target_child);

	} else {
	patch_child = find_entity(patch, crm_element_name(target_child), ID(target_child));
	}

	__subtract_xml_object(target_child, patch_child);
	}
	free(id);
	}

	static void
	__add_xml_object(xmlNode * parent, xmlNode * target, xmlNode * patch)
	{
	xmlNode *patch_child = NULL;
	xmlNode *target_child = NULL;
	xmlAttrPtr xIter = NULL;

	const char *id = NULL;
	const char *name = NULL;
	const char *value = NULL;

	if (patch == NULL) {
	return;
	} else if (parent == NULL && target == NULL) {
	return;
	}

	/* check for XML_DIFF_MARKER in a child */
	value = crm_element_value(patch, XML_DIFF_MARKER);
	if (target == NULL
	&& value != NULL
	&& strcmp(value, "added:top") == 0) {
	id = ID(patch);
	name = crm_element_name(patch);
	crm_trace("We are the root of the addition: %s.id=%s", name, id);
	add_node_copy(parent, patch);
	return;

	} else if(target == NULL) {
	id = ID(patch);
	name = crm_element_name(patch);
	crm_err("Could not locate: %s.id=%s", name, id);
	return;
	}

	if (target->type == XML_COMMENT_NODE) {
	add_xml_comment(parent, target, patch);
	}

	name = crm_element_name(target);
	CRM_CHECK(name != NULL, return);
	CRM_CHECK(safe_str_eq(crm_element_name(target), crm_element_name(patch)), return);
	CRM_CHECK(safe_str_eq(ID(target), ID(patch)), return);

	for (xIter = crm_first_attr(patch); xIter != NULL; xIter = xIter->next) {
	const char p_name = (const char )xIter->name;
	const char *p_value = crm_element_value(patch, p_name);

	xml_remove_prop(target, p_name); /* Preserve the patch order */
	crm_xml_add(target, p_name, p_value);
	}

	/* changes to child objects */
	for (patch_child = __xml_first_child(patch); patch_child != NULL;
	patch_child = __xml_next(patch_child)) {

	if (patch_child->type == XML_COMMENT_NODE) {
	target_child = find_xml_comment(target, patch_child);

	} else {
	target_child = find_entity(target, crm_element_name(patch_child), ID(patch_child));
	}

	__add_xml_object(target, target_child, patch_child);
	}
	}

	/*
	* \internal
	* \brief Find additions or removals in a patch set
	*
	* \param[in] patchset XML of patch
	* \param[in] format Patch version
	* \param[in] added TRUE if looking for additions, FALSE if removals
	* \param[in/out] patch_node Will be set to node if found
	*
	* \return TRUE if format is valid, FALSE if invalid
	*/
	static bool
	find_patch_xml_node(xmlNode *patchset, int format, bool added,
	xmlNode **patch_node)
	{
	xmlNode *cib_node;
	const char *label;

	switch(format) {
	case 1:
	label = added? "diff-added" : "diff-removed";
	*patch_node = find_xml_node(patchset, label, FALSE);
	cib_node = find_xml_node(*patch_node, "cib", FALSE);
	if (cib_node != NULL) {
	*patch_node = cib_node;
	}
	break;
	case 2:
	label = added? "target" : "source";
	*patch_node = find_xml_node(patchset, "version", FALSE);
	patch_node = find_xml_node(patch_node, label, FALSE);
	break;
	default:
	crm_warn("Unknown patch format: %d", format);
	*patch_node = NULL;
	return FALSE;
	}
	return TRUE;
	}

	bool xml_patch_versions(xmlNode *patchset, int add[3], int del[3])
	{
	int lpc = 0;
	int format = 1;
	xmlNode *tmp = NULL;

	const char *vfields[] = {
	XML_ATTR_GENERATION_ADMIN,
	XML_ATTR_GENERATION,
	XML_ATTR_NUMUPDATES,
	};


	crm_element_value_int(patchset, "format", &format);

	/* Process removals */
	if (!find_patch_xml_node(patchset, format, FALSE, &tmp)) {
	return -EINVAL;
	}
	if (tmp) {
	for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
	crm_element_value_int(tmp, vfields[lpc], &(del[lpc]));
	crm_trace("Got %d for del[%s]", del[lpc], vfields[lpc]);
	}
	}

	/* Process additions */
	if (!find_patch_xml_node(patchset, format, TRUE, &tmp)) {
	return -EINVAL;
	}
	if (tmp) {
	for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
	crm_element_value_int(tmp, vfields[lpc], &(add[lpc]));
	crm_trace("Got %d for add[%s]", add[lpc], vfields[lpc]);
	}
	}

	return pcmk_ok;
	}

	static int
	xml_patch_version_check(xmlNode xml, xmlNode patchset, int format)
	{
	int lpc = 0;
	bool changed = FALSE;

	int this[] = { 0, 0, 0 };
	int add[] = { 0, 0, 0 };
	int del[] = { 0, 0, 0 };

	const char *vfields[] = {
	XML_ATTR_GENERATION_ADMIN,
	XML_ATTR_GENERATION,
	XML_ATTR_NUMUPDATES,
	};

	for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
	crm_element_value_int(xml, vfields[lpc], &(this[lpc]));
	crm_trace("Got %d for this[%s]", this[lpc], vfields[lpc]);
	if (this[lpc] < 0) {
	this[lpc] = 0;
	}
	}

	/* Set some defaults in case nothing is present */
	add[0] = this[0];
	add[1] = this[1];
	add[2] = this[2] + 1;
	for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
	del[lpc] = this[lpc];
	}

	xml_patch_versions(patchset, add, del);

	for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
	if(this[lpc] < del[lpc]) {
	crm_debug("Current %s is too low (%d < %d)", vfields[lpc], this[lpc], del[lpc]);
	return -pcmk_err_diff_resync;

	} else if(this[lpc] > del[lpc]) {
	crm_info("Current %s is too high (%d > %d)", vfields[lpc], this[lpc], del[lpc]);
	return -pcmk_err_old_data;
	}
	}

	for(lpc = 0; lpc < DIMOF(vfields); lpc++) {
	if(add[lpc] > del[lpc]) {
	changed = TRUE;
	}
	}

	if(changed == FALSE) {
	crm_notice("Versions did not change in patch %d.%d.%d", add[0], add[1], add[2]);
	return -pcmk_err_old_data;
	}

	crm_debug("Can apply patch %d.%d.%d to %d.%d.%d",
	add[0], add[1], add[2], this[0], this[1], this[2]);
	return pcmk_ok;
	}

	static int
	xml_apply_patchset_v1(xmlNode xml, xmlNode patchset, bool check_version)
	{
	int rc = pcmk_ok;
	int root_nodes_seen = 0;
	char *version = crm_element_value_copy(xml, XML_ATTR_CRM_VERSION);

	xmlNode *child_diff = NULL;
	xmlNode *added = find_xml_node(patchset, "diff-added", FALSE);
	xmlNode *removed = find_xml_node(patchset, "diff-removed", FALSE);
	xmlNode *old = copy_xml(xml);

	crm_trace("Substraction Phase");
	for (child_diff = __xml_first_child(removed); child_diff != NULL;
	child_diff = __xml_next(child_diff)) {
	CRM_CHECK(root_nodes_seen == 0, rc = FALSE);
	if (root_nodes_seen == 0) {
	__subtract_xml_object(xml, child_diff);
	}
	root_nodes_seen++;
	}

	if (root_nodes_seen > 1) {
	crm_err("(-) Diffs cannot contain more than one change set... saw %d", root_nodes_seen);
	rc = -ENOTUNIQ;
	}

	root_nodes_seen = 0;
	crm_trace("Addition Phase");
	if (rc == pcmk_ok) {
	xmlNode *child_diff = NULL;

	for (child_diff = __xml_first_child(added); child_diff != NULL;
	child_diff = __xml_next(child_diff)) {
	CRM_CHECK(root_nodes_seen == 0, rc = FALSE);
	if (root_nodes_seen == 0) {
	__add_xml_object(NULL, xml, child_diff);
	}
	root_nodes_seen++;
	}
	}

	if (root_nodes_seen > 1) {
	crm_err("(+) Diffs cannot contain more than one change set... saw %d", root_nodes_seen);
	rc = -ENOTUNIQ;
	}

	purge_diff_markers(xml); /* Purge prior to checking the digest */

	free_xml(old);
	free(version);
	return rc;
	}

	static xmlNode *
	__first_xml_child_match(xmlNode parent, const char name, const char *id)
	{
	xmlNode *cIter = NULL;

	for (cIter = __xml_first_child(parent); cIter != NULL; cIter = __xml_next(cIter)) {
	if(strcmp((const char *)cIter->name, name) != 0) {
	continue;
	} else if(id) {
	const char *cid = ID(cIter);
	if(cid == NULL \|\| strcmp(cid, id) != 0) {
	continue;
	}
	}
	return cIter;
	}
	return NULL;
	}

	static xmlNode *
	__xml_find_path(xmlNode top, const char key)
	{
	xmlNode target = (xmlNode)top->doc;
	char *id = malloc(XML_BUFFER_SIZE);
	char *tag = malloc(XML_BUFFER_SIZE);
	char *section = malloc(XML_BUFFER_SIZE);
	char *current = strdup(key);
	char *remainder = malloc(XML_BUFFER_SIZE);
	int rc = 0;

	while(current) {
	rc = sscanf (current, "/%[^/]%s", section, remainder);
	if(rc <= 0) {
	crm_trace("Done");
	break;

	} else if(rc > 2) {
	crm_trace("Aborting on %s", current);
	target = NULL;
	break;

	} else if(tag && section) {
	int f = sscanf (section, "%[^[][@id='%[^']", tag, id);

	switch(f) {
	case 1:
	target = __first_xml_child_match(target, tag, NULL);
	break;
	case 2:
	target = __first_xml_child_match(target, tag, id);
	break;
	default:
	crm_trace("Aborting on %s", section);
	target = NULL;
	break;
	}

	if(rc == 1 \|\| target == NULL) {
	crm_trace("Done");
	break;

	} else {
	char *tmp = current;
	current = remainder;
	remainder = tmp;
	}
	}
	}

	if(target) {
	char path = (char )xmlGetNodePath(target);

	crm_trace("Found %s for %s", path, key);
	free(path);
	} else {
	crm_debug("No match for %s", key);
	}

	free(remainder);
	free(current);
	free(section);
	free(tag);
	free(id);
	return target;
	}

	static int
	xml_apply_patchset_v2(xmlNode xml, xmlNode patchset, bool check_version)
	{
	int rc = pcmk_ok;
	xmlNode *change = NULL;
	for (change = __xml_first_child(patchset); change != NULL; change = __xml_next(change)) {
	xmlNode *match = NULL;
	const char *op = crm_element_value(change, XML_DIFF_OP);
	const char *xpath = crm_element_value(change, XML_DIFF_PATH);

	crm_trace("Processing %s %s", change->name, op);
	if(op == NULL) {
	continue;
	}

	#if 0
	match = get_xpath_object(xpath, xml, LOG_TRACE);
	#else
	match = __xml_find_path(xml, xpath);
	#endif
	crm_trace("Performing %s on %s with %p", op, xpath, match);

	if(match == NULL && strcmp(op, "delete") == 0) {
	crm_debug("No %s match for %s in %p", op, xpath, xml->doc);
	continue;

	} else if(match == NULL) {
	crm_err("No %s match for %s in %p", op, xpath, xml->doc);
	rc = -pcmk_err_diff_failed;
	continue;

	} else if(strcmp(op, "create") == 0) {
	int position = 0;
	xmlNode *child = NULL;
	xmlNode *match_child = NULL;

	match_child = match->children;
	crm_element_value_int(change, XML_DIFF_POSITION, &position);

	while(match_child && position != __xml_offset(match_child)) {
	match_child = match_child->next;
	}

	child = xmlDocCopyNode(change->children, match->doc, 1);
	if(match_child) {
	crm_trace("Adding %s at position %d", child->name, position);
	xmlAddPrevSibling(match_child, child);

	} else if(match->last) { /* Add to the end */
	crm_trace("Adding %s at position %d (end)", child->name, position);
	xmlAddNextSibling(match->last, child);

	} else {
	crm_trace("Adding %s at position %d (first)", child->name, position);
	CRM_LOG_ASSERT(position == 0);
	xmlAddChild(match, child);
	}
	crm_node_created(child);

	} else if(strcmp(op, "move") == 0) {
	int position = 0;

	crm_element_value_int(change, XML_DIFF_POSITION, &position);
	if(position != __xml_offset(match)) {
	xmlNode *match_child = NULL;
	int p = position;

	if(p > __xml_offset(match)) {
	p++; /* Skip ourselves */
	}

	CRM_ASSERT(match->parent != NULL);
	match_child = match->parent->children;

	while(match_child && p != __xml_offset(match_child)) {
	match_child = match_child->next;
	}

	crm_trace("Moving %s to position %d (was %d, prev %p, %s %p)",
	match->name, position, __xml_offset(match), match->prev,
	match_child?"next":"last", match_child?match_child:match->parent->last);

	if(match_child) {
	xmlAddPrevSibling(match_child, match);

	} else {
	CRM_ASSERT(match->parent->last != NULL);
	xmlAddNextSibling(match->parent->last, match);
	}

	} else {
	crm_trace("%s is already in position %d", match->name, position);
	}

	if(position != __xml_offset(match)) {
	crm_err("Moved %s.%d to position %d instead of %d (%p)",
	match->name, ID(match), __xml_offset(match), position, match->prev);
	rc = -pcmk_err_diff_failed;
	}

	} else if(strcmp(op, "delete") == 0) {
	free_xml(match);

	} else if(strcmp(op, "modify") == 0) {
	xmlAttr *pIter = crm_first_attr(match);
	xmlNode *attrs = __xml_first_child(first_named_child(change, XML_DIFF_RESULT));

	if(attrs == NULL) {
	rc = -ENOMSG;
	continue;
	}
	while(pIter != NULL) {
	const char name = (const char )pIter->name;

	pIter = pIter->next;
	xml_remove_prop(match, name);
	}

	for (pIter = crm_first_attr(attrs); pIter != NULL; pIter = pIter->next) {
	const char name = (const char )pIter->name;
	const char *value = crm_element_value(attrs, name);

	crm_xml_add(match, name, value);
	}

	} else {
	crm_err("Unknown operation: %s", op);
	}
	}
	return rc;
	}

	int
	xml_apply_patchset(xmlNode xml, xmlNode patchset, bool check_version)
	{
	int format = 1;
	int rc = pcmk_ok;
	xmlNode *old = NULL;
	const char *digest = crm_element_value(patchset, XML_ATTR_DIGEST);

	if(patchset == NULL) {
	return rc;
	}

	xml_log_patchset(LOG_TRACE, __FUNCTION__, patchset);

	crm_element_value_int(patchset, "format", &format);
	if(check_version) {
	rc = xml_patch_version_check(xml, patchset, format);
	if(rc != pcmk_ok) {
	return rc;
	}
	}

	if(digest) {
	/* Make it available for logging if the result doesn't have the expected digest */
	old = copy_xml(xml);
	}

	if(rc == pcmk_ok) {
	switch(format) {
	case 1:
	rc = xml_apply_patchset_v1(xml, patchset, check_version);
	break;
	case 2:
	rc = xml_apply_patchset_v2(xml, patchset, check_version);
	break;
	default:
	crm_err("Unknown patch format: %d", format);
	rc = -EINVAL;
	}
	}

	if(rc == pcmk_ok && digest) {
	static struct qb_log_callsite *digest_cs = NULL;

	char *new_digest = NULL;
	char *version = crm_element_value_copy(xml, XML_ATTR_CRM_VERSION);

	if (digest_cs == NULL) {
	digest_cs =
	qb_log_callsite_get(__func__, __FILE__, "diff-digest", LOG_TRACE, __LINE__,
	crm_trace_nonlog);
	}

	new_digest = calculate_xml_versioned_digest(xml, FALSE, TRUE, version);
	if (safe_str_neq(new_digest, digest)) {
	crm_info("v%d digest mis-match: expected %s, calculated %s", format, digest, new_digest);
	rc = -pcmk_err_diff_failed;

	if (digest_cs && digest_cs->targets) {
	save_xml_to_file(old, "PatchDigest:input", NULL);
	save_xml_to_file(xml, "PatchDigest:result", NULL);
	save_xml_to_file(patchset,"PatchDigest:diff", NULL);

	} else {
	crm_trace("%p %0.6x", digest_cs, digest_cs ? digest_cs->targets : 0);
	}

	} else {
	crm_trace("v%d digest matched: expected %s, calculated %s", format, digest, new_digest);
	}
	free(new_digest);
	free(version);
	}
	free_xml(old);
	return rc;
	}

	xmlNode *
	find_xml_node(xmlNode * root, const char *search_path, gboolean must_find)
	{
	xmlNode *a_child = NULL;
	const char *name = "NULL";

	if (root != NULL) {
	name = crm_element_name(root);
	}

	if (search_path == NULL) {
	crm_warn("Will never find <NULL>");
	return NULL;
	}

	for (a_child = __xml_first_child(root); a_child != NULL; a_child = __xml_next(a_child)) {
	if (strcmp((const char *)a_child->name, search_path) == 0) {
	/* crm_trace("returning node (%s).", crm_element_name(a_child)); */
	return a_child;
	}
	}

	if (must_find) {
	crm_warn("Could not find %s in %s.", search_path, name);
	} else if (root != NULL) {
	crm_trace("Could not find %s in %s.", search_path, name);
	} else {
	crm_trace("Could not find %s in <NULL>.", search_path);
	}

	return NULL;
	}

	xmlNode *
	find_entity(xmlNode * parent, const char node_name, const char id)
	{
	xmlNode *a_child = NULL;

	for (a_child = __xml_first_child(parent); a_child != NULL; a_child = __xml_next(a_child)) {
	/* Uncertain if node_name == NULL check is strictly necessary here */
	if (node_name == NULL \|\| strcmp((const char *)a_child->name, node_name) == 0) {
	const char *cid = ID(a_child);
	if (id == NULL \|\| (cid != NULL && strcmp(id, cid) == 0)) {
	return a_child;
	}
	}
	}

	crm_trace("node <%s id=%s> not found in %s.", node_name, id, crm_element_name(parent));
	return NULL;
	}

	void
	copy_in_properties(xmlNode * target, xmlNode * src)
	{
	if (src == NULL) {
	crm_warn("No node to copy properties from");

	} else if (target == NULL) {
	crm_err("No node to copy properties into");

	} else {
	xmlAttrPtr pIter = NULL;

	for (pIter = crm_first_attr(src); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	expand_plus_plus(target, p_name, p_value);
	}
	}

	return;
	}

	void
	fix_plus_plus_recursive(xmlNode * target)
	{
	/* TODO: Remove recursion and use xpath searches for value++ */
	xmlNode *child = NULL;
	xmlAttrPtr pIter = NULL;

	for (pIter = crm_first_attr(target); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	expand_plus_plus(target, p_name, p_value);
	}
	for (child = __xml_first_child(target); child != NULL; child = __xml_next(child)) {
	fix_plus_plus_recursive(child);
	}
	}

	void
	expand_plus_plus(xmlNode * target, const char name, const char value)
	{
	int offset = 1;
	int name_len = 0;
	int int_value = 0;
	int value_len = 0;

	const char *old_value = NULL;

	if (value == NULL \|\| name == NULL) {
	return;
	}

	old_value = crm_element_value(target, name);

	if (old_value == NULL) {
	/* if no previous value, set unexpanded */
	goto set_unexpanded;

	} else if (strstr(value, name) != value) {
	goto set_unexpanded;
	}

	name_len = strlen(name);
	value_len = strlen(value);
	if (value_len < (name_len + 2)
	\|\| value[name_len] != '+' \|\| (value[name_len + 1] != '+' && value[name_len + 1] != '=')) {
	goto set_unexpanded;
	}

	/* if we are expanding ourselves,
	* then no previous value was set and leave int_value as 0
	*/
	if (old_value != value) {
	int_value = char2score(old_value);
	}

	if (value[name_len + 1] != '+') {
	const char *offset_s = value + (name_len + 2);

	offset = char2score(offset_s);
	}
	int_value += offset;

	if (int_value > INFINITY) {
	int_value = (int)INFINITY;
	}

	crm_xml_add_int(target, name, int_value);
	return;

	set_unexpanded:
	if (old_value == value) {
	/* the old value is already set, nothing to do */
	return;
	}
	crm_xml_add(target, name, value);
	return;
	}

	xmlDoc *
	getDocPtr(xmlNode * node)
	{
	xmlDoc *doc = NULL;

	CRM_CHECK(node != NULL, return NULL);

	doc = node->doc;
	if (doc == NULL) {
	doc = xmlNewDoc((const xmlChar *)"1.0");
	xmlDocSetRootElement(doc, node);
	xmlSetTreeDoc(node, doc);
	}
	return doc;
	}

	xmlNode *
	add_node_copy(xmlNode * parent, xmlNode * src_node)
	{
	xmlNode *child = NULL;
	xmlDoc *doc = getDocPtr(parent);

	CRM_CHECK(src_node != NULL, return NULL);

	child = xmlDocCopyNode(src_node, doc, 1);
	xmlAddChild(parent, child);
	crm_node_created(child);
	return child;
	}

	int
	add_node_nocopy(xmlNode * parent, const char name, xmlNode child)
	{
	add_node_copy(parent, child);
	free_xml(child);
	return 1;
	}

	static bool
	__xml_acl_check(xmlNode xml, const char name, enum xml_private_flags mode)
	{
	CRM_ASSERT(xml);
	CRM_ASSERT(xml->doc);
	CRM_ASSERT(xml->doc->_private);

	#if ENABLE_ACL
	{
	if(TRACKING_CHANGES(xml) && xml_acl_enabled(xml)) {
	int offset = 0;
	xmlNode *parent = xml;
	char buffer[XML_BUFFER_SIZE];
	xml_private_t *docp = xml->doc->_private;

	if(docp->acls == NULL) {
	crm_trace("Ordinary user %s cannot access the CIB without any defined ACLs", docp->user);
	set_doc_flag(xml, xpf_acl_denied);
	return FALSE;
	}

	offset = __get_prefix(NULL, xml, buffer, offset);
	if(name) {
	offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "[@%s]", name);
	}
	CRM_LOG_ASSERT(offset > 0);

	/* Walk the tree upwards looking for xml_acl_* flags
	* - Creating an attribute requires write permissions for the node
	* - Creating a child requires write permissions for the parent
	*/

	if(name) {
	xmlAttr attr = xmlHasProp(xml, (const xmlChar )name);

	if(attr && mode == xpf_acl_create) {
	mode = xpf_acl_write;
	}
	}

	while(parent && parent->_private) {
	xml_private_t *p = parent->_private;
	if(__xml_acl_mode_test(p->flags, mode)) {
	return TRUE;

	} else if(is_set(p->flags, xpf_acl_deny)) {
	crm_trace("%x access denied to %s: parent", mode, buffer);
	set_doc_flag(xml, xpf_acl_denied);
	return FALSE;
	}
	parent = parent->parent;
	}

	crm_trace("%x access denied to %s: default", mode, buffer);
	set_doc_flag(xml, xpf_acl_denied);
	return FALSE;
	}
	}
	#endif

	return TRUE;
	}

	const char *
	crm_xml_add(xmlNode * node, const char name, const char value)
	{
	bool dirty = FALSE;
	xmlAttr *attr = NULL;

	CRM_CHECK(node != NULL, return NULL);
	CRM_CHECK(name != NULL, return NULL);

	if (value == NULL) {
	return NULL;
	}
	#if XML_PARANOIA_CHECKS
	{
	const char *old_value = NULL;

	old_value = crm_element_value(node, name);

	/* Could be re-setting the same value */
	CRM_CHECK(old_value != value, crm_err("Cannot reset %s with crm_xml_add(%s)", name, value);
	return value);
	}
	#endif

	if(TRACKING_CHANGES(node)) {
	const char *old = crm_element_value(node, name);

	if(old == NULL \|\| value == NULL \|\| strcmp(old, value) != 0) {
	dirty = TRUE;
	}
	}

	if(dirty && __xml_acl_check(node, name, xpf_acl_create) == FALSE) {
	crm_trace("Cannot add %s=%s to %s", name, value, node->name);
	return NULL;
	}

	attr = xmlSetProp(node, (const xmlChar )name, (const xmlChar )value);
	if(dirty) {
	crm_attr_dirty(attr);
	}

	CRM_CHECK(attr && attr->children && attr->children->content, return NULL);
	return (char *)attr->children->content;
	}

	const char *
	crm_xml_replace(xmlNode * node, const char name, const char value)
	{
	bool dirty = FALSE;
	xmlAttr *attr = NULL;
	const char *old_value = NULL;

	CRM_CHECK(node != NULL, return NULL);
	CRM_CHECK(name != NULL && name[0] != 0, return NULL);

	old_value = crm_element_value(node, name);

	/* Could be re-setting the same value */
	CRM_CHECK(old_value != value, return value);

	if(__xml_acl_check(node, name, xpf_acl_write) == FALSE) {
	/* Create a fake object linked to doc->_private instead? */
	crm_trace("Cannot replace %s=%s to %s", name, value, node->name);
	return NULL;

	} else if (old_value != NULL && value == NULL) {
	xml_remove_prop(node, name);
	return NULL;

	} else if (value == NULL) {
	return NULL;
	}

	if(TRACKING_CHANGES(node)) {
	if(old_value == NULL \|\| value == NULL \|\| strcmp(old_value, value) != 0) {
	dirty = TRUE;
	}
	}

	attr = xmlSetProp(node, (const xmlChar )name, (const xmlChar )value);
	if(dirty) {
	crm_attr_dirty(attr);
	}
	CRM_CHECK(attr && attr->children && attr->children->content, return NULL);
	return (char *)attr->children->content;
	}

	const char *
	crm_xml_add_int(xmlNode * node, const char *name, int value)
	{
	char *number = crm_itoa(value);
	const char *added = crm_xml_add(node, name, number);

	free(number);
	return added;
	}

	xmlNode *
	create_xml_node(xmlNode * parent, const char *name)
	{
	xmlDoc *doc = NULL;
	xmlNode *node = NULL;

	if (name == NULL \|\| name[0] == 0) {
	CRM_CHECK(name != NULL && name[0] == 0, return NULL);
	return NULL;
	}

	if (parent == NULL) {
	doc = xmlNewDoc((const xmlChar *)"1.0");
	node = xmlNewDocRawNode(doc, NULL, (const xmlChar *)name, NULL);
	xmlDocSetRootElement(doc, node);

	} else {
	doc = getDocPtr(parent);
	node = xmlNewDocRawNode(doc, NULL, (const xmlChar *)name, NULL);
	xmlAddChild(parent, node);
	}
	crm_node_created(node);
	return node;
	}

	static inline int
	__get_prefix(const char prefix, xmlNode xml, char *buffer, int offset)
	{
	const char *id = ID(xml);

	if(offset == 0 && prefix == NULL && xml->parent) {
	offset = __get_prefix(NULL, xml->parent, buffer, offset);
	}

	if(id) {
	offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "/%s[@id='%s']", (const char *)xml->name, id);
	} else if(xml->name) {
	offset += snprintf(buffer + offset, XML_BUFFER_SIZE - offset, "/%s", (const char *)xml->name);
	}

	return offset;
	}

	char *
	xml_get_path(xmlNode *xml)
	{
	int offset = 0;
	char buffer[XML_BUFFER_SIZE];

	if(__get_prefix(NULL, xml, buffer, offset) > 0) {
	return strdup(buffer);
	}
	return NULL;
	}

	void
	free_xml(xmlNode * child)
	{
	if (child != NULL) {
	xmlNode *top = NULL;
	xmlDoc *doc = child->doc;
	xml_private_t *p = child->_private;

	if (doc != NULL) {
	top = xmlDocGetRootElement(doc);
	}

	if (doc != NULL && top == child) {
	/* Free everything */
	xmlFreeDoc(doc);

	} else if(__xml_acl_check(child, NULL, xpf_acl_write) == FALSE) {
	int offset = 0;
	char buffer[XML_BUFFER_SIZE];

	__get_prefix(NULL, child, buffer, offset);
	crm_trace("Cannot remove %s %x", buffer, p->flags);
	return;

	} else {
	if(doc && TRACKING_CHANGES(child) && is_not_set(p->flags, xpf_created)) {
	int offset = 0;
	char buffer[XML_BUFFER_SIZE];

	if(__get_prefix(NULL, child, buffer, offset) > 0) {
	crm_trace("Deleting %s %p from %p", buffer, child, doc);
	p = doc->_private;
	p->deleted_paths = g_list_append(p->deleted_paths, strdup(buffer));
	set_doc_flag(child, xpf_dirty);
	}
	}

	/* Free this particular subtree
	* Make sure to unlink it from the parent first
	*/
	xmlUnlinkNode(child);
	xmlFreeNode(child);
	}
	}
	}

	xmlNode *
	copy_xml(xmlNode * src)
	{
	xmlDoc doc = xmlNewDoc((const xmlChar )"1.0");
	xmlNode *copy = xmlDocCopyNode(src, doc, 1);

	xmlDocSetRootElement(doc, copy);
	xmlSetTreeDoc(copy, doc);
	return copy;
	}

	static void
	crm_xml_err(void ctx, const char msg, ...)
	G_GNUC_PRINTF(2, 3);

	static void
	crm_xml_err(void ctx, const char msg, ...)
	{
	int len = 0;
	va_list args;
	char *buf = NULL;
	static int buffer_len = 0;
	static char *buffer = NULL;
	static struct qb_log_callsite *xml_error_cs = NULL;

	va_start(args, msg);
	len = vasprintf(&buf, msg, args);

	if(xml_error_cs == NULL) {
	xml_error_cs = qb_log_callsite_get(
	__func__, __FILE__, "xml library error", LOG_TRACE, __LINE__, crm_trace_nonlog);
	}

	if (strchr(buf, '\n')) {
	buf[len - 1] = 0;
	if (buffer) {
	crm_err("XML Error: %s%s", buffer, buf);
	free(buffer);
	} else {
	crm_err("XML Error: %s", buf);
	}
	if (xml_error_cs && xml_error_cs->targets) {
	crm_abort(__FILE__, __PRETTY_FUNCTION__, __LINE__, "xml library error", TRUE, TRUE);
	}
	buffer = NULL;
	buffer_len = 0;

	} else if (buffer == NULL) {
	buffer_len = len;
	buffer = buf;
	buf = NULL;

	} else {
	buffer = realloc_safe(buffer, 1 + buffer_len + len);
	memcpy(buffer + buffer_len, buf, len);
	buffer_len += len;
	buffer[buffer_len] = 0;
	}

	va_end(args);
	free(buf);
	}

	xmlNode *
	string2xml(const char *input)
	{
	xmlNode *xml = NULL;
	xmlDocPtr output = NULL;
	xmlParserCtxtPtr ctxt = NULL;
	xmlErrorPtr last_error = NULL;

	if (input == NULL) {
	crm_err("Can't parse NULL input");
	return NULL;
	}

	/* create a parser context */
	ctxt = xmlNewParserCtxt();
	CRM_CHECK(ctxt != NULL, return NULL);

	/* xmlCtxtUseOptions(ctxt, XML_PARSE_NOBLANKS\|XML_PARSE_RECOVER); */

	xmlCtxtResetLastError(ctxt);
	xmlSetGenericErrorFunc(ctxt, crm_xml_err);
	/* initGenericErrorDefaultFunc(crm_xml_err); */
	output =
	xmlCtxtReadDoc(ctxt, (const xmlChar *)input, NULL, NULL,
	XML_PARSE_NOBLANKS \| XML_PARSE_RECOVER);
	if (output) {
	xml = xmlDocGetRootElement(output);
	}
	last_error = xmlCtxtGetLastError(ctxt);
	if (last_error && last_error->code != XML_ERR_OK) {
	/* crm_abort(__FILE__,__FUNCTION__,__LINE__, "last_error->code != XML_ERR_OK", TRUE, TRUE); */
	/*
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlErrorLevel
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlParserErrors
	*/
	crm_warn("Parsing failed (domain=%d, level=%d, code=%d): %s",
	last_error->domain, last_error->level, last_error->code, last_error->message);

	if (last_error->code == XML_ERR_DOCUMENT_EMPTY) {
	CRM_LOG_ASSERT("Cannot parse an empty string");

	} else if (last_error->code != XML_ERR_DOCUMENT_END) {
	crm_err("Couldn't%s parse %d chars: %s", xml ? " fully" : "", (int)strlen(input),
	input);
	if (xml != NULL) {
	crm_log_xml_err(xml, "Partial");
	}

	} else {
	int len = strlen(input);
	int lpc = 0;

	while(lpc < len) {
	crm_warn("Parse error[+%.3d]: %.80s", lpc, input+lpc);
	lpc += 80;
	}

	CRM_LOG_ASSERT("String parsing error");
	}
	}

	xmlFreeParserCtxt(ctxt);
	return xml;
	}

	xmlNode *
	stdin2xml(void)
	{
	size_t data_length = 0;
	size_t read_chars = 0;

	char *xml_buffer = NULL;
	xmlNode *xml_obj = NULL;

	do {
	size_t next = XML_BUFFER_SIZE + data_length + 1;

	if(next <= 0) {
	crm_err("Buffer size exceeded at: %l + %d", data_length, XML_BUFFER_SIZE);
	break;
	}

	xml_buffer = realloc_safe(xml_buffer, next);
	read_chars = fread(xml_buffer + data_length, 1, XML_BUFFER_SIZE, stdin);
	data_length += read_chars;
	} while (read_chars > 0);

	if (data_length == 0) {
	crm_warn("No XML supplied on stdin");
	free(xml_buffer);
	return NULL;
	}

	xml_buffer[data_length] = '\0';

	xml_obj = string2xml(xml_buffer);
	free(xml_buffer);

	crm_log_xml_trace(xml_obj, "Created fragment");
	return xml_obj;
	}

	static char *
	decompress_file(const char *filename)
	{
	char *buffer = NULL;

	#if HAVE_BZLIB_H
	int rc = 0;
	size_t length = 0, read_len = 0;

	BZFILE *bz_file = NULL;
	FILE *input = fopen(filename, "r");

	if (input == NULL) {
	crm_perror(LOG_ERR, "Could not open %s for reading", filename);
	return NULL;
	}

	bz_file = BZ2_bzReadOpen(&rc, input, 0, 0, NULL, 0);

	if (rc != BZ_OK) {
	BZ2_bzReadClose(&rc, bz_file);
	return NULL;
	}

	rc = BZ_OK;
	while (rc == BZ_OK) {
	buffer = realloc_safe(buffer, XML_BUFFER_SIZE + length + 1);
	read_len = BZ2_bzRead(&rc, bz_file, buffer + length, XML_BUFFER_SIZE);

	crm_trace("Read %ld bytes from file: %d", (long)read_len, rc);

	if (rc == BZ_OK \|\| rc == BZ_STREAM_END) {
	length += read_len;
	}
	}

	buffer[length] = '\0';

	if (rc != BZ_STREAM_END) {
	- crm_err("Couldnt read compressed xml from file");
	+ crm_err("Couldn't read compressed xml from file");
	free(buffer);
	buffer = NULL;
	}

	BZ2_bzReadClose(&rc, bz_file);
	fclose(input);

	#else
	crm_err("Cannot read compressed files:" " bzlib was not available at compile time");
	#endif
	return buffer;
	}

	void
	strip_text_nodes(xmlNode * xml)
	{
	xmlNode *iter = xml->children;

	while (iter) {
	xmlNode *next = iter->next;

	switch (iter->type) {
	case XML_TEXT_NODE:
	/* Remove it */
	xmlUnlinkNode(iter);
	xmlFreeNode(iter);
	break;

	case XML_ELEMENT_NODE:
	/* Search it */
	strip_text_nodes(iter);
	break;

	default:
	/* Leave it */
	break;
	}

	iter = next;
	}
	}

	xmlNode *
	filename2xml(const char *filename)
	{
	xmlNode *xml = NULL;
	xmlDocPtr output = NULL;
	const char *match = NULL;
	xmlParserCtxtPtr ctxt = NULL;
	xmlErrorPtr last_error = NULL;
	static int xml_options = XML_PARSE_NOBLANKS \| XML_PARSE_RECOVER;

	/* create a parser context */
	ctxt = xmlNewParserCtxt();
	CRM_CHECK(ctxt != NULL, return NULL);

	/* xmlCtxtUseOptions(ctxt, XML_PARSE_NOBLANKS\|XML_PARSE_RECOVER); */

	xmlCtxtResetLastError(ctxt);
	xmlSetGenericErrorFunc(ctxt, crm_xml_err);
	/* initGenericErrorDefaultFunc(crm_xml_err); */

	if (filename) {
	match = strstr(filename, ".bz2");
	}

	if (filename == NULL) {
	/* STDIN_FILENO == fileno(stdin) */
	output = xmlCtxtReadFd(ctxt, STDIN_FILENO, "unknown.xml", NULL, xml_options);

	} else if (match == NULL \|\| match[4] != 0) {
	output = xmlCtxtReadFile(ctxt, filename, NULL, xml_options);

	} else {
	char *input = decompress_file(filename);

	output = xmlCtxtReadDoc(ctxt, (const xmlChar *)input, NULL, NULL, xml_options);
	free(input);
	}

	if (output && (xml = xmlDocGetRootElement(output))) {
	strip_text_nodes(xml);
	}

	last_error = xmlCtxtGetLastError(ctxt);
	if (last_error && last_error->code != XML_ERR_OK) {
	/* crm_abort(__FILE__,__FUNCTION__,__LINE__, "last_error->code != XML_ERR_OK", TRUE, TRUE); */
	/*
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlErrorLevel
	* http://xmlsoft.org/html/libxml-xmlerror.html#xmlParserErrors
	*/
	crm_err("Parsing failed (domain=%d, level=%d, code=%d): %s",
	last_error->domain, last_error->level, last_error->code, last_error->message);

	if (last_error && last_error->code != XML_ERR_OK) {
	crm_err("Couldn't%s parse %s", xml ? " fully" : "", filename);
	if (xml != NULL) {
	crm_log_xml_err(xml, "Partial");
	}
	}
	}

	xmlFreeParserCtxt(ctxt);
	return xml;
	}

	/*!
	* \internal
	* \brief Add a "last written" attribute to an XML node, set to current time
	*
	* \param[in] xml_node XML node to get attribute
	*
	* \return Value that was set, or NULL on error
	*/
	const char *
	crm_xml_add_last_written(xmlNode *xml_node)
	{
	time_t now = time(NULL);
	char *now_str = ctime(&now);

	now_str[24] = EOS; /* replace the newline */
	return crm_xml_add(xml_node, XML_CIB_ATTR_WRITTEN, now_str);
	}

	static int
	write_xml_stream(xmlNode * xml_node, const char filename, FILE stream, gboolean compress)
	{
	int res = 0;
	char *buffer = NULL;
	unsigned int out = 0;

	CRM_CHECK(stream != NULL, return -1);

	crm_trace("Writing XML out to %s", filename);
	if (xml_node == NULL) {
	crm_err("Cannot write NULL to %s", filename);
	fclose(stream);
	return -1;
	}


	crm_log_xml_trace(xml_node, "Writing out");

	buffer = dump_xml_formatted(xml_node);
	CRM_CHECK(buffer != NULL && strlen(buffer) > 0, crm_log_xml_warn(xml_node, "dump:failed");
	goto bail);

	if (compress) {
	#if HAVE_BZLIB_H
	int rc = BZ_OK;
	unsigned int in = 0;
	BZFILE *bz_file = NULL;

	bz_file = BZ2_bzWriteOpen(&rc, stream, 5, 0, 30);
	if (rc != BZ_OK) {
	crm_err("bzWriteOpen failed: %d", rc);
	} else {
	BZ2_bzWrite(&rc, bz_file, buffer, strlen(buffer));
	if (rc != BZ_OK) {
	crm_err("bzWrite() failed: %d", rc);
	}
	}

	if (rc == BZ_OK) {
	BZ2_bzWriteClose(&rc, bz_file, 0, &in, &out);
	if (rc != BZ_OK) {
	crm_err("bzWriteClose() failed: %d", rc);
	out = -1;
	} else {
	crm_trace("%s: In: %d, out: %d", filename, in, out);
	}
	}
	#else
	crm_err("Cannot write compressed files:" " bzlib was not available at compile time");
	#endif
	}

	if (out <= 0) {
	res = fprintf(stream, "%s", buffer);
	if (res < 0) {
	crm_perror(LOG_ERR, "Cannot write output to %s", filename);
	goto bail;
	}
	}

	bail:

	if (fflush(stream) != 0) {
	crm_perror(LOG_ERR, "fflush for %s failed:", filename);
	res = -1;
	}

	if (fsync(fileno(stream)) < 0) {
	crm_perror(LOG_ERR, "fsync for %s failed:", filename);
	res = -1;
	}

	fclose(stream);

	crm_trace("Saved %d bytes to the Cib as XML", res);
	free(buffer);

	return res;
	}

	int
	write_xml_fd(xmlNode * xml_node, const char *filename, int fd, gboolean compress)
	{
	FILE *stream = NULL;

	CRM_CHECK(fd > 0, return -1);
	stream = fdopen(fd, "w");
	return write_xml_stream(xml_node, filename, stream, compress);
	}

	int
	write_xml_file(xmlNode * xml_node, const char *filename, gboolean compress)
	{
	FILE *stream = NULL;

	stream = fopen(filename, "w");

	return write_xml_stream(xml_node, filename, stream, compress);
	}

	xmlNode *
	get_message_xml(xmlNode * msg, const char *field)
	{
	xmlNode *tmp = first_named_child(msg, field);

	return __xml_first_child(tmp);
	}

	gboolean
	add_message_xml(xmlNode * msg, const char field, xmlNode xml)
	{
	xmlNode *holder = create_xml_node(msg, field);

	add_node_copy(holder, xml);
	return TRUE;
	}

	static char *
	crm_xml_escape_shuffle(char text, int start, int length, const char *replace)
	{
	int lpc;
	int offset = strlen(replace) - 1; /* We have space for 1 char already */

	*length += offset;
	text = realloc_safe(text, *length);

	for (lpc = (*length) - 1; lpc > (start + offset); lpc--) {
	text[lpc] = text[lpc - offset];
	}

	memcpy(text + start, replace, offset + 1);
	return text;
	}

	char *
	crm_xml_escape(const char *text)
	{
	int index;
	int changes = 0;
	int length = 1 + strlen(text);
	char *copy = strdup(text);

	/*
	* When xmlCtxtReadDoc() parses < and friends in a
	* value, it converts them to their human readable
	* form.
	*
	* If one uses xmlNodeDump() to convert it back to a
	* string, all is well, because special characters are
	* converted back to their escape sequences.
	*
	* However xmlNodeDump() is randomly dog slow, even with the same
	* input. So we need to replicate the escapeing in our custom
	* version so that the result can be re-parsed by xmlCtxtReadDoc()
	* when necessary.
	*/

	for (index = 0; index < length; index++) {
	switch (copy[index]) {
	case 0:
	break;
	case '<':
	copy = crm_xml_escape_shuffle(copy, index, &length, "<");
	changes++;
	break;
	case '>':
	copy = crm_xml_escape_shuffle(copy, index, &length, ">");
	changes++;
	break;
	case '"':
	copy = crm_xml_escape_shuffle(copy, index, &length, """);
	changes++;
	break;
	case '\'':
	copy = crm_xml_escape_shuffle(copy, index, &length, "'");
	changes++;
	break;
	case '&':
	copy = crm_xml_escape_shuffle(copy, index, &length, "&");
	changes++;
	break;
	case '\t':
	/* Might as well just expand to a few spaces... */
	copy = crm_xml_escape_shuffle(copy, index, &length, " ");
	changes++;
	break;
	case '\n':
	/* crm_trace("Convert: \\%.3o", copy[index]); */
	copy = crm_xml_escape_shuffle(copy, index, &length, "\\n");
	changes++;
	break;
	case '\r':
	copy = crm_xml_escape_shuffle(copy, index, &length, "\\r");
	changes++;
	break;
	/* For debugging...
	case '\\':
	crm_trace("Passthrough: \\%c", copy[index+1]);
	break;
	*/
	default:
	/* Check for and replace non-printing characters with their octal equivalent */
	if(copy[index] < ' ' \|\| copy[index] > '~') {
	char *replace = crm_strdup_printf("\\%.3o", copy[index]);

	/* crm_trace("Convert to octal: \\%.3o", copy[index]); */
	copy = crm_xml_escape_shuffle(copy, index, &length, replace);
	free(replace);
	changes++;
	}
	}
	}

	if (changes) {
	crm_trace("Dumped '%s'", copy);
	}
	return copy;
	}

	static inline void
	dump_xml_attr(xmlAttrPtr attr, int options, char *buffer, int offset, int *max)
	{
	char *p_value = NULL;
	const char *p_name = NULL;
	xml_private_t *p = NULL;

	CRM_ASSERT(buffer != NULL);
	if (attr == NULL \|\| attr->children == NULL) {
	return;
	}

	p = attr->_private;
	if (p && is_set(p->flags, xpf_deleted)) {
	return;
	}

	p_name = (const char *)attr->name;
	p_value = crm_xml_escape((const char *)attr->children->content);
	buffer_print(buffer, max, *offset, " %s=\"%s\"", p_name, p_value);
	free(p_value);
	}

	static void
	__xml_log_element(int log_level, const char file, const char function, int line,
	const char prefix, xmlNode data, int depth, int options)
	{
	int max = 0;
	int offset = 0;
	const char *name = NULL;
	const char *hidden = NULL;

	xmlNode *child = NULL;
	xmlAttrPtr pIter = NULL;

	if(data == NULL) {
	return;
	}

	name = crm_element_name(data);

	if(is_set(options, xml_log_option_open)) {
	char *buffer = NULL;

	insert_prefix(options, &buffer, &offset, &max, depth);
	if(data->type == XML_COMMENT_NODE) {
	buffer_print(buffer, max, offset, "<!--");
	buffer_print(buffer, max, offset, "%s", data->content);
	buffer_print(buffer, max, offset, "-->");

	} else {
	buffer_print(buffer, max, offset, "<%s", name);
	}

	hidden = crm_element_value(data, "hidden");
	for (pIter = crm_first_attr(data); pIter != NULL; pIter = pIter->next) {
	xml_private_t *p = pIter->_private;
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);
	char *p_copy = NULL;

	if(is_set(p->flags, xpf_deleted)) {
	continue;
	} else if ((is_set(options, xml_log_option_diff_plus)
	\|\| is_set(options, xml_log_option_diff_minus))
	&& strcmp(XML_DIFF_MARKER, p_name) == 0) {
	continue;

	} else if (hidden != NULL && p_name[0] != 0 && strstr(hidden, p_name) != NULL) {
	p_copy = strdup("*****");

	} else {
	p_copy = crm_xml_escape(p_value);
	}

	buffer_print(buffer, max, offset, " %s=\"%s\"", p_name, p_copy);
	free(p_copy);
	}

	if(xml_has_children(data) == FALSE) {
	buffer_print(buffer, max, offset, "/>");

	} else if(is_set(options, xml_log_option_children)) {
	buffer_print(buffer, max, offset, ">");

	} else {
	buffer_print(buffer, max, offset, "/>");
	}

	do_crm_log_alias(log_level, file, function, line, "%s %s", prefix, buffer);
	free(buffer);
	}

	if(data->type == XML_COMMENT_NODE) {
	return;

	} else if(xml_has_children(data) == FALSE) {
	return;

	} else if(is_set(options, xml_log_option_children)) {
	offset = 0;
	max = 0;

	for (child = __xml_first_child(data); child != NULL; child = __xml_next(child)) {
	__xml_log_element(log_level, file, function, line, prefix, child, depth + 1, options\|xml_log_option_open\|xml_log_option_close);
	}
	}

	if(is_set(options, xml_log_option_close)) {
	char *buffer = NULL;

	insert_prefix(options, &buffer, &offset, &max, depth);
	buffer_print(buffer, max, offset, "</%s>", name);

	do_crm_log_alias(log_level, file, function, line, "%s %s", prefix, buffer);
	free(buffer);
	}
	}

	static void
	__xml_log_change_element(int log_level, const char file, const char function, int line,
	const char prefix, xmlNode data, int depth, int options)
	{
	xml_private_t *p;
	char *prefix_m = NULL;
	xmlNode *child = NULL;
	xmlAttrPtr pIter = NULL;

	if(data == NULL) {
	return;
	}

	p = data->_private;

	prefix_m = strdup(prefix);
	prefix_m[1] = '+';

	if(is_set(p->flags, xpf_dirty) && is_set(p->flags, xpf_created)) {
	/* Continue and log full subtree */
	__xml_log_element(log_level, file, function, line,
	prefix_m, data, depth, options\|xml_log_option_open\|xml_log_option_close\|xml_log_option_children);

	} else if(is_set(p->flags, xpf_dirty)) {
	char *spaces = calloc(80, 1);
	int s_count = 0, s_max = 80;
	char *prefix_del = NULL;
	char *prefix_moved = NULL;
	const char *flags = prefix;

	insert_prefix(options, &spaces, &s_count, &s_max, depth);
	prefix_del = strdup(prefix);
	prefix_del[0] = '-';
	prefix_del[1] = '-';
	prefix_moved = strdup(prefix);
	prefix_moved[1] = '~';

	if(is_set(p->flags, xpf_moved)) {
	flags = prefix_moved;
	} else {
	flags = prefix;
	}

	__xml_log_element(log_level, file, function, line,
	flags, data, depth, options\|xml_log_option_open);

	for (pIter = crm_first_attr(data); pIter != NULL; pIter = pIter->next) {
	const char aname = (const char)pIter->name;

	p = pIter->_private;
	if(is_set(p->flags, xpf_deleted)) {
	const char *value = crm_element_value(data, aname);
	flags = prefix_del;
	do_crm_log_alias(log_level, file, function, line,
	"%s %s @%s=%s", flags, spaces, aname, value);

	} else if(is_set(p->flags, xpf_dirty)) {
	const char *value = crm_element_value(data, aname);

	if(is_set(p->flags, xpf_created)) {
	flags = prefix_m;

	} else if(is_set(p->flags, xpf_modified)) {
	flags = prefix;

	} else if(is_set(p->flags, xpf_moved)) {
	flags = prefix_moved;

	} else {
	flags = prefix;
	}
	do_crm_log_alias(log_level, file, function, line,
	"%s %s @%s=%s", flags, spaces, aname, value);
	}
	}
	free(prefix_moved);
	free(prefix_del);
	free(spaces);

	for (child = __xml_first_child(data); child != NULL; child = __xml_next(child)) {
	__xml_log_change_element(log_level, file, function, line, prefix, child, depth + 1, options);
	}

	__xml_log_element(log_level, file, function, line,
	prefix, data, depth, options\|xml_log_option_close);

	} else {
	for (child = __xml_first_child(data); child != NULL; child = __xml_next(child)) {
	__xml_log_change_element(log_level, file, function, line, prefix, child, depth + 1, options);
	}
	}

	free(prefix_m);

	}

	void
	log_data_element(int log_level, const char file, const char function, int line,
	const char prefix, xmlNode data, int depth, int options)
	{
	xmlNode *a_child = NULL;

	char *prefix_m = NULL;

	if (prefix == NULL) {
	prefix = "";
	}

	/* Since we use the same file and line, to avoid confusing libqb, we need to use the same format strings */
	if (data == NULL) {
	do_crm_log_alias(log_level, file, function, line, "%s: %s", prefix,
	"No data to dump as XML");
	return;
	}

	if(is_set(options, xml_log_option_dirty_add) \|\| is_set(options, xml_log_option_dirty_add)) {
	__xml_log_change_element(log_level, file, function, line, prefix, data, depth, options);
	return;
	}

	if (is_set(options, xml_log_option_formatted)) {
	if (is_set(options, xml_log_option_diff_plus)
	&& (data->children == NULL \|\| crm_element_value(data, XML_DIFF_MARKER))) {
	options \|= xml_log_option_diff_all;
	prefix_m = strdup(prefix);
	prefix_m[1] = '+';
	prefix = prefix_m;

	} else if (is_set(options, xml_log_option_diff_minus)
	&& (data->children == NULL \|\| crm_element_value(data, XML_DIFF_MARKER))) {
	options \|= xml_log_option_diff_all;
	prefix_m = strdup(prefix);
	prefix_m[1] = '-';
	prefix = prefix_m;
	}
	}

	if (is_set(options, xml_log_option_diff_short)
	&& is_not_set(options, xml_log_option_diff_all)) {
	/* Still searching for the actual change */
	for (a_child = __xml_first_child(data); a_child != NULL; a_child = __xml_next(a_child)) {
	log_data_element(log_level, file, function, line, prefix, a_child, depth + 1, options);
	}
	} else {
	__xml_log_element(log_level, file, function, line, prefix, data, depth,
	options\|xml_log_option_open\|xml_log_option_close\|xml_log_option_children);
	}
	free(prefix_m);
	}

	static void
	dump_filtered_xml(xmlNode * data, int options, char *buffer, int offset, int *max)
	{
	int lpc;
	xmlAttrPtr xIter = NULL;
	static int filter_len = DIMOF(filter);

	for (lpc = 0; options && lpc < filter_len; lpc++) {
	filter[lpc].found = FALSE;
	}

	for (xIter = crm_first_attr(data); xIter != NULL; xIter = xIter->next) {
	bool skip = FALSE;
	const char p_name = (const char )xIter->name;

	for (lpc = 0; skip == FALSE && lpc < filter_len; lpc++) {
	if (filter[lpc].found == FALSE && strcmp(p_name, filter[lpc].string) == 0) {
	filter[lpc].found = TRUE;
	skip = TRUE;
	break;
	}
	}

	if (skip == FALSE) {
	dump_xml_attr(xIter, options, buffer, offset, max);
	}
	}
	}

	static void
	dump_xml_element(xmlNode * data, int options, char *buffer, int offset, int *max, int depth)
	{
	const char *name = NULL;

	CRM_ASSERT(max != NULL);
	CRM_ASSERT(offset != NULL);
	CRM_ASSERT(buffer != NULL);

	if (data == NULL) {
	crm_trace("Nothing to dump");
	return;
	}

	if (*buffer == NULL) {
	*offset = 0;
	*max = 0;
	}

	name = crm_element_name(data);
	CRM_ASSERT(name != NULL);

	insert_prefix(options, buffer, offset, max, depth);
	buffer_print(buffer, max, *offset, "<%s", name);

	if (options & xml_log_option_filtered) {
	dump_filtered_xml(data, options, buffer, offset, max);

	} else {
	xmlAttrPtr xIter = NULL;

	for (xIter = crm_first_attr(data); xIter != NULL; xIter = xIter->next) {
	dump_xml_attr(xIter, options, buffer, offset, max);
	}
	}

	if (data->children == NULL) {
	buffer_print(buffer, max, *offset, "/>");

	} else {
	buffer_print(buffer, max, *offset, ">");
	}

	if (options & xml_log_option_formatted) {
	buffer_print(buffer, max, *offset, "\n");
	}

	if (data->children) {
	xmlNode *xChild = NULL;
	for(xChild = data->children; xChild != NULL; xChild = xChild->next) {
	crm_xml_dump(xChild, options, buffer, offset, max, depth + 1);
	}

	insert_prefix(options, buffer, offset, max, depth);
	buffer_print(buffer, max, *offset, "</%s>", name);

	if (options & xml_log_option_formatted) {
	buffer_print(buffer, max, *offset, "\n");
	}
	}
	}

	static void
	dump_xml_text(xmlNode * data, int options, char *buffer, int offset, int *max, int depth)
	{
	CRM_ASSERT(max != NULL);
	CRM_ASSERT(offset != NULL);
	CRM_ASSERT(buffer != NULL);

	if (data == NULL) {
	crm_trace("Nothing to dump");
	return;
	}

	if (*buffer == NULL) {
	*offset = 0;
	*max = 0;
	}

	insert_prefix(options, buffer, offset, max, depth);

	buffer_print(buffer, max, *offset, "%s", data->content);

	if (options & xml_log_option_formatted) {
	buffer_print(buffer, max, *offset, "\n");
	}
	}


	static void
	dump_xml_comment(xmlNode * data, int options, char *buffer, int offset, int *max, int depth)
	{
	CRM_ASSERT(max != NULL);
	CRM_ASSERT(offset != NULL);
	CRM_ASSERT(buffer != NULL);

	if (data == NULL) {
	crm_trace("Nothing to dump");
	return;
	}

	if (*buffer == NULL) {
	*offset = 0;
	*max = 0;
	}

	insert_prefix(options, buffer, offset, max, depth);

	buffer_print(buffer, max, *offset, "<!--");
	buffer_print(buffer, max, *offset, "%s", data->content);
	buffer_print(buffer, max, *offset, "-->");

	if (options & xml_log_option_formatted) {
	buffer_print(buffer, max, *offset, "\n");
	}
	}

	void
	crm_xml_dump(xmlNode * data, int options, char *buffer, int offset, int *max, int depth)
	{
	if(data == NULL) {
	*offset = 0;
	*max = 0;
	return;
	}
	#if 0
	if (is_not_set(options, xml_log_option_filtered)) {
	/* Turning this code on also changes the PE tests for some reason
	* (not just newlines). Figure out why before considering to
	* enable this permanently.
	*
	* It exists to help debug slowness in xmlNodeDump() and
	* potentially if we ever want to go back to it.
	*
	* In theory its a good idea (reuse) but our custom version does
	* better for the filtered case and avoids the final strdup() for
	* everything
	*/

	time_t now, next;
	xmlDoc *doc = NULL;
	xmlBuffer *xml_buffer = NULL;

	*buffer = NULL;
	doc = getDocPtr(data);
	/* doc will only be NULL if data is */
	CRM_CHECK(doc != NULL, return);

	now = time(NULL);
	xml_buffer = xmlBufferCreate();
	CRM_ASSERT(xml_buffer != NULL);

	/* The default allocator XML_BUFFER_ALLOC_EXACT does far too many
	* realloc()s and it can take upwards of 18 seconds (yes, seconds)
	* to dump a 28kb tree which XML_BUFFER_ALLOC_DOUBLEIT can do in
	* less than 1 second.
	*
	* We could also use xmlBufferCreateSize() to start with a
	* sane-ish initial size and avoid the first few doubles.
	*/
	xmlBufferSetAllocationScheme(xml_buffer, XML_BUFFER_ALLOC_DOUBLEIT);

	*max = xmlNodeDump(xml_buffer, doc, data, 0, (options & xml_log_option_formatted));
	if (*max > 0) {
	buffer = strdup((char )xml_buffer->content);
	}

	next = time(NULL);
	if ((now + 1) < next) {
	crm_log_xml_trace(data, "Long time");
	crm_err("xmlNodeDump() -> %dbytes took %ds", *max, next - now);
	}

	xmlBufferFree(xml_buffer);
	return;
	}
	#endif

	switch(data->type) {
	case XML_ELEMENT_NODE:
	/* Handle below */
	dump_xml_element(data, options, buffer, offset, max, depth);
	break;
	case XML_TEXT_NODE:
	/* if option xml_log_option_text is enabled, then dump XML_TEXT_NODE */
	if (options & xml_log_option_text) {
	dump_xml_text(data, options, buffer, offset, max, depth);
	}
	return;
	case XML_COMMENT_NODE:
	dump_xml_comment(data, options, buffer, offset, max, depth);
	break;
	default:
	crm_warn("Unhandled type: %d", data->type);
	return;

	/*
	XML_ATTRIBUTE_NODE = 2
	XML_CDATA_SECTION_NODE = 4
	XML_ENTITY_REF_NODE = 5
	XML_ENTITY_NODE = 6
	XML_PI_NODE = 7
	XML_DOCUMENT_NODE = 9
	XML_DOCUMENT_TYPE_NODE = 10
	XML_DOCUMENT_FRAG_NODE = 11
	XML_NOTATION_NODE = 12
	XML_HTML_DOCUMENT_NODE = 13
	XML_DTD_NODE = 14
	XML_ELEMENT_DECL = 15
	XML_ATTRIBUTE_DECL = 16
	XML_ENTITY_DECL = 17
	XML_NAMESPACE_DECL = 18
	XML_XINCLUDE_START = 19
	XML_XINCLUDE_END = 20
	XML_DOCB_DOCUMENT_NODE = 21
	*/
	}

	}

	void
	crm_buffer_add_char(char *buffer, int offset, int *max, char c)
	{
	buffer_print(buffer, max, *offset, "%c", c);
	}

	char *
	dump_xml_formatted_with_text(xmlNode * an_xml_node)
	{
	char *buffer = NULL;
	int offset = 0, max = 0;

	crm_xml_dump(an_xml_node, xml_log_option_formatted\|xml_log_option_text, &buffer, &offset, &max, 0);
	return buffer;
	}

	char *
	dump_xml_formatted(xmlNode * an_xml_node)
	{
	char *buffer = NULL;
	int offset = 0, max = 0;

	crm_xml_dump(an_xml_node, xml_log_option_formatted, &buffer, &offset, &max, 0);
	return buffer;
	}

	char *
	dump_xml_unformatted(xmlNode * an_xml_node)
	{
	char *buffer = NULL;
	int offset = 0, max = 0;

	crm_xml_dump(an_xml_node, 0, &buffer, &offset, &max, 0);
	return buffer;
	}

	gboolean
	xml_has_children(const xmlNode * xml_root)
	{
	if (xml_root != NULL && xml_root->children != NULL) {
	return TRUE;
	}
	return FALSE;
	}

	int
	crm_element_value_int(xmlNode * data, const char name, int dest)
	{
	const char *value = crm_element_value(data, name);

	CRM_CHECK(dest != NULL, return -1);
	if (value) {
	*dest = crm_int_helper(value, NULL);
	return 0;
	}
	return -1;
	}

	int
	crm_element_value_const_int(const xmlNode * data, const char name, int dest)
	{
	return crm_element_value_int((xmlNode *) data, name, dest);
	}

	const char *
	crm_element_value_const(const xmlNode * data, const char *name)
	{
	return crm_element_value((xmlNode *) data, name);
	}

	char *
	crm_element_value_copy(xmlNode * data, const char *name)
	{
	char *value_copy = NULL;
	const char *value = crm_element_value(data, name);

	if (value != NULL) {
	value_copy = strdup(value);
	}
	return value_copy;
	}

	void
	xml_remove_prop(xmlNode * obj, const char *name)
	{
	if(__xml_acl_check(obj, NULL, xpf_acl_write) == FALSE) {
	crm_trace("Cannot remove %s from %s", name, obj->name);

	} else if(TRACKING_CHANGES(obj)) {
	/* Leave in place (marked for removal) until after the diff is calculated */
	xml_private_t *p = NULL;
	xmlAttr attr = xmlHasProp(obj, (const xmlChar )name);

	p = attr->_private;
	set_parent_flag(obj, xpf_dirty);
	p->flags \|= xpf_deleted;
	/* crm_trace("Setting flag %x due to %s[@id=%s].%s", xpf_dirty, obj->name, ID(obj), name); */

	} else {
	xmlUnsetProp(obj, (const xmlChar *)name);
	}
	}

	void
	purge_diff_markers(xmlNode * a_node)
	{
	xmlNode *child = NULL;

	CRM_CHECK(a_node != NULL, return);

	xml_remove_prop(a_node, XML_DIFF_MARKER);
	for (child = __xml_first_child(a_node); child != NULL; child = __xml_next(child)) {
	purge_diff_markers(child);
	}
	}

	void
	save_xml_to_file(xmlNode * xml, const char desc, const char filename)
	{
	char *f = NULL;

	if (filename == NULL) {
	char *uuid = crm_generate_uuid();

	f = crm_strdup_printf("/tmp/%s", uuid);
	filename = f;
	free(uuid);
	}

	crm_info("Saving %s to %s", desc, filename);
	write_xml_file(xml, filename, FALSE);
	free(f);
	}

	gboolean
	apply_xml_diff(xmlNode * old, xmlNode * diff, xmlNode ** new)
	{
	gboolean result = TRUE;
	int root_nodes_seen = 0;
	static struct qb_log_callsite *digest_cs = NULL;
	const char *digest = crm_element_value(diff, XML_ATTR_DIGEST);
	const char *version = crm_element_value(diff, XML_ATTR_CRM_VERSION);

	xmlNode *child_diff = NULL;
	xmlNode *added = find_xml_node(diff, "diff-added", FALSE);
	xmlNode *removed = find_xml_node(diff, "diff-removed", FALSE);

	CRM_CHECK(new != NULL, return FALSE);
	if (digest_cs == NULL) {
	digest_cs =
	qb_log_callsite_get(__func__, __FILE__, "diff-digest", LOG_TRACE, __LINE__,
	crm_trace_nonlog);
	}

	crm_trace("Substraction Phase");
	for (child_diff = __xml_first_child(removed); child_diff != NULL;
	child_diff = __xml_next(child_diff)) {
	CRM_CHECK(root_nodes_seen == 0, result = FALSE);
	if (root_nodes_seen == 0) {
	*new = subtract_xml_object(NULL, old, child_diff, FALSE, NULL, NULL);
	}
	root_nodes_seen++;
	}

	if (root_nodes_seen == 0) {
	*new = copy_xml(old);

	} else if (root_nodes_seen > 1) {
	crm_err("(-) Diffs cannot contain more than one change set..." " saw %d", root_nodes_seen);
	result = FALSE;
	}

	root_nodes_seen = 0;
	crm_trace("Addition Phase");
	if (result) {
	xmlNode *child_diff = NULL;

	for (child_diff = __xml_first_child(added); child_diff != NULL;
	child_diff = __xml_next(child_diff)) {
	CRM_CHECK(root_nodes_seen == 0, result = FALSE);
	if (root_nodes_seen == 0) {
	add_xml_object(NULL, *new, child_diff, TRUE);
	}
	root_nodes_seen++;
	}
	}

	if (root_nodes_seen > 1) {
	crm_err("(+) Diffs cannot contain more than one change set..." " saw %d", root_nodes_seen);
	result = FALSE;

	} else if (result && digest) {
	char *new_digest = NULL;

	purge_diff_markers(new); / Purge now so the diff is ok */
	new_digest = calculate_xml_versioned_digest(*new, FALSE, TRUE, version);
	if (safe_str_neq(new_digest, digest)) {
	crm_info("Digest mis-match: expected %s, calculated %s", digest, new_digest);
	result = FALSE;

	crm_trace("%p %0.6x", digest_cs, digest_cs ? digest_cs->targets : 0);
	if (digest_cs && digest_cs->targets) {
	save_xml_to_file(old, "diff:original", NULL);
	save_xml_to_file(diff, "diff:input", NULL);
	save_xml_to_file(*new, "diff:new", NULL);
	}

	} else {
	crm_trace("Digest matched: expected %s, calculated %s", digest, new_digest);
	}
	free(new_digest);

	} else if (result) {
	purge_diff_markers(new); / Purge now so the diff is ok */
	}

	return result;
	}

	static void
	__xml_diff_object(xmlNode * old, xmlNode * new)
	{
	xmlNode *cIter = NULL;
	xmlAttr *pIter = NULL;

	CRM_CHECK(new != NULL, return);
	if(old == NULL) {
	crm_node_created(new);
	__xml_acl_post_process(new); /* Check creation is allowed */
	return;

	} else {
	xml_private_t *p = new->_private;

	if(p->flags & xpf_processed) {
	/* Avoid re-comparing nodes */
	return;
	}
	p->flags \|= xpf_processed;
	}

	for (pIter = crm_first_attr(new); pIter != NULL; pIter = pIter->next) {
	xml_private_t *p = pIter->_private;

	/* Assume everything was just created and take it from there */
	p->flags \|= xpf_created;
	}

	for (pIter = crm_first_attr(old); pIter != NULL; ) {
	xmlAttr *prop = pIter;
	xml_private_t *p = NULL;
	const char name = (const char )pIter->name;
	const char *old_value = crm_element_value(old, name);
	xmlAttr *exists = xmlHasProp(new, pIter->name);

	pIter = pIter->next;
	if(exists == NULL) {
	p = new->doc->_private;

	/* Prevent the dirty flag being set recursively upwards */
	clear_bit(p->flags, xpf_tracking);
	exists = xmlSetProp(new, (const xmlChar )name, (const xmlChar )old_value);
	set_bit(p->flags, xpf_tracking);

	p = exists->_private;
	p->flags = 0;

	crm_trace("Lost %s@%s=%s", old->name, name, old_value);
	xml_remove_prop(new, name);

	} else {
	int p_new = __xml_offset((xmlNode*)exists);
	int p_old = __xml_offset((xmlNode*)prop);
	const char *value = crm_element_value(new, name);

	p = exists->_private;
	p->flags = (p->flags & ~xpf_created);

	if(strcmp(value, old_value) != 0) {
	/* Restore the original value, so we can call crm_xml_add() whcih checks ACLs */
	char *vcopy = crm_element_value_copy(new, name);

	crm_trace("Modified %s@%s %s->%s", old->name, name, old_value, vcopy);
	xmlSetProp(new, prop->name, (const xmlChar *)old_value);
	crm_xml_add(new, name, vcopy);
	free(vcopy);

	} else if(p_old != p_new) {
	crm_info("Moved %s@%s (%d -> %d)", old->name, name, p_old, p_new);
	__xml_node_dirty(new);
	p->flags \|= xpf_dirty\|xpf_moved;

	if(p_old > p_new) {
	p = prop->_private;
	p->flags \|= xpf_skip;

	} else {
	p = exists->_private;
	p->flags \|= xpf_skip;
	}
	}
	}
	}

	for (pIter = crm_first_attr(new); pIter != NULL; ) {
	xmlAttr *prop = pIter;
	xml_private_t *p = pIter->_private;

	pIter = pIter->next;
	if(is_set(p->flags, xpf_created)) {
	char name = strdup((const char )prop->name);
	char *value = crm_element_value_copy(new, name);

	crm_trace("Created %s@%s=%s", new->name, name, value);
	- /* Remove plus create wont work as it will modify the relative attribute ordering */
	+ /* Remove plus create won't work as it will modify the relative attribute ordering */
	if(__xml_acl_check(new, name, xpf_acl_write)) {
	crm_attr_dirty(prop);
	} else {
	xmlUnsetProp(new, prop->name); /* Remove - change not allowed */
	}

	free(value);
	free(name);
	}
	}

	for (cIter = __xml_first_child(old); cIter != NULL; ) {
	xmlNode *old_child = cIter;
	xmlNode *new_child = find_entity(new, crm_element_name(cIter), ID(cIter));

	cIter = __xml_next(cIter);
	if(new_child) {
	__xml_diff_object(old_child, new_child);

	} else {
	xml_private_t *p = old_child->_private;

	/* Create then free (which will check the acls if necessary) */
	xmlNode *candidate = add_node_copy(new, old_child);
	xmlNode *top = xmlDocGetRootElement(candidate->doc);

	__xml_node_clean(candidate);
	__xml_acl_apply(top); /* Make sure any ACLs are applied to 'candidate' */
	free_xml(candidate);

	if(NULL == find_entity(new, crm_element_name(old_child), ID(old_child))) {
	p->flags \|= xpf_skip;
	}
	}
	}

	for (cIter = __xml_first_child(new); cIter != NULL; ) {
	xmlNode *new_child = cIter;
	xmlNode *old_child = find_entity(old, crm_element_name(cIter), ID(cIter));

	cIter = __xml_next(cIter);
	if(old_child == NULL) {
	xml_private_t *p = new_child->_private;
	p->flags \|= xpf_skip;
	__xml_diff_object(old_child, new_child);

	} else {
	/* Check for movement, we already checked for differences */
	int p_new = __xml_offset(new_child);
	int p_old = __xml_offset(old_child);
	xml_private_t *p = new_child->_private;

	if(p_old != p_new) {
	crm_info("%s.%s moved from %d to %d - %d",
	new_child->name, ID(new_child), p_old, p_new);
	__xml_node_dirty(new);
	p->flags \|= xpf_moved;

	if(p_old > p_new) {
	p = old_child->_private;
	p->flags \|= xpf_skip;

	} else {
	p = new_child->_private;
	p->flags \|= xpf_skip;
	}
	}
	}
	}
	}

	void
	xml_calculate_changes(xmlNode * old, xmlNode * new)
	{
	CRM_CHECK(safe_str_eq(crm_element_name(old), crm_element_name(new)), return);
	CRM_CHECK(safe_str_eq(ID(old), ID(new)), return);

	if(xml_tracking_changes(new) == FALSE) {
	xml_track_changes(new, NULL, NULL, FALSE);
	}

	__xml_diff_object(old, new);
	}

	xmlNode *
	diff_xml_object(xmlNode * old, xmlNode * new, gboolean suppress)
	{
	xmlNode *tmp1 = NULL;
	xmlNode *diff = create_xml_node(NULL, "diff");
	xmlNode *removed = create_xml_node(diff, "diff-removed");
	xmlNode *added = create_xml_node(diff, "diff-added");

	crm_xml_add(diff, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);

	tmp1 = subtract_xml_object(removed, old, new, FALSE, NULL, "removed:top");
	if (suppress && tmp1 != NULL && can_prune_leaf(tmp1)) {
	free_xml(tmp1);
	}

	tmp1 = subtract_xml_object(added, new, old, TRUE, NULL, "added:top");
	if (suppress && tmp1 != NULL && can_prune_leaf(tmp1)) {
	free_xml(tmp1);
	}

	if (added->children == NULL && removed->children == NULL) {
	free_xml(diff);
	diff = NULL;
	}

	return diff;
	}

	gboolean
	can_prune_leaf(xmlNode * xml_node)
	{
	xmlNode *cIter = NULL;
	xmlAttrPtr pIter = NULL;
	gboolean can_prune = TRUE;
	const char *name = crm_element_name(xml_node);

	if (safe_str_eq(name, XML_TAG_RESOURCE_REF)
	\|\| safe_str_eq(name, XML_CIB_TAG_OBJ_REF)
	\|\| safe_str_eq(name, XML_ACL_TAG_ROLE_REF)
	\|\| safe_str_eq(name, XML_ACL_TAG_ROLE_REFv1)) {
	return FALSE;
	}

	for (pIter = crm_first_attr(xml_node); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;

	if (strcmp(p_name, XML_ATTR_ID) == 0) {
	continue;
	}
	can_prune = FALSE;
	}

	cIter = __xml_first_child(xml_node);
	while (cIter) {
	xmlNode *child = cIter;

	cIter = __xml_next(cIter);
	if (can_prune_leaf(child)) {
	free_xml(child);
	} else {
	can_prune = FALSE;
	}
	}
	return can_prune;
	}

	void
	diff_filter_context(int context, int upper_bound, int lower_bound,
	xmlNode * xml_node, xmlNode * parent)
	{
	xmlNode *us = NULL;
	xmlNode *child = NULL;
	xmlAttrPtr pIter = NULL;
	xmlNode *new_parent = parent;
	const char *name = crm_element_name(xml_node);

	CRM_CHECK(xml_node != NULL && name != NULL, return);

	us = create_xml_node(parent, name);
	for (pIter = crm_first_attr(xml_node); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	lower_bound = context;
	crm_xml_add(us, p_name, p_value);
	}

	if (lower_bound >= 0 \|\| upper_bound >= 0) {
	crm_xml_add(us, XML_ATTR_ID, ID(xml_node));
	new_parent = us;

	} else {
	upper_bound = in_upper_context(0, context, xml_node);
	if (upper_bound >= 0) {
	crm_xml_add(us, XML_ATTR_ID, ID(xml_node));
	new_parent = us;
	} else {
	free_xml(us);
	us = NULL;
	}
	}

	for (child = __xml_first_child(us); child != NULL; child = __xml_next(child)) {
	diff_filter_context(context, upper_bound - 1, lower_bound - 1, child, new_parent);
	}
	}

	int
	in_upper_context(int depth, int context, xmlNode * xml_node)
	{
	if (context == 0) {
	return 0;
	}

	if (xml_node->properties) {
	return depth;

	} else if (depth < context) {
	xmlNode *child = NULL;

	for (child = __xml_first_child(xml_node); child != NULL; child = __xml_next(child)) {
	if (in_upper_context(depth + 1, context, child)) {
	return depth;
	}
	}
	}
	return 0;
	}

	static xmlNode *
	find_xml_comment(xmlNode * root, xmlNode * search_comment)
	{
	xmlNode *a_child = NULL;

	CRM_CHECK(search_comment->type == XML_COMMENT_NODE, return NULL);

	for (a_child = __xml_first_child(root); a_child != NULL; a_child = __xml_next(a_child)) {
	if (a_child->type != XML_COMMENT_NODE) {
	continue;
	}
	if (safe_str_eq((const char )a_child->content, (const char )search_comment->content)) {
	return a_child;
	}
	}

	return NULL;
	}

	static xmlNode *
	subtract_xml_comment(xmlNode * parent, xmlNode * left, xmlNode * right,
	gboolean * changed)
	{
	CRM_CHECK(left != NULL, return NULL);
	CRM_CHECK(left->type == XML_COMMENT_NODE, return NULL);

	if (right == NULL
	\|\| safe_str_neq((const char )left->content, (const char )right->content)) {
	xmlNode *deleted = NULL;

	deleted = add_node_copy(parent, left);
	*changed = TRUE;

	return deleted;
	}

	return NULL;
	}

	xmlNode *
	subtract_xml_object(xmlNode * parent, xmlNode * left, xmlNode * right,
	gboolean full, gboolean * changed, const char *marker)
	{
	gboolean dummy = FALSE;
	gboolean skip = FALSE;
	xmlNode *diff = NULL;
	xmlNode *right_child = NULL;
	xmlNode *left_child = NULL;
	xmlAttrPtr xIter = NULL;

	const char *id = NULL;
	const char *name = NULL;
	const char *value = NULL;
	const char *right_val = NULL;

	int lpc = 0;
	static int filter_len = DIMOF(filter);

	if (changed == NULL) {
	changed = &dummy;
	}

	if (left == NULL) {
	return NULL;
	}

	if (left->type == XML_COMMENT_NODE) {
	return subtract_xml_comment(parent, left, right, changed);
	}

	id = ID(left);
	if (right == NULL) {
	xmlNode *deleted = NULL;

	crm_trace("Processing <%s id=%s> (complete copy)", crm_element_name(left), id);
	deleted = add_node_copy(parent, left);
	crm_xml_add(deleted, XML_DIFF_MARKER, marker);

	*changed = TRUE;
	return deleted;
	}

	name = crm_element_name(left);
	CRM_CHECK(name != NULL, return NULL);
	CRM_CHECK(safe_str_eq(crm_element_name(left), crm_element_name(right)), return NULL);

	/* check for XML_DIFF_MARKER in a child */
	value = crm_element_value(right, XML_DIFF_MARKER);
	if (value != NULL && strcmp(value, "removed:top") == 0) {
	crm_trace("We are the root of the deletion: %s.id=%s", name, id);
	*changed = TRUE;
	return NULL;
	}

	/* Avoiding creating the full heirarchy would save even more work here */
	diff = create_xml_node(parent, name);

	/* Reset filter */
	for (lpc = 0; lpc < filter_len; lpc++) {
	filter[lpc].found = FALSE;
	}

	/* changes to child objects */
	for (left_child = __xml_first_child(left); left_child != NULL;
	left_child = __xml_next(left_child)) {
	gboolean child_changed = FALSE;

	if (left_child->type == XML_COMMENT_NODE) {
	right_child = find_xml_comment(right, left_child);

	} else {
	right_child = find_entity(right, crm_element_name(left_child), ID(left_child));
	}

	subtract_xml_object(diff, left_child, right_child, full, &child_changed, marker);
	if (child_changed) {
	*changed = TRUE;
	}
	}

	if (*changed == FALSE) {
	/* Nothing to do */

	} else if (full) {
	xmlAttrPtr pIter = NULL;

	for (pIter = crm_first_attr(left); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	xmlSetProp(diff, (const xmlChar )p_name, (const xmlChar )p_value);
	}

	/* We already have everything we need... */
	goto done;

	} else if (id) {
	xmlSetProp(diff, (const xmlChar )XML_ATTR_ID, (const xmlChar )id);
	}

	/* changes to name/value pairs */
	for (xIter = crm_first_attr(left); xIter != NULL; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;
	xmlAttrPtr right_attr = NULL;
	xml_private_t *p = NULL;

	if (strcmp(prop_name, XML_ATTR_ID) == 0) {
	continue;
	}

	skip = FALSE;
	for (lpc = 0; skip == FALSE && lpc < filter_len; lpc++) {
	if (filter[lpc].found == FALSE && strcmp(prop_name, filter[lpc].string) == 0) {
	filter[lpc].found = TRUE;
	skip = TRUE;
	break;
	}
	}

	if (skip) {
	continue;
	}

	right_attr = xmlHasProp(right, (const xmlChar *)prop_name);
	if (right_attr) {
	p = right_attr->_private;
	}

	right_val = crm_element_value(right, prop_name);
	if (right_val == NULL \|\| (p && is_set(p->flags, xpf_deleted))) {
	/* new */
	*changed = TRUE;
	if (full) {
	xmlAttrPtr pIter = NULL;

	for (pIter = crm_first_attr(left); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	xmlSetProp(diff, (const xmlChar )p_name, (const xmlChar )p_value);
	}
	break;

	} else {
	const char *left_value = crm_element_value(left, prop_name);

	xmlSetProp(diff, (const xmlChar )prop_name, (const xmlChar )value);
	crm_xml_add(diff, prop_name, left_value);
	}

	} else {
	/* Only now do we need the left value */
	const char *left_value = crm_element_value(left, prop_name);

	if (strcmp(left_value, right_val) == 0) {
	/* unchanged */

	} else {
	*changed = TRUE;
	if (full) {
	xmlAttrPtr pIter = NULL;

	crm_trace("Changes detected to %s in <%s id=%s>", prop_name,
	crm_element_name(left), id);
	for (pIter = crm_first_attr(left); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	xmlSetProp(diff, (const xmlChar )p_name, (const xmlChar )p_value);
	}
	break;

	} else {
	crm_trace("Changes detected to %s (%s -> %s) in <%s id=%s>",
	prop_name, left_value, right_val, crm_element_name(left), id);
	crm_xml_add(diff, prop_name, left_value);
	}
	}
	}
	}

	if (*changed == FALSE) {
	free_xml(diff);
	return NULL;

	} else if (full == FALSE && id) {
	crm_xml_add(diff, XML_ATTR_ID, id);
	}
	done:
	return diff;
	}

	static int
	add_xml_comment(xmlNode * parent, xmlNode * target, xmlNode * update)
	{
	CRM_CHECK(update != NULL, return 0);
	CRM_CHECK(update->type == XML_COMMENT_NODE, return 0);

	if (target == NULL) {
	target = find_xml_comment(parent, update);
	- }
	-
	+ }
	+
	if (target == NULL) {
	add_node_copy(parent, update);

	- /* We wont reach here currently */
	+ /* We won't reach here currently */
	} else if (safe_str_neq((const char )target->content, (const char )update->content)) {
	xmlFree(target->content);
	target->content = xmlStrdup(update->content);
	}

	return 0;
	}

	int
	add_xml_object(xmlNode * parent, xmlNode * target, xmlNode * update, gboolean as_diff)
	{
	xmlNode *a_child = NULL;
	const char *object_id = NULL;
	const char *object_name = NULL;

	#if XML_PARSE_DEBUG
	crm_log_xml_trace("update:", update);
	crm_log_xml_trace("target:", target);
	#endif

	CRM_CHECK(update != NULL, return 0);

	if (update->type == XML_COMMENT_NODE) {
	return add_xml_comment(parent, target, update);
	}

	object_name = crm_element_name(update);
	object_id = ID(update);

	CRM_CHECK(object_name != NULL, return 0);

	if (target == NULL && object_id == NULL) {
	/* placeholder object */
	target = find_xml_node(parent, object_name, FALSE);

	} else if (target == NULL) {
	target = find_entity(parent, object_name, object_id);
	}

	if (target == NULL) {
	target = create_xml_node(parent, object_name);
	CRM_CHECK(target != NULL, return 0);
	#if XML_PARSER_DEBUG
	crm_trace("Added <%s%s%s/>", crm_str(object_name),
	object_id ? " id=" : "", object_id ? object_id : "");

	} else {
	crm_trace("Found node <%s%s%s/> to update",
	crm_str(object_name), object_id ? " id=" : "", object_id ? object_id : "");
	#endif
	}

	CRM_CHECK(safe_str_eq(crm_element_name(target), crm_element_name(update)), return 0);

	if (as_diff == FALSE) {
	/* So that expand_plus_plus() gets called */
	copy_in_properties(target, update);

	} else {
	/* No need for expand_plus_plus(), just raw speed */
	xmlAttrPtr pIter = NULL;

	for (pIter = crm_first_attr(update); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	/* Remove it first so the ordering of the update is preserved */
	xmlUnsetProp(target, (const xmlChar *)p_name);
	xmlSetProp(target, (const xmlChar )p_name, (const xmlChar )p_value);
	}
	}

	for (a_child = __xml_first_child(update); a_child != NULL; a_child = __xml_next(a_child)) {
	#if XML_PARSER_DEBUG
	crm_trace("Updating child <%s id=%s>", crm_element_name(a_child), ID(a_child));
	#endif
	add_xml_object(target, NULL, a_child, as_diff);
	}

	#if XML_PARSER_DEBUG
	crm_trace("Finished with <%s id=%s>", crm_str(object_name), crm_str(object_id));
	#endif
	return 0;
	}

	gboolean
	update_xml_child(xmlNode * child, xmlNode * to_update)
	{
	gboolean can_update = TRUE;
	xmlNode *child_of_child = NULL;

	CRM_CHECK(child != NULL, return FALSE);
	CRM_CHECK(to_update != NULL, return FALSE);

	if (safe_str_neq(crm_element_name(to_update), crm_element_name(child))) {
	can_update = FALSE;

	} else if (safe_str_neq(ID(to_update), ID(child))) {
	can_update = FALSE;

	} else if (can_update) {
	#if XML_PARSER_DEBUG
	crm_log_xml_trace(child, "Update match found...");
	#endif
	add_xml_object(NULL, child, to_update, FALSE);
	}

	for (child_of_child = __xml_first_child(child); child_of_child != NULL;
	child_of_child = __xml_next(child_of_child)) {
	/* only update the first one */
	if (can_update) {
	break;
	}
	can_update = update_xml_child(child_of_child, to_update);
	}

	return can_update;
	}

	int
	find_xml_children(xmlNode ** children, xmlNode * root,
	const char tag, const char field, const char *value, gboolean search_matches)
	{
	int match_found = 0;

	CRM_CHECK(root != NULL, return FALSE);
	CRM_CHECK(children != NULL, return FALSE);

	if (tag != NULL && safe_str_neq(tag, crm_element_name(root))) {

	} else if (value != NULL && safe_str_neq(value, crm_element_value(root, field))) {

	} else {
	if (*children == NULL) {
	*children = create_xml_node(NULL, __FUNCTION__);
	}
	add_node_copy(*children, root);
	match_found = 1;
	}

	if (search_matches \|\| match_found == 0) {
	xmlNode *child = NULL;

	for (child = __xml_first_child(root); child != NULL; child = __xml_next(child)) {
	match_found += find_xml_children(children, child, tag, field, value, search_matches);
	}
	}

	return match_found;
	}

	gboolean
	replace_xml_child(xmlNode * parent, xmlNode * child, xmlNode * update, gboolean delete_only)
	{
	gboolean can_delete = FALSE;
	xmlNode *child_of_child = NULL;

	const char *up_id = NULL;
	const char *child_id = NULL;
	const char *right_val = NULL;

	CRM_CHECK(child != NULL, return FALSE);
	CRM_CHECK(update != NULL, return FALSE);

	up_id = ID(update);
	child_id = ID(child);

	if (up_id == NULL \|\| (child_id && strcmp(child_id, up_id) == 0)) {
	can_delete = TRUE;
	}
	if (safe_str_neq(crm_element_name(update), crm_element_name(child))) {
	can_delete = FALSE;
	}
	if (can_delete && delete_only) {
	xmlAttrPtr pIter = NULL;

	for (pIter = crm_first_attr(update); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	right_val = crm_element_value(child, p_name);
	if (safe_str_neq(p_value, right_val)) {
	can_delete = FALSE;
	}
	}
	}

	if (can_delete && parent != NULL) {
	crm_log_xml_trace(child, "Delete match found...");
	if (delete_only \|\| update == NULL) {
	free_xml(child);

	} else {
	xmlNode *tmp = copy_xml(update);
	xmlDoc *doc = tmp->doc;
	xmlNode *old = NULL;

	xml_accept_changes(tmp);
	old = xmlReplaceNode(child, tmp);

	if(xml_tracking_changes(tmp)) {
	/* Replaced sections may have included relevant ACLs */
	__xml_acl_apply(tmp);
	}

	xml_calculate_changes(old, tmp);
	xmlDocSetRootElement(doc, old);
	free_xml(old);
	}
	child = NULL;
	return TRUE;

	} else if (can_delete) {
	crm_log_xml_debug(child, "Cannot delete the search root");
	can_delete = FALSE;
	}

	child_of_child = __xml_first_child(child);
	while (child_of_child) {
	xmlNode *next = __xml_next(child_of_child);

	can_delete = replace_xml_child(child, child_of_child, update, delete_only);

	/* only delete the first one */
	if (can_delete) {
	child_of_child = NULL;
	} else {
	child_of_child = next;
	}
	}

	return can_delete;
	}

	void
	hash2nvpair(gpointer key, gpointer value, gpointer user_data)
	{
	const char *name = key;
	const char *s_value = value;

	xmlNode *xml_node = user_data;
	xmlNode *xml_child = create_xml_node(xml_node, XML_CIB_TAG_NVPAIR);

	crm_xml_add(xml_child, XML_ATTR_ID, name);
	crm_xml_add(xml_child, XML_NVPAIR_ATTR_NAME, name);
	crm_xml_add(xml_child, XML_NVPAIR_ATTR_VALUE, s_value);

	crm_trace("dumped: name=%s value=%s", name, s_value);
	}

	void
	hash2smartfield(gpointer key, gpointer value, gpointer user_data)
	{
	const char *name = key;
	const char *s_value = value;

	xmlNode *xml_node = user_data;

	if (isdigit(name[0])) {
	xmlNode *tmp = create_xml_node(xml_node, XML_TAG_PARAM);

	crm_xml_add(tmp, XML_NVPAIR_ATTR_NAME, name);
	crm_xml_add(tmp, XML_NVPAIR_ATTR_VALUE, s_value);

	} else if (crm_element_value(xml_node, name) == NULL) {
	crm_xml_add(xml_node, name, s_value);
	crm_trace("dumped: %s=%s", name, s_value);

	} else {
	crm_trace("duplicate: %s=%s", name, s_value);
	}
	}

	void
	hash2field(gpointer key, gpointer value, gpointer user_data)
	{
	const char *name = key;
	const char *s_value = value;

	xmlNode *xml_node = user_data;

	if (crm_element_value(xml_node, name) == NULL) {
	crm_xml_add(xml_node, name, s_value);

	} else {
	crm_trace("duplicate: %s=%s", name, s_value);
	}
	}

	void
	hash2metafield(gpointer key, gpointer value, gpointer user_data)
	{
	char *crm_name = NULL;

	if (key == NULL \|\| value == NULL) {
	return;
	} else if (((char *)key)[0] == '#') {
	return;
	} else if (strstr(key, ":")) {
	return;
	}

	crm_name = crm_meta_name(key);
	hash2field(crm_name, value, user_data);
	free(crm_name);
	}

	GHashTable *
	xml2list(xmlNode * parent)
	{
	xmlNode *child = NULL;
	xmlAttrPtr pIter = NULL;
	xmlNode *nvpair_list = NULL;
	GHashTable *nvpair_hash = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	CRM_CHECK(parent != NULL, return nvpair_hash);

	nvpair_list = find_xml_node(parent, XML_TAG_ATTRS, FALSE);
	if (nvpair_list == NULL) {
	crm_trace("No attributes in %s", crm_element_name(parent));
	crm_log_xml_trace(parent, "No attributes for resource op");
	}

	crm_log_xml_trace(nvpair_list, "Unpacking");

	for (pIter = crm_first_attr(nvpair_list); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	crm_trace("Added %s=%s", p_name, p_value);

	g_hash_table_insert(nvpair_hash, strdup(p_name), strdup(p_value));
	}

	for (child = __xml_first_child(nvpair_list); child != NULL; child = __xml_next(child)) {
	if (strcmp((const char *)child->name, XML_TAG_PARAM) == 0) {
	const char *key = crm_element_value(child, XML_NVPAIR_ATTR_NAME);
	const char *value = crm_element_value(child, XML_NVPAIR_ATTR_VALUE);

	crm_trace("Added %s=%s", key, value);
	if (key != NULL && value != NULL) {
	g_hash_table_insert(nvpair_hash, strdup(key), strdup(value));
	}
	}
	}

	return nvpair_hash;
	}

	typedef struct name_value_s {
	const char *name;
	const void *value;
	} name_value_t;

	static gint
	sort_pairs(gconstpointer a, gconstpointer b)
	{
	int rc = 0;
	const name_value_t *pair_a = a;
	const name_value_t *pair_b = b;

	CRM_ASSERT(a != NULL);
	CRM_ASSERT(pair_a->name != NULL);

	CRM_ASSERT(b != NULL);
	CRM_ASSERT(pair_b->name != NULL);

	rc = strcmp(pair_a->name, pair_b->name);
	if (rc < 0) {
	return -1;
	} else if (rc > 0) {
	return 1;
	}
	return 0;
	}

	static void
	dump_pair(gpointer data, gpointer user_data)
	{
	name_value_t *pair = data;
	xmlNode *parent = user_data;

	crm_xml_add(parent, pair->name, pair->value);
	}

	xmlNode *
	sorted_xml(xmlNode * input, xmlNode * parent, gboolean recursive)
	{
	xmlNode *child = NULL;
	GListPtr sorted = NULL;
	GListPtr unsorted = NULL;
	name_value_t *pair = NULL;
	xmlNode *result = NULL;
	const char *name = NULL;
	xmlAttrPtr pIter = NULL;

	CRM_CHECK(input != NULL, return NULL);

	name = crm_element_name(input);
	CRM_CHECK(name != NULL, return NULL);

	result = create_xml_node(parent, name);

	for (pIter = crm_first_attr(input); pIter != NULL; pIter = pIter->next) {
	const char p_name = (const char )pIter->name;
	const char *p_value = crm_attr_value(pIter);

	pair = calloc(1, sizeof(name_value_t));
	pair->name = p_name;
	pair->value = p_value;
	unsorted = g_list_prepend(unsorted, pair);
	pair = NULL;
	}

	sorted = g_list_sort(unsorted, sort_pairs);
	g_list_foreach(sorted, dump_pair, result);
	g_list_free_full(sorted, free);

	for (child = __xml_first_child(input); child != NULL; child = __xml_next(child)) {
	if (recursive) {
	sorted_xml(child, result, recursive);
	} else {
	add_node_copy(result, child);
	}
	}

	return result;
	}

	static gboolean
	validate_with_dtd(xmlDocPtr doc, gboolean to_logs, const char *dtd_file)
	{
	gboolean valid = TRUE;

	xmlDtdPtr dtd = NULL;
	xmlValidCtxtPtr cvp = NULL;

	CRM_CHECK(doc != NULL, return FALSE);
	CRM_CHECK(dtd_file != NULL, return FALSE);

	dtd = xmlParseDTD(NULL, (const xmlChar *)dtd_file);
	if(dtd == NULL) {
	crm_err("Could not locate/parse DTD: %s", dtd_file);
	return TRUE;
	}

	cvp = xmlNewValidCtxt();
	if(cvp) {
	if (to_logs) {
	cvp->userData = (void *)LOG_ERR;
	cvp->error = (xmlValidityErrorFunc) xml_log;
	cvp->warning = (xmlValidityWarningFunc) xml_log;
	} else {
	cvp->userData = (void *)stderr;
	cvp->error = (xmlValidityErrorFunc) fprintf;
	cvp->warning = (xmlValidityWarningFunc) fprintf;
	}

	if (!xmlValidateDtd(cvp, doc, dtd)) {
	valid = FALSE;
	}
	xmlFreeValidCtxt(cvp);

	} else {
	crm_err("Internal error: No valid context");
	}

	xmlFreeDtd(dtd);
	return valid;
	}

	xmlNode *
	first_named_child(xmlNode * parent, const char *name)
	{
	xmlNode *match = NULL;

	for (match = __xml_first_child(parent); match != NULL; match = __xml_next(match)) {
	/*
	* name == NULL gives first child regardless of name; this is
	* semantically incorrect in this funciton, but may be necessary
	* due to prior use of xml_child_iter_filter
	*/
	if (name == NULL \|\| strcmp((const char *)match->name, name) == 0) {
	return match;
	}
	}
	return NULL;
	}

	#if 0
	static void
	relaxng_invalid_stderr(void *userData, xmlErrorPtr error)
	{
	/*
	Structure xmlError
	struct _xmlError {
	int domain : What part of the library raised this er
	int code : The error code, e.g. an xmlParserError
	char * message : human-readable informative error messag
	xmlErrorLevel level : how consequent is the error
	char * file : the filename
	int line : the line number if available
	char * str1 : extra string information
	char * str2 : extra string information
	char * str3 : extra string information
	int int1 : extra number information
	int int2 : column number of the error or 0 if N/A
	void * ctxt : the parser context if available
	void * node : the node in the tree
	}
	*/
	crm_err("Structured error: line=%d, level=%d %s", error->line, error->level, error->message);
	}
	#endif

	static gboolean
	validate_with_relaxng(xmlDocPtr doc, gboolean to_logs, const char *relaxng_file,
	relaxng_ctx_cache_t ** cached_ctx)
	{
	int rc = 0;
	gboolean valid = TRUE;
	relaxng_ctx_cache_t *ctx = NULL;

	CRM_CHECK(doc != NULL, return FALSE);
	CRM_CHECK(relaxng_file != NULL, return FALSE);

	if (cached_ctx && *cached_ctx) {
	ctx = *cached_ctx;

	} else {
	crm_info("Creating RNG parser context");
	ctx = calloc(1, sizeof(relaxng_ctx_cache_t));

	xmlLoadExtDtdDefaultValue = 1;
	ctx->parser = xmlRelaxNGNewParserCtxt(relaxng_file);
	CRM_CHECK(ctx->parser != NULL, goto cleanup);

	if (to_logs) {
	xmlRelaxNGSetParserErrors(ctx->parser,
	(xmlRelaxNGValidityErrorFunc) xml_log,
	(xmlRelaxNGValidityWarningFunc) xml_log,
	GUINT_TO_POINTER(LOG_ERR));
	} else {
	xmlRelaxNGSetParserErrors(ctx->parser,
	(xmlRelaxNGValidityErrorFunc) fprintf,
	(xmlRelaxNGValidityWarningFunc) fprintf, stderr);
	}

	ctx->rng = xmlRelaxNGParse(ctx->parser);
	CRM_CHECK(ctx->rng != NULL, crm_err("Could not find/parse %s", relaxng_file);
	goto cleanup);

	ctx->valid = xmlRelaxNGNewValidCtxt(ctx->rng);
	CRM_CHECK(ctx->valid != NULL, goto cleanup);

	if (to_logs) {
	xmlRelaxNGSetValidErrors(ctx->valid,
	(xmlRelaxNGValidityErrorFunc) xml_log,
	(xmlRelaxNGValidityWarningFunc) xml_log,
	GUINT_TO_POINTER(LOG_ERR));
	} else {
	xmlRelaxNGSetValidErrors(ctx->valid,
	(xmlRelaxNGValidityErrorFunc) fprintf,
	(xmlRelaxNGValidityWarningFunc) fprintf, stderr);
	}
	}

	/* xmlRelaxNGSetValidStructuredErrors( */
	/* valid, relaxng_invalid_stderr, valid); */

	xmlLineNumbersDefault(1);
	rc = xmlRelaxNGValidateDoc(ctx->valid, doc);
	if (rc > 0) {
	valid = FALSE;

	} else if (rc < 0) {
	crm_err("Internal libxml error during validation\n");
	}

	cleanup:

	if (cached_ctx) {
	*cached_ctx = ctx;

	} else {
	if (ctx->parser != NULL) {
	xmlRelaxNGFreeParserCtxt(ctx->parser);
	}
	if (ctx->valid != NULL) {
	xmlRelaxNGFreeValidCtxt(ctx->valid);
	}
	if (ctx->rng != NULL) {
	xmlRelaxNGFree(ctx->rng);
	}
	free(ctx);
	}

	return valid;
	}

	void
	crm_xml_init(void)
	{
	static bool init = TRUE;

	if(init) {
	init = FALSE;
	/* The default allocator XML_BUFFER_ALLOC_EXACT does far too many
	* realloc_safe()s and it can take upwards of 18 seconds (yes, seconds)
	* to dump a 28kb tree which XML_BUFFER_ALLOC_DOUBLEIT can do in
	* less than 1 second.
	*/
	xmlSetBufferAllocationScheme(XML_BUFFER_ALLOC_DOUBLEIT);

	/* Populate and free the _private field when nodes are created and destroyed */
	xmlDeregisterNodeDefault(pcmkDeregisterNode);
	xmlRegisterNodeDefault(pcmkRegisterNode);

	__xml_build_schema_list();
	}
	}

	void
	crm_xml_cleanup(void)
	{
	int lpc = 0;
	relaxng_ctx_cache_t *ctx = NULL;

	crm_info("Cleaning up memory from libxml2");
	for (; lpc < xml_schema_max; lpc++) {

	switch (known_schemas[lpc].type) {
	case 0:
	/* None */
	break;
	case 1:
	/* DTD - Not cached */
	break;
	case 2:
	/* RNG - Cached */
	ctx = (relaxng_ctx_cache_t *) known_schemas[lpc].cache;
	if (ctx == NULL) {
	break;
	}
	if (ctx->parser != NULL) {
	xmlRelaxNGFreeParserCtxt(ctx->parser);
	}
	if (ctx->valid != NULL) {
	xmlRelaxNGFreeValidCtxt(ctx->valid);
	}
	if (ctx->rng != NULL) {
	xmlRelaxNGFree(ctx->rng);
	}
	free(ctx);
	known_schemas[lpc].cache = NULL;
	break;
	default:
	break;
	}
	free(known_schemas[lpc].name);
	free(known_schemas[lpc].location);
	free(known_schemas[lpc].transform);
	}
	free(known_schemas);
	xsltCleanupGlobals();
	xmlCleanupParser();
	}

	static gboolean
	validate_with(xmlNode * xml, int method, gboolean to_logs)
	{
	xmlDocPtr doc = NULL;
	gboolean valid = FALSE;
	int type = 0;
	char *file = NULL;

	if(method < 0) {
	return FALSE;
	}

	type = known_schemas[method].type;
	if(type == 0) {
	return TRUE;
	}

	CRM_CHECK(xml != NULL, return FALSE);
	doc = getDocPtr(xml);
	file = get_schema_path(known_schemas[method].name, known_schemas[method].location);

	crm_trace("Validating with: %s (type=%d)", crm_str(file), type);
	switch (type) {
	case 1:
	valid = validate_with_dtd(doc, to_logs, file);
	break;
	case 2:
	valid =
	validate_with_relaxng(doc, to_logs, file,
	(relaxng_ctx_cache_t **) & (known_schemas[method].cache));
	break;
	default:
	crm_err("Unknown validator type: %d", type);
	break;
	}

	free(file);
	return valid;
	}

	#include <stdio.h>
	static void
	dump_file(const char *filename)
	{

	FILE *fp = NULL;
	int ch, line = 0;

	CRM_CHECK(filename != NULL, return);

	fp = fopen(filename, "r");
	CRM_CHECK(fp != NULL, return);

	fprintf(stderr, "%4d ", ++line);
	do {
	ch = getc(fp);
	if (ch == EOF) {
	putc('\n', stderr);
	break;
	} else if (ch == '\n') {
	fprintf(stderr, "\n%4d ", ++line);
	} else {
	putc(ch, stderr);
	}
	} while (1);

	fclose(fp);
	}

	gboolean
	validate_xml_verbose(xmlNode * xml_blob)
	{
	int fd = 0;
	xmlDoc *doc = NULL;
	xmlNode *xml = NULL;
	gboolean rc = FALSE;
	char *filename = strdup(CRM_STATE_DIR "/cib-invalid.XXXXXX");

	umask(S_IWGRP \| S_IWOTH \| S_IROTH);
	fd = mkstemp(filename);
	write_xml_fd(xml_blob, filename, fd, FALSE);

	dump_file(filename);

	doc = xmlParseFile(filename);
	xml = xmlDocGetRootElement(doc);
	rc = validate_xml(xml, NULL, FALSE);
	free_xml(xml);

	unlink(filename);
	free(filename);

	return rc;
	}

	gboolean
	validate_xml(xmlNode * xml_blob, const char *validation, gboolean to_logs)
	{
	int version = 0;

	if (validation == NULL) {
	validation = crm_element_value(xml_blob, XML_ATTR_VALIDATION);
	}

	if (validation == NULL) {
	int lpc = 0;
	bool valid = FALSE;

	validation = crm_element_value(xml_blob, "ignore-dtd");
	if (crm_is_true(validation)) {
	/* Legacy compatibilty */
	crm_xml_add(xml_blob, XML_ATTR_VALIDATION, "none");
	return TRUE;
	}

	/* Work it out */
	for (lpc = 0; lpc < xml_schema_max; lpc++) {
	if(validate_with(xml_blob, lpc, FALSE)) {
	valid = TRUE;
	crm_xml_add(xml_blob, XML_ATTR_VALIDATION, known_schemas[lpc].name);
	crm_info("XML validated against %s", known_schemas[lpc].name);
	if(known_schemas[lpc].after_transform == 0) {
	break;
	}
	}
	}

	return valid;
	}

	version = get_schema_version(validation);
	if (strcmp(validation, "none") == 0) {
	return TRUE;

	} else if(version < xml_schema_max) {
	return validate_with(xml_blob, version, to_logs);

	}

	crm_err("Unknown validator: %s", validation);
	return FALSE;
	}

	#if HAVE_LIBXSLT
	static xmlNode *
	apply_transformation(xmlNode * xml, const char *transform)
	{
	char *xform = NULL;
	xmlNode *out = NULL;
	xmlDocPtr res = NULL;
	xmlDocPtr doc = NULL;
	xsltStylesheet *xslt = NULL;

	CRM_CHECK(xml != NULL, return FALSE);
	doc = getDocPtr(xml);
	xform = get_schema_path(NULL, transform);

	xmlLoadExtDtdDefaultValue = 1;
	xmlSubstituteEntitiesDefault(1);

	xslt = xsltParseStylesheetFile((const xmlChar *)xform);
	CRM_CHECK(xslt != NULL, goto cleanup);

	res = xsltApplyStylesheet(xslt, doc, NULL);
	CRM_CHECK(res != NULL, goto cleanup);

	out = xmlDocGetRootElement(res);

	cleanup:
	if (xslt) {
	xsltFreeStylesheet(xslt);
	}

	free(xform);

	return out;
	}
	#endif

	const char *
	get_schema_name(int version)
	{
	if (version < 0 \|\| version >= xml_schema_max) {
	return "unknown";
	}
	return known_schemas[version].name;
	}

	int
	get_schema_version(const char *name)
	{
	int lpc = 0;

	if(name == NULL) {
	name = "none";
	}
	for (; lpc < xml_schema_max; lpc++) {
	if (safe_str_eq(name, known_schemas[lpc].name)) {
	return lpc;
	}
	}
	return -1;
	}

	/* set which validation to use */
	#include <crm/cib.h>
	int
	update_validation(xmlNode ** xml_blob, int *best, int max, gboolean transform, gboolean to_logs)
	{
	xmlNode *xml = NULL;
	char *value = NULL;
	int max_stable_schemas = xml_latest_schema_index();
	int lpc = 0, match = -1, rc = pcmk_ok;

	CRM_CHECK(best != NULL, return -EINVAL);
	CRM_CHECK(xml_blob != NULL, return -EINVAL);
	CRM_CHECK(*xml_blob != NULL, return -EINVAL);

	*best = 0;
	xml = *xml_blob;
	value = crm_element_value_copy(xml, XML_ATTR_VALIDATION);

	if (value != NULL) {
	match = get_schema_version(value);

	lpc = match;
	if (lpc >= 0 && transform == FALSE) {
	lpc++;

	} else if (lpc < 0) {
	crm_debug("Unknown validation type");
	lpc = 0;
	}
	}

	if (match >= max_stable_schemas) {
	/* nothing to do */
	free(value);
	*best = match;
	return pcmk_ok;
	}

	while(lpc <= max_stable_schemas) {
	gboolean valid = TRUE;

	crm_debug("Testing '%s' validation (%d of %d)",
	known_schemas[lpc].name ? known_schemas[lpc].name : "<unset>",
	lpc, max_stable_schemas);
	valid = validate_with(xml, lpc, to_logs);

	if (valid) {
	*best = lpc;
	} else {
	crm_trace("%s validation failed", known_schemas[lpc].name ? known_schemas[lpc].name : "<unset>");
	}

	if (valid && transform) {
	xmlNode *upgrade = NULL;
	int next = known_schemas[lpc].after_transform;

	if (next < 0) {
	crm_trace("Stopping at %s", known_schemas[lpc].name);
	break;

	} else if (max > 0 && lpc == max) {
	crm_trace("Upgrade limit reached at %s (lpc=%d, next=%d, max=%d)",
	known_schemas[lpc].name, lpc, next, max);
	break;

	} else if (max > 0 && next > max) {
	crm_debug("Upgrade limit reached at %s (lpc=%d, next=%d, max=%d)",
	known_schemas[lpc].name, lpc, next, max);
	break;

	} else if (known_schemas[lpc].transform == NULL) {
	crm_debug("%s-style configuration is also valid for %s",
	known_schemas[lpc].name, known_schemas[next].name);

	if (validate_with(xml, next, to_logs)) {
	crm_debug("Configuration valid for schema: %s", known_schemas[next].name);
	lpc = next;
	*best = next;
	rc = pcmk_ok;

	} else {
	crm_info("Configuration not valid for schema: %s", known_schemas[next].name);
	}

	} else {
	crm_debug("Upgrading %s-style configuration to %s with %s",
	known_schemas[lpc].name, known_schemas[next].name,
	known_schemas[lpc].transform ? known_schemas[lpc].transform : "no-op");

	#if HAVE_LIBXSLT
	upgrade = apply_transformation(xml, known_schemas[lpc].transform);
	#endif
	if (upgrade == NULL) {
	crm_err("Transformation %s failed", known_schemas[lpc].transform);
	rc = -pcmk_err_transform_failed;

	} else if (validate_with(upgrade, next, to_logs)) {
	crm_info("Transformation %s successful", known_schemas[lpc].transform);
	lpc = next;
	*best = next;
	free_xml(xml);
	xml = upgrade;
	rc = pcmk_ok;

	} else {
	crm_err("Transformation %s did not produce a valid configuration",
	known_schemas[lpc].transform);
	crm_log_xml_info(upgrade, "transform:bad");
	free_xml(upgrade);
	rc = -pcmk_err_schema_validation;
	}
	}
	}
	}

	if (*best > match) {
	crm_info("%s the configuration from %s to %s",
	transform?"Transformed":"Upgraded",
	value ? value : "<none>", known_schemas[*best].name);
	crm_xml_add(xml, XML_ATTR_VALIDATION, known_schemas[*best].name);
	}

	*xml_blob = xml;
	free(value);
	return rc;
	}

	gboolean
	cli_config_update(xmlNode ** xml, int *best_version, gboolean to_logs)
	{
	gboolean rc = TRUE;
	const char value = crm_element_value(xml, XML_ATTR_VALIDATION);

	int version = get_schema_version(value);
	int min_version = xml_minimum_schema_index();

	if (version < min_version) {
	xmlNode *converted = NULL;

	converted = copy_xml(*xml);
	update_validation(&converted, &version, 0, TRUE, to_logs);

	value = crm_element_value(converted, XML_ATTR_VALIDATION);
	if (version < min_version) {
	if (to_logs) {
	crm_config_err("Your current configuration could only be upgraded to %s... "
	"the minimum requirement is %s.\n", crm_str(value),
	get_schema_name(min_version));
	} else {
	fprintf(stderr, "Your current configuration could only be upgraded to %s... "
	"the minimum requirement is %s.\n",
	crm_str(value), get_schema_name(min_version));
	}

	free_xml(converted);
	converted = NULL;
	rc = FALSE;

	} else {
	free_xml(*xml);
	*xml = converted;

	if (version < xml_latest_schema_index()) {
	crm_config_warn("Your configuration was internally updated to %s... "
	"which is acceptable but not the most recent",
	get_schema_name(version));

	} else if (to_logs) {
	crm_info("Your configuration was internally updated to the latest version (%s)",
	get_schema_name(version));
	}
	}

	} else if (version >= get_schema_version("none")) {
	if (to_logs) {
	crm_config_warn("Configuration validation is currently disabled."
	" It is highly encouraged and prevents many common cluster issues.");

	} else {
	fprintf(stderr, "Configuration validation is currently disabled."
	" It is highly encouraged and prevents many common cluster issues.\n");
	}
	}

	if (best_version) {
	*best_version = version;
	}

	return rc;
	}

	xmlNode *
	expand_idref(xmlNode * input, xmlNode * top)
	{
	const char *tag = NULL;
	const char *ref = NULL;
	xmlNode *result = input;
	char *xpath_string = NULL;

	if (result == NULL) {
	return NULL;

	} else if (top == NULL) {
	top = input;
	}

	tag = crm_element_name(result);
	ref = crm_element_value(result, XML_ATTR_IDREF);

	if (ref != NULL) {
	int xpath_max = 512, offset = 0;

	xpath_string = calloc(1, xpath_max);

	offset += snprintf(xpath_string + offset, xpath_max - offset, "//%s[@id='%s']", tag, ref);
	CRM_LOG_ASSERT(offset > 0);

	result = get_xpath_object(xpath_string, top, LOG_ERR);
	if (result == NULL) {
	char nodePath = (char )xmlGetNodePath(top);

	crm_err("No match for %s found in %s: Invalid configuration", xpath_string,
	crm_str(nodePath));
	free(nodePath);
	}
	}

	free(xpath_string);
	return result;
	}

	const char *
	crm_element_value(xmlNode * data, const char *name)
	{
	xmlAttr *attr = NULL;

	if (data == NULL) {
	crm_err("Couldn't find %s in NULL", name ? name : "<null>");
	CRM_LOG_ASSERT(data != NULL);
	return NULL;

	} else if (name == NULL) {
	crm_err("Couldn't find NULL in %s", crm_element_name(data));
	return NULL;
	}

	attr = xmlHasProp(data, (const xmlChar *)name);
	if (attr == NULL \|\| attr->children == NULL) {
	return NULL;
	}
	return (const char *)attr->children->content;
	}

	diff --git a/lib/pengine/native.c b/lib/pengine/native.c
	index 58f9c2e8df..5a206c492b 100644
	--- a/lib/pengine/native.c
	+++ b/lib/pengine/native.c
	@@ -1,891 +1,891 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/status.h>
	#include <crm/pengine/complex.h>
	#include <crm/pengine/internal.h>
	#include <unpack.h>
	#include <crm/msg_xml.h>

	#define VARIANT_NATIVE 1
	#include "./variant.h"

	void
	native_add_running(resource_t * rsc, node_t * node, pe_working_set_t * data_set)
	{
	GListPtr gIter = rsc->running_on;

	CRM_CHECK(node != NULL, return);
	for (; gIter != NULL; gIter = gIter->next) {
	node_t a_node = (node_t ) gIter->data;

	CRM_CHECK(a_node != NULL, return);
	if (safe_str_eq(a_node->details->id, node->details->id)) {
	return;
	}
	}

	pe_rsc_trace(rsc, "Adding %s to %s %s", rsc->id, node->details->uname,
	is_set(rsc->flags, pe_rsc_managed)?"":"(unmanaged)");

	rsc->running_on = g_list_append(rsc->running_on, node);
	if (rsc->variant == pe_native) {
	node->details->running_rsc = g_list_append(node->details->running_rsc, rsc);
	}

	if (rsc->variant == pe_native && node->details->maintenance) {
	clear_bit(rsc->flags, pe_rsc_managed);
	}

	if (is_not_set(rsc->flags, pe_rsc_managed)) {
	resource_t *p = rsc->parent;

	- pe_rsc_info(rsc, "resource %s isnt managed", rsc->id);
	+ pe_rsc_info(rsc, "resource %s isn't managed", rsc->id);
	resource_location(rsc, node, INFINITY, "not_managed_default", data_set);

	while(p && node->details->online) {
	/* add without the additional location constraint */
	p->running_on = g_list_append(p->running_on, node);
	p = p->parent;
	}
	return;
	}

	if (rsc->variant == pe_native && g_list_length(rsc->running_on) > 1) {
	switch (rsc->recovery_type) {
	case recovery_stop_only:
	{
	GHashTableIter gIter;
	node_t *local_node = NULL;

	/* make sure it doesnt come up again */
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = node_hash_from_list(data_set->nodes);
	g_hash_table_iter_init(&gIter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&gIter, NULL, (void **)&local_node)) {
	local_node->weight = -INFINITY;
	}
	}
	break;
	case recovery_stop_start:
	break;
	case recovery_block:
	clear_bit(rsc->flags, pe_rsc_managed);
	set_bit(rsc->flags, pe_rsc_block);

	/* If the group that the resource belongs to is configured with multiple-active=block, */
	/* block the whole group. */
	if (rsc->parent
	&& rsc->parent->variant == pe_group
	&& rsc->parent->recovery_type == recovery_block) {
	GListPtr gIter = rsc->parent->children;

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	clear_bit(child->flags, pe_rsc_managed);
	set_bit(child->flags, pe_rsc_block);
	}
	}
	break;
	}
	crm_debug("%s is active on %d nodes including %s: %s",
	rsc->id, g_list_length(rsc->running_on), node->details->uname,
	recovery2text(rsc->recovery_type));

	} else {
	pe_rsc_trace(rsc, "Resource %s is active on: %s", rsc->id, node->details->uname);
	}

	if (rsc->parent != NULL) {
	native_add_running(rsc->parent, node, data_set);
	}
	}

	extern void force_non_unique_clone(resource_t * rsc, const char rid, pe_working_set_t data_set);

	gboolean
	native_unpack(resource_t * rsc, pe_working_set_t * data_set)
	{
	resource_t *parent = uber_parent(rsc);
	native_variant_data_t *native_data = NULL;
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);

	pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);

	native_data = calloc(1, sizeof(native_variant_data_t));
	rsc->variant_opaque = native_data;

	if (is_set(rsc->flags, pe_rsc_unique) && rsc->parent) {

	if (safe_str_eq(class, "lsb")) {
	resource_t *top = uber_parent(rsc);

	force_non_unique_clone(top, rsc->id, data_set);
	}
	}

	if (safe_str_eq(class, "ocf") == FALSE) {
	const char *stateful = g_hash_table_lookup(parent->meta, "stateful");

	if (safe_str_eq(stateful, XML_BOOLEAN_TRUE)) {
	pe_err
	("Resource %s is of type %s and therefore cannot be used as a master/slave resource",
	rsc->id, class);
	return FALSE;
	}
	}

	return TRUE;
	}

	resource_t *
	native_find_rsc(resource_t * rsc, const char id, node_t on_node, int flags)
	{
	gboolean match = FALSE;
	resource_t *result = NULL;
	GListPtr gIter = rsc->children;

	CRM_ASSERT(id != NULL);

	if (flags & pe_find_clone) {
	const char *rid = ID(rsc->xml);

	if (rsc->parent == NULL) {
	match = FALSE;

	} else if (safe_str_eq(rsc->id, id)) {
	match = TRUE;

	} else if (safe_str_eq(rid, id)) {
	match = TRUE;
	}

	} else {
	if (strcmp(rsc->id, id) == 0) {
	match = TRUE;

	} else if (is_set(flags, pe_find_renamed)
	&& rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
	match = TRUE;
	}
	}

	if (match && on_node) {
	pe_rsc_trace(rsc, "Now checking %s is on %s", rsc->id, on_node->details->uname);
	if (is_set(flags, pe_find_current) && rsc->running_on) {

	GListPtr gIter = rsc->running_on;

	for (; gIter != NULL; gIter = gIter->next) {
	node_t loc = (node_t ) gIter->data;

	if (loc->details == on_node->details) {
	return rsc;
	}
	}

	} else if (is_set(flags, pe_find_inactive) && rsc->running_on == NULL) {
	return rsc;

	} else if (is_not_set(flags, pe_find_current) && rsc->allocated_to
	&& rsc->allocated_to->details == on_node->details) {
	return rsc;
	}

	} else if (match) {
	return rsc;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	result = rsc->fns->find_rsc(child, id, on_node, flags);
	if (result) {
	return result;
	}
	}
	return NULL;
	}

	char *
	native_parameter(resource_t * rsc, node_t * node, gboolean create, const char *name,
	pe_working_set_t * data_set)
	{
	char *value_copy = NULL;
	const char *value = NULL;
	GHashTable *hash = rsc->parameters;
	GHashTable *local_hash = NULL;

	CRM_CHECK(rsc != NULL, return NULL);
	CRM_CHECK(name != NULL && strlen(name) != 0, return NULL);

	pe_rsc_trace(rsc, "Looking up %s in %s", name, rsc->id);

	if (create \|\| g_hash_table_size(rsc->parameters) == 0) {
	if (node != NULL) {
	pe_rsc_trace(rsc, "Creating hash with node %s", node->details->uname);
	} else {
	pe_rsc_trace(rsc, "Creating default hash");
	}

	local_hash = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	get_rsc_attributes(local_hash, rsc, node, data_set);

	hash = local_hash;
	}

	value = g_hash_table_lookup(hash, name);
	if (value == NULL) {
	/* try meta attributes instead */
	value = g_hash_table_lookup(rsc->meta, name);
	}

	if (value != NULL) {
	value_copy = strdup(value);
	}
	if (local_hash != NULL) {
	g_hash_table_destroy(local_hash);
	}
	return value_copy;
	}

	gboolean
	native_active(resource_t * rsc, gboolean all)
	{
	GListPtr gIter = rsc->running_on;

	for (; gIter != NULL; gIter = gIter->next) {
	node_t a_node = (node_t ) gIter->data;

	if (a_node->details->unclean) {
	crm_debug("Resource %s: node %s is unclean", rsc->id, a_node->details->uname);
	return TRUE;
	} else if (a_node->details->online == FALSE) {
	crm_debug("Resource %s: node %s is offline", rsc->id, a_node->details->uname);
	} else {
	crm_debug("Resource %s active on %s", rsc->id, a_node->details->uname);
	return TRUE;
	}
	}

	return FALSE;
	}

	struct print_data_s {
	long options;
	void *print_data;
	};

	static void
	native_print_attr(gpointer key, gpointer value, gpointer user_data)
	{
	long options = ((struct print_data_s *)user_data)->options;
	void print_data = ((struct print_data_s )user_data)->print_data;

	status_print("Option: %s = %s\n", (char )key, (char )value);
	}

	static const char *
	native_pending_state(resource_t * rsc)
	{
	const char *pending_state = NULL;

	if (safe_str_eq(rsc->pending_task, CRMD_ACTION_START)) {
	pending_state = "Starting";

	} else if (safe_str_eq(rsc->pending_task, CRMD_ACTION_STOP)) {
	pending_state = "Stopping";

	} else if (safe_str_eq(rsc->pending_task, CRMD_ACTION_MIGRATE)) {
	pending_state = "Migrating";

	} else if (safe_str_eq(rsc->pending_task, CRMD_ACTION_MIGRATED)) {
	/* Work might be done in here. */
	pending_state = "Migrating";

	} else if (safe_str_eq(rsc->pending_task, CRMD_ACTION_PROMOTE)) {
	pending_state = "Promoting";

	} else if (safe_str_eq(rsc->pending_task, CRMD_ACTION_DEMOTE)) {
	pending_state = "Demoting";
	}

	return pending_state;
	}

	static const char *
	native_pending_task(resource_t * rsc)
	{
	const char *pending_task = NULL;

	if (safe_str_eq(rsc->pending_task, CRMD_ACTION_NOTIFY)) {
	/* "Notifying" is not very useful to be shown. */
	pending_task = NULL;

	} else if (safe_str_eq(rsc->pending_task, CRMD_ACTION_STATUS)) {
	pending_task = "Monitoring";

	/* Pending probes are not printed, even if pending
	* operations are requested. If someone ever requests that
	* behavior, uncomment this and the corresponding part of
	* unpack.c:unpack_rsc_op().
	*/
	/*
	} else if (safe_str_eq(rsc->pending_task, "probe")) {
	pending_task = "Checking";
	*/
	}

	return pending_task;
	}

	static void
	native_print_xml(resource_t * rsc, const char pre_text, long options, void print_data)
	{
	enum rsc_role_e role = rsc->role;
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	const char *rsc_state = NULL;

	if(role == RSC_ROLE_STARTED && uber_parent(rsc)->variant == pe_master) {
	role = RSC_ROLE_SLAVE;
	}

	/* resource information. */
	status_print("%s<resource ", pre_text);
	status_print("id=\"%s\" ", rsc_printable_id(rsc));
	status_print("resource_agent=\"%s%s%s:%s\" ",
	class,
	prov ? "::" : "", prov ? prov : "", crm_element_value(rsc->xml, XML_ATTR_TYPE));

	if (options & pe_print_pending) {
	rsc_state = native_pending_state(rsc);
	}
	if (rsc_state == NULL) {
	rsc_state = role2text(role);
	}
	status_print("role=\"%s\" ", rsc_state);
	status_print("active=\"%s\" ", rsc->fns->active(rsc, TRUE) ? "true" : "false");
	status_print("orphaned=\"%s\" ", is_set(rsc->flags, pe_rsc_orphan) ? "true" : "false");
	status_print("managed=\"%s\" ", is_set(rsc->flags, pe_rsc_managed) ? "true" : "false");
	status_print("failed=\"%s\" ", is_set(rsc->flags, pe_rsc_failed) ? "true" : "false");
	status_print("failure_ignored=\"%s\" ",
	is_set(rsc->flags, pe_rsc_failure_ignored) ? "true" : "false");
	status_print("nodes_running_on=\"%d\" ", g_list_length(rsc->running_on));

	if (options & pe_print_pending) {
	const char *pending_task = native_pending_task(rsc);

	if (pending_task) {
	status_print("pending=\"%s\" ", pending_task);
	}
	}

	if (options & pe_print_dev) {
	status_print("provisional=\"%s\" ",
	is_set(rsc->flags, pe_rsc_provisional) ? "true" : "false");
	status_print("runnable=\"%s\" ", is_set(rsc->flags, pe_rsc_runnable) ? "true" : "false");
	status_print("priority=\"%f\" ", (double)rsc->priority);
	status_print("variant=\"%s\" ", crm_element_name(rsc->xml));
	}

	/* print out the nodes this resource is running on */
	if (options & pe_print_rsconly) {
	status_print("/>\n");
	/* do nothing */
	} else if (g_list_length(rsc->running_on) > 0) {
	GListPtr gIter = rsc->running_on;

	status_print(">\n");
	for (; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	status_print("%s <node name=\"%s\" id=\"%s\" cached=\"%s\"/>\n", pre_text,
	node->details->uname, node->details->id,
	node->details->online ? "false" : "true");
	}
	status_print("%s</resource>\n", pre_text);
	} else {
	status_print("/>\n");
	}
	}


	void
	native_print(resource_t * rsc, const char pre_text, long options, void print_data)
	{
	node_t *node = NULL;
	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);
	const char *target_role = NULL;

	int offset = 0;
	char buffer[LINE_MAX];

	CRM_ASSERT(rsc->variant == pe_native);
	CRM_ASSERT(kind != NULL);

	if (rsc->meta) {
	const char *is_internal = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERNAL_RSC);
	if (crm_is_true(is_internal)) {
	crm_trace("skipping print of internal resource %s", rsc->id);
	return;
	}
	target_role = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);
	}

	if (pre_text == NULL && (options & pe_print_printf)) {
	pre_text = " ";
	}

	if (options & pe_print_xml) {
	native_print_xml(rsc, pre_text, options, print_data);
	return;
	}

	if (rsc->running_on != NULL) {
	node = rsc->running_on->data;
	}
	if ((options & pe_print_rsconly) \|\| g_list_length(rsc->running_on) > 1) {
	node = NULL;
	}

	if (options & pe_print_html) {
	if (is_not_set(rsc->flags, pe_rsc_managed)) {
	status_print("<font color=\"yellow\">");

	} else if (is_set(rsc->flags, pe_rsc_failed)) {
	status_print("<font color=\"red\">");

	} else if (rsc->variant == pe_native && g_list_length(rsc->running_on) == 0) {
	status_print("<font color=\"red\">");

	} else if (g_list_length(rsc->running_on) > 1) {
	status_print("<font color=\"orange\">");

	} else if (is_set(rsc->flags, pe_rsc_failure_ignored)) {
	status_print("<font color=\"yellow\">");

	} else {
	status_print("<font color=\"green\">");
	}
	}

	if(pre_text) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", pre_text);
	}
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", rsc_printable_id(rsc));
	offset += snprintf(buffer + offset, LINE_MAX - offset, "\t(%s", class);
	if (safe_str_eq(class, "ocf")) {
	const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	offset += snprintf(buffer + offset, LINE_MAX - offset, "::%s", prov);
	}
	offset += snprintf(buffer + offset, LINE_MAX - offset, ":%s):\t", kind);
	if(is_set(rsc->flags, pe_rsc_orphan)) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " ORPHANED ");
	}
	if(rsc->role > RSC_ROLE_SLAVE && is_set(rsc->flags, pe_rsc_failed)) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "FAILED %s", role2text(rsc->role));
	} else if(is_set(rsc->flags, pe_rsc_failed)) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, "FAILED");
	} else {
	const char *rsc_state = NULL;

	if (options & pe_print_pending) {
	rsc_state = native_pending_state(rsc);
	}
	if (rsc_state == NULL) {
	rsc_state = role2text(rsc->role);
	}
	if (target_role) {
	enum rsc_role_e target_role_e = text2role(target_role);

	/* Ignore target role Started, as it is the default anyways
	* (and would also allow a Master to be Master).
	* Show if current role differs from target role,
	* or if target role limits our abilities. */
	if (target_role_e != RSC_ROLE_STARTED && (
	target_role_e == RSC_ROLE_SLAVE \|\|
	target_role_e == RSC_ROLE_STOPPED \|\|
	safe_str_neq(target_role, rsc_state)))
	{
	offset += snprintf(buffer + offset, LINE_MAX - offset, "(target-role:%s) ", target_role);
	}
	}
	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", rsc_state);
	}

	if(node) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " %s", node->details->uname);

	if (node->details->online == FALSE && node->details->unclean) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (UNCLEAN)");
	}
	}

	if (options & pe_print_pending) {
	const char *pending_task = native_pending_task(rsc);

	if (pending_task) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)", pending_task);
	}
	}

	if(is_not_set(rsc->flags, pe_rsc_managed)) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (unmanaged)");
	}
	if(is_set(rsc->flags, pe_rsc_failure_ignored)) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " (failure ignored)");
	}

	if ((options & pe_print_rsconly) \|\| g_list_length(rsc->running_on) > 1) {
	const char *desc = crm_element_value(rsc->xml, XML_ATTR_DESC);
	if(desc) {
	offset += snprintf(buffer + offset, LINE_MAX - offset, " %s", desc);
	}
	}

	CRM_LOG_ASSERT(offset > 0);
	status_print("%s", buffer);

	#if CURSES_ENABLED
	if ((options & pe_print_rsconly) \|\| g_list_length(rsc->running_on) > 1) {
	/* Done */

	} else if (options & pe_print_ncurses) {
	/* coverity[negative_returns] False positive */
	move(-1, 0);
	}
	#endif

	if (options & pe_print_html) {
	status_print(" </font> ");
	}

	if ((options & pe_print_rsconly)) {

	} else if (g_list_length(rsc->running_on) > 1) {
	GListPtr gIter = rsc->running_on;
	int counter = 0;

	if (options & pe_print_html) {
	status_print("<ul>\n");
	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print("[");
	}

	for (; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	counter++;

	if (options & pe_print_html) {
	status_print("<li>\n%s", node->details->uname);

	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print(" %s", node->details->uname);

	} else if ((options & pe_print_log)) {
	status_print("\t%d : %s", counter, node->details->uname);

	} else {
	status_print("%s", node->details->uname);
	}
	if (options & pe_print_html) {
	status_print("</li>\n");

	}
	}

	if (options & pe_print_html) {
	status_print("</ul>\n");
	} else if ((options & pe_print_printf)
	\|\| (options & pe_print_ncurses)) {
	status_print(" ]");
	}
	}

	if (options & pe_print_html) {
	status_print("<br/>\n");
	} else if (options & pe_print_suppres_nl) {
	/* nothing */
	} else if ((options & pe_print_printf) \|\| (options & pe_print_ncurses)) {
	status_print("\n");
	}

	if (options & pe_print_details) {
	struct print_data_s pdata;

	pdata.options = options;
	pdata.print_data = print_data;
	g_hash_table_foreach(rsc->parameters, native_print_attr, &pdata);
	}

	if (options & pe_print_dev) {
	GHashTableIter iter;
	node_t *node = NULL;

	status_print("%s\t(%s%svariant=%s, priority=%f)", pre_text,
	is_set(rsc->flags, pe_rsc_provisional) ? "provisional, " : "",
	is_set(rsc->flags, pe_rsc_runnable) ? "" : "non-startable, ",
	crm_element_name(rsc->xml), (double)rsc->priority);
	status_print("%s\tAllowed Nodes", pre_text);
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	status_print("%s\t * %s %d", pre_text, node->details->uname, node->weight);
	}
	}

	if (options & pe_print_max_details) {
	GHashTableIter iter;
	node_t *node = NULL;

	status_print("%s\t=== Allowed Nodes\n", pre_text);
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	print_node("\t", node, FALSE);
	}
	}
	}

	void
	native_free(resource_t * rsc)
	{
	pe_rsc_trace(rsc, "Freeing resource action list (not the data)");
	common_free(rsc);
	}

	enum rsc_role_e
	native_resource_state(const resource_t * rsc, gboolean current)
	{
	enum rsc_role_e role = rsc->next_role;

	if (current) {
	role = rsc->role;
	}
	pe_rsc_trace(rsc, "%s state: %s", rsc->id, role2text(role));
	return role;
	}

	node_t *
	native_location(resource_t * rsc, GListPtr * list, gboolean current)
	{
	node_t *one = NULL;
	GListPtr result = NULL;

	if (rsc->children) {
	GListPtr gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	child->fns->location(child, &result, current);
	}

	} else if (current && rsc->running_on) {
	result = g_list_copy(rsc->running_on);

	} else if (current == FALSE && rsc->allocated_to) {
	result = g_list_append(NULL, rsc->allocated_to);
	}

	if (result && g_list_length(result) == 1) {
	one = g_list_nth_data(result, 0);
	}

	if (list) {
	GListPtr gIter = result;

	for (; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	if (list == NULL \|\| pe_find_node_id(list, node->details->id) == NULL) {
	list = g_list_append(list, node);
	}
	}
	}

	g_list_free(result);
	return one;
	}

	static void
	get_rscs_brief(GListPtr rsc_list, GHashTable * rsc_table, GHashTable * active_table)
	{
	GListPtr gIter = rsc_list;

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;

	const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
	const char *kind = crm_element_value(rsc->xml, XML_ATTR_TYPE);

	int offset = 0;
	char buffer[LINE_MAX];

	int *rsc_counter = NULL;
	int *active_counter = NULL;

	if (rsc->variant != pe_native) {
	continue;
	}

	offset += snprintf(buffer + offset, LINE_MAX - offset, "%s", class);
	if (safe_str_eq(class, "ocf")) {
	const char *prov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
	offset += snprintf(buffer + offset, LINE_MAX - offset, "::%s", prov);
	}
	offset += snprintf(buffer + offset, LINE_MAX - offset, ":%s", kind);
	CRM_LOG_ASSERT(offset > 0);

	if (rsc_table) {
	rsc_counter = g_hash_table_lookup(rsc_table, buffer);
	if (rsc_counter == NULL) {
	rsc_counter = calloc(1, sizeof(int));
	*rsc_counter = 0;
	g_hash_table_insert(rsc_table, strdup(buffer), rsc_counter);
	}
	(*rsc_counter)++;
	}

	if (active_table) {
	GListPtr gIter2 = rsc->running_on;

	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	node_t node = (node_t ) gIter2->data;
	GHashTable *node_table = NULL;

	if (node->details->unclean == FALSE && node->details->online == FALSE) {
	continue;
	}

	node_table = g_hash_table_lookup(active_table, node->details->uname);
	if (node_table == NULL) {
	node_table = g_hash_table_new_full(crm_str_hash, g_str_equal, free, free);
	g_hash_table_insert(active_table, strdup(node->details->uname), node_table);
	}

	active_counter = g_hash_table_lookup(node_table, buffer);
	if (active_counter == NULL) {
	active_counter = calloc(1, sizeof(int));
	*active_counter = 0;
	g_hash_table_insert(node_table, strdup(buffer), active_counter);
	}
	(*active_counter)++;
	}
	}
	}
	}

	static void
	destroy_node_table(gpointer data)
	{
	GHashTable *node_table = data;

	if (node_table) {
	g_hash_table_destroy(node_table);
	}
	}

	void
	print_rscs_brief(GListPtr rsc_list, const char *pre_text, long options,
	void *print_data, gboolean print_all)
	{
	GHashTable *rsc_table = g_hash_table_new_full(crm_str_hash, g_str_equal, free, free);
	GHashTable *active_table = g_hash_table_new_full(crm_str_hash, g_str_equal,
	free, destroy_node_table);
	GHashTableIter hash_iter;
	char *type = NULL;
	int *rsc_counter = NULL;

	get_rscs_brief(rsc_list, rsc_table, active_table);

	g_hash_table_iter_init(&hash_iter, rsc_table);
	while (g_hash_table_iter_next(&hash_iter, (gpointer )&type, (gpointer )&rsc_counter)) {
	GHashTableIter hash_iter2;
	char *node_name = NULL;
	GHashTable *node_table = NULL;
	int active_counter_all = 0;

	g_hash_table_iter_init(&hash_iter2, active_table);
	while (g_hash_table_iter_next(&hash_iter2, (gpointer )&node_name, (gpointer )&node_table)) {
	int *active_counter = g_hash_table_lookup(node_table, type);

	if (active_counter == NULL \|\| *active_counter == 0) {
	continue;

	} else {
	active_counter_all += *active_counter;
	}

	if (options & pe_print_rsconly) {
	node_name = NULL;
	}

	if (options & pe_print_html) {
	status_print("<li>\n");
	}

	if (print_all) {
	status_print("%s%d/%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
	active_counter ? *active_counter : 0,
	rsc_counter ? *rsc_counter : 0, type,
	active_counter && (*active_counter > 0) && node_name ? node_name : "");
	} else {
	status_print("%s%d\t(%s):\tActive %s\n", pre_text ? pre_text : "",
	active_counter ? *active_counter : 0, type,
	active_counter && (*active_counter > 0) && node_name ? node_name : "");
	}

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}

	if (print_all && active_counter_all == 0) {
	if (options & pe_print_html) {
	status_print("<li>\n");
	}

	status_print("%s%d/%d\t(%s):\tActive\n", pre_text ? pre_text : "",
	active_counter_all,
	rsc_counter ? *rsc_counter : 0, type);

	if (options & pe_print_html) {
	status_print("</li>\n");
	}
	}
	}

	if (rsc_table) {
	g_hash_table_destroy(rsc_table);
	rsc_table = NULL;
	}
	if (active_table) {
	g_hash_table_destroy(active_table);
	active_table = NULL;
	}
	}
	diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
	index c149f491f6..6a125b0b64 100644
	--- a/lib/pengine/unpack.c
	+++ b/lib/pengine/unpack.c
	@@ -1,3396 +1,3398 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/
	#include <crm_internal.h>

	#include <glib.h>

	#include <crm/crm.h>
	#include <crm/services.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <crm/common/util.h>
	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>
	#include <unpack.h>

	CRM_TRACE_INIT_DATA(pe_status);

	#define set_config_flag(data_set, option, flag) do { \
	const char *tmp = pe_pref(data_set->config_hash, option); \
	if(tmp) { \
	if(crm_is_true(tmp)) { \
	set_bit(data_set->flags, flag); \
	} else { \
	clear_bit(data_set->flags, flag); \
	} \
	} \
	} while(0)

	gboolean unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op,
	enum action_fail_response failed, pe_working_set_t data_set);
	static gboolean determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node);

	static gboolean
	is_dangling_container_remote_node(node_t *node)
	{
	/* we are looking for a remote-node that was supposed to be mapped to a
	* container resource, but all traces of that container have disappeared
	* from both the config and the status section. */
	if (is_remote_node(node) &&
	node->details->remote_rsc &&
	node->details->remote_rsc->container == NULL &&
	is_set(node->details->remote_rsc->flags, pe_rsc_orphan_container_filler)) {
	return TRUE;
	}

	return FALSE;
	}

	void
	pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason)
	{
	CRM_CHECK(node, return);

	/* fence remote nodes living in a container by marking the container as failed. */
	if (is_container_remote_node(node)) {
	resource_t *rsc = node->details->remote_rsc->container;
	if (is_set(rsc->flags, pe_rsc_failed) == FALSE) {
	crm_warn("Remote node %s will be fenced by recovering container resource %s",
	node->details->uname, rsc->id, reason);
	/* node->details->unclean = TRUE; */
	node->details->remote_requires_reset = TRUE;
	set_bit(rsc->flags, pe_rsc_failed);
	}
	} else if (is_dangling_container_remote_node(node)) {
	crm_info("Fencing remote node %s has already occurred, container no longer exists. cleaning up dangling connection resource: %s",
	node->details->uname, reason);
	set_bit(node->details->remote_rsc->flags, pe_rsc_failed);

	} else if (is_baremetal_remote_node(node)) {
	if(pe_can_fence(data_set, node)) {
	crm_warn("Node %s will be fenced %s", node->details->uname, reason);
	} else {
	crm_warn("Node %s is unclean %s", node->details->uname, reason);
	}
	node->details->unclean = TRUE;
	node->details->remote_requires_reset = TRUE;

	} else if (node->details->unclean == FALSE) {
	if(pe_can_fence(data_set, node)) {
	crm_warn("Node %s will be fenced %s", node->details->uname, reason);
	} else {
	crm_warn("Node %s is unclean %s", node->details->uname, reason);
	}
	node->details->unclean = TRUE;
	} else {
	crm_trace("Huh? %s %s", node->details->uname, reason);
	}
	}

	gboolean
	unpack_config(xmlNode * config, pe_working_set_t * data_set)
	{
	const char *value = NULL;
	GHashTable *config_hash =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	xmlXPathObjectPtr xpathObj = NULL;

	if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) {
	xpathObj = xpath_search(data_set->input, "//nvpair[@name='provides' and @value='unfencing']");
	if(xpathObj && numXpathResults(xpathObj) > 0) {
	set_bit(data_set->flags, pe_flag_enable_unfencing);
	}
	freeXpathObject(xpathObj);
	}

	if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) {
	xpathObj = xpath_search(data_set->input, "//nvpair[@name='requires' and @value='unfencing']");
	if(xpathObj && numXpathResults(xpathObj) > 0) {
	set_bit(data_set->flags, pe_flag_enable_unfencing);
	}
	freeXpathObject(xpathObj);
	}


	#ifdef REDHAT_COMPAT_6
	if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) {
	xpathObj = xpath_search(data_set->input, "//primitive[@type='fence_scsi']");
	if(xpathObj && numXpathResults(xpathObj) > 0) {
	set_bit(data_set->flags, pe_flag_enable_unfencing);
	}
	freeXpathObject(xpathObj);
	}
	#endif

	data_set->config_hash = config_hash;

	unpack_instance_attributes(data_set->input, config, XML_CIB_TAG_PROPSET, NULL, config_hash,
	CIB_OPTIONS_FIRST, FALSE, data_set->now);

	verify_pe_options(data_set->config_hash);

	set_config_flag(data_set, "enable-startup-probes", pe_flag_startup_probes);
	if(is_not_set(data_set->flags, pe_flag_startup_probes)) {
	crm_info("Startup probes: disabled (dangerous)");
	}

	value = pe_pref(data_set->config_hash, XML_ATTR_HAVE_WATCHDOG);
	if (value && crm_is_true(value)) {
	crm_notice("Relying on watchdog integration for fencing");
	set_bit(data_set->flags, pe_flag_have_stonith_resource);
	}

	value = pe_pref(data_set->config_hash, "stonith-timeout");
	data_set->stonith_timeout = crm_get_msec(value);
	crm_debug("STONITH timeout: %d", data_set->stonith_timeout);

	set_config_flag(data_set, "stonith-enabled", pe_flag_stonith_enabled);
	crm_debug("STONITH of failed nodes is %s",
	is_set(data_set->flags, pe_flag_stonith_enabled) ? "enabled" : "disabled");

	data_set->stonith_action = pe_pref(data_set->config_hash, "stonith-action");
	crm_trace("STONITH will %s nodes", data_set->stonith_action);

	set_config_flag(data_set, "stop-all-resources", pe_flag_stop_everything);
	crm_debug("Stop all active resources: %s",
	is_set(data_set->flags, pe_flag_stop_everything) ? "true" : "false");

	set_config_flag(data_set, "symmetric-cluster", pe_flag_symmetric_cluster);
	if (is_set(data_set->flags, pe_flag_symmetric_cluster)) {
	crm_debug("Cluster is symmetric" " - resources can run anywhere by default");
	}

	value = pe_pref(data_set->config_hash, "default-resource-stickiness");
	data_set->default_resource_stickiness = char2score(value);
	crm_debug("Default stickiness: %d", data_set->default_resource_stickiness);

	value = pe_pref(data_set->config_hash, "no-quorum-policy");

	if (safe_str_eq(value, "ignore")) {
	data_set->no_quorum_policy = no_quorum_ignore;

	} else if (safe_str_eq(value, "freeze")) {
	data_set->no_quorum_policy = no_quorum_freeze;

	} else if (safe_str_eq(value, "suicide")) {
	gboolean do_panic = FALSE;

	crm_element_value_int(data_set->input, XML_ATTR_QUORUM_PANIC, &do_panic);

	if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
	crm_config_err
	("Setting no-quorum-policy=suicide makes no sense if stonith-enabled=false");
	}

	if (do_panic && is_set(data_set->flags, pe_flag_stonith_enabled)) {
	data_set->no_quorum_policy = no_quorum_suicide;

	} else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE && do_panic == FALSE) {
	crm_notice("Resetting no-quorum-policy to 'stop': The cluster has never had quorum");
	data_set->no_quorum_policy = no_quorum_stop;
	}

	} else {
	data_set->no_quorum_policy = no_quorum_stop;
	}

	switch (data_set->no_quorum_policy) {
	case no_quorum_freeze:
	crm_debug("On loss of CCM Quorum: Freeze resources");
	break;
	case no_quorum_stop:
	crm_debug("On loss of CCM Quorum: Stop ALL resources");
	break;
	case no_quorum_suicide:
	crm_notice("On loss of CCM Quorum: Fence all remaining nodes");
	break;
	case no_quorum_ignore:
	crm_notice("On loss of CCM Quorum: Ignore");
	break;
	}

	set_config_flag(data_set, "stop-orphan-resources", pe_flag_stop_rsc_orphans);
	crm_trace("Orphan resources are %s",
	is_set(data_set->flags, pe_flag_stop_rsc_orphans) ? "stopped" : "ignored");

	set_config_flag(data_set, "stop-orphan-actions", pe_flag_stop_action_orphans);
	crm_trace("Orphan resource actions are %s",
	is_set(data_set->flags, pe_flag_stop_action_orphans) ? "stopped" : "ignored");

	set_config_flag(data_set, "remove-after-stop", pe_flag_remove_after_stop);
	crm_trace("Stopped resources are removed from the status section: %s",
	is_set(data_set->flags, pe_flag_remove_after_stop) ? "true" : "false");

	set_config_flag(data_set, "maintenance-mode", pe_flag_maintenance_mode);
	crm_trace("Maintenance mode: %s",
	is_set(data_set->flags, pe_flag_maintenance_mode) ? "true" : "false");

	if (is_set(data_set->flags, pe_flag_maintenance_mode)) {
	clear_bit(data_set->flags, pe_flag_is_managed_default);
	} else {
	set_config_flag(data_set, "is-managed-default", pe_flag_is_managed_default);
	}
	crm_trace("By default resources are %smanaged",
	is_set(data_set->flags, pe_flag_is_managed_default) ? "" : "not ");

	set_config_flag(data_set, "start-failure-is-fatal", pe_flag_start_failure_fatal);
	crm_trace("Start failures are %s",
	is_set(data_set->flags,
	pe_flag_start_failure_fatal) ? "always fatal" : "handled by failcount");

	node_score_red = char2score(pe_pref(data_set->config_hash, "node-health-red"));
	node_score_green = char2score(pe_pref(data_set->config_hash, "node-health-green"));
	node_score_yellow = char2score(pe_pref(data_set->config_hash, "node-health-yellow"));

	crm_debug("Node scores: 'red' = %s, 'yellow' = %s, 'green' = %s",
	pe_pref(data_set->config_hash, "node-health-red"),
	pe_pref(data_set->config_hash, "node-health-yellow"),
	pe_pref(data_set->config_hash, "node-health-green"));

	data_set->placement_strategy = pe_pref(data_set->config_hash, "placement-strategy");
	crm_trace("Placement strategy: %s", data_set->placement_strategy);

	return TRUE;
	}

	static void
	destroy_digest_cache(gpointer ptr)
	{
	op_digest_cache_t *data = ptr;

	free_xml(data->params_all);
	free_xml(data->params_secure);
	free_xml(data->params_restart);

	free(data->digest_all_calc);
	free(data->digest_restart_calc);
	free(data->digest_secure_calc);

	free(data);
	}

	static node_t *
	create_node(const char id, const char uname, const char type, const char score, pe_working_set_t * data_set)
	{
	node_t *new_node = NULL;

	if (pe_find_node(data_set->nodes, uname) != NULL) {
	crm_config_warn("Detected multiple node entries with uname=%s"
	" - this is rarely intended", uname);
	}

	new_node = calloc(1, sizeof(node_t));
	if (new_node == NULL) {
	return NULL;
	}

	new_node->weight = char2score(score);
	new_node->fixed = FALSE;
	new_node->details = calloc(1, sizeof(struct node_shared_s));

	if (new_node->details == NULL) {
	free(new_node);
	return NULL;
	}

	crm_trace("Creating node for entry %s/%s", uname, id);
	new_node->details->id = id;
	new_node->details->uname = uname;
	new_node->details->online = FALSE;
	new_node->details->shutdown = FALSE;
	new_node->details->rsc_discovery_enabled = TRUE;
	new_node->details->running_rsc = NULL;
	new_node->details->type = node_ping;

	if (safe_str_eq(type, "remote")) {
	new_node->details->type = node_remote;
	set_bit(data_set->flags, pe_flag_have_remote_nodes);
	} else if (type == NULL \|\| safe_str_eq(type, "member")
	\|\| safe_str_eq(type, NORMALNODE)) {
	new_node->details->type = node_member;
	}

	new_node->details->attrs = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str,
	g_hash_destroy_str);

	if (is_remote_node(new_node)) {
	g_hash_table_insert(new_node->details->attrs, strdup("#kind"), strdup("remote"));
	} else {
	g_hash_table_insert(new_node->details->attrs, strdup("#kind"), strdup("cluster"));
	}

	new_node->details->utilization =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
	g_hash_destroy_str);

	new_node->details->digest_cache =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
	destroy_digest_cache);

	data_set->nodes = g_list_insert_sorted(data_set->nodes, new_node, sort_node_uname);
	return new_node;
	}

	static const char *
	expand_remote_rsc_meta(xmlNode xml_obj, xmlNode parent, GHashTable **rsc_name_check)
	{
	xmlNode *xml_rsc = NULL;
	xmlNode *xml_tmp = NULL;
	xmlNode *attr_set = NULL;
	xmlNode *attr = NULL;

	const char *container_id = ID(xml_obj);
	const char *remote_name = NULL;
	const char *remote_server = NULL;
	const char *remote_port = NULL;
	const char *connect_timeout = "60s";
	const char *remote_allow_migrate=NULL;
	char *tmp_id = NULL;

	for (attr_set = __xml_first_child(xml_obj); attr_set != NULL; attr_set = __xml_next_element(attr_set)) {
	if (safe_str_neq((const char *)attr_set->name, XML_TAG_META_SETS)) {
	continue;
	}

	for (attr = __xml_first_child(attr_set); attr != NULL; attr = __xml_next_element(attr)) {
	const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
	const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);

	if (safe_str_eq(name, XML_RSC_ATTR_REMOTE_NODE)) {
	remote_name = value;
	} else if (safe_str_eq(name, "remote-addr")) {
	remote_server = value;
	} else if (safe_str_eq(name, "remote-port")) {
	remote_port = value;
	} else if (safe_str_eq(name, "remote-connect-timeout")) {
	connect_timeout = value;
	} else if (safe_str_eq(name, "remote-allow-migrate")) {
	remote_allow_migrate=value;
	}
	}
	}

	if (remote_name == NULL) {
	return NULL;
	}

	if (*rsc_name_check == NULL) {
	*rsc_name_check = g_hash_table_new(crm_str_hash, g_str_equal);
	for (xml_rsc = __xml_first_child(parent); xml_rsc != NULL; xml_rsc = __xml_next_element(xml_rsc)) {
	const char *id = ID(xml_rsc);

	/* avoiding heap allocation here because we know the duration of this hashtable allows us to */
	g_hash_table_insert(rsc_name_check, (char ) id, (char *) id);
	}
	}

	if (g_hash_table_lookup(*rsc_name_check, remote_name)) {

	crm_err("Naming conflict with remote-node=%s. remote-nodes can not have the same name as a resource.",
	remote_name);
	return NULL;
	}

	xml_rsc = create_xml_node(parent, XML_CIB_TAG_RESOURCE);

	crm_xml_add(xml_rsc, XML_ATTR_ID, remote_name);
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, "ocf");
	crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, "pacemaker");
	crm_xml_add(xml_rsc, XML_ATTR_TYPE, "remote");

	xml_tmp = create_xml_node(xml_rsc, XML_TAG_META_SETS);
	tmp_id = crm_concat(remote_name, XML_TAG_META_SETS, '_');
	crm_xml_add(xml_tmp, XML_ATTR_ID, tmp_id);
	free(tmp_id);

	attr = create_xml_node(xml_tmp, XML_CIB_TAG_NVPAIR);
	tmp_id = crm_concat(remote_name, "meta-attributes-container", '_');
	crm_xml_add(attr, XML_ATTR_ID, tmp_id);
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, XML_RSC_ATTR_CONTAINER);
	crm_xml_add(attr, XML_NVPAIR_ATTR_VALUE, container_id);
	free(tmp_id);

	attr = create_xml_node(xml_tmp, XML_CIB_TAG_NVPAIR);
	tmp_id = crm_concat(remote_name, "meta-attributes-internal", '_');
	crm_xml_add(attr, XML_ATTR_ID, tmp_id);
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, XML_RSC_ATTR_INTERNAL_RSC);
	crm_xml_add(attr, XML_NVPAIR_ATTR_VALUE, "true");
	free(tmp_id);

	if (remote_allow_migrate) {
	attr = create_xml_node(xml_tmp, XML_CIB_TAG_NVPAIR);
	tmp_id = crm_concat(remote_name, "meta-attributes-container", '_');
	crm_xml_add(attr, XML_ATTR_ID, tmp_id);
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, XML_OP_ATTR_ALLOW_MIGRATE);
	crm_xml_add(attr, XML_NVPAIR_ATTR_VALUE, remote_allow_migrate);
	free(tmp_id);
	}

	xml_tmp = create_xml_node(xml_rsc, "operations");
	attr = create_xml_node(xml_tmp, XML_ATTR_OP);
	tmp_id = crm_concat(remote_name, "monitor-interval-30s", '_');
	crm_xml_add(attr, XML_ATTR_ID, tmp_id);
	crm_xml_add(attr, XML_ATTR_TIMEOUT, "30s");
	crm_xml_add(attr, XML_LRM_ATTR_INTERVAL, "30s");
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, "monitor");
	free(tmp_id);

	if (connect_timeout) {
	attr = create_xml_node(xml_tmp, XML_ATTR_OP);
	tmp_id = crm_concat(remote_name, "start-interval-0", '_');
	crm_xml_add(attr, XML_ATTR_ID, tmp_id);
	crm_xml_add(attr, XML_ATTR_TIMEOUT, connect_timeout);
	crm_xml_add(attr, XML_LRM_ATTR_INTERVAL, "0");
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, "start");
	free(tmp_id);
	}

	if (remote_port \|\| remote_server) {
	xml_tmp = create_xml_node(xml_rsc, XML_TAG_ATTR_SETS);
	tmp_id = crm_concat(remote_name, XML_TAG_ATTR_SETS, '_');
	crm_xml_add(xml_tmp, XML_ATTR_ID, tmp_id);
	free(tmp_id);

	if (remote_server) {
	attr = create_xml_node(xml_tmp, XML_CIB_TAG_NVPAIR);
	tmp_id = crm_concat(remote_name, "instance-attributes-addr", '_');
	crm_xml_add(attr, XML_ATTR_ID, tmp_id);
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, "addr");
	crm_xml_add(attr, XML_NVPAIR_ATTR_VALUE, remote_server);
	free(tmp_id);
	}
	if (remote_port) {
	attr = create_xml_node(xml_tmp, XML_CIB_TAG_NVPAIR);
	tmp_id = crm_concat(remote_name, "instance-attributes-port", '_');
	crm_xml_add(attr, XML_ATTR_ID, tmp_id);
	crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, "port");
	crm_xml_add(attr, XML_NVPAIR_ATTR_VALUE, remote_port);
	free(tmp_id);
	}
	}

	return remote_name;
	}

	static void
	handle_startup_fencing(pe_working_set_t data_set, node_t new_node)
	{
	static const char *blind_faith = NULL;
	static gboolean unseen_are_unclean = TRUE;

	if ((new_node->details->type == node_remote) && (new_node->details->remote_rsc == NULL)) {
	/* ignore fencing remote-nodes that don't have a conneciton resource associated
	* with them. This happens when remote-node entries get left in the nodes section
	* after the connection resource is removed */
	return;
	}

	blind_faith = pe_pref(data_set->config_hash, "startup-fencing");

	if (crm_is_true(blind_faith) == FALSE) {
	unseen_are_unclean = FALSE;
	crm_warn("Blind faith: not fencing unseen nodes");
	}

	if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE
	\|\| unseen_are_unclean == FALSE) {
	/* blind faith... */
	new_node->details->unclean = FALSE;

	} else {
	/* all nodes are unclean until we've seen their
	* status entry
	*/
	new_node->details->unclean = TRUE;
	}

	/* We need to be able to determine if a node's status section
	* exists or not separate from whether the node is unclean. */
	new_node->details->unseen = TRUE;
	}

	gboolean
	unpack_nodes(xmlNode * xml_nodes, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;
	node_t *new_node = NULL;
	const char *id = NULL;
	const char *uname = NULL;
	const char *type = NULL;
	const char *score = NULL;

	for (xml_obj = __xml_first_child(xml_nodes); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
	if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_NODE, TRUE)) {
	new_node = NULL;

	id = crm_element_value(xml_obj, XML_ATTR_ID);
	uname = crm_element_value(xml_obj, XML_ATTR_UNAME);
	type = crm_element_value(xml_obj, XML_ATTR_TYPE);
	score = crm_element_value(xml_obj, XML_RULE_ATTR_SCORE);
	crm_trace("Processing node %s/%s", uname, id);

	if (id == NULL) {
	crm_config_err("Must specify id tag in <node>");
	continue;
	}
	new_node = create_node(id, uname, type, score, data_set);

	if (new_node == NULL) {
	return FALSE;
	}

	/* if(data_set->have_quorum == FALSE */
	/* && data_set->no_quorum_policy == no_quorum_stop) { */
	/* /\* start shutting resources down \/ /
	/* new_node->weight = -INFINITY; */
	/* } */

	handle_startup_fencing(data_set, new_node);

	add_node_attrs(xml_obj, new_node, FALSE, data_set);
	unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_UTILIZATION, NULL,
	new_node->details->utilization, NULL, FALSE, data_set->now);

	crm_trace("Done with node %s", crm_element_value(xml_obj, XML_ATTR_UNAME));
	}
	}

	if (data_set->localhost && pe_find_node(data_set->nodes, data_set->localhost) == NULL) {
	crm_info("Creating a fake local node");
	create_node(data_set->localhost, data_set->localhost, NULL, 0, data_set);
	}

	return TRUE;
	}

	static void
	setup_container(resource_t * rsc, pe_working_set_t * data_set)
	{
	const char *container_id = NULL;

	if (rsc->children) {
	GListPtr gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	setup_container(child_rsc, data_set);
	}
	return;
	}

	container_id = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_CONTAINER);
	if (container_id && safe_str_neq(container_id, rsc->id)) {
	resource_t *container = pe_find_resource(data_set->resources, container_id);

	if (container) {
	rsc->container = container;
	container->fillers = g_list_append(container->fillers, rsc);
	pe_rsc_trace(rsc, "Resource %s's container is %s", rsc->id, container_id);
	} else {
	pe_err("Resource %s: Unknown resource container (%s)", rsc->id, container_id);
	}
	}
	}

	gboolean
	unpack_remote_nodes(xmlNode * xml_resources, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;
	GHashTable *rsc_name_check = NULL;

	/* generate remote nodes from resource config before unpacking resources */
	for (xml_obj = __xml_first_child(xml_resources); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
	const char *new_node_id = NULL;

	/* first check if this is a bare metal remote node. Bare metal remote nodes
	* are defined as a resource primitive only. */
	if (xml_contains_remote_node(xml_obj)) {
	new_node_id = ID(xml_obj);
	/* The "pe_find_node" check is here to make sure we don't iterate over
	* an expanded node that has already been added to the node list. */
	if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
	crm_trace("Found baremetal remote node %s in container resource %s", new_node_id, ID(xml_obj));
	create_node(new_node_id, new_node_id, "remote", NULL, data_set);
	}
	continue;
	}

	/* Now check for guest remote nodes.
	* guest remote nodes are defined within a resource primitive.
	* Example1: a vm resource might be configured as a remote node.
	* Example2: a vm resource might be configured within a group to be a remote node.
	* Note: right now we only support guest remote nodes in as a standalone primitive
	* or a primitive within a group. No cloned primitives can be a guest remote node
	* right now */
	if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RESOURCE, TRUE)) {
	/* expands a metadata defined remote resource into the xml config
	* as an actual rsc primitive to be unpacked later. */
	new_node_id = expand_remote_rsc_meta(xml_obj, xml_resources, &rsc_name_check);

	if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
	crm_trace("Found guest remote node %s in container resource %s", new_node_id, ID(xml_obj));
	create_node(new_node_id, new_node_id, "remote", NULL, data_set);
	}
	continue;

	} else if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_GROUP, TRUE)) {
	xmlNode *xml_obj2 = NULL;
	/* search through a group to see if any of the primitive contain a remote node. */
	for (xml_obj2 = __xml_first_child(xml_obj); xml_obj2 != NULL; xml_obj2 = __xml_next_element(xml_obj2)) {

	new_node_id = expand_remote_rsc_meta(xml_obj2, xml_resources, &rsc_name_check);

	if (new_node_id && pe_find_node(data_set->nodes, new_node_id) == NULL) {
	crm_trace("Found guest remote node %s in container resource %s which is in group %s", new_node_id, ID(xml_obj2), ID(xml_obj));
	create_node(new_node_id, new_node_id, "remote", NULL, data_set);
	}
	}
	}
	}
	if (rsc_name_check) {
	g_hash_table_destroy(rsc_name_check);
	}

	return TRUE;
	}


	/* Call this after all the nodes and resources have been
	* unpacked, but before the status section is read.
	*
	* A remote node's online status is reflected by the state
	* of the remote node's connection resource. We need to link
	* the remote node to this connection resource so we can have
	* easy access to the connection resource during the PE calculations.
	*/
	static void
	link_rsc2remotenode(pe_working_set_t data_set, resource_t new_rsc)
	{
	node_t *remote_node = NULL;

	if (new_rsc->is_remote_node == FALSE) {
	return;
	}

	if (is_set(data_set->flags, pe_flag_quick_location)) {
	/* remote_nodes and remote_resources are not linked in quick location calculations */
	return;
	}

	print_resource(LOG_DEBUG_3, "Linking remote-node connection resource, ", new_rsc, FALSE);

	remote_node = pe_find_node(data_set->nodes, new_rsc->id);
	CRM_CHECK(remote_node != NULL, return;);

	remote_node->details->remote_rsc = new_rsc;
	/* If this is a baremetal remote-node (no container resource
	* associated with it) then we need to handle startup fencing the same way
	* as cluster nodes. */
	if (new_rsc->container == NULL) {
	handle_startup_fencing(data_set, remote_node);
	} else {
	/* At this point we know if the remote node is a container or baremetal
	* remote node, update the #kind attribute if a container is involved */
	g_hash_table_replace(remote_node->details->attrs, strdup("#kind"), strdup("container"));
	}
	}

	static void
	destroy_tag(gpointer data)
	{
	tag_t *tag = data;

	if (tag) {
	free(tag->id);
	g_list_free_full(tag->refs, free);
	free(tag);
	}
	}

	gboolean
	unpack_resources(xmlNode * xml_resources, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;
	GListPtr gIter = NULL;

	data_set->template_rsc_sets =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
	destroy_tag);

	for (xml_obj = __xml_first_child(xml_resources); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
	resource_t *new_rsc = NULL;

	if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_RSC_TEMPLATE, TRUE)) {
	const char *template_id = ID(xml_obj);

	if (template_id && g_hash_table_lookup_extended(data_set->template_rsc_sets,
	template_id, NULL, NULL) == FALSE) {
	/* Record the template's ID for the knowledge of its existence anyway. */
	g_hash_table_insert(data_set->template_rsc_sets, strdup(template_id), NULL);
	}
	continue;
	}

	crm_trace("Beginning unpack... <%s id=%s... >", crm_element_name(xml_obj), ID(xml_obj));
	if (common_unpack(xml_obj, &new_rsc, NULL, data_set)) {
	data_set->resources = g_list_append(data_set->resources, new_rsc);

	if (xml_contains_remote_node(xml_obj)) {
	new_rsc->is_remote_node = TRUE;
	}
	print_resource(LOG_DEBUG_3, "Added ", new_rsc, FALSE);

	} else {
	crm_config_err("Failed unpacking %s %s",
	crm_element_name(xml_obj), crm_element_value(xml_obj, XML_ATTR_ID));
	if (new_rsc != NULL && new_rsc->fns != NULL) {
	new_rsc->fns->free(new_rsc);
	}
	}
	}

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;

	setup_container(rsc, data_set);
	link_rsc2remotenode(data_set, rsc);
	}

	data_set->resources = g_list_sort(data_set->resources, sort_rsc_priority);
	if (is_set(data_set->flags, pe_flag_quick_location)) {
	/* Ignore */

	} else if (is_set(data_set->flags, pe_flag_stonith_enabled)
	&& is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {

	crm_config_err("Resource start-up disabled since no STONITH resources have been defined");
	crm_config_err("Either configure some or disable STONITH with the stonith-enabled option");
	crm_config_err("NOTE: Clusters with shared data need STONITH to ensure data integrity");
	}

	return TRUE;
	}

	gboolean
	unpack_tags(xmlNode * xml_tags, pe_working_set_t * data_set)
	{
	xmlNode *xml_tag = NULL;

	data_set->tags =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, destroy_tag);

	for (xml_tag = __xml_first_child(xml_tags); xml_tag != NULL; xml_tag = __xml_next_element(xml_tag)) {
	xmlNode *xml_obj_ref = NULL;
	const char *tag_id = ID(xml_tag);

	if (crm_str_eq((const char *)xml_tag->name, XML_CIB_TAG_TAG, TRUE) == FALSE) {
	continue;
	}

	if (tag_id == NULL) {
	crm_config_err("Failed unpacking %s: %s should be specified",
	crm_element_name(xml_tag), XML_ATTR_ID);
	continue;
	}

	for (xml_obj_ref = __xml_first_child(xml_tag); xml_obj_ref != NULL; xml_obj_ref = __xml_next_element(xml_obj_ref)) {
	const char *obj_ref = ID(xml_obj_ref);

	if (crm_str_eq((const char *)xml_obj_ref->name, XML_CIB_TAG_OBJ_REF, TRUE) == FALSE) {
	continue;
	}

	if (obj_ref == NULL) {
	crm_config_err("Failed unpacking %s for tag %s: %s should be specified",
	crm_element_name(xml_obj_ref), tag_id, XML_ATTR_ID);
	continue;
	}

	if (add_tag_ref(data_set->tags, tag_id, obj_ref) == FALSE) {
	return FALSE;
	}
	}
	}

	return TRUE;
	}

	/* The ticket state section:
	* "/cib/status/tickets/ticket_state" */
	static gboolean
	unpack_ticket_state(xmlNode * xml_ticket, pe_working_set_t * data_set)
	{
	const char *ticket_id = NULL;
	const char *granted = NULL;
	const char *last_granted = NULL;
	const char *standby = NULL;
	xmlAttrPtr xIter = NULL;

	ticket_t *ticket = NULL;

	ticket_id = ID(xml_ticket);
	if (ticket_id == NULL \|\| strlen(ticket_id) == 0) {
	return FALSE;
	}

	crm_trace("Processing ticket state for %s", ticket_id);

	ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
	if (ticket == NULL) {
	ticket = ticket_new(ticket_id, data_set);
	if (ticket == NULL) {
	return FALSE;
	}
	}

	for (xIter = xml_ticket->properties; xIter; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;
	const char *prop_value = crm_element_value(xml_ticket, prop_name);

	if (crm_str_eq(prop_name, XML_ATTR_ID, TRUE)) {
	continue;
	}
	g_hash_table_replace(ticket->state, strdup(prop_name), strdup(prop_value));
	}

	granted = g_hash_table_lookup(ticket->state, "granted");
	if (granted && crm_is_true(granted)) {
	ticket->granted = TRUE;
	crm_info("We have ticket '%s'", ticket->id);
	} else {
	ticket->granted = FALSE;
	crm_info("We do not have ticket '%s'", ticket->id);
	}

	last_granted = g_hash_table_lookup(ticket->state, "last-granted");
	if (last_granted) {
	ticket->last_granted = crm_parse_int(last_granted, 0);
	}

	standby = g_hash_table_lookup(ticket->state, "standby");
	if (standby && crm_is_true(standby)) {
	ticket->standby = TRUE;
	if (ticket->granted) {
	crm_info("Granted ticket '%s' is in standby-mode", ticket->id);
	}
	} else {
	ticket->standby = FALSE;
	}

	crm_trace("Done with ticket state for %s", ticket_id);

	return TRUE;
	}

	static gboolean
	unpack_tickets_state(xmlNode * xml_tickets, pe_working_set_t * data_set)
	{
	xmlNode *xml_obj = NULL;

	for (xml_obj = __xml_first_child(xml_tickets); xml_obj != NULL; xml_obj = __xml_next_element(xml_obj)) {
	if (crm_str_eq((const char *)xml_obj->name, XML_CIB_TAG_TICKET_STATE, TRUE) == FALSE) {
	continue;
	}
	unpack_ticket_state(xml_obj, data_set);
	}

	return TRUE;
	}

	/* Compatibility with the deprecated ticket state section:
	* "/cib/status/tickets/instance_attributes" */
	static void
	get_ticket_state_legacy(gpointer key, gpointer value, gpointer user_data)
	{
	const char *long_key = key;
	char *state_key = NULL;

	const char *granted_prefix = "granted-ticket-";
	const char *last_granted_prefix = "last-granted-";
	static int granted_prefix_strlen = 0;
	static int last_granted_prefix_strlen = 0;

	const char *ticket_id = NULL;
	const char *is_granted = NULL;
	const char *last_granted = NULL;
	const char *sep = NULL;

	ticket_t *ticket = NULL;
	pe_working_set_t *data_set = user_data;

	if (granted_prefix_strlen == 0) {
	granted_prefix_strlen = strlen(granted_prefix);
	}

	if (last_granted_prefix_strlen == 0) {
	last_granted_prefix_strlen = strlen(last_granted_prefix);
	}

	if (strstr(long_key, granted_prefix) == long_key) {
	ticket_id = long_key + granted_prefix_strlen;
	if (strlen(ticket_id)) {
	state_key = strdup("granted");
	is_granted = value;
	}
	} else if (strstr(long_key, last_granted_prefix) == long_key) {
	ticket_id = long_key + last_granted_prefix_strlen;
	if (strlen(ticket_id)) {
	state_key = strdup("last-granted");
	last_granted = value;
	}
	} else if ((sep = strrchr(long_key, '-'))) {
	ticket_id = sep + 1;
	state_key = strndup(long_key, strlen(long_key) - strlen(sep));
	}

	if (ticket_id == NULL \|\| strlen(ticket_id) == 0) {
	free(state_key);
	return;
	}

	if (state_key == NULL \|\| strlen(state_key) == 0) {
	free(state_key);
	return;
	}

	ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
	if (ticket == NULL) {
	ticket = ticket_new(ticket_id, data_set);
	if (ticket == NULL) {
	free(state_key);
	return;
	}
	}

	g_hash_table_replace(ticket->state, state_key, strdup(value));

	if (is_granted) {
	if (crm_is_true(is_granted)) {
	ticket->granted = TRUE;
	crm_info("We have ticket '%s'", ticket->id);
	} else {
	ticket->granted = FALSE;
	crm_info("We do not have ticket '%s'", ticket->id);
	}

	} else if (last_granted) {
	ticket->last_granted = crm_parse_int(last_granted, 0);
	}
	}

	/* remove nodes that are down, stopping */
	/* create +ve rsc_to_node constraints between resources and the nodes they are running on */
	/* anything else? */
	gboolean
	unpack_status(xmlNode * status, pe_working_set_t * data_set)
	{
	const char *id = NULL;
	const char *uname = NULL;

	xmlNode *state = NULL;
	xmlNode *lrm_rsc = NULL;
	node_t *this_node = NULL;

	crm_trace("Beginning unpack");

	if (data_set->tickets == NULL) {
	data_set->tickets =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, destroy_ticket);
	}

	for (state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
	if (crm_str_eq((const char *)state->name, XML_CIB_TAG_TICKETS, TRUE)) {
	xmlNode *xml_tickets = state;
	GHashTable *state_hash = NULL;

	/* Compatibility with the deprecated ticket state section:
	* Unpack the attributes in the deprecated "/cib/status/tickets/instance_attributes" if it exists. */
	state_hash =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str,
	g_hash_destroy_str);

	unpack_instance_attributes(data_set->input, xml_tickets, XML_TAG_ATTR_SETS, NULL,
	state_hash, NULL, TRUE, data_set->now);

	g_hash_table_foreach(state_hash, get_ticket_state_legacy, data_set);

	if (state_hash) {
	g_hash_table_destroy(state_hash);
	}

	/* Unpack the new "/cib/status/tickets/ticket_state"s */
	unpack_tickets_state(xml_tickets, data_set);
	}

	if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE)) {
	xmlNode *attrs = NULL;
	const char *resource_discovery_enabled = NULL;

	id = crm_element_value(state, XML_ATTR_ID);
	uname = crm_element_value(state, XML_ATTR_UNAME);
	this_node = pe_find_node_any(data_set->nodes, id, uname);

	if (uname == NULL) {
	/* error */
	continue;

	} else if (this_node == NULL) {
	crm_config_warn("Node %s in status section no longer exists", uname);
	continue;

	} else if (is_remote_node(this_node)) {
	/* online state for remote nodes is determined by the
	* rsc state after all the unpacking is done. we do however
	* need to mark whether or not the node has been fenced as this plays
	* a role during unpacking cluster node resource state */
	this_node->details->remote_was_fenced =
	crm_atoi(crm_element_value(state, XML_NODE_IS_FENCED), "0");
	continue;
	}

	crm_trace("Processing node id=%s, uname=%s", id, uname);

	/* Mark the node as provisionally clean
	* - at least we have seen it in the current cluster's lifetime
	*/
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;
	attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
	add_node_attrs(attrs, this_node, TRUE, data_set);

	if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "standby"))) {
	crm_info("Node %s is in standby-mode", this_node->details->uname);
	this_node->details->standby = TRUE;
	}

	if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "maintenance"))) {
	crm_info("Node %s is in maintenance-mode", this_node->details->uname);
	this_node->details->maintenance = TRUE;
	}

	resource_discovery_enabled = g_hash_table_lookup(this_node->details->attrs, XML_NODE_ATTR_RSC_DISCOVERY);
	if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
	crm_warn("ignoring %s attribute on node %s, disabling resource discovery is not allowed on cluster nodes",
	XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
	}

	crm_trace("determining node state");
	determine_online_status(state, this_node, data_set);

	if (this_node->details->online && data_set->no_quorum_policy == no_quorum_suicide) {
	/* Everything else should flow from this automatically
	* At least until the PE becomes able to migrate off healthy resources
	*/
	pe_fence_node(data_set, this_node, "because the cluster does not have quorum");
	}
	}
	}

	/* Now that we know all node states, we can safely handle migration ops */
	for (state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
	if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
	continue;
	}

	id = crm_element_value(state, XML_ATTR_ID);
	uname = crm_element_value(state, XML_ATTR_UNAME);
	this_node = pe_find_node_any(data_set->nodes, id, uname);

	if (this_node == NULL) {
	crm_info("Node %s is unknown", id);
	continue;

	} else if (is_remote_node(this_node)) {

	/* online status of remote node can not be determined until all other
	* resource status is unpacked. */
	continue;
	} else if (this_node->details->online \|\| is_set(data_set->flags, pe_flag_stonith_enabled)) {
	crm_trace("Processing lrm resource entries on healthy node: %s",
	this_node->details->uname);
	lrm_rsc = find_xml_node(state, XML_CIB_TAG_LRM, FALSE);
	lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
	unpack_lrm_resources(this_node, lrm_rsc, data_set);
	}
	}

	/* now that the rest of the cluster's status is determined
	* calculate remote-nodes */
	unpack_remote_status(status, data_set);

	return TRUE;
	}

	gboolean
	unpack_remote_status(xmlNode * status, pe_working_set_t * data_set)
	{
	const char *id = NULL;
	const char *uname = NULL;
	GListPtr gIter = NULL;

	xmlNode *state = NULL;
	xmlNode *lrm_rsc = NULL;
	node_t *this_node = NULL;

	if (is_set(data_set->flags, pe_flag_have_remote_nodes) == FALSE) {
	crm_trace("no remote nodes to unpack");
	return TRUE;
	}

	/* get online status */
	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	this_node = gIter->data;

	if ((this_node == NULL) \|\| (is_remote_node(this_node) == FALSE)) {
	continue;
	}
	determine_remote_online_status(data_set, this_node);
	}

	/* process attributes */
	for (state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
	const char *resource_discovery_enabled = NULL;
	xmlNode *attrs = NULL;
	if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
	continue;
	}

	id = crm_element_value(state, XML_ATTR_ID);
	uname = crm_element_value(state, XML_ATTR_UNAME);
	this_node = pe_find_node_any(data_set->nodes, id, uname);

	if ((this_node == NULL) \|\| (is_remote_node(this_node) == FALSE)) {
	continue;
	}
	crm_trace("Processing remote node id=%s, uname=%s", id, uname);

	if (this_node->details->remote_requires_reset == FALSE) {
	this_node->details->unclean = FALSE;
	this_node->details->unseen = FALSE;
	}
	attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
	add_node_attrs(attrs, this_node, TRUE, data_set);

	if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "standby"))) {
	crm_info("Node %s is in standby-mode", this_node->details->uname);
	this_node->details->standby = TRUE;
	}

	if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "maintenance"))) {
	crm_info("Node %s is in maintenance-mode", this_node->details->uname);
	this_node->details->maintenance = TRUE;
	}

	resource_discovery_enabled = g_hash_table_lookup(this_node->details->attrs, XML_NODE_ATTR_RSC_DISCOVERY);
	if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) {
	if (is_baremetal_remote_node(this_node) && is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
	crm_warn("ignoring %s attribute on baremetal remote node %s, disabling resource discovery requires stonith to be enabled.",
	XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname);
	} else {
	/* if we're here, this is either a baremetal node and fencing is enabled,
	* or this is a container node which we don't care if fencing is enabled
	* or not on. container nodes are 'fenced' by recovering the container resource
	* regardless of whether fencing is enabled. */
	crm_info("Node %s has resource discovery disabled", this_node->details->uname);
	this_node->details->rsc_discovery_enabled = FALSE;
	}
	}
	}

	/* process node rsc status */
	for (state = __xml_first_child(status); state != NULL; state = __xml_next_element(state)) {
	if (crm_str_eq((const char *)state->name, XML_CIB_TAG_STATE, TRUE) == FALSE) {
	continue;
	}

	id = crm_element_value(state, XML_ATTR_ID);
	uname = crm_element_value(state, XML_ATTR_UNAME);
	this_node = pe_find_node_any(data_set->nodes, id, uname);

	if ((this_node == NULL) \|\| (is_remote_node(this_node) == FALSE)) {
	continue;
	}
	crm_trace("Processing lrm resource entries on healthy remote node: %s",
	this_node->details->uname);
	lrm_rsc = find_xml_node(state, XML_CIB_TAG_LRM, FALSE);
	lrm_rsc = find_xml_node(lrm_rsc, XML_LRM_TAG_RESOURCES, FALSE);
	unpack_lrm_resources(this_node, lrm_rsc, data_set);
	}

	return TRUE;
	}

	static gboolean
	determine_online_status_no_fencing(pe_working_set_t * data_set, xmlNode * node_state,
	node_t * this_node)
	{
	gboolean online = FALSE;
	const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
	const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
	const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
	const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);

	if (!crm_is_true(in_cluster)) {
	crm_trace("Node is down: in_cluster=%s", crm_str(in_cluster));

	} else if (safe_str_eq(is_peer, ONLINESTATUS)) {
	if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {
	online = TRUE;
	} else {
	crm_debug("Node is not ready to run resources: %s", join);
	}

	} else if (this_node->details->expected_up == FALSE) {
	crm_trace("CRMd is down: in_cluster=%s", crm_str(in_cluster));
	crm_trace("\tis_peer=%s, join=%s, expected=%s",
	crm_str(is_peer), crm_str(join), crm_str(exp_state));

	} else {
	/* mark it unclean */
	pe_fence_node(data_set, this_node, "unexpectedly down");
	crm_info("\tin_cluster=%s, is_peer=%s, join=%s, expected=%s",
	crm_str(in_cluster), crm_str(is_peer), crm_str(join), crm_str(exp_state));
	}
	return online;
	}

	static gboolean
	determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_state,
	node_t * this_node)
	{
	gboolean online = FALSE;
	gboolean do_terminate = FALSE;
	const char *join = crm_element_value(node_state, XML_NODE_JOIN_STATE);
	const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER);
	const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER);
	const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
	const char *terminate = g_hash_table_lookup(this_node->details->attrs, "terminate");

	/*
	- XML_NODE_IN_CLUSTER ::= true\|false
	- XML_NODE_IS_PEER ::= true\|false\|online\|offline
	- XML_NODE_JOIN_STATE ::= member\|down\|pending\|banned
	- XML_NODE_EXPECTED ::= member\|down
	*/

	if (crm_is_true(terminate)) {
	do_terminate = TRUE;

	} else if (terminate != NULL && strlen(terminate) > 0) {
	/* could be a time() value */
	char t = terminate[0];

	if (t != '0' && isdigit(t)) {
	do_terminate = TRUE;
	}
	}

	crm_trace("%s: in_cluster=%s, is_peer=%s, join=%s, expected=%s, term=%d",
	this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
	crm_str(join), crm_str(exp_state), do_terminate);

	online = crm_is_true(in_cluster);
	if (safe_str_eq(is_peer, ONLINESTATUS)) {
	is_peer = XML_BOOLEAN_YES;
	}
	if (exp_state == NULL) {
	exp_state = CRMD_JOINSTATE_DOWN;
	}

	if (this_node->details->shutdown) {
	crm_debug("%s is shutting down", this_node->details->uname);
	- online = crm_is_true(is_peer); /* Slightly different criteria since we cant shut down a dead peer */
	+
	+ /* Slightly different criteria since we can't shut down a dead peer */
	+ online = crm_is_true(is_peer);

	} else if (in_cluster == NULL) {
	pe_fence_node(data_set, this_node, "because the peer has not been seen by the cluster");

	} else if (safe_str_eq(join, CRMD_JOINSTATE_NACK)) {
	pe_fence_node(data_set, this_node, "because it failed the pacemaker membership criteria");

	} else if (do_terminate == FALSE && safe_str_eq(exp_state, CRMD_JOINSTATE_DOWN)) {

	if (crm_is_true(in_cluster) \|\| crm_is_true(is_peer)) {
	crm_info("- Node %s is not ready to run resources", this_node->details->uname);
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	crm_trace("%s is down or still coming up", this_node->details->uname);
	}

	} else if (do_terminate && safe_str_eq(join, CRMD_JOINSTATE_DOWN)
	&& crm_is_true(in_cluster) == FALSE && crm_is_true(is_peer) == FALSE) {
	crm_info("Node %s was just shot", this_node->details->uname);
	online = FALSE;

	} else if (crm_is_true(in_cluster) == FALSE) {
	pe_fence_node(data_set, this_node, "because the node is no longer part of the cluster");

	} else if (crm_is_true(is_peer) == FALSE) {
	pe_fence_node(data_set, this_node, "because our peer process is no longer available");

	/* Everything is running at this point, now check join state */
	} else if (do_terminate) {
	pe_fence_node(data_set, this_node, "because termination was requested");

	} else if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {
	crm_info("Node %s is active", this_node->details->uname);

	} else if (safe_str_eq(join, CRMD_JOINSTATE_PENDING)
	\|\| safe_str_eq(join, CRMD_JOINSTATE_DOWN)) {
	crm_info("Node %s is not ready to run resources", this_node->details->uname);
	this_node->details->standby = TRUE;
	this_node->details->pending = TRUE;

	} else {
	pe_fence_node(data_set, this_node, "because the peer was in an unknown state");
	crm_warn("%s: in-cluster=%s, is-peer=%s, join=%s, expected=%s, term=%d, shutdown=%d",
	this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
	crm_str(join), crm_str(exp_state), do_terminate, this_node->details->shutdown);
	}

	return online;
	}

	static gboolean
	determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node)
	{
	resource_t *rsc = this_node->details->remote_rsc;
	resource_t *container = NULL;

	if (rsc == NULL) {
	this_node->details->online = FALSE;
	goto remote_online_done;
	}

	container = rsc->container;

	CRM_ASSERT(rsc != NULL);

	/* If the resource is currently started, mark it online. */
	if (rsc->role == RSC_ROLE_STARTED) {
	crm_trace("Remote node %s is set to ONLINE. role == started", this_node->details->id);
	this_node->details->online = TRUE;
	}

	/* consider this node shutting down if transitioning start->stop */
	if (rsc->role == RSC_ROLE_STARTED && rsc->next_role == RSC_ROLE_STOPPED) {
	crm_trace("Remote node %s shutdown. transition from start to stop role", this_node->details->id);
	this_node->details->shutdown = TRUE;
	}

	/* Now check all the failure conditions. */
	if(container && is_set(container->flags, pe_rsc_failed)) {
	crm_trace("Remote node %s is set to UNCLEAN. rsc failed.", this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = TRUE;

	} else if(is_set(rsc->flags, pe_rsc_failed)) {
	crm_trace("Remote node %s is set to OFFLINE. rsc failed.", this_node->details->id);
	this_node->details->online = FALSE;

	} else if (rsc->role == RSC_ROLE_STOPPED
	\|\| (container && container->role == RSC_ROLE_STOPPED)) {

	crm_trace("Remote node %s is set to OFFLINE. node is stopped.", this_node->details->id);
	this_node->details->online = FALSE;
	this_node->details->remote_requires_reset = FALSE;
	}

	remote_online_done:
	crm_trace("Remote node %s online=%s",
	this_node->details->id, this_node->details->online ? "TRUE" : "FALSE");
	return this_node->details->online;
	}

	gboolean
	determine_online_status(xmlNode * node_state, node_t * this_node, pe_working_set_t * data_set)
	{
	gboolean online = FALSE;
	const char *shutdown = NULL;
	const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);

	if (this_node == NULL) {
	crm_config_err("No node to check");
	return online;
	}

	this_node->details->shutdown = FALSE;
	this_node->details->expected_up = FALSE;
	shutdown = g_hash_table_lookup(this_node->details->attrs, XML_CIB_ATTR_SHUTDOWN);

	if (shutdown != NULL && safe_str_neq("0", shutdown)) {
	this_node->details->shutdown = TRUE;

	} else if (safe_str_eq(exp_state, CRMD_JOINSTATE_MEMBER)) {
	this_node->details->expected_up = TRUE;
	}

	if (this_node->details->type == node_ping) {
	this_node->details->unclean = FALSE;
	online = FALSE; /* As far as resource management is concerned,
	* the node is safely offline.
	* Anyone caught abusing this logic will be shot
	*/

	} else if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
	online = determine_online_status_no_fencing(data_set, node_state, this_node);

	} else {
	online = determine_online_status_fencing(data_set, node_state, this_node);
	}

	if (online) {
	this_node->details->online = TRUE;

	} else {
	/* remove node from contention */
	this_node->fixed = TRUE;
	this_node->weight = -INFINITY;
	}

	if (online && this_node->details->shutdown) {
	/* dont run resources here */
	this_node->fixed = TRUE;
	this_node->weight = -INFINITY;
	}

	if (this_node->details->type == node_ping) {
	crm_info("Node %s is not a pacemaker node", this_node->details->uname);

	} else if (this_node->details->unclean) {
	pe_proc_warn("Node %s is unclean", this_node->details->uname);

	} else if (this_node->details->online) {
	crm_info("Node %s is %s", this_node->details->uname,
	this_node->details->shutdown ? "shutting down" :
	this_node->details->pending ? "pending" :
	this_node->details->standby ? "standby" :
	this_node->details->maintenance ? "maintenance" : "online");

	} else {
	crm_trace("Node %s is offline", this_node->details->uname);
	}

	return online;
	}

	char *
	clone_strip(const char *last_rsc_id)
	{
	int lpc = 0;
	char *zero = NULL;

	CRM_CHECK(last_rsc_id != NULL, return NULL);
	lpc = strlen(last_rsc_id);
	while (--lpc > 0) {
	switch (last_rsc_id[lpc]) {
	case 0:
	crm_err("Empty string: %s", last_rsc_id);
	return NULL;
	break;
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
	break;
	case ':':
	zero = calloc(1, lpc + 1);
	memcpy(zero, last_rsc_id, lpc);
	zero[lpc] = 0;
	return zero;
	default:
	goto done;
	}
	}
	done:
	zero = strdup(last_rsc_id);
	return zero;
	}

	char *
	clone_zero(const char *last_rsc_id)
	{
	int lpc = 0;
	char *zero = NULL;

	CRM_CHECK(last_rsc_id != NULL, return NULL);
	if (last_rsc_id != NULL) {
	lpc = strlen(last_rsc_id);
	}

	while (--lpc > 0) {
	switch (last_rsc_id[lpc]) {
	case 0:
	return NULL;
	break;
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
	case '8':
	case '9':
	break;
	case ':':
	zero = calloc(1, lpc + 3);
	memcpy(zero, last_rsc_id, lpc);
	zero[lpc] = ':';
	zero[lpc + 1] = '0';
	zero[lpc + 2] = 0;
	return zero;
	default:
	goto done;
	}
	}
	done:
	lpc = strlen(last_rsc_id);
	zero = calloc(1, lpc + 3);
	memcpy(zero, last_rsc_id, lpc);
	zero[lpc] = ':';
	zero[lpc + 1] = '0';
	zero[lpc + 2] = 0;
	crm_trace("%s -> %s", last_rsc_id, zero);
	return zero;
	}

	static resource_t *
	create_fake_resource(const char rsc_id, xmlNode rsc_entry, pe_working_set_t * data_set)
	{
	resource_t *rsc = NULL;
	xmlNode *xml_rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);

	copy_in_properties(xml_rsc, rsc_entry);
	crm_xml_add(xml_rsc, XML_ATTR_ID, rsc_id);
	crm_log_xml_debug(xml_rsc, "Orphan resource");

	if (!common_unpack(xml_rsc, &rsc, NULL, data_set)) {
	return NULL;
	}

	if (xml_contains_remote_node(xml_rsc)) {
	node_t *node;

	crm_debug("Detected orphaned remote node %s", rsc_id);
	rsc->is_remote_node = TRUE;
	node = pe_find_node(data_set->nodes, rsc_id);
	if (node == NULL) {
	node = create_node(rsc_id, rsc_id, "remote", NULL, data_set);
	}
	link_rsc2remotenode(data_set, rsc);

	if (node) {
	crm_trace("Setting node %s as shutting down due to orphaned connection resource", rsc_id);
	node->details->shutdown = TRUE;
	}
	}

	if (crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER)) {
	/* This orphaned rsc needs to be mapped to a container. */
	crm_trace("Detected orphaned container filler %s", rsc_id);
	set_bit(rsc->flags, pe_rsc_orphan_container_filler);
	}
	set_bit(rsc->flags, pe_rsc_orphan);
	data_set->resources = g_list_append(data_set->resources, rsc);
	return rsc;
	}

	extern resource_t create_child_clone(resource_t rsc, int sub_id, pe_working_set_t * data_set);

	static resource_t *
	find_anonymous_clone(pe_working_set_t * data_set, node_t * node, resource_t * parent,
	const char *rsc_id)
	{
	GListPtr rIter = NULL;
	resource_t *rsc = NULL;
	gboolean skip_inactive = FALSE;

	CRM_ASSERT(parent != NULL);
	CRM_ASSERT(parent->variant == pe_clone \|\| parent->variant == pe_master);
	CRM_ASSERT(is_not_set(parent->flags, pe_rsc_unique));

	/* Find an instance active (or partially active for grouped clones) on the specified node */
	pe_rsc_trace(parent, "Looking for %s on %s in %s", rsc_id, node->details->uname, parent->id);
	for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
	GListPtr nIter = NULL;
	GListPtr locations = NULL;
	resource_t *child = rIter->data;

	child->fns->location(child, &locations, TRUE);
	if (locations == NULL) {
	pe_rsc_trace(child, "Resource %s, skip inactive", child->id);
	continue;
	}

	for (nIter = locations; nIter && rsc == NULL; nIter = nIter->next) {
	node_t *childnode = nIter->data;

	if (childnode->details == node->details) {
	/* ->find_rsc() because we might be a cloned group */
	rsc = parent->fns->find_rsc(child, rsc_id, NULL, pe_find_clone);
	if(rsc) {
	pe_rsc_trace(rsc, "Resource %s, active", rsc->id);
	}
	}

	/* Keep this block, it means we'll do the right thing if
	* anyone toggles the unique flag to 'off'
	*/
	if (rsc && rsc->running_on) {
	crm_notice("/Anonymous/ clone %s is already running on %s",
	parent->id, node->details->uname);
	skip_inactive = TRUE;
	rsc = NULL;
	}
	}

	g_list_free(locations);
	}

	/* Find an inactive instance */
	if (skip_inactive == FALSE) {
	pe_rsc_trace(parent, "Looking for %s anywhere", rsc_id);
	for (rIter = parent->children; rsc == NULL && rIter; rIter = rIter->next) {
	GListPtr locations = NULL;
	resource_t *child = rIter->data;

	if (is_set(child->flags, pe_rsc_block)) {
	pe_rsc_trace(child, "Skip: blocked in stopped state");
	continue;
	}

	child->fns->location(child, &locations, TRUE);
	if (locations == NULL) {
	/* ->find_rsc() because we might be a cloned group */
	rsc = parent->fns->find_rsc(child, rsc_id, NULL, pe_find_clone);
	pe_rsc_trace(parent, "Resource %s, empty slot", rsc->id);
	}
	g_list_free(locations);
	}
	}

	if (rsc == NULL) {
	/* Create an extra orphan */
	resource_t *top = create_child_clone(parent, -1, data_set);

	/* ->find_rsc() because we might be a cloned group */
	rsc = top->fns->find_rsc(top, rsc_id, NULL, pe_find_clone);
	CRM_ASSERT(rsc != NULL);

	pe_rsc_debug(parent, "Created orphan %s for %s: %s on %s", top->id, parent->id, rsc_id,
	node->details->uname);
	}

	if (safe_str_neq(rsc_id, rsc->id)) {
	pe_rsc_debug(rsc, "Internally renamed %s on %s to %s%s",
	rsc_id, node->details->uname, rsc->id,
	is_set(rsc->flags, pe_rsc_orphan) ? " (ORPHAN)" : "");
	}

	return rsc;
	}

	static resource_t *
	unpack_find_resource(pe_working_set_t * data_set, node_t * node, const char *rsc_id,
	xmlNode * rsc_entry)
	{
	resource_t *rsc = NULL;
	resource_t *parent = NULL;

	crm_trace("looking for %s", rsc_id);
	rsc = pe_find_resource(data_set->resources, rsc_id);

	/* no match */
	if (rsc == NULL) {
	/* Even when clone-max=0, we still create a single :0 orphan to match against */
	char *tmp = clone_zero(rsc_id);
	resource_t *clone0 = pe_find_resource(data_set->resources, tmp);

	if (clone0 && is_not_set(clone0->flags, pe_rsc_unique)) {
	rsc = clone0;
	} else {
	crm_trace("%s is not known as %s either", rsc_id, tmp);
	}

	parent = uber_parent(clone0);
	free(tmp);

	crm_trace("%s not found: %s", rsc_id, parent ? parent->id : "orphan");

	} else if (rsc->variant > pe_native) {
	crm_trace("%s is no longer a primitve resource, the lrm_resource entry is obsolete",
	rsc_id);
	return NULL;

	} else {
	parent = uber_parent(rsc);
	}

	if (parent && parent->variant > pe_group) {
	if (is_not_set(parent->flags, pe_rsc_unique)) {
	char *base = clone_strip(rsc_id);

	rsc = find_anonymous_clone(data_set, node, parent, base);
	CRM_ASSERT(rsc != NULL);
	free(base);
	}

	if (rsc && safe_str_neq(rsc_id, rsc->id)) {
	free(rsc->clone_name);
	rsc->clone_name = strdup(rsc_id);
	}
	}

	return rsc;
	}

	static resource_t *
	process_orphan_resource(xmlNode * rsc_entry, node_t * node, pe_working_set_t * data_set)
	{
	resource_t *rsc = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);

	crm_debug("Detected orphan resource %s on %s", rsc_id, node->details->uname);
	rsc = create_fake_resource(rsc_id, rsc_entry, data_set);

	if (is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
	clear_bit(rsc->flags, pe_rsc_managed);

	} else {
	print_resource(LOG_DEBUG_3, "Added orphan", rsc, FALSE);

	CRM_CHECK(rsc != NULL, return NULL);
	resource_location(rsc, NULL, -INFINITY, "__orphan_dont_run__", data_set);
	}
	return rsc;
	}

	static void
	process_rsc_state(resource_t * rsc, node_t * node,
	enum action_fail_response on_fail,
	xmlNode * migrate_op, pe_working_set_t * data_set)
	{
	node_t *tmpnode = NULL;
	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s",
	rsc->id, role2text(rsc->role), node->details->uname, fail2text(on_fail));

	/* process current state */
	if (rsc->role != RSC_ROLE_UNKNOWN) {
	resource_t *iter = rsc;

	while (iter) {
	if (g_hash_table_lookup(iter->known_on, node->details->id) == NULL) {
	node_t *n = node_copy(node);

	pe_rsc_trace(rsc, "%s (aka. %s) known on %s", rsc->id, rsc->clone_name,
	n->details->uname);
	g_hash_table_insert(iter->known_on, (gpointer) n->details->id, n);
	}
	if (is_set(iter->flags, pe_rsc_unique)) {
	break;
	}
	iter = iter->parent;
	}
	}

	if (rsc->role > RSC_ROLE_STOPPED
	&& node->details->online == FALSE && is_set(rsc->flags, pe_rsc_managed)) {

	char *reason = NULL;
	gboolean should_fence = FALSE;

	/* if this is a remote_node living in a container, fence the container
	* by recovering it. Mark the resource as unmanaged. Once the container
	* and remote connenction are re-established, the status section will
	* get reset in the crmd freeing up this resource to run again once we
	* are sure we know the resources state. */
	if (is_container_remote_node(node)) {
	set_bit(rsc->flags, pe_rsc_failed);

	should_fence = TRUE;
	} else if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
	if (is_baremetal_remote_node(node) && node->details->remote_rsc && is_not_set(node->details->remote_rsc->flags, pe_rsc_failed)) {
	/* setting unseen = true means that fencing of the remote node will
	* only occur if the connection resource is not going to start somewhere.
	* This allows connection resources on a failed cluster-node to move to
	* another node without requiring the baremetal remote nodes to be fenced
	* as well. */
	node->details->unseen = TRUE;
	reason = crm_strdup_printf("because %s is active there. Fencing will be revoked if remote-node connection can be re-established on another cluster-node.", rsc->id);
	}
	should_fence = TRUE;
	}

	if (should_fence) {
	if (reason == NULL) {
	reason = crm_strdup_printf("because %s is thought to be active there", rsc->id);
	}
	pe_fence_node(data_set, node, reason);
	}
	free(reason);
	}

	if (node->details->unclean) {
	/* No extra processing needed
	* Also allows resources to be started again after a node is shot
	*/
	on_fail = action_fail_ignore;
	}

	switch (on_fail) {
	case action_fail_ignore:
	/* nothing to do */
	break;

	case action_fail_fence:
	/* treat it as if it is still running
	* but also mark the node as unclean
	*/
	pe_fence_node(data_set, node, "because of resource failure(s)");
	break;

	case action_fail_standby:
	node->details->standby = TRUE;
	node->details->standby_onfail = TRUE;
	break;

	case action_fail_block:
	/* is_managed == FALSE will prevent any
	* actions being sent for the resource
	*/
	clear_bit(rsc->flags, pe_rsc_managed);
	set_bit(rsc->flags, pe_rsc_block);
	break;

	case action_fail_migrate:
	/* make sure it comes up somewhere else
	* or not at all
	*/
	resource_location(rsc, node, -INFINITY, "__action_migration_auto__", data_set);
	break;

	case action_fail_stop:
	rsc->next_role = RSC_ROLE_STOPPED;
	break;

	case action_fail_recover:
	if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
	set_bit(rsc->flags, pe_rsc_failed);
	stop_action(rsc, node, FALSE);
	}
	break;

	case action_fail_restart_container:
	set_bit(rsc->flags, pe_rsc_failed);

	if (rsc->container) {
	stop_action(rsc->container, node, FALSE);
	} else if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
	stop_action(rsc, node, FALSE);
	}
	break;
	case action_fail_reset_remote:
	set_bit(rsc->flags, pe_rsc_failed);
	if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
	tmpnode = NULL;
	if (rsc->is_remote_node) {
	tmpnode = pe_find_node(data_set->nodes, rsc->id);
	}
	if (tmpnode &&
	is_baremetal_remote_node(tmpnode) &&
	tmpnode->details->remote_was_fenced == 0) {

	/* connection resource to baremetal resource failed in a way that
	* should result in fencing the remote-node. */
	pe_fence_node(data_set, tmpnode, "because of connection failure(s)");
	}
	}

	/* require the stop action regardless if fencing is occuring or not. */
	if (rsc->role > RSC_ROLE_STOPPED) {
	stop_action(rsc, node, FALSE);
	}

	/* if reconnect delay is in use, prevent the connection from exiting the
	* "STOPPED" role until the failure is cleared by the delay timeout. */
	if (rsc->remote_reconnect_interval) {
	rsc->next_role = RSC_ROLE_STOPPED;
	}
	break;
	}

	/* ensure a remote-node connection failure forces an unclean remote-node
	* to be fenced. By setting unseen = FALSE, the remote-node failure will
	* result in a fencing operation regardless if we're going to attempt to
	* reconnect to the remote-node in this transition or not. */
	if (is_set(rsc->flags, pe_rsc_failed) && rsc->is_remote_node) {
	tmpnode = pe_find_node(data_set->nodes, rsc->id);
	if (tmpnode && tmpnode->details->unclean) {
	tmpnode->details->unseen = FALSE;
	}
	}

	if (rsc->role != RSC_ROLE_STOPPED && rsc->role != RSC_ROLE_UNKNOWN) {
	if (is_set(rsc->flags, pe_rsc_orphan)) {
	if (is_set(rsc->flags, pe_rsc_managed)) {
	crm_config_warn("Detected active orphan %s running on %s",
	rsc->id, node->details->uname);
	} else {
	crm_config_warn("Cluster configured not to stop active orphans."
	" %s must be stopped manually on %s",
	rsc->id, node->details->uname);
	}
	}

	native_add_running(rsc, node, data_set);
	if (on_fail != action_fail_ignore) {
	set_bit(rsc->flags, pe_rsc_failed);
	}

	} else if (rsc->clone_name && strchr(rsc->clone_name, ':') != NULL) {
	/* Only do this for older status sections that included instance numbers
	* Otherwise stopped instances will appear as orphans
	*/
	pe_rsc_trace(rsc, "Resetting clone_name %s for %s (stopped)", rsc->clone_name, rsc->id);
	free(rsc->clone_name);
	rsc->clone_name = NULL;

	} else {
	char *key = stop_key(rsc);
	GListPtr possible_matches = find_actions(rsc->actions, key, node);
	GListPtr gIter = possible_matches;

	for (; gIter != NULL; gIter = gIter->next) {
	action_t stop = (action_t ) gIter->data;

	stop->flags \|= pe_action_optional;
	}

	g_list_free(possible_matches);
	free(key);
	}
	}

	/* create active recurring operations as optional */
	static void
	process_recurring(node_t * node, resource_t * rsc,
	int start_index, int stop_index,
	GListPtr sorted_op_list, pe_working_set_t * data_set)
	{
	int counter = -1;
	const char *task = NULL;
	const char *status = NULL;
	GListPtr gIter = sorted_op_list;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "%s: Start index %d, stop index = %d", rsc->id, start_index, stop_index);

	for (; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	int interval = 0;
	char *key = NULL;
	const char *id = ID(rsc_op);
	const char *interval_s = NULL;

	counter++;

	if (node->details->online == FALSE) {
	pe_rsc_trace(rsc, "Skipping %s/%s: node is offline", rsc->id, node->details->uname);
	break;

	/* Need to check if there's a monitor for role="Stopped" */
	} else if (start_index < stop_index && counter <= stop_index) {
	pe_rsc_trace(rsc, "Skipping %s/%s: resource is not active", id, node->details->uname);
	continue;

	} else if (counter < start_index) {
	pe_rsc_trace(rsc, "Skipping %s/%s: old %d", id, node->details->uname, counter);
	continue;
	}

	interval_s = crm_element_value(rsc_op, XML_LRM_ATTR_INTERVAL);
	interval = crm_parse_int(interval_s, "0");
	if (interval == 0) {
	pe_rsc_trace(rsc, "Skipping %s/%s: non-recurring", id, node->details->uname);
	continue;
	}

	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);
	if (safe_str_eq(status, "-1")) {
	pe_rsc_trace(rsc, "Skipping %s/%s: status", id, node->details->uname);
	continue;
	}
	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	/* create the action */
	key = generate_op_key(rsc->id, task, interval);
	pe_rsc_trace(rsc, "Creating %s/%s", key, node->details->uname);
	custom_action(rsc, key, task, node, TRUE, TRUE, data_set);
	}
	}

	void
	calculate_active_ops(GListPtr sorted_op_list, int start_index, int stop_index)
	{
	int counter = -1;
	int implied_monitor_start = -1;
	int implied_master_start = -1;
	const char *task = NULL;
	const char *status = NULL;
	GListPtr gIter = sorted_op_list;

	*stop_index = -1;
	*start_index = -1;

	for (; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	counter++;

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	status = crm_element_value(rsc_op, XML_LRM_ATTR_OPSTATUS);

	if (safe_str_eq(task, CRMD_ACTION_STOP)
	&& safe_str_eq(status, "0")) {
	*stop_index = counter;

	} else if (safe_str_eq(task, CRMD_ACTION_START) \|\| safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
	*start_index = counter;

	} else if ((implied_monitor_start <= *stop_index) && safe_str_eq(task, CRMD_ACTION_STATUS)) {
	const char *rc = crm_element_value(rsc_op, XML_LRM_ATTR_RC);

	if (safe_str_eq(rc, "0") \|\| safe_str_eq(rc, "8")) {
	implied_monitor_start = counter;
	}
	} else if (safe_str_eq(task, CRMD_ACTION_PROMOTE) \|\| safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
	implied_master_start = counter;
	}
	}

	if (*start_index == -1) {
	if (implied_master_start != -1) {
	*start_index = implied_master_start;
	} else if (implied_monitor_start != -1) {
	*start_index = implied_monitor_start;
	}
	}
	}

	static resource_t *
	unpack_lrm_rsc_state(node_t * node, xmlNode * rsc_entry, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	int stop_index = -1;
	int start_index = -1;
	enum rsc_role_e req_role = RSC_ROLE_UNKNOWN;

	const char *task = NULL;
	const char *rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);

	resource_t *rsc = NULL;
	GListPtr op_list = NULL;
	GListPtr sorted_op_list = NULL;

	xmlNode *migrate_op = NULL;
	xmlNode *rsc_op = NULL;

	enum action_fail_response on_fail = FALSE;
	enum rsc_role_e saved_role = RSC_ROLE_UNKNOWN;

	crm_trace("[%s] Processing %s on %s",
	crm_element_name(rsc_entry), rsc_id, node->details->uname);

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
	if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	if (op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return NULL;
	}

	/* find the resource */
	rsc = unpack_find_resource(data_set, node, rsc_id, rsc_entry);
	if (rsc == NULL) {
	rsc = process_orphan_resource(rsc_entry, node, data_set);
	}
	CRM_ASSERT(rsc != NULL);

	/* process operations */
	saved_role = rsc->role;
	on_fail = action_fail_ignore;
	rsc->role = RSC_ROLE_UNKNOWN;
	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
	if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
	migrate_op = rsc_op;
	}

	unpack_rsc_op(rsc, node, rsc_op, &on_fail, data_set);
	}

	/* create active recurring operations as optional */
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);
	process_recurring(node, rsc, start_index, stop_index, sorted_op_list, data_set);

	/* no need to free the contents */
	g_list_free(sorted_op_list);

	process_rsc_state(rsc, node, on_fail, migrate_op, data_set);

	if (get_target_role(rsc, &req_role)) {
	if (rsc->next_role == RSC_ROLE_UNKNOWN \|\| req_role < rsc->next_role) {
	pe_rsc_debug(rsc, "%s: Overwriting calculated next role %s"
	" with requested next role %s",
	rsc->id, role2text(rsc->next_role), role2text(req_role));
	rsc->next_role = req_role;

	} else if (req_role > rsc->next_role) {
	pe_rsc_info(rsc, "%s: Not overwriting calculated next role %s"
	" with requested next role %s",
	rsc->id, role2text(rsc->next_role), role2text(req_role));
	}
	}

	if (saved_role > rsc->role) {
	rsc->role = saved_role;
	}

	return rsc;
	}

	static void
	handle_orphaned_container_fillers(xmlNode * lrm_rsc_list, pe_working_set_t * data_set)
	{
	xmlNode *rsc_entry = NULL;
	for (rsc_entry = __xml_first_child(lrm_rsc_list); rsc_entry != NULL;
	rsc_entry = __xml_next_element(rsc_entry)) {

	resource_t *rsc;
	resource_t *container;
	const char *rsc_id;
	const char *container_id;

	if (safe_str_neq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE)) {
	continue;
	}

	container_id = crm_element_value(rsc_entry, XML_RSC_ATTR_CONTAINER);
	rsc_id = crm_element_value(rsc_entry, XML_ATTR_ID);
	if (container_id == NULL \|\| rsc_id == NULL) {
	continue;
	}

	container = pe_find_resource(data_set->resources, container_id);
	if (container == NULL) {
	continue;
	}

	rsc = pe_find_resource(data_set->resources, rsc_id);
	if (rsc == NULL \|\|
	is_set(rsc->flags, pe_rsc_orphan_container_filler) == FALSE \|\|
	rsc->container != NULL) {
	continue;
	}

	pe_rsc_trace(rsc, "Mapped orphaned rsc %s's container to %s", rsc->id, container_id);
	rsc->container = container;
	container->fillers = g_list_append(container->fillers, rsc);
	}
	}

	gboolean
	unpack_lrm_resources(node_t * node, xmlNode * lrm_rsc_list, pe_working_set_t * data_set)
	{
	xmlNode *rsc_entry = NULL;
	gboolean found_orphaned_container_filler = FALSE;
	GListPtr unexpected_containers = NULL;
	GListPtr gIter = NULL;
	resource_t *remote = NULL;

	CRM_CHECK(node != NULL, return FALSE);

	crm_trace("Unpacking resources on %s", node->details->uname);

	for (rsc_entry = __xml_first_child(lrm_rsc_list); rsc_entry != NULL;
	rsc_entry = __xml_next_element(rsc_entry)) {

	if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
	resource_t *rsc;
	rsc = unpack_lrm_rsc_state(node, rsc_entry, data_set);
	if (!rsc) {
	continue;
	}
	if (is_set(rsc->flags, pe_rsc_orphan_container_filler)) {
	found_orphaned_container_filler = TRUE;
	}
	if (is_set(rsc->flags, pe_rsc_unexpectedly_running)) {
	remote = rsc_contains_remote_node(data_set, rsc);
	if (remote) {
	unexpected_containers = g_list_append(unexpected_containers, remote);
	}
	}
	}
	}

	/* If a container resource is unexpectedly up... and the remote-node
	* connection resource for that container is not up, the entire container
	* must be recovered. */
	for (gIter = unexpected_containers; gIter != NULL; gIter = gIter->next) {
	remote = (resource_t *) gIter->data;
	if (remote->role != RSC_ROLE_STARTED) {
	crm_warn("Recovering container resource %s. Resource is unexpectedly running and involves a remote-node.", remote->container->id);
	set_bit(remote->container->flags, pe_rsc_failed);
	}
	}

	/* now that all the resource state has been unpacked for this node
	* we have to go back and map any orphaned container fillers to their
	* container resource */
	if (found_orphaned_container_filler) {
	handle_orphaned_container_fillers(lrm_rsc_list, data_set);
	}
	g_list_free(unexpected_containers);
	return TRUE;
	}

	static void
	set_active(resource_t * rsc)
	{
	resource_t *top = uber_parent(rsc);

	if (top && top->variant == pe_master) {
	rsc->role = RSC_ROLE_SLAVE;
	} else {
	rsc->role = RSC_ROLE_STARTED;
	}
	}

	static void
	set_node_score(gpointer key, gpointer value, gpointer user_data)
	{
	node_t *node = value;
	int *score = user_data;

	node->weight = *score;
	}

	#define STATUS_PATH_MAX 1024
	static xmlNode *
	find_lrm_op(const char resource, const char op, const char node, const char source,
	pe_working_set_t * data_set)
	{
	int offset = 0;
	char xpath[STATUS_PATH_MAX];

	offset += snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//node_state[@uname='%s']", node);
	offset +=
	snprintf(xpath + offset, STATUS_PATH_MAX - offset, "//" XML_LRM_TAG_RESOURCE "[@id='%s']",
	resource);

	/* Need to check against transition_magic too? */
	if (source && safe_str_eq(op, CRMD_ACTION_MIGRATE)) {
	offset +=
	snprintf(xpath + offset, STATUS_PATH_MAX - offset,
	"/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_target='%s']", op,
	source);
	} else if (source && safe_str_eq(op, CRMD_ACTION_MIGRATED)) {
	offset +=
	snprintf(xpath + offset, STATUS_PATH_MAX - offset,
	"/" XML_LRM_TAG_RSC_OP "[@operation='%s' and @migrate_source='%s']", op,
	source);
	} else {
	offset +=
	snprintf(xpath + offset, STATUS_PATH_MAX - offset,
	"/" XML_LRM_TAG_RSC_OP "[@operation='%s']", op);
	}

	CRM_LOG_ASSERT(offset > 0);
	return get_xpath_object(xpath, data_set->input, LOG_DEBUG);
	}

	static void
	unpack_rsc_migration(resource_t rsc, node_t node, xmlNode xml_op, pe_working_set_t data_set)
	{

	/*
	* The normal sequence is (now): migrate_to(Src) -> migrate_from(Tgt) -> stop(Src)
	*
	* So if a migrate_to is followed by a stop, then we dont need to care what
	* happended on the target node
	*
	* Without the stop, we need to look for a successful migrate_from.
	* This would also imply we're no longer running on the source
	*
	* Without the stop, and without a migrate_from op we make sure the resource
	* gets stopped on both source and target (assuming the target is up)
	*
	*/
	int stop_id = 0;
	int task_id = 0;
	xmlNode *stop_op =
	find_lrm_op(rsc->id, CRMD_ACTION_STOP, node->details->id, NULL, data_set);

	if (stop_op) {
	crm_element_value_int(stop_op, XML_LRM_ATTR_CALLID, &stop_id);
	}

	crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &task_id);

	if (stop_op == NULL \|\| stop_id < task_id) {
	int from_rc = 0, from_status = 0;
	const char *migrate_source =
	crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
	const char *migrate_target =
	crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);

	node_t *target = pe_find_node(data_set->nodes, migrate_target);
	node_t *source = pe_find_node(data_set->nodes, migrate_source);
	xmlNode *migrate_from =
	find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, migrate_target, migrate_source,
	data_set);

	rsc->role = RSC_ROLE_STARTED; /* can be master? */
	if (migrate_from) {
	crm_element_value_int(migrate_from, XML_LRM_ATTR_RC, &from_rc);
	crm_element_value_int(migrate_from, XML_LRM_ATTR_OPSTATUS, &from_status);
	pe_rsc_trace(rsc, "%s op on %s exited with status=%d, rc=%d",
	ID(migrate_from), migrate_target, from_status, from_rc);
	}

	if (migrate_from && from_rc == PCMK_OCF_OK
	&& from_status == PCMK_LRM_OP_DONE) {
	pe_rsc_trace(rsc, "Detected dangling migration op: %s on %s", ID(xml_op),
	migrate_source);

	/* all good
	* just need to arrange for the stop action to get sent
	* but _without_ affecting the target somehow
	*/
	rsc->role = RSC_ROLE_STOPPED;
	rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);

	} else if (migrate_from) { /* Failed */
	if (target && target->details->online) {
	pe_rsc_trace(rsc, "Marking active on %s %p %d", migrate_target, target,
	target->details->online);
	native_add_running(rsc, target, data_set);
	}

	} else { /* Pending or complete but erased */
	if (target && target->details->online) {
	pe_rsc_trace(rsc, "Marking active on %s %p %d", migrate_target, target,
	target->details->online);

	native_add_running(rsc, target, data_set);
	if (source && source->details->online) {
	/* If we make it here we have a partial migration. The migrate_to
	* has completed but the migrate_from on the target has not. Hold on
	* to the target and source on the resource. Later on if we detect that
	* the resource is still going to run on that target, we may continue
	* the migration */
	rsc->partial_migration_target = target;
	rsc->partial_migration_source = source;
	}
	} else {
	/* Consider it failed here - forces a restart, prevents migration */
	set_bit(rsc->flags, pe_rsc_failed);
	clear_bit(rsc->flags, pe_rsc_allow_migrate);
	}
	}
	}
	}

	static void
	unpack_rsc_migration_failure(resource_t rsc, node_t node, xmlNode xml_op, pe_working_set_t data_set)
	{
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);

	CRM_ASSERT(rsc);
	if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
	int stop_id = 0;
	int migrate_id = 0;
	const char *migrate_source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
	const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);

	xmlNode *stop_op =
	find_lrm_op(rsc->id, CRMD_ACTION_STOP, migrate_source, NULL, data_set);
	xmlNode *migrate_op =
	find_lrm_op(rsc->id, CRMD_ACTION_MIGRATE, migrate_source, migrate_target,
	data_set);

	if (stop_op) {
	crm_element_value_int(stop_op, XML_LRM_ATTR_CALLID, &stop_id);
	}
	if (migrate_op) {
	crm_element_value_int(migrate_op, XML_LRM_ATTR_CALLID, &migrate_id);
	}

	/* Get our state right */
	rsc->role = RSC_ROLE_STARTED; /* can be master? */

	if (stop_op == NULL \|\| stop_id < migrate_id) {
	node_t *source = pe_find_node(data_set->nodes, migrate_source);

	if (source && source->details->online) {
	native_add_running(rsc, source, data_set);
	}
	}

	} else if (safe_str_eq(task, CRMD_ACTION_MIGRATE)) {
	int stop_id = 0;
	int migrate_id = 0;
	const char *migrate_source = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_SOURCE);
	const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);

	xmlNode *stop_op =
	find_lrm_op(rsc->id, CRMD_ACTION_STOP, migrate_target, NULL, data_set);
	xmlNode *migrate_op =
	find_lrm_op(rsc->id, CRMD_ACTION_MIGRATED, migrate_target, migrate_source,
	data_set);

	if (stop_op) {
	crm_element_value_int(stop_op, XML_LRM_ATTR_CALLID, &stop_id);
	}
	if (migrate_op) {
	crm_element_value_int(migrate_op, XML_LRM_ATTR_CALLID, &migrate_id);
	}

	/* Get our state right */
	rsc->role = RSC_ROLE_STARTED; /* can be master? */

	if (stop_op == NULL \|\| stop_id < migrate_id) {
	node_t *target = pe_find_node(data_set->nodes, migrate_target);

	pe_rsc_trace(rsc, "Stop: %p %d, Migrated: %p %d", stop_op, stop_id, migrate_op,
	migrate_id);
	if (target && target->details->online) {
	native_add_running(rsc, target, data_set);
	}

	} else if (migrate_op == NULL) {
	/* Make sure it gets cleaned up, the stop may pre-date the migrate_from */
	rsc->dangling_migrations = g_list_prepend(rsc->dangling_migrations, node);
	}
	}
	}

	static void
	record_failed_op(xmlNode op, node_t node, pe_working_set_t * data_set)
	{
	xmlNode *xIter = NULL;
	const char *op_key = crm_element_value(op, XML_LRM_ATTR_TASK_KEY);

	if ((node->details->shutdown) && (node->details->online == FALSE)) {
	return;
	}

	for (xIter = data_set->failed->children; xIter; xIter = xIter->next) {
	const char *key = crm_element_value(xIter, XML_LRM_ATTR_TASK_KEY);
	const char *uname = crm_element_value(xIter, XML_ATTR_UNAME);

	if(safe_str_eq(op_key, key) && safe_str_eq(uname, node->details->uname)) {
	crm_trace("Skipping duplicate entry %s on %s", op_key, node->details->uname);
	return;
	}
	}

	crm_trace("Adding entry %s on %s", op_key, node->details->uname);
	crm_xml_add(op, XML_ATTR_UNAME, node->details->uname);
	add_node_copy(data_set->failed, op);
	}

	static const char get_op_key(xmlNode xml_op)
	{
	const char *key = crm_element_value(xml_op, XML_LRM_ATTR_TASK_KEY);
	if(key == NULL) {
	key = ID(xml_op);
	}
	return key;
	}

	static void
	unpack_rsc_op_failure(resource_t rsc, node_t node, int rc, xmlNode xml_op, enum action_fail_response on_fail, pe_working_set_t * data_set)
	{
	int interval = 0;
	bool is_probe = FALSE;
	action_t *action = NULL;

	const char *key = get_op_key(xml_op);
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);

	CRM_ASSERT(rsc);
	crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);
	if(interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
	is_probe = TRUE;
	pe_rsc_trace(rsc, "is a probe: %s", key);
	}

	if (rc != PCMK_OCF_NOT_INSTALLED \|\| is_set(data_set->flags, pe_flag_symmetric_cluster)) {
	crm_warn("Processing failed op %s for %s on %s: %s (%d)",
	task, rsc->id, node->details->uname, services_ocf_exitcode_str(rc),
	rc);

	record_failed_op(xml_op, node, data_set);

	} else {
	crm_trace("Processing failed op %s for %s on %s: %s (%d)",
	task, rsc->id, node->details->uname, services_ocf_exitcode_str(rc),
	rc);
	}

	action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);
	if ((action->on_fail <= action_fail_fence && *on_fail < action->on_fail) \|\|
	(action->on_fail == action_fail_reset_remote && *on_fail <= action_fail_recover) \|\|
	(action->on_fail == action_fail_restart_container && *on_fail <= action_fail_recover) \|\|
	(*on_fail == action_fail_restart_container && action->on_fail >= action_fail_migrate)) {
	pe_rsc_trace(rsc, "on-fail %s -> %s for %s (%s)", fail2text(*on_fail),
	fail2text(action->on_fail), action->uuid, key);
	*on_fail = action->on_fail;
	}

	if (safe_str_eq(task, CRMD_ACTION_STOP)) {
	resource_location(rsc, node, -INFINITY, "__stop_fail__", data_set);

	} else if (safe_str_eq(task, CRMD_ACTION_MIGRATE) \|\| safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
	unpack_rsc_migration_failure(rsc, node, xml_op, data_set);

	} else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
	rsc->role = RSC_ROLE_MASTER;

	} else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
	/*
	* staying in role=master ends up putting the PE/TE into a loop
	* setting role=slave is not dangerous because no master will be
	* promoted until the failed resource has been fully stopped
	*/
	rsc->next_role = RSC_ROLE_STOPPED;
	if (action->on_fail == action_fail_block) {
	rsc->role = RSC_ROLE_MASTER;

	} else {
	crm_warn("Forcing %s to stop after a failed demote action", rsc->id);
	rsc->role = RSC_ROLE_SLAVE;
	}

	} else if (compare_version("2.0", op_version) > 0 && safe_str_eq(task, CRMD_ACTION_START)) {
	crm_warn("Compatibility handling for failed op %s on %s", key, node->details->uname);
	resource_location(rsc, node, -INFINITY, "__legacy_start__", data_set);
	}

	if(is_probe && rc == PCMK_OCF_NOT_INSTALLED) {
	/* leave stopped */
	pe_rsc_trace(rsc, "Leaving %s stopped", rsc->id);
	rsc->role = RSC_ROLE_STOPPED;

	} else if (rsc->role < RSC_ROLE_STARTED) {
	pe_rsc_trace(rsc, "Setting %s active", rsc->id);
	set_active(rsc);
	}

	pe_rsc_trace(rsc, "Resource %s: role=%s, unclean=%s, on_fail=%s, fail_role=%s",
	rsc->id, role2text(rsc->role),
	node->details->unclean ? "true" : "false",
	fail2text(action->on_fail), role2text(action->fail_role));

	if (action->fail_role != RSC_ROLE_STARTED && rsc->next_role < action->fail_role) {
	rsc->next_role = action->fail_role;
	}

	if (action->fail_role == RSC_ROLE_STOPPED) {
	int score = -INFINITY;

	resource_t *fail_rsc = rsc;

	if (fail_rsc->parent) {
	resource_t *parent = uber_parent(fail_rsc);

	if ((parent->variant == pe_clone \|\| parent->variant == pe_master)
	&& is_not_set(parent->flags, pe_rsc_unique)) {
	/* for clone and master resources, if a child fails on an operation
	* with on-fail = stop, all the resources fail. Do this by preventing
	* the parent from coming up again. */
	fail_rsc = parent;
	}
	}
	crm_warn("Making sure %s doesn't come up again", fail_rsc->id);
	/* make sure it doesnt come up again */
	g_hash_table_destroy(fail_rsc->allowed_nodes);
	fail_rsc->allowed_nodes = node_hash_from_list(data_set->nodes);
	g_hash_table_foreach(fail_rsc->allowed_nodes, set_node_score, &score);
	}

	pe_free_action(action);
	}

	static int
	determine_op_status(
	resource_t rsc, int rc, int target_rc, node_t node, xmlNode * xml_op, enum action_fail_response * on_fail, pe_working_set_t * data_set)
	{
	int interval = 0;
	int result = PCMK_LRM_OP_DONE;

	const char *key = get_op_key(xml_op);
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);

	bool is_probe = FALSE;

	CRM_ASSERT(rsc);
	crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);
	if (interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
	is_probe = TRUE;
	}

	if (target_rc >= 0 && target_rc != rc) {
	result = PCMK_LRM_OP_ERROR;
	pe_rsc_debug(rsc, "%s on %s returned '%s' (%d) instead of the expected value: '%s' (%d)",
	key, node->details->uname,
	services_ocf_exitcode_str(rc), rc,
	services_ocf_exitcode_str(target_rc), target_rc);
	}

	/* we could clean this up significantly except for old LRMs and CRMs that
	- * didnt include target_rc and liked to remap status
	+ * didn't include target_rc and liked to remap status
	*/
	switch (rc) {
	case PCMK_OCF_OK:
	if (is_probe && target_rc == 7) {
	result = PCMK_LRM_OP_DONE;
	set_bit(rsc->flags, pe_rsc_unexpectedly_running);
	pe_rsc_info(rsc, "Operation %s found resource %s active on %s",
	task, rsc->id, node->details->uname);

	/* legacy code for pre-0.6.5 operations */
	} else if (target_rc < 0 && interval > 0 && rsc->role == RSC_ROLE_MASTER) {
	/* catch status ops that return 0 instead of 8 while they
	* are supposed to be in master mode
	*/
	result = PCMK_LRM_OP_ERROR;
	}
	break;

	case PCMK_OCF_NOT_RUNNING:
	if (is_probe \|\| target_rc == rc \|\| is_not_set(rsc->flags, pe_rsc_managed)) {
	result = PCMK_LRM_OP_DONE;
	rsc->role = RSC_ROLE_STOPPED;

	/* clear any previous failure actions */
	*on_fail = action_fail_ignore;
	rsc->next_role = RSC_ROLE_UNKNOWN;

	} else if (safe_str_neq(task, CRMD_ACTION_STOP)) {
	result = PCMK_LRM_OP_ERROR;
	}
	break;

	case PCMK_OCF_RUNNING_MASTER:
	if (is_probe) {
	result = PCMK_LRM_OP_DONE;
	pe_rsc_info(rsc, "Operation %s found resource %s active in master mode on %s",
	task, rsc->id, node->details->uname);

	} else if (target_rc == rc) {
	/* nothing to do */

	} else if (target_rc >= 0) {
	result = PCMK_LRM_OP_ERROR;

	/* legacy code for pre-0.6.5 operations */
	} else if (safe_str_neq(task, CRMD_ACTION_STATUS)
	\|\| rsc->role != RSC_ROLE_MASTER) {
	result = PCMK_LRM_OP_ERROR;
	if (rsc->role != RSC_ROLE_MASTER) {
	crm_err("%s reported %s in master mode on %s",
	key, rsc->id, node->details->uname);
	}
	}
	rsc->role = RSC_ROLE_MASTER;
	break;

	case PCMK_OCF_DEGRADED_MASTER:
	case PCMK_OCF_FAILED_MASTER:
	rsc->role = RSC_ROLE_MASTER;
	result = PCMK_LRM_OP_ERROR;
	break;

	case PCMK_OCF_NOT_CONFIGURED:
	result = PCMK_LRM_OP_ERROR_FATAL;
	break;

	case PCMK_OCF_NOT_INSTALLED:
	case PCMK_OCF_INVALID_PARAM:
	case PCMK_OCF_INSUFFICIENT_PRIV:
	case PCMK_OCF_UNIMPLEMENT_FEATURE:
	if (rc == PCMK_OCF_UNIMPLEMENT_FEATURE && interval > 0) {
	result = PCMK_LRM_OP_NOTSUPPORTED;
	break;

	} else if(pe_can_fence(data_set, node) == FALSE
	&& safe_str_eq(task, CRMD_ACTION_STOP)) {
	/* If a stop fails and we can't fence, there's nothing else we can do */
	pe_proc_err("No further recovery can be attempted for %s: %s action failed with '%s' (%d)",
	rsc->id, task, services_ocf_exitcode_str(rc), rc);
	clear_bit(rsc->flags, pe_rsc_managed);
	set_bit(rsc->flags, pe_rsc_block);
	}
	result = PCMK_LRM_OP_ERROR_HARD;
	break;

	default:
	if (result == PCMK_LRM_OP_DONE) {
	crm_info("Treating %s (rc=%d) on %s as an ERROR",
	key, rc, node->details->uname);
	result = PCMK_LRM_OP_ERROR;
	}
	}

	return result;
	}

	static bool check_operation_expiry(resource_t rsc, node_t node, int rc, xmlNode xml_op, pe_working_set_t data_set)
	{
	bool expired = FALSE;
	time_t last_failure = 0;
	int clear_failcount = 0;
	int interval = 0;
	int failure_timeout = rsc->failure_timeout;
	const char *key = get_op_key(xml_op);
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);

	/* clearing recurring monitor operation failures automatically
	* needs to be carefully considered */
	if (safe_str_eq(crm_element_value(xml_op, XML_LRM_ATTR_TASK), "monitor") &&
	safe_str_neq(crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL), "0")) {

	/* TODO, in the future we should consider not clearing recurring monitor
	* op failures unless the last action for a resource was a "stop" action.
	* otherwise it is possible that clearing the monitor failure will result
	* in the resource being in an undeterministic state.
	*
	* For now we handle this potential undeterministic condition for remote
	* node connection resources by not clearing a recurring monitor op failure
	* until after the node has been fenced. */

	if (is_set(data_set->flags, pe_flag_stonith_enabled) &&
	(rsc->remote_reconnect_interval)) {

	node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);
	if (remote_node && remote_node->details->remote_was_fenced == 0) {
	if (strstr(ID(xml_op), "last_failure")) {
	crm_info("Waiting to clear monitor failure for remote node %s until fencing has occured", rsc->id);
	}
	/* disabling failure timeout for this operation because we believe
	* fencing of the remote node should occur first. */
	failure_timeout = 0;
	}
	}
	}

	if (failure_timeout > 0) {
	int last_run = 0;

	if (crm_element_value_int(xml_op, XML_RSC_OP_LAST_CHANGE, &last_run) == 0) {
	time_t now = get_effective_time(data_set);

	if (now > (last_run + failure_timeout)) {
	expired = TRUE;
	}
	}
	}

	if (expired) {
	if (failure_timeout > 0) {
	int fc = get_failcount_full(node, rsc, &last_failure, FALSE, xml_op, data_set);
	if(fc) {
	if (get_failcount_full(node, rsc, &last_failure, TRUE, xml_op, data_set) == 0) {
	clear_failcount = 1;
	crm_notice("Clearing expired failcount for %s on %s", rsc->id, node->details->uname);

	} else {
	expired = FALSE;
	}
	} else if (rsc->remote_reconnect_interval && strstr(ID(xml_op), "last_failure")) {
	/* always clear last failure when reconnect interval is set */
	clear_failcount = 1;
	}
	}

	} else if (strstr(ID(xml_op), "last_failure") &&
	((strcmp(task, "start") == 0) \|\| (strcmp(task, "monitor") == 0))) {

	op_digest_cache_t *digest_data = NULL;

	digest_data = rsc_action_digest_cmp(rsc, xml_op, node, data_set);

	if (digest_data->rc == RSC_DIGEST_UNKNOWN) {
	crm_trace("rsc op %s on node %s does not have a op digest to compare against", rsc->id,
	key, node->details->id);
	} else if (digest_data->rc != RSC_DIGEST_MATCH) {
	clear_failcount = 1;
	crm_info
	("Clearing failcount for %s on %s, %s failed and now resource parameters have changed.",
	task, rsc->id, node->details->uname);
	}
	}

	if (clear_failcount) {
	action_t *clear_op = NULL;

	clear_op = custom_action(rsc, crm_concat(rsc->id, CRM_OP_CLEAR_FAILCOUNT, '_'),
	CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE, data_set);
	add_hash_param(clear_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
	}

	crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);
	if(expired && interval == 0 && safe_str_eq(task, CRMD_ACTION_STATUS)) {
	switch(rc) {
	case PCMK_OCF_OK:
	case PCMK_OCF_NOT_RUNNING:
	case PCMK_OCF_RUNNING_MASTER:
	case PCMK_OCF_DEGRADED:
	case PCMK_OCF_DEGRADED_MASTER:
	/* Don't expire probes that return these values */
	expired = FALSE;
	break;
	}
	}

	return expired;
	}

	int get_target_rc(xmlNode *xml_op)
	{
	int dummy = 0;
	int target_rc = 0;
	char *dummy_string = NULL;
	const char *key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);
	if (key == NULL) {
	return -1;
	}

	decode_transition_key(key, &dummy_string, &dummy, &dummy, &target_rc);
	free(dummy_string);

	return target_rc;
	}

	static enum action_fail_response
	get_action_on_fail(resource_t rsc, const char key, const char task, pe_working_set_t data_set)
	{
	int result = action_fail_recover;
	action_t *action = custom_action(rsc, strdup(key), task, NULL, TRUE, FALSE, data_set);

	result = action->on_fail;
	pe_free_action(action);

	return result;
	}

	static void
	update_resource_state(resource_t rsc, node_t node, xmlNode * xml_op, const char *task, int rc,
	enum action_fail_response on_fail, pe_working_set_t data_set)
	{
	gboolean clear_past_failure = FALSE;

	CRM_ASSERT(rsc);
	if (rc == PCMK_OCF_NOT_RUNNING) {
	clear_past_failure = TRUE;

	} else if (rc == PCMK_OCF_NOT_INSTALLED) {
	rsc->role = RSC_ROLE_STOPPED;

	} else if (safe_str_eq(task, CRMD_ACTION_STATUS)) {
	clear_past_failure = TRUE;
	if (rsc->role < RSC_ROLE_STARTED) {
	set_active(rsc);
	}

	} else if (safe_str_eq(task, CRMD_ACTION_START)) {
	rsc->role = RSC_ROLE_STARTED;
	clear_past_failure = TRUE;

	} else if (safe_str_eq(task, CRMD_ACTION_STOP)) {
	rsc->role = RSC_ROLE_STOPPED;
	clear_past_failure = TRUE;

	} else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
	rsc->role = RSC_ROLE_MASTER;
	clear_past_failure = TRUE;

	} else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) {
	/* Demote from Master does not clear an error */
	rsc->role = RSC_ROLE_SLAVE;

	} else if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
	rsc->role = RSC_ROLE_STARTED;
	clear_past_failure = TRUE;

	} else if (safe_str_eq(task, CRMD_ACTION_MIGRATE)) {
	unpack_rsc_migration(rsc, node, xml_op, data_set);

	} else if (rsc->role < RSC_ROLE_STARTED) {
	/* migrate_to and migrate_from will land here */
	pe_rsc_trace(rsc, "%s active on %s", rsc->id, node->details->uname);
	set_active(rsc);
	}

	/* clear any previous failure actions */
	if (clear_past_failure) {
	switch (*on_fail) {
	case action_fail_stop:
	case action_fail_fence:
	case action_fail_migrate:
	case action_fail_standby:
	pe_rsc_trace(rsc, "%s.%s is not cleared by a completed stop",
	rsc->id, fail2text(*on_fail));
	break;

	case action_fail_block:
	case action_fail_ignore:
	case action_fail_recover:
	case action_fail_restart_container:
	*on_fail = action_fail_ignore;
	rsc->next_role = RSC_ROLE_UNKNOWN;
	break;
	case action_fail_reset_remote:
	if (rsc->remote_reconnect_interval == 0) {
	/* when reconnect delay is not in use, the connection is allowed
	* to start again after the remote node is fenced and completely
	* stopped. Otherwise, with reconnect delay we wait for the failure
	* to be cleared entirely before reconnected can be attempted. */
	*on_fail = action_fail_ignore;
	rsc->next_role = RSC_ROLE_UNKNOWN;
	}
	break;
	}
	}
	}

	gboolean
	unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op,
	enum action_fail_response * on_fail, pe_working_set_t * data_set)
	{
	int task_id = 0;

	const char *key = NULL;
	const char *task = NULL;
	const char *task_key = NULL;

	int rc = 0;
	int status = PCMK_LRM_OP_PENDING-1;
	int target_rc = get_target_rc(xml_op);
	int interval = 0;

	gboolean expired = FALSE;
	resource_t *parent = rsc;
	enum action_fail_response failure_strategy = action_fail_recover;

	CRM_CHECK(rsc != NULL, return FALSE);
	CRM_CHECK(node != NULL, return FALSE);
	CRM_CHECK(xml_op != NULL, return FALSE);

	task_key = get_op_key(xml_op);

	task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	key = crm_element_value(xml_op, XML_ATTR_TRANSITION_KEY);

	crm_element_value_int(xml_op, XML_LRM_ATTR_RC, &rc);
	crm_element_value_int(xml_op, XML_LRM_ATTR_CALLID, &task_id);
	crm_element_value_int(xml_op, XML_LRM_ATTR_OPSTATUS, &status);
	crm_element_value_int(xml_op, XML_LRM_ATTR_INTERVAL, &interval);

	CRM_CHECK(task != NULL, return FALSE);
	CRM_CHECK(status <= PCMK_LRM_OP_NOT_INSTALLED, return FALSE);
	CRM_CHECK(status >= PCMK_LRM_OP_PENDING, return FALSE);

	if (safe_str_eq(task, CRMD_ACTION_NOTIFY)) {
	/* safe to ignore these */
	return TRUE;
	}

	if (is_not_set(rsc->flags, pe_rsc_unique)) {
	parent = uber_parent(rsc);
	}

	pe_rsc_trace(rsc, "Unpacking task %s/%s (call_id=%d, status=%d, rc=%d) on %s (role=%s)",
	task_key, task, task_id, status, rc, node->details->uname, role2text(rsc->role));

	if (node->details->unclean) {
	pe_rsc_trace(rsc, "Node %s (where %s is running) is unclean."
	" Further action depends on the value of the stop's on-fail attribue",
	node->details->uname, rsc->id);
	}

	if (status == PCMK_LRM_OP_ERROR) {
	/* Older versions set this if rc != 0 but its up to us to decide */
	status = PCMK_LRM_OP_DONE;
	}

	if(status != PCMK_LRM_OP_NOT_INSTALLED) {
	expired = check_operation_expiry(rsc, node, rc, xml_op, data_set);
	}

	/* Degraded results are informational only, re-map them to their error-free equivalents */
	if (rc == PCMK_OCF_DEGRADED && safe_str_eq(task, CRMD_ACTION_STATUS)) {
	rc = PCMK_OCF_OK;

	/* Add them to the failed list to highlight them for the user */
	if ((node->details->shutdown == FALSE) \|\| (node->details->online == TRUE)) {
	crm_trace("Remapping %d to %d", PCMK_OCF_DEGRADED, PCMK_OCF_OK);
	record_failed_op(xml_op, node, data_set);
	}

	} else if (rc == PCMK_OCF_DEGRADED_MASTER && safe_str_eq(task, CRMD_ACTION_STATUS)) {
	rc = PCMK_OCF_RUNNING_MASTER;

	/* Add them to the failed list to highlight them for the user */
	if ((node->details->shutdown == FALSE) \|\| (node->details->online == TRUE)) {
	crm_trace("Remapping %d to %d", PCMK_OCF_DEGRADED_MASTER, PCMK_OCF_RUNNING_MASTER);
	record_failed_op(xml_op, node, data_set);
	}
	}

	if (expired && target_rc != rc) {
	const char *magic = crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC);

	pe_rsc_debug(rsc, "Expired operation '%s' on %s returned '%s' (%d) instead of the expected value: '%s' (%d)",
	key, node->details->uname,
	services_ocf_exitcode_str(rc), rc,
	services_ocf_exitcode_str(target_rc), target_rc);

	if(interval == 0) {
	crm_notice("Ignoring expired calculated failure %s (rc=%d, magic=%s) on %s",
	task_key, rc, magic, node->details->uname);
	goto done;

	} else if(node->details->online && node->details->unclean == FALSE) {
	crm_notice("Re-initiated expired calculated failure %s (rc=%d, magic=%s) on %s",
	task_key, rc, magic, node->details->uname);
	/* This is SO horrible, but we don't have access to CancelXmlOp() yet */
	crm_xml_add(xml_op, XML_LRM_ATTR_RESTART_DIGEST, "calculated-failure-timeout");
	goto done;
	}
	}

	if(status == PCMK_LRM_OP_DONE \|\| status == PCMK_LRM_OP_ERROR) {
	status = determine_op_status(rsc, rc, target_rc, node, xml_op, on_fail, data_set);
	}

	pe_rsc_trace(rsc, "Handling status: %d", status);
	switch (status) {
	case PCMK_LRM_OP_CANCELLED:
	/* do nothing?? */
	pe_err("Don't know what to do for cancelled ops yet");
	break;

	case PCMK_LRM_OP_PENDING:
	if (safe_str_eq(task, CRMD_ACTION_START)) {
	set_bit(rsc->flags, pe_rsc_start_pending);
	set_active(rsc);

	} else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) {
	rsc->role = RSC_ROLE_MASTER;

	} else if (safe_str_eq(task, CRMD_ACTION_MIGRATE) && node->details->unclean) {
	/* If a pending migrate_to action is out on a unclean node,
	* we have to force the stop action on the target. */
	const char *migrate_target = crm_element_value(xml_op, XML_LRM_ATTR_MIGRATE_TARGET);
	node_t *target = pe_find_node(data_set->nodes, migrate_target);
	if (target) {
	stop_action(rsc, target, FALSE);
	}
	}

	if (rsc->pending_task == NULL) {
	if (safe_str_eq(task, CRMD_ACTION_STATUS) && interval == 0) {
	/* Pending probes are not printed, even if pending
	* operations are requested. If someone ever requests that
	* behavior, uncomment this and the corresponding part of
	* native.c:native_pending_task().
	*/
	/rsc->pending_task = strdup("probe");/

	} else {
	rsc->pending_task = strdup(task);
	}
	}
	break;

	case PCMK_LRM_OP_DONE:
	pe_rsc_trace(rsc, "%s/%s completed on %s", rsc->id, task, node->details->uname);
	update_resource_state(rsc, node, xml_op, task, rc, on_fail, data_set);
	break;

	case PCMK_LRM_OP_NOT_INSTALLED:
	failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
	if (failure_strategy == action_fail_ignore) {
	crm_warn("Cannot ignore failed %s (status=%d, rc=%d) on %s: "
	"Resource agent doesn't exist",
	task_key, status, rc, node->details->uname);
	/* Also for printing it as "FAILED" by marking it as pe_rsc_failed later */
	*on_fail = action_fail_migrate;
	}
	resource_location(parent, node, -INFINITY, "hard-error", data_set);
	unpack_rsc_op_failure(rsc, node, rc, xml_op, on_fail, data_set);
	break;

	case PCMK_LRM_OP_ERROR:
	case PCMK_LRM_OP_ERROR_HARD:
	case PCMK_LRM_OP_ERROR_FATAL:
	case PCMK_LRM_OP_TIMEOUT:
	case PCMK_LRM_OP_NOTSUPPORTED:

	failure_strategy = get_action_on_fail(rsc, task_key, task, data_set);
	if ((failure_strategy == action_fail_ignore)
	\|\| (failure_strategy == action_fail_restart_container
	&& safe_str_eq(task, CRMD_ACTION_STOP))) {

	crm_warn("Pretending the failure of %s (rc=%d) on %s succeeded",
	task_key, rc, node->details->uname);

	update_resource_state(rsc, node, xml_op, task, target_rc, on_fail, data_set);
	crm_xml_add(xml_op, XML_ATTR_UNAME, node->details->uname);
	set_bit(rsc->flags, pe_rsc_failure_ignored);

	record_failed_op(xml_op, node, data_set);

	if (failure_strategy == action_fail_restart_container && *on_fail <= action_fail_recover) {
	*on_fail = failure_strategy;
	}

	} else {
	unpack_rsc_op_failure(rsc, node, rc, xml_op, on_fail, data_set);

	if(status == PCMK_LRM_OP_ERROR_HARD) {
	do_crm_log(rc != PCMK_OCF_NOT_INSTALLED?LOG_ERR:LOG_NOTICE,
	"Preventing %s from re-starting on %s: operation %s failed '%s' (%d)",
	parent->id, node->details->uname,
	task, services_ocf_exitcode_str(rc), rc);

	resource_location(parent, node, -INFINITY, "hard-error", data_set);

	} else if(status == PCMK_LRM_OP_ERROR_FATAL) {
	crm_err("Preventing %s from re-starting anywhere: operation %s failed '%s' (%d)",
	parent->id, task, services_ocf_exitcode_str(rc), rc);

	resource_location(parent, NULL, -INFINITY, "fatal-error", data_set);
	}
	}
	break;
	}

	done:
	pe_rsc_trace(rsc, "Resource %s after %s: role=%s", rsc->id, task, role2text(rsc->role));
	return TRUE;
	}

	gboolean
	add_node_attrs(xmlNode * xml_obj, node_t * node, gboolean overwrite, pe_working_set_t * data_set)
	{
	const char *cluster_name = NULL;

	g_hash_table_insert(node->details->attrs,
	strdup("#uname"), strdup(node->details->uname));

	g_hash_table_insert(node->details->attrs, strdup("#" XML_ATTR_ID), strdup(node->details->id));
	if (safe_str_eq(node->details->id, data_set->dc_uuid)) {
	data_set->dc_node = node;
	node->details->is_dc = TRUE;
	g_hash_table_insert(node->details->attrs,
	strdup("#" XML_ATTR_DC), strdup(XML_BOOLEAN_TRUE));
	} else {
	g_hash_table_insert(node->details->attrs,
	strdup("#" XML_ATTR_DC), strdup(XML_BOOLEAN_FALSE));
	}

	cluster_name = g_hash_table_lookup(data_set->config_hash, "cluster-name");
	if (cluster_name) {
	g_hash_table_insert(node->details->attrs, strdup("#cluster-name"), strdup(cluster_name));
	}

	unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_ATTR_SETS, NULL,
	node->details->attrs, NULL, overwrite, data_set->now);

	if (g_hash_table_lookup(node->details->attrs, "#site-name") == NULL) {
	const char *site_name = g_hash_table_lookup(node->details->attrs, "site-name");

	if (site_name) {
	/* Prefix '#' to the key */
	g_hash_table_insert(node->details->attrs, strdup("#site-name"), strdup(site_name));

	} else if (cluster_name) {
	/* Default to cluster-name if unset */
	g_hash_table_insert(node->details->attrs, strdup("#site-name"), strdup(cluster_name));
	}
	}
	return TRUE;
	}

	static GListPtr
	extract_operations(const char node, const char rsc, xmlNode * rsc_entry, gboolean active_filter)
	{
	int counter = -1;
	int stop_index = -1;
	int start_index = -1;

	xmlNode *rsc_op = NULL;

	GListPtr gIter = NULL;
	GListPtr op_list = NULL;
	GListPtr sorted_op_list = NULL;

	/* extract operations */
	op_list = NULL;
	sorted_op_list = NULL;

	for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
	if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
	crm_xml_add(rsc_op, "resource", rsc);
	crm_xml_add(rsc_op, XML_ATTR_UNAME, node);
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	if (op_list == NULL) {
	/* if there are no operations, there is nothing to do */
	return NULL;
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);

	/* create active recurring operations as optional */
	if (active_filter == FALSE) {
	return sorted_op_list;
	}

	op_list = NULL;

	calculate_active_ops(sorted_op_list, &start_index, &stop_index);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	counter++;

	if (start_index < stop_index) {
	crm_trace("Skipping %s: not active", ID(rsc_entry));
	break;

	} else if (counter < start_index) {
	crm_trace("Skipping %s: old", ID(rsc_op));
	continue;
	}
	op_list = g_list_append(op_list, rsc_op);
	}

	g_list_free(sorted_op_list);
	return op_list;
	}

	GListPtr
	find_operations(const char rsc, const char node, gboolean active_filter,
	pe_working_set_t * data_set)
	{
	GListPtr output = NULL;
	GListPtr intermediate = NULL;

	xmlNode *tmp = NULL;
	xmlNode *status = find_xml_node(data_set->input, XML_CIB_TAG_STATUS, TRUE);

	node_t *this_node = NULL;

	xmlNode *node_state = NULL;

	for (node_state = __xml_first_child(status); node_state != NULL;
	node_state = __xml_next_element(node_state)) {

	if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
	const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);

	if (node != NULL && safe_str_neq(uname, node)) {
	continue;
	}

	this_node = pe_find_node(data_set->nodes, uname);
	if(this_node == NULL) {
	CRM_LOG_ASSERT(this_node != NULL);
	continue;

	} else if (is_remote_node(this_node)) {
	determine_remote_online_status(data_set, this_node);

	} else {
	determine_online_status(node_state, this_node, data_set);
	}

	if (this_node->details->online \|\| is_set(data_set->flags, pe_flag_stonith_enabled)) {
	/* offline nodes run no resources...
	* unless stonith is enabled in which case we need to
	* make sure rsc start events happen after the stonith
	*/
	xmlNode *lrm_rsc = NULL;

	tmp = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	tmp = find_xml_node(tmp, XML_LRM_TAG_RESOURCES, FALSE);

	for (lrm_rsc = __xml_first_child(tmp); lrm_rsc != NULL;
	lrm_rsc = __xml_next_element(lrm_rsc)) {
	if (crm_str_eq((const char *)lrm_rsc->name, XML_LRM_TAG_RESOURCE, TRUE)) {

	const char *rsc_id = crm_element_value(lrm_rsc, XML_ATTR_ID);

	if (rsc != NULL && safe_str_neq(rsc_id, rsc)) {
	continue;
	}

	intermediate = extract_operations(uname, rsc_id, lrm_rsc, active_filter);
	output = g_list_concat(output, intermediate);
	}
	}
	}
	}
	}

	return output;
	}
	diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
	index 49292fc806..a891d831a2 100644
	--- a/lib/pengine/utils.c
	+++ b/lib/pengine/utils.c
	@@ -1,2232 +1,2232 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This library is free software; you can redistribute it and/or
	* modify it under the terms of the GNU Lesser General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This library is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* Lesser General Public License for more details.
	*
	* You should have received a copy of the GNU Lesser General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/
	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/util.h>

	#include <glib.h>

	#include <crm/pengine/rules.h>
	#include <crm/pengine/internal.h>

	pe_working_set_t *pe_dataset = NULL;

	extern xmlNode get_object_root(const char object_type, xmlNode * the_root);
	void print_str_str(gpointer key, gpointer value, gpointer user_data);
	gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data);
	void unpack_operation(action_t * action, xmlNode * xml_obj, resource_t * container,
	pe_working_set_t * data_set);
	static xmlNode find_rsc_op_entry_helper(resource_t rsc, const char *key,
	gboolean include_disabled);
	static gboolean is_rsc_baremetal_remote_node(resource_t rsc, pe_working_set_t data_set);

	bool pe_can_fence(pe_working_set_t * data_set, node_t *node)
	{
	if(is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
	return FALSE; /* Turned off */

	} else if (is_not_set(data_set->flags, pe_flag_have_stonith_resource)) {
	return FALSE; /* No devices */

	} else if (is_set(data_set->flags, pe_flag_have_quorum)) {
	return TRUE;

	} else if (data_set->no_quorum_policy == no_quorum_ignore) {
	return TRUE;

	} else if(node == NULL) {
	return FALSE;

	} else if(node->details->online) {
	crm_notice("We can fence %s without quorum because they're in our membership", node->details->uname);
	return TRUE;
	}

	crm_trace("Cannot fence %s", node->details->uname);
	return FALSE;
	}

	node_t *
	node_copy(node_t * this_node)
	{
	node_t *new_node = NULL;

	CRM_CHECK(this_node != NULL, return NULL);

	new_node = calloc(1, sizeof(node_t));
	CRM_ASSERT(new_node != NULL);

	crm_trace("Copying %p (%s) to %p", this_node, this_node->details->uname, new_node);

	new_node->rsc_discover_mode = this_node->rsc_discover_mode;
	new_node->weight = this_node->weight;
	new_node->fixed = this_node->fixed;
	new_node->details = this_node->details;

	return new_node;
	}

	/* any node in list1 or list2 and not in the other gets a score of -INFINITY */
	void
	node_list_exclude(GHashTable * hash, GListPtr list, gboolean merge_scores)
	{
	GHashTable *result = hash;
	node_t *other_node = NULL;
	GListPtr gIter = list;

	GHashTableIter iter;
	node_t *node = NULL;

	g_hash_table_iter_init(&iter, hash);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {

	other_node = pe_find_node_id(list, node->details->id);
	if (other_node == NULL) {
	node->weight = -INFINITY;
	} else if (merge_scores) {
	node->weight = merge_weights(node->weight, other_node->weight);
	}
	}

	for (; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	other_node = pe_hash_table_lookup(result, node->details->id);

	if (other_node == NULL) {
	node_t *new_node = node_copy(node);

	new_node->weight = -INFINITY;
	g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
	}
	}
	}

	GHashTable *
	node_hash_from_list(GListPtr list)
	{
	GListPtr gIter = list;
	GHashTable *result = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, g_hash_destroy_str);

	for (; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;
	node_t *n = node_copy(node);

	g_hash_table_insert(result, (gpointer) n->details->id, n);
	}

	return result;
	}

	GListPtr
	node_list_dup(GListPtr list1, gboolean reset, gboolean filter)
	{
	GListPtr result = NULL;
	GListPtr gIter = list1;

	for (; gIter != NULL; gIter = gIter->next) {
	node_t *new_node = NULL;
	node_t this_node = (node_t ) gIter->data;

	if (filter && this_node->weight < 0) {
	continue;
	}

	new_node = node_copy(this_node);
	if (reset) {
	new_node->weight = 0;
	}
	if (new_node != NULL) {
	result = g_list_prepend(result, new_node);
	}
	}

	return result;
	}

	gint
	sort_node_uname(gconstpointer a, gconstpointer b)
	{
	const node_t *node_a = a;
	const node_t *node_b = b;

	return strcmp(node_a->details->uname, node_b->details->uname);
	}

	void
	dump_node_scores_worker(int level, const char file, const char function, int line,
	resource_t * rsc, const char comment, GHashTable nodes)
	{
	GHashTable *hash = nodes;
	GHashTableIter iter;
	node_t *node = NULL;

	if (rsc) {
	hash = rsc->allowed_nodes;
	}

	if (rsc && is_set(rsc->flags, pe_rsc_orphan)) {
	/* Don't show the allocation scores for orphans */
	return;
	}

	if (level == 0) {
	char score[128];
	int len = sizeof(score);
	/* For now we want this in sorted order to keep the regression tests happy */
	GListPtr gIter = NULL;
	GListPtr list = g_hash_table_get_values(hash);

	list = g_list_sort(list, sort_node_uname);

	gIter = list;
	for (; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;
	/* This function is called a whole lot, use stack allocated score */
	score2char_stack(node->weight, score, len);

	if (rsc) {
	printf("%s: %s allocation score on %s: %s\n",
	comment, rsc->id, node->details->uname, score);
	} else {
	printf("%s: %s = %s\n", comment, node->details->uname, score);
	}
	}

	g_list_free(list);

	} else if (hash) {
	char score[128];
	int len = sizeof(score);
	g_hash_table_iter_init(&iter, hash);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	/* This function is called a whole lot, use stack allocated score */
	score2char_stack(node->weight, score, len);

	if (rsc) {
	do_crm_log_alias(LOG_TRACE, file, function, line,
	"%s: %s allocation score on %s: %s", comment, rsc->id,
	node->details->uname, score);
	} else {
	do_crm_log_alias(LOG_TRACE, file, function, line + 1, "%s: %s = %s", comment,
	node->details->uname, score);
	}
	}
	}

	if (rsc && rsc->children) {
	GListPtr gIter = NULL;

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	dump_node_scores_worker(level, file, function, line, child, comment, nodes);
	}
	}
	}

	static void
	append_dump_text(gpointer key, gpointer value, gpointer user_data)
	{
	char **dump_text = user_data;
	int len = 0;
	char *new_text = NULL;

	len = strlen(*dump_text) + strlen(" ") + strlen(key) + strlen("=") + strlen(value) + 1;
	new_text = calloc(1, len);
	sprintf(new_text, "%s %s=%s", dump_text, (char )key, (char *)value);

	free(*dump_text);
	*dump_text = new_text;
	}

	void
	dump_node_capacity(int level, const char comment, node_t node)
	{
	int len = 0;
	char *dump_text = NULL;

	len = strlen(comment) + strlen(": ") + strlen(node->details->uname) + strlen(" capacity:") + 1;
	dump_text = calloc(1, len);
	sprintf(dump_text, "%s: %s capacity:", comment, node->details->uname);

	g_hash_table_foreach(node->details->utilization, append_dump_text, &dump_text);

	if (level == 0) {
	fprintf(stdout, "%s\n", dump_text);
	} else {
	crm_trace("%s", dump_text);
	}

	free(dump_text);
	}

	void
	dump_rsc_utilization(int level, const char comment, resource_t rsc, node_t * node)
	{
	int len = 0;
	char *dump_text = NULL;

	len = strlen(comment) + strlen(": ") + strlen(rsc->id) + strlen(" utilization on ")
	+ strlen(node->details->uname) + strlen(":") + 1;
	dump_text = calloc(1, len);
	sprintf(dump_text, "%s: %s utilization on %s:", comment, rsc->id, node->details->uname);

	g_hash_table_foreach(rsc->utilization, append_dump_text, &dump_text);

	if (level == 0) {
	fprintf(stdout, "%s\n", dump_text);
	} else {
	crm_trace("%s", dump_text);
	}

	free(dump_text);
	}

	gint
	sort_rsc_index(gconstpointer a, gconstpointer b)
	{
	const resource_t resource1 = (const resource_t )a;
	const resource_t resource2 = (const resource_t )b;

	if (a == NULL && b == NULL) {
	return 0;
	}
	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	if (resource1->sort_index > resource2->sort_index) {
	return -1;
	}

	if (resource1->sort_index < resource2->sort_index) {
	return 1;
	}

	return 0;
	}

	gint
	sort_rsc_priority(gconstpointer a, gconstpointer b)
	{
	const resource_t resource1 = (const resource_t )a;
	const resource_t resource2 = (const resource_t )b;

	if (a == NULL && b == NULL) {
	return 0;
	}
	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	if (resource1->priority > resource2->priority) {
	return -1;
	}

	if (resource1->priority < resource2->priority) {
	return 1;
	}

	return 0;
	}

	action_t *
	custom_action(resource_t * rsc, char key, const char task,
	node_t * on_node, gboolean optional, gboolean save_action,
	pe_working_set_t * data_set)
	{
	action_t *action = NULL;
	GListPtr possible_matches = NULL;

	CRM_CHECK(key != NULL, return NULL);
	CRM_CHECK(task != NULL, free(key); return NULL);

	if (save_action && rsc != NULL) {
	possible_matches = find_actions(rsc->actions, key, on_node);
	} else if(save_action) {
	#if 0
	action = g_hash_table_lookup(data_set->singletons, key);
	#else
	/* More expensive but takes 'node' into account */
	possible_matches = find_actions(data_set->actions, key, on_node);
	#endif
	}

	if(data_set->singletons == NULL) {
	data_set->singletons = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL);
	}

	if (possible_matches != NULL) {
	if (g_list_length(possible_matches) > 1) {
	pe_warn("Action %s for %s on %s exists %d times",
	task, rsc ? rsc->id : "<NULL>",
	on_node ? on_node->details->uname : "<NULL>", g_list_length(possible_matches));
	}

	action = g_list_nth_data(possible_matches, 0);
	pe_rsc_trace(rsc, "Found existing action (%d) %s for %s on %s",
	action->id, task, rsc ? rsc->id : "<NULL>",
	on_node ? on_node->details->uname : "<NULL>");
	g_list_free(possible_matches);
	}

	if (action == NULL) {
	if (save_action) {
	pe_rsc_trace(rsc, "Creating%s action %d: %s for %s on %s %d",
	optional ? "" : " manditory", data_set->action_id, key,
	rsc ? rsc->id : "<NULL>", on_node ? on_node->details->uname : "<NULL>", optional);
	}

	action = calloc(1, sizeof(action_t));
	if (save_action) {
	action->id = data_set->action_id++;
	} else {
	action->id = 0;
	}
	action->rsc = rsc;
	CRM_ASSERT(task != NULL);
	action->task = strdup(task);
	if (on_node) {
	action->node = node_copy(on_node);
	}
	action->uuid = strdup(key);

	pe_set_action_bit(action, pe_action_runnable);
	if (optional) {
	pe_rsc_trace(rsc, "Set optional on %s", action->uuid);
	pe_set_action_bit(action, pe_action_optional);
	} else {
	pe_clear_action_bit(action, pe_action_optional);
	pe_rsc_trace(rsc, "Unset optional on %s", action->uuid);
	}

	/*
	Implied by calloc()...
	action->actions_before = NULL;
	action->actions_after = NULL;

	action->pseudo = FALSE;
	action->dumped = FALSE;
	action->processed = FALSE;
	action->seen_count = 0;
	*/

	action->extra = g_hash_table_new_full(crm_str_hash, g_str_equal, free, free);

	action->meta = g_hash_table_new_full(crm_str_hash, g_str_equal, free, free);

	if (save_action) {
	data_set->actions = g_list_prepend(data_set->actions, action);
	if(rsc == NULL) {
	g_hash_table_insert(data_set->singletons, action->uuid, action);
	}
	}

	if (rsc != NULL) {
	action->op_entry = find_rsc_op_entry_helper(rsc, key, TRUE);

	unpack_operation(action, action->op_entry, rsc->container, data_set);

	if (save_action) {
	rsc->actions = g_list_prepend(rsc->actions, action);
	}
	}

	if (save_action) {
	pe_rsc_trace(rsc, "Action %d created", action->id);
	}
	}

	if (optional == FALSE) {
	pe_rsc_trace(rsc, "Unset optional on %s", action->uuid);
	pe_clear_action_bit(action, pe_action_optional);
	}

	if (rsc != NULL) {
	enum action_tasks a_task = text2task(action->task);
	int warn_level = LOG_TRACE;

	if (save_action) {
	warn_level = LOG_WARNING;
	}

	if (is_set(action->flags, pe_action_have_node_attrs) == FALSE
	&& action->node != NULL && action->op_entry != NULL) {
	pe_set_action_bit(action, pe_action_have_node_attrs);
	unpack_instance_attributes(data_set->input, action->op_entry, XML_TAG_ATTR_SETS,
	action->node->details->attrs,
	action->extra, NULL, FALSE, data_set->now);
	}

	if (is_set(action->flags, pe_action_pseudo)) {
	/* leave untouched */

	} else if (action->node == NULL) {
	pe_rsc_trace(rsc, "Unset runnable on %s", action->uuid);
	pe_clear_action_bit(action, pe_action_runnable);

	} else if (is_not_set(rsc->flags, pe_rsc_managed)
	&& g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL) == NULL) {
	crm_debug("Action %s (unmanaged)", action->uuid);
	pe_rsc_trace(rsc, "Set optional on %s", action->uuid);
	pe_set_action_bit(action, pe_action_optional);
	/* action->runnable = FALSE; */

	} else if (action->node->details->online == FALSE) {
	pe_clear_action_bit(action, pe_action_runnable);
	do_crm_log(warn_level, "Action %s on %s is unrunnable (offline)",
	action->uuid, action->node->details->uname);
	if (is_set(action->rsc->flags, pe_rsc_managed)
	&& save_action && a_task == stop_rsc) {
	pe_fence_node(data_set, action->node, "Node is unclean");
	}

	} else if (action->node->details->pending) {
	pe_clear_action_bit(action, pe_action_runnable);
	do_crm_log(warn_level, "Action %s on %s is unrunnable (pending)",
	action->uuid, action->node->details->uname);

	} else if (action->needs == rsc_req_nothing) {
	pe_rsc_trace(rsc, "Action %s doesnt require anything", action->uuid);
	pe_set_action_bit(action, pe_action_runnable);
	#if 0
	/*
	* No point checking this
	- * - if we dont have quorum we cant stonith anyway
	+ * - if we dont have quorum we can't stonith anyway
	*/
	} else if (action->needs == rsc_req_stonith) {
	crm_trace("Action %s requires only stonith", action->uuid);
	action->runnable = TRUE;
	#endif
	} else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE
	&& data_set->no_quorum_policy == no_quorum_stop) {
	pe_clear_action_bit(action, pe_action_runnable);
	crm_debug("%s\t%s (cancelled : quorum)", action->node->details->uname, action->uuid);

	} else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE
	&& data_set->no_quorum_policy == no_quorum_freeze) {
	pe_rsc_trace(rsc, "Check resource is already active: %s %s %s %s", rsc->id, action->uuid, role2text(rsc->next_role), role2text(rsc->role));
	if (rsc->fns->active(rsc, TRUE) == FALSE \|\| rsc->next_role > rsc->role) {
	pe_clear_action_bit(action, pe_action_runnable);
	pe_rsc_debug(rsc, "%s\t%s (cancelled : quorum freeze)",
	action->node->details->uname, action->uuid);
	}

	} else {
	pe_rsc_trace(rsc, "Action %s is runnable", action->uuid);
	pe_set_action_bit(action, pe_action_runnable);
	}

	if (save_action) {
	switch (a_task) {
	case stop_rsc:
	set_bit(rsc->flags, pe_rsc_stopping);
	break;
	case start_rsc:
	clear_bit(rsc->flags, pe_rsc_starting);
	if (is_set(action->flags, pe_action_runnable)) {
	set_bit(rsc->flags, pe_rsc_starting);
	}
	break;
	default:
	break;
	}
	}
	}

	free(key);
	return action;
	}

	static const char *
	unpack_operation_on_fail(action_t * action)
	{

	const char *value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL);

	if (safe_str_eq(action->task, CRMD_ACTION_STOP) && safe_str_eq(value, "standby")) {
	crm_config_err("on-fail=standby is not allowed for stop actions: %s", action->rsc->id);
	return NULL;
	} else if (safe_str_eq(action->task, CRMD_ACTION_DEMOTE) && !value) {
	/* demote on_fail defaults to master monitor value if present */
	xmlNode *operation = NULL;
	const char *name = NULL;
	const char *role = NULL;
	const char *on_fail = NULL;
	const char *interval = NULL;
	const char *enabled = NULL;

	CRM_CHECK(action->rsc != NULL, return NULL);

	for (operation = __xml_first_child(action->rsc->ops_xml);
	operation && !value; operation = __xml_next_element(operation)) {

	if (!crm_str_eq((const char *)operation->name, "op", TRUE)) {
	continue;
	}
	name = crm_element_value(operation, "name");
	role = crm_element_value(operation, "role");
	on_fail = crm_element_value(operation, XML_OP_ATTR_ON_FAIL);
	enabled = crm_element_value(operation, "enabled");
	interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	if (!on_fail) {
	continue;
	} else if (enabled && !crm_is_true(enabled)) {
	continue;
	} else if (safe_str_neq(name, "monitor") \|\| safe_str_neq(role, "Master")) {
	continue;
	} else if (crm_get_interval(interval) <= 0) {
	continue;
	}

	value = on_fail;
	}
	}

	return value;
	}

	static xmlNode *
	find_min_interval_mon(resource_t * rsc, gboolean include_disabled)
	{
	int number = 0;
	int min_interval = -1;
	const char *name = NULL;
	const char *value = NULL;
	const char *interval = NULL;
	xmlNode *op = NULL;
	xmlNode *operation = NULL;

	for (operation = __xml_first_child(rsc->ops_xml); operation != NULL;
	operation = __xml_next_element(operation)) {

	if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
	name = crm_element_value(operation, "name");
	interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	value = crm_element_value(operation, "enabled");
	if (!include_disabled && value && crm_is_true(value) == FALSE) {
	continue;
	}

	if (safe_str_neq(name, RSC_STATUS)) {
	continue;
	}

	number = crm_get_interval(interval);
	if (number < 0) {
	continue;
	}

	if (min_interval < 0 \|\| number < min_interval) {
	min_interval = number;
	op = operation;
	}
	}
	}

	return op;
	}

	void
	unpack_operation(action_t * action, xmlNode * xml_obj, resource_t * container,
	pe_working_set_t * data_set)
	{
	int value_i = 0;
	unsigned long long interval = 0;
	unsigned long long start_delay = 0;
	char *value_ms = NULL;
	const char *value = NULL;
	const char *field = NULL;

	CRM_CHECK(action->rsc != NULL, return);

	unpack_instance_attributes(data_set->input, data_set->op_defaults, XML_TAG_META_SETS, NULL,
	action->meta, NULL, FALSE, data_set->now);

	if (xml_obj) {
	xmlAttrPtr xIter = NULL;

	for (xIter = xml_obj->properties; xIter; xIter = xIter->next) {
	const char prop_name = (const char )xIter->name;
	const char *prop_value = crm_element_value(xml_obj, prop_name);

	g_hash_table_replace(action->meta, strdup(prop_name), strdup(prop_value));
	}
	}

	unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_META_SETS,
	NULL, action->meta, NULL, FALSE, data_set->now);

	unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_ATTR_SETS,
	NULL, action->meta, NULL, FALSE, data_set->now);
	g_hash_table_remove(action->meta, "id");

	field = XML_LRM_ATTR_INTERVAL;
	value = g_hash_table_lookup(action->meta, field);
	if (value != NULL) {
	interval = crm_get_interval(value);
	if (interval > 0) {
	value_ms = crm_itoa(interval);
	g_hash_table_replace(action->meta, strdup(field), value_ms);

	} else {
	g_hash_table_remove(action->meta, field);
	}
	}

	/* Begin compatability code */
	value = g_hash_table_lookup(action->meta, "requires");

	if (safe_str_neq(action->task, RSC_START)
	&& safe_str_neq(action->task, RSC_PROMOTE)) {
	action->needs = rsc_req_nothing;
	value = "nothing (not start/promote)";

	} else if (safe_str_eq(value, "nothing")) {
	action->needs = rsc_req_nothing;

	} else if (safe_str_eq(value, "quorum")) {
	action->needs = rsc_req_quorum;

	} else if (safe_str_eq(value, "unfencing")) {
	action->needs = rsc_req_stonith;
	set_bit(action->rsc->flags, pe_rsc_needs_unfencing);
	if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
	crm_notice("%s requires (un)fencing but fencing is disabled", action->rsc->id);
	}

	} else if (is_set(data_set->flags, pe_flag_stonith_enabled)
	&& safe_str_eq(value, "fencing")) {
	action->needs = rsc_req_stonith;
	if (is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
	crm_notice("%s requires fencing but fencing is disabled", action->rsc->id);
	}
	/* End compatability code */

	} else if (is_set(action->rsc->flags, pe_rsc_needs_fencing)) {
	action->needs = rsc_req_stonith;
	value = "fencing (resource)";

	} else if (is_set(action->rsc->flags, pe_rsc_needs_quorum)) {
	action->needs = rsc_req_quorum;
	value = "quorum (resource)";

	} else {
	action->needs = rsc_req_nothing;
	value = "nothing (resource)";
	}

	pe_rsc_trace(action->rsc, "\tAction %s requires: %s", action->task, value);

	value = unpack_operation_on_fail(action);

	if (value == NULL) {

	} else if (safe_str_eq(value, "block")) {
	action->on_fail = action_fail_block;
	g_hash_table_insert(action->meta, strdup(XML_OP_ATTR_ON_FAIL), strdup("block"));

	} else if (safe_str_eq(value, "fence")) {
	action->on_fail = action_fail_fence;
	value = "node fencing";

	if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
	crm_config_err("Specifying on_fail=fence and" " stonith-enabled=false makes no sense");
	action->on_fail = action_fail_stop;
	action->fail_role = RSC_ROLE_STOPPED;
	value = "stop resource";
	}

	} else if (safe_str_eq(value, "standby")) {
	action->on_fail = action_fail_standby;
	value = "node standby";

	} else if (safe_str_eq(value, "ignore")
	\|\| safe_str_eq(value, "nothing")) {
	action->on_fail = action_fail_ignore;
	value = "ignore";

	} else if (safe_str_eq(value, "migrate")) {
	action->on_fail = action_fail_migrate;
	value = "force migration";

	} else if (safe_str_eq(value, "stop")) {
	action->on_fail = action_fail_stop;
	action->fail_role = RSC_ROLE_STOPPED;
	value = "stop resource";

	} else if (safe_str_eq(value, "restart")) {
	action->on_fail = action_fail_recover;
	value = "restart (and possibly migrate)";

	} else if (safe_str_eq(value, "restart-container")) {
	if (container) {
	action->on_fail = action_fail_restart_container;
	value = "restart container (and possibly migrate)";

	} else {
	value = NULL;
	}

	} else {
	pe_err("Resource %s: Unknown failure type (%s)", action->rsc->id, value);
	value = NULL;
	}

	/* defaults */
	if (value == NULL && container) {
	action->on_fail = action_fail_restart_container;
	value = "restart container (and possibly migrate) (default)";

	/* for barmetal remote nodes, ensure that any failure that results in
	* dropping an active connection to a remote node results in fencing of
	* the remote node.
	*
	* There are only two action failures that don't result in fencing.
	* 1. probes - probe failures are expected.
	* 2. start - a start failure indicates that an active connection does not already
	* exist. The user can set op on-fail=fence if they really want to fence start
	* failures. */
	} else if (value == NULL &&
	is_rsc_baremetal_remote_node(action->rsc, data_set) &&
	!(safe_str_eq(action->task, CRMD_ACTION_STATUS) && interval == 0) &&
	(safe_str_neq(action->task, CRMD_ACTION_START))) {

	if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
	value = "fence baremetal remote node (default)";
	} else {
	value = "recover baremetal remote node connection (default)";
	}
	if (action->rsc->remote_reconnect_interval) {
	action->fail_role = RSC_ROLE_STOPPED;
	}
	action->on_fail = action_fail_reset_remote;

	} else if (value == NULL && safe_str_eq(action->task, CRMD_ACTION_STOP)) {
	if (is_set(data_set->flags, pe_flag_stonith_enabled)) {
	action->on_fail = action_fail_fence;
	value = "resource fence (default)";

	} else {
	action->on_fail = action_fail_block;
	value = "resource block (default)";
	}

	} else if (value == NULL) {
	action->on_fail = action_fail_recover;
	value = "restart (and possibly migrate) (default)";
	}

	pe_rsc_trace(action->rsc, "\t%s failure handling: %s", action->task, value);

	value = NULL;
	if (xml_obj != NULL) {
	value = g_hash_table_lookup(action->meta, "role_after_failure");
	}
	if (value != NULL && action->fail_role == RSC_ROLE_UNKNOWN) {
	action->fail_role = text2role(value);
	}
	/* defaults */
	if (action->fail_role == RSC_ROLE_UNKNOWN) {
	if (safe_str_eq(action->task, CRMD_ACTION_PROMOTE)) {
	action->fail_role = RSC_ROLE_SLAVE;
	} else {
	action->fail_role = RSC_ROLE_STARTED;
	}
	}
	pe_rsc_trace(action->rsc, "\t%s failure results in: %s", action->task,
	role2text(action->fail_role));

	field = XML_OP_ATTR_START_DELAY;
	value = g_hash_table_lookup(action->meta, field);
	if (value != NULL) {
	value_i = crm_get_msec(value);
	if (value_i < 0) {
	value_i = 0;
	}
	start_delay = value_i;
	value_ms = crm_itoa(value_i);
	g_hash_table_replace(action->meta, strdup(field), value_ms);

	} else if (interval > 0 && g_hash_table_lookup(action->meta, XML_OP_ATTR_ORIGIN)) {
	crm_time_t *origin = NULL;

	value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ORIGIN);
	origin = crm_time_new(value);

	if (origin == NULL) {
	crm_config_err("Operation %s contained an invalid " XML_OP_ATTR_ORIGIN ": %s",
	ID(xml_obj), value);

	} else {
	crm_time_t *delay = NULL;
	int rc = crm_time_compare(origin, data_set->now);
	long long delay_s = 0;
	int interval_s = (interval / 1000);

	crm_trace("Origin: %s, interval: %d", value, interval_s);

	/* If 'origin' is in the future, find the most recent "multiple" that occurred in the past */
	while(rc > 0) {
	crm_time_add_seconds(origin, -interval_s);
	rc = crm_time_compare(origin, data_set->now);
	}

	/* Now find the first "multiple" that occurs after 'now' */
	while (rc < 0) {
	crm_time_add_seconds(origin, interval_s);
	rc = crm_time_compare(origin, data_set->now);
	}

	delay = crm_time_calculate_duration(origin, data_set->now);

	crm_time_log(LOG_TRACE, "origin", origin,
	crm_time_log_date \| crm_time_log_timeofday \|
	crm_time_log_with_timezone);
	crm_time_log(LOG_TRACE, "now", data_set->now,
	crm_time_log_date \| crm_time_log_timeofday \|
	crm_time_log_with_timezone);
	crm_time_log(LOG_TRACE, "delay", delay, crm_time_log_duration);

	delay_s = crm_time_get_seconds(delay);

	CRM_CHECK(delay_s >= 0, delay_s = 0);
	start_delay = delay_s * 1000;

	crm_info("Calculated a start delay of %llds for %s", delay_s, ID(xml_obj));
	g_hash_table_replace(action->meta, strdup(XML_OP_ATTR_START_DELAY),
	crm_itoa(start_delay));
	crm_time_free(origin);
	crm_time_free(delay);
	}
	}

	field = XML_ATTR_TIMEOUT;
	value = g_hash_table_lookup(action->meta, field);
	if (value == NULL && xml_obj == NULL && safe_str_eq(action->task, RSC_STATUS) && interval == 0) {
	xmlNode *min_interval_mon = find_min_interval_mon(action->rsc, FALSE);

	if (min_interval_mon) {
	value = crm_element_value(min_interval_mon, XML_ATTR_TIMEOUT);
	pe_rsc_trace(action->rsc,
	"\t%s uses the timeout value '%s' from the minimum interval monitor",
	action->uuid, value);
	}
	}
	if (value == NULL) {
	value = pe_pref(data_set->config_hash, "default-action-timeout");
	}
	value_i = crm_get_msec(value);
	if (value_i < 0) {
	value_i = 0;
	}
	value_i += start_delay;
	value_ms = crm_itoa(value_i);
	g_hash_table_replace(action->meta, strdup(field), value_ms);
	}

	static xmlNode *
	find_rsc_op_entry_helper(resource_t * rsc, const char *key, gboolean include_disabled)
	{
	unsigned long long number = 0;
	gboolean do_retry = TRUE;
	char *local_key = NULL;
	const char *name = NULL;
	const char *value = NULL;
	const char *interval = NULL;
	char *match_key = NULL;
	xmlNode *op = NULL;
	xmlNode *operation = NULL;

	retry:
	for (operation = __xml_first_child(rsc->ops_xml); operation != NULL;
	operation = __xml_next_element(operation)) {
	if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
	name = crm_element_value(operation, "name");
	interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
	value = crm_element_value(operation, "enabled");
	if (!include_disabled && value && crm_is_true(value) == FALSE) {
	continue;
	}

	number = crm_get_interval(interval);
	match_key = generate_op_key(rsc->id, name, number);
	if (safe_str_eq(key, match_key)) {
	op = operation;
	}
	free(match_key);

	if (rsc->clone_name) {
	match_key = generate_op_key(rsc->clone_name, name, number);
	if (safe_str_eq(key, match_key)) {
	op = operation;
	}
	free(match_key);
	}

	if (op != NULL) {
	free(local_key);
	return op;
	}
	}
	}

	free(local_key);
	if (do_retry == FALSE) {
	return NULL;
	}

	do_retry = FALSE;
	if (strstr(key, CRMD_ACTION_MIGRATE) \|\| strstr(key, CRMD_ACTION_MIGRATED)) {
	local_key = generate_op_key(rsc->id, "migrate", 0);
	key = local_key;
	goto retry;

	} else if (strstr(key, "_notify_")) {
	local_key = generate_op_key(rsc->id, "notify", 0);
	key = local_key;
	goto retry;
	}

	return NULL;
	}

	xmlNode *
	find_rsc_op_entry(resource_t * rsc, const char *key)
	{
	return find_rsc_op_entry_helper(rsc, key, FALSE);
	}

	void
	print_node(const char pre_text, node_t node, gboolean details)
	{
	if (node == NULL) {
	crm_trace("%s%s: <NULL>", pre_text == NULL ? "" : pre_text, pre_text == NULL ? "" : ": ");
	return;
	}

	CRM_ASSERT(node->details);
	crm_trace("%s%s%sNode %s: (weight=%d, fixed=%s)",
	pre_text == NULL ? "" : pre_text,
	pre_text == NULL ? "" : ": ",
	node->details->online ? "" : "Unavailable/Unclean ",
	node->details->uname, node->weight, node->fixed ? "True" : "False");

	if (details) {
	char *pe_mutable = strdup("\t\t");
	GListPtr gIter = node->details->running_rsc;

	crm_trace("\t\t===Node Attributes");
	g_hash_table_foreach(node->details->attrs, print_str_str, pe_mutable);
	free(pe_mutable);

	crm_trace("\t\t=== Resources");

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;

	print_resource(LOG_DEBUG_4, "\t\t", rsc, FALSE);
	}
	}
	}

	/*
	* Used by the HashTable for-loop
	*/
	void
	print_str_str(gpointer key, gpointer value, gpointer user_data)
	{
	crm_trace("%s%s %s ==> %s",
	user_data == NULL ? "" : (char *)user_data,
	user_data == NULL ? "" : ": ", (char )key, (char )value);
	}

	void
	print_resource(int log_level, const char pre_text, resource_t rsc, gboolean details)
	{
	long options = pe_print_log;

	if (rsc == NULL) {
	do_crm_log(log_level - 1, "%s%s: <NULL>",
	pre_text == NULL ? "" : pre_text, pre_text == NULL ? "" : ": ");
	return;
	}
	if (details) {
	options \|= pe_print_details;
	}
	rsc->fns->print(rsc, pre_text, options, &log_level);
	}

	void
	pe_free_action(action_t * action)
	{
	if (action == NULL) {
	return;
	}
	g_list_free_full(action->actions_before, free); /* action_warpper_t* */
	g_list_free_full(action->actions_after, free); /* action_warpper_t* */
	if (action->extra) {
	g_hash_table_destroy(action->extra);
	}
	if (action->meta) {
	g_hash_table_destroy(action->meta);
	}
	free(action->cancel_task);
	free(action->task);
	free(action->uuid);
	free(action->node);
	free(action);
	}

	GListPtr
	find_recurring_actions(GListPtr input, node_t * not_on_node)
	{
	const char *value = NULL;
	GListPtr result = NULL;
	GListPtr gIter = input;

	CRM_CHECK(input != NULL, return NULL);

	for (; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	value = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL);
	if (value == NULL) {
	/* skip */
	} else if (safe_str_eq(value, "0")) {
	/* skip */
	} else if (safe_str_eq(CRMD_ACTION_CANCEL, action->task)) {
	/* skip */
	} else if (not_on_node == NULL) {
	crm_trace("(null) Found: %s", action->uuid);
	result = g_list_prepend(result, action);

	} else if (action->node == NULL) {
	/* skip */
	} else if (action->node->details != not_on_node->details) {
	crm_trace("Found: %s", action->uuid);
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	enum action_tasks
	get_complex_task(resource_t * rsc, const char *name, gboolean allow_non_atomic)
	{
	enum action_tasks task = text2task(name);

	if (rsc == NULL) {
	return task;

	} else if (allow_non_atomic == FALSE \|\| rsc->variant == pe_native) {
	switch (task) {
	case stopped_rsc:
	case started_rsc:
	case action_demoted:
	case action_promoted:
	crm_trace("Folding %s back into its atomic counterpart for %s", name, rsc->id);
	return task - 1;
	break;
	default:
	break;
	}
	}
	return task;
	}

	action_t *
	find_first_action(GListPtr input, const char uuid, const char task, node_t * on_node)
	{
	GListPtr gIter = NULL;

	CRM_CHECK(uuid \|\| task, return NULL);

	for (gIter = input; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	if (uuid != NULL && safe_str_neq(uuid, action->uuid)) {
	continue;

	} else if (task != NULL && safe_str_neq(task, action->task)) {
	continue;

	} else if (on_node == NULL) {
	return action;

	} else if (action->node == NULL) {
	continue;

	} else if (on_node->details == action->node->details) {
	return action;
	}
	}

	return NULL;
	}

	GListPtr
	find_actions(GListPtr input, const char key, node_t on_node)
	{
	GListPtr gIter = input;
	GListPtr result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	for (; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	crm_trace("Matching %s against %s", key, action->uuid);
	if (safe_str_neq(key, action->uuid)) {
	continue;

	} else if (on_node == NULL) {
	result = g_list_prepend(result, action);

	} else if (action->node == NULL) {
	/* skip */
	crm_trace("While looking for %s action on %s, "
	"found an unallocated one. Assigning"
	" it to the requested node...", key, on_node->details->uname);

	action->node = node_copy(on_node);
	result = g_list_prepend(result, action);

	} else if (on_node->details == action->node->details) {
	result = g_list_prepend(result, action);
	}
	}

	return result;
	}

	GListPtr
	find_actions_exact(GListPtr input, const char key, node_t on_node)
	{
	GListPtr gIter = input;
	GListPtr result = NULL;

	CRM_CHECK(key != NULL, return NULL);

	for (; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	crm_trace("Matching %s against %s", key, action->uuid);
	if (safe_str_neq(key, action->uuid)) {
	crm_trace("Key mismatch: %s vs. %s", key, action->uuid);
	continue;

	} else if (on_node == NULL \|\| action->node == NULL) {
	crm_trace("on_node=%p, action->node=%p", on_node, action->node);
	continue;

	} else if (safe_str_eq(on_node->details->id, action->node->details->id)) {
	result = g_list_prepend(result, action);
	}
	crm_trace("Node mismatch: %s vs. %s", on_node->details->id, action->node->details->id);
	}

	return result;
	}

	static void
	resource_node_score(resource_t * rsc, node_t * node, int score, const char *tag)
	{
	node_t *match = NULL;

	if (rsc->children) {
	GListPtr gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	resource_node_score(child_rsc, node, score, tag);
	}
	}

	pe_rsc_trace(rsc, "Setting %s for %s on %s: %d", tag, rsc->id, node->details->uname, score);
	match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (match == NULL) {
	match = node_copy(node);
	match->weight = merge_weights(score, node->weight);
	g_hash_table_insert(rsc->allowed_nodes, (gpointer) match->details->id, match);
	}
	match->weight = merge_weights(match->weight, score);
	}

	void
	resource_location(resource_t * rsc, node_t * node, int score, const char *tag,
	pe_working_set_t * data_set)
	{
	if (node != NULL) {
	resource_node_score(rsc, node, score, tag);

	} else if (data_set != NULL) {
	GListPtr gIter = data_set->nodes;

	for (; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	resource_node_score(rsc, node, score, tag);
	}

	} else {
	GHashTableIter iter;
	node_t *node = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	resource_node_score(rsc, node, score, tag);
	}
	}

	if (node == NULL && score == -INFINITY) {
	if (rsc->allocated_to) {
	crm_info("Deallocating %s from %s", rsc->id, rsc->allocated_to->details->uname);
	free(rsc->allocated_to);
	rsc->allocated_to = NULL;
	}
	}
	}

	#define sort_return(an_int, why) do { \
	free(a_uuid); \
	free(b_uuid); \
	crm_trace("%s (%d) %c %s (%d) : %s", \
	a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=', \
	b_xml_id, b_call_id, why); \
	return an_int; \
	} while(0)

	gint
	sort_op_by_callid(gconstpointer a, gconstpointer b)
	{
	int a_call_id = -1;
	int b_call_id = -1;

	char *a_uuid = NULL;
	char *b_uuid = NULL;

	const xmlNode *xml_a = a;
	const xmlNode *xml_b = b;

	const char *a_xml_id = crm_element_value_const(xml_a, XML_ATTR_ID);
	const char *b_xml_id = crm_element_value_const(xml_b, XML_ATTR_ID);

	if (safe_str_eq(a_xml_id, b_xml_id)) {
	/* We have duplicate lrm_rsc_op entries in the status
	* section which is unliklely to be a good thing
	* - we can handle it easily enough, but we need to get
	* to the bottom of why its happening.
	*/
	pe_err("Duplicate lrm_rsc_op entries named %s", a_xml_id);
	sort_return(0, "duplicate");
	}

	crm_element_value_const_int(xml_a, XML_LRM_ATTR_CALLID, &a_call_id);
	crm_element_value_const_int(xml_b, XML_LRM_ATTR_CALLID, &b_call_id);

	if (a_call_id == -1 && b_call_id == -1) {
	/* both are pending ops so it doesnt matter since
	* stops are never pending
	*/
	sort_return(0, "pending");

	} else if (a_call_id >= 0 && a_call_id < b_call_id) {
	sort_return(-1, "call id");

	} else if (b_call_id >= 0 && a_call_id > b_call_id) {
	sort_return(1, "call id");

	} else if (b_call_id >= 0 && a_call_id == b_call_id) {
	/*
	* The op and last_failed_op are the same
	* Order on last-rc-change
	*/
	int last_a = -1;
	int last_b = -1;

	crm_element_value_const_int(xml_a, XML_RSC_OP_LAST_CHANGE, &last_a);
	crm_element_value_const_int(xml_b, XML_RSC_OP_LAST_CHANGE, &last_b);

	crm_trace("rc-change: %d vs %d", last_a, last_b);
	if (last_a >= 0 && last_a < last_b) {
	sort_return(-1, "rc-change");

	} else if (last_b >= 0 && last_a > last_b) {
	sort_return(1, "rc-change");
	}
	sort_return(0, "rc-change");

	} else {
	/* One of the inputs is a pending operation
	* Attempt to use XML_ATTR_TRANSITION_MAGIC to determine its age relative to the other
	*/

	int a_id = -1;
	int b_id = -1;
	int dummy = -1;

	const char *a_magic = crm_element_value_const(xml_a, XML_ATTR_TRANSITION_MAGIC);
	const char *b_magic = crm_element_value_const(xml_b, XML_ATTR_TRANSITION_MAGIC);

	CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
	if(!decode_transition_magic(a_magic, &a_uuid, &a_id, &dummy, &dummy, &dummy, &dummy)) {
	sort_return(0, "bad magic a");
	}
	if(!decode_transition_magic(b_magic, &b_uuid, &b_id, &dummy, &dummy, &dummy, &dummy)) {
	sort_return(0, "bad magic b");
	}
	- /* try and determin the relative age of the operation...
	- * some pending operations (ie. a start) may have been supuerceeded
	+ /* try to determine the relative age of the operation...
	+ * some pending operations (ie. a start) may have been superseded
	* by a subsequent stop
	*
	* [a\|b]_id == -1 means its a shutdown operation and _always_ comes last
	*/
	if (safe_str_neq(a_uuid, b_uuid) \|\| a_id == b_id) {
	/*
	* some of the logic in here may be redundant...
	*
	* if the UUID from the TE doesnt match then one better
	* be a pending operation.
	* pending operations dont survive between elections and joins
	* because we query the LRM directly
	*/

	if (b_call_id == -1) {
	sort_return(-1, "transition + call");

	} else if (a_call_id == -1) {
	sort_return(1, "transition + call");
	}

	} else if ((a_id >= 0 && a_id < b_id) \|\| b_id == -1) {
	sort_return(-1, "transition");

	} else if ((b_id >= 0 && a_id > b_id) \|\| a_id == -1) {
	sort_return(1, "transition");
	}
	}

	/* we should never end up here */
	CRM_CHECK(FALSE, sort_return(0, "default"));

	}

	time_t
	get_effective_time(pe_working_set_t * data_set)
	{
	if(data_set) {
	if (data_set->now == NULL) {
	crm_trace("Recording a new 'now'");
	data_set->now = crm_time_new(NULL);
	}
	return crm_time_get_seconds_since_epoch(data_set->now);
	}

	crm_trace("Defaulting to 'now'");
	return time(NULL);
	}

	struct fail_search {
	resource_t *rsc;
	pe_working_set_t * data_set;

	int count;
	long long last;
	char *key;
	};

	static void
	get_failcount_by_prefix(gpointer key_p, gpointer value, gpointer user_data)
	{
	struct fail_search *search = user_data;
	const char *attr_id = key_p;
	const char *match = strstr(attr_id, search->key);
	resource_t *parent = NULL;

	if (match == NULL) {
	return;
	}

	/* we are only incrementing the failcounts here if the rsc
	* that matches our prefix has the same uber parent as the rsc we're
	* calculating the failcounts for. This prevents false positive matches
	* where unrelated resources may have similar prefixes in their names.
	*
	* search->rsc is already set to be the uber parent. */
	parent = uber_parent(pe_find_resource(search->data_set->resources, match));
	if (parent == NULL \|\| parent != search->rsc) {
	return;
	}
	if (strstr(attr_id, "last-failure-") == attr_id) {
	search->last = crm_int_helper(value, NULL);

	} else if (strstr(attr_id, "fail-count-") == attr_id) {
	search->count += char2score(value);
	}
	}

	int
	get_failcount(node_t * node, resource_t * rsc, time_t last_failure, pe_working_set_t data_set)
	{
	return get_failcount_full(node, rsc, last_failure, TRUE, NULL, data_set);
	}

	static gboolean
	is_matched_failure(const char * rsc_id, xmlNode * conf_op_xml, xmlNode * lrm_op_xml)
	{
	gboolean matched = FALSE;
	const char *conf_op_name = NULL;
	int conf_op_interval = 0;
	const char *lrm_op_task = NULL;
	int lrm_op_interval = 0;
	const char *lrm_op_id = NULL;
	char *last_failure_key = NULL;

	if (rsc_id == NULL \|\| conf_op_xml == NULL \|\| lrm_op_xml == NULL) {
	return FALSE;
	}

	conf_op_name = crm_element_value(conf_op_xml, "name");
	conf_op_interval = crm_get_msec(crm_element_value(conf_op_xml, "interval"));
	lrm_op_task = crm_element_value(lrm_op_xml, XML_LRM_ATTR_TASK);
	crm_element_value_int(lrm_op_xml, XML_LRM_ATTR_INTERVAL, &lrm_op_interval);

	if (safe_str_eq(conf_op_name, lrm_op_task) == FALSE
	\|\| conf_op_interval != lrm_op_interval) {
	return FALSE;
	}

	lrm_op_id = ID(lrm_op_xml);
	last_failure_key = generate_op_key(rsc_id, "last_failure", 0);

	if (safe_str_eq(last_failure_key, lrm_op_id)) {
	matched = TRUE;

	} else {
	char *expected_op_key = generate_op_key(rsc_id, conf_op_name, conf_op_interval);

	if (safe_str_eq(expected_op_key, lrm_op_id)) {
	int rc = 0;
	int target_rc = get_target_rc(lrm_op_xml);

	crm_element_value_int(lrm_op_xml, XML_LRM_ATTR_RC, &rc);
	if (rc != target_rc) {
	matched = TRUE;
	}
	}
	free(expected_op_key);
	}

	free(last_failure_key);
	return matched;
	}

	static gboolean
	block_failure(node_t * node, resource_t * rsc, xmlNode * xml_op, pe_working_set_t * data_set)
	{
	char *xml_name = clone_strip(rsc->id);
	char *xpath = crm_strdup_printf("//primitive[@id='%s']//op[@on-fail='block']", xml_name);
	xmlXPathObject *xpathObj = xpath_search(rsc->xml, xpath);
	gboolean should_block = FALSE;

	free(xpath);

	if (xpathObj) {
	int max = numXpathResults(xpathObj);
	int lpc = 0;

	for (lpc = 0; lpc < max; lpc++) {
	xmlNode *pref = getXpathResult(xpathObj, lpc);

	if (xml_op) {
	should_block = is_matched_failure(xml_name, pref, xml_op);
	if (should_block) {
	break;
	}

	} else {
	const char *conf_op_name = NULL;
	int conf_op_interval = 0;
	char *lrm_op_xpath = NULL;
	xmlXPathObject *lrm_op_xpathObj = NULL;

	conf_op_name = crm_element_value(pref, "name");
	conf_op_interval = crm_get_msec(crm_element_value(pref, "interval"));

	lrm_op_xpath = crm_strdup_printf("//node_state[@uname='%s']"
	"//lrm_resource[@id='%s']"
	"/lrm_rsc_op[@operation='%s'][@interval='%d']",
	node->details->uname, xml_name,
	conf_op_name, conf_op_interval);
	lrm_op_xpathObj = xpath_search(data_set->input, lrm_op_xpath);

	free(lrm_op_xpath);

	if (lrm_op_xpathObj) {
	int max2 = numXpathResults(lrm_op_xpathObj);
	int lpc2 = 0;

	for (lpc2 = 0; lpc2 < max2; lpc2++) {
	xmlNode *lrm_op_xml = getXpathResult(lrm_op_xpathObj, lpc2);

	should_block = is_matched_failure(xml_name, pref, lrm_op_xml);
	if (should_block) {
	break;
	}
	}
	}
	freeXpathObject(lrm_op_xpathObj);

	if (should_block) {
	break;
	}
	}
	}
	}

	free(xml_name);
	freeXpathObject(xpathObj);

	return should_block;
	}

	int
	get_failcount_full(node_t * node, resource_t * rsc, time_t *last_failure,
	bool effective, xmlNode * xml_op, pe_working_set_t * data_set)
	{
	char *key = NULL;
	const char *value = NULL;
	struct fail_search search = { rsc, data_set, 0, 0, NULL };

	/* Optimize the "normal" case */
	key = crm_concat("fail-count", rsc->clone_name ? rsc->clone_name : rsc->id, '-');
	value = g_hash_table_lookup(node->details->attrs, key);
	search.count = char2score(value);
	crm_trace("%s = %s", key, value);
	free(key);

	if (value) {
	key = crm_concat("last-failure", rsc->clone_name ? rsc->clone_name : rsc->id, '-');
	value = g_hash_table_lookup(node->details->attrs, key);
	search.last = crm_int_helper(value, NULL);
	free(key);

	/* This block is still relevant once we omit anonymous instance numbers
	- * because stopped clones wont have clone_name set
	+ * because stopped clones won't have clone_name set
	*/
	} else if (is_not_set(rsc->flags, pe_rsc_unique)) {
	search.rsc = uber_parent(rsc);
	search.key = clone_strip(rsc->id);

	g_hash_table_foreach(node->details->attrs, get_failcount_by_prefix, &search);
	free(search.key);
	search.key = NULL;
	}

	if (search.count != 0 && search.last != 0 && last_failure) {
	*last_failure = search.last;
	}

	if(search.count && rsc->failure_timeout) {
	/* Never time-out if blocking failures are configured */
	if (block_failure(node, rsc, xml_op, data_set)) {
	pe_warn("Setting %s.failure-timeout=%d conflicts with on-fail=block: ignoring timeout", rsc->id, rsc->failure_timeout);
	rsc->failure_timeout = 0;
	#if 0
	/* A good idea? */
	} else if (rsc->container == NULL && is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
	/* In this case, stop.on-fail defaults to block in unpack_operation() */
	rsc->failure_timeout = 0;
	#endif
	}
	}

	if (effective && search.count != 0 && search.last != 0 && rsc->failure_timeout) {
	if (search.last > 0) {
	time_t now = get_effective_time(data_set);

	if (now > (search.last + rsc->failure_timeout)) {
	crm_debug("Failcount for %s on %s has expired (limit was %ds)",
	search.rsc->id, node->details->uname, rsc->failure_timeout);
	search.count = 0;
	}
	}
	}

	if (search.count != 0) {
	char *score = score2char(search.count);

	crm_info("%s has failed %s times on %s", search.rsc->id, score, node->details->uname);
	free(score);
	}

	return search.count;
	}

	/* If it's a resource container, get its failcount plus all the failcounts of the resources within it */
	int
	get_failcount_all(node_t * node, resource_t * rsc, time_t last_failure, pe_working_set_t data_set)
	{
	int failcount_all = 0;

	failcount_all = get_failcount(node, rsc, last_failure, data_set);

	if (rsc->fillers) {
	GListPtr gIter = NULL;

	for (gIter = rsc->fillers; gIter != NULL; gIter = gIter->next) {
	resource_t filler = (resource_t ) gIter->data;
	time_t filler_last_failure = 0;

	failcount_all += get_failcount(node, filler, &filler_last_failure, data_set);

	if (last_failure && filler_last_failure > *last_failure) {
	*last_failure = filler_last_failure;
	}
	}

	if (failcount_all != 0) {
	char *score = score2char(failcount_all);

	crm_info("Container %s and the resources within it have failed %s times on %s",
	rsc->id, score, node->details->uname);
	free(score);
	}
	}

	return failcount_all;
	}

	gboolean
	get_target_role(resource_t * rsc, enum rsc_role_e * role)
	{
	enum rsc_role_e local_role = RSC_ROLE_UNKNOWN;
	const char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE);

	CRM_CHECK(role != NULL, return FALSE);

	if (value == NULL \|\| safe_str_eq("started", value)
	\|\| safe_str_eq("default", value)) {
	return FALSE;
	}

	local_role = text2role(value);
	if (local_role == RSC_ROLE_UNKNOWN) {
	crm_config_err("%s: Unknown value for %s: %s", rsc->id, XML_RSC_ATTR_TARGET_ROLE, value);
	return FALSE;

	} else if (local_role > RSC_ROLE_STARTED) {
	if (uber_parent(rsc)->variant == pe_master) {
	if (local_role > RSC_ROLE_SLAVE) {
	/* This is what we'd do anyway, just leave the default to avoid messing up the placement algorithm */
	return FALSE;
	}

	} else {
	crm_config_err("%s is not part of a master/slave resource, a %s of '%s' makes no sense",
	rsc->id, XML_RSC_ATTR_TARGET_ROLE, value);
	return FALSE;
	}
	}

	*role = local_role;
	return TRUE;
	}

	gboolean
	order_actions(action_t * lh_action, action_t * rh_action, enum pe_ordering order)
	{
	GListPtr gIter = NULL;
	action_wrapper_t *wrapper = NULL;
	GListPtr list = NULL;

	if (order == pe_order_none) {
	return FALSE;
	}

	if (lh_action == NULL \|\| rh_action == NULL) {
	return FALSE;
	}

	crm_trace("Ordering Action %s before %s", lh_action->uuid, rh_action->uuid);

	/* Ensure we never create a dependency on ourselves... its happened */
	CRM_ASSERT(lh_action != rh_action);

	/* Filter dups, otherwise update_action_states() has too much work to do */
	gIter = lh_action->actions_after;
	for (; gIter != NULL; gIter = gIter->next) {
	action_wrapper_t after = (action_wrapper_t ) gIter->data;

	if (after->action == rh_action && (after->type & order)) {
	return FALSE;
	}
	}

	wrapper = calloc(1, sizeof(action_wrapper_t));
	wrapper->action = rh_action;
	wrapper->type = order;

	list = lh_action->actions_after;
	list = g_list_prepend(list, wrapper);
	lh_action->actions_after = list;

	wrapper = NULL;

	/* order \|= pe_order_implies_then; */
	/* order ^= pe_order_implies_then; */

	wrapper = calloc(1, sizeof(action_wrapper_t));
	wrapper->action = lh_action;
	wrapper->type = order;
	list = rh_action->actions_before;
	list = g_list_prepend(list, wrapper);
	rh_action->actions_before = list;
	return TRUE;
	}

	action_t *
	get_pseudo_op(const char name, pe_working_set_t data_set)
	{
	action_t *op = NULL;

	if(data_set->singletons) {
	op = g_hash_table_lookup(data_set->singletons, name);
	}
	if (op == NULL) {
	op = custom_action(NULL, strdup(name), name, NULL, TRUE, TRUE, data_set);
	set_bit(op->flags, pe_action_pseudo);
	set_bit(op->flags, pe_action_runnable);
	}

	return op;
	}

	void
	destroy_ticket(gpointer data)
	{
	ticket_t *ticket = data;

	if (ticket->state) {
	g_hash_table_destroy(ticket->state);
	}
	free(ticket->id);
	free(ticket);
	}

	ticket_t *
	ticket_new(const char ticket_id, pe_working_set_t data_set)
	{
	ticket_t *ticket = NULL;

	if (ticket_id == NULL \|\| strlen(ticket_id) == 0) {
	return NULL;
	}

	if (data_set->tickets == NULL) {
	data_set->tickets =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, destroy_ticket);
	}

	ticket = g_hash_table_lookup(data_set->tickets, ticket_id);
	if (ticket == NULL) {

	ticket = calloc(1, sizeof(ticket_t));
	if (ticket == NULL) {
	crm_err("Cannot allocate ticket '%s'", ticket_id);
	return NULL;
	}

	crm_trace("Creaing ticket entry for %s", ticket_id);

	ticket->id = strdup(ticket_id);
	ticket->granted = FALSE;
	ticket->last_granted = -1;
	ticket->standby = FALSE;
	ticket->state = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);

	g_hash_table_insert(data_set->tickets, strdup(ticket->id), ticket);
	}

	return ticket;
	}

	static void
	filter_parameters(xmlNode * param_set, const char *param_string, bool need_present)
	{
	int len = 0;
	char *name = NULL;
	char *match = NULL;

	if (param_set == NULL) {
	return;
	}

	if (param_set) {
	xmlAttrPtr xIter = param_set->properties;

	while (xIter) {
	const char prop_name = (const char )xIter->name;

	xIter = xIter->next;
	name = NULL;
	len = strlen(prop_name) + 3;

	name = malloc(len);
	if(name) {
	sprintf(name, " %s ", prop_name);
	name[len - 1] = 0;
	match = strstr(param_string, name);
	}

	if (need_present && match == NULL) {
	crm_trace("%s not found in %s", prop_name, param_string);
	xml_remove_prop(param_set, prop_name);

	} else if (need_present == FALSE && match) {
	crm_trace("%s found in %s", prop_name, param_string);
	xml_remove_prop(param_set, prop_name);
	}
	free(name);
	}
	}
	}

	op_digest_cache_t *
	rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node,
	pe_working_set_t * data_set)
	{
	op_digest_cache_t *data = NULL;

	GHashTable *local_rsc_params = NULL;

	action_t *action = NULL;
	char *key = NULL;

	int interval = 0;
	const char *op_id = ID(xml_op);
	const char *interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *digest_all;
	const char *digest_restart;
	const char *secure_list;
	const char *restart_list;
	const char *op_version;

	data = g_hash_table_lookup(node->details->digest_cache, op_id);
	if (data) {
	return data;
	}

	data = calloc(1, sizeof(op_digest_cache_t));

	digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);
	digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);

	secure_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_SECURE);
	restart_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_RESTART);

	op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);

	/* key is freed in custom_action */
	interval = crm_parse_int(interval_s, "0");
	key = generate_op_key(rsc->id, task, interval);
	action = custom_action(rsc, key, task, node, TRUE, FALSE, data_set);
	key = NULL;

	local_rsc_params = g_hash_table_new_full(crm_str_hash, g_str_equal,
	g_hash_destroy_str, g_hash_destroy_str);
	get_rsc_attributes(local_rsc_params, rsc, node, data_set);
	data->params_all = create_xml_node(NULL, XML_TAG_PARAMS);
	g_hash_table_foreach(local_rsc_params, hash2field, data->params_all);
	g_hash_table_foreach(action->extra, hash2field, data->params_all);
	g_hash_table_foreach(rsc->parameters, hash2field, data->params_all);
	g_hash_table_foreach(action->meta, hash2metafield, data->params_all);
	filter_action_parameters(data->params_all, op_version);

	data->digest_all_calc = calculate_operation_digest(data->params_all, op_version);

	if (secure_list && is_set(data_set->flags, pe_flag_sanitized)) {
	data->params_secure = copy_xml(data->params_all);

	if (secure_list) {
	filter_parameters(data->params_secure, secure_list, FALSE);
	}
	data->digest_secure_calc = calculate_operation_digest(data->params_secure, op_version);
	}

	if (digest_restart) {
	data->params_restart = copy_xml(data->params_all);

	if (restart_list) {
	filter_parameters(data->params_restart, restart_list, TRUE);
	}
	data->digest_restart_calc = calculate_operation_digest(data->params_restart, op_version);
	}

	data->rc = RSC_DIGEST_MATCH;
	if (digest_restart && strcmp(data->digest_restart_calc, digest_restart) != 0) {
	data->rc = RSC_DIGEST_RESTART;

	} else if (digest_all == NULL) {
	/* it is unknown what the previous op digest was */
	data->rc = RSC_DIGEST_UNKNOWN;

	} else if (strcmp(digest_all, data->digest_all_calc) != 0) {
	data->rc = RSC_DIGEST_ALL;
	}

	g_hash_table_insert(node->details->digest_cache, strdup(op_id), data);
	g_hash_table_destroy(local_rsc_params);
	pe_free_action(action);

	return data;
	}

	const char rsc_printable_id(resource_t rsc)
	{
	if (is_not_set(rsc->flags, pe_rsc_unique)) {
	return ID(rsc->xml);
	}
	return rsc->id;
	}

	gboolean
	is_rsc_baremetal_remote_node(resource_t rsc, pe_working_set_t data_set)
	{
	node_t *node;

	if (rsc == NULL) {
	return FALSE;
	} else if (rsc->is_remote_node == FALSE) {
	return FALSE;
	}

	node = pe_find_node(data_set->nodes, rsc->id);
	if (node == NULL) {
	return FALSE;
	}

	return is_baremetal_remote_node(node);
	}

	gboolean
	is_baremetal_remote_node(node_t *node)
	{
	if (is_remote_node(node) && (node->details->remote_rsc == FALSE \|\| node->details->remote_rsc->container == FALSE)) {
	return TRUE;
	}
	return FALSE;
	}

	gboolean
	is_container_remote_node(node_t *node)
	{
	if (is_remote_node(node) && (node->details->remote_rsc && node->details->remote_rsc->container)) {
	return TRUE;
	}
	return FALSE;
	}

	gboolean
	is_remote_node(node_t *node)
	{
	if (node && node->details->type == node_remote) {
	return TRUE;
	}
	return FALSE;
	}

	resource_t *
	rsc_contains_remote_node(pe_working_set_t * data_set, resource_t *rsc)
	{
	if (is_set(data_set->flags, pe_flag_have_remote_nodes) == FALSE) {
	return NULL;
	}

	if (rsc->fillers) {
	GListPtr gIter = NULL;
	for (gIter = rsc->fillers; gIter != NULL; gIter = gIter->next) {
	resource_t filler = (resource_t ) gIter->data;

	if (filler->is_remote_node) {
	return filler;
	}
	}
	}
	return NULL;
	}

	gboolean
	xml_contains_remote_node(xmlNode *xml)
	{
	const char *class = crm_element_value(xml, XML_AGENT_ATTR_CLASS);
	const char *provider = crm_element_value(xml, XML_AGENT_ATTR_PROVIDER);
	const char *agent = crm_element_value(xml, XML_ATTR_TYPE);

	if (safe_str_eq(agent, "remote") && safe_str_eq(provider, "pacemaker") && safe_str_eq(class, "ocf")) {
	return TRUE;
	}
	return FALSE;
	}

	void
	clear_bit_recursive(resource_t * rsc, unsigned long long flag)
	{
	GListPtr gIter = rsc->children;

	clear_bit(rsc->flags, flag);
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	clear_bit_recursive(child_rsc, flag);
	}
	}

	void
	set_bit_recursive(resource_t * rsc, unsigned long long flag)
	{
	GListPtr gIter = rsc->children;

	set_bit(rsc->flags, flag);
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	set_bit_recursive(child_rsc, flag);
	}
	}

	action_t *
	pe_fence_op(node_t * node, const char op, bool optional, pe_working_set_t data_set)
	{
	char *key = NULL;
	action_t *stonith_op = NULL;

	if(op == NULL) {
	op = data_set->stonith_action;
	}

	key = crm_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op);

	if(data_set->singletons) {
	stonith_op = g_hash_table_lookup(data_set->singletons, key);
	}

	if(stonith_op == NULL) {
	stonith_op = custom_action(NULL, key, CRM_OP_FENCE, node, optional, TRUE, data_set);

	add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname);
	add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id);
	add_hash_param(stonith_op->meta, "stonith_action", op);
	} else {
	free(key);
	}

	if(optional == FALSE) {
	crm_trace("%s is no longer optional", stonith_op->uuid);
	pe_clear_action_bit(stonith_op, pe_action_optional);
	}

	return stonith_op;
	}

	void
	trigger_unfencing(
	resource_t * rsc, node_t node, const char reason, action_t dependency, pe_working_set_t data_set)
	{
	if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) {
	/* No resources require it */
	return;

	} else if (rsc != NULL && is_not_set(rsc->flags, pe_rsc_fence_device)) {
	/* Wasnt a stonith device */
	return;

	} else if(node
	&& node->details->online
	&& node->details->unclean == FALSE
	&& node->details->shutdown == FALSE) {
	action_t *unfence = pe_fence_op(node, "on", FALSE, data_set);

	crm_notice("Unfencing %s: %s", node->details->uname, reason);
	if(dependency) {
	order_actions(unfence, dependency, pe_order_optional);
	}

	} else if(rsc) {
	GHashTableIter iter;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) {
	trigger_unfencing(rsc, node, reason, dependency, data_set);
	}
	}
	}
	}

	gboolean
	add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref)
	{
	tag_t *tag = NULL;
	GListPtr gIter = NULL;
	gboolean is_existing = FALSE;

	CRM_CHECK(tags && tag_name && obj_ref, return FALSE);

	tag = g_hash_table_lookup(tags, tag_name);
	if (tag == NULL) {
	tag = calloc(1, sizeof(tag_t));
	if (tag == NULL) {
	return FALSE;
	}
	tag->id = strdup(tag_name);
	tag->refs = NULL;
	g_hash_table_insert(tags, strdup(tag_name), tag);
	}

	for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) {
	const char existing_ref = (const char ) gIter->data;

	if (crm_str_eq(existing_ref, obj_ref, TRUE)){
	is_existing = TRUE;
	break;
	}
	}

	if (is_existing == FALSE) {
	tag->refs = g_list_append(tag->refs, strdup(obj_ref));
	crm_trace("Added: tag=%s ref=%s", tag->id, obj_ref);
	}

	return TRUE;
	}
	diff --git a/pengine/allocate.c b/pengine/allocate.c
	index 572ba74d4c..6462a0e55c 100644
	--- a/pengine/allocate.c
	+++ b/pengine/allocate.c
	@@ -1,2763 +1,2764 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <glib.h>

	#include <crm/pengine/status.h>
	#include <pengine.h>
	#include <allocate.h>
	#include <utils.h>

	CRM_TRACE_INIT_DATA(pe_allocate);

	void set_alloc_actions(pe_working_set_t * data_set);
	void migrate_reload_madness(pe_working_set_t * data_set);

	resource_alloc_functions_t resource_class_alloc_functions[] = {
	{
	native_merge_weights,
	native_color,
	native_create_actions,
	native_create_probe,
	native_internal_constraints,
	native_rsc_colocation_lh,
	native_rsc_colocation_rh,
	native_rsc_location,
	native_action_flags,
	native_update_actions,
	native_expand,
	native_append_meta,
	},
	{
	group_merge_weights,
	group_color,
	group_create_actions,
	native_create_probe,
	group_internal_constraints,
	group_rsc_colocation_lh,
	group_rsc_colocation_rh,
	group_rsc_location,
	group_action_flags,
	group_update_actions,
	group_expand,
	group_append_meta,
	},
	{
	clone_merge_weights,
	clone_color,
	clone_create_actions,
	clone_create_probe,
	clone_internal_constraints,
	clone_rsc_colocation_lh,
	clone_rsc_colocation_rh,
	clone_rsc_location,
	clone_action_flags,
	clone_update_actions,
	clone_expand,
	clone_append_meta,
	},
	{
	master_merge_weights,
	master_color,
	master_create_actions,
	clone_create_probe,
	master_internal_constraints,
	clone_rsc_colocation_lh,
	master_rsc_colocation_rh,
	clone_rsc_location,
	clone_action_flags,
	clone_update_actions,
	clone_expand,
	master_append_meta,
	}
	};

	gboolean
	update_action_flags(action_t * action, enum pe_action_flags flags)
	{
	static unsigned long calls = 0;
	gboolean changed = FALSE;
	gboolean clear = is_set(flags, pe_action_clear);
	enum pe_action_flags last = action->flags;

	if (clear) {
	pe_clear_action_bit(action, flags);
	} else {
	pe_set_action_bit(action, flags);
	}

	if (last != action->flags) {
	calls++;
	changed = TRUE;
	/* Useful for tracking down _who_ changed a specific flag */
	/* CRM_ASSERT(calls != 534); */
	clear_bit(flags, pe_action_clear);
	crm_trace("%s on %s: %sset flags 0x%.6x (was 0x%.6x, now 0x%.6x, %lu)",
	action->uuid, action->node ? action->node->details->uname : "[none]",
	clear ? "un-" : "", flags, last, action->flags, calls);
	}

	return changed;
	}

	static gboolean
	check_rsc_parameters(resource_t * rsc, node_t * node, xmlNode * rsc_entry,
	gboolean active_here, pe_working_set_t * data_set)
	{
	int attr_lpc = 0;
	gboolean force_restart = FALSE;
	gboolean delete_resource = FALSE;
	gboolean changed = FALSE;

	const char *value = NULL;
	const char *old_value = NULL;

	const char *attr_list[] = {
	XML_ATTR_TYPE,
	XML_AGENT_ATTR_CLASS,
	XML_AGENT_ATTR_PROVIDER
	};

	for (; attr_lpc < DIMOF(attr_list); attr_lpc++) {
	value = crm_element_value(rsc->xml, attr_list[attr_lpc]);
	old_value = crm_element_value(rsc_entry, attr_list[attr_lpc]);
	if (value == old_value /* ie. NULL */
	\|\| crm_str_eq(value, old_value, TRUE)) {
	continue;
	}

	changed = TRUE;
	trigger_unfencing(rsc, node, "Device definition changed", NULL, data_set);
	if (active_here) {
	force_restart = TRUE;
	crm_notice("Forcing restart of %s on %s, %s changed: %s -> %s",
	rsc->id, node->details->uname, attr_list[attr_lpc],
	crm_str(old_value), crm_str(value));
	}
	}
	if (force_restart) {
	/* make sure the restart happens */
	stop_action(rsc, node, FALSE);
	set_bit(rsc->flags, pe_rsc_start_pending);
	delete_resource = TRUE;

	} else if (changed) {
	delete_resource = TRUE;
	}
	return delete_resource;
	}

	static void
	CancelXmlOp(resource_t * rsc, xmlNode * xml_op, node_t * active_node,
	const char reason, pe_working_set_t data_set)
	{
	int interval = 0;
	action_t *cancel = NULL;

	char *key = NULL;
	const char *task = NULL;
	const char *call_id = NULL;
	const char *interval_s = NULL;

	CRM_CHECK(xml_op != NULL, return);
	CRM_CHECK(active_node != NULL, return);

	task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
	interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);

	interval = crm_parse_int(interval_s, "0");

	/* we need to reconstruct the key because of the way we used to construct resource IDs */
	key = generate_op_key(rsc->id, task, interval);

	crm_info("Action %s on %s will be stopped: %s",
	key, active_node->details->uname, reason ? reason : "unknown");

	/* TODO: This looks highly dangerous if we ever try to schedule 'key' too */
	cancel = custom_action(rsc, strdup(key), RSC_CANCEL, active_node, FALSE, TRUE, data_set);

	free(cancel->task);
	free(cancel->cancel_task);
	cancel->task = strdup(RSC_CANCEL);
	cancel->cancel_task = strdup(task);

	add_hash_param(cancel->meta, XML_LRM_ATTR_TASK, task);
	add_hash_param(cancel->meta, XML_LRM_ATTR_CALLID, call_id);
	add_hash_param(cancel->meta, XML_LRM_ATTR_INTERVAL, interval_s);

	custom_action_order(rsc, stop_key(rsc), NULL, rsc, NULL, cancel, pe_order_optional, data_set);
	free(key);
	key = NULL;
	}

	static gboolean
	check_action_definition(resource_t * rsc, node_t * active_node, xmlNode * xml_op,
	pe_working_set_t * data_set)
	{
	char *key = NULL;
	int interval = 0;
	const char *interval_s = NULL;
	const op_digest_cache_t *digest_data = NULL;
	gboolean did_change = FALSE;

	const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
	const char *op_version;
	const char *digest_secure = NULL;

	CRM_CHECK(active_node != NULL, return FALSE);
	if (safe_str_eq(task, RSC_STOP)) {
	return FALSE;
	}

	interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
	interval = crm_parse_int(interval_s, "0");

	if (interval > 0) {
	xmlNode *op_match = NULL;

	/* we need to reconstruct the key because of the way we used to construct resource IDs */
	key = generate_op_key(rsc->id, task, interval);

	pe_rsc_trace(rsc, "Checking parameters for %s", key);
	op_match = find_rsc_op_entry(rsc, key);

	if (op_match == NULL && is_set(data_set->flags, pe_flag_stop_action_orphans)) {
	CancelXmlOp(rsc, xml_op, active_node, "orphan", data_set);
	free(key);
	return TRUE;

	} else if (op_match == NULL) {
	pe_rsc_debug(rsc, "Orphan action detected: %s on %s", key, active_node->details->uname);
	free(key);
	return TRUE;
	}
	free(key);
	key = NULL;
	}

	crm_trace("Testing %s_%s_%d on %s", rsc->id, task, interval, active_node?active_node->details->uname:"N/A");
	if (interval == 0 && safe_str_eq(task, RSC_STATUS)) {
	/* Reload based on the start action not a probe */
	task = RSC_START;

	} else if (interval == 0 && safe_str_eq(task, RSC_MIGRATED)) {
	/* Reload based on the start action not a migrate */
	task = RSC_START;
	} else if (interval == 0 && safe_str_eq(task, RSC_PROMOTE)) {
	/* Reload based on the start action not a promote */
	task = RSC_START;
	}

	op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION);
	digest_data = rsc_action_digest_cmp(rsc, xml_op, active_node, data_set);

	if(is_set(data_set->flags, pe_flag_sanitized)) {
	digest_secure = crm_element_value(xml_op, XML_LRM_ATTR_SECURE_DIGEST);
	}

	if(digest_data->rc != RSC_DIGEST_MATCH
	&& digest_secure
	&& digest_data->digest_secure_calc
	&& strcmp(digest_data->digest_secure_calc, digest_secure) == 0) {
	fprintf(stdout, "Only 'private' parameters to %s_%s_%d on %s changed: %s\n",
	rsc->id, task, interval, active_node->details->uname,
	crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));

	} else if (digest_data->rc == RSC_DIGEST_RESTART) {
	/* Changes that force a restart */
	const char *digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);

	did_change = TRUE;
	key = generate_op_key(rsc->id, task, interval);
	crm_log_xml_info(digest_data->params_restart, "params:restart");
	pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (restart:%s) %s",
	key, active_node->details->uname,
	crm_str(digest_restart), digest_data->digest_restart_calc,
	op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));

	custom_action(rsc, key, task, NULL, FALSE, TRUE, data_set);
	trigger_unfencing(rsc, NULL, "Device parameters changed", NULL, data_set);

	} else if ((digest_data->rc == RSC_DIGEST_ALL) \|\| (digest_data->rc == RSC_DIGEST_UNKNOWN)) {
	/* Changes that can potentially be handled by a reload */
	const char *digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);
	const char *digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST);

	did_change = TRUE;
	trigger_unfencing(rsc, NULL, "Device parameters changed (reload)", NULL, data_set);
	crm_log_xml_info(digest_data->params_all, "params:reload");
	key = generate_op_key(rsc->id, task, interval);
	pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (reload:%s) %s",
	key, active_node->details->uname,
	crm_str(digest_all), digest_data->digest_all_calc, op_version,
	crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));

	if (interval > 0) {
	action_t *op = NULL;

	#if 0
	/* Always reload/restart the entire resource */
	op = custom_action(rsc, start_key(rsc), RSC_START, NULL, FALSE, TRUE, data_set);
	update_action_flags(op, pe_action_allow_reload_conversion);
	#else
	/* Re-sending the recurring op is sufficient - the old one will be cancelled automatically */
	op = custom_action(rsc, key, task, active_node, TRUE, TRUE, data_set);
	set_bit(op->flags, pe_action_reschedule);
	#endif

	} else if (digest_restart && rsc->isolation_wrapper == NULL && (uber_parent(rsc))->isolation_wrapper == NULL) {
	pe_rsc_trace(rsc, "Reloading '%s' action for resource %s", task, rsc->id);

	/* Allow this resource to reload - unless something else causes a full restart */
	set_bit(rsc->flags, pe_rsc_try_reload);

	/* Create these for now, it keeps the action IDs the same in the regression outputs */
	custom_action(rsc, key, task, NULL, TRUE, TRUE, data_set);

	} else {
	pe_rsc_trace(rsc, "Resource %s doesn't know how to reload", rsc->id);

	/* Re-send the start/demote/promote op
	* Recurring ops will be detected independently
	*/
	custom_action(rsc, key, task, NULL, FALSE, TRUE, data_set);
	}
	}

	return did_change;
	}

	extern gboolean DeleteRsc(resource_t * rsc, node_t * node, gboolean optional,
	pe_working_set_t * data_set);

	static void
	check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	int offset = -1;
	int interval = 0;
	int stop_index = 0;
	int start_index = 0;

	const char *task = NULL;
	const char *interval_s = NULL;

	xmlNode *rsc_op = NULL;
	GListPtr op_list = NULL;
	GListPtr sorted_op_list = NULL;
	gboolean is_probe = FALSE;
	gboolean did_change = FALSE;

	CRM_CHECK(node != NULL, return);

	if (is_set(rsc->flags, pe_rsc_orphan)) {
	resource_t *parent = uber_parent(rsc);
	if(parent == NULL
	\|\| parent->variant < pe_clone
	\|\| is_set(parent->flags, pe_rsc_unique)) {
	pe_rsc_trace(rsc, "Skipping param check for %s and deleting: orphan", rsc->id);
	DeleteRsc(rsc, node, FALSE, data_set);
	} else {
	pe_rsc_trace(rsc, "Skipping param check for %s (orphan clone)", rsc->id);
	}
	return;

	} else if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
	if (check_rsc_parameters(rsc, node, rsc_entry, FALSE, data_set)) {
	DeleteRsc(rsc, node, FALSE, data_set);
	}
	pe_rsc_trace(rsc, "Skipping param check for %s: no longer active on %s",
	rsc->id, node->details->uname);
	return;
	}

	pe_rsc_trace(rsc, "Processing %s on %s", rsc->id, node->details->uname);

	if (check_rsc_parameters(rsc, node, rsc_entry, TRUE, data_set)) {
	DeleteRsc(rsc, node, FALSE, data_set);
	}

	for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
	if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
	op_list = g_list_prepend(op_list, rsc_op);
	}
	}

	sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
	calculate_active_ops(sorted_op_list, &start_index, &stop_index);

	for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
	xmlNode rsc_op = (xmlNode ) gIter->data;

	offset++;

	if (start_index < stop_index) {
	/* stopped */
	continue;
	} else if (offset < start_index) {
	/* action occurred prior to a start */
	continue;
	}

	is_probe = FALSE;
	did_change = FALSE;
	task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);

	interval_s = crm_element_value(rsc_op, XML_LRM_ATTR_INTERVAL);
	interval = crm_parse_int(interval_s, "0");

	if (interval == 0 && safe_str_eq(task, RSC_STATUS)) {
	is_probe = TRUE;
	}

	if (interval > 0 &&
	(is_set(rsc->flags, pe_rsc_maintenance) \|\| node->details->maintenance)) {
	CancelXmlOp(rsc, rsc_op, node, "maintenance mode", data_set);

	} else if (is_probe \|\| safe_str_eq(task, RSC_START) \|\| safe_str_eq(task, RSC_PROMOTE) \|\| interval > 0
	\|\| safe_str_eq(task, RSC_MIGRATED)) {
	did_change = check_action_definition(rsc, node, rsc_op, data_set);
	}

	if (did_change && get_failcount(node, rsc, NULL, data_set)) {
	char *key = NULL;
	action_t *action_clear = NULL;

	key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
	action_clear =
	custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE, data_set);
	set_bit(action_clear->flags, pe_action_runnable);
	}
	}

	g_list_free(sorted_op_list);

	}

	static GListPtr
	find_rsc_list(GListPtr result, resource_t * rsc, const char *id, gboolean renamed_clones,
	gboolean partial, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	gboolean match = FALSE;

	if (id == NULL) {
	return NULL;

	} else if (rsc == NULL && data_set) {

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
	}

	return result;

	} else if (rsc == NULL) {
	return NULL;
	}

	if (partial) {
	if (strstr(rsc->id, id)) {
	match = TRUE;

	} else if (renamed_clones && rsc->clone_name && strstr(rsc->clone_name, id)) {
	match = TRUE;
	}

	} else {
	if (strcmp(rsc->id, id) == 0) {
	match = TRUE;

	} else if (renamed_clones && rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
	match = TRUE;
	}
	}

	if (match) {
	result = g_list_prepend(result, rsc);
	}

	if (rsc->children) {
	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
	}
	}

	return result;
	}

	static void
	check_actions(pe_working_set_t * data_set)
	{
	const char *id = NULL;
	node_t *node = NULL;
	xmlNode *lrm_rscs = NULL;
	xmlNode *status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);

	xmlNode *node_state = NULL;

	for (node_state = __xml_first_child(status); node_state != NULL;
	node_state = __xml_next_element(node_state)) {
	if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
	id = crm_element_value(node_state, XML_ATTR_ID);
	lrm_rscs = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
	lrm_rscs = find_xml_node(lrm_rscs, XML_LRM_TAG_RESOURCES, FALSE);

	node = pe_find_node_id(data_set->nodes, id);

	if (node == NULL) {
	continue;

	/* Still need to check actions for a maintenance node to cancel existing monitor operations */
	} else if (can_run_resources(node) == FALSE && node->details->maintenance == FALSE) {
	- crm_trace("Skipping param check for %s: cant run resources", node->details->uname);
	+ crm_trace("Skipping param check for %s: can't run resources",
	+ node->details->uname);
	continue;
	}

	crm_trace("Processing node %s", node->details->uname);
	if (node->details->online \|\| is_set(data_set->flags, pe_flag_stonith_enabled)) {
	xmlNode *rsc_entry = NULL;

	for (rsc_entry = __xml_first_child(lrm_rscs); rsc_entry != NULL;
	rsc_entry = __xml_next_element(rsc_entry)) {
	if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {

	if (xml_has_children(rsc_entry)) {
	GListPtr gIter = NULL;
	GListPtr result = NULL;
	const char *rsc_id = ID(rsc_entry);

	CRM_CHECK(rsc_id != NULL, return);

	result = find_rsc_list(NULL, NULL, rsc_id, TRUE, FALSE, data_set);
	for (gIter = result; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;

	if (rsc->variant != pe_native) {
	continue;
	}
	check_actions_for(rsc_entry, rsc, node, data_set);
	}
	g_list_free(result);
	}
	}
	}
	}
	}
	}
	}

	static gboolean
	apply_placement_constraints(pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	crm_trace("Applying constraints...");

	for (gIter = data_set->placement_constraints; gIter != NULL; gIter = gIter->next) {
	rsc_to_node_t cons = (rsc_to_node_t ) gIter->data;

	cons->rsc_lh->cmds->rsc_location(cons->rsc_lh, cons);
	}

	return TRUE;

	}

	static gboolean
	failcount_clear_action_exists(node_t * node, resource_t * rsc)
	{
	gboolean rc = FALSE;
	char *key = crm_concat(rsc->id, CRM_OP_CLEAR_FAILCOUNT, '_');
	GListPtr list = find_actions_exact(rsc->actions, key, node);

	if (list) {
	rc = TRUE;
	}
	g_list_free(list);
	free(key);

	return rc;
	}

	static void
	common_apply_stickiness(resource_t * rsc, node_t * node, pe_working_set_t * data_set)
	{
	int fail_count = 0;
	resource_t *failed = rsc;

	if (rsc->children) {
	GListPtr gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	common_apply_stickiness(child_rsc, node, data_set);
	}
	return;
	}

	if (is_set(rsc->flags, pe_rsc_managed)
	&& rsc->stickiness != 0 && g_list_length(rsc->running_on) == 1) {
	node_t *current = pe_find_node_id(rsc->running_on, node->details->id);
	node_t *match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);

	if (current == NULL) {

	} else if (match != NULL \|\| is_set(data_set->flags, pe_flag_symmetric_cluster)) {
	resource_t *sticky_rsc = rsc;

	resource_location(sticky_rsc, node, rsc->stickiness, "stickiness", data_set);
	pe_rsc_debug(sticky_rsc, "Resource %s: preferring current location"
	" (node=%s, weight=%d)", sticky_rsc->id,
	node->details->uname, rsc->stickiness);
	} else {
	GHashTableIter iter;
	node_t *nIter = NULL;

	pe_rsc_debug(rsc, "Ignoring stickiness for %s: the cluster is asymmetric"
	" and node %s is not explicitly allowed", rsc->id, node->details->uname);
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&nIter)) {
	crm_err("%s[%s] = %d", rsc->id, nIter->details->uname, nIter->weight);
	}
	}
	}

	/* only check failcount here if a failcount clear action
	* has not already been placed for this resource on the node.
	* There is no sense in potentially forcing the rsc from this
	* node if the failcount is being reset anyway. */
	if (failcount_clear_action_exists(node, rsc) == FALSE) {
	fail_count = get_failcount_all(node, rsc, NULL, data_set);
	}

	if (fail_count > 0 && rsc->migration_threshold != 0) {
	if (is_not_set(rsc->flags, pe_rsc_unique)) {
	failed = uber_parent(rsc);
	}
	if (rsc->migration_threshold <= fail_count) {
	resource_location(failed, node, -INFINITY, "__fail_limit__", data_set);
	crm_warn("Forcing %s away from %s after %d failures (max=%d)",
	failed->id, node->details->uname, fail_count, rsc->migration_threshold);
	} else {
	crm_info("%s can fail %d more times on %s before being forced off",
	failed->id, rsc->migration_threshold - fail_count, node->details->uname);
	}
	}
	}

	static void
	complex_set_cmds(resource_t * rsc)
	{
	GListPtr gIter = rsc->children;

	rsc->cmds = &resource_class_alloc_functions[rsc->variant];

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	complex_set_cmds(child_rsc);
	}
	}

	void
	set_alloc_actions(pe_working_set_t * data_set)
	{

	GListPtr gIter = data_set->resources;

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;

	complex_set_cmds(rsc);
	}
	}

	static void
	calculate_system_health(gpointer gKey, gpointer gValue, gpointer user_data)
	{
	const char key = (const char )gKey;
	const char value = (const char )gValue;
	int system_health = (int )user_data;

	if (!gKey \|\| !gValue \|\| !user_data) {
	return;
	}

	/* Does it start with #health? */
	if (0 == strncmp(key, "#health", 7)) {
	int score;

	/* Convert the value into an integer */
	score = char2score(value);

	/* Add it to the running total */
	system_health = merge_weights(score, system_health);
	}
	}

	static gboolean
	apply_system_health(pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	const char *health_strategy = pe_pref(data_set->config_hash, "node-health-strategy");

	if (health_strategy == NULL \|\| safe_str_eq(health_strategy, "none")) {
	/* Prevent any accidental health -> score translation */
	node_score_red = 0;
	node_score_yellow = 0;
	node_score_green = 0;
	return TRUE;

	} else if (safe_str_eq(health_strategy, "migrate-on-red")) {

	/* Resources on nodes which have health values of red are
	* weighted away from that node.
	*/
	node_score_red = -INFINITY;
	node_score_yellow = 0;
	node_score_green = 0;

	} else if (safe_str_eq(health_strategy, "only-green")) {

	/* Resources on nodes which have health values of red or yellow
	* are forced away from that node.
	*/
	node_score_red = -INFINITY;
	node_score_yellow = -INFINITY;
	node_score_green = 0;

	} else if (safe_str_eq(health_strategy, "progressive")) {
	/* Same as the above, but use the r/y/g scores provided by the user
	* Defaults are provided by the pe_prefs table
	*/

	} else if (safe_str_eq(health_strategy, "custom")) {

	/* Requires the admin to configure the rsc_location constaints for
	* processing the stored health scores
	*/
	/* TODO: Check for the existance of appropriate node health constraints */
	return TRUE;

	} else {
	crm_err("Unknown node health strategy: %s", health_strategy);
	return FALSE;
	}

	crm_info("Applying automated node health strategy: %s", health_strategy);

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	int system_health = 0;
	node_t node = (node_t ) gIter->data;

	/* Search through the node hash table for system health entries. */
	g_hash_table_foreach(node->details->attrs, calculate_system_health, &system_health);

	crm_info(" Node %s has an combined system health of %d",
	node->details->uname, system_health);

	/* If the health is non-zero, then create a new rsc2node so that the
	* weight will be added later on.
	*/
	if (system_health != 0) {

	GListPtr gIter2 = data_set->resources;

	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	resource_t rsc = (resource_t ) gIter2->data;

	rsc2node_new(health_strategy, rsc, system_health, NULL, node, data_set);
	}
	}

	}

	return TRUE;
	}

	gboolean
	stage0(pe_working_set_t * data_set)
	{
	xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);

	if (data_set->input == NULL) {
	return FALSE;
	}

	if (is_set(data_set->flags, pe_flag_have_status) == FALSE) {
	crm_trace("Calculating status");
	cluster_status(data_set);
	}

	set_alloc_actions(data_set);
	apply_system_health(data_set);
	unpack_constraints(cib_constraints, data_set);

	return TRUE;
	}

	/*
	* Check nodes for resources started outside of the LRM
	*/
	gboolean
	probe_resources(pe_working_set_t * data_set)
	{
	action_t *probe_node_complete = NULL;

	GListPtr gIter = NULL;
	GListPtr gIter2 = NULL;

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;
	const char *probed = g_hash_table_lookup(node->details->attrs, CRM_OP_PROBED);

	if (node->details->online == FALSE) {
	continue;

	} else if (node->details->unclean) {
	continue;

	} else if (is_remote_node(node) && node->details->shutdown) {
	/* Don't try and probe a remote node we're shutting down.
	* It causes constraint conflicts to try and run any sort of action
	* other that 'stop' on resources living within a remote-node when
	* it is being shutdown. */
	continue;

	} else if (is_container_remote_node(node)) {
	/* TODO enable container node probes once ordered probing is implemented. */
	continue;

	} else if (node->details->rsc_discovery_enabled == FALSE) {
	/* resource discovery is disabled for this node */
	continue;
	}

	if (probed != NULL && crm_is_true(probed) == FALSE) {
	action_t *probe_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_REPROBE, node->details->uname),
	CRM_OP_REPROBE, node, FALSE, TRUE, data_set);

	add_hash_param(probe_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
	continue;
	}

	for (gIter2 = data_set->resources; gIter2 != NULL; gIter2 = gIter2->next) {
	resource_t rsc = (resource_t ) gIter2->data;

	rsc->cmds->create_probe(rsc, node, probe_node_complete, FALSE, data_set);
	}
	}
	return TRUE;
	}

	static void
	rsc_discover_filter(resource_t rsc, node_t node)
	{
	GListPtr gIter = rsc->children;
	resource_t *top = uber_parent(rsc);
	node_t *match;

	if (rsc->exclusive_discover == FALSE && top->exclusive_discover == FALSE) {
	return;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;
	rsc_discover_filter(child_rsc, node);
	}

	match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (match && match->rsc_discover_mode != discover_exclusive) {
	match->weight = -INFINITY;
	}
	}

	/*
	* Count how many valid nodes we have (so we know the maximum number of
	* colors we can resolve).
	*
	* Apply node constraints (ie. filter the "allowed_nodes" part of resources
	*/
	gboolean
	stage2(pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	crm_trace("Applying placement constraints");

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	if (node == NULL) {
	/* error */

	} else if (node->weight >= 0.0 /* global weight */
	&& node->details->online && node->details->type != node_ping) {
	data_set->max_valid_nodes++;
	}
	}

	apply_placement_constraints(data_set);

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	GListPtr gIter2 = NULL;
	node_t node = (node_t ) gIter->data;

	gIter2 = data_set->resources;
	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	resource_t rsc = (resource_t ) gIter2->data;

	common_apply_stickiness(rsc, node, data_set);
	rsc_discover_filter(rsc, node);
	}
	}

	return TRUE;
	}

	/*
	* Create internal resource constraints before allocation
	*/
	gboolean
	stage3(pe_working_set_t * data_set)
	{

	GListPtr gIter = data_set->resources;

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;

	rsc->cmds->internal_constraints(rsc, data_set);
	}

	return TRUE;
	}

	/*
	* Check for orphaned or redefined actions
	*/
	gboolean
	stage4(pe_working_set_t * data_set)
	{
	check_actions(data_set);
	return TRUE;
	}

	static gint
	sort_rsc_process_order(gconstpointer a, gconstpointer b, gpointer data)
	{
	int rc = 0;
	int r1_weight = -INFINITY;
	int r2_weight = -INFINITY;

	const char *reason = "existance";

	const GListPtr nodes = (GListPtr) data;
	resource_t resource1 = (resource_t ) convert_const_pointer(a);
	resource_t resource2 = (resource_t ) convert_const_pointer(b);

	node_t *r1_node = NULL;
	node_t *r2_node = NULL;
	GListPtr gIter = NULL;
	GHashTable *r1_nodes = NULL;
	GHashTable *r2_nodes = NULL;

	if (a == NULL && b == NULL) {
	goto done;
	}
	if (a == NULL) {
	return 1;
	}
	if (b == NULL) {
	return -1;
	}

	reason = "priority";
	r1_weight = resource1->priority;
	r2_weight = resource2->priority;

	if (r1_weight > r2_weight) {
	rc = -1;
	goto done;
	}

	if (r1_weight < r2_weight) {
	rc = 1;
	goto done;
	}

	reason = "no node list";
	if (nodes == NULL) {
	goto done;
	}

	r1_nodes =
	rsc_merge_weights(resource1, resource1->id, NULL, NULL, 1,
	pe_weights_forward \| pe_weights_init);
	dump_node_scores(LOG_TRACE, NULL, resource1->id, r1_nodes);
	r2_nodes =
	rsc_merge_weights(resource2, resource2->id, NULL, NULL, 1,
	pe_weights_forward \| pe_weights_init);
	dump_node_scores(LOG_TRACE, NULL, resource2->id, r2_nodes);

	/* Current location score */
	reason = "current location";
	r1_weight = -INFINITY;
	r2_weight = -INFINITY;

	if (resource1->running_on) {
	r1_node = g_list_nth_data(resource1->running_on, 0);
	r1_node = g_hash_table_lookup(r1_nodes, r1_node->details->id);
	if (r1_node != NULL) {
	r1_weight = r1_node->weight;
	}
	}
	if (resource2->running_on) {
	r2_node = g_list_nth_data(resource2->running_on, 0);
	r2_node = g_hash_table_lookup(r2_nodes, r2_node->details->id);
	if (r2_node != NULL) {
	r2_weight = r2_node->weight;
	}
	}

	if (r1_weight > r2_weight) {
	rc = -1;
	goto done;
	}

	if (r1_weight < r2_weight) {
	rc = 1;
	goto done;
	}

	reason = "score";
	for (gIter = nodes; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	r1_node = NULL;
	r2_node = NULL;

	r1_weight = -INFINITY;
	if (r1_nodes) {
	r1_node = g_hash_table_lookup(r1_nodes, node->details->id);
	}
	if (r1_node) {
	r1_weight = r1_node->weight;
	}

	r2_weight = -INFINITY;
	if (r2_nodes) {
	r2_node = g_hash_table_lookup(r2_nodes, node->details->id);
	}
	if (r2_node) {
	r2_weight = r2_node->weight;
	}

	if (r1_weight > r2_weight) {
	rc = -1;
	goto done;
	}

	if (r1_weight < r2_weight) {
	rc = 1;
	goto done;
	}
	}

	done:
	crm_trace("%s (%d) on %s %c %s (%d) on %s: %s",
	resource1->id, r1_weight, r1_node ? r1_node->details->id : "n/a",
	rc < 0 ? '>' : rc > 0 ? '<' : '=',
	resource2->id, r2_weight, r2_node ? r2_node->details->id : "n/a", reason);

	if (r1_nodes) {
	g_hash_table_destroy(r1_nodes);
	}
	if (r2_nodes) {
	g_hash_table_destroy(r2_nodes);
	}

	return rc;
	}

	static void
	allocate_resources(pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	if (is_set(data_set->flags, pe_flag_have_remote_nodes)) {
	/* Force remote connection resources to be allocated first. This
	* also forces any colocation dependencies to be allocated as well */
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;
	if (rsc->is_remote_node == FALSE) {
	continue;
	}
	pe_rsc_trace(rsc, "Allocating: %s", rsc->id);
	/* for remote node connection resources, always prefer the partial migration
	* target during resource allocation if the rsc is in the middle of a
	* migration */
	rsc->cmds->allocate(rsc, rsc->partial_migration_target, data_set);
	}
	}

	/* now do the rest of the resources */
	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;
	if (rsc->is_remote_node == TRUE) {
	continue;
	}
	pe_rsc_trace(rsc, "Allocating: %s", rsc->id);
	rsc->cmds->allocate(rsc, NULL, data_set);
	}
	}

	static void
	cleanup_orphans(resource_t * rsc, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	if (is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
	return;
	}

	/* Don't recurse into ->children, those are just unallocated clone instances */
	if(is_not_set(rsc->flags, pe_rsc_orphan)) {
	return;
	}

	for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	if (node->details->online && get_failcount(node, rsc, NULL, data_set)) {
	action_t *clear_op = NULL;

	clear_op = custom_action(rsc, crm_concat(rsc->id, CRM_OP_CLEAR_FAILCOUNT, '_'),
	CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE, data_set);

	add_hash_param(clear_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
	pe_rsc_info(rsc, "Clearing failcount (%d) for orphaned resource %s on %s (%s)",
	get_failcount(node, rsc, NULL, data_set), rsc->id, node->details->uname,
	clear_op->uuid);

	custom_action_order(rsc, NULL, clear_op,
	rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL,
	pe_order_optional, data_set);
	}
	}
	}

	gboolean
	stage5(pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	if (safe_str_neq(data_set->placement_strategy, "default")) {
	GListPtr nodes = g_list_copy(data_set->nodes);

	nodes = g_list_sort_with_data(nodes, sort_node_weight, NULL);

	data_set->resources =
	g_list_sort_with_data(data_set->resources, sort_rsc_process_order, nodes);

	g_list_free(nodes);
	}

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	dump_node_capacity(show_utilization ? 0 : utilization_log_level, "Original", node);
	}

	crm_trace("Allocating services");
	/* Take (next) highest resource, assign it and create its actions */

	allocate_resources(data_set);

	gIter = data_set->nodes;
	for (; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	dump_node_capacity(show_utilization ? 0 : utilization_log_level, "Remaining", node);
	}

	if (is_set(data_set->flags, pe_flag_startup_probes)) {
	crm_trace("Calculating needed probes");
	/* This code probably needs optimization
	* ptest -x with 100 nodes, 100 clones and clone-max=100:

	With probes:

	ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
	ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
	ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
	ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:292 Check actions
	ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
	ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: stage5: allocate.c:881 Allocating services
	ptest[14781]: 2010/09/27_17:56:49 notice: TRACE: stage5: allocate.c:894 Calculating needed probes
	ptest[14781]: 2010/09/27_17:56:51 notice: TRACE: stage5: allocate.c:899 Creating actions
	ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: stage5: allocate.c:905 Creating done
	ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
	ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
	36s
	ptest[14781]: 2010/09/27_17:57:28 notice: TRACE: do_calculations: pengine.c:320 Create transition graph

	Without probes:

	ptest[14637]: 2010/09/27_17:56:21 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
	ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
	ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
	ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:292 Check actions
	ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
	ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: stage5: allocate.c:881 Allocating services
	ptest[14637]: 2010/09/27_17:56:24 notice: TRACE: stage5: allocate.c:899 Creating actions
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: stage5: allocate.c:905 Creating done
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
	ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
	*/

	probe_resources(data_set);
	}

	crm_trace("Handle orphans");

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;
	cleanup_orphans(rsc, data_set);
	}

	crm_trace("Creating actions");

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;

	rsc->cmds->create_actions(rsc, data_set);
	}

	crm_trace("Creating done");
	return TRUE;
	}

	static gboolean
	is_managed(const resource_t * rsc)
	{
	GListPtr gIter = rsc->children;

	if (is_set(rsc->flags, pe_rsc_managed)) {
	return TRUE;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	if (is_managed(child_rsc)) {
	return TRUE;
	}
	}

	return FALSE;
	}

	static gboolean
	any_managed_resources(pe_working_set_t * data_set)
	{

	GListPtr gIter = data_set->resources;

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;

	if (is_managed(rsc)) {
	return TRUE;
	}
	}
	return FALSE;
	}

	/*
	* Create dependencies for stonith and shutdown operations
	*/
	gboolean
	stage6(pe_working_set_t * data_set)
	{
	action_t *dc_down = NULL;
	action_t *dc_fence = NULL;
	action_t *stonith_op = NULL;
	action_t *last_stonith = NULL;
	gboolean integrity_lost = FALSE;
	action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);
	action_t *done = get_pseudo_op(STONITH_DONE, data_set);
	gboolean need_stonith = TRUE;
	GListPtr gIter = data_set->nodes;

	crm_trace("Processing fencing and shutdown cases");

	if (any_managed_resources(data_set) == FALSE) {
	crm_notice("Delaying fencing operations until there are resources to manage");
	need_stonith = FALSE;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	/* remote-nodes associated with a container resource (such as a vm) are not fenced */
	if (is_container_remote_node(node)) {
	/* Guest */
	if (need_stonith
	&& node->details->remote_requires_reset
	&& pe_can_fence(data_set, node)) {
	resource_t *container = node->details->remote_rsc->container;
	char *key = stop_key(container);
	GListPtr stop_list = find_actions(container->actions, key, NULL);

	- crm_info("Impliying node %s is down when container %s is stopped (%p)",
	+ crm_info("Implying node %s is down when container %s is stopped (%p)",
	node->details->uname, container->id, stop_list);
	if(stop_list) {
	stonith_constraints(node, stop_list->data, data_set);
	}

	g_list_free(stop_list);
	free(key);
	}
	continue;
	}

	stonith_op = NULL;
	if (need_stonith && node->details->unclean && pe_can_fence(data_set, node)) {
	pe_warn("Scheduling Node %s for STONITH", node->details->uname);

	stonith_op = pe_fence_op(node, NULL, FALSE, data_set);

	stonith_constraints(node, stonith_op, data_set);

	if (node->details->is_dc) {
	dc_down = stonith_op;
	dc_fence = stonith_op;

	} else {
	if (last_stonith) {
	order_actions(last_stonith, stonith_op, pe_order_optional);
	}
	last_stonith = stonith_op;
	}

	} else if (node->details->online && node->details->shutdown &&
	/* TODO define what a shutdown op means for a baremetal remote node.
	* For now we do not send shutdown operations for remote nodes, but
	* if we can come up with a good use for this in the future, we will. */
	is_remote_node(node) == FALSE) {

	action_t *down_op = NULL;

	crm_notice("Scheduling Node %s for shutdown", node->details->uname);

	down_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_SHUTDOWN, node->details->uname),
	CRM_OP_SHUTDOWN, node, FALSE, TRUE, data_set);

	shutdown_constraints(node, down_op, data_set);
	add_hash_param(down_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);

	if (node->details->is_dc) {
	dc_down = down_op;
	}
	}

	if (node->details->unclean && stonith_op == NULL) {
	integrity_lost = TRUE;
	pe_warn("Node %s is unclean!", node->details->uname);
	}
	}

	if (integrity_lost) {
	if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
	pe_warn("YOUR RESOURCES ARE NOW LIKELY COMPROMISED");
	pe_err("ENABLE STONITH TO KEEP YOUR RESOURCES SAFE");

	} else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE) {
	crm_notice("Cannot fence unclean nodes until quorum is"
	" attained (or no-quorum-policy is set to ignore)");
	}
	}

	if (dc_down != NULL) {
	GListPtr gIter = NULL;

	crm_trace("Ordering shutdowns before %s on %s (DC)",
	dc_down->task, dc_down->node->details->uname);

	add_hash_param(dc_down->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);

	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	action_t node_stop = (action_t ) gIter->data;

	if (safe_str_neq(CRM_OP_SHUTDOWN, node_stop->task)) {
	continue;
	} else if (node_stop->node->details->is_dc) {
	continue;
	}

	crm_debug("Ordering shutdown on %s before %s on %s",
	node_stop->node->details->uname,
	dc_down->task, dc_down->node->details->uname);

	order_actions(node_stop, dc_down, pe_order_optional);
	}

	if (last_stonith && dc_down != last_stonith) {
	order_actions(last_stonith, dc_down, pe_order_optional);
	}
	}


	if (dc_fence) {
	order_actions(dc_down, done, pe_order_implies_then);

	} else if (last_stonith) {
	order_actions(last_stonith, done, pe_order_implies_then);
	}

	order_actions(done, all_stopped, pe_order_implies_then);
	return TRUE;
	}

	/*
	- * Determin the sets of independent actions and the correct order for the
	+ * Determine the sets of independent actions and the correct order for the
	* actions in each set.
	*
	* Mark dependencies of un-runnable actions un-runnable
	*
	*/
	static GListPtr
	find_actions_by_task(GListPtr actions, resource_t * rsc, const char *original_key)
	{
	GListPtr list = NULL;

	list = find_actions(actions, original_key, NULL);
	if (list == NULL) {
	/* we're potentially searching a child of the original resource */
	char *key = NULL;
	char *tmp = NULL;
	char *task = NULL;
	int interval = 0;

	if (parse_op_key(original_key, &tmp, &task, &interval)) {
	key = generate_op_key(rsc->id, task, interval);
	/* crm_err("looking up %s instead of %s", key, original_key); */
	/* slist_iter(action, action_t, actions, lpc, */
	/* crm_err(" - %s", action->uuid)); */
	list = find_actions(actions, key, NULL);

	} else {
	crm_err("search key: %s", original_key);
	}

	free(key);
	free(tmp);
	free(task);
	}

	return list;
	}

	static void
	rsc_order_then(action_t * lh_action, resource_t * rsc, order_constraint_t * order)
	{
	GListPtr gIter = NULL;
	GListPtr rh_actions = NULL;
	action_t *rh_action = NULL;
	enum pe_ordering type = order->type;

	CRM_CHECK(rsc != NULL, return);
	CRM_CHECK(order != NULL, return);

	rh_action = order->rh_action;
	crm_trace("Processing RH of ordering constraint %d", order->id);

	if (rh_action != NULL) {
	rh_actions = g_list_prepend(NULL, rh_action);

	} else if (rsc != NULL) {
	rh_actions = find_actions_by_task(rsc->actions, rsc, order->rh_action_task);
	}

	if (rh_actions == NULL) {
	pe_rsc_trace(rsc, "No RH-Side (%s/%s) found for constraint..."
	" ignoring", rsc->id, order->rh_action_task);
	if (lh_action) {
	pe_rsc_trace(rsc, "LH-Side was: %s", lh_action->uuid);
	}
	return;
	}

	if (lh_action && lh_action->rsc == rsc && is_set(lh_action->flags, pe_action_dangle)) {
	pe_rsc_trace(rsc, "Detected dangling operation %s -> %s", lh_action->uuid,
	order->rh_action_task);
	clear_bit(type, pe_order_implies_then);
	}

	gIter = rh_actions;
	for (; gIter != NULL; gIter = gIter->next) {
	action_t rh_action_iter = (action_t ) gIter->data;

	if (lh_action) {
	order_actions(lh_action, rh_action_iter, type);

	} else if (type & pe_order_implies_then) {
	update_action_flags(rh_action_iter, pe_action_runnable \| pe_action_clear);
	crm_warn("Unrunnable %s 0x%.6x", rh_action_iter->uuid, type);
	} else {
	crm_warn("neither %s 0x%.6x", rh_action_iter->uuid, type);
	}
	}

	g_list_free(rh_actions);
	}

	static void
	rsc_order_first(resource_t * lh_rsc, order_constraint_t * order, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	GListPtr lh_actions = NULL;
	action_t *lh_action = order->lh_action;
	resource_t *rh_rsc = order->rh_rsc;

	crm_trace("Processing LH of ordering constraint %d", order->id);
	CRM_ASSERT(lh_rsc != NULL);

	if (lh_action != NULL) {
	lh_actions = g_list_prepend(NULL, lh_action);

	} else if (lh_action == NULL) {
	lh_actions = find_actions_by_task(lh_rsc->actions, lh_rsc, order->lh_action_task);
	}

	if (lh_actions == NULL && lh_rsc != rh_rsc) {
	char *key = NULL;
	char *rsc_id = NULL;
	char *op_type = NULL;
	int interval = 0;

	parse_op_key(order->lh_action_task, &rsc_id, &op_type, &interval);
	key = generate_op_key(lh_rsc->id, op_type, interval);

	if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_STOPPED && safe_str_eq(op_type, RSC_STOP)) {
	free(key);
	pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
	lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);

	} else if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_SLAVE && safe_str_eq(op_type, RSC_DEMOTE)) {
	free(key);
	pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
	lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);

	} else {
	pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - creating",
	lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
	lh_action = custom_action(lh_rsc, key, op_type, NULL, TRUE, TRUE, data_set);
	lh_actions = g_list_prepend(NULL, lh_action);
	}

	free(op_type);
	free(rsc_id);
	}

	gIter = lh_actions;
	for (; gIter != NULL; gIter = gIter->next) {
	action_t lh_action_iter = (action_t ) gIter->data;

	if (rh_rsc == NULL && order->rh_action) {
	rh_rsc = order->rh_action->rsc;
	}
	if (rh_rsc) {
	rsc_order_then(lh_action_iter, rh_rsc, order);

	} else if (order->rh_action) {
	order_actions(lh_action_iter, order->rh_action, order->type);
	}
	}

	g_list_free(lh_actions);
	}

	extern gboolean update_action(action_t * action);
	extern void update_colo_start_chain(action_t * action);

	static void
	apply_remote_node_ordering(pe_working_set_t *data_set)
	{
	GListPtr gIter = data_set->actions;

	if (is_set(data_set->flags, pe_flag_have_remote_nodes) == FALSE) {
	return;
	}
	for (; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;
	resource_t *remote_rsc = NULL;
	resource_t *container = NULL;

	if (action->rsc == NULL) {
	continue;
	}

	/* Special case. */
	if (action->rsc &&
	action->rsc->is_remote_node &&
	safe_str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT)) {

	/* if we are clearing the failcount of an actual remote node connect
	* resource, then make sure this happens before allowing the connection
	* to start if we are planning on starting the connection during this
	* transition */
	custom_action_order(action->rsc,
	NULL,
	action,
	action->rsc,
	generate_op_key(action->rsc->id, RSC_START, 0),
	NULL,
	pe_order_optional,
	data_set);

	continue;
	}

	/* detect if the action occurs on a remote node. if so create
	* ordering constraints that guarantee the action occurs while
	* the remote node is active (after start, before stop...) things
	* like that */
	if (action->node == NULL \|\|
	is_remote_node(action->node) == FALSE \|\|
	action->node->details->remote_rsc == NULL \|\|
	is_set(action->flags, pe_action_pseudo)) {
	continue;
	}

	remote_rsc = action->node->details->remote_rsc;
	container = remote_rsc->container;

	if (safe_str_eq(action->task, "monitor") \|\|
	safe_str_eq(action->task, "start") \|\|
	safe_str_eq(action->task, "promote") \|\|
	safe_str_eq(action->task, "notify") \|\|
	safe_str_eq(action->task, CRM_OP_LRM_REFRESH) \|\|
	safe_str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT) \|\|
	safe_str_eq(action->task, "delete")) {

	custom_action_order(remote_rsc,
	generate_op_key(remote_rsc->id, RSC_START, 0),
	NULL,
	action->rsc,
	NULL,
	action,
	pe_order_preserve \| pe_order_implies_then \| pe_order_runnable_left,
	data_set);

	} else if (safe_str_eq(action->task, "demote")) {

	/* If the connection is being torn down, we don't want
	* to build a constraint between a resource's demotion and
	* the connection resource starting... because the connection
	* resource can not start. The connection might already be up,
	* but the START action would not be allowed which in turn would
	* block the demotion of any resournces living in the remote-node.
	*
	* In this case, only build the constraint between the demotion and
	* the connection's stop action. This allows the connection and all the
	* resources within the remote-node to be torn down properly. */
	if (remote_rsc->next_role == RSC_ROLE_STOPPED) {
	custom_action_order(action->rsc,
	NULL,
	action,
	remote_rsc,
	generate_op_key(remote_rsc->id, RSC_STOP, 0),
	NULL,
	pe_order_preserve \| pe_order_implies_first,
	data_set);
	} else {

	custom_action_order(remote_rsc,
	generate_op_key(remote_rsc->id, RSC_START, 0),
	NULL,
	action->rsc,
	NULL,
	action,
	pe_order_preserve \| pe_order_implies_then \| pe_order_runnable_left,
	data_set);
	}

	} else if (safe_str_eq(action->task, "stop") &&
	container &&
	is_set(container->flags, pe_rsc_failed)) {

	/* when the container representing a remote node fails, the stop
	* action for all the resources living in that container is implied
	* by the container stopping. This is similar to how fencing operations
	* work for cluster nodes. */
	pe_set_action_bit(action, pe_action_pseudo);
	custom_action_order(container,
	generate_op_key(container->id, RSC_STOP, 0),
	NULL,
	action->rsc,
	NULL,
	action,
	pe_order_preserve \| pe_order_implies_then \| pe_order_runnable_left,
	data_set);
	} else if (safe_str_eq(action->task, "stop")) {
	gboolean after_start = FALSE;

	/* handle special case with baremetal remote where stop actions need to be
	* ordered after the connection resource starts somewhere else. */
	if (is_baremetal_remote_node(action->node)) {
	node_t *cluster_node = remote_rsc->running_on ? remote_rsc->running_on->data : NULL;

	/* if the current cluster node a baremetal connection resource
	* is residing on is unclean or went offline we can't process any
	* operations on that remote node until after it starts somewhere else. */
	if (cluster_node == NULL \|\|
	cluster_node->details->unclean == TRUE \|\|
	cluster_node->details->online == FALSE) {
	after_start = TRUE;
	} else if (g_list_length(remote_rsc->running_on) > 1 &&
	remote_rsc->partial_migration_source &&
	remote_rsc->partial_migration_target) {
	/* if we're caught in the middle of migrating a connection resource,
	* then we have to wait until after the resource migrates before performing
	* any actions. */
	after_start = TRUE;
	}
	}

	if (after_start) {
	custom_action_order(remote_rsc,
	generate_op_key(remote_rsc->id, RSC_START, 0),
	NULL,
	action->rsc,
	NULL,
	action,
	pe_order_preserve \| pe_order_implies_then \| pe_order_runnable_left,
	data_set);
	} else {
	custom_action_order(action->rsc,
	NULL,
	action,
	remote_rsc,
	generate_op_key(remote_rsc->id, RSC_STOP, 0),
	NULL,
	pe_order_preserve \| pe_order_implies_first,
	data_set);
	}
	}
	}
	}

	static void
	order_probes(pe_working_set_t * data_set)
	{
	#if 0
	GListPtr gIter = NULL;

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;

	/* Given "A then B", we would prefer to wait for A to be
	* started before probing B.
	*
	* If A was a filesystem on which the binaries and data for B
	* lived, it would have been useful if the author of B's agent
	* could assume that A is running before B.monitor will be
	* called.
	*
	* However we can't _only_ probe once A is running, otherwise
	* we'd not detect the state of B if A could not be started
	* for some reason.
	*
	* In practice however, we cannot even do an opportunistic
	* version of this because B may be moving:
	*
	* B.probe -> B.start
	* B.probe -> B.stop
	* B.stop -> B.start
	* A.stop -> A.start
	* A.start -> B.probe
	*
	* So far so good, but if we add the result of this code:
	*
	* B.stop -> A.stop
	*
	* Then we get a loop:
	*
	* B.probe -> B.stop -> A.stop -> A.start -> B.probe
	*
	* We could kill the 'B.probe -> B.stop' dependency, but that
	* could mean stopping B "too" soon, because B.start must wait
	* for the probes to complete.
	*
	* Another option is to allow it only if A is a non-unique
	* clone with clone-max == node-max (since we'll never be
	* moving it). However, we could still be stopping one
	* instance at the same time as starting another.

	* The complexity of checking for allowed conditions combined
	* with the ever narrowing usecase suggests that this code
	* should remain disabled until someone gets smarter.
	*/
	action_t *start = NULL;
	GListPtr actions = NULL;
	GListPtr probes = NULL;
	char *key = NULL;

	key = start_key(rsc);
	actions = find_actions(rsc->actions, key, NULL);
	free(key);

	if (actions) {
	start = actions->data;
	g_list_free(actions);
	}

	if(start == NULL) {
	crm_err("No start action for %s", rsc->id);
	continue;
	}

	key = generate_op_key(rsc->id, CRMD_ACTION_STATUS, 0);
	probes = find_actions(rsc->actions, key, NULL);
	free(key);

	for (actions = start->actions_before; actions != NULL; actions = actions->next) {
	action_wrapper_t before = (action_wrapper_t ) actions->data;

	GListPtr pIter = NULL;
	action_t *first = before->action;
	resource_t *first_rsc = first->rsc;

	if(first->required_runnable_before) {
	GListPtr clone_actions = NULL;
	for (clone_actions = first->actions_before; clone_actions != NULL; clone_actions = clone_actions->next) {
	before = (action_wrapper_t *) clone_actions->data;

	crm_trace("Testing %s -> %s (%p) for %s", first->uuid, before->action->uuid, before->action->rsc, start->uuid);

	CRM_ASSERT(before->action->rsc);
	first_rsc = before->action->rsc;
	break;
	}

	} else if(safe_str_neq(first->task, RSC_START)) {
	crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
	}

	if(first_rsc == NULL) {
	continue;

	} else if(uber_parent(first_rsc) == uber_parent(start->rsc)) {
	crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
	continue;

	} else if(FALSE && uber_parent(first_rsc)->variant < pe_clone) {
	crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
	continue;
	}

	crm_err("Appplying %s before %s %d", first->uuid, start->uuid, uber_parent(first_rsc)->variant);

	for (pIter = probes; pIter != NULL; pIter = pIter->next) {
	action_t probe = (action_t ) pIter->data;

	crm_err("Ordering %s before %s", first->uuid, probe->uuid);
	order_actions(first, probe, pe_order_optional);
	}
	}
	}
	#endif
	}

	gboolean
	stage7(pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;

	apply_remote_node_ordering(data_set);
	crm_trace("Applying ordering constraints");

	/* Don't ask me why, but apparently they need to be processed in
	* the order they were created in... go figure
	*
	* Also g_list_prepend() has horrendous performance characteristics
	* So we need to use g_list_prepend() and then reverse the list here
	*/
	data_set->ordering_constraints = g_list_reverse(data_set->ordering_constraints);

	for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
	order_constraint_t order = (order_constraint_t ) gIter->data;
	resource_t *rsc = order->lh_rsc;

	crm_trace("Applying ordering constraint: %d", order->id);

	if (rsc != NULL) {
	crm_trace("rsc_action-to-*");
	rsc_order_first(rsc, order, data_set);
	continue;
	}

	rsc = order->rh_rsc;
	if (rsc != NULL) {
	crm_trace("action-to-rsc_action");
	rsc_order_then(order->lh_action, rsc, order);

	} else {
	crm_trace("action-to-action");
	order_actions(order->lh_action, order->rh_action, order->type);
	}
	}

	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	update_colo_start_chain(action);
	}

	crm_trace("Ordering probes");
	order_probes(data_set);

	crm_trace("Updating %d actions", g_list_length(data_set->actions));

	for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	update_action(action);
	}

	crm_trace("Processing reloads");

	for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;

	rsc_reload(rsc, data_set);
	LogActions(rsc, data_set, FALSE);
	}
	return TRUE;
	}

	static gint
	sort_notify_entries(gconstpointer a, gconstpointer b)
	{
	int tmp;
	const notify_entry_t *entry_a = a;
	const notify_entry_t *entry_b = b;

	if (entry_a == NULL && entry_b == NULL) {
	return 0;
	}
	if (entry_a == NULL) {
	return 1;
	}
	if (entry_b == NULL) {
	return -1;
	}

	if (entry_a->rsc == NULL && entry_b->rsc == NULL) {
	return 0;
	}
	if (entry_a->rsc == NULL) {
	return 1;
	}
	if (entry_b->rsc == NULL) {
	return -1;
	}

	tmp = strcmp(entry_a->rsc->id, entry_b->rsc->id);
	if (tmp != 0) {
	return tmp;
	}

	if (entry_a->node == NULL && entry_b->node == NULL) {
	return 0;
	}
	if (entry_a->node == NULL) {
	return 1;
	}
	if (entry_b->node == NULL) {
	return -1;
	}

	return strcmp(entry_a->node->details->id, entry_b->node->details->id);
	}

	static char *
	expand_node_list(GListPtr list)
	{
	GListPtr gIter = NULL;
	char *node_list = NULL;

	if (list == NULL) {
	return strdup(" ");
	}

	for (gIter = list; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	if (node->details->uname) {
	int existing_len = 0;
	int len = 2 + strlen(node->details->uname);

	if(node_list) {
	existing_len = strlen(node_list);
	}
	crm_trace("Adding %s (%dc) at offset %d", node->details->uname, len - 2, existing_len);
	node_list = realloc_safe(node_list, len + existing_len);
	sprintf(node_list + existing_len, "%s%s", existing_len == 0 ? "":" ", node->details->uname);
	}
	}

	return node_list;
	}

	static void
	expand_list(GListPtr list, char rsc_list, char node_list)
	{
	GListPtr gIter = NULL;
	const char *uname = NULL;
	const char *rsc_id = NULL;
	const char *last_rsc_id = NULL;

	if (rsc_list) {
	*rsc_list = NULL;
	}

	if (list == NULL) {
	if (rsc_list) {
	*rsc_list = strdup(" ");
	}
	if (node_list) {
	*node_list = strdup(" ");
	}
	return;
	}

	if (node_list) {
	*node_list = NULL;
	}

	for (gIter = list; gIter != NULL; gIter = gIter->next) {
	notify_entry_t entry = (notify_entry_t ) gIter->data;

	CRM_LOG_ASSERT(entry != NULL);
	CRM_LOG_ASSERT(entry && entry->rsc != NULL);

	if(entry == NULL \|\| entry->rsc == NULL) {
	continue;
	}

	/* Uh, why? */
	CRM_LOG_ASSERT(node_list == NULL \|\| entry->node != NULL);
	if(node_list != NULL && entry->node == NULL) {
	continue;
	}

	uname = NULL;
	rsc_id = entry->rsc->id;
	CRM_ASSERT(rsc_id != NULL);

	/* filter dups */
	if (safe_str_eq(rsc_id, last_rsc_id)) {
	continue;
	}
	last_rsc_id = rsc_id;

	if (rsc_list != NULL) {
	int existing_len = 0;
	int len = 2 + strlen(rsc_id); /* +1 space, +1 EOS */

	if (rsc_list && *rsc_list) {
	existing_len = strlen(*rsc_list);
	}

	crm_trace("Adding %s (%dc) at offset %d", rsc_id, len - 2, existing_len);
	rsc_list = realloc_safe(rsc_list, len + existing_len);
	sprintf(*rsc_list + existing_len, "%s%s", existing_len == 0 ? "":" ", rsc_id);
	}

	if (entry->node != NULL) {
	uname = entry->node->details->uname;
	}

	if (node_list != NULL && uname) {
	int existing_len = 0;
	int len = 2 + strlen(uname);

	if (node_list && *node_list) {
	existing_len = strlen(*node_list);
	}

	crm_trace("Adding %s (%dc) at offset %d", uname, len - 2, existing_len);
	node_list = realloc_safe(node_list, len + existing_len);
	sprintf(*node_list + existing_len, "%s%s", existing_len == 0 ? "":" ", uname);
	}
	}

	}

	static void
	dup_attr(gpointer key, gpointer value, gpointer user_data)
	{
	add_hash_param(user_data, key, value);
	}

	static action_t *
	pe_notify(resource_t * rsc, node_t * node, action_t * op, action_t * confirm,
	notify_data_t * n_data, pe_working_set_t * data_set)
	{
	char *key = NULL;
	action_t *trigger = NULL;
	const char *value = NULL;
	const char *task = NULL;

	if (op == NULL \|\| confirm == NULL) {
	pe_rsc_trace(rsc, "Op=%p confirm=%p", op, confirm);
	return NULL;
	}

	CRM_CHECK(rsc != NULL, return NULL);
	CRM_CHECK(node != NULL, return NULL);

	if (node->details->online == FALSE) {
	pe_rsc_trace(rsc, "Skipping notification for %s: node offline", rsc->id);
	return NULL;
	} else if (is_set(op->flags, pe_action_runnable) == FALSE) {
	pe_rsc_trace(rsc, "Skipping notification for %s: not runnable", op->uuid);
	return NULL;
	}

	value = g_hash_table_lookup(op->meta, "notify_type");
	task = g_hash_table_lookup(op->meta, "notify_operation");

	pe_rsc_trace(rsc, "Creating notify actions for %s: %s (%s-%s)", op->uuid, rsc->id, value, task);

	key = generate_notify_key(rsc->id, value, task);
	trigger = custom_action(rsc, key, op->task, node,
	is_set(op->flags, pe_action_optional), TRUE, data_set);
	g_hash_table_foreach(op->meta, dup_attr, trigger->meta);
	g_hash_table_foreach(n_data->keys, dup_attr, trigger->meta);

	/* pseudo_notify before notify */
	pe_rsc_trace(rsc, "Ordering %s before %s (%d->%d)", op->uuid, trigger->uuid, trigger->id,
	op->id);

	order_actions(op, trigger, pe_order_optional);
	order_actions(trigger, confirm, pe_order_optional);
	return trigger;
	}

	static void
	pe_post_notify(resource_t * rsc, node_t * node, notify_data_t * n_data, pe_working_set_t * data_set)
	{
	action_t *notify = NULL;

	CRM_CHECK(rsc != NULL, return);

	if (n_data->post == NULL) {
	return; /* Nothing to do */
	}

	notify = pe_notify(rsc, node, n_data->post, n_data->post_done, n_data, data_set);

	if (notify != NULL) {
	notify->priority = INFINITY;
	}

	if (n_data->post_done) {
	GListPtr gIter = rsc->actions;

	for (; gIter != NULL; gIter = gIter->next) {
	action_t mon = (action_t ) gIter->data;
	const char *interval = g_hash_table_lookup(mon->meta, "interval");

	if (interval == NULL \|\| safe_str_eq(interval, "0")) {
	pe_rsc_trace(rsc, "Skipping %s: interval", mon->uuid);
	continue;
	} else if (safe_str_eq(mon->task, RSC_CANCEL)) {
	pe_rsc_trace(rsc, "Skipping %s: cancel", mon->uuid);
	continue;
	}

	order_actions(n_data->post_done, mon, pe_order_optional);
	}
	}
	}

	notify_data_t *
	create_notification_boundaries(resource_t * rsc, const char action, action_t start,
	action_t * end, pe_working_set_t * data_set)
	{
	/* Create the pseudo ops that preceed and follow the actual notifications */

	/*
	* Creates two sequences (conditional on start and end being supplied):
	* pre_notify -> pre_notify_complete -> start, and
	* end -> post_notify -> post_notify_complete
	*
	* 'start' and 'end' may be the same event or ${X} and ${X}ed as per clones
	*/
	char *key = NULL;
	notify_data_t *n_data = NULL;

	if (is_not_set(rsc->flags, pe_rsc_notify)) {
	return NULL;
	}

	n_data = calloc(1, sizeof(notify_data_t));
	n_data->action = action;
	n_data->keys =
	g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str);

	if (start) {
	/* create pre-event notification wrappers */
	key = generate_notify_key(rsc->id, "pre", start->task);
	n_data->pre =
	custom_action(rsc, key, RSC_NOTIFY, NULL, is_set(start->flags, pe_action_optional),
	TRUE, data_set);

	update_action_flags(n_data->pre, pe_action_pseudo);
	update_action_flags(n_data->pre, pe_action_runnable);

	add_hash_param(n_data->pre->meta, "notify_type", "pre");
	add_hash_param(n_data->pre->meta, "notify_operation", n_data->action);

	add_hash_param(n_data->pre->meta, "notify_key_type", "pre");
	add_hash_param(n_data->pre->meta, "notify_key_operation", start->task);

	/* create pre_notify_complete */
	key = generate_notify_key(rsc->id, "confirmed-pre", start->task);
	n_data->pre_done =
	custom_action(rsc, key, RSC_NOTIFIED, NULL, is_set(start->flags, pe_action_optional),
	TRUE, data_set);

	update_action_flags(n_data->pre_done, pe_action_pseudo);
	update_action_flags(n_data->pre_done, pe_action_runnable);

	add_hash_param(n_data->pre_done->meta, "notify_type", "pre");
	add_hash_param(n_data->pre_done->meta, "notify_operation", n_data->action);

	add_hash_param(n_data->pre_done->meta, "notify_key_type", "confirmed-pre");
	add_hash_param(n_data->pre_done->meta, "notify_key_operation", start->task);

	order_actions(n_data->pre_done, start, pe_order_optional);
	order_actions(n_data->pre, n_data->pre_done, pe_order_optional);
	}

	if (end) {
	/* create post-event notification wrappers */
	key = generate_notify_key(rsc->id, "post", end->task);
	n_data->post =
	custom_action(rsc, key, RSC_NOTIFY, NULL, is_set(end->flags, pe_action_optional), TRUE,
	data_set);

	n_data->post->priority = INFINITY;
	update_action_flags(n_data->post, pe_action_pseudo);
	if (is_set(end->flags, pe_action_runnable)) {
	update_action_flags(n_data->post, pe_action_runnable);
	} else {
	update_action_flags(n_data->post, pe_action_runnable \| pe_action_clear);
	}

	add_hash_param(n_data->post->meta, "notify_type", "post");
	add_hash_param(n_data->post->meta, "notify_operation", n_data->action);

	add_hash_param(n_data->post->meta, "notify_key_type", "post");
	add_hash_param(n_data->post->meta, "notify_key_operation", end->task);

	/* create post_notify_complete */
	key = generate_notify_key(rsc->id, "confirmed-post", end->task);
	n_data->post_done =
	custom_action(rsc, key, RSC_NOTIFIED, NULL, is_set(end->flags, pe_action_optional),
	TRUE, data_set);

	n_data->post_done->priority = INFINITY;
	update_action_flags(n_data->post_done, pe_action_pseudo);
	if (is_set(end->flags, pe_action_runnable)) {
	update_action_flags(n_data->post_done, pe_action_runnable);
	} else {
	update_action_flags(n_data->post_done, pe_action_runnable \| pe_action_clear);
	}

	add_hash_param(n_data->post_done->meta, "notify_type", "post");
	add_hash_param(n_data->post_done->meta, "notify_operation", n_data->action);

	add_hash_param(n_data->post_done->meta, "notify_key_type", "confirmed-post");
	add_hash_param(n_data->post_done->meta, "notify_key_operation", end->task);

	order_actions(end, n_data->post, pe_order_implies_then);
	order_actions(n_data->post, n_data->post_done, pe_order_implies_then);
	}

	if (start && end) {
	order_actions(n_data->pre_done, n_data->post, pe_order_optional);
	}

	if (safe_str_eq(action, RSC_STOP)) {
	action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);

	order_actions(n_data->post_done, all_stopped, pe_order_optional);
	}

	return n_data;
	}

	void
	collect_notification_data(resource_t * rsc, gboolean state, gboolean activity,
	notify_data_t * n_data)
	{

	if(n_data->allowed_nodes == NULL) {
	n_data->allowed_nodes = rsc->allowed_nodes;
	}

	if (rsc->children) {
	GListPtr gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	collect_notification_data(child, state, activity, n_data);
	}
	return;
	}

	if (state) {
	notify_entry_t *entry = NULL;

	entry = calloc(1, sizeof(notify_entry_t));
	entry->rsc = rsc;
	if (rsc->running_on) {
	/* we only take the first one */
	entry->node = rsc->running_on->data;
	}

	pe_rsc_trace(rsc, "%s state: %s", rsc->id, role2text(rsc->role));

	switch (rsc->role) {
	case RSC_ROLE_STOPPED:
	n_data->inactive = g_list_prepend(n_data->inactive, entry);
	break;
	case RSC_ROLE_STARTED:
	n_data->active = g_list_prepend(n_data->active, entry);
	break;
	case RSC_ROLE_SLAVE:
	n_data->slave = g_list_prepend(n_data->slave, entry);
	break;
	case RSC_ROLE_MASTER:
	n_data->master = g_list_prepend(n_data->master, entry);
	break;
	default:
	crm_err("Unsupported notify role");
	free(entry);
	break;
	}
	}

	if (activity) {
	notify_entry_t *entry = NULL;
	enum action_tasks task;

	GListPtr gIter = rsc->actions;

	for (; gIter != NULL; gIter = gIter->next) {
	action_t op = (action_t ) gIter->data;

	if (is_set(op->flags, pe_action_optional) == FALSE && op->node != NULL) {

	entry = calloc(1, sizeof(notify_entry_t));
	entry->node = op->node;
	entry->rsc = rsc;

	task = text2task(op->task);
	switch (task) {
	case start_rsc:
	n_data->start = g_list_prepend(n_data->start, entry);
	break;
	case stop_rsc:
	n_data->stop = g_list_prepend(n_data->stop, entry);
	break;
	case action_promote:
	n_data->promote = g_list_prepend(n_data->promote, entry);
	break;
	case action_demote:
	n_data->demote = g_list_prepend(n_data->demote, entry);
	break;
	default:
	free(entry);
	break;
	}
	}
	}
	}
	}

	gboolean
	expand_notification_data(notify_data_t * n_data, pe_working_set_t * data_set)
	{
	/* Expand the notification entries into a key=value hashtable
	* This hashtable is later used in action2xml()
	*/
	gboolean required = FALSE;
	char *rsc_list = NULL;
	char *node_list = NULL;
	GListPtr nodes = NULL;

	if (n_data->stop) {
	n_data->stop = g_list_sort(n_data->stop, sort_notify_entries);
	}
	expand_list(n_data->stop, &rsc_list, &node_list);
	if (rsc_list != NULL && safe_str_neq(" ", rsc_list)) {
	if (safe_str_eq(n_data->action, RSC_STOP)) {
	required = TRUE;
	}
	}
	g_hash_table_insert(n_data->keys, strdup("notify_stop_resource"), rsc_list);
	g_hash_table_insert(n_data->keys, strdup("notify_stop_uname"), node_list);

	if (n_data->start) {
	n_data->start = g_list_sort(n_data->start, sort_notify_entries);
	if (rsc_list && safe_str_eq(n_data->action, RSC_START)) {
	required = TRUE;
	}
	}
	expand_list(n_data->start, &rsc_list, &node_list);
	g_hash_table_insert(n_data->keys, strdup("notify_start_resource"), rsc_list);
	g_hash_table_insert(n_data->keys, strdup("notify_start_uname"), node_list);

	if (n_data->demote) {
	n_data->demote = g_list_sort(n_data->demote, sort_notify_entries);
	if (safe_str_eq(n_data->action, RSC_DEMOTE)) {
	required = TRUE;
	}
	}

	expand_list(n_data->demote, &rsc_list, &node_list);
	g_hash_table_insert(n_data->keys, strdup("notify_demote_resource"), rsc_list);
	g_hash_table_insert(n_data->keys, strdup("notify_demote_uname"), node_list);

	if (n_data->promote) {
	n_data->promote = g_list_sort(n_data->promote, sort_notify_entries);
	if (safe_str_eq(n_data->action, RSC_PROMOTE)) {
	required = TRUE;
	}
	}
	expand_list(n_data->promote, &rsc_list, &node_list);
	g_hash_table_insert(n_data->keys, strdup("notify_promote_resource"), rsc_list);
	g_hash_table_insert(n_data->keys, strdup("notify_promote_uname"), node_list);

	if (n_data->active) {
	n_data->active = g_list_sort(n_data->active, sort_notify_entries);
	}
	expand_list(n_data->active, &rsc_list, &node_list);
	g_hash_table_insert(n_data->keys, strdup("notify_active_resource"), rsc_list);
	g_hash_table_insert(n_data->keys, strdup("notify_active_uname"), node_list);

	if (n_data->slave) {
	n_data->slave = g_list_sort(n_data->slave, sort_notify_entries);
	}
	expand_list(n_data->slave, &rsc_list, &node_list);
	g_hash_table_insert(n_data->keys, strdup("notify_slave_resource"), rsc_list);
	g_hash_table_insert(n_data->keys, strdup("notify_slave_uname"), node_list);

	if (n_data->master) {
	n_data->master = g_list_sort(n_data->master, sort_notify_entries);
	}
	expand_list(n_data->master, &rsc_list, &node_list);
	g_hash_table_insert(n_data->keys, strdup("notify_master_resource"), rsc_list);
	g_hash_table_insert(n_data->keys, strdup("notify_master_uname"), node_list);

	if (n_data->inactive) {
	n_data->inactive = g_list_sort(n_data->inactive, sort_notify_entries);
	}
	expand_list(n_data->inactive, &rsc_list, NULL);
	g_hash_table_insert(n_data->keys, strdup("notify_inactive_resource"), rsc_list);

	nodes = g_hash_table_get_values(n_data->allowed_nodes);
	node_list = expand_node_list(nodes);
	g_hash_table_insert(n_data->keys, strdup("notify_available_uname"), node_list);
	g_list_free(nodes);

	node_list = expand_node_list(data_set->nodes);
	g_hash_table_insert(n_data->keys, strdup("notify_all_uname"), node_list);

	if (required && n_data->pre) {
	update_action_flags(n_data->pre, pe_action_optional \| pe_action_clear);
	update_action_flags(n_data->pre_done, pe_action_optional \| pe_action_clear);
	}

	if (required && n_data->post) {
	update_action_flags(n_data->post, pe_action_optional \| pe_action_clear);
	update_action_flags(n_data->post_done, pe_action_optional \| pe_action_clear);
	}
	return required;
	}

	void
	create_notifications(resource_t * rsc, notify_data_t * n_data, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	action_t *stop = NULL;
	action_t *start = NULL;
	enum action_tasks task = text2task(n_data->action);

	if (rsc->children) {
	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	create_notifications(child, n_data, data_set);
	}
	return;
	}

	/* Copy notification details into standard ops */

	gIter = rsc->actions;
	for (; gIter != NULL; gIter = gIter->next) {
	action_t op = (action_t ) gIter->data;

	if (is_set(op->flags, pe_action_optional) == FALSE && op->node != NULL) {
	enum action_tasks t = text2task(op->task);

	switch (t) {
	case start_rsc:
	case stop_rsc:
	case action_promote:
	case action_demote:
	g_hash_table_foreach(n_data->keys, dup_attr, op->meta);
	break;
	default:
	break;
	}
	}
	}

	pe_rsc_trace(rsc, "Creating notificaitons for: %s.%s (%s->%s)",
	n_data->action, rsc->id, role2text(rsc->role), role2text(rsc->next_role));

	stop = find_first_action(rsc->actions, NULL, RSC_STOP, NULL);
	start = find_first_action(rsc->actions, NULL, RSC_START, NULL);

	/* stop / demote */
	if (rsc->role != RSC_ROLE_STOPPED) {
	if (task == stop_rsc \|\| task == action_demote) {
	gIter = rsc->running_on;
	for (; gIter != NULL; gIter = gIter->next) {
	node_t current_node = (node_t ) gIter->data;

	/* if this stop action is a pseudo action as a result of the current
	* node being fenced, this stop action is implied by the fencing
	* action. There's no reason to send the fenced node a stop notification */
	if (stop &&
	is_set(stop->flags, pe_action_pseudo) &&
	current_node->details->unclean) {

	continue;
	}

	pe_notify(rsc, current_node, n_data->pre, n_data->pre_done, n_data, data_set);
	if (task == action_demote \|\| stop == NULL
	\|\| is_set(stop->flags, pe_action_optional)) {
	pe_post_notify(rsc, current_node, n_data, data_set);
	}
	}
	}
	}

	/* start / promote */
	if (rsc->next_role != RSC_ROLE_STOPPED) {
	if (rsc->allocated_to == NULL) {
	pe_proc_err("Next role '%s' but %s is not allocated", role2text(rsc->next_role),
	rsc->id);

	} else if (task == start_rsc \|\| task == action_promote) {
	if (task != start_rsc \|\| start == NULL \|\| is_set(start->flags, pe_action_optional)) {
	pe_notify(rsc, rsc->allocated_to, n_data->pre, n_data->pre_done, n_data, data_set);
	}
	pe_post_notify(rsc, rsc->allocated_to, n_data, data_set);
	}
	}
	}

	void
	free_notification_data(notify_data_t * n_data)
	{
	if (n_data == NULL) {
	return;
	}

	g_list_free_full(n_data->stop, free);
	g_list_free_full(n_data->start, free);
	g_list_free_full(n_data->demote, free);
	g_list_free_full(n_data->promote, free);
	g_list_free_full(n_data->master, free);
	g_list_free_full(n_data->slave, free);
	g_list_free_full(n_data->active, free);
	g_list_free_full(n_data->inactive, free);
	g_hash_table_destroy(n_data->keys);
	free(n_data);
	}

	int transition_id = -1;

	/*
	* Create a dependency graph to send to the transitioner (via the CRMd)
	*/
	gboolean
	stage8(pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	const char *value = NULL;

	transition_id++;
	crm_trace("Creating transition graph %d.", transition_id);

	data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);

	value = pe_pref(data_set->config_hash, "cluster-delay");
	crm_xml_add(data_set->graph, "cluster-delay", value);

	value = pe_pref(data_set->config_hash, "stonith-timeout");
	crm_xml_add(data_set->graph, "stonith-timeout", value);

	crm_xml_add(data_set->graph, "failed-stop-offset", "INFINITY");

	if (is_set(data_set->flags, pe_flag_start_failure_fatal)) {
	crm_xml_add(data_set->graph, "failed-start-offset", "INFINITY");
	} else {
	crm_xml_add(data_set->graph, "failed-start-offset", "1");
	}

	value = pe_pref(data_set->config_hash, "batch-limit");
	crm_xml_add(data_set->graph, "batch-limit", value);

	crm_xml_add_int(data_set->graph, "transition_id", transition_id);

	value = pe_pref(data_set->config_hash, "migration-limit");
	if (crm_int_helper(value, NULL) > 0) {
	crm_xml_add(data_set->graph, "migration-limit", value);
	}

	/* errors...
	slist_iter(action, action_t, action_list, lpc,
	if(action->optional == FALSE && action->runnable == FALSE) {
	print_action("Ignoring", action, TRUE);
	}
	);
	*/

	gIter = data_set->resources;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t rsc = (resource_t ) gIter->data;

	pe_rsc_trace(rsc, "processing actions for rsc=%s", rsc->id);
	rsc->cmds->expand(rsc, data_set);
	}

	crm_log_xml_trace(data_set->graph, "created resource-driven action list");

	/* catch any non-resource specific actions */
	crm_trace("processing non-resource actions");

	gIter = data_set->actions;
	for (; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	if (action->rsc
	&& action->node
	&& action->node->details->shutdown
	&& is_not_set(action->rsc->flags, pe_rsc_maintenance)
	&& is_not_set(action->flags, pe_action_optional)
	&& is_not_set(action->flags, pe_action_runnable)
	&& crm_str_eq(action->task, RSC_STOP, TRUE)
	) {
	/* Eventually we should just ignore the 'fence' case
	* But for now its the best way to detect (in CTS) when
	* CIB resource updates are being lost
	*/
	if (is_set(data_set->flags, pe_flag_have_quorum)
	\|\| data_set->no_quorum_policy == no_quorum_ignore) {
	crm_crit("Cannot %s node '%s' because of %s:%s%s",
	action->node->details->unclean ? "fence" : "shut down",
	action->node->details->uname, action->rsc->id,
	is_not_set(action->rsc->flags, pe_rsc_managed) ? " unmanaged" : " blocked",
	is_set(action->rsc->flags, pe_rsc_failed) ? " failed" : "");
	}
	}

	graph_element_from_action(action, data_set);
	}

	crm_log_xml_trace(data_set->graph, "created generic action list");
	crm_trace("Created transition graph %d.", transition_id);

	return TRUE;
	}

	void
	cleanup_alloc_calculations(pe_working_set_t * data_set)
	{
	if (data_set == NULL) {
	return;
	}

	crm_trace("deleting %d order cons: %p",
	g_list_length(data_set->ordering_constraints), data_set->ordering_constraints);
	pe_free_ordering(data_set->ordering_constraints);
	data_set->ordering_constraints = NULL;

	crm_trace("deleting %d node cons: %p",
	g_list_length(data_set->placement_constraints), data_set->placement_constraints);
	pe_free_rsc_to_node(data_set->placement_constraints);
	data_set->placement_constraints = NULL;

	crm_trace("deleting %d inter-resource cons: %p",
	g_list_length(data_set->colocation_constraints), data_set->colocation_constraints);
	g_list_free_full(data_set->colocation_constraints, free);
	data_set->colocation_constraints = NULL;

	crm_trace("deleting %d ticket deps: %p",
	g_list_length(data_set->ticket_constraints), data_set->ticket_constraints);
	g_list_free_full(data_set->ticket_constraints, free);
	data_set->ticket_constraints = NULL;

	cleanup_calculations(data_set);
	}
	diff --git a/pengine/clone.c b/pengine/clone.c
	index ebf53ed0f5..75127e85f3 100644
	--- a/pengine/clone.c
	+++ b/pengine/clone.c
	@@ -1,1631 +1,1631 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <crm/msg_xml.h>
	#include <allocate.h>
	#include <utils.h>
	#include <allocate.h>

	#define VARIANT_CLONE 1
	#include <lib/pengine/variant.h>

	gint sort_clone_instance(gconstpointer a, gconstpointer b, gpointer data_set);
	static void append_parent_colocation(resource_t * rsc, resource_t * child, gboolean all);

	static gint
	sort_rsc_id(gconstpointer a, gconstpointer b)
	{
	const resource_t resource1 = (const resource_t )a;
	const resource_t resource2 = (const resource_t )b;

	CRM_ASSERT(resource1 != NULL);
	CRM_ASSERT(resource2 != NULL);

	return strcmp(resource1->id, resource2->id);
	}

	static node_t *
	parent_node_instance(const resource_t * rsc, node_t * node)
	{
	node_t *ret = NULL;

	if (node != NULL) {
	ret = pe_hash_table_lookup(rsc->parent->allowed_nodes, node->details->id);
	}
	return ret;
	}

	static gboolean
	did_fail(const resource_t * rsc)
	{
	GListPtr gIter = rsc->children;

	if (is_set(rsc->flags, pe_rsc_failed)) {
	return TRUE;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	if (did_fail(child_rsc)) {
	return TRUE;
	}
	}
	return FALSE;
	}

	gint
	sort_clone_instance(gconstpointer a, gconstpointer b, gpointer data_set)
	{
	int rc = 0;
	node_t *node1 = NULL;
	node_t *node2 = NULL;

	gboolean can1 = TRUE;
	gboolean can2 = TRUE;

	const resource_t resource1 = (const resource_t )a;
	const resource_t resource2 = (const resource_t )b;

	CRM_ASSERT(resource1 != NULL);
	CRM_ASSERT(resource2 != NULL);

	/* allocation order:
	* - active instances
	* - instances running on nodes with the least copies
	- * - active instances on nodes that cant support them or are to be fenced
	+ * - active instances on nodes that can't support them or are to be fenced
	* - failed instances
	* - inactive instances
	*/

	if (resource1->running_on && resource2->running_on) {
	if (g_list_length(resource1->running_on) < g_list_length(resource2->running_on)) {
	crm_trace("%s < %s: running_on", resource1->id, resource2->id);
	return -1;

	} else if (g_list_length(resource1->running_on) > g_list_length(resource2->running_on)) {
	crm_trace("%s > %s: running_on", resource1->id, resource2->id);
	return 1;
	}
	}

	if (resource1->running_on) {
	node1 = resource1->running_on->data;
	}
	if (resource2->running_on) {
	node2 = resource2->running_on->data;
	}

	if (node1) {
	node_t *match = pe_hash_table_lookup(resource1->allowed_nodes, node1->details->id);

	if (match == NULL \|\| match->weight < 0) {
	crm_trace("%s: current location is unavailable", resource1->id);
	node1 = NULL;
	can1 = FALSE;
	}
	}

	if (node2) {
	node_t *match = pe_hash_table_lookup(resource2->allowed_nodes, node2->details->id);

	if (match == NULL \|\| match->weight < 0) {
	crm_trace("%s: current location is unavailable", resource2->id);
	node2 = NULL;
	can2 = FALSE;
	}
	}

	if (can1 != can2) {
	if (can1) {
	crm_trace("%s < %s: availability of current location", resource1->id, resource2->id);
	return -1;
	}
	crm_trace("%s > %s: availability of current location", resource1->id, resource2->id);
	return 1;
	}

	if (resource1->priority < resource2->priority) {
	crm_trace("%s < %s: priority", resource1->id, resource2->id);
	return 1;

	} else if (resource1->priority > resource2->priority) {
	crm_trace("%s > %s: priority", resource1->id, resource2->id);
	return -1;
	}

	if (node1 == NULL && node2 == NULL) {
	crm_trace("%s == %s: not active", resource1->id, resource2->id);
	return 0;
	}

	if (node1 != node2) {
	if (node1 == NULL) {
	crm_trace("%s > %s: active", resource1->id, resource2->id);
	return 1;
	} else if (node2 == NULL) {
	crm_trace("%s < %s: active", resource1->id, resource2->id);
	return -1;
	}
	}

	can1 = can_run_resources(node1);
	can2 = can_run_resources(node2);
	if (can1 != can2) {
	if (can1) {
	crm_trace("%s < %s: can", resource1->id, resource2->id);
	return -1;
	}
	crm_trace("%s > %s: can", resource1->id, resource2->id);
	return 1;
	}

	node1 = parent_node_instance(resource1, node1);
	node2 = parent_node_instance(resource2, node2);
	if (node1 != NULL && node2 == NULL) {
	crm_trace("%s < %s: not allowed", resource1->id, resource2->id);
	return -1;
	} else if (node1 == NULL && node2 != NULL) {
	crm_trace("%s > %s: not allowed", resource1->id, resource2->id);
	return 1;
	}

	if (node1 == NULL \|\| node2 == NULL) {
	crm_trace("%s == %s: not allowed", resource1->id, resource2->id);
	return 0;
	}

	if (node1->count < node2->count) {
	crm_trace("%s < %s: count", resource1->id, resource2->id);
	return -1;

	} else if (node1->count > node2->count) {
	crm_trace("%s > %s: count", resource1->id, resource2->id);
	return 1;
	}

	can1 = did_fail(resource1);
	can2 = did_fail(resource2);
	if (can1 != can2) {
	if (can1) {
	crm_trace("%s > %s: failed", resource1->id, resource2->id);
	return 1;
	}
	crm_trace("%s < %s: failed", resource1->id, resource2->id);
	return -1;
	}

	if (node1 && node2) {
	int lpc = 0;
	int max = 0;
	node_t *n = NULL;
	GListPtr gIter = NULL;
	GListPtr list1 = NULL;
	GListPtr list2 = NULL;
	GHashTable *hash1 =
	g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, g_hash_destroy_str);
	GHashTable *hash2 =
	g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, g_hash_destroy_str);

	n = node_copy(resource1->running_on->data);
	g_hash_table_insert(hash1, (gpointer) n->details->id, n);

	n = node_copy(resource2->running_on->data);
	g_hash_table_insert(hash2, (gpointer) n->details->id, n);

	for (gIter = resource1->parent->rsc_cons; gIter; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	crm_trace("Applying %s to %s", constraint->id, resource1->id);

	hash1 = native_merge_weights(constraint->rsc_rh, resource1->id, hash1,
	constraint->node_attribute,
	(float)constraint->score / INFINITY, 0);
	}

	for (gIter = resource1->parent->rsc_cons_lhs; gIter; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	crm_trace("Applying %s to %s", constraint->id, resource1->id);

	hash1 = native_merge_weights(constraint->rsc_lh, resource1->id, hash1,
	constraint->node_attribute,
	(float)constraint->score / INFINITY, pe_weights_positive);
	}

	for (gIter = resource2->parent->rsc_cons; gIter; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	crm_trace("Applying %s to %s", constraint->id, resource2->id);

	hash2 = native_merge_weights(constraint->rsc_rh, resource2->id, hash2,
	constraint->node_attribute,
	(float)constraint->score / INFINITY, 0);
	}

	for (gIter = resource2->parent->rsc_cons_lhs; gIter; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	crm_trace("Applying %s to %s", constraint->id, resource2->id);

	hash2 = native_merge_weights(constraint->rsc_lh, resource2->id, hash2,
	constraint->node_attribute,
	(float)constraint->score / INFINITY, pe_weights_positive);
	}

	/* Current location score */
	node1 = g_list_nth_data(resource1->running_on, 0);
	node1 = g_hash_table_lookup(hash1, node1->details->id);

	node2 = g_list_nth_data(resource2->running_on, 0);
	node2 = g_hash_table_lookup(hash2, node2->details->id);

	if (node1->weight < node2->weight) {
	if (node1->weight < 0) {
	crm_trace("%s > %s: current score", resource1->id, resource2->id);
	rc = -1;
	goto out;

	} else {
	crm_trace("%s < %s: current score", resource1->id, resource2->id);
	rc = 1;
	goto out;
	}

	} else if (node1->weight > node2->weight) {
	crm_trace("%s > %s: current score", resource1->id, resource2->id);
	rc = -1;
	goto out;
	}

	/* All location scores */
	list1 = g_hash_table_get_values(hash1);
	list2 = g_hash_table_get_values(hash2);

	list1 =
	g_list_sort_with_data(list1, sort_node_weight,
	g_list_nth_data(resource1->running_on, 0));
	list2 =
	g_list_sort_with_data(list2, sort_node_weight,
	g_list_nth_data(resource2->running_on, 0));
	max = g_list_length(list1);
	if (max < g_list_length(list2)) {
	max = g_list_length(list2);
	}

	for (; lpc < max; lpc++) {
	node1 = g_list_nth_data(list1, lpc);
	node2 = g_list_nth_data(list2, lpc);
	if (node1 == NULL) {
	crm_trace("%s < %s: colocated score NULL", resource1->id, resource2->id);
	rc = 1;
	break;

	} else if (node2 == NULL) {
	crm_trace("%s > %s: colocated score NULL", resource1->id, resource2->id);
	rc = -1;
	break;
	}

	if (node1->weight < node2->weight) {
	crm_trace("%s < %s: colocated score", resource1->id, resource2->id);
	rc = 1;
	break;

	} else if (node1->weight > node2->weight) {
	crm_trace("%s > %s: colocated score", resource1->id, resource2->id);
	rc = -1;
	break;
	}
	}

	/* Order by reverse uname - same as sort_node_weight() does? */
	out:
	g_hash_table_destroy(hash1); /* Free mem */
	g_hash_table_destroy(hash2); /* Free mem */
	g_list_free(list1);
	g_list_free(list2);

	if (rc != 0) {
	return rc;
	}
	}

	rc = strcmp(resource1->id, resource2->id);
	crm_trace("%s %c %s: default", resource1->id, rc < 0 ? '<' : '>', resource2->id);
	return rc;
	}

	static node_t *
	can_run_instance(resource_t * rsc, node_t * node)
	{
	node_t *local_node = NULL;
	clone_variant_data_t *clone_data = NULL;

	if (can_run_resources(node) == FALSE) {
	goto bail;

	} else if (is_set(rsc->flags, pe_rsc_orphan)) {
	goto bail;
	}

	local_node = parent_node_instance(rsc, node);
	get_clone_variant_data(clone_data, rsc->parent);

	if (local_node == NULL) {
	crm_warn("%s cannot run on %s: node not allowed", rsc->id, node->details->uname);
	goto bail;

	} else if (local_node->weight < 0) {
	common_update_score(rsc, node->details->id, local_node->weight);
	pe_rsc_trace(rsc, "%s cannot run on %s: Parent node weight doesn't allow it.",
	rsc->id, node->details->uname);
	} else if (local_node->count < clone_data->clone_node_max) {
	pe_rsc_trace(rsc, "%s can run on %s: %d", rsc->id, node->details->uname, local_node->count);
	return local_node;

	} else {
	pe_rsc_trace(rsc, "%s cannot run on %s: node full (%d >= %d)",
	rsc->id, node->details->uname, local_node->count, clone_data->clone_node_max);
	}

	bail:
	if (node) {
	common_update_score(rsc, node->details->id, -INFINITY);
	}
	return NULL;
	}

	static node_t *
	color_instance(resource_t * rsc, node_t * prefer, gboolean all_coloc, pe_working_set_t * data_set)
	{
	node_t *chosen = NULL;
	node_t *local_node = NULL;
	GHashTable *backup = NULL;

	CRM_ASSERT(rsc);
	pe_rsc_trace(rsc, "Processing %s %d", rsc->id, all_coloc);

	if (is_not_set(rsc->flags, pe_rsc_provisional)) {
	return rsc->fns->location(rsc, NULL, FALSE);

	} else if (is_set(rsc->flags, pe_rsc_allocating)) {
	pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
	return NULL;
	}

	/* Only include positive colocation preferences of dependant resources
	* if not every node will get a copy of the clone
	*/
	append_parent_colocation(rsc->parent, rsc, all_coloc);

	if (prefer) {
	node_t *local_prefer = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);

	if (local_prefer == NULL \|\| local_prefer->weight < 0) {
	pe_rsc_trace(rsc, "Not pre-allocating %s to %s - unavailable", rsc->id,
	prefer->details->uname);
	return NULL;
	}
	}

	if (rsc->allowed_nodes) {
	GHashTableIter iter;
	node_t *try_node = NULL;

	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&try_node)) {
	can_run_instance(rsc, try_node);
	}
	}

	backup = node_hash_dup(rsc->allowed_nodes);
	chosen = rsc->cmds->allocate(rsc, prefer, data_set);
	if (chosen) {
	local_node = pe_hash_table_lookup(rsc->parent->allowed_nodes, chosen->details->id);

	if (prefer && chosen && chosen->details != prefer->details) {
	crm_notice("Pre-allocation failed: got %s instead of %s",
	chosen->details->uname, prefer->details->uname);
	g_hash_table_destroy(rsc->allowed_nodes);
	rsc->allowed_nodes = backup;
	native_deallocate(rsc);
	chosen = NULL;
	backup = NULL;

	} else if (local_node) {
	local_node->count++;

	} else if (is_set(rsc->flags, pe_rsc_managed)) {
	/* what to do? we can't enforce per-node limits in this case */
	crm_config_err("%s not found in %s (list=%d)",
	chosen->details->id, rsc->parent->id,
	g_hash_table_size(rsc->parent->allowed_nodes));
	}
	}

	if(backup) {
	g_hash_table_destroy(backup);
	}
	return chosen;
	}

	static void
	append_parent_colocation(resource_t * rsc, resource_t * child, gboolean all)
	{

	GListPtr gIter = NULL;

	gIter = rsc->rsc_cons;
	for (; gIter != NULL; gIter = gIter->next) {
	rsc_colocation_t cons = (rsc_colocation_t ) gIter->data;

	if (all \|\| cons->score < 0 \|\| cons->score == INFINITY) {
	child->rsc_cons = g_list_prepend(child->rsc_cons, cons);
	}
	}

	gIter = rsc->rsc_cons_lhs;
	for (; gIter != NULL; gIter = gIter->next) {
	rsc_colocation_t cons = (rsc_colocation_t ) gIter->data;

	if (all \|\| cons->score < 0) {
	child->rsc_cons_lhs = g_list_prepend(child->rsc_cons_lhs, cons);
	}
	}
	}

	node_t *
	clone_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
	{
	GHashTableIter iter;
	GListPtr nIter = NULL;
	GListPtr gIter = NULL;
	GListPtr nodes = NULL;
	node_t *node = NULL;

	int allocated = 0;
	int loop_max = 0;
	int clone_max = 0;
	int available_nodes = 0;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	if (is_not_set(rsc->flags, pe_rsc_provisional)) {
	return NULL;

	} else if (is_set(rsc->flags, pe_rsc_allocating)) {
	pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
	return NULL;
	}

	set_bit(rsc->flags, pe_rsc_allocating);
	pe_rsc_trace(rsc, "Processing %s", rsc->id);

	/* this information is used by sort_clone_instance() when deciding in which
	* order to allocate clone instances
	*/
	gIter = rsc->rsc_cons;
	for (; gIter != NULL; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	pe_rsc_trace(rsc, "%s: Coloring %s first", rsc->id, constraint->rsc_rh->id);
	constraint->rsc_rh->cmds->allocate(constraint->rsc_rh, prefer, data_set);
	}

	gIter = rsc->rsc_cons_lhs;
	for (; gIter != NULL; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	rsc->allowed_nodes =
	constraint->rsc_lh->cmds->merge_weights(constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
	constraint->node_attribute,
	(float)constraint->score / INFINITY,
	(pe_weights_rollback \| pe_weights_positive));
	}

	dump_node_scores(show_scores ? 0 : scores_log_level, rsc, __FUNCTION__, rsc->allowed_nodes);

	/* count now tracks the number of clones currently allocated */
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	node->count = 0;
	if (can_run_resources(node)) {
	available_nodes++;
	}
	}

	clone_max = clone_data->clone_max;
	if(available_nodes) {
	loop_max = clone_data->clone_max / available_nodes;
	}
	if (loop_max < 1) {
	loop_max = 1;
	}

	rsc->children = g_list_sort_with_data(rsc->children, sort_clone_instance, data_set);

	/* Pre-allocate as many instances as we can to their current location
	* First pre-sort the list of nodes by their placement score
	*/
	nodes = g_hash_table_get_values(rsc->allowed_nodes);
	nodes = g_list_sort_with_data(nodes, sort_node_weight, NULL);

	for(nIter = nodes; nIter; nIter = nIter->next) {
	int lpc;

	node = nIter->data;

	if(clone_max <= 0) {
	break;
	}

	if (can_run_resources(node) == FALSE \|\| node->weight < 0) {
	pe_rsc_trace(rsc, "Not Pre-allocatiing %s", node->details->uname);
	continue;
	}

	clone_max--;
	pe_rsc_trace(rsc, "Pre-allocating %s (%d remaining)", node->details->uname, clone_max);
	for (lpc = 0;
	allocated < clone_data->clone_max
	&& node->count < clone_data->clone_node_max
	&& lpc < clone_data->clone_node_max && lpc < loop_max; lpc++) {
	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	if (child->running_on && is_set(child->flags, pe_rsc_provisional)
	&& is_not_set(child->flags, pe_rsc_failed)) {
	node_t *child_node = child->running_on->data;

	if (child_node->details == node->details
	&& color_instance(child, node, clone_data->clone_max < available_nodes,
	data_set)) {
	pe_rsc_trace(rsc, "Pre-allocated %s to %s", child->id,
	node->details->uname);
	allocated++;
	break;
	}
	}
	}
	}
	}

	pe_rsc_trace(rsc, "Done pre-allocating (%d of %d)", allocated, clone_data->clone_max);
	g_list_free(nodes);

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	if (g_list_length(child->running_on) > 0) {
	node_t *child_node = child->running_on->data;
	node_t *local_node = parent_node_instance(child, child->running_on->data);

	if (local_node == NULL) {
	crm_err("%s is running on %s which isn't allowed",
	child->id, child_node->details->uname);
	}
	}

	if (is_not_set(child->flags, pe_rsc_provisional)) {
	} else if (allocated >= clone_data->clone_max) {
	pe_rsc_debug(rsc, "Child %s not allocated - limit reached", child->id);
	resource_location(child, NULL, -INFINITY, "clone_color:limit_reached", data_set);

	} else if (color_instance(child, NULL, clone_data->clone_max < available_nodes, data_set)) {
	allocated++;
	}
	}

	pe_rsc_debug(rsc, "Allocated %d %s instances of a possible %d",
	allocated, rsc->id, clone_data->clone_max);

	clear_bit(rsc->flags, pe_rsc_provisional);
	clear_bit(rsc->flags, pe_rsc_allocating);

	pe_rsc_trace(rsc, "Done allocating %s", rsc->id);
	return NULL;
	}

	static void
	clone_update_pseudo_status(resource_t * rsc, gboolean * stopping, gboolean * starting,
	gboolean * active)
	{
	GListPtr gIter = NULL;

	if (rsc->children) {

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	clone_update_pseudo_status(child, stopping, starting, active);
	}

	return;
	}

	CRM_ASSERT(active != NULL);
	CRM_ASSERT(starting != NULL);
	CRM_ASSERT(stopping != NULL);

	if (rsc->running_on) {
	*active = TRUE;
	}

	gIter = rsc->actions;
	for (; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	if (starting && stopping) {
	return;

	} else if (is_set(action->flags, pe_action_optional)) {
	pe_rsc_trace(rsc, "Skipping optional: %s", action->uuid);
	continue;

	} else if (is_set(action->flags, pe_action_pseudo) == FALSE
	&& is_set(action->flags, pe_action_runnable) == FALSE) {
	pe_rsc_trace(rsc, "Skipping unrunnable: %s", action->uuid);
	continue;

	} else if (safe_str_eq(RSC_STOP, action->task)) {
	pe_rsc_trace(rsc, "Stopping due to: %s", action->uuid);
	*stopping = TRUE;

	} else if (safe_str_eq(RSC_START, action->task)) {
	if (is_set(action->flags, pe_action_runnable) == FALSE) {
	pe_rsc_trace(rsc, "Skipping pseudo-op: %s run=%d, pseudo=%d",
	action->uuid, is_set(action->flags, pe_action_runnable),
	is_set(action->flags, pe_action_pseudo));
	} else {
	pe_rsc_trace(rsc, "Starting due to: %s", action->uuid);
	pe_rsc_trace(rsc, "%s run=%d, pseudo=%d",
	action->uuid, is_set(action->flags, pe_action_runnable),
	is_set(action->flags, pe_action_pseudo));
	*starting = TRUE;
	}
	}
	}
	}

	static action_t *
	find_rsc_action(resource_t * rsc, const char key, gboolean active_only, GListPtr list)
	{
	action_t *match = NULL;
	GListPtr possible = NULL;
	GListPtr active = NULL;

	possible = find_actions(rsc->actions, key, NULL);

	if (active_only) {
	GListPtr gIter = possible;

	for (; gIter != NULL; gIter = gIter->next) {
	action_t op = (action_t ) gIter->data;

	if (is_set(op->flags, pe_action_optional) == FALSE) {
	active = g_list_prepend(active, op);
	}
	}

	if (active && g_list_length(active) == 1) {
	match = g_list_nth_data(active, 0);
	}

	if (list) {
	*list = active;
	active = NULL;
	}

	} else if (possible && g_list_length(possible) == 1) {
	match = g_list_nth_data(possible, 0);

	}
	if (list) {
	*list = possible;
	possible = NULL;
	}

	if (possible) {
	g_list_free(possible);
	}
	if (active) {
	g_list_free(active);
	}

	return match;
	}

	static void
	child_ordering_constraints(resource_t * rsc, pe_working_set_t * data_set)
	{
	char *key = NULL;
	action_t *stop = NULL;
	action_t *start = NULL;
	action_t *last_stop = NULL;
	action_t *last_start = NULL;
	GListPtr gIter = NULL;
	gboolean active_only = TRUE; /* change to false to get the old behavior */
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	if (clone_data->ordered == FALSE) {
	return;
	}
	/* we have to maintain a consistent sorted child list when building order constraints */
	rsc->children = g_list_sort(rsc->children, sort_rsc_id);

	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	key = stop_key(child);
	stop = find_rsc_action(child, key, active_only, NULL);
	free(key);

	key = start_key(child);
	start = find_rsc_action(child, key, active_only, NULL);
	free(key);

	if (stop) {
	if (last_stop) {
	/* child/child relative stop */
	order_actions(stop, last_stop, pe_order_optional);
	}
	last_stop = stop;
	}

	if (start) {
	if (last_start) {
	/* child/child relative start */
	order_actions(last_start, start, pe_order_optional);
	}
	last_start = start;
	}
	}
	}

	void
	clone_create_actions(resource_t * rsc, pe_working_set_t * data_set)
	{
	gboolean child_active = FALSE;
	gboolean child_starting = FALSE;
	gboolean child_stopping = FALSE;
	gboolean allow_dependent_migrations = TRUE;

	action_t *stop = NULL;
	action_t *stopped = NULL;

	action_t *start = NULL;
	action_t *started = NULL;

	GListPtr gIter = rsc->children;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	pe_rsc_trace(rsc, "Creating actions for %s", rsc->id);

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;
	gboolean starting = FALSE;
	gboolean stopping = FALSE;

	child_rsc->cmds->create_actions(child_rsc, data_set);
	clone_update_pseudo_status(child_rsc, &stopping, &starting, &child_active);
	if (stopping && starting) {
	allow_dependent_migrations = FALSE;
	}

	child_stopping \|= stopping;
	child_starting \|= starting;
	}

	/* start */
	start = start_action(rsc, NULL, !child_starting);
	started = custom_action(rsc, started_key(rsc),
	RSC_STARTED, NULL, !child_starting, TRUE, data_set);

	update_action_flags(start, pe_action_pseudo \| pe_action_runnable);
	update_action_flags(started, pe_action_pseudo);
	started->priority = INFINITY;

	if (child_active \|\| child_starting) {
	update_action_flags(started, pe_action_runnable);
	}

	child_ordering_constraints(rsc, data_set);
	if (clone_data->start_notify == NULL) {
	clone_data->start_notify =
	create_notification_boundaries(rsc, RSC_START, start, started, data_set);
	}

	/* stop */
	stop = stop_action(rsc, NULL, !child_stopping);
	stopped = custom_action(rsc, stopped_key(rsc),
	RSC_STOPPED, NULL, !child_stopping, TRUE, data_set);

	stopped->priority = INFINITY;
	update_action_flags(stop, pe_action_pseudo \| pe_action_runnable);
	if (allow_dependent_migrations) {
	update_action_flags(stop, pe_action_migrate_runnable);
	}
	update_action_flags(stopped, pe_action_pseudo \| pe_action_runnable);
	if (clone_data->stop_notify == NULL) {
	clone_data->stop_notify =
	create_notification_boundaries(rsc, RSC_STOP, stop, stopped, data_set);

	if (clone_data->stop_notify && clone_data->start_notify) {
	order_actions(clone_data->stop_notify->post_done, clone_data->start_notify->pre,
	pe_order_optional);
	}
	}
	}

	void
	clone_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
	{
	resource_t *last_rsc = NULL;
	GListPtr gIter;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	pe_rsc_trace(rsc, "Internal constraints for %s", rsc->id);
	new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_START, pe_order_optional, data_set);
	new_rsc_order(rsc, RSC_START, rsc, RSC_STARTED, pe_order_runnable_left, data_set);
	new_rsc_order(rsc, RSC_STOP, rsc, RSC_STOPPED, pe_order_runnable_left, data_set);

	if (rsc->variant == pe_master) {
	new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_STOP, pe_order_optional, data_set);
	new_rsc_order(rsc, RSC_STARTED, rsc, RSC_PROMOTE, pe_order_runnable_left, data_set);
	}

	if (clone_data->ordered) {
	/* we have to maintain a consistent sorted child list when building order constraints */
	rsc->children = g_list_sort(rsc->children, sort_rsc_id);
	}
	for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	child_rsc->cmds->internal_constraints(child_rsc, data_set);

	order_start_start(rsc, child_rsc, pe_order_runnable_left \| pe_order_implies_first_printed);
	new_rsc_order(child_rsc, RSC_START, rsc, RSC_STARTED, pe_order_implies_then_printed,
	data_set);
	if (clone_data->ordered && last_rsc) {
	order_start_start(last_rsc, child_rsc, pe_order_optional);
	}

	order_stop_stop(rsc, child_rsc, pe_order_implies_first_printed);
	new_rsc_order(child_rsc, RSC_STOP, rsc, RSC_STOPPED, pe_order_implies_then_printed,
	data_set);
	if (clone_data->ordered && last_rsc) {
	order_stop_stop(child_rsc, last_rsc, pe_order_optional);
	}

	last_rsc = child_rsc;
	}
	}

	static void
	assign_node(resource_t * rsc, node_t * node, gboolean force)
	{
	if (rsc->children) {

	GListPtr gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	native_assign_node(child_rsc, NULL, node, force);
	}

	return;
	}
	native_assign_node(rsc, NULL, node, force);
	}

	static resource_t *
	find_compatible_child_by_node(resource_t * local_child, node_t * local_node, resource_t * rsc,
	enum rsc_role_e filter, gboolean current)
	{
	node_t *node = NULL;
	GListPtr gIter = NULL;

	if (local_node == NULL) {
	crm_err("Can't colocate unrunnable child %s with %s", local_child->id, rsc->id);
	return NULL;
	}

	crm_trace("Looking for compatible child from %s for %s on %s",
	local_child->id, rsc->id, local_node->details->uname);

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;
	enum rsc_role_e next_role = child_rsc->fns->state(child_rsc, current);

	if (is_set_recursive(child_rsc, pe_rsc_block, TRUE) == FALSE) {
	/* We only want instances that haven't failed */
	node = child_rsc->fns->location(child_rsc, NULL, current);
	}

	if (filter != RSC_ROLE_UNKNOWN && next_role != filter) {
	crm_trace("Filtered %s", child_rsc->id);
	continue;
	}

	if (node && local_node && node->details == local_node->details) {
	crm_trace("Pairing %s with %s on %s",
	local_child->id, child_rsc->id, node->details->uname);
	return child_rsc;

	} else if (node) {
	crm_trace("%s - %s vs %s", child_rsc->id, node->details->uname,
	local_node->details->uname);

	} else {
	crm_trace("%s - not allocated %d", child_rsc->id, current);
	}
	}

	crm_trace("Can't pair %s with %s", local_child->id, rsc->id);
	return NULL;
	}

	resource_t *
	find_compatible_child(resource_t * local_child, resource_t * rsc, enum rsc_role_e filter,
	gboolean current)
	{
	resource_t *pair = NULL;
	GListPtr gIter = NULL;
	GListPtr scratch = NULL;
	node_t *local_node = NULL;

	local_node = local_child->fns->location(local_child, NULL, current);
	if (local_node) {
	return find_compatible_child_by_node(local_child, local_node, rsc, filter, current);
	}

	scratch = g_hash_table_get_values(local_child->allowed_nodes);
	scratch = g_list_sort_with_data(scratch, sort_node_weight, NULL);

	gIter = scratch;
	for (; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	pair = find_compatible_child_by_node(local_child, node, rsc, filter, current);
	if (pair) {
	goto done;
	}
	}

	pe_rsc_debug(rsc, "Can't pair %s with %s", local_child->id, rsc->id);
	done:
	g_list_free(scratch);
	return pair;
	}

	void
	clone_rsc_colocation_lh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint)
	{
	/* -- Never called --
	*
	* Instead we add the colocation constraints to the child and call from there
	*/

	GListPtr gIter = rsc_lh->children;

	CRM_CHECK(FALSE, crm_err("This functionality is not thought to be used. Please report a bug."));
	CRM_CHECK(rsc_lh, return);
	CRM_CHECK(rsc_rh, return);

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	child_rsc->cmds->rsc_colocation_lh(child_rsc, rsc_rh, constraint);
	}

	return;
	}

	void
	clone_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint)
	{
	GListPtr gIter = NULL;
	gboolean do_interleave = FALSE;
	clone_variant_data_t *clone_data = NULL;
	clone_variant_data_t *clone_data_lh = NULL;

	CRM_CHECK(constraint != NULL, return);
	CRM_CHECK(rsc_lh != NULL, pe_err("rsc_lh was NULL for %s", constraint->id); return);
	CRM_CHECK(rsc_rh != NULL, pe_err("rsc_rh was NULL for %s", constraint->id); return);
	CRM_CHECK(rsc_lh->variant == pe_native, return);

	get_clone_variant_data(clone_data, constraint->rsc_rh);
	pe_rsc_trace(rsc_rh, "Processing constraint %s: %s -> %s %d",
	constraint->id, rsc_lh->id, rsc_rh->id, constraint->score);

	if (constraint->rsc_lh->variant >= pe_clone) {

	get_clone_variant_data(clone_data_lh, constraint->rsc_lh);
	if (clone_data_lh->interleave
	&& clone_data->clone_node_max != clone_data_lh->clone_node_max) {
	crm_config_err("Cannot interleave " XML_CIB_TAG_INCARNATION " %s and %s because"
	" they do not support the same number of" " resources per node",
	constraint->rsc_lh->id, constraint->rsc_rh->id);

	/* only the LHS side needs to be labeled as interleave */
	} else if (clone_data_lh->interleave) {
	do_interleave = TRUE;
	}
	}

	if (is_set(rsc_rh->flags, pe_rsc_provisional)) {
	pe_rsc_trace(rsc_rh, "%s is still provisional", rsc_rh->id);
	return;

	} else if (do_interleave) {
	resource_t *rh_child = NULL;

	rh_child = find_compatible_child(rsc_lh, rsc_rh, RSC_ROLE_UNKNOWN, FALSE);

	if (rh_child) {
	pe_rsc_debug(rsc_rh, "Pairing %s with %s", rsc_lh->id, rh_child->id);
	rsc_lh->cmds->rsc_colocation_lh(rsc_lh, rh_child, constraint);

	} else if (constraint->score >= INFINITY) {
	crm_notice("Cannot pair %s with instance of %s", rsc_lh->id, rsc_rh->id);
	assign_node(rsc_lh, NULL, TRUE);

	} else {
	pe_rsc_debug(rsc_rh, "Cannot pair %s with instance of %s", rsc_lh->id, rsc_rh->id);
	}

	return;

	} else if (constraint->score >= INFINITY) {
	GListPtr rhs = NULL;

	gIter = rsc_rh->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;
	node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE);

	if (chosen != NULL && is_set_recursive(child_rsc, pe_rsc_block, TRUE) == FALSE) {
	rhs = g_list_prepend(rhs, chosen);
	}
	}

	node_list_exclude(rsc_lh->allowed_nodes, rhs, FALSE);
	g_list_free(rhs);
	return;
	}

	gIter = rsc_rh->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	child_rsc->cmds->rsc_colocation_rh(rsc_lh, child_rsc, constraint);
	}
	}

	static enum action_tasks
	clone_child_action(action_t * action)
	{
	enum action_tasks result = no_action;
	resource_t child = (resource_t ) action->rsc->children->data;

	if (safe_str_eq(action->task, "notify")
	\|\| safe_str_eq(action->task, "notified")) {

	/* Find the action we're notifying about instead */

	int stop = 0;
	char *key = action->uuid;
	int lpc = strlen(key);

	for (; lpc > 0; lpc--) {
	if (key[lpc] == '_' && stop == 0) {
	stop = lpc;

	} else if (key[lpc] == '_') {
	char *task_mutable = NULL;

	lpc++;
	task_mutable = strdup(key + lpc);
	task_mutable[stop - lpc] = 0;

	crm_trace("Extracted action '%s' from '%s'", task_mutable, key);
	result = get_complex_task(child, task_mutable, TRUE);
	free(task_mutable);
	break;
	}
	}

	} else {
	result = get_complex_task(child, action->task, TRUE);
	}
	return result;
	}

	enum pe_action_flags
	clone_action_flags(action_t * action, node_t * node)
	{
	GListPtr gIter = NULL;
	gboolean any_runnable = FALSE;
	gboolean check_runnable = TRUE;
	enum action_tasks task = clone_child_action(action);
	enum pe_action_flags flags = (pe_action_optional \| pe_action_runnable \| pe_action_pseudo);
	const char *task_s = task2text(task);

	gIter = action->rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	action_t *child_action = NULL;
	resource_t child = (resource_t ) gIter->data;

	child_action =
	find_first_action(child->actions, NULL, task_s, child->children ? NULL : node);
	pe_rsc_trace(child, "Checking for %s in %s on %s", task_s, child->id,
	node ? node->details->uname : "none");
	if (child_action) {
	enum pe_action_flags child_flags = child->cmds->action_flags(child_action, node);

	if (is_set(flags, pe_action_optional)
	&& is_set(child_flags, pe_action_optional) == FALSE) {
	pe_rsc_trace(child, "%s is manditory because of %s", action->uuid,
	child_action->uuid);
	flags = crm_clear_bit(__FUNCTION__, action->rsc->id, flags, pe_action_optional);
	pe_clear_action_bit(action, pe_action_optional);
	}
	if (is_set(child_flags, pe_action_runnable)) {
	any_runnable = TRUE;
	}

	} else {

	GListPtr gIter2 = child->actions;

	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	action_t op = (action_t ) gIter2->data;

	pe_rsc_trace(child, "%s on %s (%s)", op->uuid,
	op->node ? op->node->details->uname : "none", op->task);
	}
	}
	}

	if (check_runnable && any_runnable == FALSE) {
	pe_rsc_trace(action->rsc, "%s is not runnable because no children are", action->uuid);
	flags = crm_clear_bit(__FUNCTION__, action->rsc->id, flags, pe_action_runnable);
	if (node == NULL) {
	pe_clear_action_bit(action, pe_action_runnable);
	}
	}

	return flags;
	}

	static enum pe_graph_flags
	clone_update_actions_interleave(action_t * first, action_t * then, node_t * node,
	enum pe_action_flags flags, enum pe_action_flags filter,
	enum pe_ordering type)
	{
	gboolean current = FALSE;
	resource_t *first_child = NULL;
	GListPtr gIter = then->rsc->children;
	enum pe_graph_flags changed = pe_graph_none; /pe_graph_disable /

	enum action_tasks task = clone_child_action(first);
	const char *first_task = task2text(task);

	/* Fix this - lazy */
	if (strstr(first->uuid, "_stopped_0") \|\| strstr(first->uuid, "_demoted_0")) {
	current = TRUE;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t then_child = (resource_t ) gIter->data;

	CRM_ASSERT(then_child != NULL);
	first_child = find_compatible_child(then_child, first->rsc, RSC_ROLE_UNKNOWN, current);
	if (first_child == NULL && current) {
	crm_trace("Ignore");

	} else if (first_child == NULL) {
	crm_debug("No match found for %s (%d / %s / %s)", then_child->id, current, first->uuid,
	then->uuid);

	/* Me no like this hack - but what else can we do?
	*
	* If there is no-one active or about to be active
	* on the same node as then_child, then they must
	* not be allowed to start
	*/
	if (type & (pe_order_runnable_left \| pe_order_implies_then) /* Mandatory */ ) {
	pe_rsc_info(then->rsc, "Inhibiting %s from being active", then_child->id);
	assign_node(then_child, NULL, TRUE);
	/* TODO - set changed correctly? */
	}

	} else {
	action_t *first_action = NULL;
	action_t *then_action = NULL;

	pe_rsc_debug(then->rsc, "Pairing %s with %s", first_child->id, then_child->id);

	first_action = find_first_action(first_child->actions, NULL, first_task, node);
	then_action = find_first_action(then_child->actions, NULL, then->task, node);

	CRM_CHECK(first_action != NULL \|\| is_set(first_child->flags, pe_rsc_orphan),
	crm_err("No action found for %s in %s (first)", first_task, first_child->id));

	if (then_action == NULL && is_not_set(then_child->flags, pe_rsc_orphan)
	&& crm_str_eq(then->task, RSC_STOP, TRUE) == FALSE
	&& crm_str_eq(then->task, RSC_DEMOTED, TRUE) == FALSE) {
	crm_err("Internal error: No action found for %s in %s (then)", then->task,
	then_child->id);
	}

	if (first_action == NULL \|\| then_action == NULL) {
	continue;
	}
	if (order_actions(first_action, then_action, type)) {
	crm_debug("Created constraint for %s -> %s", first_action->uuid, then_action->uuid);
	changed \|= (pe_graph_updated_first \| pe_graph_updated_then);
	}
	changed \|=
	then_child->cmds->update_actions(first_action, then_action, node,
	first_child->cmds->action_flags(first_action,
	node), filter,
	type);
	}
	}
	return changed;
	}

	enum pe_graph_flags
	clone_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
	enum pe_action_flags filter, enum pe_ordering type)
	{
	const char *rsc = "none";
	gboolean interleave = FALSE;
	enum pe_graph_flags changed = pe_graph_none;

	if (first->rsc != then->rsc
	&& first->rsc && first->rsc->variant >= pe_clone
	&& then->rsc && then->rsc->variant >= pe_clone) {
	clone_variant_data_t *clone_data = NULL;

	if (strstr(then->uuid, "_stop_0") \|\| strstr(then->uuid, "_demote_0")) {
	get_clone_variant_data(clone_data, first->rsc);
	rsc = first->rsc->id;
	} else {
	get_clone_variant_data(clone_data, then->rsc);
	rsc = then->rsc->id;
	}
	interleave = clone_data->interleave;
	}

	crm_trace("Interleave %s -> %s: %s (based on %s)",
	first->uuid, then->uuid, interleave ? "yes" : "no", rsc);

	if (interleave) {
	changed = clone_update_actions_interleave(first, then, node, flags, filter, type);

	} else if (then->rsc) {
	GListPtr gIter = then->rsc->children;

	changed \|= native_update_actions(first, then, node, flags, filter, type);

	for (; gIter != NULL; gIter = gIter->next) {
	enum pe_graph_flags child_changed = pe_graph_none;
	GListPtr lpc = NULL;
	resource_t child = (resource_t ) gIter->data;
	action_t *child_action = find_first_action(child->actions, NULL, then->task, node);

	if (child_action) {
	enum pe_action_flags child_flags = child->cmds->action_flags(child_action, node);

	if (is_set(child_flags, pe_action_runnable)) {

	child_changed \|=
	child->cmds->update_actions(first, child_action, node, flags, filter, type);
	}
	changed \|= child_changed;
	if (child_changed & pe_graph_updated_then) {
	for (lpc = child_action->actions_after; lpc != NULL; lpc = lpc->next) {
	action_wrapper_t other = (action_wrapper_t ) lpc->data;
	update_action(other->action);
	}
	}
	}
	}
	}

	return changed;
	}

	void
	clone_rsc_location(resource_t * rsc, rsc_to_node_t * constraint)
	{
	GListPtr gIter = rsc->children;

	pe_rsc_trace(rsc, "Processing location constraint %s for %s", constraint->id, rsc->id);

	native_rsc_location(rsc, constraint);

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	child_rsc->cmds->rsc_location(child_rsc, constraint);
	}
	}

	void
	clone_expand(resource_t * rsc, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	gIter = rsc->actions;
	for (; gIter != NULL; gIter = gIter->next) {
	action_t op = (action_t ) gIter->data;

	rsc->cmds->action_flags(op, NULL);
	}

	if (clone_data->start_notify) {
	collect_notification_data(rsc, TRUE, TRUE, clone_data->start_notify);
	expand_notification_data(clone_data->start_notify, data_set);
	create_notifications(rsc, clone_data->start_notify, data_set);
	}

	if (clone_data->stop_notify) {
	collect_notification_data(rsc, TRUE, TRUE, clone_data->stop_notify);
	expand_notification_data(clone_data->stop_notify, data_set);
	create_notifications(rsc, clone_data->stop_notify, data_set);
	}

	if (clone_data->promote_notify) {
	collect_notification_data(rsc, TRUE, TRUE, clone_data->promote_notify);
	expand_notification_data(clone_data->promote_notify, data_set);
	create_notifications(rsc, clone_data->promote_notify, data_set);
	}

	if (clone_data->demote_notify) {
	collect_notification_data(rsc, TRUE, TRUE, clone_data->demote_notify);
	expand_notification_data(clone_data->demote_notify, data_set);
	create_notifications(rsc, clone_data->demote_notify, data_set);
	}

	/* Now that the notifcations have been created we can expand the children */

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	child_rsc->cmds->expand(child_rsc, data_set);
	}

	native_expand(rsc, data_set);

	/* The notifications are in the graph now, we can destroy the notify_data */
	free_notification_data(clone_data->demote_notify);
	clone_data->demote_notify = NULL;
	free_notification_data(clone_data->stop_notify);
	clone_data->stop_notify = NULL;
	free_notification_data(clone_data->start_notify);
	clone_data->start_notify = NULL;
	free_notification_data(clone_data->promote_notify);
	clone_data->promote_notify = NULL;
	}

	node_t *
	rsc_known_on(resource_t * rsc, GListPtr * list)
	{
	GListPtr gIter = NULL;
	node_t *one = NULL;
	GListPtr result = NULL;

	if (rsc->children) {

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	rsc_known_on(child, &result);
	}

	} else if (rsc->known_on) {
	result = g_hash_table_get_values(rsc->known_on);
	}

	if (result && g_list_length(result) == 1) {
	one = g_list_nth_data(result, 0);
	}

	if (list) {
	GListPtr gIter = NULL;

	gIter = result;
	for (; gIter != NULL; gIter = gIter->next) {
	node_t node = (node_t ) gIter->data;

	if (list == NULL \|\| pe_find_node_id(list, node->details->id) == NULL) {
	list = g_list_prepend(list, node);
	}
	}
	}

	g_list_free(result);
	return one;
	}

	static resource_t *
	find_instance_on(resource_t * rsc, node_t * node)
	{
	GListPtr gIter = NULL;

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	GListPtr gIter2 = NULL;
	GListPtr known_list = NULL;
	resource_t child = (resource_t ) gIter->data;

	rsc_known_on(child, &known_list);

	gIter2 = known_list;
	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	node_t known = (node_t ) gIter2->data;

	if (node->details == known->details) {
	g_list_free(known_list);
	return child;
	}
	}
	g_list_free(known_list);
	}

	return NULL;
	}

	gboolean
	clone_create_probe(resource_t * rsc, node_t * node, action_t * complete,
	gboolean force, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	gboolean any_created = FALSE;
	clone_variant_data_t *clone_data = NULL;

	CRM_ASSERT(rsc);
	get_clone_variant_data(clone_data, rsc);

	rsc->children = g_list_sort(rsc->children, sort_rsc_id);
	if (rsc->children == NULL) {
	pe_warn("Clone %s has no children", rsc->id);
	return FALSE;
	}

	if (rsc->exclusive_discover) {
	node_t *allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (allowed && allowed->rsc_discover_mode != discover_exclusive) {
	/* exclusive discover is enabled and this node is not marked
	* as a node this resource should be discovered on
	*
	* remove the node from allowed_nodes so that the
	* notification contains only nodes that we might ever run
	* on
	*/
	g_hash_table_remove(rsc->allowed_nodes, node->details->id);

	/* Bit of a shortcut - might as well take it */
	return FALSE;
	}
	}

	if (is_not_set(rsc->flags, pe_rsc_unique)
	&& clone_data->clone_node_max == 1) {
	/* only look for one copy */
	resource_t *child = NULL;

	/* Try whoever we probed last time */
	child = find_instance_on(rsc, node);
	if (child) {
	return child->cmds->create_probe(child, node, complete, force, data_set);
	}

	/* Try whoever we plan on starting there */
	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	node_t *local_node = NULL;
	resource_t child_rsc = (resource_t ) gIter->data;

	CRM_ASSERT(child_rsc);
	local_node = child_rsc->fns->location(child_rsc, NULL, FALSE);
	if (local_node == NULL) {
	continue;
	}

	if (local_node->details == node->details) {
	return child_rsc->cmds->create_probe(child_rsc, node, complete, force, data_set);
	}
	}

	/* Fall back to the first clone instance */
	CRM_ASSERT(rsc->children);
	child = rsc->children->data;
	return child->cmds->create_probe(child, node, complete, force, data_set);
	}

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	if (child_rsc->cmds->create_probe(child_rsc, node, complete, force, data_set)) {
	any_created = TRUE;
	}

	if (any_created && is_not_set(rsc->flags, pe_rsc_unique)
	&& clone_data->clone_node_max == 1) {
	/* only look for one copy (clone :0) */
	break;
	}
	}

	return any_created;
	}

	void
	clone_append_meta(resource_t * rsc, xmlNode * xml)
	{
	char *name = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	name = crm_meta_name(XML_RSC_ATTR_UNIQUE);
	crm_xml_add(xml, name, is_set(rsc->flags, pe_rsc_unique) ? "true" : "false");
	free(name);

	name = crm_meta_name(XML_RSC_ATTR_NOTIFY);
	crm_xml_add(xml, name, is_set(rsc->flags, pe_rsc_notify) ? "true" : "false");
	free(name);

	name = crm_meta_name(XML_RSC_ATTR_INCARNATION_MAX);
	crm_xml_add_int(xml, name, clone_data->clone_max);
	free(name);

	name = crm_meta_name(XML_RSC_ATTR_INCARNATION_NODEMAX);
	crm_xml_add_int(xml, name, clone_data->clone_node_max);
	free(name);
	}

	GHashTable *
	clone_merge_weights(resource_t * rsc, const char rhs, GHashTable nodes, const char *attr,
	float factor, enum pe_weights flags)
	{
	return rsc_merge_weights(rsc, rhs, nodes, attr, factor, flags);
	}
	diff --git a/pengine/master.c b/pengine/master.c
	index 83b9d1c638..4837a5313e 100644
	--- a/pengine/master.c
	+++ b/pengine/master.c
	@@ -1,1078 +1,1078 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <crm/msg_xml.h>
	#include <allocate.h>
	#include <utils.h>

	#define VARIANT_CLONE 1
	#include <lib/pengine/variant.h>

	extern gint sort_clone_instance(gconstpointer a, gconstpointer b, gpointer data_set);

	static int master_score(resource_t * rsc, node_t * node, int not_set_value);

	static void
	child_promoting_constraints(clone_variant_data_t * clone_data, enum pe_ordering type,
	resource_t * rsc, resource_t * child, resource_t * last,
	pe_working_set_t * data_set)
	{
	if (child == NULL) {
	if (clone_data->ordered && last != NULL) {
	pe_rsc_trace(rsc, "Ordered version (last node)");
	/* last child promote before promoted started */
	new_rsc_order(last, RSC_PROMOTE, rsc, RSC_PROMOTED, type, data_set);
	}
	return;
	}

	/* child promote before global promoted */
	new_rsc_order(child, RSC_PROMOTE, rsc, RSC_PROMOTED, type, data_set);

	/* global promote before child promote */
	new_rsc_order(rsc, RSC_PROMOTE, child, RSC_PROMOTE, type, data_set);

	if (clone_data->ordered) {
	pe_rsc_trace(rsc, "Ordered version");
	if (last == NULL) {
	/* global promote before first child promote */
	last = rsc;

	}
	/* else: child/child relative promote */
	order_start_start(last, child, type);
	new_rsc_order(last, RSC_PROMOTE, child, RSC_PROMOTE, type, data_set);

	} else {
	pe_rsc_trace(rsc, "Un-ordered version");
	}
	}

	static void
	child_demoting_constraints(clone_variant_data_t * clone_data, enum pe_ordering type,
	resource_t * rsc, resource_t * child, resource_t * last,
	pe_working_set_t * data_set)
	{
	if (child == NULL) {
	if (clone_data->ordered && last != NULL) {
	pe_rsc_trace(rsc, "Ordered version (last node)");
	/* global demote before first child demote */
	new_rsc_order(rsc, RSC_DEMOTE, last, RSC_DEMOTE, pe_order_optional, data_set);
	}
	return;
	}

	/* child demote before global demoted */
	new_rsc_order(child, RSC_DEMOTE, rsc, RSC_DEMOTED, pe_order_implies_then_printed, data_set);

	/* global demote before child demote */
	new_rsc_order(rsc, RSC_DEMOTE, child, RSC_DEMOTE, pe_order_implies_first_printed, data_set);

	if (clone_data->ordered && last != NULL) {
	pe_rsc_trace(rsc, "Ordered version");

	/* child/child relative demote */
	new_rsc_order(child, RSC_DEMOTE, last, RSC_DEMOTE, type, data_set);

	} else if (clone_data->ordered) {
	pe_rsc_trace(rsc, "Ordered version (1st node)");
	/* first child stop before global stopped */
	new_rsc_order(child, RSC_DEMOTE, rsc, RSC_DEMOTED, type, data_set);

	} else {
	pe_rsc_trace(rsc, "Un-ordered version");
	}
	}

	static void
	master_update_pseudo_status(resource_t * rsc, gboolean * demoting, gboolean * promoting)
	{
	GListPtr gIter = NULL;

	if (rsc->children) {
	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	master_update_pseudo_status(child, demoting, promoting);
	}
	return;
	}

	CRM_ASSERT(demoting != NULL);
	CRM_ASSERT(promoting != NULL);

	gIter = rsc->actions;
	for (; gIter != NULL; gIter = gIter->next) {
	action_t action = (action_t ) gIter->data;

	if (promoting && demoting) {
	return;

	} else if (is_set(action->flags, pe_action_optional)) {
	continue;

	} else if (safe_str_eq(RSC_DEMOTE, action->task)) {
	*demoting = TRUE;

	} else if (safe_str_eq(RSC_PROMOTE, action->task)) {
	*promoting = TRUE;
	}
	}
	}

	#define apply_master_location(list) do { \
	gIter2 = list; \
	for(; gIter2 != NULL; gIter2 = gIter2->next) { \
	rsc_to_node_t cons = (rsc_to_node_t)gIter2->data; \
	\
	cons_node = NULL; \
	if(cons->role_filter == RSC_ROLE_MASTER) { \
	pe_rsc_trace(rsc, "Applying %s to %s", \
	cons->id, child_rsc->id); \
	cons_node = pe_find_node_id( \
	cons->node_list_rh, chosen->details->id); \
	} \
	if(cons_node != NULL) { \
	int new_priority = merge_weights( \
	child_rsc->priority, cons_node->weight); \
	pe_rsc_trace(rsc, "\t%s: %d->%d (%d)", child_rsc->id, \
	child_rsc->priority, new_priority, cons_node->weight); \
	child_rsc->priority = new_priority; \
	} \
	} \
	} while(0)

	static node_t *
	can_be_master(resource_t * rsc)
	{
	node_t *node = NULL;
	node_t *local_node = NULL;
	resource_t *parent = uber_parent(rsc);
	clone_variant_data_t *clone_data = NULL;

	#if 0
	enum rsc_role_e role = RSC_ROLE_UNKNOWN;

	role = rsc->fns->state(rsc, FALSE);
	crm_info("%s role: %s", rsc->id, role2text(role));
	#endif

	if (rsc->children) {
	GListPtr gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	if (can_be_master(child) == NULL) {
	pe_rsc_trace(rsc, "Child %s of %s can't be promoted", child->id, rsc->id);
	return NULL;
	}
	}
	}

	node = rsc->fns->location(rsc, NULL, FALSE);
	if (node == NULL) {
	pe_rsc_trace(rsc, "%s cannot be master: not allocated", rsc->id);
	return NULL;

	} else if (is_not_set(rsc->flags, pe_rsc_managed)) {
	if (rsc->fns->state(rsc, TRUE) == RSC_ROLE_MASTER) {
	crm_notice("Forcing unmanaged master %s to remain promoted on %s",
	rsc->id, node->details->uname);

	} else {
	return NULL;
	}

	} else if (rsc->priority < 0) {
	pe_rsc_trace(rsc, "%s cannot be master: preference: %d", rsc->id, rsc->priority);
	return NULL;

	} else if (can_run_resources(node) == FALSE) {
	- crm_trace("Node cant run any resources: %s", node->details->uname);
	+ crm_trace("Node can't run any resources: %s", node->details->uname);
	return NULL;
	}

	get_clone_variant_data(clone_data, parent);
	local_node = pe_hash_table_lookup(parent->allowed_nodes, node->details->id);

	if (local_node == NULL) {
	crm_err("%s cannot run on %s: node not allowed", rsc->id, node->details->uname);
	return NULL;

	} else if (local_node->count < clone_data->master_node_max
	\|\| is_not_set(rsc->flags, pe_rsc_managed)) {
	return local_node;

	} else {
	pe_rsc_trace(rsc, "%s cannot be master on %s: node full", rsc->id, node->details->uname);
	}

	return NULL;
	}

	static gint
	sort_master_instance(gconstpointer a, gconstpointer b, gpointer data_set)
	{
	int rc;
	enum rsc_role_e role1 = RSC_ROLE_UNKNOWN;
	enum rsc_role_e role2 = RSC_ROLE_UNKNOWN;

	const resource_t resource1 = (const resource_t )a;
	const resource_t resource2 = (const resource_t )b;

	CRM_ASSERT(resource1 != NULL);
	CRM_ASSERT(resource2 != NULL);

	role1 = resource1->fns->state(resource1, TRUE);
	role2 = resource2->fns->state(resource2, TRUE);

	rc = sort_rsc_index(a, b);
	if (rc != 0) {
	crm_trace("%s %c %s (index)", resource1->id, rc < 0 ? '<' : '>', resource2->id);
	return rc;
	}

	if (role1 > role2) {
	crm_trace("%s %c %s (role)", resource1->id, '<', resource2->id);
	return -1;

	} else if (role1 < role2) {
	crm_trace("%s %c %s (role)", resource1->id, '>', resource2->id);
	return 1;
	}

	return sort_clone_instance(a, b, data_set);
	}

	GHashTable *
	master_merge_weights(resource_t * rsc, const char rhs, GHashTable nodes, const char *attr,
	float factor, enum pe_weights flags)
	{
	return rsc_merge_weights(rsc, rhs, nodes, attr, factor, flags);
	}

	static void
	master_promotion_order(resource_t * rsc, pe_working_set_t * data_set)
	{
	GListPtr gIter = NULL;
	node_t *node = NULL;
	node_t *chosen = NULL;
	clone_variant_data_t *clone_data = NULL;
	char score[33];
	size_t len = sizeof(score);

	get_clone_variant_data(clone_data, rsc);

	if (clone_data->merged_master_weights) {
	return;
	}
	clone_data->merged_master_weights = TRUE;
	pe_rsc_trace(rsc, "Merging weights for %s", rsc->id);
	set_bit(rsc->flags, pe_rsc_merging);

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	pe_rsc_trace(rsc, "Sort index: %s = %d", child->id, child->sort_index);
	}
	dump_node_scores(LOG_DEBUG_3, rsc, "Before", rsc->allowed_nodes);

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	chosen = child->fns->location(child, NULL, FALSE);
	if (chosen == NULL \|\| child->sort_index < 0) {
	pe_rsc_trace(rsc, "Skipping %s", child->id);
	continue;
	}

	node = (node_t *) pe_hash_table_lookup(rsc->allowed_nodes, chosen->details->id);
	CRM_ASSERT(node != NULL);
	/* adds in master preferences and rsc_location.role=Master */
	score2char_stack(child->sort_index, score, len);
	pe_rsc_trace(rsc, "Adding %s to %s from %s", score,
	node->details->uname, child->id);
	node->weight = merge_weights(child->sort_index, node->weight);
	}

	dump_node_scores(LOG_DEBUG_3, rsc, "Middle", rsc->allowed_nodes);

	gIter = rsc->rsc_cons;
	for (; gIter != NULL; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	/* (re-)adds location preferences of resources that the
	* master instance should/must be colocated with
	*/
	if (constraint->role_lh == RSC_ROLE_MASTER) {
	enum pe_weights flags = constraint->score == INFINITY ? 0 : pe_weights_rollback;

	pe_rsc_trace(rsc, "RHS: %s with %s: %d", constraint->rsc_lh->id, constraint->rsc_rh->id,
	constraint->score);
	rsc->allowed_nodes =
	constraint->rsc_rh->cmds->merge_weights(constraint->rsc_rh, rsc->id,
	rsc->allowed_nodes,
	constraint->node_attribute,
	(float)constraint->score / INFINITY, flags);
	}
	}

	gIter = rsc->rsc_cons_lhs;
	for (; gIter != NULL; gIter = gIter->next) {
	rsc_colocation_t constraint = (rsc_colocation_t ) gIter->data;

	/* (re-)adds location preferences of resource that wish to be
	* colocated with the master instance
	*/
	if (constraint->role_rh == RSC_ROLE_MASTER) {
	pe_rsc_trace(rsc, "LHS: %s with %s: %d", constraint->rsc_lh->id, constraint->rsc_rh->id,
	constraint->score);
	rsc->allowed_nodes =
	constraint->rsc_lh->cmds->merge_weights(constraint->rsc_lh, rsc->id,
	rsc->allowed_nodes,
	constraint->node_attribute,
	(float)constraint->score / INFINITY,
	(pe_weights_rollback \|
	pe_weights_positive));
	}
	}

	gIter = rsc->rsc_tickets;
	for (; gIter != NULL; gIter = gIter->next) {
	rsc_ticket_t rsc_ticket = (rsc_ticket_t ) gIter->data;

	if (rsc_ticket->role_lh == RSC_ROLE_MASTER
	&& (rsc_ticket->ticket->granted == FALSE \|\| rsc_ticket->ticket->standby)) {
	resource_location(rsc, NULL, -INFINITY, "__stateful_without_ticket__", data_set);
	}
	}

	dump_node_scores(LOG_DEBUG_3, rsc, "After", rsc->allowed_nodes);

	/* write them back and sort */

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;

	chosen = child->fns->location(child, NULL, FALSE);
	if (is_not_set(child->flags, pe_rsc_managed) && child->next_role == RSC_ROLE_MASTER) {
	child->sort_index = INFINITY;

	} else if (chosen == NULL \|\| child->sort_index < 0) {
	pe_rsc_trace(rsc, "%s: %d", child->id, child->sort_index);

	} else {
	node = (node_t *) pe_hash_table_lookup(rsc->allowed_nodes, chosen->details->id);
	CRM_ASSERT(node != NULL);

	child->sort_index = node->weight;
	}
	pe_rsc_trace(rsc, "Set sort index: %s = %d", child->id, child->sort_index);
	}

	rsc->children = g_list_sort_with_data(rsc->children, sort_master_instance, data_set);
	clear_bit(rsc->flags, pe_rsc_merging);
	}

	static gboolean
	filter_anonymous_instance(resource_t * rsc, node_t * node)
	{
	GListPtr rIter = NULL;
	char *key = clone_strip(rsc->id);
	resource_t *parent = uber_parent(rsc);

	for (rIter = parent->children; rIter; rIter = rIter->next) {
	resource_t *child = rIter->data;
	resource_t *active = parent->fns->find_rsc(child, key, node, pe_find_clone\|pe_find_current);

	/*
	* Look for an active instance on $node, if there is one, only it receives the master score
	* Use ->find_rsc() because we might be a cloned group
	*/
	if(rsc == active) {
	pe_rsc_trace(rsc, "Found %s for %s active on %s: done", active->id, key, node->details->uname);
	free(key);
	return TRUE;
	} else if(active) {
	pe_rsc_trace(rsc, "Found %s for %s on %s: not %s", active->id, key, node->details->uname, rsc->id);
	free(key);
	return FALSE;
	} else {
	pe_rsc_trace(rsc, "%s on %s: not active", key, node->details->uname);
	}
	}

	for (rIter = parent->children; rIter; rIter = rIter->next) {
	resource_t *child = rIter->data;

	/*
	* We know its not running, but any score will still count if
	* the instance has been probed on $node
	*
	* Again use ->find_rsc() because we might be a cloned group
	* and knowing that other members of the group are known here
	* implies nothing
	*/
	rsc = parent->fns->find_rsc(child, key, NULL, pe_find_clone);
	CRM_LOG_ASSERT(rsc);
	if(rsc) {
	pe_rsc_trace(rsc, "Checking %s for %s on %s", rsc->id, key, node->details->uname);
	if (g_hash_table_lookup(rsc->known_on, node->details->id)) {
	free(key);
	return TRUE;
	}
	}
	}
	free(key);
	return FALSE;
	}

	static int
	master_score(resource_t * rsc, node_t * node, int not_set_value)
	{
	char *attr_name;
	char *name = rsc->id;
	const char *attr_value = NULL;
	int score = not_set_value, len = 0;

	if (rsc->children) {
	GListPtr gIter = rsc->children;

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child = (resource_t ) gIter->data;
	int c_score = master_score(child, node, not_set_value);

	if (score == not_set_value) {
	score = c_score;
	} else {
	score += c_score;
	}
	}
	return score;
	}

	if (node == NULL) {
	if (rsc->fns->state(rsc, TRUE) < RSC_ROLE_STARTED) {
	pe_rsc_trace(rsc, "Ingoring master score for %s: unknown state", rsc->id);
	return score;
	}

	} else {
	node_t *match = pe_find_node_id(rsc->running_on, node->details->id);
	node_t *known = pe_hash_table_lookup(rsc->known_on, node->details->id);

	if (is_not_set(rsc->flags, pe_rsc_unique) && filter_anonymous_instance(rsc, node)) {
	pe_rsc_trace(rsc, "Anonymous clone %s is allowed on %s", rsc->id, node->details->uname);

	} else if (match == NULL && known == NULL) {
	pe_rsc_trace(rsc, "%s (aka. %s) has been filtered on %s - ignoring", rsc->id,
	rsc->clone_name, node->details->uname);
	return score;
	}

	match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
	if (match == NULL) {
	return score;

	} else if (match->weight < 0) {
	pe_rsc_trace(rsc, "%s on %s has score: %d - ignoring",
	rsc->id, match->details->uname, match->weight);
	return score;
	}
	}

	if (rsc->clone_name) {
	/* Use the name the lrm knows this resource as,
	* since that's what crm_master would have used too
	*/
	name = rsc->clone_name;
	}

	len = 8 + strlen(name);
	attr_name = calloc(1, len);
	sprintf(attr_name, "master-%s", name);

	if (node) {
	attr_value = g_hash_table_lookup(node->details->attrs, attr_name);
	pe_rsc_trace(rsc, "%s: %s[%s] = %s", rsc->id, attr_name, node->details->uname,
	crm_str(attr_value));
	}

	if (attr_value != NULL) {
	score = char2score(attr_value);
	}

	free(attr_name);
	return score;
	}

	#define max(a, b) a<b?b:a

	static void
	apply_master_prefs(resource_t * rsc)
	{
	int score, new_score;
	GListPtr gIter = rsc->children;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	if (clone_data->applied_master_prefs) {
	/* Make sure we only do this once */
	return;
	}

	clone_data->applied_master_prefs = TRUE;

	for (; gIter != NULL; gIter = gIter->next) {
	GHashTableIter iter;
	node_t *node = NULL;
	resource_t child_rsc = (resource_t ) gIter->data;

	g_hash_table_iter_init(&iter, child_rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	if (can_run_resources(node) == FALSE) {
	/* This node will never be promoted to master,
	* so don't apply the master score as that may
	* lead to clone shuffling
	*/
	continue;
	}

	score = master_score(child_rsc, node, 0);
	if (score > 0) {
	new_score = merge_weights(node->weight, score);
	if (new_score != node->weight) {
	pe_rsc_trace(rsc, "\t%s: Updating preference for %s (%d->%d)",
	child_rsc->id, node->details->uname, node->weight, new_score);
	node->weight = new_score;
	}
	}

	new_score = max(child_rsc->priority, score);
	if (new_score != child_rsc->priority) {
	pe_rsc_trace(rsc, "\t%s: Updating priority (%d->%d)",
	child_rsc->id, child_rsc->priority, new_score);
	child_rsc->priority = new_score;
	}
	}
	}
	}

	static void
	set_role_slave(resource_t * rsc, gboolean current)
	{
	GListPtr gIter = rsc->children;

	if (current) {
	if (rsc->role == RSC_ROLE_STARTED) {
	rsc->role = RSC_ROLE_SLAVE;
	}

	} else {
	GListPtr allocated = NULL;

	rsc->fns->location(rsc, &allocated, FALSE);

	if (allocated) {
	rsc->next_role = RSC_ROLE_SLAVE;

	} else {
	rsc->next_role = RSC_ROLE_STOPPED;
	}
	g_list_free(allocated);
	}

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	set_role_slave(child_rsc, current);
	}
	}

	static void
	set_role_master(resource_t * rsc)
	{
	GListPtr gIter = rsc->children;

	if (rsc->next_role == RSC_ROLE_UNKNOWN) {
	rsc->next_role = RSC_ROLE_MASTER;
	}

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	set_role_master(child_rsc);
	}
	}

	node_t *
	master_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
	{
	int promoted = 0;
	GListPtr gIter = NULL;
	GListPtr gIter2 = NULL;

	GHashTableIter iter;
	node_t *node = NULL;
	node_t *chosen = NULL;
	node_t *cons_node = NULL;
	enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;

	char score[33];
	size_t len = sizeof(score);

	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	if (is_not_set(rsc->flags, pe_rsc_provisional)) {
	return NULL;

	} else if (is_set(rsc->flags, pe_rsc_allocating)) {
	pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
	return NULL;
	}

	apply_master_prefs(rsc);

	clone_color(rsc, prefer, data_set);

	set_bit(rsc->flags, pe_rsc_allocating);

	/* count now tracks the number of masters allocated */
	g_hash_table_iter_init(&iter, rsc->allowed_nodes);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	node->count = 0;
	}

	/*
	* assign priority
	*/
	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	GListPtr list = NULL;
	resource_t child_rsc = (resource_t ) gIter->data;

	pe_rsc_trace(rsc, "Assigning priority for %s: %s", child_rsc->id,
	role2text(child_rsc->next_role));

	if (child_rsc->fns->state(child_rsc, TRUE) == RSC_ROLE_STARTED) {
	set_role_slave(child_rsc, TRUE);
	}

	chosen = child_rsc->fns->location(child_rsc, &list, FALSE);
	if (g_list_length(list) > 1) {
	crm_config_err("Cannot promote non-colocated child %s", child_rsc->id);
	}

	g_list_free(list);
	if (chosen == NULL) {
	continue;
	}

	next_role = child_rsc->fns->state(child_rsc, FALSE);
	switch (next_role) {
	case RSC_ROLE_STARTED:
	case RSC_ROLE_UNKNOWN:
	CRM_CHECK(chosen != NULL, break);
	/*
	* Default to -1 if no value is set
	*
	* This allows master locations to be specified
	* based solely on rsc_location constraints,
	* but prevents anyone from being promoted if
	* neither a constraint nor a master-score is present
	*/
	child_rsc->priority = master_score(child_rsc, chosen, -1);
	break;

	case RSC_ROLE_SLAVE:
	case RSC_ROLE_STOPPED:
	child_rsc->priority = -INFINITY;
	break;
	case RSC_ROLE_MASTER:
	/* We will arrive here if we're re-creating actions after a stonith
	*/
	break;
	default:
	CRM_CHECK(FALSE /* unhandled */ ,
	crm_err("Unknown resource role: %d for %s", next_role, child_rsc->id));
	}

	apply_master_location(child_rsc->rsc_location);
	apply_master_location(rsc->rsc_location);

	gIter2 = child_rsc->rsc_cons;
	for (; gIter2 != NULL; gIter2 = gIter2->next) {
	rsc_colocation_t cons = (rsc_colocation_t ) gIter2->data;

	child_rsc->cmds->rsc_colocation_lh(child_rsc, cons->rsc_rh, cons);
	}

	child_rsc->sort_index = child_rsc->priority;
	pe_rsc_trace(rsc, "Assigning priority for %s: %d", child_rsc->id, child_rsc->priority);

	if (next_role == RSC_ROLE_MASTER) {
	child_rsc->sort_index = INFINITY;
	}
	}

	dump_node_scores(LOG_DEBUG_3, rsc, "Pre merge", rsc->allowed_nodes);
	master_promotion_order(rsc, data_set);

	/* mark the first N as masters */

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;
	score2char_stack(child_rsc->sort_index, score, len);

	chosen = child_rsc->fns->location(child_rsc, NULL, FALSE);
	if (show_scores) {
	fprintf(stdout, "%s promotion score on %s: %s\n",
	child_rsc->id, chosen ? chosen->details->uname : "none", score);

	} else {
	do_crm_log(scores_log_level, "%s promotion score on %s: %s",
	child_rsc->id, chosen ? chosen->details->uname : "none", score);
	}

	chosen = NULL; /* nuke 'chosen' so that we don't promote more than the
	* required number of instances
	*/

	if (child_rsc->sort_index < 0) {
	pe_rsc_trace(rsc, "Not supposed to promote child: %s", child_rsc->id);

	} else if (promoted < clone_data->master_max \|\| is_not_set(rsc->flags, pe_rsc_managed)) {
	chosen = can_be_master(child_rsc);
	}

	pe_rsc_debug(rsc, "%s master score: %d", child_rsc->id, child_rsc->priority);

	if (chosen == NULL) {
	set_role_slave(child_rsc, FALSE);
	continue;

	} else if(child_rsc->role < RSC_ROLE_MASTER
	&& is_set(data_set->flags, pe_flag_have_quorum) == FALSE
	&& data_set->no_quorum_policy == no_quorum_freeze) {
	crm_notice("Resource %s cannot be elevated from %s to %s: no-quorum-policy=freeze",
	child_rsc->id, role2text(child_rsc->role), role2text(child_rsc->next_role));
	set_role_slave(child_rsc, FALSE);
	continue;
	}

	chosen->count++;
	pe_rsc_info(rsc, "Promoting %s (%s %s)",
	child_rsc->id, role2text(child_rsc->role), chosen->details->uname);
	set_role_master(child_rsc);
	promoted++;
	}

	clone_data->masters_allocated = promoted;
	pe_rsc_info(rsc, "%s: Promoted %d instances of a possible %d to master",
	rsc->id, promoted, clone_data->master_max);

	clear_bit(rsc->flags, pe_rsc_provisional);
	clear_bit(rsc->flags, pe_rsc_allocating);

	return NULL;
	}

	void
	master_create_actions(resource_t * rsc, pe_working_set_t * data_set)
	{
	action_t *action = NULL;
	GListPtr gIter = rsc->children;
	action_t *action_complete = NULL;
	gboolean any_promoting = FALSE;
	gboolean any_demoting = FALSE;
	resource_t *last_promote_rsc = NULL;
	resource_t *last_demote_rsc = NULL;

	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	pe_rsc_debug(rsc, "Creating actions for %s", rsc->id);

	/* create actions as normal */
	clone_create_actions(rsc, data_set);

	for (; gIter != NULL; gIter = gIter->next) {
	gboolean child_promoting = FALSE;
	gboolean child_demoting = FALSE;
	resource_t child_rsc = (resource_t ) gIter->data;

	pe_rsc_trace(rsc, "Creating actions for %s", child_rsc->id);
	child_rsc->cmds->create_actions(child_rsc, data_set);
	master_update_pseudo_status(child_rsc, &child_demoting, &child_promoting);

	any_demoting = any_demoting \|\| child_demoting;
	any_promoting = any_promoting \|\| child_promoting;
	pe_rsc_trace(rsc, "Created actions for %s: %d %d", child_rsc->id, child_promoting,
	child_demoting);
	}

	/* promote */
	action = promote_action(rsc, NULL, !any_promoting);
	action_complete = custom_action(rsc, promoted_key(rsc),
	RSC_PROMOTED, NULL, !any_promoting, TRUE, data_set);

	action_complete->priority = INFINITY;
	update_action_flags(action, pe_action_pseudo);
	update_action_flags(action, pe_action_runnable);
	update_action_flags(action_complete, pe_action_pseudo);
	update_action_flags(action_complete, pe_action_runnable);

	if (clone_data->masters_allocated > 0) {
	update_action_flags(action, pe_action_runnable);
	update_action_flags(action_complete, pe_action_runnable);
	}

	child_promoting_constraints(clone_data, pe_order_optional,
	rsc, NULL, last_promote_rsc, data_set);

	if (clone_data->promote_notify == NULL) {
	clone_data->promote_notify =
	create_notification_boundaries(rsc, RSC_PROMOTE, action, action_complete, data_set);
	}

	/* demote */
	action = demote_action(rsc, NULL, !any_demoting);
	action_complete = custom_action(rsc, demoted_key(rsc),
	RSC_DEMOTED, NULL, !any_demoting, TRUE, data_set);
	action_complete->priority = INFINITY;

	update_action_flags(action, pe_action_pseudo);
	update_action_flags(action, pe_action_runnable);
	update_action_flags(action_complete, pe_action_pseudo);
	update_action_flags(action_complete, pe_action_runnable);

	child_demoting_constraints(clone_data, pe_order_optional, rsc, NULL, last_demote_rsc, data_set);

	if (clone_data->demote_notify == NULL) {
	clone_data->demote_notify =
	create_notification_boundaries(rsc, RSC_DEMOTE, action, action_complete, data_set);

	if (clone_data->promote_notify) {
	/* If we ever wanted groups to have notifications we'd need to move this to native_internal_constraints() one day
	* Requires exposing *_notify
	*/
	order_actions(clone_data->stop_notify->post_done, clone_data->promote_notify->pre,
	pe_order_optional);
	order_actions(clone_data->start_notify->post_done, clone_data->promote_notify->pre,
	pe_order_optional);
	order_actions(clone_data->demote_notify->post_done, clone_data->promote_notify->pre,
	pe_order_optional);
	order_actions(clone_data->demote_notify->post_done, clone_data->start_notify->pre,
	pe_order_optional);
	order_actions(clone_data->demote_notify->post_done, clone_data->stop_notify->pre,
	pe_order_optional);
	}
	}

	/* restore the correct priority */

	gIter = rsc->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	child_rsc->priority = rsc->priority;
	}
	}

	void
	master_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
	{
	GListPtr gIter = rsc->children;
	resource_t *last_rsc = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	clone_internal_constraints(rsc, data_set);

	/* global stopped before start */
	new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_START, pe_order_optional, data_set);

	/* global stopped before promote */
	new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_PROMOTE, pe_order_optional, data_set);

	/* global demoted before start */
	new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_START, pe_order_optional, data_set);

	/* global started before promote */
	new_rsc_order(rsc, RSC_STARTED, rsc, RSC_PROMOTE, pe_order_optional, data_set);

	/* global demoted before stop */
	new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_STOP, pe_order_optional, data_set);

	/* global demote before demoted */
	new_rsc_order(rsc, RSC_DEMOTE, rsc, RSC_DEMOTED, pe_order_optional, data_set);

	/* global demoted before promote */
	new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_PROMOTE, pe_order_optional, data_set);

	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	/* child demote before promote */
	new_rsc_order(child_rsc, RSC_DEMOTE, child_rsc, RSC_PROMOTE, pe_order_optional, data_set);

	child_promoting_constraints(clone_data, pe_order_optional,
	rsc, child_rsc, last_rsc, data_set);

	child_demoting_constraints(clone_data, pe_order_optional,
	rsc, child_rsc, last_rsc, data_set);

	last_rsc = child_rsc;
	}
	}

	static void
	node_hash_update_one(GHashTable * hash, node_t * other, const char *attr, int score)
	{
	GHashTableIter iter;
	node_t *node = NULL;
	const char *value = NULL;

	if (other == NULL) {
	return;

	} else if (attr == NULL) {
	attr = "#" XML_ATTR_UNAME;
	}

	value = g_hash_table_lookup(other->details->attrs, attr);
	g_hash_table_iter_init(&iter, hash);
	while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
	const char *tmp = g_hash_table_lookup(node->details->attrs, attr);

	if (safe_str_eq(value, tmp)) {
	crm_trace("%s: %d + %d", node->details->uname, node->weight, other->weight);
	node->weight = merge_weights(node->weight, score);
	}
	}
	}

	void
	master_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint)
	{
	GListPtr gIter = NULL;

	CRM_CHECK(rsc_rh != NULL, return);
	if (is_set(rsc_rh->flags, pe_rsc_provisional)) {
	return;

	} else if (constraint->role_rh == RSC_ROLE_UNKNOWN) {
	pe_rsc_trace(rsc_rh, "Handling %s as a clone colocation", constraint->id);
	clone_rsc_colocation_rh(rsc_lh, rsc_rh, constraint);
	return;
	}

	CRM_CHECK(rsc_lh != NULL, return);
	CRM_CHECK(rsc_lh->variant == pe_native, return);
	pe_rsc_trace(rsc_rh, "Processing constraint %s: %d", constraint->id, constraint->score);

	if (constraint->role_rh == RSC_ROLE_UNKNOWN) {

	gIter = rsc_rh->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;

	child_rsc->cmds->rsc_colocation_rh(rsc_lh, child_rsc, constraint);
	}

	} else if (is_set(rsc_lh->flags, pe_rsc_provisional)) {
	GListPtr rhs = NULL;

	gIter = rsc_rh->children;
	for (; gIter != NULL; gIter = gIter->next) {
	resource_t child_rsc = (resource_t ) gIter->data;
	node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE);
	enum rsc_role_e next_role = child_rsc->fns->state(child_rsc, FALSE);

	pe_rsc_trace(rsc_rh, "Processing: %s", child_rsc->id);
	if (chosen != NULL && next_role == constraint->role_rh) {
	pe_rsc_trace(rsc_rh, "Applying: %s %s %s %d", child_rsc->id,
	role2text(next_role), chosen->details->uname, constraint->score);
	if (constraint->score < INFINITY) {
	node_hash_update_one(rsc_lh->allowed_nodes, chosen,
	constraint->node_attribute, constraint->score);
	}
	rhs = g_list_prepend(rhs, chosen);
	}
	}

	/* Only do this if its not a master-master colocation
	* Doing this unconditionally would prevent the slaves from being started
	*/
	if (constraint->role_lh != RSC_ROLE_MASTER \|\| constraint->role_rh != RSC_ROLE_MASTER) {
	if (constraint->score >= INFINITY) {
	node_list_exclude(rsc_lh->allowed_nodes, rhs, TRUE);
	}
	}
	g_list_free(rhs);

	} else if (constraint->role_lh == RSC_ROLE_MASTER) {
	resource_t *rh_child = find_compatible_child(rsc_lh, rsc_rh, constraint->role_rh, FALSE);

	if (rh_child == NULL && constraint->score >= INFINITY) {
	pe_rsc_trace(rsc_lh, "%s can't be promoted %s", rsc_lh->id, constraint->id);
	rsc_lh->priority = -INFINITY;

	} else if (rh_child != NULL) {
	int new_priority = merge_weights(rsc_lh->priority, constraint->score);

	pe_rsc_debug(rsc_lh, "Applying %s to %s", constraint->id, rsc_lh->id);
	pe_rsc_debug(rsc_lh, "\t%s: %d->%d", rsc_lh->id, rsc_lh->priority, new_priority);
	rsc_lh->priority = new_priority;
	}
	}

	return;
	}

	void
	master_append_meta(resource_t * rsc, xmlNode * xml)
	{
	char *name = NULL;
	clone_variant_data_t *clone_data = NULL;

	get_clone_variant_data(clone_data, rsc);

	clone_append_meta(rsc, xml);

	name = crm_meta_name(XML_RSC_ATTR_MASTER_MAX);
	crm_xml_add_int(xml, name, clone_data->master_max);
	free(name);

	name = crm_meta_name(XML_RSC_ATTR_MASTER_NODEMAX);
	crm_xml_add_int(xml, name, clone_data->master_node_max);
	free(name);
	}
	diff --git a/tools/crmadmin.c b/tools/crmadmin.c
	index 0e8c91dfb2..293c099a8e 100644
	--- a/tools/crmadmin.c
	+++ b/tools/crmadmin.c
	@@ -1,575 +1,575 @@

	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>

	#include <sys/param.h>

	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>
	#include <libgen.h>

	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <crm/common/mainloop.h>

	#include <crm/cib.h>

	int message_timer_id = -1;
	int message_timeout_ms = 30 * 1000;

	GMainLoop *mainloop = NULL;
	crm_ipc_t *crmd_channel = NULL;
	char *admin_uuid = NULL;

	void usage(const char *cmd, int exit_status);
	gboolean do_init(void);
	int do_work(void);
	void crmadmin_ipc_connection_destroy(gpointer user_data);

	int admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata);
	char pluralSection(const char a_section);
	xmlNode *handleCibMod(void);
	int do_find_node_list(xmlNode * xml_node);
	gboolean admin_message_timeout(gpointer data);
	gboolean is_node_online(xmlNode * node_state);

	enum debug {
	debug_none,
	debug_dec,
	debug_inc
	};

	gboolean BE_VERBOSE = FALSE;
	int expected_responses = 1;

	gboolean BASH_EXPORT = FALSE;
	gboolean DO_HEALTH = FALSE;
	gboolean DO_RESET = FALSE;
	gboolean DO_RESOURCE = FALSE;
	gboolean DO_ELECT_DC = FALSE;
	gboolean DO_WHOIS_DC = FALSE;
	gboolean DO_NODE_LIST = FALSE;
	gboolean BE_SILENT = FALSE;
	gboolean DO_RESOURCE_LIST = FALSE;
	enum debug DO_DEBUG = debug_none;
	const char *crmd_operation = NULL;

	xmlNode *msg_options = NULL;

	const char *standby_on_off = "on";
	const char *admin_verbose = XML_BOOLEAN_FALSE;
	char *id = NULL;
	char *disconnect = NULL;
	char *dest_node = NULL;
	char *rsc_name = NULL;
	char *crm_option = NULL;

	int operation_status = 0;
	const char *sys_to = NULL;

	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	/* Top-level Options */
	{"help", 0, 0, '?', "\tThis text"},
	{"version", 0, 0, '$', "\tVersion information" },
	{"quiet", 0, 0, 'q', "\tDisplay only the essential query information"},
	{"verbose", 0, 0, 'V', "\tIncrease debug output"},

	{"-spacer-", 1, 0, '-', "\nCommands:"},
	/* daemon options */
	{"debug_inc", 1, 0, 'i', "Increase the crmd's debug level on the specified host"},
	{"debug_dec", 1, 0, 'd', "Decrease the crmd's debug level on the specified host"},
	{"status", 1, 0, 'S', "Display the status of the specified node." },
	{"-spacer-", 1, 0, '-', "\n\tResult is the node's internal FSM state which can be useful for debugging\n"},
	{"dc_lookup", 0, 0, 'D', "Display the uname of the node co-ordinating the cluster."},
	{"-spacer-", 1, 0, '-', "\n\tThis is an internal detail and is rarely useful to administrators except when deciding on which node to examine the logs.\n"},
	{"nodes", 0, 0, 'N', "\tDisplay the uname of all member nodes"},
	{"election", 0, 0, 'E', "(Advanced) Start an election for the cluster co-ordinator"},
	{"kill", 1, 0, 'K', "(Advanced) Shut down the crmd (not the rest of the clusterstack ) on the specified node"},
	{"health", 0, 0, 'H', NULL, 1},

	{"-spacer-", 1, 0, '-', "\nAdditional Options:"},
	{XML_ATTR_TIMEOUT, 1, 0, 't', "Time (in milliseconds) to wait before declaring the operation failed"},
	{"bash-export", 0, 0, 'B', "Create Bash export entries of the form 'export uname=uuid'\n"},

	{"-spacer-", 1, 0, '-', "Notes:"},
	{"-spacer-", 1, 0, '-', " The -i,-d,-K and -E commands are rarely used and may be removed in future versions."},

	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	int
	main(int argc, char **argv)
	{
	int option_index = 0;
	int argerr = 0;
	int flag;

	crm_log_cli_init("crmadmin");
	crm_set_options(NULL, "command [options]", long_options,
	"Development tool for performing some crmd-specific commands."
	"\n Likely to be replaced by crm_node in the future");
	if (argc < 2) {
	crm_help('?', EX_USAGE);
	}

	while (1) {
	flag = crm_get_option(argc, argv, &option_index);
	if (flag == -1)
	break;

	switch (flag) {
	case 'V':
	BE_VERBOSE = TRUE;
	admin_verbose = XML_BOOLEAN_TRUE;
	crm_bump_log_level(argc, argv);
	break;
	case 't':
	message_timeout_ms = atoi(optarg);
	if (message_timeout_ms < 1) {
	message_timeout_ms = 30 * 1000;
	}
	break;

	case '$':
	case '?':
	crm_help(flag, EX_OK);
	break;
	case 'D':
	DO_WHOIS_DC = TRUE;
	break;
	case 'B':
	BASH_EXPORT = TRUE;
	break;
	case 'K':
	DO_RESET = TRUE;
	crm_trace("Option %c => %s", flag, optarg);
	dest_node = strdup(optarg);
	crmd_operation = CRM_OP_LOCAL_SHUTDOWN;
	break;
	case 'q':
	BE_SILENT = TRUE;
	break;
	case 'i':
	DO_DEBUG = debug_inc;
	crm_trace("Option %c => %s", flag, optarg);
	dest_node = strdup(optarg);
	break;
	case 'd':
	DO_DEBUG = debug_dec;
	crm_trace("Option %c => %s", flag, optarg);
	dest_node = strdup(optarg);
	break;
	case 'S':
	DO_HEALTH = TRUE;
	crm_trace("Option %c => %s", flag, optarg);
	dest_node = strdup(optarg);
	break;
	case 'E':
	DO_ELECT_DC = TRUE;
	break;
	case 'N':
	DO_NODE_LIST = TRUE;
	break;
	case 'H':
	DO_HEALTH = TRUE;
	break;
	default:
	printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag);
	++argerr;
	break;
	}
	}

	if (optind < argc) {
	printf("non-option ARGV-elements: ");
	while (optind < argc)
	printf("%s ", argv[optind++]);
	printf("\n");
	}

	if (optind > argc) {
	++argerr;
	}

	if (argerr) {
	crm_help('?', EX_USAGE);
	}

	if (do_init()) {
	int res = 0;

	res = do_work();
	if (res > 0) {
	/* wait for the reply by creating a mainloop and running it until
	* the callbacks are invoked...
	*/
	mainloop = g_main_new(FALSE);
	crm_trace("Waiting for %d replies from the local CRM", expected_responses);

	message_timer_id = g_timeout_add(message_timeout_ms, admin_message_timeout, NULL);

	g_main_run(mainloop);

	} else if (res < 0) {
	crm_err("No message to send");
	operation_status = -1;
	}
	} else {
	crm_warn("Init failed, could not perform requested operations");
	operation_status = -2;
	}

	crm_trace("%s exiting normally", crm_system_name);
	return operation_status;
	}

	int
	do_work(void)
	{
	int ret = 1;

	/* construct the request */
	xmlNode *msg_data = NULL;
	gboolean all_is_good = TRUE;

	msg_options = create_xml_node(NULL, XML_TAG_OPTIONS);
	crm_xml_add(msg_options, XML_ATTR_VERBOSE, admin_verbose);
	crm_xml_add(msg_options, XML_ATTR_TIMEOUT, "0");

	if (DO_HEALTH == TRUE) {
	crm_trace("Querying the system");

	sys_to = CRM_SYSTEM_DC;

	if (dest_node != NULL) {
	sys_to = CRM_SYSTEM_CRMD;
	crmd_operation = CRM_OP_PING;

	if (BE_VERBOSE) {
	expected_responses = 1;
	}

	crm_xml_add(msg_options, XML_ATTR_TIMEOUT, "0");

	} else {
	crm_info("Cluster-wide health not available yet");
	all_is_good = FALSE;
	}

	} else if (DO_ELECT_DC) {
	/* tell the local node to initiate an election */

	dest_node = NULL;
	sys_to = CRM_SYSTEM_CRMD;
	crmd_operation = CRM_OP_VOTE;

	crm_xml_add(msg_options, XML_ATTR_TIMEOUT, "0");
	ret = 0; /* no return message */

	} else if (DO_WHOIS_DC) {
	dest_node = NULL;
	sys_to = CRM_SYSTEM_DC;
	crmd_operation = CRM_OP_PING;
	crm_xml_add(msg_options, XML_ATTR_TIMEOUT, "0");

	} else if (DO_NODE_LIST) {

	cib_t *the_cib = cib_new();
	xmlNode *output = NULL;

	int rc = the_cib->cmds->signon(the_cib, crm_system_name, cib_command);

	if (rc != pcmk_ok) {
	return -1;
	}

	rc = the_cib->cmds->query(the_cib, NULL, &output, cib_scope_local \| cib_sync_call);
	if(rc == pcmk_ok) {
	do_find_node_list(output);

	free_xml(output);
	}
	the_cib->cmds->signoff(the_cib);
	crm_exit(rc);

	} else if (DO_RESET) {
	- /* tell dest_node to initiate the shutdown proceedure
	+ /* tell dest_node to initiate the shutdown procedure
	*
	* if dest_node is NULL, the request will be sent to the
	* local node
	*/
	sys_to = CRM_SYSTEM_CRMD;
	crm_xml_add(msg_options, XML_ATTR_TIMEOUT, "0");

	ret = 0; /* no return message */

	} else if (DO_DEBUG == debug_inc) {
	/* tell dest_node to increase its debug level
	*
	* if dest_node is NULL, the request will be sent to the
	* local node
	*/
	sys_to = CRM_SYSTEM_CRMD;
	crmd_operation = CRM_OP_DEBUG_UP;

	ret = 0; /* no return message */

	} else if (DO_DEBUG == debug_dec) {
	/* tell dest_node to increase its debug level
	*
	* if dest_node is NULL, the request will be sent to the
	* local node
	*/
	sys_to = CRM_SYSTEM_CRMD;
	crmd_operation = CRM_OP_DEBUG_DOWN;

	ret = 0; /* no return message */

	} else {
	crm_err("Unknown options");
	all_is_good = FALSE;
	}

	if (all_is_good == FALSE) {
	crm_err("Creation of request failed. No message to send");
	return -1;
	}

	/* send it */
	if (crmd_channel == NULL) {
	crm_err("The IPC connection is not valid, cannot send anything");
	return -1;
	}

	if (sys_to == NULL) {
	if (dest_node != NULL) {
	sys_to = CRM_SYSTEM_CRMD;
	} else {
	sys_to = CRM_SYSTEM_DC;
	}
	}

	{
	xmlNode *cmd = create_request(crmd_operation, msg_data, dest_node, sys_to,
	crm_system_name, admin_uuid);

	crm_ipc_send(crmd_channel, cmd, 0, 0, NULL);
	free_xml(cmd);
	}

	return ret;
	}

	void
	crmadmin_ipc_connection_destroy(gpointer user_data)
	{
	crm_err("Connection to CRMd was terminated");
	if (mainloop) {
	g_main_quit(mainloop);
	} else {
	crm_exit(ENOTCONN);
	}
	}

	struct ipc_client_callbacks crm_callbacks = {
	.dispatch = admin_msg_callback,
	.destroy = crmadmin_ipc_connection_destroy
	};

	gboolean
	do_init(void)
	{
	mainloop_io_t *source =
	mainloop_add_ipc_client(CRM_SYSTEM_CRMD, G_PRIORITY_DEFAULT, 0, NULL, &crm_callbacks);

	admin_uuid = calloc(1, 11);
	if (admin_uuid != NULL) {
	snprintf(admin_uuid, 10, "%d", getpid());
	admin_uuid[10] = '\0';
	}

	crmd_channel = mainloop_get_ipc_client(source);

	if (DO_RESOURCE \|\| DO_RESOURCE_LIST \|\| DO_NODE_LIST) {
	return TRUE;

	} else if (crmd_channel != NULL) {
	xmlNode *xml = create_hello_message(admin_uuid, crm_system_name, "0", "1");

	crm_ipc_send(crmd_channel, xml, 0, 0, NULL);
	return TRUE;
	}
	return FALSE;
	}

	static bool
	validate_crm_message(xmlNode * msg, const char sys, const char uuid, const char *msg_type)
	{
	const char *type = NULL;
	const char *crm_msg_reference = NULL;

	if (msg == NULL) {
	return FALSE;
	}

	type = crm_element_value(msg, F_CRM_MSG_TYPE);
	crm_msg_reference = crm_element_value(msg, XML_ATTR_REFERENCE);

	if (type == NULL) {
	crm_info("No message type defined.");
	return FALSE;

	} else if (msg_type != NULL && strcasecmp(msg_type, type) != 0) {
	crm_info("Expecting a (%s) message but received a (%s).", msg_type, type);
	return FALSE;
	}

	if (crm_msg_reference == NULL) {
	crm_info("No message crm_msg_reference defined.");
	return FALSE;
	}

	return TRUE;
	}

	int
	admin_msg_callback(const char *buffer, ssize_t length, gpointer userdata)
	{
	static int received_responses = 0;
	xmlNode *xml = string2xml(buffer);

	received_responses++;
	g_source_remove(message_timer_id);

	crm_log_xml_trace(xml, "ipc");

	if (xml == NULL) {
	crm_info("XML in IPC message was not valid... " "discarding.");

	} else if (validate_crm_message(xml, crm_system_name, admin_uuid, XML_ATTR_RESPONSE) == FALSE) {
	crm_trace("Message was not a CRM response. Discarding.");

	} else if (DO_HEALTH) {
	xmlNode *data = get_message_xml(xml, F_CRM_DATA);
	const char *state = crm_element_value(data, "crmd_state");

	printf("Status of %s@%s: %s (%s)\n",
	crm_element_value(data, XML_PING_ATTR_SYSFROM),
	crm_element_value(xml, F_CRM_HOST_FROM),
	state, crm_element_value(data, XML_PING_ATTR_STATUS));

	if (BE_SILENT && state != NULL) {
	fprintf(stderr, "%s\n", state);
	}

	} else if (DO_WHOIS_DC) {
	const char *dc = crm_element_value(xml, F_CRM_HOST_FROM);

	printf("Designated Controller is: %s\n", dc);
	if (BE_SILENT && dc != NULL) {
	fprintf(stderr, "%s\n", dc);
	}
	crm_exit(pcmk_ok);
	}

	free_xml(xml);

	if (received_responses >= expected_responses) {
	crm_trace("Received expected number (%d) of messages from Heartbeat."
	" Exiting normally.", expected_responses);
	crm_exit(pcmk_ok);
	}

	message_timer_id = g_timeout_add(message_timeout_ms, admin_message_timeout, NULL);
	return 0;
	}

	gboolean
	admin_message_timeout(gpointer data)
	{
	fprintf(stderr, "No messages received in %d seconds.. aborting\n",
	(int)message_timeout_ms / 1000);
	crm_err("No messages received in %d seconds", (int)message_timeout_ms / 1000);
	operation_status = -3;
	g_main_quit(mainloop);
	return FALSE;
	}

	gboolean
	is_node_online(xmlNode * node_state)
	{
	const char *uname = crm_element_value(node_state, XML_ATTR_UNAME);
	const char *join_state = crm_element_value(node_state, XML_NODE_JOIN_STATE);
	const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED);
	const char *crm_state = crm_element_value(node_state, XML_NODE_IS_PEER);
	const char *ccm_state = crm_element_value(node_state, XML_NODE_IN_CLUSTER);

	if (safe_str_neq(join_state, CRMD_JOINSTATE_DOWN)
	&& crm_is_true(ccm_state)
	&& safe_str_eq(crm_state, "online")) {
	crm_trace("Node %s is online", uname);
	return TRUE;
	}
	crm_trace("Node %s: ccm=%s join=%s exp=%s crm=%s",
	uname, crm_str(ccm_state),
	crm_str(join_state), crm_str(exp_state), crm_str(crm_state));
	crm_trace("Node %s is offline", uname);
	return FALSE;
	}

	int
	do_find_node_list(xmlNode * xml_node)
	{
	int found = 0;
	xmlNode *node = NULL;
	xmlNode *nodes = get_object_root(XML_CIB_TAG_NODES, xml_node);

	for (node = __xml_first_child(nodes); node != NULL; node = __xml_next(node)) {
	if (crm_str_eq((const char *)node->name, XML_CIB_TAG_NODE, TRUE)) {

	if (BASH_EXPORT) {
	printf("export %s=%s\n",
	crm_element_value(node, XML_ATTR_UNAME),
	crm_element_value(node, XML_ATTR_ID));
	} else {
	printf("%s node: %s (%s)\n",
	crm_element_value(node, XML_ATTR_TYPE),
	crm_element_value(node, XML_ATTR_UNAME),
	crm_element_value(node, XML_ATTR_ID));
	}
	found++;
	}
	}

	if (found == 0) {
	printf("NO nodes configured\n");
	}

	return found;
	}
	diff --git a/tools/test.iso8601.c b/tools/test.iso8601.c
	index 6d70ce4461..6f54602404 100644
	--- a/tools/test.iso8601.c
	+++ b/tools/test.iso8601.c
	@@ -1,234 +1,234 @@
	/*
	* Copyright (C) 2005 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	*/

	#include <crm_internal.h>
	#include <crm/crm.h>
	#include <crm/common/iso8601.h>
	#include <unistd.h>

	char command = 0;

	/* INDENT-OFF */
	static struct crm_option long_options[] = {
	/* Top-level Options */
	{"help", 0, 0, '?', "\tThis text"},
	{"version", 0, 0, '$', "\tVersion information" },
	{"verbose", 0, 0, 'V', "\tIncrease debug output"},

	{"-spacer-", 0, 0, '-', "\nCommands:"},
	{"now", 0, 0, 'n', "\tDisplay the current date/time"},
	{"date", 1, 0, 'd', "Parse an ISO8601 date/time. Eg. '2005-01-20 00:30:00 +01:00' or '2005-040'"},
	{"period", 1, 0, 'p', "Parse an ISO8601 date/time with interval/period (wth start time). Eg. '2005-040/2005-043'"},
	{"duration", 1, 0, 'D', "Parse an ISO8601 date/time with duration (wth start time). Eg. '2005-040/P1M'"},
	{"expected", 1, 0, 'E', "Parse an ISO8601 date/time with duration (wth start time). Eg. '2005-040/P1M'"},

	{"-spacer-",0, 0, '-', "\nOutput Modifiers:"},
	{"seconds", 0, 0, 's', "\tShow result as a seconds since 0000-001 00:00:00Z"},
	{"epoch", 0, 0, 'S', "\tShow result as a seconds since EPOCH (1970-001 00:00:00Z)"},
	{"local", 0, 0, 'L', "\tShow result as a 'local' date/time"},
	{"ordinal", 0, 0, 'O', "\tShow result as an 'ordinal' date/time"},
	{"week", 0, 0, 'W', "\tShow result as an 'calendar week' date/time"},
	{"-spacer-",0, 0, '-', "\nFor more information on the ISO8601 standard, see: http://en.wikipedia.org/wiki/ISO_8601"},

	{0, 0, 0, 0}
	};
	/* INDENT-ON */

	static void
	log_time_period(int log_level, crm_time_period_t * dtp, int flags)
	{
	char *end = NULL;
	char *start = NULL;

	if(dtp) {
	start = crm_time_as_string(dtp->start, flags);
	end = crm_time_as_string(dtp->end, flags);
	}

	if (log_level < LOG_CRIT) {
	printf("Period: %s to %s\n", start, end);
	} else {
	do_crm_log(log_level, "Period: %s to %s", start, end);
	}
	free(start);
	free(end);
	}

	int
	main(int argc, char **argv)
	{
	int rc = 0;
	int argerr = 0;
	int flag;
	int index = 0;
	int print_options = 0;
	crm_time_t *duration = NULL;
	crm_time_t *date_time = NULL;
	crm_time_period_t *interval = NULL;

	const char *period_s = NULL;
	const char *duration_s = NULL;
	const char *date_time_s = NULL;
	const char *expected_s = NULL;

	crm_log_cli_init("iso8601");
	crm_set_options(NULL, "command [output modifier] ", long_options,
	"Display and parse ISO8601 dates and times");

	if (argc < 2) {
	argerr++;
	}

	while (1) {
	flag = crm_get_option(argc, argv, &index);
	if (flag == -1)
	break;

	switch (flag) {
	case 'V':
	crm_bump_log_level(argc, argv);
	break;
	case '?':
	case '$':
	crm_help(flag, 0);
	break;
	case 'n':
	date_time_s = "now";
	break;
	case 'd':
	date_time_s = optarg;
	break;
	case 'p':
	period_s = optarg;
	break;
	case 'D':
	duration_s = optarg;
	break;
	case 'E':
	expected_s = optarg;
	break;
	case 'S':
	print_options \|= crm_time_epoch;
	break;
	case 's':
	print_options \|= crm_time_seconds;
	break;
	case 'W':
	print_options \|= crm_time_weeks;
	break;
	case 'O':
	print_options \|= crm_time_ordinal;
	break;
	case 'L':
	print_options \|= crm_time_log_with_timezone;
	break;
	break;
	}
	}

	if (safe_str_eq("now", date_time_s)) {
	date_time = crm_time_new(NULL);

	if (date_time == NULL) {
	- fprintf(stderr, "Internal error: couldnt determin 'now' !\n");
	+ fprintf(stderr, "Internal error: couldn't determine 'now'!\n");
	crm_help('?', 1);
	}
	crm_time_log(LOG_TRACE, "Current date/time", date_time,
	crm_time_ordinal \| crm_time_log_date \| crm_time_log_timeofday);
	crm_time_log(-1, "Current date/time", date_time,
	print_options \| crm_time_log_date \| crm_time_log_timeofday);

	} else if (date_time_s) {
	date_time = crm_time_new(date_time_s);

	if (date_time == NULL) {
	fprintf(stderr, "Invalid date/time specified: %s\n", optarg);
	crm_help('?', 1);
	}
	crm_time_log(LOG_TRACE, "Date", date_time,
	crm_time_ordinal \| crm_time_log_date \| crm_time_log_timeofday);
	crm_time_log(-1, "Date", date_time,
	print_options \| crm_time_log_date \| crm_time_log_timeofday);
	}

	if (duration_s) {
	duration = crm_time_parse_duration(duration_s);

	if (duration == NULL) {
	fprintf(stderr, "Invalid duration specified: %s\n", duration_s);
	crm_help('?', 1);
	}
	crm_time_log(LOG_TRACE, "Duration", duration, crm_time_log_duration);
	crm_time_log(-1, "Duration", duration, print_options \| crm_time_log_duration);
	}

	if (period_s) {
	interval = crm_time_parse_period(period_s);

	if (interval == NULL) {
	fprintf(stderr, "Invalid interval specified: %s\n", optarg);
	crm_help('?', 1);
	}
	log_time_period(LOG_TRACE, interval,
	print_options \| crm_time_log_date \| crm_time_log_timeofday);
	log_time_period(-1, interval, print_options \| crm_time_log_date \| crm_time_log_timeofday);
	}

	if (date_time && duration) {
	crm_time_t *later = crm_time_add(date_time, duration);

	crm_time_log(LOG_TRACE, "Duration ends at", later,
	crm_time_ordinal \| crm_time_log_date \| crm_time_log_timeofday);
	crm_time_log(-1, "Duration ends at", later,
	print_options \| crm_time_log_date \| crm_time_log_timeofday \|
	crm_time_log_with_timezone);
	if (expected_s) {
	char *dt_s = crm_time_as_string(later,
	print_options \| crm_time_log_date \|
	crm_time_log_timeofday);
	if (safe_str_neq(expected_s, dt_s)) {
	rc = 1;
	}
	free(dt_s);
	}
	crm_time_free(later);

	} else if (date_time && expected_s) {
	char *dt_s = crm_time_as_string(date_time,
	print_options \| crm_time_log_date \| crm_time_log_timeofday);

	if (safe_str_neq(expected_s, dt_s)) {
	rc = 1;
	}
	free(dt_s);
	}

	/* if(date_time && interval) { */
	/* } */

	crm_time_free(date_time);
	crm_time_free(duration);
	if (interval) {
	crm_time_free(interval->start);
	crm_time_free(interval->end);
	crm_time_free(interval->diff);
	free(interval);
	}

	return crm_exit(rc);
	}

File Metadata

Mime Type: text/x-diff
Expires: Thu, Jul 10, 3:20 AM (17 h, 52 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1990439
Default Alt Text: (1000 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions