No OneTemporary
Actions

Size

180 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/crm/cib/cibio.c b/crm/cib/cibio.c
	index 4fe2c688af..ace0abc18d 100644
	--- a/crm/cib/cibio.c
	+++ b/crm/cib/cibio.c
	@@ -1,432 +1,434 @@
	-/* $Id: cibio.c,v 1.21 2004/04/29 15:33:03 andrew Exp $ */
	+/* $Id: cibio.c,v 1.22 2004/05/11 17:54:02 andrew Exp $ */
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/

	#include <portability.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>
	#include <sys/stat.h>

	#include <string.h>
	#include <stdlib.h>

	#include <errno.h>
	#include <fcntl.h>

	#include <crm/crm.h>

	#include <libxml/tree.h>
	#include <crm/cib.h>
	#include <cibio.h>
	#include <crm/common/msgutils.h> // for getNow()
	#include <crm/msg_xml.h>
	#include <crm/common/xmlutils.h>

	#include <crm/dmalloc_wrapper.h>

	const char * local_resource_path[] =
	{
	XML_CIB_TAG_STATUS,
	};

	const char * resource_path[] =
	{
	XML_CIB_TAG_RESOURCES,
	};

	const char * node_path[] =
	{
	XML_CIB_TAG_NODES,
	};

	const char * constraint_path[] =
	{
	XML_CIB_TAG_CONSTRAINTS,
	};

	gboolean initialized = FALSE;
	xmlNodePtr the_cib = NULL;
	xmlNodePtr node_search = NULL;
	xmlNodePtr resource_search = NULL;
	xmlNodePtr constraint_search = NULL;
	xmlNodePtr status_search = NULL;

	/*
	* It is the callers responsibility to free both the new CIB (output)
	* and the new CIB (input)
	*/
	xmlNodePtr
	createEmptyCib(void)
	{
	xmlNodePtr cib_root = NULL, config = NULL, status = NULL;

	cib_root = create_xml_node(NULL, XML_TAG_CIB);

	config = create_xml_node(cib_root, XML_CIB_TAG_CONFIGURATION);
	status = create_xml_node(cib_root, XML_CIB_TAG_STATUS);

	set_node_tstamp(cib_root);
	set_node_tstamp(config);
	set_node_tstamp(status);

	set_xml_property_copy(cib_root, "version", "1");
	set_xml_property_copy(cib_root, "generated", "true");

	create_xml_node(config, XML_CIB_TAG_NODES);
	create_xml_node(config, XML_CIB_TAG_RESOURCES);
	create_xml_node(config, XML_CIB_TAG_CONSTRAINTS);

	if (verifyCibXml(cib_root)) {
	FNRET(cib_root);
	}
	cl_log(LOG_CRIT,
	"The generated CIB did not pass integrity testing!!"
	" All hope is lost.");
	FNRET(NULL);
	}

	gboolean
	verifyCibXml(xmlNodePtr cib)
	{
	gboolean is_valid = TRUE;
	xmlNodePtr tmp_node = NULL;
	FNIN();

	if (cib == NULL) {
	cl_log(LOG_ERR, "XML Buffer was empty.");
	FNRET(FALSE);
	}

	tmp_node = get_object_root(XML_CIB_TAG_NODES, cib);
	if (tmp_node == NULL) is_valid = FALSE;

	tmp_node = get_object_root(XML_CIB_TAG_RESOURCES, cib);
	if (tmp_node == NULL) is_valid = FALSE;

	tmp_node = get_object_root(XML_CIB_TAG_CONSTRAINTS, cib);
	if (tmp_node == NULL) is_valid = FALSE;

	tmp_node = get_object_root(XML_CIB_TAG_STATUS, cib);
	if (tmp_node == NULL) is_valid = FALSE;

	// more integrity tests

	FNRET(is_valid);
	}

	/*
	* It is the callers responsibility to free the output of this function
	*/
	xmlNodePtr
	readCibXml(char *buffer)
	{
	xmlNodePtr root = string2xml(buffer);
	if (verifyCibXml(root) == FALSE) {
	free_xml(root);
	FNRET(createEmptyCib());
	}
	FNRET(root);
	}

	/*
	* It is the callers responsibility to free the output of this function
	*/
	xmlNodePtr
	readCibXmlFile(const char *filename)
	{
	int s_res = -1;
	struct stat buf;
	xmlNodePtr root = NULL;
	FNIN();

	if(filename != NULL) {
	s_res = stat(filename, &buf);
	}

	if (s_res == 0) {
	FILE *cib_file = fopen(filename, "r");
	root = file2xml(cib_file);
	set_xml_property_copy(root, "generated", "false");
	fclose(cib_file);

	} else {
	cl_log(LOG_WARNING,
	"Stat of (%s) failed, file does not exist.",
	CIB_FILENAME);
	}

	if (verifyCibXml(root) == FALSE) {
	free_xml(root);
	// FNRET(createEmptyCib());
	root = NULL;
	}

	FNRET(root);
	}

	/*
	* The caller should never free the return value
	*/
	xmlNodePtr
	get_the_CIB(void)
	{
	FNIN();
	FNRET(the_cib);
	}

	gboolean
	uninitializeCib(void)
	{
	xmlNodePtr tmp_cib = the_cib;
	FNIN();

	if(tmp_cib == NULL) {
	cl_log(LOG_ERR, "The CIB has already been deallocated.");
	FNRET(FALSE);
	}

	initialized = FALSE;
	the_cib = NULL;
	node_search = NULL;
	resource_search = NULL;
	constraint_search = NULL;
	status_search = NULL;

	cl_log(LOG_WARNING, "Deallocating the CIB.");

	free_xml(tmp_cib);

	cl_log(LOG_WARNING, "The CIB has been deallocated.");

	FNRET(TRUE);
	}




	/*
	* This method will not free the old CIB pointer or the new one.
	* We rely on the caller to have saved a pointer to the old CIB
	* and to free the old/bad one depending on what is appropriate.
	*/
	gboolean
	initializeCib(xmlNodePtr new_cib)
	{
	if (verifyCibXml(new_cib)) {

	initialized = FALSE;
	the_cib = new_cib;

	// update search paths
	/* not used yet...
	node_search =
	get_object_root(XML_CIB_TAG_NODES, new_cib);
	resource_search =
	get_object_root(XML_CIB_TAG_RESOURCES, new_cib);
	constraint_search =
	get_object_root(XML_CIB_TAG_CONSTRAINTS, new_cib);
	status_search =
	get_object_root(XML_CIB_TAG_STATUS, new_cib);
	*/
	initialized = TRUE;

	CRM_DEBUG("CIB initialized");
	FNRET(TRUE);
	}
	else {
	cl_log(LOG_ERR, "CIB Verification failed");
	}

	FNRET(FALSE);

	}

	int
	moveFile(const char *oldname,
	const char *newname,
	gboolean backup,
	char *ext)
	{
	/* move 'oldname' to 'newname' by creating a hard link to it
	* and then removing the original hard link
	*/
	int res = 0;
	struct stat tmp;
	int s_res = stat(newname, &tmp);
	FNIN();

	cl_log(LOG_INFO, "Stat of %s (code: %d).", newname, s_res);
	if (s_res >= 0)
	{
	if (backup == TRUE) {
	char backname[1024];
	static const char *back_ext = "bak";
	if (ext != NULL) back_ext = (char*)ext;

	snprintf(backname, sizeof(backname)-1,
	"%s.%s", newname, back_ext);
	moveFile(newname, backname, FALSE, NULL);
	} else {
	res = unlink(newname);
	if (res < 0) {
	perror("Could not remove the current backup of Cib");
	FNRET(-1);
	}
	}
	}

	s_res = stat(oldname, &tmp);
	cl_log(LOG_INFO, "Stat of %s (code: %d).", oldname, s_res);

	if (s_res >= 0) {
	res = link(oldname, newname);
	if (res < 0) {
	perror("Could not create backup of current Cib");
	FNRET(-2);
	}
	res = unlink(oldname);
	if (res < 0) {
	perror("Could not unlink the current Cib");
	FNRET(-3);
	}
	}

	FNRET(0);

	}


	int
	activateCibBuffer(char buffer, const char filename)
	{
	int result = -1;
	xmlNodePtr local_cib = NULL;
	FNIN();

	local_cib = readCibXml(buffer);
	result = activateCibXml(local_cib, filename);

	FNRET(result);
	}

	/*
	* This method will free the old CIB pointer on success and the new one
	* on failure.
	*/
	int
	activateCibXml(xmlNodePtr new_cib, const char *filename)
	{
	int error_code = 0;
	xmlNodePtr saved_cib = get_the_CIB();
	const char *filename_bak = CIB_BACKUP; // calculate
	xmlDocPtr foo;
	FNIN();

	-
	-
	if (initializeCib(new_cib) == TRUE) {
	int res = moveFile(filename, filename_bak, FALSE, NULL);

	if (res < 0) {
	cl_log(LOG_INFO,
	"Could not make backup of the current Cib "
	"(code: %d)... aborting update.", res);
	error_code = -1;
	} else {
	cl_log(LOG_INFO,
	"Writing CIB out to %s",
	CIB_FILENAME);

	if (new_cib->doc == NULL) {
	cl_log(LOG_INFO,
	"Writing of a node tree with a NULL "
	"document will fail, creating a new "
	"back link.");
	foo = xmlNewDoc("1.0");
	xmlDocSetRootElement(foo, new_cib);
	xmlSetTreeDoc(new_cib,foo);
	}

	-
	+ time_t now = time(NULL);
	+ char *now_str = asctime(localtime(&now));
	+ set_xml_property_copy(new_cib, "last_written",now_str);
	+ free(now_str);
	+
	/* save it.
	* set arg 3 to 0 to disable line breaks,1 to enable
	* res == num bytes saved
	*/
	res = xmlSaveFormatFile(filename,
	new_cib->doc,
	1);

	/* for some reason, reading back after saving with
	* line-breaks doesnt go real well
	*/
	cl_log(LOG_INFO,
	"Saved %d bytes to the Cib as XML",
	res);

	if (res < 0) {
	// assume 0 is good
	if (moveFile(filename_bak,
	filename,
	FALSE,
	NULL) < -1) {
	cl_log(LOG_CRIT,
	"Could not restore the "
	"backup of the current Cib "
	"(code: %d)... panic!",
	res);
	error_code = -2;
	// should probably exit here
	} else if (initializeCib(saved_cib) == FALSE) {
	// oh we are so dead
	cl_log(LOG_CRIT,
	"Could not re-initialize "
	"with the old CIB. "
	"Everything is about to go "
	"pear shaped");
	error_code = -3;
	} else {
	cl_log(LOG_CRIT,
	"Update of Cib failed "
	"(code: %d)... reverted to "
	"last known valid version",
	res);

	error_code = -4;
	}
	}
	}
	}
	else
	{
	cl_log(LOG_INFO, "Ignoring invalid or NULL Cib");
	error_code = -5;
	}

	// Make sure memory is cleaned up appropriately
	if (error_code != 0) {
	// CRM_DEBUG("Freeing new CIB %p", new_cib);
	free_xml(new_cib);
	} else {
	// CRM_DEBUG("Freeing saved CIB %p", saved_cib);
	free_xml(saved_cib);
	}

	FNRET(error_code);

	}
	diff --git a/crm/cib/cibmessages.c b/crm/cib/cibmessages.c
	index 4f15bb416c..133e0d20d0 100644
	--- a/crm/cib/cibmessages.c
	+++ b/crm/cib/cibmessages.c
	@@ -1,463 +1,462 @@
	-/* $Id: cibmessages.c,v 1.32 2004/04/29 15:33:03 andrew Exp $ */
	+/* $Id: cibmessages.c,v 1.33 2004/05/11 17:54:02 andrew Exp $ */
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/

	#include <portability.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>

	#include <crm/crm.h>

	#include <clplumbing/cl_log.h>

	#include <libxml/tree.h>
	#include <time.h>

	#include <crm/common/msgutils.h>
	#include <crm/common/xmlutils.h>
	#include <crm/cib.h>
	#include <cibio.h>
	#include <crm/msg_xml.h>

	#include <clplumbing/cl_log.h>
	#include <cibprimatives.h>
	#include <cibmessages.h>

	#include <crm/dmalloc_wrapper.h>

	enum cib_result updateList(xmlNodePtr local_cib,
	xmlNodePtr update_command,
	xmlNodePtr failed,
	int operation,
	const char *section);

	xmlNodePtr createCibFragmentAnswer(const char *section, xmlNodePtr failed);

	gboolean replace_section(const char *section,
	xmlNodePtr tmpCib,
	xmlNodePtr command);

	gboolean check_generation(xmlNodePtr newCib, xmlNodePtr oldCib);

	gboolean update_results(xmlNodePtr failed,
	xmlNodePtr target,
	int operation,
	int return_code);

	xmlNodePtr
	cib_process_request(const char *op,
	const xmlNodePtr options,
	const xmlNodePtr fragment,
	enum cib_result *result)
	{

	const char *verbose = NULL;
	const char *section = NULL;
	const char *output_section = NULL;
	xmlNodePtr failed = NULL;
	xmlNodePtr cib_answer = NULL;

	gboolean update_the_cib = FALSE;
	int cib_update_op = CIB_OP_NONE;
	xmlNodePtr tmpCib;
	char *new_value = NULL;
	char *old_value = NULL;
	int int_value = -1;

	FNIN();

	*result = CIBRES_OK;
	verbose = xmlGetProp(options, XML_ATTR_VERBOSE);
	section = xmlGetProp(options, XML_ATTR_FILTER_TYPE);
	failed = create_xml_node(NULL, XML_TAG_FAILED);

	cl_log(LOG_DEBUG, "[cib] Processing \"%s\" event", op);

	if(op == NULL) {
	*result = CIBRES_FAILED;
	cl_log(LOG_ERR, "No operation specified\n");

	} else if(strcmp("noop", op) == 0) {
	;

	} else if(strcmp("quit", op) == 0) {
	cl_log(LOG_WARNING,
	"The CRMd has asked us to exit... complying");
	exit(0);

	} else if (strcmp(CRM_OPERATION_PING, op) == 0) {
	cib_answer =
	createPingAnswerFragment(CRM_SYSTEM_CIB, "ok");

	} else if (strcmp(CRM_OPERATION_BUMP, op) == 0) {
	tmpCib = get_cib_copy();
	CRM_DEBUG("Handling a %s for section=%s of the cib",
	CRM_OPERATION_BUMP, section);

	// modify the timestamp
	set_node_tstamp(tmpCib);
	old_value =
	xmlGetProp(get_the_CIB(), XML_ATTR_GENERATION);

	if(old_value != NULL) {
	new_value = (char)cl_malloc(128(sizeof(char)));
	int_value = atoi(old_value);
	sprintf(new_value, "%d", ++int_value);
	} else {
	new_value = cl_strdup("0");
	}

	cl_log(LOG_DEBUG, "Generation %d(%s)->%s",
	int_value, old_value, new_value);

	set_xml_property_copy(tmpCib, XML_ATTR_GENERATION, new_value);
	cl_free(new_value);

	if(activateCibXml(tmpCib, CIB_FILENAME) >= 0) {
	verbose = "true";
	} else {
	*result = CIBRES_FAILED;
	}


	} else if (strcmp("query", op) == 0) {
	CRM_DEBUG("Handling a query for section=%s of the cib",
	section);
	/* force a pick-up of the relevant section before
	* returning
	*/
	verbose = "true";

	} else if (strcmp(CRM_OPERATION_ERASE, op) == 0) {
	xmlNodePtr new_cib = createEmptyCib();

	// Preserve generation counters etc
	copy_in_properties(new_cib, get_the_CIB());

	if (activateCibXml(new_cib, CIB_FILENAME) < 0) {
	*result = CIBRES_FAILED;
	}

	} else if (strcmp(CRM_OPERATION_CREATE, op) == 0) {
	update_the_cib = TRUE;
	cib_update_op = CIB_OP_ADD;

	} else if (strcmp(CRM_OPERATION_UPDATE, op) == 0
	\|\| strcmp(CRM_OPERATION_WELCOME, op) == 0
	\|\| strcmp(CRM_OPERATION_SHUTDOWN_REQ, op) == 0) {
	update_the_cib = TRUE;
	cib_update_op = CIB_OP_MODIFY;

	} else if (strcmp(CRM_OPERATION_DELETE, op) == 0) {
	update_the_cib = TRUE;
	cib_update_op = CIB_OP_DELETE;

	} else if (strcmp(CRM_OPERATION_REPLACE, op) == 0) {
	CRM_DEBUG("Replacing section=%s of the cib", section);
	section = xmlGetProp(fragment, XML_ATTR_SECTION);

	if (section == NULL
	\|\| strlen(section) == 0
	\|\| strcmp("all", section) == 0) {
	tmpCib = copy_xml_node_recursive(
	find_xml_node(fragment, XML_TAG_CIB));

	} else {
	tmpCib = copy_xml_node_recursive(get_the_CIB());
	replace_section(section, tmpCib, fragment);
	}

	/*if(check_generation(cib_updates, tmpCib) == FALSE)
	*result = "discarded old update";
	else */
	if (activateCibXml(tmpCib, CIB_FILENAME) < 0)
	*result = CIBRES_FAILED;
	} else {
	*result = CIBRES_FAILED_NOTSUPPORTED;
	cl_log(LOG_ERR, "Action [%s] is not supported by the CIB", op);
	}

	if (update_the_cib) {
	CRM_DEBUG("Backing up CIB");
	tmpCib = copy_xml_node_recursive(get_the_CIB());
	section = xmlGetProp(fragment, XML_ATTR_SECTION);

	CRM_DEBUG("Updating section=%s of the cib (op=%s)",
	section, op);

	// should we be doing this?
	// do logging

	// make changes to a temp copy then activate
	if(section == NULL) {
	cl_log(LOG_ERR, "No section specified in %s",
	XML_ATTR_FILTER_TYPE);
	*result = CIBRES_FAILED_NOSECTION;

	} else if(strcmp("all", section) == 0
	&& cib_update_op == CIB_OP_DELETE) {
	// delete

	/* order is no longer important here */
	updateList(tmpCib, fragment, failed, cib_update_op,
	XML_CIB_TAG_STATUS);
	updateList(tmpCib, fragment, failed, cib_update_op,
	XML_CIB_TAG_CONSTRAINTS);
	updateList(tmpCib, fragment, failed, cib_update_op,
	XML_CIB_TAG_RESOURCES);
	updateList(tmpCib, fragment, failed, cib_update_op,
	XML_CIB_TAG_NODES);

	} else if(strcmp("all", section) == 0) {
	/* order is no longer important here */
	updateList(tmpCib, fragment, failed, cib_update_op,
	XML_CIB_TAG_NODES);
	updateList(tmpCib, fragment, failed, cib_update_op,
	XML_CIB_TAG_RESOURCES);
	updateList(tmpCib, fragment, failed, cib_update_op,
	XML_CIB_TAG_CONSTRAINTS);
	updateList(tmpCib, fragment, failed, cib_update_op,
	XML_CIB_TAG_STATUS);
	} else {
	*result = updateList(tmpCib, fragment, failed,
	cib_update_op, section);
	}

	CRM_DEBUG("Activating temporary CIB");
	/* if(check_generation(cib_updates, tmpCib) == FALSE) */
	/* status = "discarded old update"; */
	/* else */
	if (activateCibXml(tmpCib, CIB_FILENAME) < 0) {
	*result = CIBRES_FAILED_ACTIVATION;

	} else if (failed->children != NULL) {
	*result = CIBRES_FAILED;

	}

	CRM_DEBUG("CIB update status: %d", *result);
	}

	output_section = section;

	if (failed->children != NULL \|\| *result != CIBRES_OK) {
	cib_answer = createCibFragmentAnswer(NULL /"all"/, failed);

	} else if (verbose != NULL && strcmp("true", verbose) == 0) {
	cib_answer = createCibFragmentAnswer(output_section, failed);

	}

	free_xml(failed);

	FNRET(cib_answer);
	}

	gboolean
	replace_section(const char *section, xmlNodePtr tmpCib, xmlNodePtr fragment)
	{
	xmlNodePtr parent = NULL,
	cib_updates = NULL,
	new_section = NULL,
	old_section = NULL;
	FNIN();

	cib_updates = find_xml_node(fragment, XML_TAG_CIB);

	/* find the old and new versions of the section */
	new_section = get_object_root(section, cib_updates);
	old_section = get_object_root(section, tmpCib);

	if(old_section == NULL) {
	cl_log(LOG_ERR,
	"The CIB is corrupt, cannot replace missing section %s",
	section);
	FNRET(FALSE);

	} else if(new_section == NULL) {
	cl_log(LOG_ERR,
	"The CIB is corrupt, cannot set section %s to nothing",
	section);
	FNRET(FALSE);
	}

	parent = old_section->parent;

	/* unlink and free the old one */
	unlink_xml_node(old_section);
	free_xml(old_section);

	/* add the new copy */
	add_node_copy(parent, new_section);

	FNRET(TRUE);
	}



	enum cib_result
	updateList(xmlNodePtr local_cib, xmlNodePtr update_fragment, xmlNodePtr failed,
	int operation, const char *section)
	{
	xmlNodePtr child = NULL;
	xmlNodePtr this_section = get_object_root(section, local_cib);
	xmlNodePtr cib_updates = find_xml_node(update_fragment, XML_TAG_CIB);
	xmlNodePtr xml_section = get_object_root(section, cib_updates);

	if (section == NULL \|\| xml_section == NULL) {
	cl_log(LOG_ERR, "Section %s not found in message."
	" CIB update is corrupt, ignoring.", section);
	return CIBRES_FAILED_NOSECTION;
	}

	if(CIB_OP_NONE > operation > CIB_OP_MAX) {
	cl_log(LOG_ERR, "Invalid operation on section %s", section);
	return CIBRES_FAILED;
	}

	set_node_tstamp(this_section);
	child = xml_section->children;

	while(child != NULL) {
	if(operation == CIB_OP_DELETE) {
	update_results(failed, child, operation,
	delete_cib_object(this_section, child));

	} else if(operation == CIB_OP_MODIFY) {
	update_results(failed, child, operation,
	update_cib_object(this_section, child,
	FALSE));

	} else {
	update_results(failed, child, operation,
	add_cib_object(this_section, child));
	}

	child = child->next;
	}

	if (failed->children != NULL)
	return CIBRES_FAILED;
	else
	return CIBRES_OK;

	}

	xmlNodePtr
	createCibFragmentAnswer(const char *section, xmlNodePtr failed)
	{
	xmlNodePtr fragment = create_xml_node(NULL, XML_TAG_FRAGMENT);

	FNIN();

	set_xml_property_copy(fragment, XML_ATTR_SECTION, section);

	if (section == NULL
	\|\| strlen(section) == 0
	\|\| strcmp("all", section) == 0) {
	add_node_copy(fragment, get_the_CIB());

	} else {
	xmlNodePtr cib = create_xml_node(fragment, XML_TAG_CIB);
	add_node_copy(cib, get_object_root(section, get_the_CIB()));
	copy_in_properties(cib, get_the_CIB());

	}

	if (failed != NULL && failed->children != NULL) {
	add_node_copy(fragment, failed);
	}

	FNRET(fragment);
	}


	gboolean
	check_generation(xmlNodePtr newCib, xmlNodePtr oldCib)
	{
	char *new_value = xmlGetProp(newCib, XML_ATTR_GENERATION);
	char *old_value = xmlGetProp(oldCib, XML_ATTR_GENERATION);
	int int_new_value = -1;
	int int_old_value = -1;
	if(old_value != NULL) int_old_value = atoi(old_value);
	if(new_value != NULL) int_new_value = atoi(new_value);

	if(int_new_value >= int_old_value) {
	return TRUE;
	} else {
	cl_log(LOG_ERR, "Generation from update (%d) is older than %d",
	int_new_value, int_old_value);
	}

	return FALSE;
	}

	gboolean
	update_results(xmlNodePtr failed,
	xmlNodePtr target,
	int operation,
	int return_code)
	{
	gboolean was_error = FALSE;
	const char *error_msg = NULL;
	const char *operation_msg = NULL;
	xmlNodePtr xml_node;
	FNIN();

	if (return_code != CIBRES_OK)
	{
	error_msg = cib_error2string(return_code);
	operation_msg = cib_op2string(operation);

	- xml_node = create_xml_node(failed,
	- XML_FAIL_TAG_CIB);
	+ xml_node = create_xml_node(failed, XML_FAIL_TAG_CIB);

	was_error = TRUE;

	set_xml_property_copy(xml_node,
	XML_FAILCIB_ATTR_ID,
	ID(target));

	set_xml_property_copy(xml_node,
	XML_FAILCIB_ATTR_OBJTYPE,
	TYPE(target));

	set_xml_property_copy(xml_node,
	XML_FAILCIB_ATTR_OP,
	operation_msg);

	set_xml_property_copy(xml_node,
	XML_FAILCIB_ATTR_REASON,
	error_msg);

	cl_log(LOG_DEBUG,
	"Action %s failed: %s (cde=%d)",
	operation_msg,
	error_msg,
	return_code);

	}

	FNRET(was_error);
	}

	diff --git a/crm/crmd/election.c b/crm/crmd/election.c
	index f6334fa6e2..92b44d83eb 100644
	--- a/crm/crmd/election.c
	+++ b/crm/crmd/election.c
	@@ -1,586 +1,608 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <portability.h>
	#include <crm/crm.h>
	#include <crmd_fsa.h>
	#include <libxml/tree.h>
	#include <crm/msg_xml.h>

	#include <crm/common/xmlutils.h>
	#include <crm/common/ipcutils.h>
	#include <crm/common/msgutils.h>
	#include <crm/cib.h>
	#include <string.h>
	#include <crmd_messages.h>

	#include <crm/dmalloc_wrapper.h>

	GHashTable *joined_nodes = NULL;

	/* A_ELECTION_VOTE */
	enum crmd_fsa_input
	do_election_vote(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input election_result = I_NULL;
	FNIN();

	/* dont vote if we're in one of these states or wanting to shut down */
	switch(cur_state) {
	case S_RECOVERY:
	case S_RECOVERY_DC:
	case S_STOPPING:
	case S_RELEASE_DC:
	case S_TERMINATE:
	FNRET(I_NULL);
	// log warning
	break;
	default:
	if(is_set(fsa_input_register, R_SHUTDOWN)) {
	FNRET(I_NULL);
	// log warning
	}
	break;
	}

	send_request(NULL, NULL, CRM_OPERATION_VOTE, NULL, CRM_SYSTEM_CRMD);

	FNRET(election_result);
	}

	gboolean
	timer_popped(gpointer data)
	{
	fsa_timer_t timer = (fsa_timer_t )data;

	cl_log(LOG_INFO, "#!!#!!# Timer %s just popped!",
	fsa_input2string(timer->fsa_input));

	stopTimer(timer); // dont make it go off again

	s_crmd_fsa(C_TIMER_POPPED, timer->fsa_input, NULL);

	return TRUE;
	}

	gboolean
	do_dc_heartbeat(gpointer data)
	{
	fsa_timer_t timer = (fsa_timer_t )data;
	// cl_log(LOG_DEBUG, "#!!#!!# Heartbeat timer just popped!");

	gboolean was_sent = send_request(NULL, NULL, CRM_OPERATION_HBEAT,
	NULL, CRM_SYSTEM_CRMD);

	if(was_sent == FALSE) {
	// this is bad
	stopTimer(timer); // dont make it go off again
	s_crmd_fsa(C_HEARTBEAT_FAILED, I_SHUTDOWN, NULL);
	}

	return TRUE;
	}


	/* A_ELECTION_COUNT */
	enum crmd_fsa_input
	do_election_count_vote(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	gboolean we_loose = FALSE;
	xmlNodePtr vote = (xmlNodePtr)data;
	unsigned int my_born = -1, your_born = -1;
	int lpc = 0, my_index = -1, your_index = -1;
	enum crmd_fsa_input election_result = I_NULL;
	const char *vote_from = xmlGetProp(vote, XML_ATTR_HOSTFROM);
	const char *lowest_uname = NULL;
	int lowest_bornon = 0;

	FNIN();

	if(vote_from == NULL \|\| strcmp(vote_from, fsa_our_uname) == 0) {
	// dont count our own vote
	FNRET(election_result);
	}

	if(fsa_membership_copy->members_size < 1) {
	// if even we are not in the cluster then we should not vote
	FNRET(I_FAIL);

	}

	lowest_uname = fsa_membership_copy->members[0].node_uname;
	lowest_bornon = fsa_membership_copy->members[0].node_born_on;

	for(; lpc < fsa_membership_copy->members_size; lpc++) {

	const char *node_uname =
	fsa_membership_copy->members[lpc].node_uname;
	int this_born_on =
	fsa_membership_copy->members[lpc].node_born_on;

	if(node_uname == NULL) {
	continue;
	}

	if(strcmp(vote_from, node_uname) == 0) {
	your_born = this_born_on;
	your_index = lpc;

	} else if (strcmp(fsa_our_uname, node_uname) == 0) {
	my_born = this_born_on;
	my_index = lpc;
	}

	if(lowest_bornon > this_born_on) {
	lowest_uname = node_uname;
	lowest_bornon = this_born_on;

	} else if(lowest_bornon == this_born_on
	&& strcmp(lowest_uname, node_uname) > 0) {
	lowest_uname = node_uname;
	lowest_bornon = this_born_on;
	}
	}

	#if 0
	cl_log(LOG_DEBUG, "%s (bornon=%d), our bornon (%d)",
	vote_from, your_born, my_born);

	cl_log(LOG_DEBUG, "%s %s %s",
	fsa_our_uname,
	strcmp(fsa_our_uname, vote_from) < 0?"<":">=",
	vote_from);
	#endif

	cl_log(LOG_DEBUG, "Election winner should be %s (born_on=%d)",
	lowest_uname, lowest_bornon);


	if(lowest_uname != NULL && strcmp(lowest_uname, fsa_our_uname) == 0){
	cl_log(LOG_DEBUG, "Election win: lowest born_on and uname");
	election_result = I_ELECTION_DC;

	} else if(your_born < my_born) {
	cl_log(LOG_DEBUG, "Election fail: born_on");
	we_loose = TRUE;
	} else if(your_born == my_born
	&& strcmp(fsa_our_uname, vote_from) > 0) {
	cl_log(LOG_DEBUG, "Election fail: uname");
	we_loose = TRUE;
	} else {
	CRM_DEBUG("We might win... we should vote (possibly again)");
	election_result = I_DC_TIMEOUT;
	}

	if(we_loose) {
	if(fsa_input_register & R_THE_DC) {
	cl_log(LOG_DEBUG, "Give up the DC");
	election_result = I_RELEASE_DC;
	} else {
	cl_log(LOG_DEBUG, "We werent the DC anyway");
	election_result = I_NOT_DC;
	}
	}

	if(we_loose \|\| election_result == I_ELECTION_DC) {
	// cancel timer, its been decided
	stopTimer(election_timeout);
	}

	FNRET(election_result);
	}


	/* A_ELECT_TIMER_START, A_ELECTION_TIMEOUT */
	// we won
	enum crmd_fsa_input
	do_election_timer_ctrl(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	FNIN();

	if(action & A_ELECT_TIMER_START) {
	CRM_DEBUG("Starting the election timer...");
	startTimer(election_timeout);

	} else if(action & A_ELECT_TIMER_STOP \|\| action & A_ELECTION_TIMEOUT) {
	CRM_DEBUG("Stopping the election timer...");
	stopTimer(election_timeout);

	} else {
	cl_log(LOG_ERR, "unexpected action %s",
	fsa_action2string(action));
	}

	if(action & A_ELECTION_TIMEOUT) {
	CRM_DEBUG("The election timer went off, we win!");

	FNRET(I_ELECTION_DC);

	}


	FNRET(I_NULL);
	}

	/* A_DC_TIMER_STOP, A_DC_TIMER_START */
	enum crmd_fsa_input
	do_dc_timer_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	gboolean timer_op_ok = TRUE;
	FNIN();

	if(action & A_DC_TIMER_STOP) {
	timer_op_ok = stopTimer(election_trigger);
	}

	/* dont start a timer that wasnt already running */
	if(action & A_DC_TIMER_START && timer_op_ok) {
	startTimer(election_trigger);
	}

	FNRET(I_NULL);
	}


	/* A_DC_TAKEOVER */
	enum crmd_fsa_input
	do_dc_takeover(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	FNIN();

	CRM_DEBUG("################## Taking over the DC ##################");
	set_bit_inplace(&fsa_input_register, R_THE_DC);

	CRM_DEBUG("Am I the DC? %s", AM_I_DC?"yes":"no");

	set_bit_inplace(&fsa_input_register, R_JOIN_OK);
	set_bit_inplace(&fsa_input_register, R_INVOKE_PE);

	clear_bit_inplace(&fsa_input_register, R_CIB_DONE);
	clear_bit_inplace(&fsa_input_register, R_HAVE_CIB);

	startTimer(dc_heartbeat);
	+
	+ /* Update the CIB to indicate we are the DC
	+ *
	+ * Has the side benefit of setting our state to active
	+ * if we are the only node around
	+ */
	+ xmlNodePtr tmp1 = create_xml_node(NULL, XML_CIB_TAG_STATE);
	+
	+ set_node_tstamp(tmp1);
	+ set_xml_property_copy(tmp1, XML_ATTR_ID, fsa_our_uname);
	+ set_xml_property_copy(tmp1, "source", fsa_our_uname);
	+ set_xml_property_copy(tmp1, "state", "active");
	+ set_xml_property_copy(tmp1, "exp_state", "active");
	+ set_xml_property_copy(tmp1, "is_dc", "true");
	+
	+ xmlNodePtr fragment = create_cib_fragment(tmp1, NULL);
	+
	+ send_request(NULL, fragment, CRM_OPERATION_UPDATE,
	+ NULL, CRM_SYSTEM_DCIB);
	+
	+ free_xml(fragment);
	+ free_xml(tmp1);

	FNRET(I_NULL);
	}

	/* A_DC_RELEASE */
	enum crmd_fsa_input
	do_dc_release(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input result = I_NULL;
	FNIN();

	CRM_DEBUG("################## Releasing the DC ##################");

	stopTimer(dc_heartbeat);

	if(action & A_DC_RELEASE) {
	clear_bit_inplace(&fsa_input_register, R_THE_DC);

	/* get a new CIB from the new DC */
	clear_bit_inplace(&fsa_input_register, R_HAVE_CIB);
	} else if (action & A_DC_RELEASED) {

	if(cur_state == S_STOPPING) {
	result = I_SHUTDOWN; // necessary?
	result = I_RELEASE_SUCCESS;
	}
	#if 0
	else if( are there errors ) {
	// we cant stay up if not healthy
	// or perhaps I_ERROR and go to S_RECOVER?
	result = I_SHUTDOWN;
	}
	#endif
	else
	result = I_RELEASE_SUCCESS;

	} else {
	cl_log(LOG_ERR, "Warning, do_dc_release invoked for action %s",
	fsa_action2string(action));
	}

	CRM_DEBUG("Am I still the DC? %s", AM_I_DC?"yes":"no");

	FNRET(result);
	}

	/* A_JOIN_WELCOME, A_JOIN_WELCOME_ALL */
	enum crmd_fsa_input
	do_send_welcome(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	int lpc = 0, size = 0, num_sent = 0;
	oc_node_t *members;
	gboolean was_sent = TRUE;
	FNIN();

	if(action & A_JOIN_WELCOME && data == NULL) {
	cl_log(LOG_ERR,
	"Attempt to send welcome message "
	"without a message to reply to!");
	FNRET(I_NULL);

	} else if(action & A_JOIN_WELCOME) {
	xmlNodePtr welcome = (xmlNodePtr)data;

	const char *join_to = xmlGetProp(welcome, XML_ATTR_HOSTFROM);
	if(join_to != NULL) {
	send_request(NULL, NULL, CRM_OPERATION_WELCOME,
	join_to, CRM_SYSTEM_CRMD);
	}

	FNRET(I_NULL);
	}

	// welcome everyone...

	/* Give everyone a chance to join before invoking the PolicyEngine */
	stopTimer(integration_timer);
	startTimer(integration_timer);

	members = fsa_membership_copy->members;
	size = fsa_membership_copy->members_size;

	if(joined_nodes != NULL) {
	g_hash_table_destroy(joined_nodes);
	joined_nodes = g_hash_table_new(&g_str_hash, &g_str_equal);

	}


	for(; members != NULL && lpc < size; lpc++) {
	const char *new_node = members[lpc].node_uname;
	if(strcmp(fsa_our_uname, new_node) == 0) {
	// dont send one to ourselves
	continue;
	}

	CRM_DEBUG("Sending welcome message to %s (%d)",
	new_node, was_sent);
	num_sent++;
	was_sent = was_sent
	&& send_request(NULL, NULL, CRM_OPERATION_WELCOME,
	new_node, CRM_SYSTEM_CRMD);
	CRM_DEBUG("Sent welcome message to %s (%d)",
	new_node, was_sent);
	}

	if(was_sent == FALSE)
	FNRET(I_FAIL);

	/* No point hanging around in S_INTEGRATION if we're the only ones here! */
	if(num_sent == 0) {
	// that was the last outstanding join ack)
	cl_log(LOG_INFO,"That was the last outstanding join ack");
	FNRET(I_SUCCESS);
	} else {
	cl_log(LOG_DEBUG,
	"Still waiting on %d outstanding join acks",
	num_sent);
	//dont waste time by invoking the pe yet;
	}

	FNRET(I_NULL);
	}

	/* A_JOIN_ACK */
	enum crmd_fsa_input
	do_ack_welcome(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	xmlNodePtr welcome = (xmlNodePtr)data;
	xmlNodePtr cib_copy;
	xmlNodePtr tmp1;
	xmlNodePtr tmp2;

	FNIN();

	#if 0
	if(we are sick) {
	log error ;
	FNRET(I_NULL);
	}
	#endif

	cib_copy = get_cib_copy();
	tmp1 = get_object_root(XML_CIB_TAG_STATUS, cib_copy);
	tmp2 = create_cib_fragment(tmp1, NULL);

	send_ha_reply(fsa_cluster_conn, welcome, tmp2);

	free_xml(tmp2);
	free_xml(cib_copy);

	FNRET(I_NULL);
	}

	/* A_ANNOUNCE */
	enum crmd_fsa_input
	do_announce(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	FNIN();

	/* Once we hear from the DC, we can stop the timer
	*
	* This timer was started either on startup or when a node
	* left the CCM list
	*/

	/* dont announce if we're in one of these states */
	switch(cur_state) {
	case S_RECOVERY:
	case S_RECOVERY_DC:
	case S_RELEASE_DC:
	case S_TERMINATE:
	FNRET(I_NULL);
	// log warning
	break;
	default:
	break;
	}

	if(AM_I_OPERATIONAL) {
	send_request(NULL, NULL, CRM_OPERATION_ANNOUNCE,
	NULL, CRM_SYSTEM_DC);
	} else {
	/* Delay announce until we have finished local startup */
	FNRET(I_NULL);
	}

	FNRET(I_NULL);
	}


	/* A_JOIN_PROCESS_ACK */
	enum crmd_fsa_input
	do_process_welcome_ack(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	int lpc = 0, size = 0;
	oc_node_t *members;
	gboolean is_a_member = FALSE;
	xmlNodePtr tmp1;
	xmlNodePtr join_ack = (xmlNodePtr)data;
	xmlNodePtr cib_fragment;
	const char *join_from = xmlGetProp(join_ack, XML_ATTR_HOSTFROM);

	FNIN();


	FNIN();

	members = fsa_membership_copy->members;
	size = fsa_membership_copy->members_size;

	for(; lpc < size; lpc++) {
	const char *new_node = members[lpc].node_uname;
	if(strcmp(join_from, new_node) == 0) {
	is_a_member = TRUE;
	}
	}

	cib_fragment = find_xml_node(join_ack, XML_TAG_FRAGMENT);

	if(is_a_member == FALSE) {
	cl_log(LOG_ERR, "Node %s is not known to us", join_from);

	/* make sure any information from this node is discarded,
	* it is invalid
	*/
	free_xml(cib_fragment);
	FNRET(I_FAIL);
	}

	cl_log(LOG_DEBUG, "Welcoming node %s after ACK", join_from);

	// add them to our list of "active" nodes
	g_hash_table_insert(joined_nodes, strdup(join_from),strdup(join_from));


	if(cib_fragment == NULL) {
	cl_log(LOG_ERR,
	"No status information was part of the"
	" Welcome ACK from %s",
	join_from);
	FNRET(I_NULL);
	}

	/* TODO: check the fragment is only for the status section
	= get_xml_attr(cib_fragment, NULL,
	XML_ATTR_FILTER_TYPE, TRUE); */

	/* Make changes so that state=active for this node when the update
	* is processed by A_CIB_INVOKE
	*/
	tmp1 = find_xml_node(cib_fragment, XML_TAG_CIB);
	tmp1 = get_object_root(XML_CIB_TAG_STATUS, tmp1);
	tmp1 = find_entity(tmp1, XML_CIB_TAG_STATE, join_from, FALSE);

	set_xml_property_copy(tmp1, "state", "active");

	if(g_hash_table_size(joined_nodes)
	== fsa_membership_copy->members_size) {
	// that was the last outstanding join ack)
	cl_log(LOG_INFO,"That was the last outstanding join ack");
	FNRET(I_SUCCESS);
	} else {
	cl_log(LOG_DEBUG,
	"Still waiting on %d outstanding join acks",
	size);
	//dont waste time by invoking the pe yet;
	}
	FNRET(I_CIB_OP);
	}
	diff --git a/crm/crmd/fsa.c b/crm/crmd/fsa.c
	index 904e6658ce..994f9667e2 100644
	--- a/crm/crmd/fsa.c
	+++ b/crm/crmd/fsa.c
	@@ -1,1088 +1,1098 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <portability.h>
	#include <crm/crm.h>
	#include <crmd_fsa.h>
	#include <fsa_matrix.h>
	#include <fsa_proto.h>
	#include <stdio.h>

	#include <crm/common/xmlutils.h>
	#include <crm/common/msgutils.h>
	#include <crm/msg_xml.h>

	#include <clplumbing/Gmain_timeout.h>

	#include <crmd_messages.h>
	#include <string.h>
	#include <time.h>

	#include <crm/dmalloc_wrapper.h>

	long long
	do_state_transition(long long actions,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_state next_state,
	enum crmd_fsa_input current_input,
	void *data);

	#ifdef DOT_FSA_ACTIONS
	# ifdef FSA_TRACE
	# define IF_FSA_ACTION(x,y) \
	if(is_set(actions,x)) { \
	CRM_DEBUG("Invoking action %s (%.16llx)", \
	fsa_action2string(x), x); \
	last_action = x; \
	actions = clear_bit(actions, x); \
	next_input = y(x, cause, cur_state, last_input, data); \
	if( (x & O_DC_TICKLE) == 0 && next_input != I_DC_HEARTBEAT ) \
	fprintf(dot_strm, \
	"\t// %s:\t%s\t(data? %s)\t(result=%s)\n", \
	fsa_input2string(cur_input), \
	fsa_action2string(x), \
	data==NULL?"no":"yes", \
	fsa_input2string(next_input)); \
	fflush(dot_strm); \
	CRM_DEBUG("Result of action %s was %s", \
	fsa_action2string(x), fsa_input2string(next_input)); \
	}
	# else
	# define IF_FSA_ACTION(x,y) \
	if(is_set(actions,x)) { \
	last_action = x; \
	actions = clear_bit(actions, x); \
	next_input = y(x, cause, cur_state, last_input, data); \
	if( (x & O_DC_TICKLE) == 0 && next_input != I_DC_HEARTBEAT ) \
	fprintf(dot_strm, \
	"\t// %s:\t%s\t(data? %s)\t(result=%s)\n", \
	fsa_input2string(cur_input), \
	fsa_action2string(x), \
	data==NULL?"no":"yes", \
	fsa_input2string(next_input)); \
	fflush(dot_strm); \
	}
	# endif
	#else
	# ifdef FSA_TRACE
	# define IF_FSA_ACTION(x,y) \
	if(is_set(actions,x)) { \
	CRM_DEBUG("Invoking action %s (%.16llx)", \
	fsa_action2string(x), x); \
	last_action = x; \
	actions = clear_bit(actions, x); \
	next_input = y(x, cause, cur_state, last_input, data); \
	CRM_DEBUG("Result of action %s was %s", \
	fsa_action2string(x), fsa_input2string(next_input)); \
	}
	# else
	# define IF_FSA_ACTION(x,y) \
	if(is_set(actions,x)) { \
	last_action = x; \
	actions = clear_bit(actions, x); \
	next_input = y(x, cause, cur_state, last_input, data); \
	}
	# endif
	#endif

	#define ELSEIF_FSA_ACTION(x,y) else IF_FSA_ACTION(x,y)

	const char *dot_intro = "digraph \"g\" {\n"
	" size = \"30,30\"\n"
	" graph [\n"
	" fontsize = \"12\"\n"
	" fontname = \"Times-Roman\"\n"
	" fontcolor = \"black\"\n"
	" bb = \"0,0,398.922306,478.927856\"\n"
	" color = \"black\"\n"
	" ]\n"
	" node [\n"
	" fontsize = \"12\"\n"
	" fontname = \"Times-Roman\"\n"
	" fontcolor = \"black\"\n"
	" shape = \"ellipse\"\n"
	" color = \"black\"\n"
	" ]\n"
	" edge [\n"
	" fontsize = \"12\"\n"
	" fontname = \"Times-Roman\"\n"
	" fontcolor = \"black\"\n"
	" color = \"black\"\n"
	" ]\n"
	"// special nodes\n"
	" \"S_PENDING\" \n"
	" [\n"
	" color = \"blue\"\n"
	" fontcolor = \"blue\"\n"
	" ]\n"
	" \"S_TERMINATE\" \n"
	" [\n"
	" color = \"red\"\n"
	" fontcolor = \"red\"\n"
	" ]\n"
	"\n"
	"// DC only nodes\n"
	" \"S_RECOVERY_DC\" [ fontcolor = \"green\" ]\n"
	" \"S_INTEGRATION\" [ fontcolor = \"green\" ]\n"
	" \"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]\n"
	" \"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]\n"
	" \"S_RELEASE_DC\" [ fontcolor = \"green\" ]\n"
	" \"S_IDLE\" [ fontcolor = \"green\" ]\n";


	static FILE *dot_strm = NULL;

	enum crmd_fsa_state fsa_state;
	oc_node_list_t *fsa_membership_copy;
	ll_cluster_t *fsa_cluster_conn;
	ll_lrm_t *fsa_lrm_conn;
	long long fsa_input_register;
	long long fsa_actions = A_NOTHING;
	const char *fsa_our_uname;

	fsa_timer_t election_trigger = NULL; / */
	fsa_timer_t election_timeout = NULL; / */
	fsa_timer_t shutdown_escalation_timmer = NULL; / */
	fsa_timer_t *integration_timer = NULL;
	fsa_timer_t *dc_heartbeat = NULL;

	long long
	toggle_bit(long long action_list, long long action)
	{
	// CRM_DEBUG("Toggling bit %.16llx", action);
	action_list ^= action;
	// CRM_DEBUG("Result %.16llx", action_list & action);
	return action_list;
	}

	long long
	clear_bit(long long action_list, long long action)
	{
	// CRM_DEBUG("Clearing bit\t%.16llx", action);

	// ensure its set
	action_list \|= action;

	// then toggle
	action_list = action_list ^ action;

	return action_list;
	}

	long long
	set_bit(long long action_list, long long action)
	{
	// CRM_DEBUG("Adding bit\t%.16llx", action);
	action_list \|= action;
	return action_list;
	}

	void
	toggle_bit_inplace(long long *action_list, long long action)
	{
	action_list = toggle_bit(action_list, action);
	}

	void
	clear_bit_inplace(long long *action_list, long long action)
	{
	action_list = clear_bit(action_list, action);
	}

	void
	set_bit_inplace(long long *action_list, long long action)
	{
	action_list = set_bit(action_list, action);
	}



	gboolean
	is_set(long long action_list, long long action)
	{
	// CRM_DEBUG("Checking bit\t%.16llx", action);
	return ((action_list & action) == action);
	}

	gboolean
	startTimer(fsa_timer_t *timer)
	{
	if(((int)timer->source_id) < 0) {
	timer->source_id =
	Gmain_timeout_add(timer->period_ms,
	timer->callback,
	(void*)timer);
	/*
	CRM_DEBUG("#!!#!!# Started %s timer (%d)",
	fsa_input2string(timer->fsa_input),
	timer->source_id);
	*/
	} else {
	cl_log(LOG_INFO, "#!!#!!# Timer %s already running (%d)",
	fsa_input2string(timer->fsa_input),
	timer->source_id);
	return FALSE;
	}
	return TRUE;
	}


	gboolean
	stopTimer(fsa_timer_t *timer)
	{
	if(((int)timer->source_id) > 0) {
	/*
	CRM_DEBUG("#!!#!!# Stopping %s timer (%d)",
	fsa_input2string(timer->fsa_input),
	timer->source_id);
	*/
	g_source_remove(timer->source_id);
	timer->source_id = -2;

	} else {
	cl_log(LOG_INFO, "#!!#!!# Timer %s already stopped (%d)",
	fsa_input2string(timer->fsa_input),
	timer->source_id);
	return FALSE;
	}
	return TRUE;
	}

	enum crmd_fsa_state
	s_crmd_fsa(enum crmd_fsa_cause cause,
	enum crmd_fsa_input initial_input,
	void *data)
	{
	long long actions = fsa_actions;
	long long new_actions = A_NOTHING;
	long long last_action = A_NOTHING;
	enum crmd_fsa_input last_input = initial_input;
	enum crmd_fsa_input cur_input;
	enum crmd_fsa_input next_input;
	enum crmd_fsa_state last_state, cur_state, next_state, starting_state;

	FNIN();

	starting_state = fsa_state;
	cur_input = initial_input;
	next_input = initial_input;

	last_state = starting_state;
	cur_state = starting_state;
	next_state = starting_state;

	#ifdef FSA_TRACE
	CRM_DEBUG("FSA invoked with Cause: %s\n\tState: %s, Input: %s",
	fsa_cause2string(cause),
	fsa_state2string(cur_state),
	fsa_input2string(cur_input));
	#endif

	if(dot_strm == NULL) {
	dot_strm = fopen("/tmp/live.dot", "w");
	fprintf(dot_strm, "%s", dot_intro);
	}

	/*
	* Process actions in order of priority but do only one
	* action at a time to avoid complicating the ordering.
	*
	* Actions may result in a new I_ event, these are added to
	* (not replace) existing actions before the next iteration.
	*
	*/
	while(next_input != I_NULL \|\| actions != A_NOTHING) {

	if(next_input == I_WAIT_FOR_EVENT) {
	/* we may be waiting for an a-sync task to "happen"
	* and until it does, we cant do anything else
	*
	* Re-add the last action
	*/

	actions \|= last_action;

	cl_log(LOG_INFO, "Wait until something else happens");
	break;
	}

	#ifdef FSA_TRACE
	CRM_DEBUG("FSA while loop:\tState: %s, Input: %s",
	fsa_state2string(cur_state),
	fsa_input2string(cur_input));
	#endif

	/* update input variables */
	cur_input = next_input;
	if(cur_input != I_NULL) {
	last_input = cur_input;
	}

	/* get the next batch of actions */
	new_actions = crmd_fsa_actions[cur_input][cur_state];
	if(new_actions != A_NOTHING) {
	#ifdef FSA_TRACE
	CRM_DEBUG("Adding actions %.16llx", new_actions);
	#endif
	actions \|= new_actions;
	}

	/* logging : before the state is changed */
	IF_FSA_ACTION(A_ERROR, do_log)
	ELSEIF_FSA_ACTION(A_WARN, do_log)
	ELSEIF_FSA_ACTION(A_LOG, do_log)

	/* update state variables */
	next_state = crmd_fsa_state[cur_input][cur_state];
	last_state = cur_state;
	cur_state = next_state;
	fsa_state = next_state;

	/* start doing things... */


	/*
	* Hook for change of state.
	* Allows actions to be added or removed when entering a state
	*/
	if(last_state != cur_state){
	actions = do_state_transition(actions, cause,
	last_state, cur_state,
	last_input, data);
	}

	/* this is always run, some inputs/states may make various
	* actions irrelevant/invalid
	*/
	actions = clear_flags(actions, cause, cur_state, cur_input);

	/* regular action processing in order of action priority
	*
	* Make sure all actions that connect to required systems
	* are performed first
	*/
	if(actions == A_NOTHING) {

	cl_log(LOG_INFO, "Nothing to do");
	next_input = I_NULL;

	/* // check registers, see if anything is pending
	if(is_set(fsa_input_register, R_SHUTDOWN)) {
	CRM_DEBUG("(Re-)invoking shutdown");
	next_input = I_SHUTDOWN;
	} else if(is_set(fsa_input_register, R_INVOKE_PE)) {
	CRM_DEBUG("Invoke the PE somehow");
	}
	*/
	}


	/* get out of here NOW! before anything worse happens */
	ELSEIF_FSA_ACTION(A_EXIT_1, do_exit)

	ELSEIF_FSA_ACTION(A_STARTUP, do_startup)

	ELSEIF_FSA_ACTION(A_CIB_START, do_cib_control)
	ELSEIF_FSA_ACTION(A_HA_CONNECT, do_ha_control)
	ELSEIF_FSA_ACTION(A_LRM_CONNECT,do_lrm_control)
	ELSEIF_FSA_ACTION(A_CCM_CONNECT,do_ccm_control)
	ELSEIF_FSA_ACTION(A_ANNOUNCE, do_announce)

	/* sub-system start */
	ELSEIF_FSA_ACTION(A_PE_START, do_pe_control)
	ELSEIF_FSA_ACTION(A_TE_START, do_te_control)

	/* sub-system restart
	*/
	ELSEIF_FSA_ACTION(O_CIB_RESTART,do_cib_control)
	ELSEIF_FSA_ACTION(O_PE_RESTART, do_pe_control)
	ELSEIF_FSA_ACTION(O_TE_RESTART, do_te_control)

	ELSEIF_FSA_ACTION(A_STARTED, do_started)

	/* DC Timer */
	ELSEIF_FSA_ACTION(O_DC_TIMER_RESTART, do_dc_timer_control)
	ELSEIF_FSA_ACTION(A_DC_TIMER_STOP, do_dc_timer_control)
	ELSEIF_FSA_ACTION(A_DC_TIMER_START, do_dc_timer_control)

	/*
	* Highest priority actions
	*/
	ELSEIF_FSA_ACTION(A_TE_COPYTO, do_te_copyto)
	ELSEIF_FSA_ACTION(A_SHUTDOWN_REQ, do_shutdown_req)
	ELSEIF_FSA_ACTION(A_MSG_ROUTE, do_msg_route)
	ELSEIF_FSA_ACTION(A_RECOVER, do_recover)
	ELSEIF_FSA_ACTION(A_ELECTION_VOTE, do_election_vote)
	ELSEIF_FSA_ACTION(A_ELECT_TIMER_START, do_election_timer_ctrl)
	ELSEIF_FSA_ACTION(A_ELECT_TIMER_STOP, do_election_timer_ctrl)
	ELSEIF_FSA_ACTION(A_ELECTION_COUNT, do_election_count_vote)
	ELSEIF_FSA_ACTION(A_ELECTION_TIMEOUT, do_election_timer_ctrl)

	/*
	* "Get this over with" actions
	*/
	ELSEIF_FSA_ACTION(A_MSG_STORE, do_msg_store)
	ELSEIF_FSA_ACTION(A_NODE_BLOCK, do_node_block)

	/*
	* High priority actions
	* Update the cache first
	*/
	ELSEIF_FSA_ACTION(A_CCM_UPDATE_CACHE, do_ccm_update_cache)
	ELSEIF_FSA_ACTION(A_CCM_EVENT, do_ccm_event)

	/*
	* Medium priority actions
	*/
	ELSEIF_FSA_ACTION(A_DC_TAKEOVER, do_dc_takeover)
	ELSEIF_FSA_ACTION(A_DC_RELEASE, do_dc_release)
	ELSEIF_FSA_ACTION(A_JOIN_WELCOME_ALL, do_send_welcome)
	ELSEIF_FSA_ACTION(A_JOIN_WELCOME, do_send_welcome)
	ELSEIF_FSA_ACTION(A_JOIN_ACK, do_ack_welcome)
	ELSEIF_FSA_ACTION(A_JOIN_PROCESS_ACK, do_process_welcome_ack)

	/*
	* Low(er) priority actions
	* Make sure the CIB is always updated before invoking the
	* PE, and the PE before the TE
	*/
	ELSEIF_FSA_ACTION(A_UPDATE_NODESTATUS, do_lrm_invoke)
	ELSEIF_FSA_ACTION(A_CIB_INVOKE_LOCAL, do_cib_invoke)
	ELSEIF_FSA_ACTION(A_CIB_INVOKE, do_cib_invoke)
	ELSEIF_FSA_ACTION(A_CIB_BUMPGEN, do_cib_invoke)
	ELSEIF_FSA_ACTION(A_LRM_INVOKE, do_lrm_invoke)
	ELSEIF_FSA_ACTION(A_LRM_EVENT, do_lrm_event)
	ELSEIF_FSA_ACTION(A_TE_CANCEL, do_te_invoke)
	ELSEIF_FSA_ACTION(A_PE_INVOKE, do_pe_invoke)
	ELSEIF_FSA_ACTION(A_TE_INVOKE, do_te_invoke)

	/* sub-system stop */
	ELSEIF_FSA_ACTION(A_PE_STOP, do_pe_control)
	ELSEIF_FSA_ACTION(A_TE_STOP, do_te_control)
	ELSEIF_FSA_ACTION(A_DC_RELEASED, do_dc_release)

	ELSEIF_FSA_ACTION(A_HA_DISCONNECT, do_ha_control)
	ELSEIF_FSA_ACTION(A_CCM_DISCONNECT, do_ccm_control)
	ELSEIF_FSA_ACTION(A_LRM_DISCONNECT, do_lrm_control)
	ELSEIF_FSA_ACTION(A_CIB_STOP, do_cib_control)
	/* time to go now... */

	/* Some of these can probably be consolidated */
	ELSEIF_FSA_ACTION(A_SHUTDOWN, do_shutdown)
	ELSEIF_FSA_ACTION(A_STOP, do_stop)

	/* exit gracefully */
	ELSEIF_FSA_ACTION(A_EXIT_0, do_exit)

	// ELSEIF_FSA_ACTION(A_, do_)

	- else if(is_message()) {
	+ else if((actions & A_MSG_PROCESS) != 0
	+ \|\| is_message()) {
	xmlNodePtr stored_msg = NULL;

	fsa_message_queue_t msg = get_message();

	if(is_message() == FALSE) {
	actions = clear_bit(actions, A_MSG_PROCESS);
	}

	if(msg == NULL \|\| msg->message == NULL) {
	cl_log(LOG_ERR,
	"Invalid stored message");
	continue;
	}
	-
	+
	+ /*
	+ * This is where we should clean up old messages
	+ * The problem is that we dont always know the
	+ * type of the data (and therefore the correct way
	+ * to free it). A wrapper is probably required.
	+ */
	data = msg->message;

	#ifdef DOT_FSA_ACTIONS
	fprintf(dot_strm,
	"\t// %s:\t%s\t(data? %s)",
	fsa_input2string(cur_input),
	fsa_action2string(A_MSG_PROCESS),
	stored_msg==NULL?"no":"yes");
	fflush(dot_strm);
	#endif
	#ifdef FSA_TRACE
	CRM_DEBUG("Invoking action %s (%.16llx)",
	fsa_action2string(A_MSG_PROCESS),
	A_MSG_PROCESS);
	#endif

	stored_msg = (xmlNodePtr)data;

	#ifdef FSA_TRACE
	xml_message_debug(stored_msg,"FSA processing message");
	#endif

	next_input = handle_message(stored_msg);

	#ifdef DOT_FSA_ACTIONS
	fprintf(dot_strm, "\t(result=%s)\n",
	fsa_input2string(next_input));
	#endif
	CRM_DEBUG("Result of action %s was %s",
	fsa_action2string(A_MSG_PROCESS),
	fsa_input2string(next_input));

	/* Error checking and reporting */
	} else if(cur_input != I_NULL && is_set(actions, A_NOTHING)) {
	cl_log(LOG_WARNING,
	"No action specified for input,state (%s,%s)",
	fsa_input2string(cur_input),
	fsa_state2string(cur_state));

	next_input = I_NULL;

	} else if(cur_input == I_NULL && is_set(actions, A_NOTHING)) {
	#ifdef FSA_TRACE
	cl_log(LOG_INFO, "Nothing left to do");
	#endif
	} else {
	cl_log(LOG_ERR, "Action %s (0x%llx) not supported ",
	fsa_action2string(actions), actions);
	next_input = I_ERROR;
	}
	}

	#ifdef FSA_TRACE
	CRM_DEBUG("################# Exiting the FSA (%s) ##################",
	fsa_state2string(fsa_state));
	#endif

	#ifdef DOT_FSA_ACTIONS
	fprintf(dot_strm,
	"\t// ### Exiting the FSA (%s)\n",
	fsa_state2string(fsa_state));
	fflush(dot_strm);
	#endif

	// cleanup inputs?
	fsa_actions = actions;

	FNRET(fsa_state);
	}


	/* A_NODE_BLOCK */
	enum crmd_fsa_input
	do_node_block(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{

	xmlNodePtr xml_message = (xmlNodePtr)data;
	const char *host_from = xmlGetProp(xml_message, XML_ATTR_HOSTFROM);

	FNIN();

	(void)host_from;


	FNRET(I_NULL);
	}


	const char *
	fsa_input2string(int input)
	{
	const char *inputAsText = NULL;

	switch(input){
	case I_NULL:
	inputAsText = "I_NULL";
	break;
	case I_CCM_EVENT:
	inputAsText = "I_CCM_EVENT";
	break;
	case I_CIB_OP:
	inputAsText = "I_CIB_OP";
	break;
	case I_CIB_UPDATE:
	inputAsText = "I_CIB_UPDATE";
	break;
	case I_DC_TIMEOUT:
	inputAsText = "I_DC_TIMEOUT";
	break;
	case I_ELECTION:
	inputAsText = "I_ELECTION";
	break;
	case I_RELEASE_DC:
	inputAsText = "I_RELEASE_DC";
	break;
	case I_ELECTION_DC:
	inputAsText = "I_ELECTION_DC";
	break;
	case I_ERROR:
	inputAsText = "I_ERROR";
	break;
	case I_FAIL:
	inputAsText = "I_FAIL";
	break;
	case I_INTEGRATION_TIMEOUT:
	inputAsText = "I_INTEGRATION_TIMEOUT";
	break;
	case I_NODE_JOIN:
	inputAsText = "I_NODE_JOIN";
	break;
	case I_NODE_LEFT:
	inputAsText = "I_NODE_LEFT";
	break;
	case I_NODE_LEAVING:
	inputAsText = "I_NODE_LEAVING";
	break;
	case I_NOT_DC:
	inputAsText = "I_NOT_DC";
	break;
	case I_RECOVERED:
	inputAsText = "I_RECOVERED";
	break;
	case I_RELEASE_FAIL:
	inputAsText = "I_RELEASE_FAIL";
	break;
	case I_RELEASE_SUCCESS:
	inputAsText = "I_RELEASE_SUCCESS";
	break;
	case I_RESTART:
	inputAsText = "I_RESTART";
	break;
	case I_REQUEST:
	inputAsText = "I_REQUEST";
	break;
	case I_ROUTER:
	inputAsText = "I_ROUTER";
	break;
	case I_SHUTDOWN:
	inputAsText = "I_SHUTDOWN";
	break;
	/* case I_SHUTDOWN_REQ: */
	/* inputAsText = "I_SHUTDOWN_REQ"; */
	/* break; */
	case I_STARTUP:
	inputAsText = "I_STARTUP";
	break;
	case I_SUCCESS:
	inputAsText = "I_SUCCESS";
	break;
	case I_TERMINATE:
	inputAsText = "I_TERMINATE";
	break;
	case I_WELCOME:
	inputAsText = "I_WELCOME";
	break;
	case I_WELCOME_ACK:
	inputAsText = "I_WELCOME_ACK";
	break;
	case I_DC_HEARTBEAT:
	inputAsText = "I_DC_HEARTBEAT";
	break;
	case I_WAIT_FOR_EVENT:
	inputAsText = "I_WAIT_FOR_EVENT";
	break;
	case I_ILLEGAL:
	inputAsText = "I_ILLEGAL";
	break;
	}

	if(inputAsText == NULL) {
	cl_log(LOG_ERR, "Input %d is unknown", input);
	inputAsText = "<UNKNOWN_INPUT>";
	}

	return inputAsText;
	}

	const char *
	fsa_state2string(int state)
	{
	const char *stateAsText = NULL;

	switch(state){
	case S_IDLE:
	stateAsText = "S_IDLE";
	break;
	case S_ELECTION:
	stateAsText = "S_ELECTION";
	break;
	case S_INTEGRATION:
	stateAsText = "S_INTEGRATION";
	break;
	case S_NOT_DC:
	stateAsText = "S_NOT_DC";
	break;
	case S_POLICY_ENGINE:
	stateAsText = "S_POLICY_ENGINE";
	break;
	case S_RECOVERY:
	stateAsText = "S_RECOVERY";
	break;
	case S_RECOVERY_DC:
	stateAsText = "S_RECOVERY_DC";
	break;
	case S_RELEASE_DC:
	stateAsText = "S_RELEASE_DC";
	break;
	case S_PENDING:
	stateAsText = "S_PENDING";
	break;
	case S_STOPPING:
	stateAsText = "S_STOPPING";
	break;
	case S_TERMINATE:
	stateAsText = "S_TERMINATE";
	break;
	case S_TRANSITION_ENGINE:
	stateAsText = "S_TRANSITION_ENGINE";
	break;
	case S_ILLEGAL:
	stateAsText = "S_ILLEGAL";
	break;
	}

	if(stateAsText == NULL) {
	cl_log(LOG_ERR, "State %d is unknown", state);
	stateAsText = "<UNKNOWN_STATE>";
	}

	return stateAsText;
	}

	const char *
	fsa_cause2string(int cause)
	{
	const char *causeAsText = NULL;

	switch(cause){
	case C_UNKNOWN:
	causeAsText = "C_UNKNOWN";
	break;
	case C_STARTUP:
	causeAsText = "C_STARTUP";
	break;
	case C_IPC_MESSAGE:
	causeAsText = "C_IPC_MESSAGE";
	break;
	case C_HA_MESSAGE:
	causeAsText = "C_HA_MESSAGE";
	break;
	case C_CCM_CALLBACK:
	causeAsText = "C_CCM_CALLBACK";
	break;
	case C_TIMER_POPPED:
	causeAsText = "C_TIMER_POPPED";
	break;
	case C_SHUTDOWN:
	causeAsText = "C_SHUTDOWN";
	break;
	case C_HEARTBEAT_FAILED:
	causeAsText = "C_HEARTBEAT_FAILED";
	break;
	case C_SUBSYSTEM_CONNECT:
	causeAsText = "C_SUBSYSTEM_CONNECT";
	break;
	case C_ILLEGAL:
	causeAsText = "C_ILLEGAL";
	break;
	}

	if(causeAsText == NULL) {
	cl_log(LOG_ERR, "Cause %d is unknown", cause);
	causeAsText = "<UNKNOWN_CAUSE>";
	}

	return causeAsText;
	}

	const char *
	fsa_action2string(long long action)
	{
	const char *actionAsText = NULL;

	switch(action){

	case A_NOTHING:
	actionAsText = "A_NOTHING";
	break;
	case O_SHUTDOWN:
	actionAsText = "O_SHUTDOWN";
	break;
	case O_RELEASE:
	actionAsText = "O_RELEASE";
	break;
	case A_STARTUP:
	actionAsText = "A_STARTUP";
	break;
	case A_STARTED:
	actionAsText = "A_STARTED";
	break;
	case A_HA_CONNECT:
	actionAsText = "A_HA_CONNECT";
	break;
	case A_HA_DISCONNECT:
	actionAsText = "A_HA_DISCONNECT";
	break;
	case A_LRM_CONNECT:
	actionAsText = "A_LRM_CONNECT";
	break;
	case A_LRM_DISCONNECT:
	actionAsText = "A_LRM_DISCONNECT";
	break;
	case O_DC_TIMER_RESTART:
	actionAsText = "O_DC_TIMER_RESTART";
	break;
	case A_DC_TIMER_STOP:
	actionAsText = "A_DC_TIMER_STOP";
	break;
	case A_DC_TIMER_START:
	actionAsText = "A_DC_TIMER_START";
	break;
	case A_ELECTION_COUNT:
	actionAsText = "A_ELECTION_COUNT";
	break;
	case A_ELECTION_TIMEOUT:
	actionAsText = "A_ELECTION_TIMEOUT";
	break;
	case A_ELECT_TIMER_START:
	actionAsText = "A_ELECT_TIMER_START";
	break;
	case A_ELECT_TIMER_STOP:
	actionAsText = "A_ELECT_TIMER_STOP";
	break;
	case A_ELECTION_VOTE:
	actionAsText = "A_ELECTION_VOTE";
	break;
	case A_ANNOUNCE:
	actionAsText = "A_ANNOUNCE";
	break;
	case A_JOIN_ACK:
	actionAsText = "A_JOIN_ACK";
	break;
	case A_JOIN_WELCOME:
	actionAsText = "A_JOIN_WELCOME";
	break;
	case A_JOIN_WELCOME_ALL:
	actionAsText = "A_JOIN_WELCOME_ALL";
	break;
	case A_JOIN_PROCESS_ACK:
	actionAsText = "A_JOIN_PROCESS_ACK";
	break;
	case A_MSG_PROCESS:
	actionAsText = "A_MSG_PROCESS";
	break;
	case A_MSG_ROUTE:
	actionAsText = "A_MSG_ROUTE";
	break;
	case A_MSG_STORE:
	actionAsText = "A_MSG_STORE";
	break;
	case A_RECOVER:
	actionAsText = "A_RECOVER";
	break;
	case A_DC_RELEASE:
	actionAsText = "A_DC_RELEASE";
	break;
	case A_DC_RELEASED:
	actionAsText = "A_DC_RELEASED";
	break;
	case A_DC_TAKEOVER:
	actionAsText = "A_DC_TAKEOVER";
	break;
	case A_SHUTDOWN:
	actionAsText = "A_SHUTDOWN";
	break;
	case A_SHUTDOWN_REQ:
	actionAsText = "A_SHUTDOWN_REQ";
	break;
	case A_STOP:
	actionAsText = "A_STOP ";
	break;
	case A_EXIT_0:
	actionAsText = "A_EXIT_0";
	break;
	case A_EXIT_1:
	actionAsText = "A_EXIT_1";
	break;
	case A_CCM_CONNECT:
	actionAsText = "A_CCM_CONNECT";
	break;
	case A_CCM_DISCONNECT:
	actionAsText = "A_CCM_DISCONNECT";
	break;
	case A_CCM_EVENT:
	actionAsText = "A_CCM_EVENT";
	break;
	case A_CCM_UPDATE_CACHE:
	actionAsText = "A_CCM_UPDATE_CACHE";
	break;
	case A_CIB_BUMPGEN:
	actionAsText = "A_CIB_BUMPGEN";
	break;
	case A_CIB_INVOKE:
	actionAsText = "A_CIB_INVOKE";
	break;
	case O_CIB_RESTART:
	actionAsText = "O_CIB_RESTART";
	break;
	case A_CIB_START:
	actionAsText = "A_CIB_START";
	break;
	case A_CIB_STOP:
	actionAsText = "A_CIB_STOP";
	break;
	case A_TE_INVOKE:
	actionAsText = "A_TE_INVOKE";
	break;
	case O_TE_RESTART:
	actionAsText = "O_TE_RESTART";
	break;
	case A_TE_START:
	actionAsText = "A_TE_START";
	break;
	case A_TE_STOP:
	actionAsText = "A_TE_STOP";
	break;
	+ case A_TE_COPYTO:
	+ actionAsText = "A_TE_COPYTO";
	+ break;
	case A_PE_INVOKE:
	actionAsText = "A_PE_INVOKE";
	break;
	case O_PE_RESTART:
	actionAsText = "O_PE_RESTART";
	break;
	case A_PE_START:
	actionAsText = "A_PE_START";
	break;
	case A_PE_STOP:
	actionAsText = "A_PE_STOP";
	break;
	case A_NODE_BLOCK:
	actionAsText = "A_NODE_BLOCK";
	break;
	case A_UPDATE_NODESTATUS:
	actionAsText = "A_UPDATE_NODESTATUS";
	break;
	case A_LOG:
	actionAsText = "A_LOG ";
	break;
	case A_ERROR:
	actionAsText = "A_ERROR ";
	break;
	case A_WARN:
	actionAsText = "A_WARN ";
	break;
	}

	if(actionAsText == NULL) {
	cl_log(LOG_ERR, "Action %.16llx is unknown", action);
	actionAsText = "<UNKNOWN_ACTION>";
	}

	return actionAsText;
	}

	long long
	do_state_transition(long long actions,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_state next_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	long long tmp = A_NOTHING;

	if(current_input != I_NULL
	&& (current_input != I_DC_HEARTBEAT \|\| cur_state != S_NOT_DC)){
	const char *state_from = fsa_state2string(cur_state);
	const char *state_to = fsa_state2string(next_state);
	const char *input = fsa_input2string(current_input);

	time_t now = time(NULL);

	fprintf(dot_strm,
	"\t\"%s\" -> \"%s\" [ label =\"%s\" ] // %s",
	state_from, state_to, input,
	asctime(localtime(&now)));
	fflush(dot_strm);
	}

	switch(next_state) {
	case S_PENDING:
	case S_NOT_DC:
	if(is_set(fsa_input_register, R_SHUTDOWN)){
	tmp = set_bit(actions, A_SHUTDOWN_REQ);
	}
	tmp = clear_bit(actions, A_RECOVER);
	break;
	case S_RECOVERY_DC:
	case S_RECOVERY:
	tmp = set_bit(actions, A_RECOVER);
	break;
	default:
	tmp = clear_bit(actions, A_RECOVER);
	break;
	}

	if(tmp != actions) {
	cl_log(LOG_INFO, "Action b4 %.16llx ", actions);
	cl_log(LOG_INFO, "Action after %.16llx ", tmp);
	actions = tmp;
	}

	return actions;
	}

	long long
	clear_flags(long long actions,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input cur_input)
	{

	if(is_set(fsa_input_register, R_SHUTDOWN)){
	clear_bit_inplace(&actions, A_DC_TIMER_START);
	}


	switch(cur_state) {
	case S_IDLE:
	break;
	case S_ELECTION:
	break;
	case S_INTEGRATION:
	break;
	case S_NOT_DC:
	break;
	case S_POLICY_ENGINE:
	break;
	case S_RECOVERY:
	break;
	case S_RECOVERY_DC:
	break;
	case S_RELEASE_DC:
	break;
	case S_PENDING:
	break;
	case S_STOPPING:
	break;
	case S_TERMINATE:
	break;
	case S_TRANSITION_ENGINE:
	break;
	case S_ILLEGAL:
	break;
	}
	return actions;
	}
	diff --git a/crm/crmd/messages.c b/crm/crmd/messages.c
	index 2bceb701f5..b1e6036c35 100644
	--- a/crm/crmd/messages.c
	+++ b/crm/crmd/messages.c
	@@ -1,865 +1,866 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <portability.h>
	#include <crm/crm.h>
	#include <string.h>
	#include <crmd_fsa.h>
	#include <libxml/tree.h>


	#include <crm/msg_xml.h>
	#include <crm/common/xmlutils.h>
	#include <crm/common/msgutils.h>
	#include <crm/cib.h>

	#include <crmd.h>
	#include <crmd_messages.h>

	#include <crm/dmalloc_wrapper.h>

	FILE *msg_in_strm = NULL;
	FILE *router_strm = NULL;

	fsa_message_queue_t fsa_message_queue = NULL;

	gboolean relay_message(xmlNodePtr xml_relay_message,
	gboolean originated_locally);

	#ifdef MSG_LOG

	# define ROUTER_RESULT(x) char *msg_text = dump_xml(xml_relay_message);\
	if(router_strm == NULL) { \
	router_strm = fopen("/tmp/router.log", "w"); \
	} \
	fprintf(router_strm, "[%d RESULT (%s)]\t%s\t%s\n", \
	AM_I_DC, \
	xmlGetProp(xml_relay_message, XML_ATTR_REFERENCE),\
	x, msg_text); \
	fflush(router_strm); \
	cl_free(msg_text);
	#else
	# define ROUTER_RESULT(x) CRM_DEBUG(x);
	#endif



	/* returns the current head of the FIFO queue */
	fsa_message_queue_t
	put_message(xmlNodePtr new_message)
	{
	fsa_message_queue_t next_message = (fsa_message_queue_t)
	cl_malloc(sizeof(struct fsa_message_queue_s));

	CRM_DEBUG("Adding msg to queue");
	-
	- next_message->message = new_message;
	+
	+ // make sure to free it properly later
	+ next_message->message = copy_xml_node_recursive(new_message);
	next_message->next = NULL;

	if(fsa_message_queue == NULL) {
	fsa_message_queue = next_message;
	} else {
	fsa_message_queue->next = next_message;
	}

	CRM_DEBUG("Added msg to queue");

	return fsa_message_queue;
	}

	/* returns the next message */
	fsa_message_queue_t
	get_message(void)
	{
	fsa_message_queue_t next_message = NULL;

	if(fsa_message_queue != NULL) {
	next_message = fsa_message_queue;
	fsa_message_queue = fsa_message_queue->next;
	next_message->next = NULL;
	}


	return next_message;
	}

	/* returns the current head of the FIFO queue */
	gboolean
	is_message(void)
	{
	return (fsa_message_queue != NULL
	&& fsa_message_queue->message != NULL);
	}


	/* A_MSG_STORE */
	enum crmd_fsa_input
	do_msg_store(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	// xmlNodePtr new_message = (xmlNodePtr)data;
	FNIN();

	// put_message(new_message);

	FNRET(I_NULL);
	}


	/* A_MSG_ROUTE */
	enum crmd_fsa_input
	do_msg_route(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input result = I_NULL;
	xmlNodePtr xml_message = (xmlNodePtr)data;
	gboolean routed = FALSE, defer = TRUE, do_process = TRUE;

	FNIN();

	#if 0
	// if(cause == C_IPC_MESSAGE) {
	if (crmd_authorize_message(root_xml_node,
	msg,
	curr_client) == FALSE) {
	CRM_DEBUG("Message not authorized");
	do_process = FALSE;
	}
	// }
	#endif
	if(do_process) {
	/* try passing the buck first */
	routed = relay_message(xml_message, cause==C_IPC_MESSAGE);

	if(routed == FALSE) {

	defer = TRUE;
	/* calculate defer */
	result = handle_message(xml_message);
	switch(result) {
	case I_NULL:
	defer = FALSE;
	break;
	case I_DC_HEARTBEAT:
	defer = FALSE;
	break;

	/* what else should go here? */
	default:
	CRM_DEBUG("Defering local processing of message");

	put_message(xml_message);
	result = I_REQUEST;
	break;
	}
	}
	}

	FNRET(result);
	}


	void
	crmd_ha_input_callback(const struct ha_msg* msg, void* private_data)
	{
	const char *from = ha_msg_value(msg, F_ORIG);
	const char *to = NULL;
	xmlNodePtr root_xml_node;

	FNIN();

	#ifdef MSG_LOG
	if(msg_in_strm == NULL) {
	msg_in_strm = fopen("/tmp/inbound.log", "w");
	}
	#endif

	if(from == NULL \|\| strcmp(from, fsa_our_uname) == 0) {
	#ifdef MSG_LOG
	fprintf(msg_in_strm,
	"Discarded message [F_SEQ=%s] from ourselves.\n",
	ha_msg_value(msg, F_SEQ));
	#endif
	FNOUT();
	}

	#ifdef MSG_LOG
	fprintf(msg_in_strm, "[%s (%s:%s)]\t%s\n",
	from,
	ha_msg_value(msg, F_SEQ),
	ha_msg_value(msg, F_TYPE),
	ha_msg_value(msg, "xml")
	);
	fflush(msg_in_strm);
	#endif

	root_xml_node = find_xml_in_hamessage(msg);
	to = xmlGetProp(root_xml_node, XML_ATTR_HOSTTO);

	if(to != NULL && strlen(to) > 0 && strcmp(to, fsa_our_uname) != 0) {
	#ifdef MSG_LOG
	fprintf(msg_in_strm,
	"Discarding message [F_SEQ=%s] for someone else.",
	ha_msg_value(msg, F_SEQ));
	#endif
	FNOUT();
	}

	set_xml_property_copy(root_xml_node, XML_ATTR_HOSTFROM, from);
	s_crmd_fsa(C_HA_MESSAGE, I_ROUTER, root_xml_node);

	free_xml(root_xml_node);

	FNOUT();
	}

	/*
	* Apparently returning TRUE means "stay connected, keep doing stuff".
	* Returning FALSE means "we're all done, close the connection"
	*/
	gboolean
	crmd_ipc_input_callback(IPC_Channel *client, gpointer user_data)
	{
	int lpc = 0;
	char *buffer = NULL;
	IPC_Message *msg = NULL;
	gboolean hack_return_good = TRUE;
	xmlNodePtr root_xml_node;
	crmd_client_t curr_client = (crmd_client_t)user_data;

	FNIN();
	CRM_DEBUG("Processing IPC message from %s",
	curr_client->table_key);

	while(client->ops->is_message_pending(client)) {
	if (client->ch_status == IPC_DISCONNECT) {
	/* The message which was pending for us is that
	* the IPC status is now IPC_DISCONNECT */
	break;
	}
	if (client->ops->recv(client, &msg) != IPC_OK) {
	perror("Receive failure:");
	FNRET(!hack_return_good);
	}
	if (msg == NULL) {
	CRM_DEBUG("No message this time");
	continue;
	}

	lpc++;
	buffer = (char*)msg->msg_body;
	CRM_DEBUG("Processing xml from %s [text=%s]",
	curr_client->table_key, buffer);

	root_xml_node =
	find_xml_in_ipcmessage(msg, FALSE);
	if (root_xml_node != NULL) {
	if (crmd_authorize_message(root_xml_node,
	msg,
	curr_client)) {
	CRM_DEBUG("Message authorized,about to relay");
	s_crmd_fsa(C_IPC_MESSAGE,
	I_ROUTER,
	root_xml_node);

	} else {
	CRM_DEBUG("Message not authorized");
	}
	} else {
	cl_log(LOG_INFO,
	"IPC Message was not valid... discarding.");
	}
	free_xml(root_xml_node);
	msg->msg_done(msg);

	msg = NULL;
	buffer = NULL;
	root_xml_node = NULL;
	}

	CRM_DEBUG("Processed %d messages", lpc);

	if (client->ch_status == IPC_DISCONNECT)
	{
	cl_log(LOG_INFO,
	"received HUP from %s",
	curr_client->table_key);
	if (curr_client != NULL) {
	struct crm_subsystem_s *the_subsystem = NULL;

	if (curr_client->sub_sys == NULL)
	CRM_DEBUG("Client had not registered with us yet");
	else if (strcmp(CRM_SYSTEM_PENGINE, curr_client->sub_sys) == 0)
	the_subsystem = pe_subsystem;
	else if (strcmp(CRM_SYSTEM_TENGINE, curr_client->sub_sys) == 0)
	the_subsystem = te_subsystem;
	else if (strcmp(CRM_SYSTEM_CIB, curr_client->sub_sys) == 0)
	the_subsystem = cib_subsystem;

	if(the_subsystem != NULL) {
	cleanup_subsystem(the_subsystem);
	} // else that was a transient client

	if (curr_client->table_key != NULL) {
	/*
	* Key is destroyed below: curr_client->table_key
	* Value is cleaned up by G_main_del_IPC_Channel
	*/
	g_hash_table_remove(ipc_clients,
	curr_client->table_key);
	}


	if(curr_client->client_source != NULL) {
	gboolean det =
	G_main_del_IPC_Channel(curr_client->client_source);

	CRM_DEBUG("crm_client was %s detached",
	det?"successfully":"not");
	}

	cl_free(curr_client->table_key);
	cl_free(curr_client->sub_sys);
	cl_free(curr_client->uuid);
	cl_free(curr_client);
	}
	CRM_DEBUG("this client has now left the building.");
	FNRET(!hack_return_good);
	}

	FNRET(hack_return_good);
	}


	/*
	* This method adds a copy of xml_response_data
	*/
	gboolean
	send_request(xmlNodePtr msg_options, xmlNodePtr msg_data,
	const char operation, const char host_to, const char *sys_to)
	{
	gboolean was_sent = FALSE;
	xmlNodePtr request = NULL;

	FNIN();


	msg_options = set_xml_attr(msg_options, XML_TAG_OPTIONS,
	XML_ATTR_OP, operation, TRUE);

	request = create_request(msg_options,
	msg_data,
	host_to,
	sys_to,
	AM_I_DC?CRM_SYSTEM_DC:CRM_SYSTEM_CRMD,
	NULL,
	NULL);

	// xml_message_debug(request, "Final request...");

	was_sent = relay_message(request, TRUE);

	if(was_sent == FALSE) {
	put_message(request);
	}

	free_xml(request);

	FNRET(was_sent);
	}

	/*
	* This method adds a copy of xml_response_data
	*/
	gboolean
	store_request(xmlNodePtr msg_options, xmlNodePtr msg_data,
	const char operation, const char host_to, const char *sys_to)
	{
	xmlNodePtr request = NULL;
	FNIN();


	msg_options = set_xml_attr(msg_options, XML_TAG_OPTIONS,
	XML_ATTR_OP, operation, TRUE);

	request = create_request(msg_options,
	msg_data,
	host_to,
	sys_to,
	AM_I_DC?CRM_SYSTEM_DC:CRM_SYSTEM_CRMD,
	NULL,
	NULL);

	put_message(request);

	FNRET(TRUE);
	}

	gboolean
	relay_message(xmlNodePtr xml_relay_message, gboolean originated_locally)
	{
	int is_for_dc = 0;
	int is_for_dcib = 0;
	int is_for_crm = 0;
	int is_for_cib = 0;
	int is_local = 0;
	gboolean dont_cc= TRUE;
	gboolean processing_complete = FALSE;
	const char *host_to = xmlGetProp(xml_relay_message,XML_ATTR_HOSTTO);
	const char *sys_to = xmlGetProp(xml_relay_message,XML_ATTR_SYSTO);

	FNIN();

	if(xml_relay_message == NULL) {
	cl_log(LOG_ERR, "Cannot route empty message");
	FNRET(TRUE);
	}

	if(strcmp("hello", xml_relay_message->name) == 0) {
	/* quietly ignore */
	FNRET(TRUE);
	}

	if(strcmp(XML_MSG_TAG, xml_relay_message->name) != 0) {

	xml_message_debug(xml_relay_message,
	"Bad message type, should be crm_message");
	cl_log(LOG_ERR, "Ignoring message of type %s",
	xml_relay_message->name);
	FNRET(TRUE);
	}


	if(sys_to == NULL) {
	xml_message_debug(xml_relay_message,
	"Message did not have any value for sys_to");
	cl_log(LOG_ERR, "Message did not have any value for %s",
	XML_ATTR_SYSTO);
	FNRET(TRUE);
	}

	is_for_dc = (strcmp(CRM_SYSTEM_DC, sys_to) == 0);
	is_for_dcib = (strcmp(CRM_SYSTEM_DCIB, sys_to) == 0);
	is_for_cib = (strcmp(CRM_SYSTEM_CIB, sys_to) == 0);
	is_for_crm = (strcmp(CRM_SYSTEM_CRMD, sys_to) == 0);

	is_local = 0;
	if(host_to == NULL \|\| strlen(host_to) == 0) {
	if(is_for_dc)
	is_local = 0;
	else if(is_for_crm && originated_locally)
	is_local = 0;
	else
	is_local = 1;

	} else if(strcmp(fsa_our_uname, host_to) == 0) {
	is_local=1;
	}

	#if 0
	CRM_DEBUG("is_local %d", is_local);
	CRM_DEBUG("is_for_dcib %d", is_for_dcib);
	CRM_DEBUG("is_for_dc %d", is_for_dc);
	CRM_DEBUG("is_for_crm %d", is_for_crm);
	CRM_DEBUG("AM_I_DC %d", AM_I_DC);
	CRM_DEBUG("sys_to %s", sys_to);
	CRM_DEBUG("host_to %s", host_to);
	#endif

	if(is_for_dc \|\| is_for_dcib) {
	if(AM_I_DC) {
	ROUTER_RESULT("Message result: DC/CRMd process");
	processing_complete = FALSE; // more to be done by caller

	} else if(originated_locally) {
	ROUTER_RESULT("Message result: External relay to DC");
	send_msg_via_ha(xml_relay_message, NULL);
	processing_complete = TRUE;

	} else {
	ROUTER_RESULT("Message result: Discard, not DC");
	processing_complete = TRUE; // discard
	}

	} else if(is_local && (is_for_crm \|\| is_for_cib)) {
	ROUTER_RESULT("Message result: CRMd process");

	} else if(is_local) {
	if(dont_cc) {
	ROUTER_RESULT("Message result: Local relay");
	} else {
	/* The DC should also get this message */
	ROUTER_RESULT("Message result: Local relay with CC");
	}
	send_msg_via_ipc(xml_relay_message, sys_to);
	processing_complete = TRUE & dont_cc;

	} else {
	if(dont_cc) {
	ROUTER_RESULT("Message result: External relay");
	} else {
	/* The DC should also get this message */
	ROUTER_RESULT("Message result: External relay with CC");
	}
	send_msg_via_ha(xml_relay_message, host_to);
	processing_complete = TRUE & dont_cc;

	}

	FNRET(processing_complete);
	}

	void
	send_msg_via_ha(xmlNodePtr action, const char *dest_node)
	{
	FNIN();
	if (action == NULL) FNOUT();

	if (validate_crm_message(action, NULL, NULL, NULL) == NULL)
	{
	cl_log(LOG_ERR,
	"Relay message to (%s) via HA was invalid, ignoring",
	dest_node);
	FNOUT();
	}
	// CRM_DEBUG("Relaying message to (%s) via HA", dest_node);
	set_xml_property_copy(action, XML_ATTR_HOSTTO, dest_node);

	send_xmlha_message(fsa_cluster_conn, action);
	FNOUT();
	}


	void
	send_msg_via_ipc(xmlNodePtr action, const char *sys)
	{
	IPC_Channel *client_channel;

	FNIN();
	// cl_log(LOG_DEBUG, "relaying msg to sub_sys=%s via IPC", sys);

	client_channel =
	(IPC_Channel*)g_hash_table_lookup (ipc_clients, sys);

	if (client_channel != NULL) {
	cl_log(LOG_DEBUG, "Sending message via channel %s.", sys);
	send_xmlipc_message(client_channel, action);
	} else if(sys != NULL && strcmp(sys, CRM_SYSTEM_CIB) == 0) {
	cl_log(LOG_ERR,
	"Sub-system (%s) has been incorporated into the CRMd.",
	sys);
	xml_message_debug(action, "Change the way we handle");
	relay_message(process_cib_message(action, TRUE), TRUE);

	} else if(sys != NULL && strcmp(sys, CRM_SYSTEM_LRMD) == 0) {

	do_lrm_invoke(A_LRM_INVOKE, C_IPC_MESSAGE,
	fsa_state, I_MESSAGE, action);

	} else {
	cl_log(LOG_ERR,
	"Unknown Sub-system (%s)... discarding message.",
	sys);
	}
	FNOUT();
	}

	gboolean
	crmd_authorize_message(xmlNodePtr root_xml_node,
	IPC_Message *client_msg,
	crmd_client_t *curr_client)
	{
	// check the best case first
	const char *sys_from = xmlGetProp(root_xml_node,
	XML_ATTR_SYSFROM);
	char *uuid = NULL;
	char *client_name = NULL;
	char *major_version = NULL;
	char *minor_version = NULL;
	const char *filtered_from;
	gpointer table_key = NULL;
	gboolean result;

	FNIN();

	if (sys_from != NULL) {
	gboolean can_reply = FALSE; // no-one has registered with this id
	filtered_from = sys_from;

	/* The CIB can have two names on the DC */
	if(strcmp(sys_from, CRM_SYSTEM_DCIB) == 0)
	filtered_from = CRM_SYSTEM_CIB;

	if (g_hash_table_lookup (ipc_clients, filtered_from) != NULL)
	can_reply = TRUE; // reply can be routed


	CRM_DEBUG("Message reply can%s be routed from %s.",
	can_reply?"":" not", sys_from);
	FNRET(can_reply);
	}

	// otherwise, check if it was a hello message

	cl_log(LOG_INFO,
	"received client join msg: %s",
	(char*)client_msg->msg_body);

	result = process_hello_message(client_msg,
	&uuid,
	&client_name,
	&major_version,
	&minor_version);

	if (result == TRUE) {
	// check version
	int mav = atoi(major_version);
	int miv = atoi(minor_version);
	if (mav < 0 \|\| miv < 0) {
	cl_log(LOG_ERR,
	"Client version (%d:%d) is not acceptable",
	mav,
	miv);
	result = FALSE;
	}
	cl_free(major_version);
	cl_free(minor_version);
	}

	struct crm_subsystem_s *the_subsystem = NULL;

	if (result == TRUE) {
	/* if we already have one of those clients
	* only applies to te, pe etc. not admin clients
	*/


	if (client_name == NULL)
	CRM_DEBUG("Client had not registered with us yet");
	else if (strcmp(CRM_SYSTEM_PENGINE, client_name) == 0)
	the_subsystem = pe_subsystem;
	else if (strcmp(CRM_SYSTEM_TENGINE, client_name) == 0)
	the_subsystem = te_subsystem;
	else if (strcmp(CRM_SYSTEM_CIB, client_name) == 0)
	the_subsystem = cib_subsystem;

	if (the_subsystem != NULL) {
	// do we already have one?
	result =(fsa_input_register & the_subsystem->flag)==0;
	if(result) {
	the_subsystem->ipc =
	curr_client->client_channel;

	} // else we didnt ask for the client to start

	} else if(client_name != NULL && uuid != NULL) {
	table_key = (gpointer)
	generate_hash_key(client_name, uuid);
	} else {
	result = FALSE;
	cl_log(LOG_ERR,
	"Bad client details (client_name=%s, uuid=%s)",
	client_name, uuid);
	}
	}

	if(result == TRUE && table_key == NULL)
	table_key = (gpointer)cl_strdup(client_name);

	if (result == TRUE) {
	cl_log(LOG_INFO, "Accepted client %s", (char*)table_key);

	curr_client->table_key = table_key;
	curr_client->sub_sys = cl_strdup(client_name);
	curr_client->uuid = cl_strdup(uuid);

	g_hash_table_insert (ipc_clients,
	table_key,
	curr_client->client_channel);

	send_hello_message(curr_client->client_channel,
	"n/a", CRM_SYSTEM_CRMD,
	"0", "1");

	cl_log(LOG_INFO, "Updated client list with %s",
	(char*)table_key);

	if(the_subsystem != NULL) {
	set_bit_inplace(&fsa_input_register,
	the_subsystem->flag);
	}

	s_crmd_fsa(C_SUBSYSTEM_CONNECT, I_NULL, NULL);

	} else {
	cl_log(LOG_ERR, "Rejected client logon request");
	curr_client->client_channel->ch_status = IPC_DISC_PENDING;
	}

	if(uuid != NULL) cl_free(uuid);
	if(client_name != NULL) cl_free(client_name);

	/* hello messages should never be processed further */
	FNRET(FALSE);
	}

	enum crmd_fsa_input
	handle_message(xmlNodePtr stored_msg)
	{
	enum crmd_fsa_input next_input = I_NULL;

	const char *sys_to = get_xml_attr(stored_msg, NULL,
	XML_ATTR_SYSTO, TRUE);

	// const char *sys_from = get_xml_attr(stored_msg, NULL,
	// XML_ATTR_SYSFROM, TRUE);

	const char *type = get_xml_attr(stored_msg, NULL,
	XML_ATTR_MSGTYPE, TRUE);

	const char *op = get_xml_attr(stored_msg, XML_TAG_OPTIONS,
	XML_ATTR_OP, TRUE);

	// xml_message_debug(stored_msg, "Processing message");

	if(type == NULL \|\| op == NULL) {
	cl_log(LOG_ERR, "Ignoring message (type=%s), (op=%s)",
	type, op);
	xml_message_debug(stored_msg, "Bad message");

	} else if(strcmp(type, XML_ATTR_REQUEST) == 0){
	if(strcmp(op, CRM_OPERATION_VOTE) == 0) {
	next_input = I_ELECTION;

	} else if(AM_I_DC && strcmp(op, "te_abort") == 0) {
	next_input = I_PE_CALC;

	} else if(AM_I_DC
	&& fsa_state == S_TRANSITION_ENGINE
	&& strcmp(op, "te_complete") == 0) {
	next_input = I_SUCCESS;

	} else if(strcmp(op, CRM_OPERATION_HBEAT) == 0) {
	next_input = I_DC_HEARTBEAT;

	} else if(strcmp(op, CRM_OPERATION_WELCOME) == 0) {
	next_input = I_WELCOME;

	} else if(strcmp(op, CRM_OPERATION_SHUTDOWN_REQ) == 0) {
	next_input = I_CIB_OP;

	} else if(strcmp(op, CRM_OPERATION_SHUTDOWN) == 0) {
	next_input = I_TERMINATE;

	} else if(strcmp(op, CRM_OPERATION_ANNOUNCE) == 0) {
	next_input = I_NODE_JOIN;

	} else if(strcmp(op, CRM_OPERATION_REPLACE) == 0
	\|\| strcmp(op, CRM_OPERATION_ERASE) == 0) {
	next_input = I_CIB_OP;
	fprintf(router_strm, "Message result: CIB Op\n");

	} else if(AM_I_DC
	&& (strcmp(op, CRM_OPERATION_CREATE) == 0
	\|\| strcmp(op, CRM_OPERATION_UPDATE) == 0
	\|\| strcmp(op, CRM_OPERATION_DELETE) == 0)) {
	/* updates should only be performed on the DC */
	next_input = I_CIB_OP;

	} else if(strcmp(op, CRM_OPERATION_PING) == 0) {
	/* eventually do some stuff to figure out
	* if we /are/ ok
	*/
	xmlNodePtr ping =
	createPingAnswerFragment(sys_to, "ok");

	xmlNodePtr wrapper = create_reply(stored_msg, ping);
	relay_message(wrapper, TRUE);
	free_xml(wrapper);

	} else {
	cl_log(LOG_ERR,
	"Unexpected request (op=%s) sent to the %s",
	op, AM_I_DC?"DC":"CRMd");
	}

	} else if(strcmp(type, XML_ATTR_RESPONSE) == 0) {

	if(strcmp(op, CRM_OPERATION_WELCOME) == 0) {
	next_input = I_WELCOME_ACK;

	} else if(strcmp(op, "pecalc") == 0) {
	// results int eh TE being invoked
	next_input = I_SUCCESS;

	} else if(strcmp(op, CRM_OPERATION_VOTE) == 0
	\|\| strcmp(op, CRM_OPERATION_HBEAT) == 0
	\|\| strcmp(op, CRM_OPERATION_WELCOME) == 0
	\|\| strcmp(op, CRM_OPERATION_SHUTDOWN_REQ) == 0
	\|\| strcmp(op, CRM_OPERATION_SHUTDOWN) == 0
	\|\| strcmp(op, CRM_OPERATION_ANNOUNCE) == 0) {
	next_input = I_NULL;

	/* } else if(AM_I_DC */
	/* && (strcmp(op, CRM_OPERATION_CREATE) == 0 */
	/* \|\| strcmp(op, CRM_OPERATION_UPDATE) == 0 */
	/* \|\| strcmp(op, CRM_OPERATION_DELETE) == 0 */
	/* \|\| strcmp(op, CRM_OPERATION_REPLACE) == 0 */
	/* \|\| strcmp(op, CRM_OPERATION_ERASE) == 0)) { */

	/* // perhaps we should do somethign with these replies */
	/* fprintf(router_strm, "Message result: CIB Reply\n"); */

	} else {
	cl_log(LOG_ERR,
	"Unexpected response (op=%s) sent to the %s",
	op, AM_I_DC?"DC":"CRMd");
	next_input = I_NULL;
	}
	} else {
	cl_log(LOG_ERR, "Unexpected message type %s", type);

	}

	/* CRM_DEBUG("%s: Next input is %s", __FUNCTION__, */
	/* fsa_input2string(next_input)); */


	return next_input;

	}


	void lrm_op_callback (lrm_op_t* op)
	{
	CRM_DEBUG("In lrm_op_callback()");

	s_crmd_fsa(C_LRM_OP_CALLBACK, I_LRM_EVENT, op);
	}

	void lrm_monitor_callback (lrm_mon_t* monitor)
	{
	CRM_DEBUG("In lrm_monitor_callback()");
	s_crmd_fsa(C_LRM_MONITOR_CALLBACK, I_LRM_EVENT, monitor);
	}

	diff --git a/crm/crmd/subsystems.c b/crm/crmd/subsystems.c
	index f28c27abff..8509d4ed1a 100644
	--- a/crm/crmd/subsystems.c
	+++ b/crm/crmd/subsystems.c
	@@ -1,1174 +1,1176 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <portability.h>
	#include <crm/crm.h>
	#include <crmd_fsa.h>

	#include <sys/types.h>
	#include <sys/wait.h>

	#include <unistd.h> // for access
	#include <clplumbing/cl_signal.h>
	#include <clplumbing/realtime.h>
	#include <sys/types.h> // for calls to open
	#include <sys/stat.h> // for calls to open
	#include <fcntl.h> // for calls to open
	#include <pwd.h> // for getpwuid
	#include <grp.h> // for initgroups

	#include <sys/time.h> // for getrlimit
	#include <sys/resource.h>// for getrlimit

	#include <crm/common/crmutils.h>
	#include <crm/common/ipcutils.h>
	#include <crm/common/msgutils.h>

	#include <crm/msg_xml.h>
	#include <crm/common/xmlutils.h>


	#include <crm/cib.h>

	#include <crmd.h>
	#include <crmd_messages.h>
	#include <string.h>
	#include <errno.h>

	#include <crm/dmalloc_wrapper.h>

	#define CLIENT_EXIT_WAIT 10

	static gboolean stop_subsystem (struct crm_subsystem_s *centry);
	static gboolean start_subsystem(struct crm_subsystem_s *centry);
	static gboolean run_command (struct crm_subsystem_s *centry,
	const char *options,
	gboolean update_pid);

	xmlNodePtr do_lrm_query(void);

	GHashTable xml2list(xmlNodePtr parent, const char *attr_path, int depth);

	gboolean lrm_dispatch(int fd, gpointer user_data);

	void do_update_resource(lrm_rsc_t rsc, int status, int rc, const char op_type);

	struct crm_subsystem_s *cib_subsystem = NULL;
	struct crm_subsystem_s *te_subsystem = NULL;
	struct crm_subsystem_s *pe_subsystem = NULL;

	void
	cleanup_subsystem(struct crm_subsystem_s *the_subsystem)
	{
	int pid_status = -1;
	the_subsystem->ipc = NULL;
	clear_bit_inplace(&fsa_input_register,
	the_subsystem->flag);

	/* Forcing client to die */
	kill(the_subsystem->pid, -9);

	// cleanup the ps entry
	waitpid(the_subsystem->pid, &pid_status, WNOHANG);
	the_subsystem->pid = -1;
	}

	/* A_CIB_STOP, A_CIB_START, A_CIB_RESTART, */
	enum crmd_fsa_input
	do_cib_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input result = I_NULL;
	struct crm_subsystem_s *this_subsys = cib_subsystem;

	long long stop_actions = A_CIB_STOP;
	long long start_actions = A_CIB_START;

	FNIN();

	if(action & stop_actions) {
	// dont do anything, its embedded now
	}

	if(action & start_actions) {

	if(cur_state != S_STOPPING) {
	if(startCib(CIB_FILENAME) == FALSE)
	result = I_FAIL;

	} else {
	cl_log(LOG_INFO,
	"Ignoring request to start %s after shutdown",
	this_subsys->command);
	}
	}

	FNRET(result);
	}

	/* A_CIB_INVOKE, A_CIB_BUMPGEN, A_UPDATE_NODESTATUS */
	enum crmd_fsa_input
	do_cib_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	xmlNodePtr cib_msg = NULL;
	xmlNodePtr answer = NULL;
	xmlNodePtr tmp1 = NULL;
	xmlNodePtr tmp2 = NULL;
	xmlNodePtr new_options = NULL;
	const char *req_from;
	const char *section = NULL;

	FNIN();

	if(data != NULL)
	cib_msg = (xmlNodePtr)data;


	if(action & A_CIB_INVOKE) {

	- const char *op = xmlGetProp(cib_msg, XML_ATTR_OP);
	+ const char *op = get_xml_attr(cib_msg, XML_TAG_OPTIONS,
	+ XML_ATTR_OP, TRUE);
	+
	if(safe_str_eq(op, CRM_OPERATION_SHUTDOWN_REQ)){
	// create update section
	tmp2 =
	create_xml_node(NULL, XML_CIB_TAG_STATE);
	req_from =
	xmlGetProp(cib_msg, XML_ATTR_HOSTFROM);

	set_xml_property_copy(tmp1, "id", req_from);
	set_xml_property_copy(tmp1, "exp_state", "shutdown");

	// create fragment
	tmp1 = create_cib_fragment(tmp2, NULL);

	// add to cib_msg
	add_node_copy(cib_msg, tmp1);

	free_xml(tmp2);
	free_xml(tmp1);
	}

	set_xml_property_copy(cib_msg, XML_ATTR_SYSTO, "cib");
	answer = process_cib_message(cib_msg, TRUE);
	if(relay_message(answer, TRUE) == FALSE) {
	cl_log(LOG_ERR, "Confused what to do with cib result");
	xml_message_debug(answer, "Couldnt route: ");
	}


	if(op != NULL && AM_I_DC
	&& (strcmp(op, CRM_OPERATION_CREATE) == 0
	\|\| strcmp(op, CRM_OPERATION_UPDATE) == 0
	\|\| strcmp(op, CRM_OPERATION_DELETE) == 0
	\|\| strcmp(op, CRM_OPERATION_REPLACE) == 0
	\|\| strcmp(op, CRM_OPERATION_WELCOME) == 0
	\|\| strcmp(op, CRM_OPERATION_SHUTDOWN_REQ) == 0
	\|\| strcmp(op, CRM_OPERATION_ERASE) == 0)) {
	FNRET(I_CIB_UPDATE);
	}

	if(op == NULL) {
	xml_message_debug(cib_msg, "Invalid CIB Message");
	}



	// check the answer, see if we are interested in it also
	#if 0
	if(interested in reply) {
	put_message(answer);
	FNRET(I_REQUEST);
	}
	#endif

	free_xml(answer);

	/* experimental */
	} else if(action & A_CIB_INVOKE_LOCAL) {
	answer = process_cib_message(cib_msg, TRUE);
	put_message(answer);
	FNRET(I_REQUEST);

	} else if(action & A_CIB_BUMPGEN) {
	// check if the response was ok before next bit

	section = get_xml_attr(cib_msg, XML_TAG_OPTIONS,
	XML_ATTR_FILTER_TYPE, FALSE);

	/* set the section so that we dont always send the
	* whole thing
	*/

	if(section != NULL) {
	new_options = set_xml_attr(NULL, XML_TAG_OPTIONS,
	XML_ATTR_FILTER_TYPE,
	section, TRUE);
	}

	answer = process_cib_request(CRM_OPERATION_BUMP,
	new_options, NULL);

	free_xml(new_options);

	if(answer == NULL) {
	cl_log(LOG_ERR, "Result of BUMP in %s was NULL",
	__FUNCTION__);
	FNRET(I_FAIL);
	}

	send_request(NULL, answer, CRM_OPERATION_REPLACE,
	NULL, CRM_SYSTEM_CRMD);

	free_xml(answer);

	} else {
	cl_log(LOG_ERR, "Unexpected action %s in %s",
	fsa_action2string(action), __FUNCTION__);
	}


	FNRET(I_NULL);
	}


	/* A_PE_START, A_PE_STOP, A_TE_RESTART */
	enum crmd_fsa_input
	do_pe_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input result = I_NULL;
	struct crm_subsystem_s *this_subsys = pe_subsystem;

	long long stop_actions = A_PE_STOP;
	long long start_actions = A_PE_START;

	FNIN();

	if(action & stop_actions) {
	if(stop_subsystem(this_subsys) == FALSE)
	result = I_FAIL;
	else if(this_subsys->pid > 0){
	int lpc = CLIENT_EXIT_WAIT;
	int pid_status = -1;
	while(lpc-- > 0
	&& this_subsys->pid > 0
	&& CL_PID_EXISTS(this_subsys->pid)) {

	sleep(1);
	waitpid(this_subsys->pid, &pid_status, WNOHANG);
	}

	if(CL_PID_EXISTS(this_subsys->pid)) {
	cl_log(LOG_ERR,
	"Process %s is still active with pid=%d",
	this_subsys->command, this_subsys->pid);
	result = I_FAIL;
	}
	}

	cleanup_subsystem(this_subsys);
	}

	if(action & start_actions) {

	if(cur_state != S_STOPPING) {
	if(start_subsystem(this_subsys) == FALSE) {
	result = I_FAIL;
	cleanup_subsystem(this_subsys);
	}
	} else {
	cl_log(LOG_INFO,
	"Ignoring request to start %s while shutting down",
	this_subsys->command);
	}
	}

	FNRET(result);
	}

	/* A_PE_INVOKE */
	enum crmd_fsa_input
	do_pe_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	FNIN();

	stopTimer(integration_timer);

	if(is_set(fsa_input_register, R_PE_CONNECTED) == FALSE){

	cl_log(LOG_INFO, "Waiting for the PE to connect");
	FNRET(I_WAIT_FOR_EVENT);

	}

	xmlNodePtr local_cib = get_cib_copy();

	CRM_DEBUG("Invoking %s with %p", CRM_SYSTEM_PENGINE, local_cib);

	send_request(NULL, local_cib, "pecalc",
	NULL, CRM_SYSTEM_PENGINE);

	FNRET(I_NULL);
	}

	/* A_TE_START, A_TE_STOP, A_TE_RESTART */
	enum crmd_fsa_input
	do_te_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input result = I_NULL;
	struct crm_subsystem_s *this_subsys = te_subsystem;

	long long stop_actions = A_TE_STOP;
	long long start_actions = A_TE_START;

	FNIN();

	/* if(action & stop_actions && cur_state != S_STOPPING */
	/* && is_set(fsa_input_register, R_TE_PEND)) { */
	/* result = I_WAIT_FOR_EVENT; */
	/* FNRET(result); */
	/* } */

	if(action & stop_actions) {
	if(stop_subsystem(this_subsys) == FALSE)
	result = I_FAIL;
	else if(this_subsys->pid > 0){
	int lpc = CLIENT_EXIT_WAIT;
	int pid_status = -1;
	while(lpc-- > 0
	&& this_subsys->pid > 0
	&& CL_PID_EXISTS(this_subsys->pid)) {

	sleep(1);
	waitpid(this_subsys->pid, &pid_status, WNOHANG);
	}

	if(CL_PID_EXISTS(this_subsys->pid)) {
	cl_log(LOG_ERR,
	"Process %s is still active with pid=%d",
	this_subsys->command, this_subsys->pid);
	result = I_FAIL;
	}
	}

	cleanup_subsystem(this_subsys);
	}

	if(action & start_actions) {

	if(cur_state != S_STOPPING) {
	if(start_subsystem(this_subsys) == FALSE) {
	result = I_FAIL;
	cleanup_subsystem(this_subsys);
	}
	} else {
	cl_log(LOG_INFO,
	"Ignoring request to start %s while shutting down",
	this_subsys->command);
	}
	}

	FNRET(result);
	}

	/* A_TE_COPYTO */
	enum crmd_fsa_input
	do_te_copyto(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	xmlNodePtr message = copy_xml_node_recursive((xmlNodePtr)data);
	xmlNodePtr opts = find_xml_node(message, "options");
	const char *true_op = xmlGetProp(opts, XML_ATTR_OP);

	FNIN();

	set_xml_property_copy(opts, XML_ATTR_OP, "event");
	set_xml_property_copy(message, XML_ATTR_SYSTO, CRM_SYSTEM_TENGINE);
	set_xml_property_copy(opts, "true_op", true_op);

	relay_message(message, FALSE);

	free_xml(message);

	FNRET(I_NULL);
	}

	static xmlNodePtr te_last_input = NULL;

	/* A_TE_INVOKE, A_TE_CANCEL */
	enum crmd_fsa_input
	do_te_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	xmlNodePtr graph = NULL;
	xmlNodePtr msg = (xmlNodePtr)data;
	FNIN();

	if(is_set(fsa_input_register, R_TE_CONNECTED) == FALSE){
	cl_log(LOG_INFO, "Waiting for the TE to connect");
	if(data != NULL) {
	free_xml(te_last_input);
	te_last_input = copy_xml_node_recursive(msg);
	}
	FNRET(I_WAIT_FOR_EVENT);

	}

	if(msg == NULL) {
	msg = te_last_input;
	} else {
	free_xml(te_last_input);
	}

	if(action & A_TE_INVOKE) {
	graph = find_xml_node(msg, "transition_graph");
	if(graph == NULL) {
	FNRET(I_FAIL);
	}

	send_request(NULL, graph, "transition",
	NULL, CRM_SYSTEM_TENGINE);
	} else {
	send_request(NULL, graph, "abort",
	NULL, CRM_SYSTEM_TENGINE);
	}


	FNRET(I_NULL);
	}

	gboolean
	crmd_client_connect(IPC_Channel *client_channel, gpointer user_data)
	{
	FNIN();

	CRM_DEBUG("A client tried to connect... and there was much rejoicing.");

	if (client_channel == NULL) {
	cl_log(LOG_ERR, "Channel was NULL");
	} else if (client_channel->ch_status == IPC_DISCONNECT) {
	cl_log(LOG_ERR, "Channel was disconnected");
	} else {
	crmd_client_t *blank_client =
	(crmd_client_t *)cl_malloc(sizeof(crmd_client_t));

	if (blank_client == NULL) {
	cl_log(LOG_ERR,
	"Could not allocate memory for a blank crmd_client_t");
	FNRET(FALSE);
	}
	client_channel->ops->set_recv_qlen(client_channel, 100);
	client_channel->ops->set_send_qlen(client_channel, 100);

	blank_client->client_channel = client_channel;
	blank_client->sub_sys = NULL;
	blank_client->uuid = NULL;
	blank_client->table_key = NULL;

	CRM_DEBUG("Adding IPC Channel to main thread.");
	blank_client->client_source =
	G_main_add_IPC_Channel(G_PRIORITY_LOW,
	client_channel,
	FALSE,
	crmd_ipc_input_callback,
	blank_client,
	default_ipc_input_destroy);
	}

	FNRET(TRUE);
	}

	static gboolean
	stop_subsystem(struct crm_subsystem_s* centry)
	{
	cl_log(LOG_INFO, "Stopping sub-system \"%s\"", centry->name);
	if (centry->pid <= 0) {
	cl_log(LOG_ERR,
	"OOPS! client %s not running yet",
	centry->command);

	} else {
	cl_log(LOG_INFO, "Sending quit message to %s.", centry->name);
	send_request(NULL, NULL, "quit", NULL, centry->name);

	}

	return TRUE;
	}


	static gboolean
	start_subsystem(struct crm_subsystem_s* centry)
	{
	cl_log(LOG_INFO, "Starting sub-system \"%s\"", centry->command);

	if (centry->pid != 0) {
	cl_log(LOG_ERR, "OOPS! client %s already running as pid %d"
	, centry->command, (int) centry->pid);
	}

	return run_command(centry, "-r", TRUE);
	}


	static gboolean
	run_command(struct crm_subsystem_s *centry,
	const char *options,
	gboolean update_pid)
	{
	pid_t pid;
	struct stat buf;
	int s_res,size;
	char *cmd_with_options = NULL;

	/*
	* We need to ensure that the exec will succeed before
	* we bother forking. We don't want to respawn something that
	* won't exec in the first place.
	*/

	if (access(centry->path, F_OK\|X_OK) != 0) {
	cl_perror("Cannot (access) exec %s", centry->path);
	return FALSE;
	}

	s_res = stat(centry->command, &buf);
	if(s_res != 0) {
	cl_perror("Cannot (stat) exec %s", centry->command);
	return FALSE;
	}


	/* We need to fork so we can make child procs not real time */
	switch(pid=fork()) {

	case -1: cl_log(LOG_ERR
	, "start_a_child_client: Cannot fork.");
	return FALSE;

	default: /* Parent */
	#if 0
	NewTrackedProc(pid, 1, PT_LOGVERBOSE
	, centry, &ManagedChildTrackOps);
	#else
	if(update_pid)
	centry->pid = pid;
	#endif
	return TRUE;

	case 0: /* Child */
	break;
	}

	/* Child process: start the managed child */
	cl_make_normaltime();
	setpgid(0,0);

	/* Limit peak resource usage, maximize success chances */
	if (centry->shortrcount > 0) {
	alarm(0);
	sleep(1);
	}

	size = strlen(options);
	size += strlen(centry->command);
	size += 2; // ' ' + \0

	cmd_with_options = cl_malloc((1+size)*sizeof(char));
	sprintf(cmd_with_options, "%s %s", centry->command, options);
	cmd_with_options[size] = 0;


	cl_log(LOG_INFO, "Executing \"%s\" (pid %d)",
	cmd_with_options, (int) getpid());

	if(CL_SIGINTERRUPT(SIGALRM, 0) < 0) {
	cl_perror("Cannot set interrupt for child process %s",
	cmd_with_options);
	}else{
	const char * devnull = "/dev/null";
	unsigned int j;
	struct rlimit oflimits;
	CL_SIGNAL(SIGCHLD, SIG_DFL);
	alarm(0);
	CL_IGNORE_SIG(SIGALRM);

	/* A precautionary measure */
	getrlimit(RLIMIT_NOFILE, &oflimits);
	for (j=0; j < oflimits.rlim_cur; ++j) {
	close(j);
	}
	(void)devnull;

	(void)open(devnull, O_RDONLY); /* Stdin: fd 0 */
	(void)open(devnull, O_WRONLY); /* Stdout: fd 1 */
	(void)open(devnull, O_WRONLY); /* Stderr: fd 2 */

	(void)execl("/bin/sh", "sh", "-c", cmd_with_options, (const char *)NULL);

	/* Should not happen */
	cl_perror("Cannot exec %s", cmd_with_options);
	}
	/* Suppress respawning */
	exit(100);

	// never reached
	return TRUE;
	}


	/* A_LRM_CONNECT */
	enum crmd_fsa_input
	do_lrm_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input failed = I_NULL;//I_FAIL;
	int ret = HA_OK;
	FNIN();

	if(action & A_LRM_DISCONNECT) {
	fsa_lrm_conn->lrm_ops->signoff(fsa_lrm_conn);
	}

	if(action & A_LRM_CONNECT) {

	CRM_DEBUG("LRM: connect...");
	fsa_lrm_conn = ll_lrm_new("lrm");
	if(NULL == fsa_lrm_conn) {
	return failed;
	}

	CRM_DEBUG("LRM: sigon...");
	ret = fsa_lrm_conn->lrm_ops->signon(fsa_lrm_conn,
	"crmd");

	if(ret != HA_OK) {
	cl_log(LOG_ERR, "Failed to sign on to the LRM");
	return failed;
	}

	CRM_DEBUG("LRM: set_lrm_callback...");
	ret = fsa_lrm_conn->lrm_ops->set_lrm_callback(fsa_lrm_conn,
	lrm_op_callback,
	lrm_monitor_callback);

	if(ret != HA_OK) {
	cl_log(LOG_ERR, "Failed to set LRM callbacks");
	return failed;
	}

	/* TODO: create a destroy handler that causes
	* some recovery to happen
	*/
	G_main_add_fd(G_PRIORITY_LOW,
	fsa_lrm_conn->lrm_ops->inputfd(fsa_lrm_conn),
	FALSE,
	lrm_dispatch, fsa_lrm_conn,
	default_ipc_input_destroy);
	}

	if(action & ~(A_LRM_CONNECT\|A_LRM_DISCONNECT)) {
	cl_log(LOG_ERR, "Unexpected action %s in %s",
	fsa_action2string(action), __FUNCTION__);
	}


	FNRET(I_NULL);
	}

	gboolean lrm_dispatch(int fd, gpointer user_data)
	{
	ll_lrm_t lrm = (ll_lrm_t)user_data;
	lrm->lrm_ops->rcvmsg(lrm, FALSE);
	return TRUE;
	}

	xmlNodePtr
	do_lrm_query(void)
	{
	GList* lrm_list = NULL;
	GList* element = NULL;
	GList* op_list = NULL;
	xmlNodePtr agent = NULL;
	xmlNodePtr data = create_xml_node(NULL, "lrm");
	xmlNodePtr agent_list = create_xml_node(data, "lrm_agents");
	xmlNodePtr rsc_list;
	char *rsc_type = NULL;
	state_flag_t cur_state = 0;
	const char *this_op = NULL;
	GList* node = NULL;

	lrm_list = fsa_lrm_conn->lrm_ops->get_ra_supported(fsa_lrm_conn);
	if (NULL != lrm_list) {
	GList* element = g_list_first(lrm_list);
	while (NULL != element) {
	rsc_type = (char*)element->data;

	agent =
	create_xml_node(agent_list, "lrm_agent");

	set_xml_property_copy(agent, "class", rsc_type);

	/* we dont have these yet */
	set_xml_property_copy(agent, "type", NULL);
	set_xml_property_copy(agent, "version", NULL);

	element = g_list_next(element);
	}
	}

	g_list_free(lrm_list);
	lrm_list = fsa_lrm_conn->lrm_ops->get_all_rscs(fsa_lrm_conn);

	rsc_list = create_xml_node(data, "lrm_resources");

	if (NULL != lrm_list) {
	element = g_list_first(lrm_list);
	}

	while (NULL != element) {
	lrm_rsc_t the_rsc = (lrm_rsc_t)element->data;

	/* const char* ra_type; */
	/* GHashTable* params; */

	xmlNodePtr xml_rsc = create_xml_node(rsc_list, "rsc_state");

	set_xml_property_copy(xml_rsc, "id", the_rsc->id);
	set_xml_property_copy(xml_rsc, "rsc_id", the_rsc->name);
	set_xml_property_copy(xml_rsc, "node_id",fsa_our_uname);

	CRM_DEBUG("get_cur_state...");

	op_list = the_rsc->ops->get_cur_state(the_rsc,
	&cur_state);
	CRM_DEBUG("\tcurrent state:%s\n",
	cur_state==LRM_RSC_IDLE?"Idel":"Busy");

	node = g_list_first(op_list);

	while(NULL != node){
	lrm_op_t* op = (lrm_op_t*)node->data;
	this_op = op->op_type;
	if(this_op == NULL
	\|\| strcmp(this_op, "status") != 0){

	const char *status_text = "<unknown>";
	switch(op->status) {
	case LRM_OP_DONE:
	status_text = "done";
	break;
	case LRM_OP_CANCELLED:
	status_text = "cancelled";
	break;
	case LRM_OP_TIMEOUT:
	status_text = "timeout";
	break;
	case LRM_OP_NOTSUPPORTED:
	status_text = "not suported";
	break;
	case LRM_OP_ERROR:
	status_text = "error";
	break;
	}


	set_xml_property_copy(xml_rsc,
	"op_result",
	status_text);

	set_xml_property_copy(xml_rsc,
	"rsc_op",
	this_op);

	// we only want the last one
	break;
	}

	node = g_list_next(node);
	}

	element = g_list_next(element);
	}

	if (NULL != lrm_list) {
	g_list_free(lrm_list);
	}

	return data;
	}


	/* A_LRM_INVOKE */
	enum crmd_fsa_input
	do_lrm_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input next_input = I_NULL;
	xmlNodePtr fragment, tmp1;
	xmlNodePtr msg;
	const char *rsc_path[] =
	{
	"msg_data",
	"rsc_op",
	"resource",
	"instance_attributes",
	"parameters"
	};
	const char *operation = NULL;
	rsc_id_t rid;
	const char *id_from_cib = NULL;
	const char *crm_op = NULL;
	lrm_rsc_t *rsc = NULL;
	lrm_mon_t* mon = NULL;
	lrm_op_t* op = NULL;

	FNIN();

	#ifdef USE_FAKE_LRM
	msg = (xmlNodePtr)data;

	operation = get_xml_attr_nested(msg, rsc_path, DIMOF(rsc_path) -3,
	- XML_ATTR_OP, TRUE);
	+ "task", TRUE);

	id_from_cib = get_xml_attr_nested(msg, rsc_path, DIMOF(rsc_path) -2,
	"id", TRUE);

	crm_op = get_xml_attr(msg, XML_TAG_OPTIONS, XML_ATTR_OP, TRUE);


	if(safe_str_eq(crm_op, "rsc_op")) {

	CRM_DEBUG("performing op %s...", operation);

	- xmlNodePtr iter = create_xml_node(iter, "lrm_resource");
	+ xmlNodePtr iter = create_xml_node(NULL, "lrm_resource");

	set_xml_property_copy(iter, XML_ATTR_ID, id_from_cib);
	set_xml_property_copy(iter, "last_op", operation);

	if(safe_str_eq(operation, "start")){
	set_xml_property_copy(iter, "op_status", "started");
	} else {
	set_xml_property_copy(iter, "op_status", "stopped");
	}

	set_xml_property_copy(iter, "op_code", "0");
	set_xml_property_copy(iter, "op_node", fsa_our_uname);

	send_request(NULL, iter, "event",
	NULL, CRM_SYSTEM_TENGINE);
	}

	FNRET(I_NULL);
	#endif

	if(action & A_UPDATE_NODESTATUS) {

	xmlNodePtr data = do_lrm_query();
	set_xml_property_copy(data, "replace_lrm", "true");

	tmp1 = create_xml_node(NULL, XML_CIB_TAG_STATE);
	set_xml_property_copy(tmp1, XML_ATTR_ID, fsa_our_uname);
	fragment = create_cib_fragment(tmp1, NULL);

	set_xml_property_copy(data, "replace_lrm", "true");
	add_node_copy(tmp1, data);

	send_request(NULL, fragment, CRM_OPERATION_UPDATE,
	NULL, CRM_SYSTEM_DC);

	free_xml(fragment);
	free_xml(tmp1);
	free_xml(data);

	FNRET(next_input);
	}

	cl_log(LOG_WARNING, "Action %s (%.16llx) only kind of supported\n",
	fsa_action2string(action), action);


	msg = (xmlNodePtr)data;

	operation = get_xml_attr_nested(msg, rsc_path, DIMOF(rsc_path) -3,
	XML_ATTR_OP, TRUE);


	id_from_cib = get_xml_attr_nested(msg, rsc_path, DIMOF(rsc_path) -2,
	"id", TRUE);

	// only the first 16 chars are used by the LRM
	strncpy(rid, id_from_cib, 16);


	crm_op = get_xml_attr(msg, XML_TAG_OPTIONS, XML_ATTR_OP, TRUE);

	rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rid);

	if(crm_op != NULL && strcmp(crm_op, "lrm_query") == 0) {

	xmlNodePtr data, tmp1, tmp2, reply;

	tmp1 = create_xml_node(NULL, XML_CIB_TAG_STATE);
	set_xml_property_copy(tmp1, XML_ATTR_ID, fsa_our_uname);

	data = create_cib_fragment(tmp1, NULL);

	tmp2 = do_lrm_query();
	add_node_copy(tmp1, tmp2);

	reply = create_reply(msg, data);

	relay_message(reply, TRUE);

	free_xml(data);
	free_xml(reply);
	free_xml(tmp2);
	free_xml(tmp1);

	} else if(operation != NULL && strcmp(operation, "monitor") == 0) {
	if(rsc == NULL) {
	cl_log(LOG_ERR, "Could not find resource to monitor");
	FNRET(I_FAIL);
	}

	mon = g_new(lrm_mon_t, 1);
	mon->op_type = "status";
	mon->params = NULL;
	mon->timeout = 0;
	mon->user_data = rsc;
	mon->mode = LRM_MONITOR_SET;
	mon->interval = 2;
	mon->target = 1;
	rsc->ops->set_monitor(rsc,mon);
	mon = g_new(lrm_mon_t, 1);

	} else if(operation != NULL) {
	if(rsc == NULL) {
	// add it to the list
	CRM_DEBUG("add_rsc...");
	fsa_lrm_conn->lrm_ops->add_rsc(
	fsa_lrm_conn, rid,
	get_xml_attr_nested(msg,
	rsc_path,
	DIMOF(rsc_path) -2,
	"class", TRUE),
	get_xml_attr_nested(msg,
	rsc_path,
	DIMOF(rsc_path) -2,
	"type", TRUE),
	NULL);

	rsc = fsa_lrm_conn->lrm_ops->get_rsc(
	fsa_lrm_conn, rid);
	}

	if(rsc == NULL) {
	cl_log(LOG_ERR, "Could not add resource to LRM");
	FNRET(I_FAIL);
	}

	// now do the op
	CRM_DEBUG("performing op %s...", operation);
	op = g_new(lrm_op_t, 1);
	op->op_type = operation;
	op->params = xml2list(msg, rsc_path, DIMOF(rsc_path));
	op->timeout = 0;
	op->user_data = rsc;
	rsc->ops->perform_op(rsc, op);
	}

	FNRET(next_input);
	}

	GHashTable *
	xml2list(xmlNodePtr parent, const char**attr_path, int depth)
	{
	xmlNodePtr node_iter = NULL;

	GHashTable *nvpair_hash =
	g_hash_table_new(&g_str_hash, &g_str_equal);

	xmlNodePtr nvpair_list =
	find_xml_node_nested(parent, attr_path, depth);

	if(nvpair_list != NULL){
	node_iter = nvpair_list->children;
	while(node_iter != NULL) {

	const char *key = xmlGetProp(node_iter, "name");
	const char *value = xmlGetProp(node_iter, "value");

	CRM_DEBUG("Added %s=%s", key, value);

	g_hash_table_insert (nvpair_hash,
	cl_strdup(key),
	cl_strdup(value));

	node_iter = node_iter->next;
	}
	}

	return nvpair_hash;
	}


	void
	do_update_resource(lrm_rsc_t rsc, int status, int rc, const char op_type)
	{
	/*
	<status>
	<nodes_status id=uname>
	<lrm>
	<lrm_resources>
	<lrm_resource id=>
	</...>
	*/
	xmlNodePtr update, iter;
	char *tmp = NULL;
	xmlNodePtr fragment, tmp1;


	update = create_xml_node(NULL, "node_state");
	set_xml_property_copy(update, XML_ATTR_ID, fsa_our_uname);
	iter = create_xml_node(update, "lrm");
	iter = create_xml_node(iter, "lrm_resources");
	iter = create_xml_node(iter, "lrm_resource");

	set_xml_property_copy(iter, XML_ATTR_ID, rsc->id);
	set_xml_property_copy(iter, "last_op", op_type);

	tmp = crm_itoa(status);
	set_xml_property_copy(iter, "op_status", tmp);
	cl_free(tmp);

	tmp = crm_itoa(rc);
	set_xml_property_copy(iter, "op_code", tmp);
	cl_free(tmp);

	set_xml_property_copy(iter, "op_node", fsa_our_uname);

	tmp1 = create_xml_node(NULL, XML_CIB_TAG_STATE);
	set_xml_property_copy(tmp1, XML_ATTR_ID, fsa_our_uname);
	add_node_copy(tmp1, update);

	fragment = create_cib_fragment(tmp1, NULL);

	send_request(NULL, fragment, CRM_OPERATION_UPDATE,
	NULL, CRM_SYSTEM_DCIB);

	free_xml(fragment);
	free_xml(update);
	free_xml(tmp1);
	}

	enum crmd_fsa_input
	do_lrm_event(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input cur_input,
	void *data)
	{
	FNIN();
	if(cause == C_LRM_MONITOR_CALLBACK) {
	lrm_mon_t* monitor = (lrm_mon_t*)data;
	lrm_rsc_t* rsc = monitor->rsc;


	switch(monitor->status) {
	case LRM_OP_DONE:
	CRM_DEBUG("An LRM monitor operation passed");
	FNRET(I_NULL);
	break;

	case LRM_OP_CANCELLED:
	case LRM_OP_TIMEOUT:
	case LRM_OP_NOTSUPPORTED:
	case LRM_OP_ERROR:
	cl_log(LOG_ERR,
	"An LRM monitor operation failed"
	" or was aborted");

	do_update_resource(rsc,
	monitor->status,
	monitor->rc,
	monitor->op_type);

	break;
	}

	} else if(cause == C_LRM_OP_CALLBACK) {
	lrm_op_t* op = (lrm_op_t*)data;
	lrm_rsc_t* rsc = op->rsc;

	switch(op->status) {
	case LRM_OP_CANCELLED:
	case LRM_OP_TIMEOUT:
	case LRM_OP_NOTSUPPORTED:
	case LRM_OP_ERROR:
	cl_log(LOG_ERR,
	"An LRM operation failed"
	" or was aborted");
	// keep going
	case LRM_OP_DONE:

	do_update_resource(rsc,
	op->status,
	op->rc,
	op->op_type);

	break;
	}

	} else {

	FNRET(I_FAIL);
	}

	FNRET(I_NULL);
	}


	diff --git a/crm/pengine/pengine.c b/crm/pengine/pengine.c
	index 33d8636f1f..3cb03e4e61 100755
	--- a/crm/pengine/pengine.c
	+++ b/crm/pengine/pengine.c
	@@ -1,1888 +1,1890 @@
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xmlutils.h>
	#include <crm/common/crmutils.h>
	#include <crm/common/msgutils.h>
	#include <crm/cib.h>
	#include <glib.h>
	#include <libxml/tree.h>

	#include <pengine.h>
	#include <pe_utils.h>

	xmlNodePtr do_calculations(xmlNodePtr cib_object);

	gboolean process_pe_message(xmlNodePtr msg, IPC_Channel *sender);


	void color_resource(resource_t *lh_resource,
	GSListPtr *colors,
	GSListPtr resources);

	gboolean create_rsc_to_rsc(const char *id, enum con_strength strength,
	resource_t rsc_lh, resource_t rsc_rh);

	gboolean create_ordering(const char *id, enum con_strength strength,
	resource_t rsc_lh, resource_t rsc_rh,
	GSListPtr *action_constraints);

	gboolean unpack_constraints(xmlNodePtr xml_constraints,
	GSListPtr nodes, GSListPtr resources,
	GSListPtr *node_constraints,
	GSListPtr *action_constraints);

	gboolean unpack_resources(xmlNodePtr xml_resources,
	GSListPtr *resources,
	GSListPtr *actions,
	GSListPtr *action_cons,
	GSListPtr all_nodes);

	gboolean unpack_nodes(xmlNodePtr xml_nodes, GSListPtr *nodes);

	gboolean unpack_status(xmlNodePtr status,
	GSListPtr nodes,
	GSListPtr rsc_list,
	GSListPtr *node_constraints);

	gboolean apply_node_constraints(GSListPtr constraints,
	GSListPtr resources,
	GSListPtr nodes);

	gboolean is_active(rsc_to_node_t *cons);

	gboolean choose_node_from_list(GSListPtr colors,
	color_t *color,
	GSListPtr nodes);

	gboolean unpack_rsc_to_attr(xmlNodePtr xml_obj,
	GSListPtr rsc_list,
	GSListPtr node_list,
	GSListPtr *node_constraints);

	gboolean unpack_rsc_to_node(xmlNodePtr xml_obj,
	GSListPtr rsc_list,
	GSListPtr node_list,
	GSListPtr *node_constraints);

	gboolean unpack_rsc_to_rsc(xmlNodePtr xml_obj,
	GSListPtr rsc_list,
	GSListPtr *action_constraints);

	gboolean choose_color(resource_t *lh_resource, GSListPtr candidate_colors);

	gboolean strict_postproc(rsc_to_rsc_t *constraint,
	color_t *local_color,
	color_t *other_color,
	GSListPtr *colors,
	GSListPtr resources);

	gboolean strict_preproc(rsc_to_rsc_t *constraint,
	color_t *local_color,
	color_t *other_color,
	GSListPtr *colors,
	GSListPtr resources);

	gboolean update_node_weight(rsc_to_node_t cons, char id, GSListPtr nodes);

	gboolean process_node_lrm_state(node_t *node,
	xmlNodePtr lrm_state,
	GSListPtr rsc_list,
	GSListPtr nodes,
	GSListPtr *node_constraints);

	GSListPtr match_attrs(xmlNodePtr attr_exp, GSListPtr node_list);
	gboolean update_runnable(GSListPtr actions);
	GSListPtr create_action_set(action_t *action);

	/*
	GSListPtr rsc_list = NULL;
	GSListPtr node_list = NULL;
	GSListPtr node_cons_list = NULL;
	GSListPtr rsc_cons_list = NULL;
	GSListPtr action_list = NULL;
	GSListPtr action_set_list = NULL;
	GSListPtr action_cons_list = NULL;
	GSListPtr colors = NULL;
	GSListPtr stonith_list = NULL;
	GSListPtr shutdown_list = NULL;
	color_t *current_color = NULL;
	xmlNodePtr xml_set_of_sets = NULL;
	*/
	color_t *no_color = NULL;
	int max_valid_nodes = 0;
	int order_id = 1;
	int action_id = 1;

	gboolean pe_debug = FALSE;
	gboolean pe_debug_saved = FALSE;

	/*
	* Unpack everything
	* At the end you'll have:
	* - A list of nodes
	* - A list of resources (each with any dependancies on other resources)
	* - A list of constraints between resources and nodes
	* - A list of constraints between start/stop actions
	* - A list of nodes that need to be stonith'd
	* - A list of nodes that need to be shutdown
	* - A list of the possible stop/start actions (without dependancies)
	*/
	gboolean
	stage0(xmlNodePtr cib,
	GSListPtr *resources,
	GSListPtr nodes, GSListPtr node_constraints,
	GSListPtr actions, GSListPtr action_constraints,
	GSListPtr stonith_list, GSListPtr shutdown_list)
	{
	int lpc = 0;
	xmlNodePtr cib_nodes = get_object_root("nodes", cib);
	xmlNodePtr cib_status = get_object_root("status", cib);
	xmlNodePtr cib_resources = get_object_root("resources", cib);
	xmlNodePtr cib_constraints = get_object_root("constraints", cib);

	unpack_nodes(safe_val(NULL, cib_nodes, children), nodes);

	unpack_resources(safe_val(NULL, cib_resources, children),
	resources, actions, action_constraints, *nodes);

	unpack_status(safe_val(NULL, cib_status, children),
	nodes, resources, node_constraints);

	unpack_constraints(safe_val(NULL, cib_constraints, children),
	nodes, resources,
	node_constraints, action_constraints);

	slist_iter(
	node, node_t, *nodes, lpc,
	if(node->details->shutdown) {
	shutdown_list = g_slist_append(shutdown_list, node);
	} else if(node->details->unclean) {
	stonith_list = g_slist_append(stonith_list, node);
	}
	);

	return TRUE;
	}

	/*
	* Count how many valid nodes we have (so we know the maximum number of
	* colors we can resolve).
	*
	* Apply node constraints (ie. filter the "allowed_nodes" part of resources
	*/
	gboolean
	stage1(GSListPtr node_constraints, GSListPtr nodes, GSListPtr resources)
	{
	int lpc = 0;

	slist_iter(
	node, node_t, nodes, lpc,
	if(node == NULL) {
	// error
	} else if(node->weight >= 0.0
	&& node->details->online
	&& node->details->type == node_member) {
	max_valid_nodes++;
	}
	);

	apply_node_constraints(node_constraints, nodes, resources);

	return TRUE;
	}

	/*
	* Choose a color for all resources from highest priority and "must"
	* dependancies to lowest, creating new colors as necessary (returned
	* as "colors").
	*
	* Some nodes may be colored as a "no_color" meaning that it was unresolvable
	* given the current node stati and constraints.
	*/
	gboolean
	stage2(GSListPtr sorted_rscs, GSListPtr sorted_nodes, GSListPtr *colors)
	{
	int lpc = 0;
	color_t *current_color = NULL;

	// Set initial color
	// Set color.candidate_nodes = all active nodes
	no_color = create_color(colors, NULL, NULL);
	current_color = create_color(colors, sorted_nodes, sorted_rscs);

	// Set resource.color = color (all resources)
	// Set resource.provisional = TRUE (all resources)
	slist_iter(
	this_resource, resource_t, sorted_rscs, lpc,

	this_resource->color = current_color;
	this_resource->provisional = TRUE;
	);

	pdebug("initialized resources to default color");

	// Take (next) highest resource
	slist_iter(
	lh_resource, resource_t, sorted_rscs, lpc,
	// if resource.provisional == FALSE, repeat
	if(lh_resource->provisional == FALSE) {
	// already processed this resource
	continue;
	}
	color_resource(lh_resource, colors, sorted_rscs);
	// next resource
	);

	return TRUE;
	}

	/*
	* not sure if this is a good idea or not, but eventually we might like
	* to utilize as many nodes as possible... and this might be a convienient
	* hook
	*/
	gboolean
	stage3(GSListPtr colors)
	{
	// not sure if this is a good idea or not
	if(g_slist_length(colors) > max_valid_nodes) {
	// we need to consolidate some
	} else if(g_slist_length(colors) < max_valid_nodes) {
	// we can create a few more
	}
	return TRUE;
	}

	#define color_n_nodes color_n->details->candidate_nodes
	#define color_n_plus_1_nodes color_n_plus_1->details->candidate_nodes

	/*
	* Choose a node for each (if possible) color
	*/
	gboolean
	stage4(GSListPtr colors)
	{
	int lpc = 0;
	color_t *color_n = NULL;
	color_t *color_n_plus_1 = NULL;

	for(lpc = 0; lpc < g_slist_length(colors); lpc++) {
	color_n = color_n_plus_1;
	color_n_plus_1 = (color_t*)g_slist_nth_data(colors, lpc);

	pdebug_action(print_color("Choose node for...", color_n, FALSE));

	if(color_n == NULL) {
	continue;
	}


	GSListPtr xor = node_list_xor(color_n_nodes,
	color_n_plus_1_nodes);
	GSListPtr minus = node_list_minus(color_n_nodes,
	color_n_plus_1_nodes);

	if(g_slist_length(xor) == 0 \|\| g_slist_length(minus) == 0) {
	pdebug("Choose any node from our list");
	choose_node_from_list(colors, color_n, color_n_nodes);

	} else {
	pdebug("Choose a node not in n+1");
	choose_node_from_list(colors, color_n, minus);
	}

	}

	// choose last color
	if(color_n_plus_1 != NULL) {
	pdebug_action(print_color("Choose node for last color...",
	color_n_plus_1,
	FALSE));

	choose_node_from_list(colors,
	color_n_plus_1,
	color_n_plus_1_nodes);
	}
	pdebug("done %s", __FUNCTION__);
	return TRUE;

	}

	/*
	* Attach nodes to the actions that need to be taken
	*
	* Mark actions "optional" if possible (Ie. if the start and stop are
	* for the same node)
	*
	* Mark unrunnable actions
	*/
	gboolean
	stage5(GSListPtr resources)
	{

	pdebug("filling in the nodes to perform the actions on");
	int lpc = 0;
	slist_iter(
	rsc, resource_t, resources, lpc,

	print_resource("Processing", rsc, FALSE);

	if(safe_val(NULL, rsc, stop) == NULL
	\|\| safe_val(NULL, rsc, start) == NULL) {
	// error
	continue;
	}
	if(safe_val4(NULL, rsc, color, details, chosen_node) == NULL) {
	rsc->stop->node = safe_val(NULL, rsc, cur_node);
	rsc->start->node = NULL;

	} else if(safe_str_eq(safe_val4(NULL, rsc, cur_node, details, id),
	safe_val6(NULL, rsc, color ,details,
	chosen_node, details, id))){
	cl_log(LOG_DEBUG,
	"No change for Resource %s (%s)",
	safe_val(NULL, rsc, id),
	safe_val4(NULL, rsc, cur_node, details, id));

	rsc->stop->optional = TRUE;
	rsc->start->optional = TRUE;
	rsc->stop->node = safe_val(NULL, rsc, cur_node);
	rsc->start->node = safe_val4(NULL, rsc, color,
	details, chosen_node);

	} else if(safe_val4(NULL, rsc, cur_node, details, id) == NULL) {
	rsc->stop->optional = TRUE;
	rsc->start->node = safe_val4(NULL, rsc, color,
	details, chosen_node);

	} else {
	rsc->stop->node = safe_val(NULL, rsc, cur_node);
	rsc->start->node = safe_val4(NULL, rsc, color,
	details, chosen_node);
	}

	if(rsc->stop->node != NULL) {
	rsc->stop->runnable = TRUE;
	}
	if(rsc->start->node != NULL) {
	rsc->start->runnable = TRUE;
	}

	);

	return TRUE;
	}

	/*
	* Create dependacies for stonith and shutdown operations
	*/
	gboolean
	stage6(GSListPtr actions, GSListPtr action_constraints,
	GSListPtr stonith_nodes, GSListPtr shutdown_nodes)
	{
	int lpc = 0;
	int llpc = 0;
	slist_iter(
	node, node_t, shutdown_nodes, lpc,

	action_t *down_node =
	action_new(action_id++, NULL, shutdown_crm);
	down_node->node = node;
	down_node->runnable = TRUE;

	actions = g_slist_append(actions, down_node);

	slist_iter(
	rsc, resource_t, node->details->running_rsc, llpc,

	order_constraint_t order = (order_constraint_t)
	cl_malloc(sizeof(order_constraint_t));

	/* stop resources before shutdown */
	order->id = order_id++;
	order->lh_action = rsc->stop;
	order->rh_action = down_node;
	order->strength = must;
	*action_constraints =
	g_slist_append(*action_constraints, order);
	);
	);

	slist_iter(
	node, node_t, stonith_nodes, lpc,

	action_t *stonith_node =
	action_new(action_id++, NULL, stonith_op);
	stonith_node->node = node;
	stonith_node->runnable = TRUE;

	actions = g_slist_append(actions, stonith_node);

	slist_iter(
	rsc, resource_t, node->details->running_rsc, llpc,

	order_constraint_t order = (order_constraint_t)
	cl_malloc(sizeof(order_constraint_t));

	/* try stopping the resource before stonithing the node
	*
	* if the stop succeeds, the transitioner can then
	* decided if stonith is needed
	*/
	order->id = order_id++;
	order->lh_action = rsc->stop;
	order->rh_action = stonith_node;
	order->strength = must;
	*action_constraints =
	g_slist_append(*action_constraints, order);

	/* stonith before start */
	order = (order_constraint_t*)
	cl_malloc(sizeof(order_constraint_t));

	// try stopping the node first
	order->id = order_id++;
	order->lh_action = stonith_node;
	order->rh_action = rsc->start;
	order->strength = must;
	*action_constraints =
	g_slist_append(*action_constraints, order);
	);
	);


	return TRUE;
	}


	/*
	* Determin the sets of independant actions and the correct order for the
	* actions in each set.
	*
	* Mark dependancies f un-runnable actions un-runnable
	*
	*/
	gboolean
	stage7(GSListPtr resources, GSListPtr actions, GSListPtr action_constraints,
	GSListPtr *action_sets)
	{
	int lpc = 0;

	slist_iter(
	order, order_constraint_t, action_constraints, lpc,

	action_wrapper_t wrapper = (action_wrapper_t)
	cl_malloc(sizeof(action_wrapper_t));
	wrapper->action = order->rh_action;
	wrapper->strength = order->strength;

	order->lh_action->actions_after =
	g_slist_append(order->lh_action->actions_after,
	wrapper);

	wrapper = (action_wrapper_t*)
	cl_malloc(sizeof(action_wrapper_t));
	wrapper->action = order->lh_action;
	wrapper->strength = order->strength;

	order->rh_action->actions_before =
	g_slist_append(order->rh_action->actions_before,
	wrapper);

	);

	update_runnable(actions);

	slist_iter(
	rsc, resource_t, resources, lpc,
	GSListPtr action_set = NULL;
	if(rsc->stop->runnable) {
	action_set = create_action_set(rsc->stop);
	if(action_set != NULL) {
	action_sets = g_slist_append(action_sets,
	action_set);
	} else {
	pdebug("No actions resulting from %s->stop",
	rsc->id);
	}


	}
	if(rsc->start->runnable) {
	action_set = create_action_set(rsc->start);
	if(action_set != NULL) {
	action_sets = g_slist_append(action_sets,
	action_set);
	} else {
	pdebug("No actions resulting from %s->start",
	rsc->id);
	}
	}

	);

	return TRUE;
	}

	/*
	* Create a dependancy graph to send to the transitioner (via the CRMd)
	*/
	gboolean
	stage8(GSListPtr action_sets, xmlNodePtr *graph)
	{
	int lpc = 0;
	xmlNodePtr xml_action_set = NULL;

	*graph = create_xml_node(NULL, "transition_graph");

	/* errors...
	slist_iter(action, action_t, action_list, lpc,
	if(action->optional == FALSE && action->runnable == FALSE) {
	print_action("Ignoring", action, TRUE);
	}
	);
	*/
	int lpc2;
	slist_iter(action_set, GSList, action_sets, lpc,
	pdebug("Processing Action Set %d", lpc);
	xml_action_set = create_xml_node(NULL, "actions");
	set_xml_property_copy(xml_action_set, "id", crm_itoa(lpc));

	slist_iter(action, action_t, action_set, lpc2,
	xmlNodePtr xml_action = action2xml(action);
	xmlAddChild(xml_action_set, xml_action);
	)
	xmlAddChild(*graph, xml_action_set);
	);

	xml_message_debug(*graph, "created action list");

	return TRUE;
	}

	/*
	* Print a nice human readable high-level summary of what we're going to do
	*/
	gboolean
	summary(GSListPtr resources)
	{
	int lpc = 0;
	slist_iter(
	rsc, resource_t, resources, lpc,
	char *rsc_id = safe_val(NULL, rsc, id);
	char *node_id = safe_val4(NULL, rsc, cur_node, details, id);
	char *new_node_id = safe_val6(NULL, rsc, color, details,
	chosen_node, details, id);
	if(rsc->runnable == FALSE) {
	cl_log(LOG_ERR,
	"Resource %s was not runnable",
	rsc_id);
	if(node_id != NULL) {
	cl_log(LOG_WARNING,
	"Stopping Resource (%s) on node %s",
	rsc_id,
	node_id);
	}

	} else if(safe_val4(NULL, rsc, color, details, chosen_node) == NULL) {
	cl_log(LOG_ERR,
	"Could not allocate Resource %s",
	rsc_id);
	if(node_id != NULL) {

	cl_log(LOG_WARNING,
	"Stopping Resource (%s) on node %s",
	rsc_id,
	node_id);
	}

	} else if(safe_str_eq(node_id, new_node_id)){
	cl_log(LOG_DEBUG,
	"No change for Resource %s (%s)",
	rsc_id,
	safe_val4(NULL, rsc, cur_node, details, id));

	} else if(node_id == NULL) {
	cl_log(LOG_INFO,
	"Starting Resource %s on %s",
	rsc_id,
	new_node_id);

	} else {
	cl_log(LOG_INFO,
	"Moving Resource %s from %s to %s",
	rsc_id,
	node_id,
	new_node_id);
	}
	);


	return TRUE;
	}


	gboolean
	choose_node_from_list(GSListPtr colors, color_t *color, GSListPtr nodes)
	{
	/*
	1. Sort by weight
	2. color.chosen_node = highest wieghted node
	3. remove color.chosen_node from all other colors
	*/
	int lpc = 0;
	nodes = g_slist_sort(nodes, sort_node_weight);
	color->details->chosen_node = (node_t*)g_slist_nth_data(nodes, 0);

	if(color->details->chosen_node == NULL) {
	cl_log(LOG_ERR, "Could not allocate a node for color %d", color->id);
	return FALSE;
	}

	slist_iter(
	color_n, color_t, colors, lpc,
	node_t *other_node = pe_find_node(color_n->details->candidate_nodes,
	color->details->chosen_node->details->id);
	color_n->details->candidate_nodes =
	g_slist_remove(color_n->details->candidate_nodes,
	other_node);
	);

	return TRUE;
	}


	gboolean
	unpack_nodes(xmlNodePtr xml_nodes, GSListPtr *nodes)
	{
	pdebug("Begining unpack... %s", __FUNCTION__);
	while(xml_nodes != NULL) {
	pdebug("Processing node...");
	xmlNodePtr xml_obj = xml_nodes;
	xmlNodePtr attrs = xml_obj->children;
	const char *id = xmlGetProp(xml_obj, "id");
	const char *type = xmlGetProp(xml_obj, "type");
	if(attrs != NULL) {
	attrs = attrs->children;
	}

	xml_nodes = xml_nodes->next;

	if(id == NULL) {
	cl_log(LOG_ERR, "Must specify id tag in <node>");
	continue;
	}
	if(type == NULL) {
	cl_log(LOG_ERR, "Must specify type tag in <node>");
	continue;
	}
	node_t *new_node = cl_malloc(sizeof(node_t));
	new_node->weight = 1.0;
	new_node->fixed = FALSE;
	new_node->details = (struct node_shared_s*)
	cl_malloc(sizeof(struct node_shared_s*));
	new_node->details->online = FALSE;
	new_node->details->unclean = FALSE;
	new_node->details->shutdown = FALSE;
	new_node->details->running_rsc = NULL;
	new_node->details->id = cl_strdup(id);
	new_node->details->attrs =
	g_hash_table_new(g_str_hash, g_str_equal);
	new_node->details->type = node_ping;
	if(safe_str_eq(type, "node")) {
	new_node->details->type = node_member;
	}


	while(attrs != NULL){
	const char *name = xmlGetProp(attrs, "name");
	const char *value = xmlGetProp(attrs, "value");
	if(name != NULL && value != NULL) {
	g_hash_table_insert(new_node->details->attrs,
	cl_strdup(name),
	cl_strdup(value));
	}
	attrs = attrs->next;
	}

	- pdebug("Adding node id... %s (%p)", id, new_node);
	+ pdebug_action(print_node("Added", new_node, FALSE));

	nodes = g_slist_append(nodes, new_node);
	}

	nodes = g_slist_sort(nodes, sort_node_weight);

	return TRUE;
	}

	gboolean
	unpack_resources(xmlNodePtr xml_resources,
	GSListPtr *resources,
	GSListPtr *actions,
	GSListPtr *action_cons,
	GSListPtr all_nodes)
	{
	pdebug("Begining unpack... %s", __FUNCTION__);
	while(xml_resources != NULL) {
	xmlNodePtr xml_obj = xml_resources;
	const char *id = xmlGetProp(xml_obj, "id");
	const char *priority = xmlGetProp(xml_obj, "priority");
	float priority_f = atof(priority);

	xml_resources = xml_resources->next;

	pdebug("Processing resource...");

	if(id == NULL) {
	cl_log(LOG_ERR, "Must specify id tag in <resource>");
	continue;
	}
	resource_t *new_rsc = cl_malloc(sizeof(resource_t));
	new_rsc->xml = xml_obj; // copy first
	new_rsc->priority = priority_f;
	new_rsc->candidate_colors = NULL;
	new_rsc->color = NULL;
	new_rsc->runnable = TRUE;
	new_rsc->provisional = TRUE;
	new_rsc->allowed_nodes = node_list_dup(all_nodes);
	new_rsc->rsc_cons = NULL;
	new_rsc->node_cons = NULL;
	new_rsc->id = cl_strdup(id);
	-
	+ new_rsc->cur_node = NULL;

	action_t *action_stop = action_new(action_id++, new_rsc,
	stop_rsc);

	action_t *action_start = action_new(action_id++, new_rsc,
	start_rsc);

	new_rsc->stop = action_stop;
	actions = g_slist_append(actions, action_stop);

	new_rsc->start = action_start;
	actions = g_slist_append(actions, action_start);

	order_constraint_t order = (order_constraint_t)
	cl_malloc(sizeof(order_constraint_t));
	order->id = order_id++;
	order->lh_action = action_stop;
	order->rh_action = action_start;
	order->strength = startstop;
	action_cons = g_slist_append(action_cons, order);

	pdebug_action(print_resource("Added", new_rsc, FALSE));
	resources = g_slist_append(resources, new_rsc);
	}
	resources = g_slist_sort(resources, sort_rsc_priority);

	return TRUE;
	}



	gboolean
	unpack_constraints(xmlNodePtr xml_constraints,
	GSListPtr nodes, GSListPtr resources,
	GSListPtr *node_constraints,
	GSListPtr *action_constraints)
	{
	pdebug("Begining unpack... %s", __FUNCTION__);
	while(xml_constraints != NULL) {
	const char *id = xmlGetProp(xml_constraints, "id");
	xmlNodePtr xml_obj = xml_constraints;
	xml_constraints = xml_constraints->next;
	if(id == NULL) {
	cl_log(LOG_ERR, "Constraint must have an id");
	continue;
	}

	pdebug("Processing constraint %s %s",
	xml_obj->name,id);
	if(safe_str_eq("rsc_to_rsc", xml_obj->name)) {
	unpack_rsc_to_rsc(xml_obj, resources,
	action_constraints);

	} else if(safe_str_eq("rsc_to_node", xml_obj->name)) {
	unpack_rsc_to_node(xml_obj, resources, nodes,
	node_constraints);

	} else if(safe_str_eq("rsc_to_attr", xml_obj->name)) {
	unpack_rsc_to_attr(xml_obj, resources, nodes,
	node_constraints);

	} else {
	cl_log(LOG_ERR, "Unsupported constraint type: %s",
	xml_obj->name);
	}
	}

	return TRUE;
	}


	gboolean
	apply_node_constraints(GSListPtr constraints,
	GSListPtr resources,
	GSListPtr nodes)
	{
	pdebug("Applying constraints... %s", __FUNCTION__);
	int lpc = 0;
	slist_iter(
	cons, rsc_to_node_t, constraints, lpc,
	pdebug_action(print_rsc_to_node("Applying", cons, FALSE));
	// take "lifetime" into account
	if(cons == NULL) {
	cl_log(LOG_ERR, "Constraint (%d) is NULL", lpc);
	continue;

	} else if(is_active(cons) == FALSE) {
	cl_log(LOG_INFO, "Constraint (%d) is not active", lpc);
	// warning
	continue;
	}

	resource_t *rsc_lh = cons->rsc_lh;
	if(rsc_lh == NULL) {
	cl_log(LOG_ERR, "LHS of rsc_to_node (%s) is NULL", cons->id);
	continue;
	}

	cons->rsc_lh->node_cons =
	g_slist_append(cons->rsc_lh->node_cons, cons);

	if(cons->node_list_rh == NULL) {
	cl_log(LOG_ERR,
	"RHS of rsc_to_node (%s) is NULL",
	cons->id);
	continue;
	} else {
	int llpc = 0;
	slist_iter(node_rh, node_t, cons->node_list_rh, llpc,
	update_node_weight(cons,
	node_rh->details->id,
	nodes));
	}

	/* dont add it to the resource,
	* the information is in the resouce's node list
	*/
	);

	return TRUE;

	}


	// remove nodes that are down, stopping
	// create +ve rsc_to_node constraints between resources and the nodes they are running on
	// anything else?
	gboolean
	unpack_status(xmlNodePtr status,
	GSListPtr nodes,
	GSListPtr rsc_list,
	GSListPtr *node_constraints)
	{
	pdebug("Begining unpack %s", __FUNCTION__);
	while(status != NULL) {
	const char *id = xmlGetProp(status, "id");
	const char *state = xmlGetProp(status, "state");
	const char *exp_state = xmlGetProp(status, "exp_state");
	xmlNodePtr lrm_state = find_xml_node(status, "lrm");
	xmlNodePtr attrs = find_xml_node(status, "attributes");

	lrm_state = find_xml_node(lrm_state, "lrm_resources");
	lrm_state = find_xml_node(lrm_state, "rsc_state");
	status = status->next;

	pdebug("Processing node %s", id);

	if(id == NULL){
	// error
	continue;
	}
	pdebug("Processing node attrs");

	node_t *this_node = pe_find_node(nodes, id);
	while(attrs != NULL){
	const char *name = xmlGetProp(attrs, "name");
	const char *value = xmlGetProp(attrs, "value");

	if(name != NULL && value != NULL
	&& safe_val(NULL, this_node, details) != NULL) {
	pdebug("Adding %s => %s",
	name, value);
	g_hash_table_insert(this_node->details->attrs,
	cl_strdup(name),
	cl_strdup(value));
	}
	attrs = attrs->next;
	}

	pdebug("determining node state");

	if(safe_str_eq(exp_state, "active")
	&& safe_str_eq(state, "active")) {
	// process resource, make +ve preference
	this_node->details->online = TRUE;

	} else {
	pdebug("remove %s", __FUNCTION__);
	// remove node from contention
	this_node->weight = -1;
	this_node->fixed = TRUE;

	pdebug("state %s, expected %s",
	state, exp_state);

	if(safe_str_eq(state, "shutdown")){
	// create shutdown req
	this_node->details->shutdown = TRUE;

	} else if(safe_str_eq(exp_state, "active")
	&& safe_str_neq(state, "active")) {
	// mark unclean in the xml
	this_node->details->unclean = TRUE;

	// remove any running resources from being allocated
	}
	}

	pdebug("Processing node lrm state");
	process_node_lrm_state(this_node, lrm_state,
	rsc_list, nodes,
	node_constraints);
	}

	return TRUE;

	}

	gboolean
	is_active(rsc_to_node_t *cons)
	{
	return TRUE;
	}



	gboolean
	strict_preproc(rsc_to_rsc_t *constraint,
	color_t *local_color,
	color_t *other_color,
	GSListPtr *colors,
	GSListPtr resources)
	{
	resource_t * lh_resource = constraint->rsc_lh;
	switch(constraint->strength) {
	case must:
	if(constraint->rsc_rh->runnable == FALSE) {
	cl_log(LOG_WARNING,
	"Resource %s must run on the same node"
	" as %s (cons %s), but %s is not"
	" runnable.",
	constraint->rsc_lh->id,
	constraint->rsc_rh->id,
	constraint->id,
	constraint->rsc_rh->id);
	constraint->rsc_lh->runnable = FALSE;
	}
	break;

	// x * should * should_not = x
	case should:
	if(constraint->rsc_rh->provisional == FALSE) {
	local_color->local_weight =
	local_color->local_weight * 2.0;
	}
	break;
	case should_not:
	if(constraint->rsc_rh->provisional == FALSE) {
	local_color->local_weight =
	local_color->local_weight * 0.5;
	}
	pdebug("# Colors %d, Nodes %d",
	g_slist_length(*colors),
	max_valid_nodes);

	if(g_slist_length(*colors) < max_valid_nodes
	// && g_slist_length(lh_resource->candidate_colors)==1
	) {
	create_color(colors,
	lh_resource->allowed_nodes,
	resources);
	}


	break;
	case must_not:
	if(constraint->rsc_rh->provisional == FALSE) {
	lh_resource->candidate_colors =
	g_slist_remove(
	lh_resource->candidate_colors,
	local_color);
	}
	break;
	default:
	// error
	break;
	}
	return TRUE;
	}

	gboolean
	strict_postproc(rsc_to_rsc_t *constraint,
	color_t *local_color,
	color_t *other_color,
	GSListPtr *colors,
	GSListPtr resources)
	{
	print_rsc_to_rsc("Post processing", constraint, FALSE);

	switch(constraint->strength) {
	case must:
	if(constraint->rsc_rh->provisional == TRUE) {
	constraint->rsc_rh->color = other_color;
	constraint->rsc_rh->provisional = FALSE;
	color_resource(constraint->rsc_rh,
	colors, resources);
	}
	// else check for error
	if(constraint->rsc_lh->runnable == FALSE) {
	cl_log(LOG_WARNING,
	"Resource %s must run on the same node"
	" as %s (cons %s), but %s is not"
	" runnable.",
	constraint->rsc_rh->id,
	constraint->rsc_lh->id,
	constraint->id,
	constraint->rsc_lh->id);
	constraint->rsc_rh->runnable = FALSE;

	}

	break;

	case should:
	break;
	case should_not:
	break;
	case must_not:
	if(constraint->rsc_rh->provisional == TRUE) {
	// check for error
	}
	break;
	default:
	// error
	break;
	}
	return TRUE;
	}

	gboolean
	choose_color(resource_t *lh_resource, GSListPtr candidate_colors)
	{
	int lpc = 0;

	if(lh_resource->runnable == FALSE) {
	lh_resource->color = no_color;
	lh_resource->provisional = FALSE;
	} else {
	GSListPtr sorted_colors = g_slist_sort(candidate_colors,
	sort_color_weight);

	lh_resource->candidate_colors = sorted_colors;

	pdebug( "Choose a color from %d possibilities",
	g_slist_length(sorted_colors));
	}

	if(lh_resource->provisional) {
	slist_iter(
	this_color, color_t,lh_resource->candidate_colors, lpc,
	GSListPtr intersection = node_list_and(
	this_color->details->candidate_nodes,
	lh_resource->allowed_nodes);

	if(g_slist_length(intersection) != 0) {
	// TODO: merge node weights
	g_slist_free(this_color->details->candidate_nodes);
	this_color->details->candidate_nodes = intersection;
	lh_resource->color = this_color;
	lh_resource->provisional = FALSE;
	break;
	}
	);
	}
	return !lh_resource->provisional;
	}

	gboolean
	unpack_rsc_to_node(xmlNodePtr xml_obj,
	GSListPtr rsc_list,
	GSListPtr node_list,
	GSListPtr *node_constraints)
	{

	xmlNodePtr node_ref = xml_obj->children;
	rsc_to_node_t *new_con = cl_malloc(sizeof(rsc_to_node_t));
	const char *id_lh = xmlGetProp(xml_obj, "from");
	const char *id = xmlGetProp(xml_obj, "id");

	const char *mod = xmlGetProp(xml_obj, "modifier");
	const char *weight = xmlGetProp(xml_obj, "weight");
	float weight_f = atof(weight);

	resource_t *rsc_lh = pe_find_resource(rsc_list, id_lh);
	if(rsc_lh == NULL) {
	cl_log(LOG_ERR, "No resource (con=%s, rsc=%s)",
	id, id_lh);
	}

	new_con->id = cl_strdup(id);
	new_con->rsc_lh = rsc_lh;
	new_con->weight = weight_f;

	if(safe_str_eq(mod, "set")){
	new_con->modifier = set;
	} else if(safe_str_eq(mod, "inc")){
	new_con->modifier = inc;
	} else if(safe_str_eq(mod, "dec")){
	new_con->modifier = dec;
	} else {
	// error
	}
	/*
	<rsc_to_node>
	<node_ref id= type= name=/>
	<node_ref id= type= name=/>
	<node_ref id= type= name=/>
	*/
	//

	while(node_ref != NULL) {
	const char *id_rh = xmlGetProp(node_ref, "name");
	node_t *node_rh = pe_find_node(node_list, id_rh);
	if(node_rh == NULL) {
	// error
	cl_log(LOG_ERR,
	"node %s (from %s) not found",
	id_rh, node_ref->name);
	continue;
	}

	new_con->node_list_rh =
	g_slist_append(new_con->node_list_rh,
	node_rh);


	/* dont add it to the resource,
	* the information is in the resouce's node list
	*/
	node_ref = node_ref->next;
	}
	node_constraints = g_slist_append(node_constraints, new_con);

	return TRUE;
	}


	gboolean
	unpack_rsc_to_attr(xmlNodePtr xml_obj,
	GSListPtr rsc_list,
	GSListPtr node_list,
	GSListPtr *node_constraints)
	{
	/*
	<rsc_to_attr id="cons4" from="rsc2" weight="20.0" modifier="inc">
	<attr_expression id="attr_exp_1"/>
	<node_match id="node_match_1" type="has_attr" target="cpu"/>
	<node_match id="node_match_2" type="attr_value" target="kernel" value="2.6"/>
	</attr_expression>
	<attr_expression id="attr_exp_2"/>
	<node_match id="node_match_3" type="has_attr" target="hdd"/>
	<node_match id="node_match_4" type="attr_value" target="kernel" value="2.4"/>
	</attr_expression>

	Translation:
	give any node a +ve weight of 20.0 to run rsc2 if:
	attr "cpu" is set _and_ "kernel"="2.6", _or_
	attr "hdd" is set _and_ "kernel"="2.4"

	Further translation:
	2 constraints that give any node a +ve weight of 20.0 to run rsc2
	cons1: attr "cpu" is set and "kernel"="2.6"
	cons2: attr "hdd" is set and "kernel"="2.4"

	*/

	xmlNodePtr attr_exp = xml_obj->children;
	const char *id_lh = xmlGetProp(xml_obj, "from");
	const char *mod = xmlGetProp(xml_obj, "modifier");
	const char *weight = xmlGetProp(xml_obj, "weight");
	const char *id = xmlGetProp(attr_exp, "id");
	float weight_f = atof(weight);
	enum con_modifier a_modifier = modifier_none;

	resource_t *rsc_lh = pe_find_resource(rsc_list, id_lh);
	if(rsc_lh == NULL) {
	cl_log(LOG_ERR, "No resource (con=%s, rsc=%s)",
	id, id_lh);
	return FALSE;
	}

	if(safe_str_eq(mod, "set")){
	a_modifier = set;
	} else if(safe_str_eq(mod, "inc")){
	a_modifier = inc;
	} else if(safe_str_eq(mod, "dec")){
	a_modifier = dec;
	} else {
	// error
	}

	if(attr_exp == NULL) {
	cl_log(LOG_WARNING, "no attrs for constraint %s", id);
	}

	while(attr_exp != NULL) {
	const char *id_rh = xmlGetProp(attr_exp, "name");
	const char *id = xmlGetProp(attr_exp, "id");
	rsc_to_node_t *new_con = cl_malloc(sizeof(rsc_to_node_t));
	new_con->id = cl_strdup(id);
	new_con->rsc_lh = rsc_lh;
	new_con->weight = weight_f;
	new_con->modifier = a_modifier;

	new_con->node_list_rh = match_attrs(attr_exp, node_list);

	if(new_con->node_list_rh == NULL) {
	// error
	cl_log(LOG_ERR,
	"node %s (from %s) not found",
	id_rh, attr_exp->name);
	}
	pdebug_action(print_rsc_to_node("Added", new_con, FALSE));
	node_constraints = g_slist_append(node_constraints, new_con);

	/* dont add it to the resource,
	* the information is in the resouce's node list
	*/
	attr_exp = attr_exp->next;
	}
	return TRUE;
	}

	gboolean
	update_node_weight(rsc_to_node_t cons, char id, GSListPtr nodes)
	{
	node_t *node_rh = pe_find_node(cons->rsc_lh->allowed_nodes, id);

	if(node_rh == NULL) {
	node_t *node_tmp = pe_find_node(nodes, id);
	node_rh = node_copy(node_tmp);
	cons->rsc_lh->allowed_nodes =
	g_slist_append(cons->rsc_lh->allowed_nodes,
	node_rh);
	}

	if(node_rh == NULL) {
	// error
	return FALSE;
	}

	if(node_rh->fixed) {
	// warning
	cl_log(LOG_WARNING,
	"Constraint %s is irrelevant as the"
	" weight of node %s is fixed as %f.",
	cons->id,
	node_rh->details->id,
	node_rh->weight);
	return TRUE;
	}

	pdebug( "Constraint %s: node %s weight %s %f.",
	cons->id,
	node_rh->details->id,
	modifier2text(cons->modifier),
	node_rh->weight);

	switch(cons->modifier) {
	case set:
	node_rh->weight = cons->weight;
	node_rh->fixed = TRUE;
	break;
	case inc:
	node_rh->weight += cons->weight;
	break;
	case dec:
	node_rh->weight -= cons->weight;
	break;
	case modifier_none:
	// warning
	break;
	}
	return TRUE;
	}

	gboolean
	process_node_lrm_state(node_t *node, xmlNodePtr lrm_state,
	GSListPtr rsc_list, GSListPtr nodes,
	GSListPtr *node_constraints)
	{
	pdebug("here %s", __FUNCTION__);

	while(lrm_state != NULL) {
	const char *id = xmlGetProp(lrm_state, "id");
	const char *rsc_id = xmlGetProp(lrm_state, "rsc_id");
	const char *node_id = xmlGetProp(lrm_state, "node_id");
	const char *rsc_state = xmlGetProp(lrm_state, "rsc_state");
	resource_t *rsc_lh = pe_find_resource(rsc_list, rsc_id);
	rsc_lh->cur_node = node;

	node->details->running_rsc =
	g_slist_append(node->details->running_rsc, rsc_lh);

	/* it is runnable, but depends on a stonith op
	if(safe_val3(FALSE, node, details, unclean)) {
	rsc_lh->runnable = FALSE;
	}
	*/

	if((safe_str_eq(rsc_state, "starting"))
	\|\| (safe_str_eq(rsc_state, "started"))) {

	node_t *node_rh;
	rsc_to_node_t *new_cons =
	cl_malloc(sizeof(rsc_to_node_t));
	new_cons->id = cl_strdup(id); // genereate one
	new_cons->weight = 100.0;
	new_cons->modifier = inc;

	new_cons->rsc_lh = rsc_lh;
	node_rh = pe_find_node(nodes, node_id);

	new_cons->node_list_rh = g_slist_append(NULL, node_rh);

	*node_constraints =
	g_slist_append(*node_constraints, new_cons);

	pdebug_action(print_rsc_to_node(
	"Added", new_cons, FALSE));

	} else if(safe_str_eq(rsc_state, "stop_fail")) {
	// do soemthing
	} // else no preference

	lrm_state = lrm_state->next;
	}
	return TRUE;
	}

	GSListPtr
	match_attrs(xmlNodePtr attr_exp, GSListPtr node_list)
	{
	int lpc = 0;
	GSListPtr result = NULL;
	slist_iter(
	node, node_t, node_list, lpc,
	xmlNodePtr node_match = attr_exp->children;
	gboolean accept = TRUE;

	while(accept && node_match != NULL) {
	const char *type =xmlGetProp(node_match, "type");
	const char *value=xmlGetProp(node_match, "value");
	const char *name =xmlGetProp(node_match, "target");
	node_match = node_match->next;

	if(name == NULL \|\| type == NULL) {
	// error
	continue;
	}

	const char h_val = (const char)
	g_hash_table_lookup(node->details->attrs, name);

	if(h_val != NULL && safe_str_eq(type, "has_attr")){
	accept = TRUE;
	} else if(h_val == NULL
	&& safe_str_eq(type, "not_attr")) {
	accept = TRUE;
	} else if(h_val != NULL
	&& safe_str_eq(type, "attr_value")
	&& safe_str_eq(h_val, value)) {
	accept = TRUE;
	} else {
	accept = FALSE;
	}
	}

	if(accept) {
	result = g_slist_append(result, node);

	}
	);

	return result;
	}

	gboolean
	create_rsc_to_rsc(const char *id, enum con_strength strength,
	resource_t rsc_lh, resource_t rsc_rh)
	{
	if(rsc_lh == NULL \|\| rsc_rh == NULL){
	// error
	return FALSE;
	}

	rsc_to_rsc_t *new_con = cl_malloc(sizeof(rsc_to_node_t));
	rsc_to_rsc_t *inverted_con = NULL;

	new_con->id = cl_strdup(id);
	new_con->rsc_lh = rsc_lh;
	new_con->rsc_rh = rsc_rh;
	new_con->strength = strength;

	inverted_con = invert_constraint(new_con);

	rsc_lh->rsc_cons = g_slist_insert_sorted(rsc_lh->rsc_cons,
	inverted_con, sort_cons_strength);
	rsc_rh->rsc_cons = g_slist_insert_sorted(rsc_rh->rsc_cons,
	new_con, sort_cons_strength);

	return TRUE;
	}

	gboolean
	create_ordering(const char *id, enum con_strength strength,
	resource_t rsc_lh, resource_t rsc_rh,
	GSListPtr *action_constraints)
	{
	if(rsc_lh == NULL \|\| rsc_rh == NULL){
	// error
	return FALSE;
	}

	action_t *lh_stop = rsc_lh->stop;
	action_t *lh_start = rsc_lh->start;
	action_t *rh_stop = rsc_rh->stop;
	action_t *rh_start = rsc_rh->start;

	order_constraint_t order = (order_constraint_t)
	cl_malloc(sizeof(order_constraint_t));
	order->id = order_id++;
	order->lh_action = lh_stop;
	order->rh_action = rh_stop;
	order->strength = strength;
	action_constraints = g_slist_append(action_constraints, order);

	order = (order_constraint_t*)
	cl_malloc(sizeof(order_constraint_t));
	order->id = order_id++;
	order->lh_action = rh_start;
	order->rh_action = lh_start;
	order->strength = strength;
	action_constraints = g_slist_append(action_constraints, order);

	return TRUE;
	}

	gboolean
	unpack_rsc_to_rsc(xmlNodePtr xml_obj,
	GSListPtr rsc_list,
	GSListPtr *action_constraints)
	{
	const char *id_lh = xmlGetProp(xml_obj, "from");
	const char *id = xmlGetProp(xml_obj, "id");
	resource_t *rsc_lh = pe_find_resource(rsc_list, id_lh);
	const char *id_rh = xmlGetProp(xml_obj, "to");
	resource_t *rsc_rh = pe_find_resource(rsc_list, id_rh);
	const char *strength = xmlGetProp(xml_obj, "strength");
	const char *type = xmlGetProp(xml_obj, "type");
	enum con_strength strength_e = ignore;

	if(rsc_lh == NULL) {
	cl_log(LOG_ERR, "No resource (con=%s, rsc=%s)",
	id, id_lh);
	return FALSE;
	}
	if(safe_str_eq(strength, "must")) {
	strength_e = must;

	} else if(safe_str_eq(strength, "should")) {
	strength_e = should;

	} else if(safe_str_eq(strength, "should_not")) {
	strength_e = should_not;

	} else if(safe_str_eq(strength, "must_not")) {
	strength_e = must_not;
	} else {
	// error
	}

	if(safe_str_eq(type, "ordering")) {
	// make an action_cons instead
	return create_ordering(id, strength_e, rsc_lh, rsc_rh,
	action_constraints);
	}

	return create_rsc_to_rsc(id, strength_e, rsc_lh, rsc_rh);
	}


	GSListPtr
	create_action_set(action_t *action)
	{
	int lpc = 0;
	GSListPtr result = NULL;
	GSListPtr tmp = NULL;

	if(action->processed) {
	return NULL;
	}

	pdebug_action(print_action("Create action set for", action, FALSE));

	// process actions_before
	if(action->seen_count == 0) {
	pdebug("Processing \"before\" for action %d", action->id);
	slist_iter(
	other, action_wrapper_t, action->actions_before, lpc,
	tmp = create_action_set(other->action);
	pdebug("%d (%d total) \"before\" actions for %d)",
	g_slist_length(tmp), g_slist_length(result),action->id);
	result = g_slist_concat(result, tmp);
	);

	// add ourselves
	pdebug("Adding self %d", action->id);
	if(action->processed == FALSE) {
	result = g_slist_append(result, action);
	action->processed = TRUE;
	}

	} else {
	pdebug("Already seen action %d", action->id);
	pdebug("Processing \"before\" for action %d", action->id);
	slist_iter(
	other, action_wrapper_t, action->actions_before, lpc,

	if(other->action->seen_count > action->seen_count
	&& other->strength == must) {
	tmp = create_action_set(other->action);
	pdebug("%d (%d total) \"before\" actions for %d)",
	g_slist_length(tmp), g_slist_length(result),action->id);
	result = g_slist_concat(result, tmp);
	}

	);

	// add ourselves
	pdebug("Adding self %d", action->id);
	if(action->processed == FALSE) {
	result = g_slist_append(result, action);
	action->processed = TRUE;
	}

	// add strength == !MUST
	slist_iter(
	other, action_wrapper_t, action->actions_before, lpc,

	tmp = create_action_set(other->action);
	pdebug("%d (%d total) post-self \"before\" actions for %d)",
	g_slist_length(tmp), g_slist_length(result),action->id);
	result = g_slist_concat(result, tmp);
	);
	}

	action->seen_count = action->seen_count + 1;

	// process actions_after
	pdebug("Processing \"after\" for action %d", action->id);
	slist_iter(
	other, action_wrapper_t, action->actions_after, lpc,
	tmp = create_action_set(other->action);
	pdebug("%d (%d total) \"after\" actions for %d)",
	g_slist_length(tmp), g_slist_length(result),action->id);
	result = g_slist_concat(result, tmp);
	);

	return result;
	}


	gboolean
	update_runnable(GSListPtr actions)
	{

	int lpc = 0, lpc2 = 0;
	gboolean change = TRUE;

	while(change) {
	change = FALSE;
	slist_iter(
	action, action_t, actions, lpc,

	if(action->runnable) {
	continue;
	} else if(action->optional) {
	continue;
	}

	slist_iter(
	other, action_wrapper_t, action->actions_before, lpc2,
	if(other->action->runnable) {
	change = TRUE;
	}
	other->action->runnable = FALSE;
	);
	);
	}
	return TRUE;
	}

	void
	color_resource(resource_t lh_resource, GSListPtr colors, GSListPtr resources)
	{
	int lpc = 0;

	pdebug_action(print_resource("Coloring", lh_resource, FALSE));

	if(lh_resource->provisional == FALSE) {
	// already processed this resource
	return;
	}

	lh_resource->rsc_cons = g_slist_sort(lh_resource->rsc_cons,
	sort_cons_strength);

	pdebug("=== Pre-processing");
	//------ Pre-processing
	slist_iter(
	constraint, rsc_to_rsc_t, lh_resource->rsc_cons, lpc,
	color_t *other_color = NULL;
	color_t *local_color = NULL;
	if(lh_resource->runnable == FALSE) {
	break;
	}
	pdebug_action(print_rsc_to_rsc(
	"Processing constraint",
	constraint, FALSE));

	if(constraint->rsc_rh == NULL) {
	cl_log(LOG_ERR,
	"rsc_rh was NULL for %s",
	constraint->id);
	continue;
	}
	other_color = constraint->rsc_rh->color;
	local_color = find_color(lh_resource->candidate_colors,
	other_color);
	strict_preproc(constraint, local_color, other_color,
	colors, resources);
	);


	// filter out nodes with a negative weight
	filter_nodes(lh_resource);

	/* Choose a color from the candidates or,
	* create a new one if no color is suitable
	* (this may need modification pending further napkin drawings)
	*/
	choose_color(lh_resource, lh_resource->candidate_colors);

	pdebug("* Colors %d, Nodes %d",
	g_slist_length(*colors),
	max_valid_nodes);

	if(lh_resource->provisional
	&& g_slist_length(*colors) < max_valid_nodes) {
	// Create new color
	pdebug("Create a new color");
	lh_resource->color = create_color(colors,
	lh_resource->allowed_nodes,
	resources);
	lh_resource->provisional = FALSE;

	} else if(lh_resource->provisional) {
	cl_log(LOG_ERR, "Could not color resource %s", lh_resource->id);
	print_resource("ERROR: No color", lh_resource, FALSE);
	lh_resource->color = no_color;
	lh_resource->provisional = FALSE;

	}

	pdebug_action(print_resource("Post-processing", lh_resource, FALSE));

	//------ Post-processing

	color_t *local_color = lh_resource->color;
	slist_iter(
	constraint, rsc_to_rsc_t, lh_resource->rsc_cons, lpc,
	color_t *other_color =
	find_color(constraint->rsc_rh->candidate_colors,
	local_color);

	strict_postproc(constraint, local_color, other_color,
	colors, resources);
	);

	pdebug_action(print_resource("Colored", lh_resource, FALSE));
	}

	gboolean
	process_pe_message(xmlNodePtr msg, IPC_Channel *sender)
	{
	const char *op = get_xml_attr (msg, XML_TAG_OPTIONS,
	XML_ATTR_OP, TRUE);

	const char *ref = xmlGetProp(msg, XML_ATTR_REFERENCE);

	CRM_DEBUG("Processing %s op (ref=%s)...", op, ref);

	const char *sys_to = xmlGetProp(msg, XML_ATTR_SYSTO);

	if(op == NULL){
	// error
	} else if(sys_to == NULL \|\| strcmp(sys_to, "pengine") != 0) {
	CRM_DEBUG("Bad sys-to %s", sys_to);
	return FALSE;

	} else if(strcmp(op, "pecalc") == 0) {
	xmlNodePtr input_cib = find_xml_node(msg, XML_TAG_CIB);
	if (send_ipc_reply(sender, msg,
	do_calculations(input_cib)) ==FALSE) {

	cl_log(LOG_WARNING,
	"Answer could not be sent");
	}

	} else if(strcmp(op, "quit") == 0) {
	cl_log(LOG_WARNING, "Received quit message, terminating");
	exit(0);
	}

	return TRUE;
	}

	xmlNodePtr
	do_calculations(xmlNodePtr cib_object)
	{
	int lpc, lpc2;

	GSListPtr resources = NULL;
	GSListPtr nodes = NULL;
	GSListPtr node_constraints = NULL;
	GSListPtr actions = NULL;
	GSListPtr action_constraints = NULL;
	GSListPtr stonith_list = NULL;
	GSListPtr shutdown_list = NULL;

	GSListPtr colors = NULL;
	GSListPtr action_sets = NULL;

	xmlNodePtr graph = NULL;

	+// pe_debug_on();
	+
	pdebug("=#=#=#=#= Stage 0 =#=#=#=#=");

	stage0(cib_object,
	&resources,
	&nodes, &node_constraints,
	&actions, &action_constraints,
	&stonith_list, &shutdown_list);

	pdebug("=#=#=#=#= Stage 1 =#=#=#=#=");
	stage1(node_constraints, nodes, resources);

	pdebug("=#=#=#=#= Stage 2 =#=#=#=#=");
	stage2(resources, nodes, &colors);

	pdebug("========= Nodes =========");
	pdebug_action(
	slist_iter(node, node_t, nodes, lpc,
	print_node(NULL, node, TRUE)
	)
	);

	pdebug("========= Resources =========");
	pdebug_action(
	slist_iter(resource, resource_t, resources, lpc,
	print_resource(NULL, resource, TRUE)
	)
	);

	pdebug("=#=#=#=#= Stage 3 =#=#=#=#=");
	stage3(colors);

	pdebug("=#=#=#=#= Stage 4 =#=#=#=#=");
	stage4(colors);
	pdebug("========= Colors =========");
	pdebug_action(
	slist_iter(color, color_t, colors, lpc,
	print_color(NULL, color, FALSE)
	)
	);

	pdebug("=#=#=#=#= Stage 5 =#=#=#=#=");
	stage5(resources);

	pdebug("=#=#=#=#= Stage 6 =#=#=#=#=");
	stage6(&actions, &action_constraints,
	stonith_list, shutdown_list);

	pdebug("========= Action List =========");
	pdebug_action(
	slist_iter(action, action_t, actions, lpc,
	print_action(NULL, action, TRUE)
	)
	);

	pdebug("=#=#=#=#= Stage 7 =#=#=#=#=");
	stage7(resources, actions, action_constraints, &action_sets);

	pdebug("=#=#=#=#= Summary =#=#=#=#=");
	summary(resources);

	pdebug("========= Action Sets =========");

	pdebug("\t========= Set %d (Un-runnable) =========", -1);
	pdebug_action(
	slist_iter(action, action_t, actions, lpc,
	if(action->optional == FALSE
	&& action->runnable == FALSE) {
	print_action("\t", action, TRUE);
	}
	)
	);

	pdebug_action(
	slist_iter(action_set, GSList, action_sets, lpc,
	pdebug("\t========= Set %d =========", lpc);
	slist_iter(action, action_t, action_set, lpc2,
	print_action("\t", action, TRUE);
	)
	)
	);


	pdebug("========= Stonith List =========");
	pdebug_action(
	slist_iter(node, node_t, stonith_list, lpc,
	print_node(NULL, node, FALSE);
	)
	);

	pdebug("========= Shutdown List =========");
	pdebug_action(
	slist_iter(node, node_t, shutdown_list, lpc,
	print_node(NULL, node, FALSE);
	)
	);

	pdebug("=#=#=#=#= Stage 8 =#=#=#=#=");
	stage8(action_sets, &graph);

	return graph;
	}
	diff --git a/crm/tengine/tengine.c b/crm/tengine/tengine.c
	index 7cb6c94360..cbf5522336 100644
	--- a/crm/tengine/tengine.c
	+++ b/crm/tengine/tengine.c
	@@ -1,488 +1,543 @@
	#include <crm/crm.h>
	#include <crm/msg_xml.h>
	#include <crm/common/msgutils.h>
	#include <crm/common/xmlutils.h>
	#include <crm/cib.h>
	#include <tengine.h>

	GSListPtr graph = NULL;
	IPC_Channel *crm_ch = NULL;

	typedef struct action_list_s
	{
	int index;
	int index_max;
	GSListPtr actions;
	} action_list_t;

	gboolean initialize_graph(void);
	gboolean unpack_graph(xmlNodePtr xml_graph);
	gboolean extract_event(xmlNodePtr msg);
	gboolean initiate_transition(void);
	-gboolean initiate_action(xmlNodePtr xml_action);
	+gboolean initiate_action(action_list_t *list);
	gboolean process_graph_event(const char *event_node,
	const char *event_rsc,
	const char *event_action,
	const char *event_status,
	const char *event_rc);

	void send_success(void);
	void send_abort(void);
	gboolean process_fake_event(xmlNodePtr msg);


	gboolean
	initialize_graph(void)
	{
	while(g_slist_length(graph) > 0) {
	action_list_t *action_list = g_slist_nth_data(graph, 0);
	while(g_slist_length(action_list->actions) > 0) {
	- GSListPtr action = g_slist_nth(action_list->actions, 0);
	- g_slist_remove(action_list->actions, action);
	- cl_free(action->data);
	+ xmlNodePtr action =
	+ g_slist_nth_data(action_list->actions, 0);
	+ action_list->actions =
	+ g_slist_remove(action_list->actions, action);
	+ free_xml(action);
	}
	- g_slist_remove(graph, action_list);
	+ graph = g_slist_remove(graph, action_list);
	+ cl_free(action_list);
	}

	graph = NULL;

	return TRUE;
	}


	gboolean
	unpack_graph(xmlNodePtr xml_graph)
	{
	/*
	<transition_graph>
	<actions id="0">
	<rsc_op id="5" runnable="false" optional="true" task="stop">
	<resource id="rsc3" priority="3.0"/>
	</rsc_op>
	*/
	xmlNodePtr xml_action_list = xml_graph?xml_graph->children:NULL;
	if(xml_action_list == NULL) {
	// nothing to do
	return FALSE;
	}

	while(xml_action_list != NULL) {
	xmlNodePtr xml_obj = xml_action_list;
	xmlNodePtr xml_action = xml_obj->children;
	action_list_t action_list = (action_list_t)
	cl_malloc(sizeof(action_list_t));

	xml_action_list = xml_action_list->next;

	- action_list->index = 0;
	+ action_list->index = -1;
	action_list->index_max = 0;
	+ action_list->actions = NULL;

	while(xml_action != NULL) {
	- xmlNodePtr action = copy_xml_node_recursive(xml_action);
	+ xmlNodePtr action =
	+ copy_xml_node_recursive(xml_action);

	action_list->actions =
	g_slist_append(action_list->actions, action);
	-
	+
	action_list->index_max++;
	+ xml_action = xml_action->next;
	}

	graph = g_slist_append(graph, action_list);
	}


	return TRUE;
	}

	gboolean
	process_fake_event(xmlNodePtr msg)
	{
	+
	xmlNodePtr data = find_xml_node(msg, "lrm_resource");
	+ CRM_DEBUG("Processing fake LRM event...");
	+ const char *last_op = "start";
	+ if(safe_str_eq("stopped", xmlGetProp(data, "last_op"))) {
	+ last_op = "stop";
	+ }
	+ CRM_DEBUG("Fake LRM event... %s, %s, %s, %s, %s",
	+ xmlGetProp(data, "op_node"),
	+ xmlGetProp(data, "id"),
	+ last_op,
	+ xmlGetProp(data, "op_status"),
	+ xmlGetProp(data, "op_code"));
	+
	return process_graph_event(xmlGetProp(data, "op_node"),
	xmlGetProp(data, "id"),
	xmlGetProp(data, "last_op"),
	xmlGetProp(data, "op_status"),
	xmlGetProp(data, "op_code"));
	}

	gboolean
	extract_event(xmlNodePtr msg)
	{
	gboolean abort = FALSE;
	xmlNodePtr iter = NULL;
	const char *section = NULL;

	const char *event_action = NULL;
	const char *event_node = NULL;
	const char *event_rsc = NULL;
	const char *event_status = NULL;
	const char *event_rc = NULL;

	/*
	<status>
	<node_state id="node1" state="active" exp_state="active">
	<lrm>
	<lrm_resources>
	<rsc_state id="" rsc_id="rsc4" node_id="node1" rsc_state="stopped"/>
	*/

	xml_message_debug(msg, "TE Event");

	iter = find_xml_node(msg, XML_TAG_FRAGMENT);
	section = xmlGetProp(iter, "section");

	if(safe_str_neq(section, XML_CIB_TAG_STATUS)) {
	// these too are never expected
	send_abort();
	return FALSE;

	}

	iter = find_xml_node(msg, XML_TAG_CIB);
	iter = get_object_root(XML_CIB_TAG_STATUS, iter);
	iter = iter->children;

	while(abort == FALSE && iter != NULL) {
	xmlNodePtr node_state = iter;
	xmlNodePtr child = iter->children;
	const char *state = xmlGetProp(node_state, "state");
	iter = iter->next;

	if(state != NULL && child == NULL) {
	/* node state update,
	* possibly from a shutdown we requested
	*/
	event_status = state;
	event_node = xmlGetProp(node_state, XML_ATTR_ID);
	if(safe_str_eq(event_status, "down")) {
	event_action = "shutdown";
	}

	abort = !process_graph_event(event_node,
	event_rsc,
	event_action,
	event_status,
	event_rc);

	} else if(state != NULL && child != NULL) {
	/* this is a complex eventand could not be completely
	* due to any request we made
	*/
	send_abort();
	abort = TRUE;

	} else {
	child = find_xml_node(node_state, "lrm");
	child = find_xml_node(child, "lrm_resources");
	child = child->children;

	while(abort == FALSE && child != NULL) {
	event_action = xmlGetProp(child, "last_op");
	event_node = xmlGetProp(child, "op_node");
	event_rsc = xmlGetProp(child, "id");
	event_status = xmlGetProp(child, "op_status");
	event_rc = xmlGetProp(child, "op_code");

	abort = !process_graph_event(event_node,
	event_rsc,
	event_action,
	event_status,
	event_rc);

	child = child->next;
	}
	}
	}

	return !abort;
	}


	gboolean
	process_graph_event(const char *event_node,
	const char *event_rsc,
	const char *event_action,
	const char *event_status,
	const char *event_rc)
	{
	int lpc;
	xmlNodePtr action = NULL; // <rsc_op> or <crm_event>
	xmlNodePtr next_action = NULL;
	+ action_list_t *matched_action_list = NULL;

	// Find the action corresponding to this event
	slist_iter(
	action_list, action_list_t, graph, lpc,
	action = g_slist_nth_data(action_list->actions,
	action_list->index);
	/*
	<rsc_op id= runnable= optional= task= on_node= >
	<resource id="rsc3" priority="3.0"/>
	</rsc_op>
	*/
	const char *this_action = xmlGetProp(action, "task");
	const char *this_node = xmlGetProp(action, "on_node");
	const char *this_rsc = xmlGetProp(action->children, "id");

	if(safe_str_neq(this_node, event_node)) {
	continue;

	} else if(safe_str_neq(this_action, event_action)) {
	continue;

	} else if(safe_str_eq(action->name, "rsc_op")
	&& safe_str_eq(this_rsc, event_rsc)) {
	- action_list->index++;
	- next_action = g_slist_nth_data(action_list->actions,
	- action_list->index);
	+ matched_action_list = action_list;

	} else if(safe_str_eq(action->name, "crm_event")) {
	- action_list->index++;
	- next_action = g_slist_nth_data(action_list->actions,
	- action_list->index);
	-
	+ matched_action_list = action_list;
	}
	- );
	+ );

	// for the moment all actions succeed

	- if(action == NULL) {
	+ if(matched_action_list == NULL) {
	// unexpected event, trigger a pe-recompute
	// possibly do this only for certain types of actions

	send_abort();
	-
	- } else if(next_action == NULL) {
	- /* last action in that list, check if there are
	- * anymore actions at all
	- */
	- gboolean more_to_do = FALSE;
	- slist_iter(
	- action_list, action_list_t, graph, lpc,
	- if(action_list->index <= action_list->index_max){
	- more_to_do = TRUE;
	- break;
	- }
	- );
	- if(more_to_do == FALSE) {
	- // indicate to the CRMd that we're done
	- xmlNodePtr options = create_xml_node(NULL, "options");
	- set_xml_property_copy(options, XML_ATTR_OP, "te_complete");
	+ return FALSE;
	+ }

	- send_ipc_request(crm_ch, options, NULL,
	- NULL, "dc", "tengine",
	- NULL, NULL);
	-
	- free_xml(options);
	+ gboolean more_to_do = FALSE;
	+ while(matched_action_list->index < matched_action_list->index_max) {
	+ next_action = g_slist_nth_data(matched_action_list->actions,
	+ matched_action_list->index);

	- return TRUE;
	- } // else wait for the next event
	+ gboolean failed = initiate_action(matched_action_list);
	+
	+ if(failed) {
	+ send_abort();
	+
	+ } else if(matched_action_list->index >
	+ matched_action_list->index_max) {
	+ /* last action in that list, check if there are
	+ * anymore actions at all
	+ */
	+ slist_iter(
	+ action_list, action_list_t, graph, lpc,
	+ if(action_list->index <=
	+ action_list->index_max){
	+ more_to_do = TRUE;
	+ break;
	+ }
	+ );
	+ } else {
	+ more_to_do = TRUE;
	+
	+ }

	- } else {
	- return initiate_action(next_action);
	}

	+ if(more_to_do == FALSE) {
	+ // indicate to the CRMd that we're done
	+ xmlNodePtr options = create_xml_node(NULL, "options");
	+ set_xml_property_copy(options, XML_ATTR_OP,
	+ "te_complete");
	+
	+ send_ipc_request(crm_ch, options, NULL,
	+ NULL, "dc", "tengine",
	+ NULL, NULL);
	+
	+ free_xml(options);
	+
	+ return TRUE;
	+ } // else wait for the next event
	+
	return FALSE;
	}

	gboolean
	initiate_transition(void)
	{
	int lpc;
	gboolean anything = FALSE;
	- xmlNodePtr action = NULL;

	FNIN();

	slist_iter(
	action_list, action_list_t, graph, lpc,
	- action = g_slist_nth_data(action_list->actions,
	- action_list->index);
	-
	- if(action != NULL) {
	+ if(initiate_action(action_list) == FALSE) {
	anything = TRUE;
	- initiate_action(action);
	}
	-
	- action_list->index++;
	);

	FNRET(anything);
	}

	gboolean
	-initiate_action(xmlNodePtr xml_action)
	+initiate_action(action_list_t *list)
	{
	- // initiate the next action
	-
	- const char *on_node = xmlGetProp(xml_action, "on_node");
	- const char *id = xmlGetProp(xml_action, "id");
	-// const char *runnable = xmlGetProp(xml_action, "runnable");
	-// const char *optional = xmlGetProp(xml_action, "optional");
	- const char *task = xmlGetProp(xml_action, "task");
	-
	- FNIN();
	-
	-
	- cl_log(LOG_INFO, "Invoking action %s (id=%s) on %s", task, id, on_node);
	-
	-
	- if(id == NULL \|\| strlen(id) == 0
	- \|\| on_node == NULL \|\| strlen(on_node) == 0
	- \|\| task == NULL \|\| strlen(task) == 0) {
	- // error
	- cl_log(LOG_ERR,
	- "Command: \"%s (id=%s) on %s\" was corrupted.",
	+ gboolean was_error = FALSE;
	+ xmlNodePtr xml_action = NULL;
	+ const char *on_node = NULL;
	+ const char *id = NULL;
	+ const char *runnable = NULL;
	+ const char *optional = NULL;
	+ const char *task = NULL;
	+
	+ while(TRUE) {
	+
	+ list->index++;
	+ xml_action = g_slist_nth_data(list->actions, list->index);
	+
	+ if(xml_action == NULL) {
	+ cl_log(LOG_INFO, "No tasks left on this list");
	+ list->index = list->index_max + 1;
	+
	+ return was_error;
	+ }
	+
	+
	+ on_node = xmlGetProp(xml_action, "on_node");
	+ id = xmlGetProp(xml_action, "id");
	+ runnable = xmlGetProp(xml_action, "runnable");
	+ optional = xmlGetProp(xml_action, "optional");
	+ task = xmlGetProp(xml_action, "task");
	+
	+ cl_log(LOG_INFO,
	+ "Invoking action %s (id=%s) on %s",
	task, id, on_node);

	- FNRET(FALSE);
	+ if(safe_str_eq(optional, "true")) {
	+ cl_log(LOG_INFO, "Skipping optional command");
	+
	+ } else if(safe_str_eq(runnable, "false")) {
	+ cl_log(LOG_ERR, "Skipping un-runnable command");
	+ return !was_error;
	+
	+ } else if(id == NULL \|\| strlen(id) == 0
	+ \|\| on_node == NULL \|\| strlen(on_node) == 0
	+ \|\| task == NULL \|\| strlen(task) == 0) {
	+ // error
	+ cl_log(LOG_ERR,
	+ "Command: \"%s (id=%s) on %s\" was corrupted.",
	+ task, id, on_node);
	+
	+ return !was_error;

	// } else if(safe_str_eq(xml_action->name, "pseduo_event")){

	- } else if(safe_str_eq(xml_action->name, "crm_event")){
	- /*
	- <crm_msg op="task" to="on_node">
	- */
	- xmlNodePtr options = create_xml_node(NULL, "options");
	- set_xml_property_copy(options, XML_ATTR_OP, task);
	-
	- send_ipc_request(crm_ch, options, NULL,
	- on_node, "crmd", "tengine",
	- NULL, NULL);
	+ } else if(safe_str_eq(xml_action->name, "crm_event")){
	+ /*
	+ <crm_msg op="task" to="on_node">
	+ */
	+ xmlNodePtr options = create_xml_node(NULL, "options");
	+ set_xml_property_copy(options, XML_ATTR_OP, task);

	- free_xml(options);
	+ send_ipc_request(crm_ch, options, NULL,
	+ on_node, "crmd", "tengine",
	+ NULL, NULL);
	+
	+ free_xml(options);
	+ return was_error;

	- } else if(safe_str_eq(xml_action->name, "rsc_op")){
	- /*
	- <msg_data>
	+ } else if(safe_str_eq(xml_action->name, "rsc_op")){
	+ /*
	+ <msg_data>
	<rsc_op id="operation number" on_node="" task="">
	- <resource>...</resource>
	- */
	- xmlNodePtr options = create_xml_node(NULL, "options");
	- xmlNodePtr data = create_xml_node(NULL, "msg_data");
	- xmlNodePtr rsc_op = create_xml_node(data, "rsc_op");
	-
	- set_xml_property_copy(options, XML_ATTR_OP, "rsc_op");
	-
	- set_xml_property_copy(rsc_op, "id", id);
	- set_xml_property_copy(rsc_op, "task", task);
	- set_xml_property_copy(rsc_op, "on_node", on_node);
	-
	- add_node_copy(rsc_op, xml_action->children);
	-
	- send_ipc_request(crm_ch, options, data,
	- on_node, "lrmd", "tengine",
	- NULL, NULL);
	+ <resource>...</resource>
	+ */
	+ xmlNodePtr options = create_xml_node(NULL, "options");
	+ xmlNodePtr data = create_xml_node(NULL, "msg_data");
	+ xmlNodePtr rsc_op = create_xml_node(data, "rsc_op");

	- free_xml(options);
	- free_xml(data);
	+ set_xml_property_copy(options, XML_ATTR_OP, "rsc_op");

	- } else {
	- // error
	- cl_log(LOG_ERR, "Action %s is not (yet?) supported",
	- xml_action->name);
	-
	- FNRET(FALSE);
	+ set_xml_property_copy(rsc_op, "id", id);
	+ set_xml_property_copy(rsc_op, "task", task);
	+ set_xml_property_copy(rsc_op, "on_node", on_node);
	+
	+ add_node_copy(rsc_op, xml_action->children);
	+
	+ send_ipc_request(crm_ch, options, data,
	+ on_node, "lrmd", "tengine",
	+ NULL, NULL);
	+
	+ free_xml(options);
	+ free_xml(data);
	+ return was_error;
	+
	+ } else {
	+ // error
	+ cl_log(LOG_ERR, "Action %s is not (yet?) supported",
	+ xml_action->name);
	+
	+ }
	}
	-
	- FNRET(TRUE);
	-
	+
	+ return !was_error;
	}

	gboolean
	process_te_message(xmlNodePtr msg, IPC_Channel *sender)
	{
	const char *op = get_xml_attr (msg, XML_TAG_OPTIONS,
	XML_ATTR_OP, TRUE);

	const char *sys_to = xmlGetProp(msg, XML_ATTR_SYSTO);

	cl_log(LOG_DEBUG, "Processing %s message", op);

	if(op == NULL){
	// error
	} else if(sys_to == NULL \|\| strcmp(sys_to, "tengine") != 0) {
	CRM_DEBUG("Bad sys-to %s", sys_to);
	return FALSE;

	} else if(strcmp(op, "transition") == 0) {
	+
	+ CRM_DEBUG("Initializing graph...");
	initialize_graph();

	xmlNodePtr graph = find_xml_node(msg, "transition_graph");
	+ CRM_DEBUG("Unpacking graph...");
	unpack_graph(graph);
	+ CRM_DEBUG("Initiating transition...");
	if(initiate_transition() == FALSE) {
	// nothing to be done.. means we're done.
	cl_log(LOG_INFO, "No actions to be taken..."
	" transition compelte.");
	send_success();
	}
	+ CRM_DEBUG("Processing complete...");


	} else if(strcmp(op, "event") == 0) {
	const char *true_op = get_xml_attr (msg, XML_TAG_OPTIONS,
	"true_op", TRUE);
	if(true_op == NULL) {
	#ifdef USE_FAKE_LRM
	process_fake_event(msg);
	#else
	// error
	#endif
	} else if(strcmp(op, CRM_OPERATION_CREATE) == 0
	\|\| strcmp(op, CRM_OPERATION_DELETE) == 0
	\|\| strcmp(op, CRM_OPERATION_REPLACE) == 0
	\|\| strcmp(op, CRM_OPERATION_WELCOME) == 0
	\|\| strcmp(op, CRM_OPERATION_SHUTDOWN_REQ) == 0
	\|\| strcmp(op, CRM_OPERATION_ERASE) == 0) {

	// these are always unexpected, trigger the PE
	send_abort();

	} else if(strcmp(op, CRM_OPERATION_UPDATE) == 0) {
	// this may not be un-expected
	extract_event(msg);

	} else {
	cl_log(LOG_ERR,
	"Did not expect copy of action %s", op);
	}

	} else if(strcmp(op, "abort") == 0) {
	initialize_graph();

	} else if(strcmp(op, "quit") == 0) {
	cl_log(LOG_WARNING, "Received quit message, terminating");
	exit(0);
	}

	return TRUE;
	}

	void
	send_abort(void)
	{
	xmlNodePtr options = create_xml_node(NULL, "options");

	CRM_DEBUG("Sending \"abort\" message");

	set_xml_property_copy(options, XML_ATTR_OP, "te_abort");

	send_ipc_request(crm_ch, options, NULL,
	NULL, "dc", "tengine",
	NULL, NULL);

	free_xml(options);
	}

	void
	send_success(void)
	{
	xmlNodePtr options = create_xml_node(NULL, "options");

	CRM_DEBUG("Sending \"complete\" message");

	set_xml_property_copy(options, XML_ATTR_OP, "te_complete");

	send_ipc_request(crm_ch, options, NULL,
	NULL, "dc", "tengine",
	NULL, NULL);

	free_xml(options);
	}

File Metadata

Mime Type: text/x-diff
Expires: Wed, Jun 25, 4:15 AM (1 d, 11 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1952113
Default Alt Text: (180 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions