No OneTemporary
Actions

Size

167 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/crm/crmd/Makefile.am b/crm/crmd/Makefile.am
	index 3a6f775bab..5b431fd2a2 100644
	--- a/crm/crmd/Makefile.am
	+++ b/crm/crmd/Makefile.am
	@@ -1,94 +1,94 @@
	#
	# Copyright (C) 2004 Andrew Beekhof
	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU General Public License
	# as published by the Free Software Foundation; either version 2
	# of the License, or (at your option) any later version.
	#
	# This program is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	# GNU General Public License for more details.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; if not, write to the Free Software
	# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	#
	MAINTAINERCLEANFILES = Makefile.in

	INCLUDES = -I$(top_builddir)/include -I$(top_srcdir)/include \
	-I$(top_builddir)/libltdl -I$(top_srcdir)/libltdl \
	-I$(top_builddir)/linux-ha -I$(top_srcdir)/linux-ha \
	-I$(top_builddir) -I$(top_srcdir)

	hadir = $(sysconfdir)/ha.d
	halibdir = $(libdir)/@HB_PKG@
	commmoddir = $(halibdir)/modules/comm
	havarlibdir = $(localstatedir)/lib/@HB_PKG@
	PIDFILE = $(localstatedir)/run/crmd.pid
	XML_FLAGS = `xml2-config --cflags`
	XML_LIBS = `xml2-config --libs`

	# sockets with path
	crmdir = $(havarlibdir)/crm
	apigid = @HA_APIGID@
	crmuid = @HA_CCMUID@

	COMMONLIBS = $(CRM_DEBUG_LIBS) \
	$(top_builddir)/lib/clplumbing/libplumb.la \
	$(top_builddir)/$(CRM_DIR)/common/libcrmcommon.la \
	$(top_builddir)/$(CRM_DIR)/cib/libcib.la \
	$(top_builddir)/lib/apphb/libapphb.la \
	$(top_builddir)/lib/hbclient/libhbclient.la \
	$(GLIBLIB) \
	$(LIBRT)

	LIBRT = @LIBRT@
	AM_CFLAGS = @CFLAGS@ \
	-DPIDFILE='"$(PIDFILE)"' \
	$(CRM_DEBUG_FLAGS)

	## libraries
	lib_LTLIBRARIES =

	## binary progs
	halib_PROGRAMS = crmd

	## SOURCES

	-noinst_HEADERS = crmd.h crmd_fsa.h crmd_messages.h fsa_defines.h \
	- fsa_matrix.h fsa_proto.h crmd_utils.h
	+noinst_HEADERS = crmd.h crmd_fsa.h crmd_messages.h fsa_defines.h \
	+ fsa_matrix.h fsa_proto.h crmd_utils.h crmd_callbacks.h

	crmd_SOURCES = crmdmain.c crmd.c \
	- fsa.c control.c messages.c ccm.c \
	+ fsa.c control.c messages.c ccm.c callbacks.c \
	election.c subsystems.c lrm.c join.c utils.c misc.c

	crmd_CFLAGS = $(XML_FLAGS) -DHA_VARLIBDIR='"@HA_VARLIBDIR@"'
	crmd_LDFLAGS = $(XML_LIBS)
	crmd_LDADD = $(COMMONLIBS) \
	$(top_builddir)/$(CRM_DIR)/cib/libcib.la \
	$(top_builddir)/lib/lrm/liblrm.la \
	$(top_builddir)/membership/ccm/libccmclient.la

	# Simple HA client app
	#clnt_SOURCES = clnt.c

	#clnt_CFLAGS = $(XML_FLAGS) -DHA_VARLIBDIR='"@HA_VARLIBDIR@"'
	#clnt_LDFLAGS = $(XML_LIBS)
	#clnt_LDADD = $(COMMONLIBS)

	clean-generic:
	rm -f .log .debug .xml ~

	install-exec-local:

	uninstall-local:

	graphs: fsa_inputs.png fsa_inputs_by_action.png fsa_actions_by_state.png

	%.png: %.dot
	dot -Tpng $< > $@

	%.dot : fsa_matrix.h
	perl make_dot.pl
	diff --git a/crm/crmd/callbacks.c b/crm/crmd/callbacks.c
	new file mode 100644
	index 0000000000..5b653e4514
	--- /dev/null
	+++ b/crm/crmd/callbacks.c
	@@ -0,0 +1,322 @@
	+/*
	+ * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	+ *
	+ * This program is free software; you can redistribute it and/or
	+ * modify it under the terms of the GNU General Public
	+ * License as published by the Free Software Foundation; either
	+ * version 2.1 of the License, or (at your option) any later version.
	+ *
	+ * This software is distributed in the hope that it will be useful,
	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	+ * General Public License for more details.
	+ *
	+ * You should have received a copy of the GNU General Public
	+ * License along with this library; if not, write to the Free Software
	+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	+ */
	+#include <portability.h>
	+#include <crm/crm.h>
	+#include <string.h>
	+#include <crmd_fsa.h>
	+#include <libxml/tree.h>
	+
	+#include <heartbeat.h>
	+
	+#include <hb_api.h>
	+#include <crm/msg_xml.h>
	+#include <crm/common/xml.h>
	+#include <crm/common/msg.h>
	+#include <crm/cib.h>
	+
	+#include <crmd.h>
	+#include <crmd_messages.h>
	+
	+#include <crm/dmalloc_wrapper.h>
	+
	+FILE *msg_in_strm = NULL;
	+
	+xmlNodePtr find_xml_in_hamessage(const struct ha_msg* msg);
	+
	+void
	+crmd_ha_input_callback(const struct ha_msg* msg, void* private_data)
	+{
	+ const char *from = ha_msg_value(msg, F_ORIG);
	+ const char *to = NULL;
	+ xmlNodePtr root_xml_node;
	+
	+ FNIN();
	+
	+#ifdef MSG_LOG
	+ if(msg_in_strm == NULL) {
	+ msg_in_strm = fopen("/tmp/inbound.log", "w");
	+ }
	+#endif
	+
	+ if(from == NULL \|\| strcmp(from, fsa_our_uname) == 0) {
	+#ifdef MSG_LOG
	+ fprintf(msg_in_strm,
	+ "Discarded message [F_SEQ=%s] from ourselves.\n",
	+ ha_msg_value(msg, F_SEQ));
	+#endif
	+ FNOUT();
	+ }
	+
	+#ifdef MSG_LOG
	+ fprintf(msg_in_strm, "[%s (%s:%s)]\t%s\n",
	+ from,
	+ ha_msg_value(msg, F_SEQ),
	+ ha_msg_value(msg, F_TYPE),
	+ ha_msg_value(msg, "xml")
	+ );
	+ fflush(msg_in_strm);
	+#endif
	+
	+ root_xml_node = find_xml_in_hamessage(msg);
	+ to = xmlGetProp(root_xml_node, XML_ATTR_HOSTTO);
	+
	+ if(to != NULL && strlen(to) > 0 && strcmp(to, fsa_our_uname) != 0) {
	+#ifdef MSG_LOG
	+ fprintf(msg_in_strm,
	+ "Discarding message [F_SEQ=%s] for someone else.",
	+ ha_msg_value(msg, F_SEQ));
	+#endif
	+ FNOUT();
	+ }
	+
	+ set_xml_property_copy(root_xml_node, XML_ATTR_HOSTFROM, from);
	+ s_crmd_fsa(C_HA_MESSAGE, I_ROUTER, root_xml_node);
	+
	+ free_xml(root_xml_node);
	+
	+ FNOUT();
	+}
	+
	+/*
	+ * Apparently returning TRUE means "stay connected, keep doing stuff".
	+ * Returning FALSE means "we're all done, close the connection"
	+ */
	+gboolean
	+crmd_ipc_input_callback(IPC_Channel *client, gpointer user_data)
	+{
	+ int lpc = 0;
	+ char *buffer = NULL;
	+ IPC_Message *msg = NULL;
	+ gboolean hack_return_good = TRUE;
	+ xmlNodePtr root_xml_node;
	+ crmd_client_t curr_client = (crmd_client_t)user_data;
	+
	+ FNIN();
	+ CRM_DEBUG("Processing IPC message from %s",
	+ curr_client->table_key);
	+
	+ while(client->ops->is_message_pending(client)) {
	+ if (client->ch_status == IPC_DISCONNECT) {
	+ /* The message which was pending for us is that
	+ * the IPC status is now IPC_DISCONNECT */
	+ break;
	+ }
	+ if (client->ops->recv(client, &msg) != IPC_OK) {
	+ perror("Receive failure:");
	+ FNRET(!hack_return_good);
	+ }
	+ if (msg == NULL) {
	+ cl_log(LOG_WARNING, "No message this time");
	+ continue;
	+ }
	+
	+ lpc++;
	+ buffer = (char*)msg->msg_body;
	+ CRM_DEBUG("Processing xml from %s [text=%s]",
	+ curr_client->table_key, buffer);
	+
	+ root_xml_node =
	+ find_xml_in_ipcmessage(msg, FALSE);
	+ if (root_xml_node != NULL) {
	+
	+ if (crmd_authorize_message(root_xml_node,
	+ msg,
	+ curr_client)) {
	+ s_crmd_fsa(C_IPC_MESSAGE,
	+ I_ROUTER,
	+ root_xml_node);
	+ }
	+ } else {
	+ cl_log(LOG_INFO,
	+ "IPC Message was not valid... discarding.");
	+ }
	+ free_xml(root_xml_node);
	+ msg->msg_done(msg);
	+
	+ msg = NULL;
	+ buffer = NULL;
	+ root_xml_node = NULL;
	+ }
	+
	+ CRM_DEBUG("Processed %d messages", lpc);
	+
	+ if (client->ch_status == IPC_DISCONNECT)
	+ {
	+ cl_log(LOG_INFO,
	+ "received HUP from %s",
	+ curr_client->table_key);
	+ if (curr_client != NULL) {
	+ struct crm_subsystem_s *the_subsystem = NULL;
	+
	+ if (curr_client->sub_sys == NULL) {
	+ cl_log(LOG_WARNING,
	+ "Client hadn't registered with us yet");
	+
	+ } else if (strcmp(CRM_SYSTEM_PENGINE,
	+ curr_client->sub_sys) == 0) {
	+ the_subsystem = pe_subsystem;
	+
	+ } else if (strcmp(CRM_SYSTEM_TENGINE,
	+ curr_client->sub_sys) == 0) {
	+ the_subsystem = te_subsystem;
	+
	+ } else if (strcmp(CRM_SYSTEM_CIB,
	+ curr_client->sub_sys) == 0){
	+ the_subsystem = cib_subsystem;
	+ }
	+
	+
	+ if(the_subsystem != NULL) {
	+ cleanup_subsystem(the_subsystem);
	+ } // else that was a transient client
	+
	+ if (curr_client->table_key != NULL) {
	+ /*
	+ * Key is destroyed below:
	+ * curr_client->table_key
	+ * Value is cleaned up by:
	+ * G_main_del_IPC_Channel
	+ */
	+ g_hash_table_remove(
	+ ipc_clients, curr_client->table_key);
	+ }
	+
	+
	+ if(curr_client->client_source != NULL) {
	+ gboolean det = G_main_del_IPC_Channel(
	+ curr_client->client_source);
	+
	+ CRM_DEBUG("crm_client was %s detached",
	+ det?"successfully":"not");
	+ }
	+
	+ crm_free(curr_client->table_key);
	+ crm_free(curr_client->sub_sys);
	+ crm_free(curr_client->uuid);
	+ crm_free(curr_client);
	+ }
	+ FNRET(!hack_return_good);
	+ }
	+
	+ FNRET(hack_return_good);
	+}
	+
	+
	+void
	+lrm_op_callback (lrm_op_t* op)
	+{
	+ s_crmd_fsa(C_LRM_OP_CALLBACK, I_LRM_EVENT, op);
	+}
	+
	+void
	+lrm_monitor_callback (lrm_mon_t* monitor)
	+{
	+ s_crmd_fsa(C_LRM_MONITOR_CALLBACK, I_LRM_EVENT, monitor);
	+}
	+
	+void
	+CrmdClientStatus(const char * node, const char * client,
	+ const char * status, void * private)
	+{
	+ const char *join = NULL;
	+ const char *extra = NULL;
	+ xmlNodePtr update = NULL;
	+ xmlNodePtr fragment = NULL;
	+
	+ if(safe_str_eq(status, JOINSTATUS)){
	+ status = ONLINESTATUS;
	+ extra = XML_CIB_ATTR_CLEAR_SHUTDOWN;
	+
	+ } else if(safe_str_eq(status, LEAVESTATUS)){
	+ status = OFFLINESTATUS;
	+ join = CRMD_JOINSTATE_DOWN;
	+ extra = XML_CIB_ATTR_CLEAR_SHUTDOWN;
	+ }
	+
	+ cl_log(LOG_NOTICE,
	+ "Status update: Client %s/%s now has status [%s]\n",
	+ node, client, status);
	+
	+ if(AM_I_DC) {
	+ update = create_node_state(node, NULL, status, join);
	+
	+ if(extra != NULL) {
	+ set_xml_property_copy(update, extra, XML_BOOLEAN_TRUE);
	+ }
	+
	+ fragment = create_cib_fragment(update, NULL);
	+ store_request(NULL, fragment,
	+ CRM_OP_UPDATE, CRM_SYSTEM_DCIB);
	+
	+ free_xml(fragment);
	+ free_xml(update);
	+
	+ s_crmd_fsa(C_CRMD_STATUS_CALLBACK, I_NULL, NULL);
	+
	+ } else {
	+ cl_log(LOG_ERR, "Got client status callback in non-DC mode");
	+ }
	+}
	+
	+
	+xmlNodePtr
	+find_xml_in_hamessage(const struct ha_msg* msg)
	+{
	+ const char *xml;
	+ xmlDocPtr doc;
	+ xmlNodePtr root;
	+
	+ FNIN();
	+ if (msg == NULL) {
	+ cl_log(LOG_INFO,
	+ "**** ha_crm_msg_callback called on a NULL message");
	+ FNRET(NULL);
	+ }
	+
	+#if 0
	+ cl_log(LOG_DEBUG, "[F_TYPE=%s]", ha_msg_value(msg, F_TYPE));
	+ cl_log(LOG_DEBUG, "[F_ORIG=%s]", ha_msg_value(msg, F_ORIG));
	+ cl_log(LOG_DEBUG, "[F_TO=%s]", ha_msg_value(msg, F_TO));
	+ cl_log(LOG_DEBUG, "[F_COMMENT=%s]", ha_msg_value(msg, F_COMMENT));
	+ cl_log(LOG_DEBUG, "[F_XML=%s]", ha_msg_value(msg, "xml"));
	+// cl_log(LOG_DEBUG, "[F_=%s]", ha_msg_value(ha_msg, F_));
	+#endif
	+
	+ if (strcmp("CRM", ha_msg_value(msg, F_TYPE)) != 0) {
	+ cl_log(LOG_INFO, "Received a (%s) message by mistake.",
	+ ha_msg_value(msg, F_TYPE));
	+ FNRET(NULL);
	+ }
	+ xml = ha_msg_value(msg, "xml");
	+ if (xml == NULL) {
	+ cl_log(LOG_INFO, "No XML attached to this message.");
	+ FNRET(NULL);
	+ }
	+ doc = xmlParseMemory(xml, strlen(xml));
	+ if (doc == NULL) {
	+ cl_log(LOG_INFO, "XML Buffer was not valid.");
	+ FNRET(NULL);
	+ }
	+
	+ root = xmlDocGetRootElement(doc);
	+ if (root == NULL) {
	+ cl_log(LOG_INFO, "Root node was NULL.");
	+ FNRET(NULL);
	+ }
	+ FNRET(root);
	+}
	diff --git a/crm/crmd/ccm.c b/crm/crmd/ccm.c
	index fa84d4f149..279a4c0d21 100644
	--- a/crm/crmd/ccm.c
	+++ b/crm/crmd/ccm.c
	@@ -1,577 +1,577 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/

	/* put these first so that uuid_t is defined without conflicts */
	#include <portability.h>
	#include <ocf/oc_event.h>
	#include <ocf/oc_membership.h>

	+#include <clplumbing/GSource.h>
	+#include <string.h>
	+
	#include <crm/crm.h>
	-#include <crm/msg_xml.h>
	#include <crm/cib.h>
	+#include <crm/msg_xml.h>
	+#include <crm/common/xml.h>
	+#include <crmd_messages.h>
	#include <crmd_fsa.h>
	#include <fsa_proto.h>

	-void oc_ev_special(const oc_ev_t *, oc_ev_class_t , int );
	-
	-#include <clplumbing/GSource.h>
	-#include <crm/common/ipcutils.h>
	-#include <crm/common/xmlutils.h>
	-#include <crmd_messages.h>
	-#include <string.h>
	#include <crm/dmalloc_wrapper.h>

	+void oc_ev_special(const oc_ev_t *, oc_ev_class_t , int );
	+
	int register_with_ccm(ll_cluster_t *hb_cluster);

	void msg_ccm_join(const struct ha_msg msg, void foo);

	void crmd_ccm_input_callback(oc_ed_t event,
	void *cookie,
	size_t size,
	const void *data);

	void ccm_event_detail(const oc_ev_membership_t *oc, oc_ed_t event);
	gboolean ccm_dispatch(int fd, gpointer user_data);

	gboolean ghash_node_clfree(gpointer key, gpointer value, gpointer user_data);
	void ghash_update_cib_node(gpointer key, gpointer value, gpointer user_data);

	#define CCM_EVENT_DETAIL 1
	oc_ev_t *fsa_ev_token;

	/* A_CCM_CONNECT */
	enum crmd_fsa_input
	do_ccm_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	int ret;
	int fsa_ev_fd;

	FNIN();

	if(action & A_CCM_DISCONNECT){
	oc_ev_unregister(fsa_ev_token);

	}

	if(action & A_CCM_CONNECT) {

	cl_log(LOG_INFO, "Registering with CCM");
	oc_ev_register(&fsa_ev_token);

	cl_log(LOG_INFO, "Setting up CCM callbacks");
	oc_ev_set_callback(fsa_ev_token, OC_EV_MEMB_CLASS,
	crmd_ccm_input_callback,
	NULL);

	oc_ev_special(fsa_ev_token, OC_EV_MEMB_CLASS, 0/don't care/);

	cl_log(LOG_INFO, "Activating CCM token");
	ret = oc_ev_activate(fsa_ev_token, &fsa_ev_fd);
	if (ret){
	cl_log(LOG_INFO, "CCM Activation failed... unregistering");
	oc_ev_unregister(fsa_ev_token);
	return(I_FAIL);
	}
	cl_log(LOG_INFO, "CCM Activation passed... all set to go!");

	//GFDSource*
	G_main_add_fd(G_PRIORITY_LOW, fsa_ev_fd, FALSE, ccm_dispatch,
	fsa_ev_token,
	default_ipc_input_destroy);

	}

	if(action & ~(A_CCM_CONNECT\|A_CCM_DISCONNECT)) {
	cl_log(LOG_ERR, "Unexpected action %s in %s",
	fsa_action2string(action), __FUNCTION__);
	}

	FNRET(I_NULL);
	}


	/* A_CCM_EVENT */
	enum crmd_fsa_input
	do_ccm_event(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input return_input = I_NULL;
	const oc_ev_membership_t oc = ((struct ccm_data )data)->oc;
	oc_ed_t event = ((struct ccm_data )data)->event;

	FNIN();

	cl_log(LOG_INFO,"event=%s",
	event==OC_EV_MS_NEW_MEMBERSHIP?"NEW MEMBERSHIP":
	event==OC_EV_MS_NOT_PRIMARY?"NOT PRIMARY":
	event==OC_EV_MS_PRIMARY_RESTORED?"PRIMARY RESTORED":
	event==OC_EV_MS_EVICTED?"EVICTED":
	"NO QUORUM MEMBERSHIP");

	if(CCM_EVENT_DETAIL) {
	ccm_event_detail(oc, event);
	}

	if (OC_EV_MS_EVICTED == event) {
	/* get out... NOW! */
	return_input = I_SHUTDOWN;

	}

	if(return_input == I_SHUTDOWN) {
	; /* ignore everything, the new DC will handle it */
	} else {
	/* My understanding is that we will never get both
	* node leaving and node joining callbacks at the
	* same time.
	*
	* This logic would need to change if this is not
	* the case
	*/

	if(oc->m_n_out !=0) {
	return_input = I_NODE_LEFT;

	} else if(oc->m_n_in !=0) {
	/* delay the I_NODE_JOIN until they acknowledge our
	* DC status and send us their CIB
	*/
	return_input = I_NULL;
	} else {
	cl_log(LOG_INFO,
	"So why are we here? What CCM event happened?");
	}
	}

	FNRET(return_input);
	}

	/* A_CCM_UPDATE_CACHE */
	/*
	* Take the opportunity to update the node status in the CIB as well
	* (but only if we are the DC)
	*/
	enum crmd_fsa_input
	do_ccm_update_cache(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input next_input = I_NULL;
	int lpc, offset;
	GHashTable *members = NULL;
	oc_ed_t event = ((struct ccm_data )data)->event;
	const oc_ev_membership_t oc = ((struct ccm_data )data)->oc;

	oc_node_list_t tmp = NULL, membership_copy = (oc_node_list_t *)
	crm_malloc(sizeof(oc_node_list_t));

	FNIN();

	cl_log(LOG_INFO,"Updating CCM cache after a \"%s\" event.",
	event==OC_EV_MS_NEW_MEMBERSHIP?"NEW MEMBERSHIP":
	event==OC_EV_MS_NOT_PRIMARY?"NOT PRIMARY":
	event==OC_EV_MS_PRIMARY_RESTORED?"PRIMARY RESTORED":
	event==OC_EV_MS_EVICTED?"EVICTED":
	"NO QUORUM MEMBERSHIP");

	/---- All Member Nodes ----/
	offset = oc->m_memb_idx;
	membership_copy->members_size = oc->m_n_member;

	if(membership_copy->members_size > 0) {
	membership_copy->members =
	g_hash_table_new(g_str_hash, g_str_equal);
	members = membership_copy->members;

	for(lpc=0; lpc < membership_copy->members_size; lpc++) {
	oc_node_t member = (oc_node_t )
	crm_malloc(sizeof(oc_node_t));

	member->node_id =
	oc->m_array[offset+lpc].node_id;

	member->node_born_on =
	oc->m_array[offset+lpc].node_born_on;

	member->node_uname =
	crm_strdup(oc->m_array[offset+lpc].node_uname);

	g_hash_table_insert(members, member->node_uname, member);
	}

	} else {
	membership_copy->members = NULL;
	}

	/---- New Member Nodes ----/
	offset = oc->m_in_idx;
	membership_copy->new_members_size = oc->m_n_in;

	if(membership_copy->new_members_size > 0) {
	membership_copy->new_members =
	g_hash_table_new(g_str_hash, g_str_equal);
	members = membership_copy->new_members;

	for(lpc=0; lpc < membership_copy->new_members_size; lpc++) {
	oc_node_t member = (oc_node_t )
	crm_malloc(sizeof(oc_node_t));

	member->node_id =
	oc->m_array[offset+lpc].node_id;

	member->node_born_on =
	oc->m_array[offset+lpc].node_born_on;

	member->node_uname =
	crm_strdup(oc->m_array[offset+lpc].node_uname);

	g_hash_table_insert(members, member->node_uname, member);
	}

	} else {
	membership_copy->new_members = NULL;
	}

	/---- Recently Dead Member Nodes ----/
	offset = oc->m_out_idx;
	membership_copy->dead_members_size = oc->m_n_out;
	if(membership_copy->dead_members_size > 0) {
	membership_copy->dead_members =
	g_hash_table_new(g_str_hash, g_str_equal);

	members = membership_copy->dead_members;

	for(lpc=0; lpc < membership_copy->dead_members_size; lpc++) {
	oc_node_t member = (oc_node_t )
	crm_malloc(sizeof(oc_node_t));

	member->node_id =
	oc->m_array[offset+lpc].node_id;

	member->node_born_on =
	oc->m_array[offset+lpc].node_born_on;

	member->node_uname =
	crm_strdup(oc->m_array[offset+lpc].node_uname);

	g_hash_table_insert(members, member->node_uname, member);
	}

	} else {
	membership_copy->dead_members = NULL;
	}

	tmp = fsa_membership_copy;
	fsa_membership_copy = membership_copy;

	if(AM_I_DC) {
	// should be sufficient for only the DC to do this
	free_xml(do_update_cib_nodes(NULL, FALSE));
	}

	/* Free the old copy */
	if(tmp != NULL) {
	if(tmp->members != NULL)
	g_hash_table_foreach_remove(
	tmp->members, ghash_node_clfree, NULL);
	if(tmp->new_members != NULL)
	g_hash_table_foreach_remove(
	tmp->new_members, ghash_node_clfree, NULL);
	if(tmp->dead_members != NULL)
	g_hash_table_foreach_remove(
	tmp->dead_members, ghash_node_clfree, NULL);
	crm_free(tmp);
	}

	FNRET(next_input);
	}

	void
	ccm_event_detail(const oc_ev_membership_t *oc, oc_ed_t event)
	{
	int member_id = -1;
	gboolean member = FALSE;
	int lpc;
	int node_list_size;

	cl_log(LOG_INFO,"trans=%d, nodes=%d, new=%d, lost=%d n_idx=%d, "
	"new_idx=%d, old_idx=%d",
	oc->m_instance,
	oc->m_n_member,
	oc->m_n_in,
	oc->m_n_out,
	oc->m_memb_idx,
	oc->m_in_idx,
	oc->m_out_idx);

	cl_log(LOG_INFO, "NODES IN THE PRIMARY MEMBERSHIP");

	node_list_size = oc->m_n_member;
	for(lpc=0; lpc<node_list_size; lpc++) {
	cl_log(LOG_INFO,"\t%s [nodeid=%d, born=%d]",
	oc->m_array[oc->m_memb_idx+lpc].node_uname,
	oc->m_array[oc->m_memb_idx+lpc].node_id,
	oc->m_array[oc->m_memb_idx+lpc].node_born_on);

	CRM_DEBUG("%s ? %s", fsa_our_uname,
	oc->m_array[oc->m_memb_idx+lpc].node_uname);
	if(safe_str_eq(fsa_our_uname,
	oc->m_array[oc->m_memb_idx+lpc].node_uname)) {
	member = TRUE;
	member_id = oc->m_array[oc->m_memb_idx+lpc].node_id;
	}
	}

	if (member == FALSE) {
	cl_log(LOG_WARNING,
	"MY NODE IS NOT IN CCM THE MEMBERSHIP LIST");
	} else {
	cl_log(LOG_INFO, "MY NODE ID IS %d", member_id);
	}


	cl_log(LOG_INFO, "NEW MEMBERS");
	if (oc->m_n_in==0)
	cl_log(LOG_INFO, "\tNONE");

	for(lpc=0; lpc<oc->m_n_in; lpc++) {
	cl_log(LOG_INFO,"\t%s [nodeid=%d, born=%d]",
	oc->m_array[oc->m_in_idx+lpc].node_uname,
	oc->m_array[oc->m_in_idx+lpc].node_id,
	oc->m_array[oc->m_in_idx+lpc].node_born_on);
	}

	cl_log(LOG_INFO, "MEMBERS LOST");
	if (oc->m_n_out==0)
	cl_log(LOG_INFO, "\tNONE");

	for(lpc=0; lpc<oc->m_n_out; lpc++) {
	cl_log(LOG_INFO,"\t%s [nodeid=%d, born=%d]",
	oc->m_array[oc->m_out_idx+lpc].node_uname,
	oc->m_array[oc->m_out_idx+lpc].node_id,
	oc->m_array[oc->m_out_idx+lpc].node_born_on);
	if(fsa_our_uname != NULL
	&& strcmp(fsa_our_uname, oc->m_array[oc->m_memb_idx+lpc].node_uname)) {
	cl_log(LOG_ERR,
	"We're not part of the cluster anymore");
	}
	}

	cl_log(LOG_INFO, "-----------------------");

	}

	int
	register_with_ccm(ll_cluster_t *hb_cluster)
	{
	FNRET(0);
	}

	gboolean ccm_dispatch(int fd, gpointer user_data)
	{
	oc_ev_t ccm_token = (oc_ev_t)user_data;
	oc_ev_handle_event(ccm_token);
	return TRUE;
	}


	void
	crmd_ccm_input_callback(oc_ed_t event,
	void *cookie,
	size_t size,
	const void *data)
	{
	struct ccm_data *event_data = NULL;

	FNIN();

	if(data != NULL) {
	event_data = (struct ccm_data *)
	crm_malloc(sizeof(struct ccm_data));

	event_data->event = &event;
	event_data->oc = (const oc_ev_membership_t *)data;

	s_crmd_fsa(C_CCM_CALLBACK, I_CCM_EVENT, (void*)event_data);

	event_data->event = NULL;
	event_data->oc = NULL;

	crm_free(event_data);

	} else {
	cl_log(LOG_INFO, "CCM Callback with NULL data... "
	"I dont /think/ this is bad");
	}

	oc_ev_callback_done(cookie);

	FNOUT();
	}

	void
	msg_ccm_join(const struct ha_msg msg, void foo)
	{
	FNIN();
	cl_log(LOG_INFO, "\n###### Recieved ccm_join message...");
	if (msg != NULL)
	{
	cl_log(LOG_INFO,
	"[type=%s]",
	ha_msg_value(msg, F_TYPE));
	cl_log(LOG_INFO,
	"[orig=%s]",
	ha_msg_value(msg, F_ORIG));
	cl_log(LOG_INFO,
	"[to=%s]",
	ha_msg_value(msg, F_TO));
	cl_log(LOG_INFO,
	"[status=%s]",
	ha_msg_value(msg, F_STATUS));
	cl_log(LOG_INFO,
	"[info=%s]",
	ha_msg_value(msg, F_COMMENT));
	cl_log(LOG_INFO,
	"[rsc_hold=%s]",
	ha_msg_value(msg, F_RESOURCES));
	cl_log(LOG_INFO,
	"[stable=%s]",
	ha_msg_value(msg, F_ISSTABLE));
	cl_log(LOG_INFO,
	"[rtype=%s]",
	ha_msg_value(msg, F_RTYPE));
	cl_log(LOG_INFO,
	"[ts=%s]",
	ha_msg_value(msg, F_TIME));
	cl_log(LOG_INFO,
	"[seq=%s]",
	ha_msg_value(msg, F_SEQ));
	cl_log(LOG_INFO,
	"[generation=%s]",
	ha_msg_value(msg, F_HBGENERATION));
	// cl_log(LOG_INFO, "[=%s]", ha_msg_value(msg, F_));
	}
	FNOUT();
	}

	struct update_data_s
	{
	xmlNodePtr updates;
	const char *state;
	const char *join;
	};

	xmlNodePtr
	do_update_cib_nodes(xmlNodePtr updates, gboolean overwrite)
	{

	struct update_data_s update_data;
	update_data.updates = updates;

	update_data.state = XML_BOOLEAN_NO;
	update_data.join = CRMD_JOINSTATE_DOWN;
	if(fsa_membership_copy->dead_members != NULL) {
	g_hash_table_foreach(fsa_membership_copy->dead_members,
	ghash_update_cib_node, &update_data);
	}

	update_data.state = XML_BOOLEAN_YES;
	update_data.join = NULL;
	if(overwrite) {
	update_data.join = CRMD_JOINSTATE_PENDING;
	}

	if(fsa_membership_copy->members != NULL) {
	g_hash_table_foreach(fsa_membership_copy->members,
	ghash_update_cib_node, &update_data);
	}

	/* this is most likely overkill...
	*
	* make sure that the join status of nodes entering the ccm list
	* is reset
	*
	update_data.join = CRMD_JOINSTATE_PENDING;
	if(fsa_membership_copy->new_members != NULL) {
	g_hash_table_foreach(fsa_membership_copy->new_members,
	ghash_update_cib_node, &update_data);
	}
	*/
	if(update_data.updates != NULL) {
	xmlNodePtr fragment =
	create_cib_fragment(update_data.updates, NULL);

	store_request(NULL, fragment,
	CRM_OP_UPDATE, CRM_SYSTEM_DCIB);

	free_xml(fragment);
	}

	return update_data.updates;

	}

	void
	ghash_update_cib_node(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNodePtr tmp1 = NULL;
	const char node_uname = (const char)key;
	struct update_data_s* data = (struct update_data_s*)user_data;
	const char *state = data->join;

	crm_debug("%s processing %s (%s)",
	__FUNCTION__, node_uname, data->state);

	if(state != NULL
	&& safe_str_eq(fsa_our_uname, node_uname)) {
	/* the DC is always a member */
	state = CRMD_JOINSTATE_MEMBER;
	}

	tmp1 = create_node_state(node_uname, data->state, NULL, state);

	if(data->updates == NULL) {
	crm_debug("Creating first update");
	data->updates = tmp1;
	} else {
	xmlAddNextSibling(data->updates, tmp1);
	}

	}

	gboolean
	ghash_node_clfree(gpointer key, gpointer value, gpointer user_data)
	{
	// value->node_uname is free'd as "key"
	if(key != NULL) {
	crm_free(key);
	}
	if(value != NULL) {
	crm_free(value);
	}
	return TRUE;
	}
	diff --git a/crm/crmd/control.c b/crm/crmd/control.c
	index bec444e709..03c1adfe11 100644
	--- a/crm/crmd/control.c
	+++ b/crm/crmd/control.c
	@@ -1,415 +1,553 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/

	#include <portability.h>
	+
	#include <crm/crm.h>
	+#include <crm/common/ctrl.h>
	+
	+#include <crmd.h>
	#include <crmd_fsa.h>
	#include <fsa_proto.h>
	-#include <crmd.h>
	-#include <crm/common/ipcutils.h>
	-#include <crm/common/crmutils.h>
	#include <crmd_messages.h>
	+#include <crmd_callbacks.h>

	-#include <clplumbing/Gmain_timeout.h>
	+#include <sys/types.h>
	+#include <sys/stat.h>

	#include <crm/dmalloc_wrapper.h>

	#define PID_FILE WORKING_DIR"/crm.pid"
	#define DAEMON_LOG LOG_DIR"/crm.log"
	#define DAEMON_DEBUG LOG_DIR"/crm.debug"

	gboolean crmd_ha_input_dispatch(int fd, gpointer user_data);
	+
	void crmd_ha_input_destroy(gpointer user_data);
	+
	void crm_shutdown(int nsig);

	+IPC_WaitConnection *wait_channel_init(char daemonsocket[]);
	+
	+int init_server_ipc_comms(
	+ const char *child,
	+ gboolean (channel_client_connect)(IPC_Channel newclient,
	+ gpointer user_data),
	+ void (*channel_input_destroy)(gpointer user_data));
	+
	+
	+gboolean
	+register_with_ha(ll_cluster_t hb_cluster, const char client_name,
	+ gboolean (*dispatch_method)(int fd, gpointer user_data),
	+ void (message_callback)(const struct ha_msg msg,
	+ void* private_data),
	+ GDestroyNotify cleanup_method);
	+
	GHashTable *ipc_clients = NULL;

	/* A_HA_CONNECT */
	enum crmd_fsa_input
	do_ha_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	gboolean registered = FALSE;

	FNIN();

	if(action & A_HA_DISCONNECT) {
	if(fsa_cluster_conn != NULL) {
	fsa_cluster_conn->llc_ops->signoff(fsa_cluster_conn);
	}

	}

	if(action & A_HA_CONNECT) {
	if(fsa_cluster_conn == NULL)
	fsa_cluster_conn = ll_cluster_new("heartbeat");

	// make sure we are disconnected first
	fsa_cluster_conn->llc_ops->signoff(fsa_cluster_conn);

	registered = register_with_ha(fsa_cluster_conn,
	crm_system_name,
	crmd_ha_input_dispatch,
	crmd_ha_input_callback,
	crmd_ha_input_destroy);

	if(registered == FALSE) {
	FNRET(I_FAIL);
	}
	}

	if(action & ~(A_HA_CONNECT\|A_HA_DISCONNECT)) {
	cl_log(LOG_ERR, "Unexpected action %s in %s",
	fsa_action2string(action), __FUNCTION__);
	}


	FNRET(I_NULL);
	}

	/* A_SHUTDOWN */
	enum crmd_fsa_input
	do_shutdown(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input next_input = I_NULL;
	enum crmd_fsa_input tmp = I_NULL;

	FNIN();

	/* last attempt to shut these down */
	if(is_set(fsa_input_register, R_PE_CONNECTED)) {
	cl_log(LOG_WARNING,
	"Last attempt to shutdown the PolicyEngine");
	tmp = do_pe_control(A_PE_STOP, cause, cur_state,
	current_input, data);
	if(tmp != I_NULL) {
	next_input = I_ERROR;
	cl_log(LOG_ERR, "Failed to shutdown the PolicyEngine");
	}
	}

	if(is_set(fsa_input_register, R_TE_CONNECTED)) {
	cl_log(LOG_WARNING,
	"Last attempt to shutdown the Transitioner");
	tmp = do_pe_control(A_TE_STOP, cause, cur_state,
	current_input, data);
	if(tmp != I_NULL) {
	next_input = I_ERROR;
	cl_log(LOG_ERR, "Failed to shutdown the Transitioner");
	}

	}

	/* TODO: shutdown all remaining resources? */

	FNRET(next_input);
	}

	/* A_SHUTDOWN_REQ */
	enum crmd_fsa_input
	do_shutdown_req(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input next_input = I_NULL;
	FNIN();

	if(send_request(NULL, NULL, CRM_OP_SHUTDOWN_REQ,
	NULL, CRM_SYSTEM_DC, NULL) == FALSE){
	next_input = I_ERROR;
	}

	FNRET(next_input);
	}

	gboolean
	crmd_ha_input_dispatch(int fd, gpointer user_data)
	{
	int lpc = 0;
	ll_cluster_t* hb_cluster = (ll_cluster_t*)user_data;

	FNIN();

	while(hb_cluster->llc_ops->msgready(hb_cluster))
	{
	lpc++;
	// invoke the callbacks but dont block
	hb_cluster->llc_ops->rcvmsg(hb_cluster, 0);
	}

	if(lpc == 0){
	// hey what happened??
	cl_log(LOG_ERR, "We were called but no message was ready.\n"
	"\tLikely the connection to Heartbeat failed, check the logs");

	// TODO: feed this back into the FSA

	FNRET(FALSE);
	}


	FNRET(TRUE);
	}

	void
	crmd_ha_input_destroy(gpointer user_data)
	{
	cl_log(LOG_INFO, "in my hb_input_destroy");
	}

	/* A_EXIT_0, A_EXIT_1 */
	enum crmd_fsa_input
	do_exit(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	FNIN();

	cl_log(LOG_ERR, "Action %s (%.16llx) not supported\n", fsa_action2string(action), action);

	if(action & A_EXIT_0) {
	g_main_quit(crmd_mainloop);
	} else {
	exit(1);
	}

	FNRET(I_NULL);
	}


	/* A_STARTUP */
	enum crmd_fsa_input
	do_startup(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	int facility;
	int was_error = 0;
	int interval = 1; // seconds between DC heartbeats


	FNIN();

	fsa_input_register = 0; // zero out the regester

	cl_log(LOG_INFO, "Register PID");
	register_pid(PID_FILE, FALSE, crm_shutdown);

	cl_log_set_logfile(DAEMON_LOG);
	/* if (crm_debug()) { */
	cl_log_set_debugfile(DAEMON_DEBUG);
	/* cl_log_enable_stderr(FALSE);
	} */

	ipc_clients = g_hash_table_new(&g_str_hash, &g_str_equal);

	/* change the logging facility to the one used by heartbeat daemon */
	fsa_cluster_conn = ll_cluster_new("heartbeat");

	cl_log(LOG_INFO, "Switching to Heartbeat logger");
	if ((facility =
	fsa_cluster_conn->llc_ops->get_logfacility(
	fsa_cluster_conn)) > 0) {
	cl_log_set_facility(facility);
	}

	CRM_DEBUG("Facility: %d", facility);

	if(was_error == 0) {
	cl_log(LOG_INFO, "Init server comms");
	was_error = init_server_ipc_comms(CRM_SYSTEM_CRMD,
	crmd_client_connect,
	default_ipc_input_destroy);
	}

	if (was_error == 0) {
	fsa_our_uname = fsa_cluster_conn->llc_ops->get_mynodeid(
	fsa_cluster_conn);

	if (fsa_our_uname == NULL) {
	cl_log(LOG_ERR, "get_mynodeid() failed");
	was_error = 1;
	}
	cl_log(LOG_INFO, "FSA Hostname: %s", fsa_our_uname);
	}

	/* set up the timers */
	dc_heartbeat = (fsa_timer_t *)crm_malloc(sizeof(fsa_timer_t));
	integration_timer= (fsa_timer_t *)crm_malloc(sizeof(fsa_timer_t));
	election_trigger = (fsa_timer_t *)crm_malloc(sizeof(fsa_timer_t));
	election_timeout = (fsa_timer_t *)crm_malloc(sizeof(fsa_timer_t));
	shutdown_escalation_timmer = (fsa_timer_t *)
	crm_malloc(sizeof(fsa_timer_t));


	interval = interval * 1000;

	election_trigger->source_id = -1;
	election_trigger->period_ms = interval*4;
	election_trigger->fsa_input = I_DC_TIMEOUT;
	election_trigger->callback = timer_popped;

	dc_heartbeat->source_id = -1;
	dc_heartbeat->period_ms = interval;
	dc_heartbeat->fsa_input = I_NULL;
	dc_heartbeat->callback = do_dc_heartbeat;

	election_timeout->source_id = -1;
	election_timeout->period_ms = interval*6;
	election_timeout->fsa_input = I_ELECTION_DC;
	election_timeout->callback = timer_popped;

	integration_timer->source_id = -1;
	integration_timer->period_ms = interval*6;
	integration_timer->fsa_input = I_INTEGRATION_TIMEOUT;
	integration_timer->callback = timer_popped;

	shutdown_escalation_timmer->source_id = -1;
	shutdown_escalation_timmer->period_ms = interval*130;
	shutdown_escalation_timmer->fsa_input = I_TERMINATE;
	shutdown_escalation_timmer->callback = timer_popped;

	/* set up the sub systems */
	cib_subsystem = (struct crm_subsystem_s*)
	crm_malloc(sizeof(struct crm_subsystem_s));

	cib_subsystem->pid = 0;
	cib_subsystem->respawn = 1;
	cib_subsystem->path = crm_strdup(BIN_DIR);
	cib_subsystem->name = crm_strdup(CRM_SYSTEM_CIB);
	cib_subsystem->command = BIN_DIR"/cib";
	cib_subsystem->flag = R_CIB_CONNECTED;

	te_subsystem = (struct crm_subsystem_s*)
	crm_malloc(sizeof(struct crm_subsystem_s));

	te_subsystem->pid = 0;
	te_subsystem->respawn = 1;
	te_subsystem->path = crm_strdup(BIN_DIR);
	te_subsystem->name = crm_strdup(CRM_SYSTEM_TENGINE);
	te_subsystem->command = BIN_DIR"/tengine";
	te_subsystem->flag = R_TE_CONNECTED;

	pe_subsystem = (struct crm_subsystem_s*)
	crm_malloc(sizeof(struct crm_subsystem_s));

	pe_subsystem->pid = 0;
	pe_subsystem->respawn = 1;
	pe_subsystem->path = crm_strdup(BIN_DIR);
	pe_subsystem->name = crm_strdup(CRM_SYSTEM_PENGINE);
	pe_subsystem->command = BIN_DIR"/pengine";
	pe_subsystem->flag = R_PE_CONNECTED;


	if(was_error)
	FNRET(I_FAIL);

	FNRET(I_NULL);
	}


	/* A_STOP */
	enum crmd_fsa_input
	do_stop(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	FNIN();

	cl_log(LOG_ERR, "Action %s (%.16llx) not supported\n",
	fsa_action2string(action), action);

	FNRET(I_NULL);
	}

	/* A_STARTED */
	enum crmd_fsa_input
	do_started(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	FNIN();

	clear_bit_inplace(&fsa_input_register, R_STARTING);

	FNRET(I_NULL);
	}

	/* A_RECOVER */
	enum crmd_fsa_input
	do_recover(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	FNIN();

	cl_log(LOG_ERR, "Action %s (%.16llx) not supported\n",
	fsa_action2string(action), action);

	FNRET(I_SHUTDOWN);
	}

	void
	crm_shutdown(int nsig)
	{
	FNIN();

	CL_SIGNAL(nsig, crm_shutdown);


	if (crmd_mainloop != NULL && g_main_is_running(crmd_mainloop)) {

	if(is_set(fsa_input_register, R_SHUTDOWN)) {
	cl_log(LOG_WARNING, "Escalating the shutdown");
	s_crmd_fsa(C_SHUTDOWN, I_ERROR, NULL);

	} else {
	set_bit_inplace(&fsa_input_register, R_SHUTDOWN);

	// cant rely on this...
	startTimer(shutdown_escalation_timmer);

	s_crmd_fsa(C_SHUTDOWN, I_SHUTDOWN, NULL);
	}

	} else {
	cl_log(LOG_INFO, "exit from shutdown");
	exit(LSB_EXIT_OK);

	}
	FNOUT();
	}
	+
	+
	+IPC_WaitConnection *
	+wait_channel_init(char daemonsocket[])
	+{
	+ IPC_WaitConnection *wait_ch;
	+ mode_t mask;
	+ char path[] = IPC_PATH_ATTR;
	+ GHashTable * attrs;
	+
	+ FNIN();
	+ attrs = g_hash_table_new(g_str_hash,g_str_equal);
	+ g_hash_table_insert(attrs, path, daemonsocket);
	+
	+ mask = umask(0);
	+ wait_ch = ipc_wait_conn_constructor(IPC_ANYTYPE, attrs);
	+ if (wait_ch == NULL) {
	+ cl_perror("Can't create wait channel of type %s",
	+ IPC_ANYTYPE);
	+ exit(1);
	+ }
	+ mask = umask(mask);
	+
	+ g_hash_table_destroy(attrs);
	+
	+ FNRET(wait_ch);
	+}
	+
	+int
	+init_server_ipc_comms(
	+ const char *child,
	+ gboolean (channel_client_connect)(IPC_Channel newclient,
	+ gpointer user_data),
	+ void (*channel_input_destroy)(gpointer user_data))
	+{
	+ /* the clients wait channel is the other source of events.
	+ * This source delivers the clients connection events.
	+ * listen to this source at a relatively lower priority.
	+ */
	+
	+ char commpath[SOCKET_LEN];
	+ IPC_WaitConnection *wait_ch;
	+
	+ FNIN();
	+ sprintf(commpath, WORKING_DIR "/%s", child);
	+
	+ wait_ch = wait_channel_init(commpath);
	+
	+ if (wait_ch == NULL) FNRET(1);
	+ G_main_add_IPC_WaitConnection(G_PRIORITY_LOW,
	+ wait_ch,
	+ NULL,
	+ FALSE,
	+ channel_client_connect,
	+ wait_ch, // user data passed to ??
	+ channel_input_destroy);
	+
	+ cl_log(LOG_DEBUG, "Listening on: %s", commpath);
	+
	+ FNRET(0);
	+}
	+
	+gboolean
	+register_with_ha(ll_cluster_t hb_cluster, const char client_name,
	+ gboolean (*dispatch_method)(int fd, gpointer user_data),
	+ void (message_callback)(const struct ha_msg msg,
	+ void* private_data),
	+ GDestroyNotify cleanup_method)
	+{
	+ const char* ournode = NULL;
	+
	+ cl_log(LOG_INFO, "Signing in with Heartbeat");
	+ if (hb_cluster->llc_ops->signon(hb_cluster, client_name)!= HA_OK) {
	+ cl_log(LOG_ERR, "Cannot sign on with heartbeat");
	+ cl_log(LOG_ERR,
	+ "REASON: %s",
	+ hb_cluster->llc_ops->errmsg(hb_cluster));
	+ return FALSE;
	+ }
	+
	+ cl_log(LOG_DEBUG, "Finding our node name");
	+ if ((ournode =
	+ hb_cluster->llc_ops->get_mynodeid(hb_cluster)) == NULL) {
	+ cl_log(LOG_ERR, "get_mynodeid() failed");
	+ return FALSE;
	+ }
	+ cl_log(LOG_INFO, "hostname: %s", ournode);
	+
	+ cl_log(LOG_DEBUG, "Be informed of CRM messages");
	+ if (hb_cluster->llc_ops->set_msg_callback(hb_cluster,
	+ "CRM",
	+ message_callback,
	+ hb_cluster)
	+ !=HA_OK){
	+ cl_log(LOG_ERR, "Cannot set CRM message callback");
	+ cl_log(LOG_ERR,
	+ "REASON: %s",
	+ hb_cluster->llc_ops->errmsg(hb_cluster));
	+ return FALSE;
	+ }
	+
	+
	+ G_main_add_fd(G_PRIORITY_HIGH,
	+ hb_cluster->llc_ops->inputfd(hb_cluster),
	+ FALSE,
	+ dispatch_method,
	+ hb_cluster, // usrdata
	+ cleanup_method);
	+
	+ /* it seems we need to poke the message receiving stuff in order for it to
	+ * start seeing messages. Its like it gets blocked or something.
	+ */
	+ dispatch_method(0, hb_cluster);
	+
	+ return TRUE;
	+
	+}
	diff --git a/crm/crmd/crmd.h b/crm/crmd/crmd.h
	index 6f98d2a177..eaf8cc8b20 100644
	--- a/crm/crmd/crmd.h
	+++ b/crm/crmd/crmd.h
	@@ -1,35 +1,44 @@
	-/* $Id: crmd.h,v 1.5 2004/02/27 13:41:45 andrew Exp $ */
	+/* $Id: crmd.h,v 1.6 2004/06/02 12:31:34 andrew Exp $ */
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#ifndef CRMD__H
	#define CRMD__H

	-extern GMainLoop* crmd_mainloop;
	+#include <ha_msg.h>

	-extern const char* crm_system_name;
	+extern GMainLoop *crmd_mainloop;

	-extern GHashTable *pending_remote_replies;
	-extern GHashTable *ipc_clients;
	+extern const char *crm_system_name;
	+
	+extern GHashTable *ipc_clients;
	+
	+extern GHashTable *pending_remote_replies;

	extern void msg_ccm_join(const struct ha_msg msg, void foo);
	-extern gboolean crmd_client_connect(IPC_Channel *newclient, gpointer user_data);
	-extern void crmd_ha_input_callback(const struct ha_msg* msg, void* private_data);
	-extern gboolean crmd_ipc_input_callback(IPC_Channel *client, gpointer user_data);
	+
	+extern gboolean crmd_client_connect(IPC_Channel *newclient,
	+ gpointer user_data);
	+
	+extern void crmd_ha_input_callback(const struct ha_msg* msg,
	+ void* private_data);
	+
	+extern gboolean crmd_ipc_input_callback(IPC_Channel *client,
	+ gpointer user_data);


	#endif
	diff --git a/crm/crmd/crmd.h b/crm/crmd/crmd_callbacks.h
	similarity index 57%
	copy from crm/crmd/crmd.h
	copy to crm/crmd/crmd_callbacks.h
	index 6f98d2a177..f9f80aec80 100644
	--- a/crm/crmd/crmd.h
	+++ b/crm/crmd/crmd_callbacks.h
	@@ -1,35 +1,38 @@
	-/* $Id: crmd.h,v 1.5 2004/02/27 13:41:45 andrew Exp $ */
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	-#ifndef CRMD__H
	-#define CRMD__H

	-extern GMainLoop* crmd_mainloop;
	+#include <hb_api.h>
	+#include <clplumbing/ipc.h>

	-extern const char* crm_system_name;

	-extern GHashTable *pending_remote_replies;
	-extern GHashTable *ipc_clients;
	+extern void crmd_ha_input_callback(const struct ha_msg* msg,
	+ void* private_data);

	-extern void msg_ccm_join(const struct ha_msg msg, void foo);
	-extern gboolean crmd_client_connect(IPC_Channel *newclient, gpointer user_data);
	-extern void crmd_ha_input_callback(const struct ha_msg* msg, void* private_data);
	-extern gboolean crmd_ipc_input_callback(IPC_Channel *client, gpointer user_data);
	-
	+/*
	+ * Apparently returning TRUE means "stay connected, keep doing stuff".
	+ * Returning FALSE means "we're all done, close the connection"
	+ */
	+extern gboolean crmd_ipc_input_callback(IPC_Channel *client,
	+ gpointer user_data);
	+
	+extern void lrm_op_callback (lrm_op_t* op);
	+
	+extern void lrm_monitor_callback (lrm_mon_t* monitor);

	-#endif
	+extern void CrmdClientStatus(const char * node, const char * client,
	+ const char * status, void * private);
	diff --git a/crm/crmd/crmd_messages.h b/crm/crmd/crmd_messages.h
	index dddfa990a6..37eca91fd9 100644
	--- a/crm/crmd/crmd_messages.h
	+++ b/crm/crmd/crmd_messages.h
	@@ -1,82 +1,86 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#ifndef XML_CRM_MESSAGES__H
	#define XML_CRM_MESSAGES__H


	#include <crm/crm.h>
	-#include <crm/common/ipcutils.h>
	+#include <crm/common/ipc.h>
	#include <crmd_fsa.h>
	#include <libxml/tree.h>

	typedef GSList *fsa_message_queue_t;

	fsa_message_queue_t put_message(xmlNodePtr new_message);
	xmlNodePtr get_message(void);
	gboolean is_message(void);

	extern gboolean relay_message(xmlNodePtr xml_relay_message,
	gboolean originated_locally);

	extern void crmd_ha_input_callback(const struct ha_msg* msg,
	void* private_data);

	extern gboolean crmd_ipc_input_callback(IPC_Channel *client,
	gpointer user_data);

	extern void process_message(xmlNodePtr root_xml_node,
	gboolean originated_locally,
	const char *src_node_name);

	extern gboolean crm_dc_process_message(xmlNodePtr whole_message,
	xmlNodePtr action,
	const char *host_from,
	const char *sys_from,
	const char *sys_to,
	const char *op,
	gboolean dc_mode);

	extern void send_msg_via_ha(xmlNodePtr action, const char *dest_node);
	extern void send_msg_via_ipc(xmlNodePtr action, const char *sys);

	extern gboolean add_pending_outgoing_reply(const char *originating_node_name,
	const char *crm_msg_reference,
	const char *sys_to,
	const char *sys_from);

	extern gboolean crmd_authorize_message(xmlNodePtr root_xml_node,
	IPC_Message *client_msg,
	crmd_client_t *curr_client);
	extern gboolean send_request(xmlNodePtr msg_options,
	xmlNodePtr msg_data,
	const char *operation,
	const char *host_to,
	const char *sys_to,
	char **msg_reference);

	extern gboolean store_request(xmlNodePtr msg_options,
	xmlNodePtr msg_data,
	const char *operation,
	const char *sys_to);

	extern enum crmd_fsa_input handle_message(xmlNodePtr stored_msg);

	+extern gboolean send_ha_reply(ll_cluster_t *hb_cluster,
	+ xmlNodePtr xml_request,
	+ xmlNodePtr xml_response_data);
	+
	extern void lrm_op_callback (lrm_op_t* op);
	extern void lrm_monitor_callback (lrm_mon_t* monitor);

	#endif
	diff --git a/crm/crmd/crmdmain.c b/crm/crmd/crmdmain.c
	index adddd34bb6..9cdf802048 100644
	--- a/crm/crmd/crmdmain.c
	+++ b/crm/crmd/crmdmain.c
	@@ -1,209 +1,204 @@
	-/* $Id: crmdmain.c,v 1.17 2004/06/01 16:12:49 andrew Exp $ */
	+/* $Id: crmdmain.c,v 1.18 2004/06/02 12:31:34 andrew Exp $ */
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <portability.h>
	#include <stdio.h>
	#include <sys/types.h>
	#include <unistd.h>

	#include <stdlib.h>
	#include <errno.h>
	#include <fcntl.h>

	-#include <hb_api.h>
	#include <apphb.h>

	-#include <crm/crm.h>
	-
	#include <clplumbing/ipc.h>
	#include <clplumbing/Gmain_timeout.h>
	#include <clplumbing/cl_log.h>
	#include <clplumbing/cl_signal.h>
	#include <clplumbing/lsb_exitcodes.h>
	#include <clplumbing/uids.h>
	#include <clplumbing/realtime.h>
	#include <clplumbing/GSource.h>
	#include <clplumbing/cl_poll.h>

	-#include <libxml/tree.h>
	-
	-const char* crm_system_name = CRM_SYSTEM_CRMD;
	-
	-#include <crm/common/crmutils.h>
	-#include <crm/common/ipcutils.h>
	+#include <crm/crm.h>
	+#include <crm/common/ctrl.h>
	+#include <crm/common/ipc.h>
	+#include <crm/common/xml.h>

	#include <crmd.h>
	-
	#include <crmd_fsa.h>

	#include <crm/dmalloc_wrapper.h>

	+const char* crm_system_name = CRM_SYSTEM_CRMD;
	#define PID_FILE WORKING_DIR"/crm.pid"
	#define OPTARGS "skrh"


	void usage(const char* cmd, int exit_status);
	int init_start(void);
	void crmd_hamsg_callback(const struct ha_msg* msg, void* private_data);
	gboolean crmd_tickle_apphb(gpointer data);

	GMainLoop* crmd_mainloop = NULL;
	gboolean crm_debug_state = TRUE;

	int
	main(int argc, char ** argv)
	{

	int req_restart = FALSE;
	int req_status = FALSE;
	int req_stop = FALSE;
	int argerr = 0;
	int flag;

	cl_log_set_entity(crm_system_name);
	cl_log_enable_stderr(TRUE);
	cl_log_set_facility(LOG_USER);

	while ((flag = getopt(argc, argv, OPTARGS)) != EOF) {
	switch(flag) {
	case 's': /* Status */
	req_status = TRUE;
	break;
	case 'k': /* Stop (kill) */
	req_stop = TRUE;
	break;
	case 'r': /* Restart */
	req_restart = TRUE;
	break;
	case 'h': /* Help message */
	usage(crm_system_name, LSB_EXIT_OK);
	break;
	default:
	++argerr;
	break;
	}
	}

	if (optind > argc) {
	++argerr;
	}

	if (argerr) {
	usage(crm_system_name,LSB_EXIT_GENERIC);
	}

	// read local config file

	if (req_status){
	FNRET(init_status(PID_FILE, crm_system_name));
	}

	if (req_stop){
	FNRET(init_stop(PID_FILE));
	}

	if (req_restart) {
	init_stop(PID_FILE);
	}

	FNRET(init_start());
	}


	int
	init_start(void)
	{
	long pid;
	enum crmd_fsa_state state;

	if ((pid = get_running_pid(PID_FILE, NULL)) > 0) {
	cl_log(LOG_CRIT, "already running: [pid %ld].", pid);
	exit(LSB_EXIT_OK);
	}

	fsa_state = S_PENDING;
	state = s_crmd_fsa(C_STARTUP, I_STARTUP, NULL);

	if (state == S_PENDING) {
	/* Create the mainloop and run it... */
	crmd_mainloop = g_main_new(FALSE);
	cl_log(LOG_INFO, "Starting %s", crm_system_name);

	#ifdef REALTIME_SUPPORT
	static int crm_realtime = 1;
	if (crm_realtime == 1){
	cl_enable_realtime();
	}else if (crm_realtime == 0){
	cl_disable_realtime();
	}
	cl_make_realtime(SCHED_RR, 5, 64, 64);
	#endif

	g_main_run(crmd_mainloop);
	return_to_orig_privs();
	} else {

	cl_log(LOG_ERR, "Startup of CRMd failed. Current state: %s",
	fsa_state2string(state));

	}


	if (unlink(PID_FILE) == 0) {
	cl_log(LOG_INFO, "[%s] stopped", crm_system_name);
	}

	FNRET(state != S_PENDING);
	}



	void
	usage(const char* cmd, int exit_status)
	{
	FILE* stream;

	stream = exit_status ? stderr : stdout;

	fprintf(stream, "usage: %s [-srkh]"
	"[-c configure file]\n", cmd);
	/* fprintf(stream, "\t-d\tsets debug level\n"); */
	/* fprintf(stream, "\t-s\tgets daemon status\n"); */
	/* fprintf(stream, "\t-r\trestarts daemon\n"); */
	/* fprintf(stream, "\t-k\tstops daemon\n"); */
	/* fprintf(stream, "\t-h\thelp message\n"); */
	fflush(stream);

	exit(exit_status);
	}


	gboolean
	crmd_tickle_apphb(gpointer data)
	{
	char app_instance[APPNAME_LEN];
	int rc = 0;
	sprintf(app_instance, "%s_%ld", crm_system_name, (long)getpid());

	rc = apphb_hb();
	if (rc < 0) {
	cl_perror("%s apphb_hb failure", app_instance);
	exit(3);
	}
	return TRUE;
	}
	diff --git a/crm/crmd/election.c b/crm/crmd/election.c
	index 9f498069e9..72231f2772 100644
	--- a/crm/crmd/election.c
	+++ b/crm/crmd/election.c
	@@ -1,389 +1,385 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <portability.h>
	-#include <crm/crm.h>
	-#include <crmd_fsa.h>
	-#include <libxml/tree.h>
	-#include <crm/msg_xml.h>

	-#include <crm/common/xmlutils.h>
	-#include <crm/common/ipcutils.h>
	-#include <crm/common/msgutils.h>
	+#include <heartbeat.h>
	+
	#include <crm/cib.h>
	-#include <string.h>
	+#include <crm/msg_xml.h>
	+#include <crm/common/xml.h>
	+#include <crm/crm.h>
	+#include <crmd_fsa.h>
	#include <crmd_messages.h>

	-#include <heartbeat.h>
	-
	#include <crm/dmalloc_wrapper.h>

	void ghash_count_vote(gpointer key, gpointer value, gpointer user_data);

	/* A_ELECTION_VOTE */
	enum crmd_fsa_input
	do_election_vote(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input election_result = I_NULL;
	FNIN();

	/* dont vote if we're in one of these states or wanting to shut down */
	switch(cur_state) {
	case S_RECOVERY:
	case S_RECOVERY_DC:
	case S_STOPPING:
	case S_RELEASE_DC:
	case S_TERMINATE:
	FNRET(I_NULL);
	// log warning
	break;
	default:
	if(is_set(fsa_input_register, R_SHUTDOWN)) {
	FNRET(I_NULL);
	// log warning
	}
	break;
	}

	send_request(NULL, NULL, CRM_OP_VOTE,
	NULL, CRM_SYSTEM_CRMD, NULL);

	FNRET(election_result);
	}

	gboolean
	do_dc_heartbeat(gpointer data)
	{
	fsa_timer_t timer = (fsa_timer_t )data;
	// cl_log(LOG_DEBUG, "#!!#!!# Heartbeat timer just popped!");

	gboolean was_sent = send_request(NULL, NULL, CRM_OP_HBEAT,
	NULL, CRM_SYSTEM_CRMD, NULL);

	if(was_sent == FALSE) {
	// this is bad
	stopTimer(timer); // dont make it go off again
	s_crmd_fsa(C_HEARTBEAT_FAILED, I_SHUTDOWN, NULL);
	}

	return TRUE;
	}

	struct election_data_s
	{
	const char *winning_uname;
	unsigned int winning_bornon;
	};

	/* A_ELECTION_COUNT */
	enum crmd_fsa_input
	do_election_count_vote(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	gboolean we_loose = FALSE;
	xmlNodePtr vote = (xmlNodePtr)data;
	enum crmd_fsa_input election_result = I_NULL;
	const char *vote_from = xmlGetProp(vote, XML_ATTR_HOSTFROM);

	FNIN();

	if(vote_from == NULL \|\| strcmp(vote_from, fsa_our_uname) == 0) {
	// dont count our own vote
	FNRET(election_result);
	}

	if(fsa_membership_copy->members_size < 1) {
	// if even we are not in the cluster then we should not vote
	FNRET(I_FAIL);

	}

	oc_node_t our_node = (oc_node_t)
	g_hash_table_lookup(fsa_membership_copy->members, fsa_our_uname);

	oc_node_t your_node = (oc_node_t)
	g_hash_table_lookup(fsa_membership_copy->members, vote_from);

	#if 0
	cl_log(LOG_DEBUG, "%s (bornon=%d), our bornon (%d)",
	vote_from, our_node->born, my_born);

	cl_log(LOG_DEBUG, "%s %s %s",
	fsa_our_uname,
	strcmp(fsa_our_uname, vote_from) < 0?"<":">=",
	vote_from);
	#endif

	if(is_set(fsa_input_register, R_SHUTDOWN)) {
	cl_log(LOG_DEBUG,
	"Election fail: we are shutting down");
	we_loose = TRUE;

	} else if(our_node == NULL) {
	cl_log(LOG_DEBUG,
	"Election fail: we dont exist in the CCM list");
	we_loose = TRUE;

	} else if(your_node == NULL) {
	cl_log(LOG_ERR, "The other side doesnt exist in the CCM list");

	} else if(your_node->node_born_on < our_node->node_born_on) {
	cl_log(LOG_DEBUG, "Election fail: born_on");
	we_loose = TRUE;

	} else if(your_node->node_born_on == our_node->node_born_on
	&& strcmp(fsa_our_uname, vote_from) > 0) {
	cl_log(LOG_DEBUG, "Election fail: uname");
	we_loose = TRUE;

	} else {
	struct election_data_s election_data;
	election_data.winning_uname = NULL;
	election_data.winning_bornon = -1; // maximum integer

	CRM_NOTE("We might win... we should vote (possibly again)");
	election_result = I_DC_TIMEOUT; // new "default"

	g_hash_table_foreach(fsa_membership_copy->members,
	ghash_count_vote, &election_data);

	cl_log(LOG_DEBUG, "Election winner should be %s (born_on=%d)",
	election_data.winning_uname, election_data.winning_bornon);


	if(safe_str_eq(election_data.winning_uname, fsa_our_uname)){
	cl_log(LOG_DEBUG, "Election win: lowest born_on and uname");
	election_result = I_ELECTION_DC;
	}
	}

	if(we_loose) {
	if(fsa_input_register & R_THE_DC) {
	cl_log(LOG_DEBUG, "Give up the DC");
	election_result = I_RELEASE_DC;

	} else {
	cl_log(LOG_DEBUG, "We werent the DC anyway");
	election_result = I_NOT_DC;

	}
	}

	if(we_loose \|\| election_result == I_ELECTION_DC) {
	// cancel timer, its been decided
	stopTimer(election_timeout);
	}

	FNRET(election_result);
	}

	/* A_ELECT_TIMER_START, A_ELECTION_TIMEOUT */
	// we won
	enum crmd_fsa_input
	do_election_timer_ctrl(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	FNIN();

	if(action & A_ELECT_TIMER_START) {
	startTimer(election_timeout);

	} else if(action & A_ELECT_TIMER_STOP \|\| action & A_ELECTION_TIMEOUT) {
	stopTimer(election_timeout);

	} else {
	cl_log(LOG_ERR, "unexpected action %s",
	fsa_action2string(action));
	}

	if(action & A_ELECTION_TIMEOUT) {
	CRM_NOTE("The election timer went off, we win!");

	FNRET(I_ELECTION_DC);

	}


	FNRET(I_NULL);
	}

	/* A_DC_TIMER_STOP, A_DC_TIMER_START */
	enum crmd_fsa_input
	do_dc_timer_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	gboolean timer_op_ok = TRUE;
	FNIN();

	if(action & A_DC_TIMER_STOP) {
	timer_op_ok = stopTimer(election_trigger);
	}

	/* dont start a timer that wasnt already running */
	if(action & A_DC_TIMER_START && timer_op_ok) {
	startTimer(election_trigger);
	}

	FNRET(I_NULL);
	}


	/* A_DC_TAKEOVER */
	enum crmd_fsa_input
	do_dc_takeover(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	xmlNodePtr update = NULL, fragment = NULL;
	FNIN();

	CRM_NOTE("################## Taking over the DC ##################");
	set_bit_inplace(&fsa_input_register, R_THE_DC);

	CRM_DEBUG("Am I the DC? %s", AM_I_DC?XML_BOOLEAN_YES:XML_BOOLEAN_NO);

	fsa_our_dc = NULL;
	set_bit_inplace(&fsa_input_register, R_JOIN_OK);
	set_bit_inplace(&fsa_input_register, R_INVOKE_PE);

	clear_bit_inplace(&fsa_input_register, R_CIB_DONE);
	clear_bit_inplace(&fsa_input_register, R_HAVE_CIB);

	startTimer(dc_heartbeat);

	if (fsa_cluster_conn->llc_ops->set_cstatus_callback(
	fsa_cluster_conn, CrmdClientStatus, NULL)!=HA_OK){
	cl_log(LOG_ERR, "Cannot set client status callback\n");
	cl_log(LOG_ERR, "REASON: %s\n",
	fsa_cluster_conn->llc_ops->errmsg(fsa_cluster_conn));
	}

	/* store our state in the CIB (since some fields will not be
	* filled in because the DC doesnt go through the join process
	* with itself
	*
	* bypass the TE for now, it will be informed in good time
	*/
	update = create_node_state(
	fsa_our_uname, NULL, ONLINESTATUS, CRMD_JOINSTATE_MEMBER);
	set_xml_property_copy(
	update,XML_CIB_ATTR_EXPSTATE, CRMD_STATE_ACTIVE);

	fragment = create_cib_fragment(update, NULL);
	store_request(NULL, fragment, CRM_OP_UPDATE, CRM_SYSTEM_DCIB);

	free_xml(update);
	free_xml(fragment);

	/* Async get client status information in the cluster */
	fsa_cluster_conn->llc_ops->client_status(
	fsa_cluster_conn, NULL, CRM_SYSTEM_CRMD, -1);

	FNRET(I_NULL);
	}

	/* A_DC_RELEASE */
	enum crmd_fsa_input
	do_dc_release(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input result = I_NULL;
	FNIN();

	CRM_NOTE("################## Releasing the DC ##################");

	stopTimer(dc_heartbeat);
	if (fsa_cluster_conn->llc_ops->set_cstatus_callback(
	fsa_cluster_conn, NULL, NULL)!=HA_OK){
	cl_log(LOG_ERR, "Cannot unset client status callback\n");
	cl_log(LOG_ERR, "REASON: %s\n",
	fsa_cluster_conn->llc_ops->errmsg(fsa_cluster_conn));
	result = I_ERROR;
	}

	if(action & A_DC_RELEASE) {
	clear_bit_inplace(&fsa_input_register, R_THE_DC);

	/* get a new CIB from the new DC */
	clear_bit_inplace(&fsa_input_register, R_HAVE_CIB);
	} else if (action & A_DC_RELEASED) {

	if(cur_state == S_STOPPING) {
	result = I_SHUTDOWN; // necessary?
	result = I_RELEASE_SUCCESS;
	}
	#if 0
	else if( are there errors ) {
	// we cant stay up if not healthy
	// or perhaps I_ERROR and go to S_RECOVER?
	result = I_SHUTDOWN;
	}
	#endif
	else
	result = I_RELEASE_SUCCESS;

	} else {
	cl_log(LOG_ERR, "Warning, do_dc_release invoked for action %s",
	fsa_action2string(action));
	}

	CRM_DEBUG("Am I still the DC? %s", AM_I_DC?XML_BOOLEAN_YES:XML_BOOLEAN_NO);

	FNRET(result);
	}

	void
	ghash_count_vote(gpointer key, gpointer value, gpointer user_data)
	{

	struct election_data_s *election_data =
	(struct election_data_s *)user_data;

	oc_node_t cur_node = (oc_node_t)value;
	const char node_uname = (const char)key;

	if(election_data->winning_bornon > cur_node->node_born_on) {
	election_data->winning_uname = node_uname;
	election_data->winning_bornon = cur_node->node_born_on;

	} else if(election_data->winning_bornon == cur_node->node_born_on
	&& (election_data->winning_uname == NULL
	\|\| strcmp(election_data->winning_uname, node_uname) > 0)) {
	election_data->winning_uname = node_uname;
	election_data->winning_bornon = cur_node->node_born_on;

	}
	}

	diff --git a/crm/crmd/fsa.c b/crm/crmd/fsa.c
	index c9b19922cf..478e9e26f8 100644
	--- a/crm/crmd/fsa.c
	+++ b/crm/crmd/fsa.c
	@@ -1,602 +1,602 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <portability.h>
	-#include <crm/crm.h>
	-#include <crmd_fsa.h>
	-#include <fsa_matrix.h>
	-#include <fsa_proto.h>
	#include <stdio.h>
	+#include <string.h>
	+#include <time.h>

	-#include <crm/common/xmlutils.h>
	-#include <crm/common/msgutils.h>
	+#include <crm/crm.h>
	#include <crm/msg_xml.h>
	+#include <crm/common/xml.h>
	+#include <crm/common/msg.h>

	#include <clplumbing/Gmain_timeout.h>

	#include <crmd_messages.h>
	-#include <string.h>
	-#include <time.h>
	+#include <crmd_fsa.h>
	+#include <fsa_proto.h>
	+#include <fsa_matrix.h>

	#include <crm/dmalloc_wrapper.h>

	long long
	do_state_transition(long long actions,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_state next_state,
	enum crmd_fsa_input current_input,
	void *data);

	// delete this
	extern fsa_message_queue_t fsa_message_queue;

	#ifdef DOT_FSA_ACTIONS
	# ifdef FSA_TRACE
	# define IF_FSA_ACTION(x,y) \
	if(is_set(actions,x)) { \
	CRM_DEBUG("Invoking action %s (%.16llx)", \
	fsa_action2string(x), x); \
	last_action = x; \
	actions = clear_bit(actions, x); \
	next_input = y(x, cause, cur_state, last_input, data); \
	if( (x & O_DC_TICKLE) == 0 && next_input != I_DC_HEARTBEAT ) \
	fprintf(dot_strm, \
	"\t// %s:\t%s\t(data? %s)\t(result=%s)\n", \
	fsa_input2string(cur_input), \
	fsa_action2string(x), \
	data==NULL?XML_BOOLEAN_NO:XML_BOOLEAN_YES, \
	fsa_input2string(next_input)); \
	fflush(dot_strm); \
	CRM_DEBUG("Result of action %s was %s", \
	fsa_action2string(x), fsa_input2string(next_input)); \
	}
	# else
	# define IF_FSA_ACTION(x,y) \
	if(is_set(actions,x)) { \
	last_action = x; \
	actions = clear_bit(actions, x); \
	next_input = y(x, cause, cur_state, last_input, data); \
	if( (x & O_DC_TICKLE) == 0 && next_input != I_DC_HEARTBEAT ) \
	fprintf(dot_strm, \
	"\t// %s:\t%s\t(data? %s)\t(result=%s)\n", \
	fsa_input2string(cur_input), \
	fsa_action2string(x), \
	data==NULL?XML_BOOLEAN_NO:XML_BOOLEAN_YES, \
	fsa_input2string(next_input)); \
	fflush(dot_strm); \
	}
	# endif
	#else
	# ifdef FSA_TRACE
	# define IF_FSA_ACTION(x,y) \
	if(is_set(actions,x)) { \
	CRM_DEBUG("Invoking action %s (%.16llx)", \
	fsa_action2string(x), x); \
	last_action = x; \
	actions = clear_bit(actions, x); \
	next_input = y(x, cause, cur_state, last_input, data); \
	CRM_DEBUG("Result of action %s was %s", \
	fsa_action2string(x), fsa_input2string(next_input)); \
	}
	# else
	# define IF_FSA_ACTION(x,y) \
	if(is_set(actions,x)) { \
	last_action = x; \
	actions = clear_bit(actions, x); \
	next_input = y(x, cause, cur_state, last_input, data); \
	}
	# endif
	#endif

	#define ELSEIF_FSA_ACTION(x,y) else IF_FSA_ACTION(x,y)

	const char *dot_intro = "digraph \"g\" {\n"
	" size = \"30,30\"\n"
	" graph [\n"
	" fontsize = \"12\"\n"
	" fontname = \"Times-Roman\"\n"
	" fontcolor = \"black\"\n"
	" bb = \"0,0,398.922306,478.927856\"\n"
	" color = \"black\"\n"
	" ]\n"
	" node [\n"
	" fontsize = \"12\"\n"
	" fontname = \"Times-Roman\"\n"
	" fontcolor = \"black\"\n"
	" shape = \"ellipse\"\n"
	" color = \"black\"\n"
	" ]\n"
	" edge [\n"
	" fontsize = \"12\"\n"
	" fontname = \"Times-Roman\"\n"
	" fontcolor = \"black\"\n"
	" color = \"black\"\n"
	" ]\n"
	"// special nodes\n"
	" \"S_PENDING\" \n"
	" [\n"
	" color = \"blue\"\n"
	" fontcolor = \"blue\"\n"
	" ]\n"
	" \"S_TERMINATE\" \n"
	" [\n"
	" color = \"red\"\n"
	" fontcolor = \"red\"\n"
	" ]\n"
	"\n"
	"// DC only nodes\n"
	" \"S_RECOVERY_DC\" [ fontcolor = \"green\" ]\n"
	" \"S_INTEGRATION\" [ fontcolor = \"green\" ]\n"
	" \"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]\n"
	" \"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]\n"
	" \"S_RELEASE_DC\" [ fontcolor = \"green\" ]\n"
	" \"S_IDLE\" [ fontcolor = \"green\" ]\n";


	static FILE *dot_strm = NULL;

	enum crmd_fsa_state fsa_state;
	oc_node_list_t *fsa_membership_copy;
	ll_cluster_t *fsa_cluster_conn;
	ll_lrm_t *fsa_lrm_conn;
	long long fsa_input_register;
	long long fsa_actions = A_NOTHING;
	const char *fsa_our_uname;
	const char *fsa_our_dc;

	fsa_timer_t election_trigger = NULL; / */
	fsa_timer_t election_timeout = NULL; / */
	fsa_timer_t shutdown_escalation_timmer = NULL; / */
	fsa_timer_t *integration_timer = NULL;
	fsa_timer_t *dc_heartbeat = NULL;

	enum crmd_fsa_state
	s_crmd_fsa(enum crmd_fsa_cause cause,
	enum crmd_fsa_input initial_input,
	void *data)
	{
	long long actions = fsa_actions;
	long long new_actions = A_NOTHING;
	long long last_action = A_NOTHING;
	enum crmd_fsa_input last_input = initial_input;
	enum crmd_fsa_input cur_input;
	enum crmd_fsa_input next_input;
	enum crmd_fsa_state last_state, cur_state, next_state, starting_state;

	FNIN();

	starting_state = fsa_state;
	cur_input = initial_input;
	next_input = initial_input;

	last_state = starting_state;
	cur_state = starting_state;
	next_state = starting_state;

	#ifdef FSA_TRACE
	CRM_DEBUG("FSA invoked with Cause: %s\n\tState: %s, Input: %s",
	fsa_cause2string(cause),
	fsa_state2string(cur_state),
	fsa_input2string(cur_input));
	#endif

	#ifdef DOT_FSA_ACTIONS
	if(dot_strm == NULL) {
	dot_strm = fopen("/tmp/live.dot", "w");
	fprintf(dot_strm, "%s", dot_intro);
	}
	#endif
	/*
	* Process actions in order of priority but do only one
	* action at a time to avoid complicating the ordering.
	*
	* Actions may result in a new I_ event, these are added to
	* (not replace) existing actions before the next iteration.
	*
	*/
	while(next_input != I_NULL \|\| actions != A_NOTHING \|\| is_message()) {

	if(next_input == I_WAIT_FOR_EVENT) {
	/* we may be waiting for an a-sync task to "happen"
	* and until it does, we cant do anything else
	*
	* Re-add the last action
	*/

	actions \|= last_action;

	cl_log(LOG_INFO, "Wait until something else happens");
	break;
	}

	#ifdef FSA_TRACE
	CRM_DEBUG("FSA while loop:\tState: %s, Input: %s",
	fsa_state2string(cur_state),
	fsa_input2string(cur_input));
	#endif

	/* update input variables */
	cur_input = next_input;
	if(cur_input != I_NULL) {
	last_input = cur_input;
	}

	/* get the next batch of actions */
	new_actions = crmd_fsa_actions[cur_input][cur_state];
	if(new_actions != A_NOTHING) {
	#ifdef FSA_TRACE
	CRM_DEBUG("Adding actions %.16llx", new_actions);
	#endif
	actions \|= new_actions;
	}

	/* logging : before the state is changed */
	IF_FSA_ACTION(A_ERROR, do_log)
	ELSEIF_FSA_ACTION(A_WARN, do_log)
	ELSEIF_FSA_ACTION(A_LOG, do_log)

	/* update state variables */
	next_state = crmd_fsa_state[cur_input][cur_state];
	last_state = cur_state;
	cur_state = next_state;
	fsa_state = next_state;

	/* start doing things... */


	/*
	* Hook for change of state.
	* Allows actions to be added or removed when entering a state
	*/
	if(last_state != cur_state){
	actions = do_state_transition(actions, cause,
	last_state, cur_state,
	last_input, data);
	}

	/* this is always run, some inputs/states may make various
	* actions irrelevant/invalid
	*/
	actions = clear_flags(actions, cause, cur_state, cur_input);

	/* regular action processing in order of action priority
	*
	* Make sure all actions that connect to required systems
	* are performed first
	*/
	if(actions == A_NOTHING) {

	cl_log(LOG_INFO, "Nothing to do");
	next_input = I_NULL;

	/* // check registers, see if anything is pending
	if(is_set(fsa_input_register, R_SHUTDOWN)) {
	CRM_DEBUG("(Re-)invoking shutdown");
	next_input = I_SHUTDOWN;
	} else if(is_set(fsa_input_register, R_INVOKE_PE)) {
	CRM_DEBUG("Invoke the PE somehow");
	}
	*/
	}


	/* get out of here NOW! before anything worse happens */
	ELSEIF_FSA_ACTION(A_EXIT_1, do_exit)

	ELSEIF_FSA_ACTION(A_STARTUP, do_startup)

	ELSEIF_FSA_ACTION(A_CIB_START, do_cib_control)
	ELSEIF_FSA_ACTION(A_HA_CONNECT, do_ha_control)
	ELSEIF_FSA_ACTION(A_LRM_CONNECT,do_lrm_control)
	ELSEIF_FSA_ACTION(A_CCM_CONNECT,do_ccm_control)

	/* sub-system start */
	ELSEIF_FSA_ACTION(A_TE_START, do_te_control)
	ELSEIF_FSA_ACTION(A_PE_START, do_pe_control)

	/* sub-system restart
	*/
	ELSEIF_FSA_ACTION(O_CIB_RESTART,do_cib_control)
	ELSEIF_FSA_ACTION(O_PE_RESTART, do_pe_control)
	ELSEIF_FSA_ACTION(O_TE_RESTART, do_te_control)

	ELSEIF_FSA_ACTION(A_STARTED, do_started)

	/* DC Timer */
	ELSEIF_FSA_ACTION(O_DC_TIMER_RESTART, do_dc_timer_control)
	ELSEIF_FSA_ACTION(A_DC_TIMER_STOP, do_dc_timer_control)
	ELSEIF_FSA_ACTION(A_DC_TIMER_START, do_dc_timer_control)

	/*
	* Highest priority actions
	*/
	ELSEIF_FSA_ACTION(A_CIB_BUMPGEN, do_cib_invoke)
	ELSEIF_FSA_ACTION(A_TE_COPYTO, do_te_copyto)
	ELSEIF_FSA_ACTION(A_MSG_ROUTE, do_msg_route)
	ELSEIF_FSA_ACTION(A_RECOVER, do_recover)
	ELSEIF_FSA_ACTION(A_UPDATE_NODESTATUS, do_lrm_invoke)
	ELSEIF_FSA_ACTION(A_JOIN_ACK, do_ack_welcome)
	ELSEIF_FSA_ACTION(A_SHUTDOWN_REQ, do_shutdown_req)
	ELSEIF_FSA_ACTION(A_ELECTION_VOTE, do_election_vote)
	ELSEIF_FSA_ACTION(A_ELECT_TIMER_STOP, do_election_timer_ctrl)
	ELSEIF_FSA_ACTION(A_ELECT_TIMER_START, do_election_timer_ctrl)
	ELSEIF_FSA_ACTION(A_ELECTION_COUNT, do_election_count_vote)
	ELSEIF_FSA_ACTION(A_ELECTION_TIMEOUT, do_election_timer_ctrl)

	/*
	* "Get this over with" actions
	*/
	ELSEIF_FSA_ACTION(A_MSG_STORE, do_msg_store)

	/*
	* High priority actions
	* Update the cache first
	*/
	ELSEIF_FSA_ACTION(A_CCM_UPDATE_CACHE, do_ccm_update_cache)
	ELSEIF_FSA_ACTION(A_CCM_EVENT, do_ccm_event)

	/*
	* Medium priority actions
	*/
	ELSEIF_FSA_ACTION(A_DC_TAKEOVER, do_dc_takeover)
	ELSEIF_FSA_ACTION(A_DC_RELEASE, do_dc_release)
	ELSEIF_FSA_ACTION(A_JOIN_WELCOME_ALL, do_send_welcome_all)
	ELSEIF_FSA_ACTION(A_JOIN_WELCOME, do_send_welcome)
	ELSEIF_FSA_ACTION(A_JOIN_PROCESS_ACK, do_process_welcome_ack)

	/*
	* Low(er) priority actions
	* Make sure the CIB is always updated before invoking the
	* PE, and the PE before the TE
	*/
	ELSEIF_FSA_ACTION(A_CIB_INVOKE_LOCAL, do_cib_invoke)
	ELSEIF_FSA_ACTION(A_CIB_INVOKE, do_cib_invoke)
	ELSEIF_FSA_ACTION(A_LRM_INVOKE, do_lrm_invoke)
	ELSEIF_FSA_ACTION(A_LRM_EVENT, do_lrm_event)
	ELSEIF_FSA_ACTION(A_TE_CANCEL, do_te_invoke)
	ELSEIF_FSA_ACTION(A_PE_INVOKE, do_pe_invoke)
	ELSEIF_FSA_ACTION(A_TE_INVOKE, do_te_invoke)
	ELSEIF_FSA_ACTION(A_ANNOUNCE, do_announce)

	/* sub-system stop */
	ELSEIF_FSA_ACTION(A_PE_STOP, do_pe_control)
	ELSEIF_FSA_ACTION(A_TE_STOP, do_te_control)
	ELSEIF_FSA_ACTION(A_DC_RELEASED, do_dc_release)

	ELSEIF_FSA_ACTION(A_HA_DISCONNECT, do_ha_control)
	ELSEIF_FSA_ACTION(A_CCM_DISCONNECT, do_ccm_control)
	ELSEIF_FSA_ACTION(A_LRM_DISCONNECT, do_lrm_control)
	ELSEIF_FSA_ACTION(A_CIB_STOP, do_cib_control)
	/* time to go now... */

	/* Some of these can probably be consolidated */
	ELSEIF_FSA_ACTION(A_SHUTDOWN, do_shutdown)
	ELSEIF_FSA_ACTION(A_STOP, do_stop)

	/* exit gracefully */
	ELSEIF_FSA_ACTION(A_EXIT_0, do_exit)

	// ELSEIF_FSA_ACTION(A_, do_)

	else if((actions & A_MSG_PROCESS) != 0
	\|\| is_message()) {
	xmlNodePtr stored_msg = NULL;
	crm_debug("Checking messages... %d",
	g_slist_length(fsa_message_queue));

	stored_msg = get_message();

	if(is_message() == FALSE) {
	actions = clear_bit(actions, A_MSG_PROCESS);
	}

	if(stored_msg == NULL) {
	cl_log(LOG_ERR, "Invalid stored message");
	continue;
	}

	/*
	* This is where we should clean up old messages
	* The problem is that we dont always know the
	* type of the data (and therefore the correct way
	* to free it). A wrapper is probably required.
	*/
	data = stored_msg;

	#ifdef DOT_FSA_ACTIONS
	fprintf(dot_strm,
	"\t// %s:\t%s\t(data? %s)",
	fsa_input2string(cur_input),
	fsa_action2string(A_MSG_PROCESS),
	stored_msg==NULL?XML_BOOLEAN_NO:XML_BOOLEAN_YES);
	fflush(dot_strm);
	#endif
	#ifdef FSA_TRACE
	CRM_DEBUG("Invoking action %s (%.16llx)",
	fsa_action2string(A_MSG_PROCESS),
	A_MSG_PROCESS);
	#endif

	//#ifdef FSA_TRACE
	xml_message_debug(stored_msg,"FSA processing message");
	//#endif

	next_input = handle_message(stored_msg);

	#ifdef DOT_FSA_ACTIONS
	fprintf(dot_strm, "\t(result=%s)\n",
	fsa_input2string(next_input));
	#endif
	CRM_DEBUG("Result of action %s was %s",
	fsa_action2string(A_MSG_PROCESS),
	fsa_input2string(next_input));

	/* Error checking and reporting */
	} else if(cur_input != I_NULL && is_set(actions, A_NOTHING)) {
	cl_log(LOG_WARNING,
	"No action specified for input,state (%s,%s)",
	fsa_input2string(cur_input),
	fsa_state2string(cur_state));

	next_input = I_NULL;

	} else if(cur_input == I_NULL && is_set(actions, A_NOTHING)) {
	#ifdef FSA_TRACE
	cl_log(LOG_INFO, "Nothing left to do");
	#endif
	} else {
	cl_log(LOG_ERR, "Action %s (0x%llx) not supported ",
	fsa_action2string(actions), actions);
	next_input = I_ERROR;
	}

	if(is_message()) {
	actions \|= A_MSG_PROCESS;
	}
	}

	#ifdef FSA_TRACE
	CRM_DEBUG("################# Exiting the FSA (%s) ##################",
	fsa_state2string(fsa_state));
	#endif

	#ifdef DOT_FSA_ACTIONS
	fprintf(dot_strm,
	"\t// ### Exiting the FSA (%s)\n",
	fsa_state2string(fsa_state));
	fflush(dot_strm);
	#endif

	// cleanup inputs?
	fsa_actions = actions;

	FNRET(fsa_state);
	}




	long long
	do_state_transition(long long actions,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_state next_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	long long tmp = actions;
	const char *state_from = fsa_state2string(cur_state);
	const char *state_to = fsa_state2string(next_state);
	const char *input = fsa_input2string(current_input);

	time_t now = time(NULL);

	if(cur_state == next_state) {
	cl_log(LOG_ERR,
	"%s called in state %s with no transtion",
	__FUNCTION__, state_from);
	return A_NOTHING;
	}


	// if(current_input != I_NULL
	// && (current_input != I_DC_HEARTBEAT \|\| cur_state != S_NOT_DC)){

	fprintf(dot_strm,
	"\t\"%s\" -> \"%s\" [ label =\"%s\" ] // %s",
	state_from, state_to, input,
	asctime(localtime(&now)));
	fflush(dot_strm);
	//}

	cl_log(LOG_INFO,
	"State transition \"%s\" -> \"%s\" [ cause =\"%s\" %s ]",
	state_from, state_to, input, asctime(localtime(&now)));

	switch(next_state) {
	case S_PENDING:
	break;
	case S_NOT_DC:
	if(is_set(fsa_input_register, R_SHUTDOWN)){
	cl_log(LOG_INFO,
	"(Re)Issuing shutdown request now"
	" that we have a new DC");
	tmp = set_bit(tmp, A_SHUTDOWN_REQ);
	}
	tmp = clear_bit(tmp, A_RECOVER);
	break;
	case S_RECOVERY_DC:
	case S_RECOVERY:
	tmp = set_bit(tmp, A_RECOVER);
	break;
	default:
	tmp = clear_bit(tmp, A_RECOVER);
	break;
	}

	if(tmp != actions) {
	cl_log(LOG_INFO, "Action b4 %.16llx ", actions);
	cl_log(LOG_INFO, "Action after %.16llx ", tmp);
	actions = tmp;
	}

	return actions;
	}

	long long
	clear_flags(long long actions,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input cur_input)
	{

	if(is_set(fsa_input_register, R_SHUTDOWN)){
	clear_bit_inplace(&actions, A_DC_TIMER_START);
	}


	switch(cur_state) {
	case S_IDLE:
	break;
	case S_ELECTION:
	break;
	case S_INTEGRATION:
	break;
	case S_NOT_DC:
	break;
	case S_POLICY_ENGINE:
	break;
	case S_RECOVERY:
	break;
	case S_RECOVERY_DC:
	break;
	case S_RELEASE_DC:
	break;
	case S_PENDING:
	break;
	case S_STOPPING:
	break;
	case S_TERMINATE:
	break;
	case S_TRANSITION_ENGINE:
	break;
	case S_ILLEGAL:
	break;
	}
	return actions;
	}
	diff --git a/crm/crmd/join.c b/crm/crmd/join.c
	index dd801b6c27..6f3250531e 100644
	--- a/crm/crmd/join.c
	+++ b/crm/crmd/join.c
	@@ -1,368 +1,365 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <portability.h>
	+
	+#include <heartbeat.h>
	+
	#include <crm/crm.h>
	-#include <crmd_fsa.h>
	-#include <libxml/tree.h>
	+#include <crm/cib.h>
	#include <crm/msg_xml.h>
	+#include <crm/common/xml.h>

	-#include <crm/common/xmlutils.h>
	-#include <crm/common/ipcutils.h>
	-#include <crm/common/msgutils.h>
	-#include <crm/cib.h>
	-#include <string.h>
	+#include <crmd_fsa.h>
	#include <crmd_messages.h>

	-#include <heartbeat.h>
	-
	#include <crm/dmalloc_wrapper.h>

	GHashTable *joined_nodes = NULL;

	/* A_JOIN_WELCOME, A_JOIN_WELCOME_ALL */
	enum crmd_fsa_input
	do_send_welcome(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	FNIN();

	if(action & A_JOIN_WELCOME && data == NULL) {
	cl_log(LOG_ERR,
	"Attempt to send welcome message "
	"without a message to reply to!");
	FNRET(I_NULL);

	} else if(action & A_JOIN_WELCOME) {
	xmlNodePtr welcome = (xmlNodePtr)data;

	const char *join_to = xmlGetProp(welcome, XML_ATTR_HOSTFROM);
	if(join_to != NULL) {

	xmlNodePtr update = create_node_state(
	join_to, NULL, NULL, CRMD_JOINSTATE_PENDING);

	xmlNodePtr tmp1 = create_cib_fragment(update, NULL);
	store_request(NULL, tmp1, CRM_OP_UPDATE, CRM_SYSTEM_DCIB);

	send_request(NULL, NULL, CRM_OP_WELCOME,
	join_to, CRM_SYSTEM_CRMD, NULL);

	free_xml(update);
	free_xml(tmp1);

	} else {
	cl_log(LOG_ERR, "No recipient for welcome message");
	}

	FNRET(I_NULL);

	}
	FNRET(I_ERROR);
	}

	// welcome everyone...

	enum crmd_fsa_input
	do_send_welcome_all(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	FNIN();

	// reset everyones status back to down or in_ccm in the CIB
	xmlNodePtr update = NULL;
	xmlNodePtr cib_copy = get_cib_copy();
	xmlNodePtr tmp1 = get_object_root(XML_CIB_TAG_STATUS, cib_copy);
	xmlNodePtr node_entry = tmp1->children;

	/* Give everyone a chance to join before invoking the PolicyEngine */
	stopTimer(integration_timer);
	startTimer(integration_timer);

	if(joined_nodes != NULL) {
	g_hash_table_destroy(joined_nodes);
	joined_nodes = g_hash_table_new(&g_str_hash, &g_str_equal);

	}

	// catch any nodes that are active in the CIB but not in the CCM list
	while(node_entry != NULL){
	const char *node_id = xmlGetProp(node_entry, XML_ATTR_ID);

	gpointer a_node =
	g_hash_table_lookup(fsa_membership_copy->members,
	node_id);

	node_entry = node_entry->next;

	if(a_node != NULL \|\| (safe_str_eq(fsa_our_uname, node_id))) {
	/* handled by do_update_cib_node() */
	continue;
	}

	tmp1 = create_node_state(node_id, XML_BOOLEAN_NO, NULL, CRMD_JOINSTATE_DOWN);

	if(update == NULL) {
	update = tmp1;
	} else {
	update = xmlAddSibling(update, tmp1);
	}
	}

	// now process the CCM data
	free_xml(do_update_cib_nodes(update, TRUE));
	free_xml(cib_copy);

	/* Avoid ordered message delays caused when the CRMd proc
	* isnt running yet (ie. send as a broadcast msg which are never
	* sent ordered.
	*/
	send_request(NULL, NULL, CRM_OP_WELCOME,
	NULL, CRM_SYSTEM_CRMD, NULL);

	/* No point hanging around in S_INTEGRATION if we're the only ones here! */
	if(g_hash_table_size(joined_nodes)
	== fsa_membership_copy->members_size) {
	// that was the last outstanding join ack)
	cl_log(LOG_INFO,"That was the last outstanding join ack");
	FNRET(I_SUCCESS);

	} else {
	cl_log(LOG_DEBUG,
	"Still waiting on %d outstanding join acks",
	fsa_membership_copy->members_size - g_hash_table_size(joined_nodes));
	// dont waste time by invoking the pe yet;
	}

	FNRET(I_NULL);
	}


	/* A_JOIN_ACK */
	enum crmd_fsa_input
	do_ack_welcome(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	xmlNodePtr welcome = (xmlNodePtr)data;
	xmlNodePtr cib_copy;
	xmlNodePtr tmp1;
	xmlNodePtr tmp2;

	FNIN();

	#if 0
	if(we are sick) {
	log error ;
	FNRET(I_NULL);
	}
	#endif
	fsa_our_dc = xmlGetProp(welcome, XML_ATTR_HOSTFROM);

	if(fsa_our_dc == NULL) {
	cl_log(LOG_ERR, "Failed to determin our DC");
	FNRET(I_FAIL);
	}

	/* send our status section to the DC */
	cib_copy = get_cib_copy();
	tmp1 = get_object_root(XML_CIB_TAG_STATUS, cib_copy);
	tmp2 = create_cib_fragment(tmp1, NULL);

	send_ha_reply(fsa_cluster_conn, welcome, tmp2);

	free_xml(tmp2);
	free_xml(cib_copy);

	FNRET(I_NULL);
	}

	/* A_ANNOUNCE */
	enum crmd_fsa_input
	do_announce(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	xmlNodePtr msg = (xmlNodePtr)data;
	FNIN();

	/* Once we hear from the DC, we can stop the timer
	*
	* This timer was started either on startup or when a node
	* left the CCM list
	*/

	/* dont announce if we're in one of these states */
	switch(cur_state) {
	case S_RECOVERY:
	case S_RECOVERY_DC:
	case S_RELEASE_DC:
	case S_TERMINATE:
	cl_log(LOG_WARNING,
	"Do not announce ourselves in state %s",
	fsa_state2string(cur_state));
	FNRET(I_NULL);
	break;
	default:
	break;
	}

	if(AM_I_OPERATIONAL) {
	const char *from = xmlGetProp(msg, XML_ATTR_HOSTFROM);

	if(from == NULL) {
	cl_log(LOG_ERR, "Failed to origin of ping message");
	FNRET(I_FAIL);
	}

	send_request(NULL, NULL, CRM_OP_ANNOUNCE,
	from, CRM_SYSTEM_DC, NULL);
	} else {
	/* Delay announce until we have finished local startup */
	cl_log(LOG_WARNING,
	"Delaying announce until local startup is complete");
	FNRET(I_NULL);
	}

	FNRET(I_NULL);
	}


	/* A_JOIN_PROCESS_ACK */
	enum crmd_fsa_input
	do_process_welcome_ack(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	xmlNodePtr tmp1;
	xmlNodePtr tmp2;
	xmlNodePtr cib_fragment;
	xmlNodePtr msg_cib;
	xmlNodePtr join_ack = (xmlNodePtr)data;

	int size = 0;
	gboolean is_a_member = FALSE;
	const char *join_from = xmlGetProp(join_ack, XML_ATTR_HOSTFROM);
	const char *ref = xmlGetProp(join_ack, XML_ATTR_REFERENCE);

	FNIN();

	gpointer join_node =
	g_hash_table_lookup(fsa_membership_copy->members, join_from);

	if(join_node != NULL) {
	is_a_member = TRUE;
	}

	cib_fragment = find_xml_node(join_ack, XML_TAG_FRAGMENT);

	if(is_a_member == FALSE) {
	cl_log(LOG_ERR, "Node %s is not known to us (ref %s)",
	join_from, ref);

	/* make sure any information from this node is discarded,
	* it is invalid
	*/
	free_xml(cib_fragment);
	FNRET(I_FAIL);
	}

	cl_log(LOG_DEBUG, "Welcoming node %s after ACK (ref %s)",
	join_from, ref);

	/* add them to our list of CRMD_STATE_ACTIVE nodes
	TODO: still used?
	*/
	g_hash_table_insert(joined_nodes, strdup(join_from),strdup(join_from));

	if(cib_fragment == NULL) {
	cl_log(LOG_ERR,
	"No status information was part of the"
	" Welcome ACK from %s",
	join_from);
	FNRET(I_NULL);
	}

	/* make sure a node entry exists for the new node
	*
	* this will add anyone except the first ever node in the cluster
	* since it will also be the DC which doesnt go through the
	* join process (with itself). We can include a special case
	* later if desired.
	*/
	tmp1 = create_xml_node(NULL, XML_CIB_TAG_NODE);
	set_xml_property_copy(tmp1, XML_ATTR_ID, join_from);
	set_xml_property_copy(tmp1, "uname", join_from);
	set_xml_property_copy(tmp1, XML_ATTR_TYPE, "node");

	tmp2 = create_cib_fragment(tmp1, NULL);

	/* do not forward this to the TE */
	invoke_local_cib(NULL, tmp2, CRM_OP_UPDATE);

	free_xml(tmp2);
	free_xml(tmp1);


	/* Make changes so that exp_state=active for this node when the update
	* is processed by A_CIB_INVOKE
	*/
	msg_cib = find_xml_node(cib_fragment, XML_TAG_CIB);
	tmp1 = get_object_root(XML_CIB_TAG_STATUS, msg_cib);
	tmp2 = find_entity(tmp1, XML_CIB_TAG_STATE, join_from, FALSE);

	if(tmp2 == NULL) {
	cl_log(LOG_ERR,
	"Status entry for %s not found in update, adding",
	join_from);

	tmp2 = create_xml_node(tmp1, XML_CIB_TAG_STATE);
	set_xml_property_copy(tmp2, XML_ATTR_ID, join_from);
	}

	set_xml_property_copy(tmp2, XML_CIB_ATTR_EXPSTATE, CRMD_STATE_ACTIVE);
	set_xml_property_copy(tmp2, XML_CIB_ATTR_JOINSTATE, CRMD_JOINSTATE_MEMBER);


	if(g_hash_table_size(joined_nodes)
	== fsa_membership_copy->members_size) {
	cl_log(LOG_INFO,"That was the last outstanding join ack");
	FNRET(I_SUCCESS);
	/* The update isnt lost, the A_CIB_OP action is part of the
	* matrix for S_INTEGRATION + I_SUCCESS.
	*/

	} else {
	cl_log(LOG_DEBUG,
	"Still waiting on %d outstanding join acks",
	size);
	/* dont waste time by invoking the pe yet */
	}
	FNRET(I_CIB_OP);
	}
	diff --git a/crm/crmd/lrm.c b/crm/crmd/lrm.c
	index 395e05994b..11ffeefac3 100644
	--- a/crm/crmd/lrm.c
	+++ b/crm/crmd/lrm.c
	@@ -1,624 +1,609 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <portability.h>
	#include <crm/crm.h>
	#include <crmd_fsa.h>

	#include <sys/types.h>
	#include <sys/wait.h>

	#include <unistd.h> // for access
	#include <clplumbing/cl_signal.h>
	-#include <clplumbing/realtime.h>
	-#include <sys/types.h> // for calls to open
	-#include <sys/stat.h> // for calls to open
	-#include <fcntl.h> // for calls to open
	-#include <pwd.h> // for getpwuid
	-#include <grp.h> // for initgroups
	-
	-#include <sys/time.h> // for getrlimit
	-#include <sys/resource.h>// for getrlimit
	-
	-#include <crm/common/crmutils.h>
	-#include <crm/common/ipcutils.h>
	-#include <crm/common/msgutils.h>
	-
	-#include <crm/msg_xml.h>
	-#include <crm/common/xmlutils.h>

	+#include <errno.h>

	#include <crm/cib.h>
	+#include <crm/msg_xml.h>
	+#include <crm/common/xml.h>

	#include <crmd.h>
	#include <crmd_messages.h>
	-#include <string.h>
	-#include <errno.h>

	#include <crm/dmalloc_wrapper.h>

	xmlNodePtr do_lrm_query(void);

	GHashTable xml2list(xmlNodePtr parent, const char *attr_path, int depth);

	gboolean lrm_dispatch(int fd, gpointer user_data);

	void do_update_resource(lrm_rsc_t *rsc,
	int status,
	int rc,
	const char *op_type);

	/* A_LRM_CONNECT */
	enum crmd_fsa_input
	do_lrm_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input failed = I_NULL;//I_FAIL;
	int ret = HA_OK;
	FNIN();

	if(action & A_LRM_DISCONNECT) {
	fsa_lrm_conn->lrm_ops->signoff(fsa_lrm_conn);
	}

	if(action & A_LRM_CONNECT) {

	CRM_NOTE("LRM: connect...");
	fsa_lrm_conn = ll_lrm_new(XML_CIB_TAG_LRM);
	if(NULL == fsa_lrm_conn) {
	return failed;
	}

	CRM_NOTE("LRM: sigon...");
	ret = fsa_lrm_conn->lrm_ops->signon(fsa_lrm_conn,
	CRM_SYSTEM_CRMD);

	if(ret != HA_OK) {
	cl_log(LOG_ERR, "Failed to sign on to the LRM");
	return failed;
	}

	CRM_NOTE("LRM: set_lrm_callback...");
	ret = fsa_lrm_conn->lrm_ops->set_lrm_callback(
	fsa_lrm_conn, lrm_op_callback, lrm_monitor_callback);

	if(ret != HA_OK) {
	cl_log(LOG_ERR, "Failed to set LRM callbacks");
	return failed;
	}

	/* TODO: create a destroy handler that causes
	* some recovery to happen
	*/
	G_main_add_fd(G_PRIORITY_LOW,
	fsa_lrm_conn->lrm_ops->inputfd(fsa_lrm_conn),
	FALSE,
	lrm_dispatch, fsa_lrm_conn,
	default_ipc_input_destroy);
	}

	if(action & ~(A_LRM_CONNECT\|A_LRM_DISCONNECT)) {
	cl_log(LOG_ERR, "Unexpected action %s in %s",
	fsa_action2string(action), __FUNCTION__);
	}


	FNRET(I_NULL);
	}

	gboolean lrm_dispatch(int fd, gpointer user_data)
	{
	ll_lrm_t lrm = (ll_lrm_t)user_data;
	lrm->lrm_ops->rcvmsg(lrm, FALSE);
	return TRUE;
	}

	xmlNodePtr
	do_lrm_query(void)
	{
	GList* lrm_list = NULL;
	GList* element = NULL;
	GList* op_list = NULL;
	xmlNodePtr agent = NULL;
	xmlNodePtr data = create_xml_node(NULL, XML_CIB_TAG_LRM);
	xmlNodePtr agent_list = create_xml_node(data, "lrm_agents");
	xmlNodePtr rsc_list;
	char *rsc_type = NULL;
	state_flag_t cur_state = 0;
	const char *this_op = NULL;
	GList* node = NULL;

	lrm_list = fsa_lrm_conn->lrm_ops->get_ra_supported(fsa_lrm_conn);
	if (NULL != lrm_list) {
	GList* element = g_list_first(lrm_list);
	while (NULL != element) {
	rsc_type = (char*)element->data;

	agent =
	create_xml_node(agent_list, "lrm_agent");

	set_xml_property_copy(agent, "class", rsc_type);

	/* we dont have these yet */
	set_xml_property_copy(agent, XML_ATTR_TYPE, NULL);
	set_xml_property_copy(agent, "version", NULL);

	element = g_list_next(element);
	}
	}

	g_list_free(lrm_list);
	lrm_list = fsa_lrm_conn->lrm_ops->get_all_rscs(fsa_lrm_conn);

	rsc_list = create_xml_node(data, XML_LRM_TAG_RESOURCES);

	if (NULL != lrm_list) {
	element = g_list_first(lrm_list);
	}

	while (NULL != element) {
	lrm_rsc_t the_rsc = (lrm_rsc_t)element->data;

	/* const char* ra_type; */
	/* GHashTable* params; */

	xmlNodePtr xml_rsc = create_xml_node(rsc_list, "rsc_state");

	set_xml_property_copy(xml_rsc, XML_ATTR_ID, the_rsc->id);
	set_xml_property_copy(xml_rsc, "rsc_id", the_rsc->name);
	set_xml_property_copy(xml_rsc, "node_id",fsa_our_uname);

	op_list = the_rsc->ops->get_cur_state(the_rsc,
	&cur_state);
	CRM_DEBUG("\tcurrent state:%s\n",
	cur_state==LRM_RSC_IDLE?"Idle":"Busy");

	node = g_list_first(op_list);

	while(NULL != node){
	lrm_op_t* op = (lrm_op_t*)node->data;
	this_op = op->op_type;
	if(this_op == NULL
	\|\| strcmp(this_op, "status") != 0){

	const char *status_text = "<unknown>";
	switch(op->status) {
	case LRM_OP_DONE:
	status_text = "done";
	break;
	case LRM_OP_CANCELLED:
	status_text = "cancelled";
	break;
	case LRM_OP_TIMEOUT:
	status_text = "timeout";
	break;
	case LRM_OP_NOTSUPPORTED:
	status_text = "not suported";
	break;
	case LRM_OP_ERROR:
	status_text = "error";
	break;
	}


	set_xml_property_copy(xml_rsc,
	"op_result",
	status_text);

	set_xml_property_copy(xml_rsc,
	"rsc_op",
	this_op);

	// we only want the last one
	break;
	}

	node = g_list_next(node);
	}

	element = g_list_next(element);
	}

	if (NULL != lrm_list) {
	g_list_free(lrm_list);
	}

	return data;
	}


	/* A_LRM_INVOKE */
	enum crmd_fsa_input
	do_lrm_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input next_input = I_NULL;
	xmlNodePtr fragment, tmp1;
	xmlNodePtr msg;
	const char *rsc_path[] =
	{
	"msg_data",
	"rsc_op",
	"resource",
	"instance_attributes",
	"parameters"
	};
	const char *operation = NULL;
	rsc_id_t rid;
	const char *id_from_cib = NULL;
	const char *crm_op = NULL;
	lrm_rsc_t *rsc = NULL;
	lrm_mon_t* mon = NULL;
	lrm_op_t* op = NULL;

	FNIN();

	if(action & A_UPDATE_NODESTATUS) {

	xmlNodePtr data = NULL;
	#ifndef USE_FAKE_LRM
	data = do_lrm_query();
	#endif
	set_xml_property_copy(data, "replace", XML_CIB_TAG_LRM);

	tmp1 = create_xml_node(NULL, XML_CIB_TAG_STATE);
	set_xml_property_copy(tmp1, XML_ATTR_ID, fsa_our_uname);

	fragment = create_cib_fragment(tmp1, NULL);
	add_node_copy(tmp1, data);

	/* this only happens locally. the updates are pushed out
	* as part of the join process
	*/
	store_request(NULL, fragment, CRM_OP_UPDATE, CRM_SYSTEM_DC);

	free_xml(fragment);
	free_xml(tmp1);
	free_xml(data);

	FNRET(next_input);
	}

	#ifdef USE_FAKE_LRM
	if(data == NULL) {
	FNRET(I_ERROR);
	}

	msg = (xmlNodePtr)data;

	operation = get_xml_attr_nested(msg, rsc_path, DIMOF(rsc_path) -3,
	XML_LRM_ATTR_TASK, TRUE);

	id_from_cib = get_xml_attr_nested(msg, rsc_path, DIMOF(rsc_path) -2,
	XML_ATTR_ID, TRUE);

	crm_op = get_xml_attr(msg, XML_TAG_OPTIONS, XML_ATTR_OP, TRUE);

	if(safe_str_eq(crm_op, "rsc_op")) {

	const char *op_status = NULL;
	xmlNodePtr update = NULL;
	xmlNodePtr state = create_xml_node(NULL, XML_CIB_TAG_STATE);
	xmlNodePtr iter = create_xml_node(state, XML_CIB_TAG_LRM);

	CRM_DEBUG("performing op %s...", operation);

	// so we can identify where to do the update
	set_xml_property_copy(state, XML_ATTR_ID, fsa_our_uname);

	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
	iter = create_xml_node(iter, "lrm_resource");

	set_xml_property_copy(iter, XML_ATTR_ID, id_from_cib);
	set_xml_property_copy(iter, XML_LRM_ATTR_LASTOP, operation);

	long int op_code = 0;

	#if 0
	/* introduce a 10% chance of an action failing */
	op_code = random();
	#endif
	if((op_code % 10) == 1) {
	op_code = 1;
	} else {
	op_code = 0;
	}
	char *op_code_s = crm_itoa(op_code);

	if(op_code) {
	// fail
	if(safe_str_eq(operation, "start")){
	op_status = "stopped";
	} else {
	op_status = "started";
	}
	} else {
	// pass
	if(safe_str_eq(operation, "start")){
	op_status = "started";
	} else {
	op_status = "stopped";
	}
	}

	set_xml_property_copy(iter, XML_LRM_ATTR_OPSTATE,op_status);
	set_xml_property_copy(iter, XML_LRM_ATTR_OPCODE, op_code_s);
	set_xml_property_copy(iter, XML_LRM_ATTR_TARGET, fsa_our_uname);

	crm_free(op_code_s);

	update = create_cib_fragment(state, NULL);

	send_request(NULL, update, CRM_OP_UPDATE,
	fsa_our_dc, CRM_SYSTEM_DCIB, NULL);
	}

	FNRET(I_NULL);
	#endif


	cl_log(LOG_WARNING, "Action %s (%.16llx) only kind of supported\n",
	fsa_action2string(action), action);


	msg = (xmlNodePtr)data;

	operation = get_xml_attr_nested(msg, rsc_path, DIMOF(rsc_path) -3,
	XML_ATTR_OP, TRUE);


	id_from_cib = get_xml_attr_nested(msg, rsc_path, DIMOF(rsc_path) -2,
	XML_ATTR_ID, TRUE);

	// only the first 16 chars are used by the LRM
	strncpy(rid, id_from_cib, 16);


	crm_op = get_xml_attr(msg, XML_TAG_OPTIONS, XML_ATTR_OP, TRUE);

	rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rid);

	if(crm_op != NULL && strcmp(crm_op, "lrm_query") == 0) {

	xmlNodePtr data, tmp1, tmp2, reply;

	tmp1 = create_xml_node(NULL, XML_CIB_TAG_STATE);
	set_xml_property_copy(tmp1, XML_ATTR_ID, fsa_our_uname);

	data = create_cib_fragment(tmp1, NULL);

	tmp2 = do_lrm_query();
	add_node_copy(tmp1, tmp2);

	reply = create_reply(msg, data);

	relay_message(reply, TRUE);

	free_xml(data);
	free_xml(reply);
	free_xml(tmp2);
	free_xml(tmp1);

	} else if(operation != NULL && strcmp(operation, "monitor") == 0) {
	if(rsc == NULL) {
	cl_log(LOG_ERR, "Could not find resource to monitor");
	FNRET(I_FAIL);
	}

	mon = g_new(lrm_mon_t, 1);
	mon->op_type = "status";
	mon->params = NULL;
	mon->timeout = 0;
	mon->user_data = rsc;
	mon->mode = LRM_MONITOR_SET;
	mon->interval = 2;
	mon->target = 1;
	rsc->ops->set_monitor(rsc,mon);
	mon = g_new(lrm_mon_t, 1);

	} else if(operation != NULL) {
	if(rsc == NULL) {
	// add it to the list
	CRM_DEBUG("adding rsc %s before operation", rid);
	fsa_lrm_conn->lrm_ops->add_rsc(
	fsa_lrm_conn, rid,
	get_xml_attr_nested(msg,
	rsc_path,
	DIMOF(rsc_path) -2,
	"class", TRUE),
	get_xml_attr_nested(msg,
	rsc_path,
	DIMOF(rsc_path) -2,
	XML_ATTR_TYPE, TRUE),
	NULL);

	rsc = fsa_lrm_conn->lrm_ops->get_rsc(
	fsa_lrm_conn, rid);
	}

	if(rsc == NULL) {
	cl_log(LOG_ERR, "Could not add resource to LRM");
	FNRET(I_FAIL);
	}

	// now do the op
	CRM_DEBUG("performing op %s...", operation);
	op = g_new(lrm_op_t, 1);
	op->op_type = operation;
	op->params = xml2list(msg, rsc_path, DIMOF(rsc_path));
	op->timeout = 0;
	op->user_data = rsc;
	rsc->ops->perform_op(rsc, op);
	}

	FNRET(next_input);
	}

	GHashTable *
	xml2list(xmlNodePtr parent, const char**attr_path, int depth)
	{
	xmlNodePtr node_iter = NULL;

	GHashTable *nvpair_hash =
	g_hash_table_new(&g_str_hash, &g_str_equal);

	xmlNodePtr nvpair_list =
	find_xml_node_nested(parent, attr_path, depth);

	if(nvpair_list != NULL){
	node_iter = nvpair_list->children;
	while(node_iter != NULL) {

	const char *key = xmlGetProp(
	node_iter, XML_NVPAIR_ATTR_NAME);
	const char *value = xmlGetProp(
	node_iter, XML_NVPAIR_ATTR_VALUE);

	CRM_DEBUG("Added %s=%s", key, value);

	g_hash_table_insert (nvpair_hash,
	crm_strdup(key),
	crm_strdup(value));

	node_iter = node_iter->next;
	}
	}

	return nvpair_hash;
	}


	void
	do_update_resource(lrm_rsc_t rsc, int status, int rc, const char op_type)
	{
	/*
	<status>
	<nodes_status id=uname>
	<lrm>
	<lrm_resources>
	<lrm_resource id=>
	</...>
	*/
	xmlNodePtr update, iter;
	char *tmp = NULL;
	xmlNodePtr fragment, tmp1;


	update = create_xml_node(NULL, "node_state");
	set_xml_property_copy(update, XML_ATTR_ID, fsa_our_uname);
	iter = create_xml_node(update, XML_CIB_TAG_LRM);
	iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES);
	iter = create_xml_node(iter, "lrm_resource");

	set_xml_property_copy(iter, XML_ATTR_ID, rsc->id);
	set_xml_property_copy(iter, XML_LRM_ATTR_LASTOP, op_type);

	tmp = crm_itoa(status);
	set_xml_property_copy(iter, XML_LRM_ATTR_OPSTATE, tmp);
	crm_free(tmp);

	tmp = crm_itoa(rc);
	set_xml_property_copy(iter, XML_LRM_ATTR_OPCODE, tmp);
	crm_free(tmp);

	set_xml_property_copy(iter, XML_LRM_ATTR_TARGET, fsa_our_uname);

	tmp1 = create_xml_node(NULL, XML_CIB_TAG_STATE);
	set_xml_property_copy(tmp1, XML_ATTR_ID, fsa_our_uname);
	add_node_copy(tmp1, update);

	fragment = create_cib_fragment(tmp1, NULL);

	send_request(NULL, fragment, CRM_OP_UPDATE,
	fsa_our_dc, CRM_SYSTEM_DCIB, NULL);

	free_xml(fragment);
	free_xml(update);
	free_xml(tmp1);
	}

	enum crmd_fsa_input
	do_lrm_event(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input cur_input,
	void *data)
	{
	FNIN();
	if(cause == C_LRM_MONITOR_CALLBACK) {
	lrm_mon_t* monitor = (lrm_mon_t*)data;
	lrm_rsc_t* rsc = monitor->rsc;


	switch(monitor->status) {
	case LRM_OP_DONE:
	CRM_NOTE("An LRM monitor operation passed");
	FNRET(I_NULL);
	break;

	case LRM_OP_CANCELLED:
	case LRM_OP_TIMEOUT:
	case LRM_OP_NOTSUPPORTED:
	case LRM_OP_ERROR:
	cl_log(LOG_ERR,
	"An LRM monitor operation failed"
	" or was aborted");

	do_update_resource(rsc,
	monitor->status,
	monitor->rc,
	monitor->op_type);

	break;
	}

	} else if(cause == C_LRM_OP_CALLBACK) {
	lrm_op_t* op = (lrm_op_t*)data;
	lrm_rsc_t* rsc = op->rsc;

	switch(op->status) {
	case LRM_OP_CANCELLED:
	case LRM_OP_TIMEOUT:
	case LRM_OP_NOTSUPPORTED:
	case LRM_OP_ERROR:
	cl_log(LOG_ERR,
	"An LRM operation failed"
	" or was aborted");
	// keep going
	case LRM_OP_DONE:

	do_update_resource(rsc,
	op->status,
	op->rc,
	op->op_type);

	break;
	}

	} else {

	FNRET(I_FAIL);
	}

	FNRET(I_NULL);
	}
	diff --git a/crm/crmd/messages.c b/crm/crmd/messages.c
	index d52b7a8dce..ad452b0916 100644
	--- a/crm/crmd/messages.c
	+++ b/crm/crmd/messages.c
	@@ -1,939 +1,903 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <portability.h>
	#include <crm/crm.h>
	#include <string.h>
	#include <crmd_fsa.h>
	#include <libxml/tree.h>


	+#include <hb_api.h>
	#include <crm/msg_xml.h>
	-#include <crm/common/crmutils.h>
	-#include <crm/common/xmlutils.h>
	-#include <crm/common/msgutils.h>
	+#include <crm/common/xml.h>
	+#include <crm/common/msg.h>
	#include <crm/cib.h>

	#include <crmd.h>
	#include <crmd_messages.h>

	#include <crm/dmalloc_wrapper.h>

	-FILE *msg_in_strm = NULL;
	+FILE *msg_out_strm = NULL;
	FILE *router_strm = NULL;

	fsa_message_queue_t fsa_message_queue = NULL;

	gboolean relay_message(xmlNodePtr xml_relay_message,
	gboolean originated_locally);

	+
	+gboolean send_ha_reply(ll_cluster_t *hb_cluster,
	+ xmlNodePtr xml_request,
	+ xmlNodePtr xml_response_data);
	+
	+gboolean send_xmlha_message(ll_cluster_t *hb_fd, xmlNodePtr root);
	+
	#ifdef MSG_LOG

	# define ROUTER_RESULT(x) char *msg_text = dump_xml(xml_relay_message);\
	if(router_strm == NULL) { \
	router_strm = fopen("/tmp/router.log", "w"); \
	} \
	fprintf(router_strm, "[%d RESULT (%s)]\t%s\t%s\n", \
	AM_I_DC, \
	xmlGetProp(xml_relay_message, XML_ATTR_REFERENCE),\
	x, msg_text); \
	fflush(router_strm); \
	crm_free(msg_text);
	#else
	# define ROUTER_RESULT(x) CRM_DEBUG(x, NULL);
	#endif



	/* returns the current head of the FIFO queue */
	fsa_message_queue_t
	put_message(xmlNodePtr new_message)
	{
	int old_len = g_slist_length(fsa_message_queue);

	// make sure to free it properly later
	fsa_message_queue = g_slist_append(fsa_message_queue,
	copy_xml_node_recursive(new_message));

	CRM_DEBUG("Queue len: %d -> %d", old_len,
	g_slist_length(fsa_message_queue));


	if(old_len == g_slist_length(fsa_message_queue)){
	cl_log(LOG_ERR, "Couldnt add message to the queue");
	}

	return fsa_message_queue;
	}

	/* returns the next message */
	xmlNodePtr
	get_message(void)
	{
	xmlNodePtr message = g_slist_nth_data(fsa_message_queue, 0);
	fsa_message_queue = g_slist_remove(fsa_message_queue, message);
	return message;
	}

	/* returns the current head of the FIFO queue */
	gboolean
	is_message(void)
	{
	return (g_slist_length(fsa_message_queue) > 0);
	}


	/* A_MSG_STORE */
	enum crmd_fsa_input
	do_msg_store(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	// xmlNodePtr new_message = (xmlNodePtr)data;
	FNIN();

	// put_message(new_message);

	FNRET(I_NULL);
	}


	/* A_MSG_ROUTE */
	enum crmd_fsa_input
	do_msg_route(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input result = I_NULL;
	xmlNodePtr xml_message = (xmlNodePtr)data;
	gboolean routed = FALSE, defer = TRUE, do_process = TRUE;

	FNIN();

	#if 0
	// if(cause == C_IPC_MESSAGE) {
	if (crmd_authorize_message(root_xml_node,
	msg,
	curr_client) == FALSE) {
	CRM_DEBUG("Message not authorized");
	do_process = FALSE;
	}
	// }
	#endif
	if(do_process) {
	/* try passing the buck first */
	routed = relay_message(xml_message, cause==C_IPC_MESSAGE);

	if(routed == FALSE) {

	defer = TRUE;
	/* calculate defer */
	result = handle_message(xml_message);
	switch(result) {
	case I_NULL:
	defer = FALSE;
	break;
	case I_DC_HEARTBEAT:
	defer = FALSE;
	break;

	/* what else should go here? */
	default:
	CRM_NOTE("Defering local processing of message");

	put_message(xml_message);
	result = I_REQUEST;
	break;
	}
	}
	}

	FNRET(result);
	}

	-
	-void
	-crmd_ha_input_callback(const struct ha_msg* msg, void* private_data)
	-{
	- const char *from = ha_msg_value(msg, F_ORIG);
	- const char *to = NULL;
	- xmlNodePtr root_xml_node;
	-
	- FNIN();
	-
	-#ifdef MSG_LOG
	- if(msg_in_strm == NULL) {
	- msg_in_strm = fopen("/tmp/inbound.log", "w");
	- }
	-#endif
	-
	- if(from == NULL \|\| strcmp(from, fsa_our_uname) == 0) {
	-#ifdef MSG_LOG
	- fprintf(msg_in_strm,
	- "Discarded message [F_SEQ=%s] from ourselves.\n",
	- ha_msg_value(msg, F_SEQ));
	-#endif
	- FNOUT();
	- }
	-
	-#ifdef MSG_LOG
	- fprintf(msg_in_strm, "[%s (%s:%s)]\t%s\n",
	- from,
	- ha_msg_value(msg, F_SEQ),
	- ha_msg_value(msg, F_TYPE),
	- ha_msg_value(msg, "xml")
	- );
	- fflush(msg_in_strm);
	-#endif
	-
	- root_xml_node = find_xml_in_hamessage(msg);
	- to = xmlGetProp(root_xml_node, XML_ATTR_HOSTTO);
	-
	- if(to != NULL && strlen(to) > 0 && strcmp(to, fsa_our_uname) != 0) {
	-#ifdef MSG_LOG
	- fprintf(msg_in_strm,
	- "Discarding message [F_SEQ=%s] for someone else.",
	- ha_msg_value(msg, F_SEQ));
	-#endif
	- FNOUT();
	- }
	-
	- set_xml_property_copy(root_xml_node, XML_ATTR_HOSTFROM, from);
	- s_crmd_fsa(C_HA_MESSAGE, I_ROUTER, root_xml_node);
	-
	- free_xml(root_xml_node);
	-
	- FNOUT();
	-}
	-
	-/*
	- * Apparently returning TRUE means "stay connected, keep doing stuff".
	- * Returning FALSE means "we're all done, close the connection"
	- */
	-gboolean
	-crmd_ipc_input_callback(IPC_Channel *client, gpointer user_data)
	-{
	- int lpc = 0;
	- char *buffer = NULL;
	- IPC_Message *msg = NULL;
	- gboolean hack_return_good = TRUE;
	- xmlNodePtr root_xml_node;
	- crmd_client_t curr_client = (crmd_client_t)user_data;
	-
	- FNIN();
	- CRM_DEBUG("Processing IPC message from %s",
	- curr_client->table_key);
	-
	- while(client->ops->is_message_pending(client)) {
	- if (client->ch_status == IPC_DISCONNECT) {
	- /* The message which was pending for us is that
	- * the IPC status is now IPC_DISCONNECT */
	- break;
	- }
	- if (client->ops->recv(client, &msg) != IPC_OK) {
	- perror("Receive failure:");
	- FNRET(!hack_return_good);
	- }
	- if (msg == NULL) {
	- cl_log(LOG_WARNING, "No message this time");
	- continue;
	- }
	-
	- lpc++;
	- buffer = (char*)msg->msg_body;
	- CRM_DEBUG("Processing xml from %s [text=%s]",
	- curr_client->table_key, buffer);
	-
	- root_xml_node =
	- find_xml_in_ipcmessage(msg, FALSE);
	- if (root_xml_node != NULL) {
	-
	- if (crmd_authorize_message(root_xml_node,
	- msg,
	- curr_client)) {
	- s_crmd_fsa(C_IPC_MESSAGE,
	- I_ROUTER,
	- root_xml_node);
	- }
	- } else {
	- cl_log(LOG_INFO,
	- "IPC Message was not valid... discarding.");
	- }
	- free_xml(root_xml_node);
	- msg->msg_done(msg);
	-
	- msg = NULL;
	- buffer = NULL;
	- root_xml_node = NULL;
	- }
	-
	- CRM_DEBUG("Processed %d messages", lpc);
	-
	- if (client->ch_status == IPC_DISCONNECT)
	- {
	- cl_log(LOG_INFO,
	- "received HUP from %s",
	- curr_client->table_key);
	- if (curr_client != NULL) {
	- struct crm_subsystem_s *the_subsystem = NULL;
	-
	- if (curr_client->sub_sys == NULL) {
	- cl_log(LOG_WARNING,
	- "Client had not registered with us yet");
	- } else if (strcmp(CRM_SYSTEM_PENGINE,
	- curr_client->sub_sys) == 0) {
	- the_subsystem = pe_subsystem;
	- } else if (strcmp(CRM_SYSTEM_TENGINE,
	- curr_client->sub_sys) == 0) {
	- the_subsystem = te_subsystem;
	- } else if (strcmp(CRM_SYSTEM_CIB,
	- curr_client->sub_sys) == 0){
	- the_subsystem = cib_subsystem;
	- }
	-
	-
	- if(the_subsystem != NULL) {
	- cleanup_subsystem(the_subsystem);
	- } // else that was a transient client
	-
	- if (curr_client->table_key != NULL) {
	- /*
	- * Key is destroyed below: curr_client->table_key
	- * Value is cleaned up by G_main_del_IPC_Channel
	- */
	- g_hash_table_remove(ipc_clients,
	- curr_client->table_key);
	- }
	-
	-
	- if(curr_client->client_source != NULL) {
	- gboolean det = G_main_del_IPC_Channel(
	- curr_client->client_source);
	-
	- CRM_DEBUG("crm_client was %s detached",
	- det?"successfully":"not");
	- }
	-
	- crm_free(curr_client->table_key);
	- crm_free(curr_client->sub_sys);
	- crm_free(curr_client->uuid);
	- crm_free(curr_client);
	- }
	- FNRET(!hack_return_good);
	- }
	-
	- FNRET(hack_return_good);
	-}
	-
	/*
	* This method adds a copy of xml_response_data
	*/
	gboolean
	send_request(xmlNodePtr msg_options, xmlNodePtr msg_data,
	const char operation, const char host_to, const char *sys_to,
	char **msg_reference)
	{
	gboolean was_sent = FALSE;
	xmlNodePtr request = NULL;

	FNIN();

	msg_options = set_xml_attr(msg_options, XML_TAG_OPTIONS,
	XML_ATTR_OP, operation, TRUE);

	request = create_request(msg_options,
	msg_data,
	host_to,
	sys_to,
	AM_I_DC?CRM_SYSTEM_DC:CRM_SYSTEM_CRMD,
	NULL,
	NULL);

	// xml_message_debug(request, "Final request...");

	if(msg_reference != NULL) {
	*msg_reference = crm_strdup(xmlGetProp(request, XML_ATTR_REFERENCE));
	}

	was_sent = relay_message(request, TRUE);

	if(was_sent == FALSE) {
	put_message(request);
	}

	free_xml(request);

	FNRET(was_sent);
	}

	/*
	* This method adds a copy of xml_response_data
	*/
	gboolean
	store_request(xmlNodePtr msg_options, xmlNodePtr msg_data,
	const char operation, const char sys_to)
	{
	xmlNodePtr request = NULL;
	FNIN();

	msg_options = set_xml_attr(msg_options, XML_TAG_OPTIONS,
	XML_ATTR_OP, operation, TRUE);

	crm_debug("Storing op=%s message for later processing", operation);

	request = create_request(msg_options,
	msg_data,
	NULL,
	sys_to,
	AM_I_DC?CRM_SYSTEM_DC:CRM_SYSTEM_CRMD,
	NULL,
	NULL);

	put_message(request);
	free_xml(request);

	FNRET(TRUE);
	}

	gboolean
	relay_message(xmlNodePtr xml_relay_message, gboolean originated_locally)
	{
	int is_for_dc = 0;
	int is_for_dcib = 0;
	int is_for_crm = 0;
	int is_for_cib = 0;
	int is_local = 0;
	gboolean dont_cc= TRUE;
	gboolean processing_complete = FALSE;
	const char *host_to = xmlGetProp(xml_relay_message,XML_ATTR_HOSTTO);
	const char *sys_to = xmlGetProp(xml_relay_message,XML_ATTR_SYSTO);

	FNIN();

	if(xml_relay_message == NULL) {
	cl_log(LOG_ERR, "Cannot route empty message");
	FNRET(TRUE);
	}

	if(strcmp(CRM_OP_HELLO, xml_relay_message->name) == 0) {
	/* quietly ignore */
	FNRET(TRUE);
	}

	if(strcmp(XML_MSG_TAG, xml_relay_message->name) != 0) {

	xml_message_debug(xml_relay_message,
	"Bad message type, should be crm_message");
	cl_log(LOG_ERR, "Ignoring message of type %s",
	xml_relay_message->name);
	FNRET(TRUE);
	}


	if(sys_to == NULL) {
	xml_message_debug(xml_relay_message,
	"Message did not have any value for sys_to");
	cl_log(LOG_ERR, "Message did not have any value for %s",
	XML_ATTR_SYSTO);
	FNRET(TRUE);
	}

	is_for_dc = (strcmp(CRM_SYSTEM_DC, sys_to) == 0);
	is_for_dcib = (strcmp(CRM_SYSTEM_DCIB, sys_to) == 0);
	is_for_cib = (strcmp(CRM_SYSTEM_CIB, sys_to) == 0);
	is_for_crm = (strcmp(CRM_SYSTEM_CRMD, sys_to) == 0);

	is_local = 0;
	if(host_to == NULL \|\| strlen(host_to) == 0) {
	if(is_for_dc)
	is_local = 0;
	else if(is_for_crm && originated_locally)
	is_local = 0;
	else
	is_local = 1;

	} else if(strcmp(fsa_our_uname, host_to) == 0) {
	is_local=1;
	}

	#if 0
	CRM_DEBUG("is_local %d", is_local);
	CRM_DEBUG("is_for_dcib %d", is_for_dcib);
	CRM_DEBUG("is_for_dc %d", is_for_dc);
	CRM_DEBUG("is_for_crm %d", is_for_crm);
	CRM_DEBUG("AM_I_DC %d", AM_I_DC);
	CRM_DEBUG("sys_to %s", sys_to);
	CRM_DEBUG("host_to %s", host_to);
	#endif

	if(is_for_dc \|\| is_for_dcib) {
	if(AM_I_DC) {
	ROUTER_RESULT("Message result: DC/CRMd process");
	processing_complete = FALSE; // more to be done by caller

	} else if(originated_locally) {
	ROUTER_RESULT("Message result: External relay to DC");
	send_msg_via_ha(xml_relay_message, NULL);
	processing_complete = TRUE;

	} else {
	ROUTER_RESULT("Message result: Discard, not DC");
	processing_complete = TRUE; // discard
	}

	} else if(is_local && (is_for_crm \|\| is_for_cib)) {
	ROUTER_RESULT("Message result: CRMd process");

	} else if(is_local) {
	if(dont_cc) {
	ROUTER_RESULT("Message result: Local relay");

	} else {
	/* The DC should also get this message */
	ROUTER_RESULT("Message result: Local relay with CC");
	}
	send_msg_via_ipc(xml_relay_message, sys_to);
	processing_complete = TRUE & dont_cc;

	} else {
	if(dont_cc) {
	ROUTER_RESULT("Message result: External relay");
	} else {
	/* The DC should also get this message */
	ROUTER_RESULT("Message result: External relay with CC");
	}
	send_msg_via_ha(xml_relay_message, host_to);
	processing_complete = TRUE & dont_cc;

	}

	FNRET(processing_complete);
	}

	-void
	-send_msg_via_ha(xmlNodePtr action, const char *dest_node)
	-{
	- FNIN();
	- if (action == NULL) FNOUT();
	-
	- if (validate_crm_message(action, NULL, NULL, NULL) == NULL)
	- {
	- cl_log(LOG_ERR,
	- "Relay message to (%s) via HA was invalid, ignoring",
	- dest_node);
	- FNOUT();
	- }
	-// CRM_DEBUG("Relaying message to (%s) via HA", dest_node);
	- set_xml_property_copy(action, XML_ATTR_HOSTTO, dest_node);
	-
	- send_xmlha_message(fsa_cluster_conn, action);
	- FNOUT();
	-}
	-
	-
	-void
	-send_msg_via_ipc(xmlNodePtr action, const char *sys)
	-{
	- IPC_Channel *client_channel;
	-
	- FNIN();
	-// cl_log(LOG_DEBUG, "relaying msg to sub_sys=%s via IPC", sys);
	-
	- client_channel =
	- (IPC_Channel*)g_hash_table_lookup (ipc_clients, sys);
	-
	- if (client_channel != NULL) {
	- cl_log(LOG_DEBUG, "Sending message via channel %s.", sys);
	- send_xmlipc_message(client_channel, action);
	- } else if(sys != NULL && strcmp(sys, CRM_SYSTEM_CIB) == 0) {
	- cl_log(LOG_ERR,
	- "Sub-system (%s) has been incorporated into the CRMd.",
	- sys);
	- xml_message_debug(action, "Change the way we handle");
	- relay_message(process_cib_message(action, TRUE), TRUE);
	-
	- } else if(sys != NULL && strcmp(sys, CRM_SYSTEM_LRMD) == 0) {
	-
	- do_lrm_invoke(A_LRM_INVOKE, C_IPC_MESSAGE,
	- fsa_state, I_MESSAGE, action);
	-
	- } else {
	- cl_log(LOG_ERR,
	- "Unknown Sub-system (%s)... discarding message.",
	- sys);
	- }
	- FNOUT();
	-}
	-
	gboolean
	crmd_authorize_message(xmlNodePtr root_xml_node,
	IPC_Message *client_msg,
	crmd_client_t *curr_client)
	{
	// check the best case first
	const char *sys_from = xmlGetProp(root_xml_node,
	XML_ATTR_SYSFROM);
	char *uuid = NULL;
	char *client_name = NULL;
	char *major_version = NULL;
	char *minor_version = NULL;
	const char *filtered_from;
	gpointer table_key = NULL;
	gboolean result;
	const char *op = get_xml_attr(root_xml_node, XML_TAG_OPTIONS,
	XML_ATTR_OP, TRUE);

	FNIN();

	if (safe_str_neq(CRM_OP_HELLO, op)) {

	if(sys_from == NULL) {
	return FALSE;
	}

	gboolean can_reply = FALSE; // no-one has registered with this id
	filtered_from = sys_from;

	/* The CIB can have two names on the DC */
	if(strcmp(sys_from, CRM_SYSTEM_DCIB) == 0)
	filtered_from = CRM_SYSTEM_CIB;

	if (g_hash_table_lookup (ipc_clients, filtered_from) != NULL)
	can_reply = TRUE; // reply can be routed


	CRM_DEBUG("Message reply can%s be routed from %s.",
	can_reply?"":" not", sys_from);

	if(can_reply == FALSE) {
	cl_log(LOG_ERR, "Message not authorized");
	}

	return can_reply;
	}

	cl_log(LOG_INFO,
	"received client join msg: %s",
	(char*)client_msg->msg_body);

	result = process_hello_message(root_xml_node,
	&uuid,
	&client_name,
	&major_version,
	&minor_version);

	if (result == TRUE) {
	// check version
	int mav = atoi(major_version);
	int miv = atoi(minor_version);
	if (mav < 0 \|\| miv < 0) {
	cl_log(LOG_ERR,
	"Client version (%d:%d) is not acceptable",
	mav,
	miv);
	result = FALSE;
	}
	crm_free(major_version);
	crm_free(minor_version);
	}

	struct crm_subsystem_s *the_subsystem = NULL;

	if (result == TRUE) {
	/* if we already have one of those clients
	* only applies to te, pe etc. not admin clients
	*/


	if (client_name == NULL)
	cl_log(LOG_WARNING,
	"Client had not registered with us yet");
	else if (strcmp(CRM_SYSTEM_PENGINE, client_name) == 0)
	the_subsystem = pe_subsystem;
	else if (strcmp(CRM_SYSTEM_TENGINE, client_name) == 0)
	the_subsystem = te_subsystem;
	else if (strcmp(CRM_SYSTEM_CIB, client_name) == 0)
	the_subsystem = cib_subsystem;

	if (the_subsystem != NULL) {
	// do we already have one?
	result =(fsa_input_register & the_subsystem->flag)==0;
	if(result) {
	the_subsystem->ipc =
	curr_client->client_channel;

	} // else we didnt ask for the client to start

	} else if(client_name != NULL && uuid != NULL) {
	table_key = (gpointer)
	generate_hash_key(client_name, uuid);
	} else {
	result = FALSE;
	cl_log(LOG_ERR,
	"Bad client details (client_name=%s, uuid=%s)",
	client_name, uuid);
	}
	}

	if(result == TRUE && table_key == NULL) {
	table_key = (gpointer)crm_strdup(client_name);
	}

	if (result == TRUE) {
	cl_log(LOG_INFO, "Accepted client %s", (char*)table_key);

	curr_client->table_key = table_key;
	curr_client->sub_sys = crm_strdup(client_name);
	curr_client->uuid = crm_strdup(uuid);

	g_hash_table_insert (ipc_clients,
	table_key,
	curr_client->client_channel);

	send_hello_message(curr_client->client_channel,
	"n/a", CRM_SYSTEM_CRMD,
	"0", "1");

	cl_log(LOG_INFO, "Updated client list with %s",
	(char*)table_key);

	if(the_subsystem != NULL) {
	set_bit_inplace(&fsa_input_register,
	the_subsystem->flag);
	}

	s_crmd_fsa(C_SUBSYSTEM_CONNECT, I_NULL, NULL);

	} else {
	cl_log(LOG_ERR, "Rejected client logon request");
	curr_client->client_channel->ch_status = IPC_DISC_PENDING;
	}

	if(uuid != NULL) crm_free(uuid);
	if(client_name != NULL) crm_free(client_name);

	/* hello messages should never be processed further */
	return FALSE;
	}

	enum crmd_fsa_input
	handle_message(xmlNodePtr stored_msg)
	{
	enum crmd_fsa_input next_input = I_NULL;

	- const char *sys_to = get_xml_attr(stored_msg, NULL,
	- XML_ATTR_SYSTO, TRUE);
	+ const char *sys_to = get_xml_attr(
	+ stored_msg, NULL, XML_ATTR_SYSTO, TRUE);

	- const char *sys_from = get_xml_attr(stored_msg, NULL,
	- XML_ATTR_SYSFROM, TRUE);
	+ const char *sys_from = get_xml_attr(
	+ stored_msg, NULL, XML_ATTR_SYSFROM, TRUE);

	- const char *host_from= get_xml_attr(stored_msg, NULL,
	- XML_ATTR_HOSTFROM, TRUE);
	+ const char *host_from= get_xml_attr(
	+ stored_msg, NULL, XML_ATTR_HOSTFROM, TRUE);

	- const char *msg_ref = get_xml_attr(stored_msg, NULL,
	- XML_ATTR_REFERENCE, TRUE);
	+ const char *msg_ref = get_xml_attr(
	+ stored_msg, NULL, XML_ATTR_REFERENCE, TRUE);

	- const char *type = get_xml_attr(stored_msg, NULL,
	- XML_ATTR_MSGTYPE, TRUE);
	+ const char *type = get_xml_attr(
	+ stored_msg, NULL, XML_ATTR_MSGTYPE, TRUE);

	- const char *op = get_xml_attr(stored_msg, XML_TAG_OPTIONS,
	- XML_ATTR_OP, TRUE);
	+ const char *op = get_xml_attr(
	+ stored_msg, XML_TAG_OPTIONS, XML_ATTR_OP, TRUE);

	// xml_message_debug(stored_msg, "Processing message");

	cl_log(LOG_DEBUG,
	"Received %s %s in state %s",
	op, type, fsa_state2string(fsa_state));

	if(type == NULL \|\| op == NULL) {
	cl_log(LOG_ERR, "Ignoring message (type=%s), (op=%s)",
	type, op);
	xml_message_debug(stored_msg, "Bad message");

	} else if(strcmp(type, XML_ATTR_REQUEST) == 0){
	if(strcmp(op, CRM_OP_VOTE) == 0) {
	next_input = I_ELECTION;

	} else if(AM_I_DC && strcmp(op, CRM_OP_TEABORT) == 0) {
	next_input = I_PE_CALC;

	} else if(AM_I_DC
	&& strcmp(op, CRM_OP_TECOMPLETE) == 0) {
	if(fsa_state == S_TRANSITION_ENGINE) {
	next_input = I_SUCCESS;
	/* silently ignore? probably means the TE is signaling OK too early
	} else {
	cl_log(LOG_WARNING,
	"Op %s is only valid in state %s (%s)",
	op,
	fsa_state2string(S_TRANSITION_ENGINE),
	fsa_state2string(fsa_state));
	*/
	}

	} else if(strcmp(op, CRM_OP_HBEAT) == 0) {
	next_input = I_DC_HEARTBEAT;

	} else if(strcmp(op, CRM_OP_WELCOME) == 0) {
	next_input = I_WELCOME;

	} else if(strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0) {

	/* create cib fragment and add to message */

	/* handle here to avoid potential version issues
	* where the shutdown message/proceedure may have
	* been changed in later versions.
	*
	* This way the DC is always in control of the shutdown
	*/

	xmlNodePtr frag = NULL;
	time_t now = time(NULL);
	char *now_s = crm_itoa((int)now);
	xmlNodePtr node_state =
	create_xml_node(NULL, XML_CIB_TAG_STATE);

	cl_log(LOG_INFO, "Creating shutdown request for %s",
	host_from);

	set_xml_property_copy(
	node_state, XML_ATTR_ID, host_from);
	set_xml_property_copy(
	node_state, XML_CIB_ATTR_SHUTDOWN, now_s);
	set_xml_property_copy(
	node_state,
	XML_CIB_ATTR_EXPSTATE,
	CRMD_STATE_INACTIVE);

	frag = create_cib_fragment(node_state, NULL);
	xmlAddChild(stored_msg, frag);

	free_xml(node_state);
	crm_free(now_s);

	next_input = I_CIB_OP;

	} else if(strcmp(op, CRM_OP_SHUTDOWN) == 0) {
	next_input = I_TERMINATE;

	} else if(strcmp(op, CRM_OP_ANNOUNCE) == 0) {
	next_input = I_NODE_JOIN;

	} else if(strcmp(op, CRM_OP_REPLACE) == 0
	\|\| strcmp(op, CRM_OP_ERASE) == 0) {
	next_input = I_CIB_OP;
	fprintf(router_strm, "Message result: CIB Op\n");

	} else if(AM_I_DC
	&& (strcmp(op, CRM_OP_CREATE) == 0
	\|\| strcmp(op, CRM_OP_UPDATE) == 0
	\|\| strcmp(op, CRM_OP_DELETE) == 0)) {
	/* updates should only be performed on the DC */
	next_input = I_CIB_OP;

	} else if(strcmp(op, CRM_OP_PING) == 0) {
	/* eventually do some stuff to figure out
	* if we /are/ ok
	*/
	xmlNodePtr ping =
	createPingAnswerFragment(sys_to, "ok");

	xmlNodePtr wrapper = create_reply(stored_msg, ping);
	relay_message(wrapper, TRUE);
	free_xml(wrapper);

	} else {
	cl_log(LOG_ERR,
	"Unexpected request (op=%s) sent to the %s",
	op, AM_I_DC?"DC":"CRMd");
	}

	} else if(strcmp(type, XML_ATTR_RESPONSE) == 0) {

	if(strcmp(op, CRM_OP_WELCOME) == 0) {
	next_input = I_WELCOME_ACK;

	} else if(AM_I_DC
	&& strcmp(op, CRM_OP_PECALC) == 0) {

	if(fsa_state == S_POLICY_ENGINE
	&& safe_str_eq(msg_ref, fsa_pe_ref)) {
	next_input = I_SUCCESS;
	} else if(fsa_state != S_POLICY_ENGINE) {
	cl_log(LOG_ERR,
	"Reply to %s is only valid in state %s",
	op, fsa_state2string(S_POLICY_ENGINE));

	} else {
	CRM_DEBUG("Skipping superceeded reply from %s",
	sys_from);
	}

	} else if(strcmp(op, CRM_OP_VOTE) == 0
	\|\| strcmp(op, CRM_OP_HBEAT) == 0
	\|\| strcmp(op, CRM_OP_WELCOME) == 0
	\|\| strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0
	\|\| strcmp(op, CRM_OP_SHUTDOWN) == 0
	\|\| strcmp(op, CRM_OP_ANNOUNCE) == 0) {
	next_input = I_NULL;

	} else if(strcmp(op, CRM_OP_CREATE) == 0
	\|\| strcmp(op, CRM_OP_UPDATE) == 0
	\|\| strcmp(op, CRM_OP_DELETE) == 0
	\|\| strcmp(op, CRM_OP_REPLACE) == 0
	\|\| strcmp(op, CRM_OP_ERASE) == 0) {

	/* perhaps we should do somethign with these replies,
	* especially check that the actions passed
	*/
	/* fprintf(router_strm, "Message result: CIB Reply\n"); */

	} else {
	cl_log(LOG_ERR,
	"Unexpected response (op=%s) sent to the %s",
	op, AM_I_DC?"DC":"CRMd");
	next_input = I_NULL;
	}
	} else {
	cl_log(LOG_ERR, "Unexpected message type %s", type);

	}

	/* CRM_DEBUG("%s: Next input is %s", __FUNCTION__, */
	/* fsa_input2string(next_input)); */


	return next_input;

	}

	+gboolean
	+send_xmlha_message(ll_cluster_t *hb_fd, xmlNodePtr root)
	+{
	+ int xml_len = -1;
	+ int send_result = -1;
	+ char *xml_text = NULL;
	+ const char *host_to = NULL;
	+ const char *sys_to = NULL;
	+ struct ha_msg *msg = NULL;
	+ gboolean all_is_good = TRUE;
	+ gboolean broadcast = FALSE;
	+ int log_level = LOG_DEBUG;
	+
	+ xmlNodePtr opts = find_xml_node(root, XML_TAG_OPTIONS);
	+ const char *op = xmlGetProp(opts, XML_ATTR_OP);

	-void lrm_op_callback (lrm_op_t* op)
	+#ifdef MSG_LOG
	+ char *msg_text = NULL;
	+#endif
	+
	+ FNIN();
	+
	+ if (root == NULL) {
	+ cl_log(LOG_ERR, "Attempt to send NULL Message via HA failed.");
	+ all_is_good = FALSE;
	+ }
	+
	+ host_to = xmlGetProp(root, XML_ATTR_HOSTTO);
	+ sys_to = xmlGetProp(root, XML_ATTR_SYSTO);
	+
	+ if (all_is_good) {
	+ msg = ha_msg_new(4);
	+ ha_msg_add(msg, F_TYPE, "CRM");
	+ ha_msg_add(msg, F_COMMENT, "A CRM xml message");
	+ xml_text = dump_xml(root);
	+ xml_len = strlen(xml_text);
	+
	+ if (xml_text == NULL \|\| xml_len <= 0) {
	+ cl_log(LOG_ERR,
	+ "Failed sending an invalid XML Message via HA");
	+ all_is_good = FALSE;
	+ xml_message_debug(root, "Bad message was");
	+
	+ } else {
	+ if(ha_msg_add(msg, "xml", xml_text) == HA_FAIL) {
	+ cl_log(LOG_ERR,
	+ "Could not add xml to HA message");
	+ all_is_good = FALSE;
	+ }
	+ }
	+ }
	+
	+ if (all_is_good) {
	+ if (sys_to == NULL \|\| strlen(sys_to) == 0)
	+ {
	+ cl_log(LOG_ERR,
	+ "You did not specify a destination sub-system"
	+ " for this message.");
	+ all_is_good = FALSE;
	+ }
	+ }
	+
	+
	+ /* There are a number of messages may not need to be ordered.
	+ * At a later point perhaps we should detect them and send them
	+ * as unordered messages.
	+ */
	+ if (all_is_good) {
	+ if (host_to == NULL
	+ \|\| strlen(host_to) == 0) {
	+ broadcast = TRUE;
	+ send_result =
	+ hb_fd->llc_ops->sendclustermsg(hb_fd, msg);
	+ }
	+ else {
	+ send_result = hb_fd->llc_ops->send_ordered_nodemsg(
	+ hb_fd, msg, host_to);
	+ }
	+
	+ if(send_result != HA_OK)
	+ all_is_good = FALSE;
	+ }
	+
	+ if(all_is_good == FALSE) {
	+ log_level = LOG_ERR;
	+ }
	+
	+ if(log_level == LOG_ERR
	+ \|\| (safe_str_neq(op, CRM_OP_HBEAT))) {
	+ cl_log(log_level,
	+ "Sending %s HA message (ref=%s, len=%d) to %s@%s %s.",
	+ broadcast?"broadcast":"directed",
	+ xmlGetProp(root, XML_ATTR_REFERENCE), xml_len,
	+ sys_to, host_to==NULL?"<all>":host_to,
	+ all_is_good?"succeeded":"failed");
	+ }
	+
	+#ifdef MSG_LOG
	+ msg_text = dump_xml(root);
	+ if(msg_out_strm == NULL) {
	+ msg_out_strm = fopen("/tmp/outbound.log", "w");
	+ }
	+ fprintf(msg_out_strm, "[%d HA (%s:%d)]\t%s\n",
	+ all_is_good,
	+ xmlGetProp(root, XML_ATTR_REFERENCE),
	+ send_result,
	+ msg_text);
	+
	+ fflush(msg_out_strm);
	+ crm_free(msg_text);
	+ if(msg != NULL) {
	+ ha_msg_del(msg);
	+ }
	+#endif
	+
	+ FNRET(all_is_good);
	+}
	+
	+
	+
	+// required? or just send to self an let relay_message do its thing?
	+/*
	+ * This method adds a copy of xml_response_data
	+ */
	+gboolean
	+send_ha_reply(ll_cluster_t *hb_cluster,
	+ xmlNodePtr xml_request,
	+ xmlNodePtr xml_response_data)
	{
	- s_crmd_fsa(C_LRM_OP_CALLBACK, I_LRM_EVENT, op);
	+ gboolean was_sent = FALSE;
	+ xmlNodePtr reply;
	+
	+ FNIN();
	+ was_sent = FALSE;
	+ reply = create_reply(xml_request, xml_response_data);
	+ if (reply != NULL) {
	+ was_sent = send_xmlha_message(hb_cluster, reply);
	+ free_xml(reply);
	+ }
	+ FNRET(was_sent);
	}

	-void lrm_monitor_callback (lrm_mon_t* monitor)
	+
	+void
	+send_msg_via_ha(xmlNodePtr action, const char *dest_node)
	{
	- s_crmd_fsa(C_LRM_MONITOR_CALLBACK, I_LRM_EVENT, monitor);
	+ FNIN();
	+ if (action == NULL) FNOUT();
	+
	+ if (validate_crm_message(action, NULL, NULL, NULL) == NULL)
	+ {
	+ cl_log(LOG_ERR,
	+ "Relay message to (%s) via HA was invalid, ignoring",
	+ dest_node);
	+ FNOUT();
	+ }
	+// CRM_DEBUG("Relaying message to (%s) via HA", dest_node);
	+ set_xml_property_copy(action, XML_ATTR_HOSTTO, dest_node);
	+
	+ send_xmlha_message(fsa_cluster_conn, action);
	+ FNOUT();
	}

	+
	+void
	+send_msg_via_ipc(xmlNodePtr action, const char *sys)
	+{
	+ IPC_Channel *client_channel;
	+
	+ FNIN();
	+// cl_log(LOG_DEBUG, "relaying msg to sub_sys=%s via IPC", sys);
	+
	+ client_channel =
	+ (IPC_Channel*)g_hash_table_lookup (ipc_clients, sys);
	+
	+ if (client_channel != NULL) {
	+ cl_log(LOG_DEBUG, "Sending message via channel %s.", sys);
	+ send_xmlipc_message(client_channel, action);
	+ } else if(sys != NULL && strcmp(sys, CRM_SYSTEM_CIB) == 0) {
	+ cl_log(LOG_ERR,
	+ "Sub-system (%s) has been incorporated into the CRMd.",
	+ sys);
	+ xml_message_debug(action, "Change the way we handle");
	+ relay_message(process_cib_message(action, TRUE), TRUE);
	+
	+ } else if(sys != NULL && strcmp(sys, CRM_SYSTEM_LRMD) == 0) {
	+
	+ do_lrm_invoke(A_LRM_INVOKE, C_IPC_MESSAGE,
	+ fsa_state, I_MESSAGE, action);
	+
	+ } else {
	+ cl_log(LOG_ERR,
	+ "Unknown Sub-system (%s)... discarding message.",
	+ sys);
	+ }
	+ FNOUT();
	+}
	diff --git a/crm/crmd/misc.c b/crm/crmd/misc.c
	index 8c258b5896..ff372015c3 100644
	--- a/crm/crmd/misc.c
	+++ b/crm/crmd/misc.c
	@@ -1,97 +1,54 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <portability.h>
	+#include <heartbeat.h>
	+
	#include <crm/crm.h>
	-#include <crm/msg_xml.h>
	#include <crm/cib.h>
	+#include <crm/msg_xml.h>
	+#include <crm/common/xml.h>
	+
	#include <crmd_fsa.h>
	#include <crmd_messages.h>
	-#include <crm/common/xmlutils.h>
	-
	-#include <heartbeat.h>

	#include <crm/dmalloc_wrapper.h>

	/* A_LOG, A_WARN, A_ERROR */
	enum crmd_fsa_input
	do_log(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	int log_type = LOG_DEBUG;

	FNIN();

	if(action & A_LOG) log_type = LOG_INFO;
	if(action & A_WARN) log_type = LOG_WARNING;
	if(action & A_ERROR) log_type = LOG_ERR;

	cl_log(log_type,
	"[[FSA]] Input (%s) was received while in state (%s)",
	fsa_input2string(current_input),
	fsa_state2string(cur_state));

	FNRET(I_NULL);
	}

	-void
	-CrmdClientStatus(const char * node, const char * client,
	- const char * status, void * private)
	-{
	- const char *join = NULL;
	- const char *extra = NULL;
	- xmlNodePtr update = NULL;
	- xmlNodePtr fragment = NULL;
	-
	- if(safe_str_eq(status, JOINSTATUS)){
	- status = ONLINESTATUS;
	- extra = XML_CIB_ATTR_CLEAR_SHUTDOWN;
	-
	- } else if(safe_str_eq(status, LEAVESTATUS)){
	- status = OFFLINESTATUS;
	- join = CRMD_JOINSTATE_DOWN;
	- extra = XML_CIB_ATTR_CLEAR_SHUTDOWN;
	- }
	-
	- cl_log(LOG_NOTICE,
	- "Status update: Client %s/%s now has status [%s]\n",
	- node, client, status);
	-
	- if(AM_I_DC) {
	- update = create_node_state(node, NULL, status, join);
	-
	- if(extra != NULL) {
	- set_xml_property_copy(update, extra, XML_BOOLEAN_TRUE);
	- }
	-
	- fragment = create_cib_fragment(update, NULL);
	- store_request(NULL, fragment,
	- CRM_OP_UPDATE, CRM_SYSTEM_DCIB);
	-
	- free_xml(fragment);
	- free_xml(update);
	-
	- s_crmd_fsa(C_CRMD_STATUS_CALLBACK, I_NULL, NULL);
	-
	- } else {
	- cl_log(LOG_ERR, "Got client status callback in non-DC mode");
	- }
	-}
	-
	diff --git a/crm/crmd/subsystems.c b/crm/crmd/subsystems.c
	index 7705a5215e..5d462690f2 100644
	--- a/crm/crmd/subsystems.c
	+++ b/crm/crmd/subsystems.c
	@@ -1,641 +1,635 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <portability.h>
	#include <crm/crm.h>
	#include <crmd_fsa.h>

	#include <sys/types.h>
	#include <sys/wait.h>

	#include <unistd.h> // for access
	#include <clplumbing/cl_signal.h>
	#include <clplumbing/realtime.h>
	#include <sys/types.h> // for calls to open
	#include <sys/stat.h> // for calls to open
	#include <fcntl.h> // for calls to open
	#include <pwd.h> // for getpwuid
	#include <grp.h> // for initgroups

	#include <sys/time.h> // for getrlimit
	#include <sys/resource.h>// for getrlimit

	-#include <crm/common/crmutils.h>
	-#include <crm/common/ipcutils.h>
	-#include <crm/common/msgutils.h>
	+#include <errno.h>

	#include <crm/msg_xml.h>
	-#include <crm/common/xmlutils.h>
	-
	+#include <crm/common/xml.h>
	+#include <crmd_messages.h>

	#include <crm/cib.h>
	-
	#include <crmd.h>
	-#include <crmd_messages.h>
	-#include <string.h>
	-#include <errno.h>

	#include <crm/dmalloc_wrapper.h>

	#define CLIENT_EXIT_WAIT 10

	static gboolean stop_subsystem (struct crm_subsystem_s *centry);
	static gboolean start_subsystem(struct crm_subsystem_s *centry);

	struct crm_subsystem_s *cib_subsystem = NULL;
	struct crm_subsystem_s *te_subsystem = NULL;
	struct crm_subsystem_s *pe_subsystem = NULL;


	/* A_CIB_STOP, A_CIB_START, A_CIB_RESTART, */
	enum crmd_fsa_input
	do_cib_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input result = I_NULL;
	struct crm_subsystem_s *this_subsys = cib_subsystem;

	long long stop_actions = A_CIB_STOP;
	long long start_actions = A_CIB_START;

	FNIN();

	if(action & stop_actions) {
	// dont do anything, its embedded now
	}

	if(action & start_actions) {

	if(cur_state != S_STOPPING) {
	if(startCib(CIB_FILENAME) == FALSE)
	result = I_FAIL;

	} else {
	cl_log(LOG_INFO,
	"Ignoring request to start %s after shutdown",
	this_subsys->command);
	}
	}

	FNRET(result);
	}


	/* A_CIB_INVOKE, A_CIB_BUMPGEN, A_UPDATE_NODESTATUS */
	enum crmd_fsa_input
	do_cib_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	xmlNodePtr cib_msg = NULL;
	xmlNodePtr answer = NULL;
	xmlNodePtr new_options = NULL;
	const char *section = NULL;

	FNIN();

	if(data != NULL) {
	cib_msg = (xmlNodePtr)data;
	}


	if(action & A_CIB_INVOKE) {

	const char *op = get_xml_attr(cib_msg, XML_TAG_OPTIONS,
	XML_ATTR_OP, TRUE);

	xml_message_debug(cib_msg, "[CIB] Invoking with");
	if(cib_msg == NULL) {
	cl_log(LOG_ERR, "No message for CIB command");
	FNRET(I_NULL); // I_ERROR
	}

	set_xml_property_copy(cib_msg, XML_ATTR_SYSTO, "cib");
	answer = process_cib_message(cib_msg, TRUE);
	if(relay_message(answer, TRUE) == FALSE) {
	cl_log(LOG_ERR, "Confused what to do with cib result");
	xml_message_debug(answer, "Couldnt route: ");
	}


	if(op != NULL && AM_I_DC
	&& (strcmp(op, CRM_OP_CREATE) == 0
	\|\| strcmp(op, CRM_OP_UPDATE) == 0
	\|\| strcmp(op, CRM_OP_DELETE) == 0
	\|\| strcmp(op, CRM_OP_REPLACE) == 0
	\|\| strcmp(op, CRM_OP_WELCOME) == 0
	\|\| strcmp(op, CRM_OP_SHUTDOWN_REQ) == 0
	\|\| strcmp(op, CRM_OP_ERASE) == 0)) {
	FNRET(I_CIB_UPDATE);
	}

	if(op == NULL) {
	xml_message_debug(cib_msg, "Invalid CIB Message");
	}



	// check the answer, see if we are interested in it also
	#if 0
	if(interested in reply) {
	put_message(answer);
	FNRET(I_REQUEST);
	}
	#endif

	free_xml(answer);

	/* experimental */
	} else if(action & A_CIB_INVOKE_LOCAL) {
	xml_message_debug(cib_msg, "[CIB] Invoking with");
	if(cib_msg == NULL) {
	cl_log(LOG_ERR, "No message for CIB command");
	FNRET(I_NULL); // I_ERROR
	}

	answer = process_cib_message(cib_msg, TRUE);
	put_message(answer);
	FNRET(I_REQUEST);

	} else if(action & A_CIB_BUMPGEN) {
	// check if the response was ok before next bit

	section = get_xml_attr(cib_msg, XML_TAG_OPTIONS,
	XML_ATTR_FILTER_TYPE, FALSE);

	/* set the section so that we dont always send the
	* whole thing
	*/

	if(section != NULL) {
	new_options = set_xml_attr(NULL, XML_TAG_OPTIONS,
	XML_ATTR_FILTER_TYPE,
	section, TRUE);
	}

	answer = process_cib_request(CRM_OP_BUMP,
	new_options, NULL);

	free_xml(new_options);

	if(answer == NULL) {
	cl_log(LOG_ERR, "Result of BUMP in %s was NULL",
	__FUNCTION__);
	FNRET(I_FAIL);
	}

	send_request(NULL, answer, CRM_OP_REPLACE,
	NULL, CRM_SYSTEM_CRMD, NULL);

	free_xml(answer);

	} else {
	cl_log(LOG_ERR, "Unexpected action %s in %s",
	fsa_action2string(action), __FUNCTION__);
	}


	FNRET(I_NULL);
	}


	/* A_PE_START, A_PE_STOP, A_TE_RESTART */
	enum crmd_fsa_input
	do_pe_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input result = I_NULL;
	struct crm_subsystem_s *this_subsys = pe_subsystem;

	long long stop_actions = A_PE_STOP;
	long long start_actions = A_PE_START;

	FNIN();

	if(action & stop_actions) {
	if(stop_subsystem(this_subsys) == FALSE)
	result = I_FAIL;
	else if(this_subsys->pid > 0){
	int lpc = CLIENT_EXIT_WAIT;
	int pid_status = -1;
	while(lpc-- > 0
	&& this_subsys->pid > 0
	&& CL_PID_EXISTS(this_subsys->pid)) {

	sleep(1);
	waitpid(this_subsys->pid, &pid_status, WNOHANG);
	}

	if(CL_PID_EXISTS(this_subsys->pid)) {
	cl_log(LOG_ERR,
	"Process %s is still active with pid=%d",
	this_subsys->command, this_subsys->pid);
	result = I_FAIL;
	}
	}

	cleanup_subsystem(this_subsys);
	}

	if(action & start_actions) {

	if(cur_state != S_STOPPING) {
	if(start_subsystem(this_subsys) == FALSE) {
	result = I_FAIL;
	cleanup_subsystem(this_subsys);
	}
	} else {
	cl_log(LOG_INFO,
	"Ignoring request to start %s while shutting down",
	this_subsys->command);
	}
	}

	FNRET(result);
	}

	char *fsa_pe_ref = NULL;

	/* A_PE_INVOKE */
	enum crmd_fsa_input
	do_pe_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	FNIN();

	stopTimer(integration_timer);

	if(is_set(fsa_input_register, R_PE_CONNECTED) == FALSE){

	cl_log(LOG_INFO, "Waiting for the PE to connect");
	FNRET(I_WAIT_FOR_EVENT);

	}

	xmlNodePtr local_cib = get_cib_copy();

	CRM_DEBUG("Invoking %s with %p", CRM_SYSTEM_PENGINE, local_cib);

	if(fsa_pe_ref) {
	crm_free(fsa_pe_ref);
	fsa_pe_ref = NULL;
	}

	send_request(NULL, local_cib, CRM_OP_PECALC,
	NULL, CRM_SYSTEM_PENGINE, &fsa_pe_ref);

	FNRET(I_NULL);
	}

	/* A_TE_START, A_TE_STOP, A_TE_RESTART */
	enum crmd_fsa_input
	do_te_control(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	enum crmd_fsa_input result = I_NULL;
	struct crm_subsystem_s *this_subsys = te_subsystem;

	long long stop_actions = A_TE_STOP;
	long long start_actions = A_TE_START;

	FNIN();

	/* if(action & stop_actions && cur_state != S_STOPPING */
	/* && is_set(fsa_input_register, R_TE_PEND)) { */
	/* result = I_WAIT_FOR_EVENT; */
	/* FNRET(result); */
	/* } */

	if(action & stop_actions) {
	if(stop_subsystem(this_subsys) == FALSE)
	result = I_FAIL;
	else if(this_subsys->pid > 0){
	int lpc = CLIENT_EXIT_WAIT;
	int pid_status = -1;
	while(lpc-- > 0
	&& this_subsys->pid > 0
	&& CL_PID_EXISTS(this_subsys->pid)) {

	sleep(1);
	waitpid(this_subsys->pid, &pid_status, WNOHANG);
	}

	if(CL_PID_EXISTS(this_subsys->pid)) {
	cl_log(LOG_ERR,
	"Process %s is still active with pid=%d",
	this_subsys->command, this_subsys->pid);
	result = I_FAIL;
	}
	}

	cleanup_subsystem(this_subsys);
	}

	if(action & start_actions) {

	if(cur_state != S_STOPPING) {
	if(start_subsystem(this_subsys) == FALSE) {
	result = I_FAIL;
	cleanup_subsystem(this_subsys);
	}
	} else {
	cl_log(LOG_INFO,
	"Ignoring request to start %s while shutting down",
	this_subsys->command);
	}
	}

	FNRET(result);
	}

	static xmlNodePtr te_last_input = NULL;
	static xmlNodePtr te_lastcc = NULL;

	/* A_TE_COPYTO */
	enum crmd_fsa_input
	do_te_copyto(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	xmlNodePtr message = NULL;
	xmlNodePtr opts = NULL;
	const char *true_op = NULL;

	FNIN();

	if(data != NULL) {
	message = copy_xml_node_recursive((xmlNodePtr)data);
	opts = find_xml_node(message, XML_TAG_OPTIONS);
	true_op = xmlGetProp(opts, XML_ATTR_OP);

	set_xml_property_copy(opts, XML_ATTR_OP, CRM_OP_EVENTCC);
	set_xml_property_copy(opts, XML_ATTR_TRUEOP, true_op);

	set_xml_property_copy(message,
	XML_ATTR_SYSTO,
	CRM_SYSTEM_TENGINE);
	}

	if(is_set(fsa_input_register, R_TE_CONNECTED) == FALSE){
	cl_log(LOG_INFO, "Waiting for the TE to connect");
	if(data != NULL) {
	free_xml(te_lastcc);
	te_lastcc = message;
	}
	FNRET(I_WAIT_FOR_EVENT);

	}

	if(message == NULL) {
	message = te_lastcc;
	te_lastcc = NULL;

	} else {
	free_xml(te_lastcc);
	}

	relay_message(message, FALSE);

	// only free it if it was a local copy
	if(data == NULL) {
	free_xml(message);
	}

	FNRET(I_NULL);
	}


	/* A_TE_INVOKE, A_TE_CANCEL */
	enum crmd_fsa_input
	do_te_invoke(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	void *data)
	{
	xmlNodePtr graph = NULL;
	xmlNodePtr msg = (xmlNodePtr)data;
	FNIN();

	if(is_set(fsa_input_register, R_TE_CONNECTED) == FALSE){
	cl_log(LOG_INFO, "Waiting for the TE to connect");
	if(data != NULL) {
	free_xml(te_last_input);
	te_last_input = copy_xml_node_recursive(msg);
	}
	FNRET(I_WAIT_FOR_EVENT);

	}

	if(msg == NULL) {
	msg = te_last_input;
	te_last_input = NULL;

	} else {
	free_xml(te_last_input);
	}

	if(action & A_TE_INVOKE) {
	graph = find_xml_node(msg, "transition_graph");
	if(graph == NULL) {
	FNRET(I_FAIL);
	}

	send_request(NULL, graph, CRM_OP_TRANSITION,
	NULL, CRM_SYSTEM_TENGINE, NULL);
	} else {
	send_request(NULL, graph, CRM_OP_ABORT,
	NULL, CRM_SYSTEM_TENGINE, NULL);
	}

	// only free it if it was a local copy
	if(data == NULL) {
	free_xml(msg);
	}

	FNRET(I_NULL);
	}

	gboolean
	crmd_client_connect(IPC_Channel *client_channel, gpointer user_data)
	{
	FNIN();

	if (client_channel == NULL) {
	cl_log(LOG_ERR, "Channel was NULL");
	} else if (client_channel->ch_status == IPC_DISCONNECT) {
	cl_log(LOG_ERR, "Channel was disconnected");
	} else {
	crmd_client_t *blank_client =
	(crmd_client_t *)crm_malloc(sizeof(crmd_client_t));

	if (blank_client == NULL) {
	cl_log(LOG_ERR,
	"Could not allocate memory for a blank crmd_client_t");
	FNRET(FALSE);
	}
	client_channel->ops->set_recv_qlen(client_channel, 100);
	client_channel->ops->set_send_qlen(client_channel, 100);

	blank_client->client_channel = client_channel;
	blank_client->sub_sys = NULL;
	blank_client->uuid = NULL;
	blank_client->table_key = NULL;

	blank_client->client_source =
	G_main_add_IPC_Channel(G_PRIORITY_LOW,
	client_channel,
	FALSE,
	crmd_ipc_input_callback,
	blank_client,
	default_ipc_input_destroy);
	}

	FNRET(TRUE);
	}

	static gboolean
	stop_subsystem(struct crm_subsystem_s* centry)
	{
	cl_log(LOG_INFO, "Stopping sub-system \"%s\"", centry->name);
	if (centry->pid <= 0) {
	cl_log(LOG_ERR,
	"OOPS! client %s not running yet",
	centry->command);

	} else {
	cl_log(LOG_INFO, "Sending quit message to %s.", centry->name);
	send_request(NULL, NULL, CRM_OP_QUIT, NULL, centry->name, NULL);

	}

	return TRUE;
	}


	static gboolean
	start_subsystem(struct crm_subsystem_s* centry)
	{
	pid_t pid;
	struct stat buf;
	int s_res;

	cl_log(LOG_INFO, "Starting sub-system \"%s\"", centry->command);

	if (centry->pid != 0) {
	cl_log(LOG_ERR, "OOPS! client %s already running as pid %d"
	, centry->command, (int) centry->pid);
	}

	/*
	* We need to ensure that the exec will succeed before
	* we bother forking. We don't want to respawn something that
	* won't exec in the first place.
	*/

	if (access(centry->path, F_OK\|X_OK) != 0) {
	cl_perror("Cannot (access) exec %s", centry->path);
	return FALSE;
	}

	s_res = stat(centry->command, &buf);
	if(s_res != 0) {
	cl_perror("Cannot (stat) exec %s", centry->command);
	return FALSE;
	}


	/* We need to fork so we can make child procs not real time */
	switch(pid=fork()) {

	case -1:
	cl_log(LOG_ERR, "start_a_child_client: Cannot fork.");
	return FALSE;

	default: /* Parent */
	centry->pid = pid;
	return TRUE;

	case 0: /* Child */
	break;
	}

	/* Child process: start the managed child */
	cl_make_normaltime();
	setpgid(0,0);

	/* Limit peak resource usage, maximize success chances */
	if (centry->shortrcount > 0) {
	alarm(0);
	sleep(1);
	}

	cl_log(LOG_INFO, "Executing \"%s\" (pid %d)",
	centry->command, (int) getpid());

	if(CL_SIGINTERRUPT(SIGALRM, 0) < 0) {
	cl_perror("Cannot set interrupt for child process %s",
	centry->command);
	}else{
	const char * devnull = "/dev/null";
	unsigned int j;
	struct rlimit oflimits;
	CL_SIGNAL(SIGCHLD, SIG_DFL);
	alarm(0);
	CL_IGNORE_SIG(SIGALRM);

	/* A precautionary measure */
	getrlimit(RLIMIT_NOFILE, &oflimits);
	for (j=0; j < oflimits.rlim_cur; ++j) {
	close(j);
	}
	(void)devnull;

	(void)open(devnull, O_RDONLY); /* Stdin: fd 0 */
	(void)open(devnull, O_WRONLY); /* Stdout: fd 1 */
	(void)open(devnull, O_WRONLY); /* Stderr: fd 2 */

	(void)execl("/bin/sh", "sh", "-c", centry->command, (const char *)NULL);

	/* Should not happen */
	cl_perror("Cannot exec %s", centry->command);
	}
	/* Suppress respawning */
	exit(100);

	// never reached
	return TRUE;
	}



	diff --git a/crm/crmd/utils.c b/crm/crmd/utils.c
	index c87ab7dc99..1b956a4e9d 100644
	--- a/crm/crmd/utils.c
	+++ b/crm/crmd/utils.c
	@@ -1,648 +1,643 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <portability.h>
	#include <crm/crm.h>
	#include <crmd_fsa.h>
	-#include <libxml/tree.h>
	-#include <crm/msg_xml.h>

	#include <clplumbing/Gmain_timeout.h>
	#include <sys/types.h>
	#include <sys/wait.h>

	-#include <crm/common/xmlutils.h>
	-#include <crm/common/ipcutils.h>
	-#include <crm/common/msgutils.h>
	-#include <crm/cib.h>
	-#include <string.h>
	-#include <crmd_messages.h>
	+#include <crm/msg_xml.h>
	+#include <crm/common/xml.h>
	+#include <crm/common/msg.h>

	#include <crm/dmalloc_wrapper.h>


	gboolean
	timer_popped(gpointer data)
	{
	fsa_timer_t timer = (fsa_timer_t )data;

	cl_log(LOG_INFO, "#!!#!!# Timer %s just popped!",
	fsa_input2string(timer->fsa_input));

	stopTimer(timer); // dont make it go off again

	s_crmd_fsa(C_TIMER_POPPED, timer->fsa_input, NULL);

	return TRUE;
	}

	gboolean
	startTimer(fsa_timer_t *timer)
	{
	if(((int)timer->source_id) < 0) {
	timer->source_id =
	Gmain_timeout_add(timer->period_ms,
	timer->callback,
	(void*)timer);
	/*
	CRM_DEBUG("#!!#!!# Started %s timer (%d)",
	fsa_input2string(timer->fsa_input),
	timer->source_id);
	*/
	} else {
	cl_log(LOG_INFO, "#!!#!!# Timer %s already running (%d)",
	fsa_input2string(timer->fsa_input),
	timer->source_id);
	return FALSE;
	}
	return TRUE;
	}


	gboolean
	stopTimer(fsa_timer_t *timer)
	{
	if(((int)timer->source_id) > 0) {
	/*
	CRM_DEBUG("#!!#!!# Stopping %s timer (%d)",
	fsa_input2string(timer->fsa_input),
	timer->source_id);
	*/
	g_source_remove(timer->source_id);
	timer->source_id = -2;

	} else {
	cl_log(LOG_INFO, "#!!#!!# Timer %s already stopped (%d)",
	fsa_input2string(timer->fsa_input),
	timer->source_id);
	return FALSE;
	}
	return TRUE;
	}


	long long
	toggle_bit(long long action_list, long long action)
	{
	// CRM_DEBUG("Toggling bit %.16llx", action);
	action_list ^= action;
	// CRM_DEBUG("Result %.16llx", action_list & action);
	return action_list;
	}

	long long
	clear_bit(long long action_list, long long action)
	{
	// CRM_DEBUG("Clearing bit\t%.16llx", action);

	// ensure its set
	action_list \|= action;

	// then toggle
	action_list = action_list ^ action;

	return action_list;
	}

	long long
	set_bit(long long action_list, long long action)
	{
	// CRM_DEBUG("Adding bit\t%.16llx", action);
	action_list \|= action;
	return action_list;
	}

	void
	toggle_bit_inplace(long long *action_list, long long action)
	{
	action_list = toggle_bit(action_list, action);
	}

	void
	clear_bit_inplace(long long *action_list, long long action)
	{
	action_list = clear_bit(action_list, action);
	}

	void
	set_bit_inplace(long long *action_list, long long action)
	{
	action_list = set_bit(action_list, action);
	}



	gboolean
	is_set(long long action_list, long long action)
	{
	// CRM_DEBUG("Checking bit\t%.16llx", action);
	return ((action_list & action) == action);
	}


	xmlNodePtr
	create_node_state(const char *node,
	const char *ccm_state,
	const char *crmd_state,
	const char *join_state)
	{
	xmlNodePtr node_state = create_xml_node(NULL, XML_CIB_TAG_STATE);

	set_xml_property_copy(node_state, XML_ATTR_ID, node);
	if(ccm_state != NULL) {
	set_xml_property_copy(node_state, XML_CIB_ATTR_INCCM, ccm_state);
	}

	if(crmd_state != NULL) {
	set_xml_property_copy(node_state, XML_CIB_ATTR_CRMDSTATE, crmd_state);
	}

	if(join_state != NULL) {
	set_xml_property_copy(node_state, XML_CIB_ATTR_JOINSTATE, join_state);
	}

	/* if(exp_state != NULL) { */
	/* set_xml_property_copy(node_state, XML_CIB_ATTR_EXPSTATE, exp_state); */
	/* } */

	/* if(lrm_data != NULL) { */
	/* // set_xml_property_copy(data, "replace", XML_CIB_TAG_LRM); */
	/* add_node_copy(node_state, lrm_data); */
	/* } */

	xml_message_debug(node_state, "created");

	return node_state;
	}

	const char *
	fsa_input2string(enum crmd_fsa_input input)
	{
	const char *inputAsText = NULL;

	switch(input){
	case I_NULL:
	inputAsText = "I_NULL";
	break;
	case I_CCM_EVENT:
	inputAsText = "I_CCM_EVENT";
	break;
	case I_CIB_OP:
	inputAsText = "I_CIB_OP";
	break;
	case I_CIB_UPDATE:
	inputAsText = "I_CIB_UPDATE";
	break;
	case I_DC_TIMEOUT:
	inputAsText = "I_DC_TIMEOUT";
	break;
	case I_ELECTION:
	inputAsText = "I_ELECTION";
	break;
	case I_PE_CALC:
	inputAsText = "I_PE_CALC";
	break;
	case I_RELEASE_DC:
	inputAsText = "I_RELEASE_DC";
	break;
	case I_ELECTION_DC:
	inputAsText = "I_ELECTION_DC";
	break;
	case I_ERROR:
	inputAsText = "I_ERROR";
	break;
	case I_FAIL:
	inputAsText = "I_FAIL";
	break;
	case I_INTEGRATION_TIMEOUT:
	inputAsText = "I_INTEGRATION_TIMEOUT";
	break;
	case I_NODE_JOIN:
	inputAsText = "I_NODE_JOIN";
	break;
	case I_NODE_LEFT:
	inputAsText = "I_NODE_LEFT";
	break;
	case I_NOT_DC:
	inputAsText = "I_NOT_DC";
	break;
	case I_RECOVERED:
	inputAsText = "I_RECOVERED";
	break;
	case I_RELEASE_FAIL:
	inputAsText = "I_RELEASE_FAIL";
	break;
	case I_RELEASE_SUCCESS:
	inputAsText = "I_RELEASE_SUCCESS";
	break;
	case I_RESTART:
	inputAsText = "I_RESTART";
	break;
	case I_REQUEST:
	inputAsText = "I_REQUEST";
	break;
	case I_ROUTER:
	inputAsText = "I_ROUTER";
	break;
	case I_SHUTDOWN:
	inputAsText = "I_SHUTDOWN";
	break;
	case I_STARTUP:
	inputAsText = "I_STARTUP";
	break;
	case I_SUCCESS:
	inputAsText = "I_SUCCESS";
	break;
	case I_TERMINATE:
	inputAsText = "I_TERMINATE";
	break;
	case I_WELCOME:
	inputAsText = "I_WELCOME";
	break;
	case I_WELCOME_ACK:
	inputAsText = "I_WELCOME_ACK";
	break;
	case I_DC_HEARTBEAT:
	inputAsText = "I_DC_HEARTBEAT";
	break;
	case I_WAIT_FOR_EVENT:
	inputAsText = "I_WAIT_FOR_EVENT";
	break;
	case I_LRM_EVENT:
	inputAsText = "I_LRM_EVENT";
	break;
	case I_ILLEGAL:
	inputAsText = "I_ILLEGAL";
	break;
	}

	if(inputAsText == NULL) {
	cl_log(LOG_ERR, "Input %d is unknown", input);
	inputAsText = "<UNKNOWN_INPUT>";
	}

	return inputAsText;
	}

	const char *
	fsa_state2string(enum crmd_fsa_state state)
	{
	const char *stateAsText = NULL;

	switch(state){
	case S_IDLE:
	stateAsText = "S_IDLE";
	break;
	case S_ELECTION:
	stateAsText = "S_ELECTION";
	break;
	case S_INTEGRATION:
	stateAsText = "S_INTEGRATION";
	break;
	case S_NOT_DC:
	stateAsText = "S_NOT_DC";
	break;
	case S_POLICY_ENGINE:
	stateAsText = "S_POLICY_ENGINE";
	break;
	case S_RECOVERY:
	stateAsText = "S_RECOVERY";
	break;
	case S_RECOVERY_DC:
	stateAsText = "S_RECOVERY_DC";
	break;
	case S_RELEASE_DC:
	stateAsText = "S_RELEASE_DC";
	break;
	case S_PENDING:
	stateAsText = "S_PENDING";
	break;
	case S_STOPPING:
	stateAsText = "S_STOPPING";
	break;
	case S_TERMINATE:
	stateAsText = "S_TERMINATE";
	break;
	case S_TRANSITION_ENGINE:
	stateAsText = "S_TRANSITION_ENGINE";
	break;
	case S_ILLEGAL:
	stateAsText = "S_ILLEGAL";
	break;
	}

	if(stateAsText == NULL) {
	cl_log(LOG_ERR, "State %d is unknown", state);
	stateAsText = "<UNKNOWN_STATE>";
	}

	return stateAsText;
	}

	const char *
	fsa_cause2string(enum crmd_fsa_cause cause)
	{
	const char *causeAsText = NULL;

	switch(cause){
	case C_UNKNOWN:
	causeAsText = "C_UNKNOWN";
	break;
	case C_STARTUP:
	causeAsText = "C_STARTUP";
	break;
	case C_IPC_MESSAGE:
	causeAsText = "C_IPC_MESSAGE";
	break;
	case C_HA_MESSAGE:
	causeAsText = "C_HA_MESSAGE";
	break;
	case C_CCM_CALLBACK:
	causeAsText = "C_CCM_CALLBACK";
	break;
	case C_TIMER_POPPED:
	causeAsText = "C_TIMER_POPPED";
	break;
	case C_SHUTDOWN:
	causeAsText = "C_SHUTDOWN";
	break;
	case C_HEARTBEAT_FAILED:
	causeAsText = "C_HEARTBEAT_FAILED";
	break;
	case C_SUBSYSTEM_CONNECT:
	causeAsText = "C_SUBSYSTEM_CONNECT";
	break;
	case C_LRM_OP_CALLBACK:
	causeAsText = "C_LRM_OP_CALLBACK";
	break;
	case C_LRM_MONITOR_CALLBACK:
	causeAsText = "C_LRM_MONITOR_CALLBACK";
	break;
	case C_CRMD_STATUS_CALLBACK:
	causeAsText = "C_CRMD_STATUS_CALLBACK";
	break;
	case C_ILLEGAL:
	causeAsText = "C_ILLEGAL";
	break;
	}

	if(causeAsText == NULL) {
	cl_log(LOG_ERR, "Cause %d is unknown", cause);
	causeAsText = "<UNKNOWN_CAUSE>";
	}

	return causeAsText;
	}

	const char *
	fsa_action2string(long long action)
	{
	const char *actionAsText = NULL;

	switch(action){

	case A_NOTHING:
	actionAsText = "A_NOTHING";
	break;
	case O_SHUTDOWN:
	actionAsText = "O_SHUTDOWN";
	break;
	case O_RELEASE:
	actionAsText = "O_RELEASE";
	break;
	case A_STARTUP:
	actionAsText = "A_STARTUP";
	break;
	case A_STARTED:
	actionAsText = "A_STARTED";
	break;
	case A_HA_CONNECT:
	actionAsText = "A_HA_CONNECT";
	break;
	case A_HA_DISCONNECT:
	actionAsText = "A_HA_DISCONNECT";
	break;
	case A_LRM_CONNECT:
	actionAsText = "A_LRM_CONNECT";
	break;
	case A_LRM_DISCONNECT:
	actionAsText = "A_LRM_DISCONNECT";
	break;
	case O_DC_TIMER_RESTART:
	actionAsText = "O_DC_TIMER_RESTART";
	break;
	case A_DC_TIMER_STOP:
	actionAsText = "A_DC_TIMER_STOP";
	break;
	case A_DC_TIMER_START:
	actionAsText = "A_DC_TIMER_START";
	break;
	case A_ELECTION_COUNT:
	actionAsText = "A_ELECTION_COUNT";
	break;
	case A_ELECTION_TIMEOUT:
	actionAsText = "A_ELECTION_TIMEOUT";
	break;
	case A_ELECT_TIMER_START:
	actionAsText = "A_ELECT_TIMER_START";
	break;
	case A_ELECT_TIMER_STOP:
	actionAsText = "A_ELECT_TIMER_STOP";
	break;
	case A_ELECTION_VOTE:
	actionAsText = "A_ELECTION_VOTE";
	break;
	case A_ANNOUNCE:
	actionAsText = "A_ANNOUNCE";
	break;
	case A_JOIN_ACK:
	actionAsText = "A_JOIN_ACK";
	break;
	case A_JOIN_WELCOME:
	actionAsText = "A_JOIN_WELCOME";
	break;
	case A_JOIN_WELCOME_ALL:
	actionAsText = "A_JOIN_WELCOME_ALL";
	break;
	case A_JOIN_PROCESS_ACK:
	actionAsText = "A_JOIN_PROCESS_ACK";
	break;
	case A_MSG_PROCESS:
	actionAsText = "A_MSG_PROCESS";
	break;
	case A_MSG_ROUTE:
	actionAsText = "A_MSG_ROUTE";
	break;
	case A_MSG_STORE:
	actionAsText = "A_MSG_STORE";
	break;
	case A_RECOVER:
	actionAsText = "A_RECOVER";
	break;
	case A_DC_RELEASE:
	actionAsText = "A_DC_RELEASE";
	break;
	case A_DC_RELEASED:
	actionAsText = "A_DC_RELEASED";
	break;
	case A_DC_TAKEOVER:
	actionAsText = "A_DC_TAKEOVER";
	break;
	case A_SHUTDOWN:
	actionAsText = "A_SHUTDOWN";
	break;
	case A_SHUTDOWN_REQ:
	actionAsText = "A_SHUTDOWN_REQ";
	break;
	case A_STOP:
	actionAsText = "A_STOP ";
	break;
	case A_EXIT_0:
	actionAsText = "A_EXIT_0";
	break;
	case A_EXIT_1:
	actionAsText = "A_EXIT_1";
	break;
	case A_CCM_CONNECT:
	actionAsText = "A_CCM_CONNECT";
	break;
	case A_CCM_DISCONNECT:
	actionAsText = "A_CCM_DISCONNECT";
	break;
	case A_CCM_EVENT:
	actionAsText = "A_CCM_EVENT";
	break;
	case A_CCM_UPDATE_CACHE:
	actionAsText = "A_CCM_UPDATE_CACHE";
	break;
	case A_CIB_BUMPGEN:
	actionAsText = "A_CIB_BUMPGEN";
	break;
	case A_CIB_INVOKE:
	actionAsText = "A_CIB_INVOKE";
	break;
	case O_CIB_RESTART:
	actionAsText = "O_CIB_RESTART";
	break;
	case A_CIB_START:
	actionAsText = "A_CIB_START";
	break;
	case A_CIB_STOP:
	actionAsText = "A_CIB_STOP";
	break;
	case A_TE_INVOKE:
	actionAsText = "A_TE_INVOKE";
	break;
	case O_TE_RESTART:
	actionAsText = "O_TE_RESTART";
	break;
	case A_TE_START:
	actionAsText = "A_TE_START";
	break;
	case A_TE_STOP:
	actionAsText = "A_TE_STOP";
	break;
	case A_TE_CANCEL:
	actionAsText = "A_TE_CANCEL";
	break;
	case A_TE_COPYTO:
	actionAsText = "A_TE_COPYTO";
	break;
	case A_PE_INVOKE:
	actionAsText = "A_PE_INVOKE";
	break;
	case O_PE_RESTART:
	actionAsText = "O_PE_RESTART";
	break;
	case A_PE_START:
	actionAsText = "A_PE_START";
	break;
	case A_PE_STOP:
	actionAsText = "A_PE_STOP";
	break;
	case A_NODE_BLOCK:
	actionAsText = "A_NODE_BLOCK";
	break;
	case A_UPDATE_NODESTATUS:
	actionAsText = "A_UPDATE_NODESTATUS";
	break;
	case A_LOG:
	actionAsText = "A_LOG ";
	break;
	case A_ERROR:
	actionAsText = "A_ERROR ";
	break;
	case A_WARN:
	actionAsText = "A_WARN ";
	break;
	}

	if(actionAsText == NULL) {
	cl_log(LOG_ERR, "Action %.16llx is unknown", action);
	actionAsText = "<UNKNOWN_ACTION>";
	}

	return actionAsText;
	}


	void
	cleanup_subsystem(struct crm_subsystem_s *the_subsystem)
	{
	int pid_status = -1;
	the_subsystem->ipc = NULL;
	clear_bit_inplace(&fsa_input_register,
	the_subsystem->flag);

	/* Forcing client to die */
	kill(the_subsystem->pid, -9);

	// cleanup the ps entry
	waitpid(the_subsystem->pid, &pid_status, WNOHANG);
	the_subsystem->pid = -1;
	}

	enum crmd_fsa_input
	invoke_local_cib(xmlNodePtr msg_options,
	xmlNodePtr msg_data,
	const char *operation)
	{
	enum crmd_fsa_input result = I_NULL;
	xmlNodePtr request = NULL;
	FNIN();

	msg_options = set_xml_attr(msg_options, XML_TAG_OPTIONS,
	XML_ATTR_OP, operation, TRUE);

	request = create_request(msg_options,
	msg_data,
	NULL,
	CRM_SYSTEM_CIB,
	AM_I_DC?CRM_SYSTEM_DC:CRM_SYSTEM_CRMD,
	NULL,
	NULL);

	result = do_cib_invoke(A_CIB_INVOKE_LOCAL,
	C_UNKNOWN,
	fsa_state,
	I_CIB_OP,
	request);

	free_xml(request);

	FNRET(result);
	}

File Metadata

Mime Type: text/x-diff
Expires: Wed, Jun 4, 6:23 AM (6 h, 52 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1845448
Default Alt Text: (167 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions