No OneTemporary
Actions

Size

55 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/crm/crmd/fsa.c b/crm/crmd/fsa.c
	index f092d441ae..5db69dfbdf 100644
	--- a/crm/crmd/fsa.c
	+++ b/crm/crmd/fsa.c
	@@ -1,749 +1,753 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <sys/param.h>
	#include <stdio.h>
	#include <string.h>
	#include <time.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>
	#include <crm/common/msg.h>

	#include <clplumbing/Gmain_timeout.h>

	#include <crmd_messages.h>
	#include <crmd_fsa.h>
	#include <fsa_proto.h>
	#include <fsa_matrix.h>

	#include <crm/dmalloc_wrapper.h>

	extern int num_join_invites;
	extern GHashTable *join_requests;
	extern GHashTable *confirmed_nodes;
	+extern void initialize_join(gboolean before);

	long long
	do_state_transition(long long actions,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_state next_state,
	enum crmd_fsa_input current_input,
	fsa_data_t *msg_data);

	long long clear_flags(long long actions,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input cur_input);

	void dump_rsc_info(void);

	#ifdef DOT_FSA_ACTIONS
	# ifdef FSA_TRACE
	# define IF_FSA_ACTION(x,y) \
	if(is_set(actions,x)) { \
	last_action = x; \
	actions = clear_bit(actions, x); \
	crm_verbose("Invoking action %s (%.16llx)", \
	fsa_action2string(x), x); \
	next_input = y(x, cause, cur_state, last_input, fsa_data); \
	crm_verbose("Action complete: %s (%.16llx)", \
	fsa_action2string(x), x); \
	if(next_input != I_NULL) { \
	crm_warn("Action %s returned %s", \
	fsa_action2string(x), \
	fsa_input2string(next_input)); \
	} \
	if((x & O_DC_TICKLE) == 0 && next_input != I_DC_HEARTBEAT ){ \
	fprintf(dot_strm, "\t// %s\n", \
	fsa_action2string(x)); \
	} \
	fflush(dot_strm); \
	}
	# else
	# define IF_FSA_ACTION(x,y) \
	if(is_set(actions,x)) { \
	last_action = x; \
	actions = clear_bit(actions, x); \
	next_input = y(x, cause, cur_state, last_input, fsa_data); \
	if( (x & O_DC_TICKLE) == 0 && next_input != I_DC_HEARTBEAT ) \
	fprintf(dot_strm, "\t// %s\n", fsa_action2string(x)); \
	fflush(dot_strm); \
	}
	# endif
	#else
	# ifdef FSA_TRACE
	# define IF_FSA_ACTION(x,y) \
	if(is_set(actions,x)) { \
	last_action = x; \
	actions = clear_bit(actions, x); \
	crm_verbose("Invoking action %s (%.16llx)", \
	fsa_action2string(x), x); \
	next_input = y(x, cause, cur_state, last_input, fsa_data); \
	crm_verbose("Action complete: %s (%.16llx)", \
	fsa_action2string(x), x); \
	if(next_input != I_NULL) { \
	crm_warn("Action %s returned %s", \
	fsa_action2string(x), \
	fsa_input2string(next_input)); \
	} \
	}
	# else
	# define IF_FSA_ACTION(x,y) \
	if(is_set(actions,x)) { \
	last_action = x; \
	actions = clear_bit(actions, x); \
	next_input = y(x, cause, cur_state, last_input, fsa_data); \
	}
	# endif
	#endif

	/* #define ELSEIF_FSA_ACTION(x,y) else IF_FSA_ACTION(x,y) */

	const char *dot_intro = "digraph \"g\" {\n"
	" size = \"30,30\"\n"
	" graph [\n"
	" fontsize = \"12\"\n"
	" fontname = \"Times-Roman\"\n"
	" fontcolor = \"black\"\n"
	" bb = \"0,0,398.922306,478.927856\"\n"
	" color = \"black\"\n"
	" ]\n"
	" node [\n"
	" fontsize = \"12\"\n"
	" fontname = \"Times-Roman\"\n"
	" fontcolor = \"black\"\n"
	" shape = \"ellipse\"\n"
	" color = \"black\"\n"
	" ]\n"
	" edge [\n"
	" fontsize = \"12\"\n"
	" fontname = \"Times-Roman\"\n"
	" fontcolor = \"black\"\n"
	" color = \"black\"\n"
	" ]\n"
	"// special nodes\n"
	" \"S_PENDING\" \n"
	" [\n"
	" color = \"blue\"\n"
	" fontcolor = \"blue\"\n"
	" ]\n"
	" \"S_TERMINATE\" \n"
	" [\n"
	" color = \"red\"\n"
	" fontcolor = \"red\"\n"
	" ]\n"
	"\n"
	"// DC only nodes\n"
	" \"S_INTEGRATION\" [ fontcolor = \"green\" ]\n"
	" \"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]\n"
	" \"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]\n"
	" \"S_RELEASE_DC\" [ fontcolor = \"green\" ]\n"
	" \"S_IDLE\" [ fontcolor = \"green\" ]\n";


	static FILE *dot_strm = NULL;

	volatile enum crmd_fsa_state fsa_state = S_STARTING;
	oc_node_list_t *fsa_membership_copy;
	ll_cluster_t *fsa_cluster_conn;
	ll_lrm_t *fsa_lrm_conn;
	volatile long long fsa_input_register;
	volatile long long fsa_actions = A_NOTHING;
	const char *fsa_our_uname;
	char *fsa_our_dc;

	fsa_timer_t election_trigger = NULL; / */
	fsa_timer_t election_timeout = NULL; / */
	fsa_timer_t shutdown_escalation_timer = NULL; / */
	fsa_timer_t *integration_timer = NULL;
	fsa_timer_t *finalization_timer = NULL;
	fsa_timer_t *dc_heartbeat = NULL;
	fsa_timer_t *wait_timer = NULL;

	int fsa_join_reannouce = 0;
	volatile gboolean do_fsa_stall = FALSE;


	enum crmd_fsa_state
	s_crmd_fsa(enum crmd_fsa_cause cause)
	{
	fsa_data_t *fsa_data = NULL;
	long long actions = fsa_actions;
	long long new_actions = A_NOTHING;
	long long last_action = A_NOTHING;
	enum crmd_fsa_input last_input = I_NULL;
	enum crmd_fsa_input cur_input = I_NULL;
	enum crmd_fsa_input next_input = I_NULL;
	enum crmd_fsa_state starting_state = fsa_state;
	enum crmd_fsa_state last_state = starting_state;
	enum crmd_fsa_state cur_state = starting_state;
	enum crmd_fsa_state next_state = starting_state;

	#ifdef FSA_TRACE
	crm_verbose("FSA invoked with Cause: %s\tState: %s",
	fsa_cause2string(cause),
	fsa_state2string(cur_state));
	#endif

	#ifdef DOT_FSA_ACTIONS
	if(dot_strm == NULL) {
	dot_strm = fopen(DEVEL_DIR"/live.dot", "w");
	fprintf(dot_strm, "%s", dot_intro);
	fflush(dot_strm);
	}
	#endif

	/*
	* Process actions in order of priority but do only one
	* action at a time to avoid complicating the ordering.
	*
	* Actions may result in a new I_ event, these are added to
	* (not replace) existing actions before the next iteration.
	*
	*/
	do_fsa_stall = FALSE;
	while(next_input != I_NULL \|\| actions != A_NOTHING \|\| is_message()) {

	if(do_fsa_stall) {
	/* we may be waiting for an a-sync task to "happen"
	* and until it does, we cant do anything else
	*/
	crm_info("Wait until something else happens");
	break;

	} else if((is_message() && fsa_data == NULL)
	\|\| (is_message() && actions == A_NOTHING && next_input == I_NULL)) {
	fsa_data_t *stored_msg = NULL;
	crm_debug("Finished with current input..."
	" Checking messages (%d remaining)",
	g_list_length(fsa_message_queue));

	next_input = I_NULL;
	stored_msg = get_message();

	if(stored_msg == NULL) {
	crm_crit("Invalid stored message");
	exit(1);
	}

	delete_fsa_input(fsa_data);

	if(stored_msg->fsa_cause == C_CCM_CALLBACK) {
	crm_devel("FSA processing CCM callback from %s",
	stored_msg->where);

	} else if(stored_msg->fsa_cause == C_LRM_OP_CALLBACK) {
	crm_devel("FSA processing LRM callback from %s",
	stored_msg->where);

	} else if(stored_msg->data == NULL) {
	crm_devel("FSA processing input from %s",
	stored_msg->where);

	} else {
	crm_devel("FSA processing XML message from %s",
	stored_msg->where);

	crm_xml_devel(stored_msg->data,
	"FSA processing message");
	}

	fsa_data = stored_msg;

	/* set up the input */
	next_input = fsa_data->fsa_input;
	/* add any actions back to the queue */
	actions \|= fsa_data->actions;
	/* update the cause */
	cause = fsa_data->fsa_cause;

	fsa_dump_actions(fsa_data->actions, "\tadded back");

	crm_debug("FSA input: State=%s\tCause=%s"
	"\tInput=%s\tOrigin=%s()",
	fsa_state2string(cur_state),
	fsa_cause2string(fsa_data->fsa_cause),
	fsa_input2string(fsa_data->fsa_input),
	fsa_data->where);
	#ifdef DOT_FSA_ACTIONS
	fprintf(dot_strm,
	"\t// FSA input: State=%s\tCause=%s"
	"\tInput=%s\tOrigin=%s()\n",
	fsa_state2string(cur_state),
	fsa_cause2string(fsa_data->fsa_cause),
	fsa_input2string(fsa_data->fsa_input),
	fsa_data->where);

	fflush(dot_strm);
	#endif

	} else if(fsa_data == NULL) {
	crm_malloc(fsa_data, sizeof(fsa_data_t));
	fsa_data->fsa_input = I_NULL;
	fsa_data->fsa_cause = cause;
	fsa_data->actions = A_NOTHING;
	fsa_data->where = crm_strdup("s_crmd_fsa (enter)");
	fsa_data->data = NULL;
	if(fsa_data->where == NULL) {
	crm_crit("Out of memory");
	exit(1);
	}

	}


	/* update input variables */
	cur_input = next_input;
	if(cur_input != I_NULL) {
	/* record the most recent non I_NULL input */
	crm_devel("Updating last_input to %s",
	fsa_input2string(cur_input));
	last_input = cur_input;
	}

	/* get the next batch of actions */
	new_actions = crmd_fsa_actions[cur_input][cur_state];
	if(new_actions != A_NOTHING) {
	#ifdef FSA_TRACE
	crm_verbose("Adding actions %.16llx", new_actions);
	fsa_dump_actions(new_actions, "\tscheduled");
	#endif
	actions \|= new_actions;
	}

	if(fsa_data == NULL) {
	crm_err("No input for FSA.... terminating");
	exit(1);
	}

	#ifdef FSA_TRACE
	crm_verbose("FSA while loop:\tState: %s, Cause: %s,"
	" Input: %s, Origin=%s",
	fsa_state2string(cur_state),
	fsa_cause2string(fsa_data->fsa_cause),
	fsa_input2string(fsa_data->fsa_input),
	fsa_data->where);
	#endif


	/* logging : before the state is changed */
	IF_FSA_ACTION(A_ERROR, do_log)
	else IF_FSA_ACTION(A_WARN, do_log)
	else IF_FSA_ACTION(A_LOG, do_log)

	/* update state variables */
	next_state = crmd_fsa_state[cur_input][cur_state];
	last_state = cur_state;
	cur_state = next_state;
	fsa_state = next_state;

	/* start doing things... */

	/*
	* Hook for change of state.
	* Allows actions to be added or removed when entering a state
	*/
	if(last_state != cur_state){
	actions = do_state_transition(
	actions, cause, last_state, cur_state,
	cur_input, fsa_data);
	}

	/* this is always run, some inputs/states may make various
	* actions irrelevant/invalid
	*/
	actions = clear_flags(actions, cause, cur_state, cur_input);


	/* regular action processing in order of action priority
	*
	* Make sure all actions that connect to required systems
	* are performed first
	*/

	/* get out of here NOW! before anything worse happens */
	IF_FSA_ACTION(A_EXIT_1, do_exit)

	/* essential start tasks */
	else IF_FSA_ACTION(A_HA_CONNECT, do_ha_control)
	else IF_FSA_ACTION(A_STARTUP, do_startup)
	else IF_FSA_ACTION(A_CIB_START, do_cib_control)
	else IF_FSA_ACTION(A_READCONFIG, do_read_config)

	/* sub-system start/connect */
	else IF_FSA_ACTION(A_LRM_CONNECT, do_lrm_control)
	else IF_FSA_ACTION(A_CCM_CONNECT, do_ccm_control)
	else IF_FSA_ACTION(A_TE_START, do_te_control)
	else IF_FSA_ACTION(A_PE_START, do_pe_control)

	/* sub-system restart
	*/
	else IF_FSA_ACTION(O_CIB_RESTART, do_cib_control)
	else IF_FSA_ACTION(O_PE_RESTART, do_pe_control)
	else IF_FSA_ACTION(O_TE_RESTART, do_te_control)

	/* Timers */
	else IF_FSA_ACTION(O_DC_TIMER_RESTART, do_timer_control)
	else IF_FSA_ACTION(A_DC_TIMER_STOP, do_timer_control)
	else IF_FSA_ACTION(A_DC_TIMER_START, do_timer_control)
	else IF_FSA_ACTION(A_INTEGRATE_TIMER_STOP, do_timer_control)
	else IF_FSA_ACTION(A_INTEGRATE_TIMER_START, do_timer_control)
	else IF_FSA_ACTION(A_FINALIZE_TIMER_STOP, do_timer_control)
	else IF_FSA_ACTION(A_FINALIZE_TIMER_START, do_timer_control)

	/*
	* Highest priority actions
	*/
	else IF_FSA_ACTION(A_TE_COPYTO, do_te_copyto)
	else IF_FSA_ACTION(A_CIB_BUMPGEN, do_cib_invoke)

	else IF_FSA_ACTION(A_MSG_ROUTE, do_msg_route)
	else IF_FSA_ACTION(A_RECOVER, do_recover)
	else IF_FSA_ACTION(A_CL_JOIN_REQUEST, do_cl_join_request)
	else IF_FSA_ACTION(A_CL_JOIN_RESULT, do_cl_join_result)
	else IF_FSA_ACTION(A_SHUTDOWN_REQ, do_shutdown_req)
	else IF_FSA_ACTION(A_ELECTION_VOTE, do_election_vote)
	else IF_FSA_ACTION(A_ELECTION_COUNT, do_election_count_vote)

	/*
	* "Get this over with" actions
	*/
	else IF_FSA_ACTION(A_MSG_STORE, do_msg_store)

	/*
	* High priority actions
	* Update the cache first
	*/
	else IF_FSA_ACTION(A_CCM_UPDATE_CACHE, do_ccm_update_cache)
	else IF_FSA_ACTION(A_CCM_EVENT, do_ccm_event)
	else IF_FSA_ACTION(A_STARTED, do_started)

	/*
	* Medium priority actions
	*/
	else IF_FSA_ACTION(A_DC_TAKEOVER, do_dc_takeover)
	else IF_FSA_ACTION(A_DC_RELEASE, do_dc_release)
	else IF_FSA_ACTION(A_DC_JOIN_OFFER_ALL, do_dc_join_offer_all)
	else IF_FSA_ACTION(A_DC_JOIN_OFFER_ONE, do_dc_join_offer_one)
	else IF_FSA_ACTION(A_DC_JOIN_PROCESS_REQ,do_dc_join_req)
	else IF_FSA_ACTION(A_DC_JOIN_PROCESS_ACK,do_dc_join_ack)

	/*
	* Low(er) priority actions
	* Make sure the CIB is always updated before invoking the
	* PE, and the PE before the TE
	*/
	else IF_FSA_ACTION(A_CIB_INVOKE_LOCAL, do_cib_invoke)
	else IF_FSA_ACTION(A_CIB_INVOKE, do_cib_invoke)
	else IF_FSA_ACTION(A_DC_JOIN_FINALIZE, do_dc_join_finalize)
	else IF_FSA_ACTION(A_LRM_INVOKE, do_lrm_invoke)
	else IF_FSA_ACTION(A_LRM_EVENT, do_lrm_event)
	else IF_FSA_ACTION(A_TE_CANCEL, do_te_invoke)
	else IF_FSA_ACTION(A_PE_INVOKE, do_pe_invoke)
	else IF_FSA_ACTION(A_TE_INVOKE, do_te_invoke)
	else IF_FSA_ACTION(A_CL_JOIN_ANNOUNCE, do_cl_join_announce)

	/* sub-system stop */
	else IF_FSA_ACTION(A_PE_STOP, do_pe_control)
	else IF_FSA_ACTION(A_TE_STOP, do_te_control)
	else IF_FSA_ACTION(A_DC_RELEASED, do_dc_release)

	else IF_FSA_ACTION(A_HA_DISCONNECT, do_ha_control)
	else IF_FSA_ACTION(A_CCM_DISCONNECT, do_ccm_control)
	else IF_FSA_ACTION(A_LRM_DISCONNECT, do_lrm_control)
	else IF_FSA_ACTION(A_CIB_STOP, do_cib_control)

	/* time to go now... */

	/* Some of these can probably be consolidated */
	else IF_FSA_ACTION(A_SHUTDOWN, do_shutdown)
	else IF_FSA_ACTION(A_STOP, do_stop)

	/* exit gracefully */
	else IF_FSA_ACTION(A_EXIT_0, do_exit)

	/* else IF_FSA_ACTION(A_, do_) */

	/* Error checking and reporting */
	else if(cur_input != I_NULL && actions == A_NOTHING) {
	crm_warn(
	"No action specified for input,state (%s,%s)",
	fsa_input2string(cur_input),
	fsa_state2string(cur_state));

	next_input = I_NULL;

	} else if(cur_input == I_NULL && actions == A_NOTHING) {
	#ifdef FSA_TRACE
	crm_info("Nothing left to do...");
	fsa_dump_actions(actions, "still here");
	#endif
	break;

	} else {
	crm_err("Action %s (0x%llx) not supported ",
	fsa_action2string(actions), actions);
	next_input = I_ERROR;
	}

	}

	#ifdef FSA_TRACE
	crm_verbose("################# Exiting the FSA (%s) ##################",
	fsa_state2string(fsa_state));
	#endif

	#ifdef DOT_FSA_ACTIONS
	fprintf(dot_strm,
	"\t// ### Exiting the FSA (%s)\n",
	fsa_state2string(fsa_state));
	fflush(dot_strm);
	#endif

	/* cleanup inputs? */
	fsa_actions = actions;
	delete_fsa_input(fsa_data);

	return fsa_state;
	}




	long long
	do_state_transition(long long actions,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_state next_state,
	enum crmd_fsa_input current_input,
	fsa_data_t *msg_data)
	{
	gboolean clear_recovery_bit = TRUE;
	long long tmp = actions;
	const char *state_from = fsa_state2string(cur_state);
	const char *state_to = fsa_state2string(next_state);
	const char *input = fsa_input2string(current_input);

	time_t now = time(NULL);

	if(cur_state == next_state) {
	crm_err("%s called in state %s with no transtion",
	__FUNCTION__, state_from);
	return A_NOTHING;
	}

	if(current_input != I_DC_HEARTBEAT && cur_state != S_NOT_DC){
	fprintf(dot_strm,
	"\t\"%s\" -> \"%s\" [ label=\"%s\" cause=%s origin=%s ] // %s",
	state_from, state_to, input, fsa_cause2string(cause), msg_data->where,
	asctime(localtime(&now)));
	fflush(dot_strm);
	}

	crm_info("State transition \"%s\" -> \"%s\""
	" [ input=%s cause=%s origin=%s %s ]",
	state_from, state_to, input,
	fsa_cause2string(cause), msg_data->where,
	asctime(localtime(&now)));

	if(next_state != S_ELECTION) {
	stopTimer(election_timeout);
	/* } else { */
	/* startTimer(election_timeout); */
	}

	if(is_set(fsa_input_register, R_SHUTDOWN)){
	set_bit_inplace(tmp, A_DC_TIMER_STOP);

	} else if(next_state == S_PENDING \|\| next_state == S_NOT_DC) {
	set_bit_inplace(tmp, A_DC_TIMER_START);

	/* } else we are the DC and dont want to shut down { */
	}

	switch(next_state) {
	case S_PENDING:
	case S_ELECTION:
	crm_info("Resetting our DC to NULL on election");
	crm_free(fsa_our_dc);
	fsa_our_dc = NULL;
	break;
	case S_NOT_DC:
	if(is_set(fsa_input_register, R_SHUTDOWN)){
	crm_info("(Re)Issuing shutdown request now"
	" that we have a new DC");
	set_bit_inplace(tmp, A_SHUTDOWN_REQ);
	}

	if(fsa_our_dc == NULL) {
	crm_err("Reached S_NOT_DC without a DC"
	" being recorded");
	}
	break;
	case S_RECOVERY:
	clear_recovery_bit = FALSE;
	break;

	case S_INTEGRATION:
	+ initialize_join(TRUE);
	set_bit_inplace(tmp, A_INTEGRATE_TIMER_START);
	set_bit_inplace(tmp, A_FINALIZE_TIMER_STOP);
	+
	break;

	case S_FINALIZE_JOIN:
	set_bit_inplace(tmp, A_INTEGRATE_TIMER_STOP);
	set_bit_inplace(tmp, A_FINALIZE_TIMER_START);

	if(cause != C_FSA_INTERNAL) {
	crm_warn("Progressed to state %s after %s",
	fsa_state2string(cur_state),
	fsa_cause2string(cause));
	}
	if(g_hash_table_size(join_requests)
	!= fsa_membership_copy->members_size) {
	crm_warn("Only %d (of %d) cluster nodes "
	"responded to the join offer.",
	g_hash_table_size(join_requests),
	fsa_membership_copy->members_size);
	} else {
	crm_info("All %d clusters nodes "
	"responded to the join offer.",
	fsa_membership_copy->members_size);
	}
	break;

	case S_POLICY_ENGINE:
	+ initialize_join(FALSE);
	set_bit_inplace(tmp, A_FINALIZE_TIMER_STOP);
	if(cause != C_FSA_INTERNAL) {
	crm_warn("Progressed to state %s after %s",
	fsa_state2string(cur_state),
	fsa_cause2string(cause));
	}

	if(g_hash_table_size(confirmed_nodes)
	== fsa_membership_copy->members_size) {
	crm_info("All %d clusters nodes are"
	" eligable to run resources.",
	fsa_membership_copy->members_size);

	} else if(g_hash_table_size(confirmed_nodes)
	== num_join_invites) {
	crm_warn("All %d (%d total) cluster "
	"nodes are eligable to run resources",
	g_hash_table_size(confirmed_nodes),
	fsa_membership_copy->members_size);

	} else {
	crm_warn("Only %d of %d (%d total) cluster "
	"nodes are eligable to run resources",
	num_join_invites,
	g_hash_table_size(confirmed_nodes),
	fsa_membership_copy->members_size);
	}
	break;

	case S_IDLE:
	dump_rsc_info();
	break;

	default:
	break;
	}

	if(clear_recovery_bit && next_state != S_PENDING) {
	tmp = clear_bit(tmp, A_RECOVER);
	} else if(clear_recovery_bit == FALSE) {
	tmp = set_bit(tmp, A_RECOVER);
	}

	if(tmp != actions) {
	crm_info("Action b4 %.16llx ", actions);
	crm_info("Action after %.16llx ", tmp);
	actions = tmp;
	}

	return actions;
	}

	long long
	clear_flags(long long actions,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input cur_input)
	{

	if(is_set(fsa_input_register, R_SHUTDOWN)){
	clear_bit_inplace(actions, A_DC_TIMER_START);
	}

	if(cur_state == S_STOPPING) {
	clear_bit_inplace(
	actions,
	A_CCM_CONNECT\|A_STARTED\|A_LRM_CONNECT\|
	A_HA_CONNECT\|A_CIB_START);
	}

	return actions;
	}

	void
	dump_rsc_info(void)
	{
	xmlNodePtr local_cib = get_cib_copy();
	xmlNodePtr root = get_object_root(XML_CIB_TAG_STATUS, local_cib);
	xmlNodePtr resources = NULL;
	const char *rsc_id = NULL;
	const char *node_id = NULL;
	const char *rsc_state = NULL;
	const char *op_status = NULL;
	const char *last_rc = NULL;
	const char *last_op = NULL;


	const char *path[] = {
	XML_CIB_TAG_LRM,
	XML_LRM_TAG_RESOURCES
	};

	xml_child_iter(
	root, node, XML_CIB_TAG_STATE,

	resources = find_xml_node_nested(node, path, DIMOF(path));

	xml_child_iter(
	resources, rsc, XML_LRM_TAG_RESOURCE,

	rsc_id = xmlGetProp(rsc, XML_ATTR_ID);
	node_id = xmlGetProp(rsc, XML_LRM_ATTR_TARGET);
	rsc_state = xmlGetProp(rsc, XML_LRM_ATTR_RSCSTATE);
	op_status = xmlGetProp(rsc, XML_LRM_ATTR_OPSTATUS);
	last_rc = xmlGetProp(rsc, XML_LRM_ATTR_RC);
	last_op = xmlGetProp(rsc, XML_LRM_ATTR_LASTOP);

	/* if(safe_str_eq(rsc_state, "stopped")) { */
	/* continue; */
	/* } */

	crm_info("Resource state: %s %s "
	"[%s (rc=%s) after %s] on %s",
	rsc_id, rsc_state,
	op_status, last_rc, last_op, node_id);
	);
	);
	}
	diff --git a/crm/crmd/fsa_defines.h b/crm/crmd/fsa_defines.h
	index 0829b5485a..5269292a2e 100644
	--- a/crm/crmd/fsa_defines.h
	+++ b/crm/crmd/fsa_defines.h
	@@ -1,468 +1,468 @@
	-/* $Id: fsa_defines.h,v 1.25 2004/10/19 11:23:45 andrew Exp $ */
	+/* $Id: fsa_defines.h,v 1.26 2004/10/23 11:58:42 andrew Exp $ */
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#ifndef FSA_DEFINES__H
	#define FSA_DEFINES__H

	/*======================================
	* States the DC/CRMd can be in
	======================================/
	enum crmd_fsa_state {
	S_IDLE = 0, /* Nothing happening */

	S_ELECTION, /* Take part in the election algorithm as
	* described below
	*/
	S_INTEGRATION, /* integrate that status of new nodes (which is
	* all of them if we have just been elected DC)
	* to form a complete and up-to-date picture of
	* the CIB
	*/
	S_FINALIZE_JOIN,/* integrate that status of new nodes (which is
	* all of them if we have just been elected DC)
	* to form a complete and up-to-date picture of
	* the CIB
	*/
	S_NOT_DC, /* we are in crmd/slave mode */
	S_POLICY_ENGINE,/* Determin the next stable state of the cluster
	*/
	S_RECOVERY, /* Something bad happened, check everything is ok
	* before continuing and attempt to recover if
	* required
	*/
	S_RELEASE_DC, /* we were the DC, but now we arent anymore,
	* possibly by our own request, and we should
	* release all unnecessary sub-systems, finish
	* any pending actions, do general cleanup and
	* unset anything that makes us think we are
	* special :)
	*/
	S_STARTING, /* we are just starting out */
	S_PENDING, /* we are not a full/active member yet */
	S_STOPPING, /* We are in the final stages of shutting down */
	S_TERMINATE, /* We are going to shutdown, this is the equiv of
	* "Sending TERM signal to all processes" in Linux
	* and in worst case scenarios could be considered
	* a self STONITH
	*/
	S_TRANSITION_ENGINE,/* Attempt to make the calculated next stable
	* state of the cluster a reality
	*/

	/* ----------- Last input found in table is above ---------- */
	S_ILLEGAL, /* This is an illegal FSA state */
	/* (must be last) */
	};
	#define MAXSTATE S_ILLEGAL
	/*
	A state diagram can be constructed from the dc_fsa.dot with the
	following command:

	dot -Tpng crmd_fsa.dot > crmd_fsa.png

	Description:

	Once we start and do some basic sanity checks, we go into the
	S_NOT_DC state and await instructions from the DC or input from
	the CCM which indicates the election algorithm needs to run.

	If the election algorithm is triggered we enter the S_ELECTION state
	from where we can either go back to the S_NOT_DC state or progress
	to the S_INTEGRATION state (or S_RELEASE_DC if we used to be the DC
	but arent anymore).

	The election algorithm has been adapted from
	http://www.cs.indiana.edu/cgi-bin/techreports/TRNNN.cgi?trnum=TR521

	Loosly known as the Bully Algorithm, its major points are:
	- Election is initiated by any node (N) notices that the coordinator
	is no longer responding
	- Concurrent multiple elections are possible
	- Algorithm
	+ N sends ELECTION messages to all nodes that occur earlier in
	the CCM's membership list.
	+ If no one responds, N wins and becomes coordinator
	+ N sends out COORDINATOR messages to all other nodes in the
	partition
	+ If one of higher-ups answers, it takes over. N is done.

	Once the election is complete, if we are the DC, we enter the
	S_INTEGRATION state which is a DC-in-waiting style state. We are
	the DC, but we shouldnt do anything yet because we may not have an
	up-to-date picture of the cluster. There may of course be times
	when this fails, so we should go back to the S_RECOVERY stage and
	check everything is ok. We may also end up here if a new node came
	online, since each node is authorative on itself and we would want
	to incorporate its information into the CIB.

	Once we have the latest CIB, we then enter the S_POLICY_ENGINE state
	where invoke the Policy Engine. It is possible that between
	invoking the Policy Engine and recieving an answer, that we recieve
	more input. In this case we would discard the orginal result and
	invoke it again.

	Once we are satisfied with the output from the Policy Engine we
	enter S_TRANSITION_ENGINE and feed the Policy Engine's output to the
	Transition Engine who attempts to make the Policy Engine's
	calculation a reality. If the transition completes successfully,
	we enter S_IDLE, otherwise we go back to S_POLICY_ENGINE with the
	current unstable state and try again.

	Of course we may be asked to shutdown at any time, however we must
	progress to S_NOT_DC before doing so. Once we have handed over DC
	duties to another node, we can then shut down like everyone else,
	that is by asking the DC for permission and waiting it to take all
	our resources away.

	The case where we are the DC and the only node in the cluster is a
	special case and handled as an escalation which takes us to
	S_SHUTDOWN. Similarly if any other point in the shutdown
	fails or stalls, this is escalated and we end up in S_TERMINATE.

	At any point, the CRMd/DC can relay messages for its sub-systems,
	but outbound messages (from sub-systems) should probably be blocked
	until S_INTEGRATION (for the DC case) or the join protocol has
	completed (for the CRMd case)

	*/

	/*======================================
	*
	* Inputs/Events/Stimuli to be given to the finite state machine
	*
	* Some of these a true events, and others a synthesised based on
	* the "register" (see below) and the contents or source of messages.
	*
	* At this point, my plan is to have a loop of some sort that keeps
	* going until recieving I_NULL
	*
	======================================/
	enum crmd_fsa_input {
	/* 0 */
	I_NULL, /* Nothing happened */
	/* 1 */
	I_CCM_EVENT,

	I_CIB_OP, /* An update to the CIB occurred */
	I_CIB_UPDATE, /* An update to the CIB occurred */
	I_DC_TIMEOUT, /* We have lost communication with the DC */
	I_ELECTION, /* Someone started an election */
	I_PE_CALC, /* The Policy Engine needs to be invoked */
	I_RELEASE_DC, /* The election completed and we were not
	* elected, but we were the DC beforehand
	*/
	I_ELECTION_DC, /* The election completed and we were (re-)elected
	* DC
	*/
	I_ERROR, /* Something bad happened (more serious than
	* I_FAIL) and may not have been due to the action
	* being performed. For example, we may have lost
	* our connection to the CIB.
	*/
	/* 10 */
	I_FAIL, /* The action failed to complete successfully */
	I_INTEGRATED,
	I_FINALIZED,
	I_NODE_JOIN, /* A node has entered the cluster */
	I_NOT_DC, /* We are not and were not the DC before or after
	* the current operation or state
	*/
	I_RECOVERED, /* The recovery process completed successfully */
	I_RELEASE_FAIL, /* We could not give up DC status for some reason
	*/
	I_RELEASE_SUCCESS, /* We are no longer the DC */
	I_RESTART, /* The current set of actions needs to be
	* restarted
	*/
	I_TE_SUCCESS, /* Some non-resource, non-ccm action is required
	* of us, eg. ping
	*/
	/* 20 */
	I_ROUTER, /* Do our job as router and forward this to the
	* right place
	*/
	I_SHUTDOWN, /* We are asking to shutdown */
	I_TERMINATE, /* We have been told to shutdown */
	I_STARTUP,
	I_PE_SUCCESS, /* The action completed successfully */

	I_JOIN_OFFER, /* The DC is offering membership */
	I_JOIN_REQUEST, /* The client is requesting membership */
	I_JOIN_RESULT, /* If not the DC: The result of a join request
	* Else: A client is responding with its local state info
	*/

	I_WAIT_FOR_EVENT, /* we may be waiting for an async task to "happen"
	* and until it does, we cant do anything else
	*/

	I_DC_HEARTBEAT, /* The DC is telling us that it is alive and well */


	I_LRM_EVENT,

	/* 30 */
	I_PENDING,

	/* ------------ Last input found in table is above ----------- */
	I_ILLEGAL, /* This is an illegal value for an FSA input */
	/* (must be last) */
	};
	#define MAXINPUT I_ILLEGAL

	#define I_MESSAGE I_ROUTER

	/*======================================
	*
	* actions
	*
	* Some of the actions below will always occur together for now, but I can
	* forsee that this may not always be the case. So I've spilt them up so
	* that if they ever do need to be called independantly in the future, it
	* wont be a problem.
	*
	* For example, separating A_LRM_CONNECT from A_STARTUP might be useful
	* if we ever try to recover from a faulty or disconnected LRM.
	*
	======================================/


	/* Dont do anything */
	#define A_NOTHING 0x0000000000000000ULL

	/* -- Startup actions -- */
	/* Hook to perform any actions (other than starting the CIB,
	* connecting to HA or the CCM) that might be needed as part
	* of the startup.
	*/
	#define A_STARTUP 0x0000000000000001ULL
	/* Hook to perform any actions that might be needed as part
	* after startup is successful.
	*/
	#define A_STARTED 0x0000000000000002ULL
	/* Connect to Heartbeat */
	#define A_HA_CONNECT 0x0000000000000004ULL
	#define A_HA_DISCONNECT 0x0000000000000008ULL

	/* -- Election actions -- */
	#define A_DC_TIMER_START 0x0000000000000010ULL
	#define A_DC_TIMER_STOP 0x0000000000000020ULL
	#define A_ELECTION_COUNT 0x0000000000000040ULL
	#define A_ELECTION_VOTE 0x0000000000000080ULL

	#define A_INTEGRATE_TIMER_START 0x0000000000000100ULL
	#define A_INTEGRATE_TIMER_STOP 0x0000000000000200ULL
	#define A_FINALIZE_TIMER_START 0x0000000000000400ULL
	#define A_FINALIZE_TIMER_STOP 0x0000000000000800ULL

	/* -- Message processing -- */
	/* Process the queue of requests */
	#define A_MSG_PROCESS 0x0000000000001000ULL
	/* Send the message to the correct recipient */
	#define A_MSG_ROUTE 0x0000000000002000ULL
	/* Put the request into a queue for processing. We do this every
	* time so that the processing is consistent. The intent is to
	* allow the DC to keep doing important work while still not
	* loosing requests.
	* Messages are not considered recieved until processed.
	*/
	#define A_MSG_STORE 0x0000000000004000ULL


	/* -- Client Join protocol actions -- */
	#define A_CL_JOIN_ANNOUNCE 0x0000000000010000ULL
	/* Request membership to the DC list */
	#define A_CL_JOIN_REQUEST 0x0000000000020000ULL
	/* Did the DC accept or reject the request */
	#define A_CL_JOIN_RESULT 0x0000000000040000ULL

	/* -- Server Join protocol actions -- */
	/* Send a welcome message to new node(s) */
	#define A_DC_JOIN_OFFER_ONE 0x0000000000080000ULL
	/* Send a welcome message to all nodes */
	#define A_DC_JOIN_OFFER_ALL 0x0000000000100000ULL
	/* Process the remote node's ack of our join message */
	#define A_DC_JOIN_PROCESS_REQ 0x0000000000200000ULL
	/* Send out the reults of the Join phase */
	#define A_DC_JOIN_FINALIZE 0x0000000000400000ULL
	/* Send out the reults of the Join phase */
	#define A_DC_JOIN_PROCESS_ACK 0x0000000000800000ULL

	/* -- Recovery, DC start/stop -- */
	/* Something bad happened, try to recover */
	#define A_RECOVER 0x0000000001000000ULL
	/* Hook to perform any actions (apart from starting, the TE, PE
	* and gathering the latest CIB) that might be necessary before
	* giving up the responsibilities of being the DC.
	*/
	#define A_DC_RELEASE 0x0000000002000000ULL
	/* */
	#define A_DC_RELEASED 0x0000000004000000ULL
	/* Hook to perform any actions (apart from starting, the TE, PE
	* and gathering the latest CIB) that might be necessary before
	* taking over the responsibilities of being the DC.
	*/
	#define A_DC_TAKEOVER 0x0000000008000000ULL

	/* -- Shutdown actions -- */
	#define A_SHUTDOWN 0x0000000010000000ULL
	#define A_STOP 0x0000000020000000ULL
	#define A_EXIT_0 0x0000000040000000ULL
	#define A_EXIT_1 0x0000000080000000ULL

	#define A_SHUTDOWN_REQ 0x0000000100000000ULL

	/* -- CCM actions -- */
	#define A_CCM_CONNECT 0x0000001000000000ULL
	#define A_CCM_DISCONNECT 0x0000002000000000ULL
	/* Process whatever it is the CCM is trying to tell us.
	* This will generate inputs such as I_NODE_JOIN,
	* I_NODE_LEAVE, I_SHUTDOWN, I_DC_RELEASE, I_DC_TAKEOVER
	*/
	#define A_CCM_EVENT 0x0000004000000000ULL
	#define A_CCM_UPDATE_CACHE 0x0000008000000000ULL

	/* -- CBI actions -- */
	#define A_CIB_INVOKE 0x0000010000000000ULL
	#define A_CIB_START 0x0000020000000000ULL
	#define A_CIB_STOP 0x0000040000000000ULL
	#define A_CIB_INVOKE_LOCAL 0x0000080000000000ULL

	/* -- Transition Engine actions -- */
	/* Attempt to reach the newly calculated cluster state. This is
	* only called once per transition (except if it is asked to
	* stop the transition or start a new one).
	* Once given a cluster state to reach, the TE will determin
	* tasks that can be performed in parallel, execute them, wait
	* for replies and then determin the next set until the new
	* state is reached or no further tasks can be taken.
	*/
	#define A_TE_INVOKE 0x0000100000000000ULL
	#define A_TE_START 0x0000200000000000ULL
	#define A_TE_STOP 0x0000400000000000ULL
	#define A_TE_CANCEL 0x0000800000000000ULL
	#define A_TE_COPYTO 0x0001000000000000ULL

	/* -- Policy Engine actions -- */
	/* Calculate the next state for the cluster. This is only
	* invoked once per needed calculation.
	*/
	#define A_PE_INVOKE 0x0002000000000000ULL
	#define A_PE_START 0x0004000000000000ULL
	#define A_PE_STOP 0x0008000000000000ULL
	/* -- Misc actions -- */
	/* Add a system generate "block" so that resources arent moved
	* to or are activly moved away from the affected node. This
	* way we can return quickly even if busy with other things.
	*/
	#define A_NODE_BLOCK 0x0010000000000000ULL
	/* Update our information in the local CIB */
	#define A_UPDATE_NODESTATUS 0x0020000000000000ULL
	#define A_CIB_BUMPGEN 0x0040000000000000ULL
	#define A_READCONFIG 0x0080000000000000ULL

	/* -- LRM Actions -- */
	/* Connect to the Local Resource Manager */
	#define A_LRM_CONNECT 0x0100000000000000ULL
	/* Disconnect from the Local Resource Manager */
	#define A_LRM_DISCONNECT 0x0200000000000000ULL
	#define A_LRM_INVOKE 0x0400000000000000ULL
	#define A_LRM_EVENT 0x0800000000000000ULL

	/* -- Logging actions -- */
	#define A_LOG 0x1000000000000000ULL
	#define A_ERROR 0x2000000000000000ULL
	#define A_WARN 0x4000000000000000ULL

	#define O_SHUTDOWN (A_CCM_DISCONNECT\|A_LRM_DISCONNECT\|A_HA_DISCONNECT\|A_SHUTDOWN\|A_STOP\|A_EXIT_0\|A_CIB_STOP)
	#define O_RELEASE (A_DC_TIMER_STOP\|A_DC_RELEASE\|A_PE_STOP\|A_TE_STOP\|A_DC_RELEASED)
	#define O_DC_TIMER_RESTART (A_DC_TIMER_STOP\|A_DC_TIMER_START)
	#define O_PE_RESTART (A_PE_START\|A_PE_STOP)
	#define O_TE_RESTART (A_TE_START\|A_TE_STOP)
	#define O_CIB_RESTART (A_CIB_START\|A_CIB_STOP)

	#define O_DC_TICKLE O_DC_TIMER_RESTART

	/*======================================
	*
	* "register" contents
	*
	* Things we may want to remember regardless of which state we are in.
	*
	* These also count as inputs for synthesizing I_*
	*
	======================================/
	#define R_THE_DC 0x00000001 /* Are we the DC? */
	#define R_STARTING 0x00000002 /* Are we starting up? */
	#define R_SHUTDOWN 0x00000004 /* Are we trying to shut down? */

	#define R_JOIN_OK 0x00000010 /* Have we completed the join process */
	#define R_HAVE_RES 0x00000040 /* Do we have any resources running
	locally */
	#define R_INVOKE_PE 0x00000080 /* Does the PE needed to be invoked at
	the next appropriate point? */

	#define R_CIB_CONNECTED 0x00000100 /* Is the CIB connected? */
	#define R_PE_CONNECTED 0x00000200 /* Is the Policy Engine connected? */
	#define R_TE_CONNECTED 0x00000400 /* Is the Transition Engine connected? */
	#define R_LRM_CONNECTED 0x00000800 /* Is the Local Resource Manager
	connected? */

	#define R_REQ_PEND 0x00001000 /* Are there Requests waiting for
	processing? */
	#define R_PE_PEND 0x00002000 /* Has the PE been invoked and we're
	awaiting a reply? */
	#define R_TE_PEND 0x00004000 /* Has the TE been invoked and we're
	awaiting completion? */
	#define R_RESP_PEND 0x00008000 /* Do we have clients waiting on a
	response? if so perhaps we shouldnt
	stop yet */

	#define R_CIB_DONE 0x00010000 /* Have we calculated the CIB? */
	#define R_HAVE_CIB 0x00020000 /* Do we have an up-to-date CIB */
	#define R_CIB_ASKED 0x00040000 /* Have we asked for an up-to-date CIB */

	#define R_CCM_DATA 0x00100000 /* Have we got CCM data yet */
	-#define R_PEER_DATA 0x00200000 /* Have we got CCM data yet */
	+#define R_PEER_DATA 0x00200000 /* Have we got T_CL_STATUS data yet */

	enum crmd_fsa_cause
	{
	C_UNKNOWN = 0,
	C_STARTUP,
	C_IPC_MESSAGE,
	C_HA_MESSAGE,
	C_CCM_CALLBACK,
	C_CRMD_STATUS_CALLBACK,
	C_LRM_OP_CALLBACK,
	C_LRM_MONITOR_CALLBACK,
	C_TIMER_POPPED,
	C_SHUTDOWN,
	C_HEARTBEAT_FAILED,
	C_SUBSYSTEM_CONNECT,
	C_HA_DISCONNECT,
	C_FSA_INTERNAL,
	C_ILLEGAL
	};

	extern const char *fsa_input2string(enum crmd_fsa_input input);
	extern const char *fsa_state2string(enum crmd_fsa_state state);
	extern const char *fsa_cause2string(enum crmd_fsa_cause cause);
	extern const char *fsa_action2string(long long action);


	#endif
	diff --git a/crm/crmd/join_dc.c b/crm/crmd/join_dc.c
	index 10dcdd97bc..cb418b81b4 100644
	--- a/crm/crmd/join_dc.c
	+++ b/crm/crmd/join_dc.c
	@@ -1,524 +1,530 @@
	/*
	* Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public
	* License as published by the Free Software Foundation; either
	* version 2.1 of the License, or (at your option) any later version.
	*
	* This software is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	* General Public License for more details.
	*
	* You should have received a copy of the GNU General Public
	* License along with this library; if not, write to the Free Software
	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	*/
	#include <portability.h>

	#include <heartbeat.h>

	#include <crm/crm.h>
	#include <crm/cib.h>
	#include <crm/msg_xml.h>
	#include <crm/common/xml.h>

	#include <crmd_fsa.h>
	#include <crmd_messages.h>

	#include <crm/dmalloc_wrapper.h>

	int num_join_invites = 0;
	GHashTable *join_offers = NULL;
	GHashTable *join_requests = NULL;
	GHashTable *confirmed_nodes = NULL;
	xmlNodePtr our_generation = NULL;
	const char *max_generation_from = NULL;

	-void initialize_join(void);
	+void initialize_join(gboolean before);
	void finalize_join_for(gpointer key, gpointer value, gpointer user_data);
	void join_send_offer(gpointer key, gpointer value, gpointer user_data);

	/* A_DC_JOIN_OFFER_ALL */
	enum crmd_fsa_input
	do_dc_join_offer_all(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	fsa_data_t *msg_data)
	{
	/* reset everyones status back to down or in_ccm in the CIB */
	xmlNodePtr update = NULL;
	xmlNodePtr cib_copy = get_cib_copy();
	xmlNodePtr tmp1 = get_object_root(XML_CIB_TAG_STATUS, cib_copy);
	xmlNodePtr tmp2 = NULL;

	- initialize_join();
	-
	/* catch any nodes that are active in the CIB but not in the CCM list*/
	xml_child_iter(
	tmp1, node_entry, XML_CIB_TAG_STATE,

	const char *node_id = xmlGetProp(node_entry, XML_ATTR_UNAME);
	gpointer a_node = g_hash_table_lookup(
	fsa_membership_copy->members, node_id);

	if(a_node != NULL \|\| (safe_str_eq(fsa_our_uname, node_id))) {
	/* handled by do_update_cib_node() */
	continue;
	}

	tmp2 = create_node_state(
	node_id, node_id, NULL,
	XML_BOOLEAN_NO, NULL, CRMD_JOINSTATE_PENDING, NULL);

	if(update == NULL) {
	update = tmp2;
	} else {
	update = xmlAddSibling(update, tmp2);
	}
	);

	/* now process the CCM data */
	free_xml(do_update_cib_nodes(update, TRUE));
	free_xml(cib_copy);

	#if 0
	/* Avoid ordered message delays caused when the CRMd proc
	* isnt running yet (ie. send as a broadcast msg which are never
	* sent ordered.
	*/
	send_request(NULL, NULL, CRM_OP_WELCOME,
	NULL, CRM_SYSTEM_CRMD, NULL);
	#else

	crm_debug("Offering membership to %d clients",
	fsa_membership_copy->members_size);

	g_hash_table_foreach(fsa_membership_copy->members,
	join_send_offer, NULL);

	#endif
	/* No point hanging around in S_INTEGRATION if we're the only ones here! */
	if(g_hash_table_size(join_requests)
	>= fsa_membership_copy->members_size) {
	crm_info("Not expecting any join acks");
	register_fsa_input(C_FSA_INTERNAL, I_INTEGRATED, NULL);
	return I_NULL;
	}

	/* dont waste time by invoking the pe yet; */
	crm_debug("Still waiting on %d outstanding join acks",
	fsa_membership_copy->members_size
	- g_hash_table_size(join_requests));

	return I_NULL;
	}

	/* A_DC_JOIN_OFFER_ONE */
	enum crmd_fsa_input
	do_dc_join_offer_one(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	fsa_data_t *msg_data)
	{
	gpointer a_node = NULL;
	xmlNodePtr welcome = NULL;
	const char *join_to = NULL;

	if(msg_data->data == NULL) {
	crm_err("Attempt to send welcome message "
	"without a message to reply to!");
	return I_NULL;
	}

	welcome = (xmlNodePtr)msg_data->data;

	join_to = xmlGetProp(welcome, XML_ATTR_HOSTFROM);

	- a_node = g_hash_table_lookup(join_offers, join_to);
	- if(a_node != NULL) {
	+ a_node = g_hash_table_lookup(join_requests, join_to);
	+ if(a_node != NULL
	+ && (cur_state == S_INTEGRATION \|\| cur_state == S_FINALIZE_JOIN)) {
	/* note: it _is_ possible that a node will have been
	* sick or starting up when the original offer was made.
	* however, it will either re-announce itself in due course
	* _or_ we can re-store the original offer on the client.
	*/
	crm_warn("Already offered membership to %s... discarding",
	join_to);

	/* Make sure we end up in the correct state again */
	if(g_hash_table_size(join_requests)
	>= fsa_membership_copy->members_size) {

	crm_info("False alarm, returning to %s",
	fsa_state2string(S_FINALIZE_JOIN));

	register_fsa_input(C_FSA_INTERNAL, I_INTEGRATED, NULL);
	return I_NULL;
	}

	crm_debug("Still waiting on %d outstanding join acks",
	fsa_membership_copy->members_size
	- g_hash_table_size(join_requests));



	} else {
	oc_node_t member;
	+
	+ crm_debug("Processing annouce request from %s in state %s",
	+ join_to, fsa_state2string(cur_state));
	+
	member.node_uname = crm_strdup(join_to);
	join_send_offer(NULL, &member, NULL);
	crm_free(member.node_uname);

	/* this was a genuine join request, cancel any existing
	* transition and invoke the PE
	*/
	register_fsa_input_w_actions(
	msg_data->fsa_cause, I_NULL, NULL, A_TE_CANCEL);
	}

	return I_NULL;
	}

	/* A_DC_JOIN_PROCESS_REQ */
	enum crmd_fsa_input
	do_dc_join_req(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	fsa_data_t *msg_data)
	{
	xmlNodePtr generation;
	xmlNodePtr join_ack = (xmlNodePtr)msg_data->data;
	const char *ack_nack = CRMD_JOINSTATE_MEMBER;

	gboolean is_a_member = FALSE;
	const char *join_from = xmlGetProp(join_ack, XML_ATTR_HOSTFROM);
	const char *ref = xmlGetProp(join_ack, XML_ATTR_REFERENCE);

	gpointer join_node =
	g_hash_table_lookup(fsa_membership_copy->members, join_from);

	crm_debug("Processing req from %s", join_from);

	if(join_node != NULL) {
	is_a_member = TRUE;
	}

	generation = find_xml_node(join_ack, "generation_tuple");
	if(compare_cib_generation(our_generation, generation) < 0) {
	clear_bit_inplace(fsa_input_register, R_HAVE_CIB);
	crm_debug("%s has a better generation number than us",
	join_from);
	crm_xml_debug(our_generation, "Our generation");
	crm_xml_debug(generation, "Their generation");
	max_generation_from = join_from;
	}

	crm_debug("Welcoming node %s after ACK (ref %s)", join_from, ref);

	if(is_a_member == FALSE) {
	/* nack them now so they are not counted towards the
	* expected responses
	*/
	char *local_from = crm_strdup(join_from);
	char *local_down = crm_strdup(CRMD_JOINSTATE_DOWN);

	crm_err("Node %s is not known to us (ref %s)", join_from, ref);
	finalize_join_for(local_from, local_down, NULL);
	crm_free(local_from);
	crm_free(local_down);

	return I_FAIL;

	} else if(/* some reason */ 0) {
	/* NACK this client */
	ack_nack = CRMD_JOINSTATE_DOWN;
	}

	/* add them to our list of CRMD_STATE_ACTIVE nodes
	TODO: check its not already there
	*/
	g_hash_table_insert(
	join_requests, crm_strdup(join_from), crm_strdup(ack_nack));

	if(g_hash_table_size(join_requests)
	>= fsa_membership_copy->members_size) {
	crm_info("That was the last outstanding join ack");
	register_fsa_input(C_FSA_INTERNAL, I_INTEGRATED, NULL);
	return I_NULL;
	}

	/* dont waste time by invoking the PE yet; */
	crm_debug("Still waiting on %d (of %d) outstanding join acks",
	fsa_membership_copy->members_size
	- g_hash_table_size(join_requests),
	fsa_membership_copy->members_size);

	return I_NULL;
	}



	/* A_DC_JOIN_FINALIZE */
	enum crmd_fsa_input
	do_dc_join_finalize(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	fsa_data_t *msg_data)
	{
	if(max_generation_from == NULL) {
	crm_warn("There is no CIB to get..."
	" how did R_HAVE_CIB get unset?");
	set_bit_inplace(fsa_input_register, R_HAVE_CIB);
	}

	if(! is_set(fsa_input_register, R_HAVE_CIB)) {
	if(is_set(fsa_input_register, R_CIB_ASKED)) {
	crm_info("Waiting for the CIB from %s",
	crm_str(max_generation_from));
	crmd_fsa_stall();
	return I_NULL;
	}

	set_bit_inplace(fsa_input_register, R_CIB_ASKED);

	/* ask for the agreed best CIB */
	crm_info("Asking %s for its copy of the CIB",
	crm_str(max_generation_from));

	send_request(NULL, NULL, CRM_OP_RETRIVE_CIB,
	max_generation_from, CRM_SYSTEM_CRMD, NULL);

	crmd_fsa_stall();
	return I_NULL;
	}

	num_join_invites = 0;
	crm_debug("Notifying %d clients of join results",
	g_hash_table_size(join_requests));
	g_hash_table_foreach(join_requests, finalize_join_for, NULL);

	if(num_join_invites <= g_hash_table_size(confirmed_nodes)) {
	crm_info("Not expecting any join confirmations");

	register_fsa_input(C_FSA_INTERNAL, I_FINALIZED, NULL);
	return I_NULL;
	}

	/* dont waste time by invoking the PE yet; */
	crm_debug("Still waiting on %d outstanding join confirmations",
	num_join_invites - g_hash_table_size(confirmed_nodes));

	return I_NULL;
	}

	/* A_DC_JOIN_PROCESS_ACK */
	enum crmd_fsa_input
	do_dc_join_ack(long long action,
	enum crmd_fsa_cause cause,
	enum crmd_fsa_state cur_state,
	enum crmd_fsa_input current_input,
	fsa_data_t *msg_data)
	{
	/* now update them to "member" */
	xmlNodePtr tmp1 = NULL, update = NULL;
	xmlNodePtr join_ack = (xmlNodePtr)msg_data->data;
	const char *join_from = xmlGetProp(join_ack, XML_ATTR_HOSTFROM);

	const char *join_state = NULL;
	crm_debug("Processing ack from %s", join_from);

	join_state = (const char *)
	g_hash_table_lookup(join_requests, join_from);

	if(join_state == NULL) {
	crm_err("Join not in progress: ignoring join from %s",
	join_from);
	return I_FAIL;

	} else if(safe_str_neq(join_state, CRMD_JOINSTATE_MEMBER)) {
	crm_err("Node %s wasnt invited to join the cluster",join_from);
	return I_NULL;
	}

	g_hash_table_insert(confirmed_nodes, crm_strdup(join_from),
	crm_strdup(CRMD_JOINSTATE_MEMBER));

	/* update node entry in the status section */
	crm_debug("Updating node state to %s for %s", join_state, join_from);
	update = create_node_state(
	join_from, join_from,
	ACTIVESTATUS, NULL, ONLINESTATUS, join_state, join_state);

	set_xml_property_copy(update,XML_CIB_ATTR_EXPSTATE, CRMD_STATE_ACTIVE);

	tmp1 = create_cib_fragment(update, NULL);
	invoke_local_cib(NULL, tmp1, CRM_OP_UPDATE);

	free_xml(tmp1);

	register_fsa_input(cause, I_CIB_OP, msg_data->data);

	if(num_join_invites <= g_hash_table_size(confirmed_nodes)) {
	crm_info("That was the last outstanding join confirmation");
	register_fsa_input_later(C_FSA_INTERNAL, I_FINALIZED, NULL);
	return I_NULL;
	}

	/* dont waste time by invoking the pe yet; */
	crm_debug("Still waiting on %d outstanding join confirmations",
	num_join_invites - g_hash_table_size(confirmed_nodes));

	return I_NULL;
	}

	void
	finalize_join_for(gpointer key, gpointer value, gpointer user_data)
	{
	xmlNodePtr tmp1 = NULL;
	xmlNodePtr tmp2 = NULL;
	xmlNodePtr cib_copy = NULL;
	xmlNodePtr options = NULL;

	const char *join_to = NULL;
	const char *join_state = NULL;

	if(key == NULL \|\| value == NULL) {
	return;
	}

	options = create_xml_node(NULL, XML_TAG_OPTIONS);
	join_to = (const char *)key;
	join_state = (const char *)value;

	/* make sure the node exists in the config section */
	create_node_entry(join_to, join_to, CRMD_JOINSTATE_MEMBER);

	/* perhaps we shouldnt special case this... */
	if(safe_str_eq(join_to, fsa_our_uname)) {
	/* mark ourselves confirmed */
	g_hash_table_insert(confirmed_nodes, crm_strdup(fsa_our_uname),
	crm_strdup(CRMD_JOINSTATE_MEMBER));

	/* make sure our cluster state is set correctly */
	tmp1 = create_node_state(
	join_to, join_to, ACTIVESTATUS,
	NULL, ONLINESTATUS, join_state, join_state);

	if(tmp1 != NULL) {
	tmp2 = create_cib_fragment(tmp1, NULL);
	invoke_local_cib(NULL, tmp2, CRM_OP_UPDATE);
	free_xml(tmp2);
	free_xml(tmp1);

	} else {
	crm_err("Could not create our node state");
	/* TODO: raise an error input */
	}

	/* update our LRM data */
	tmp1 = do_lrm_query(TRUE);
	if(tmp1 != NULL) {
	invoke_local_cib(NULL, tmp1, CRM_OP_UPDATE);
	free_xml(tmp1);

	} else {
	crm_err("Could not determin current LRM state");
	/* TODO: raise an error input */
	}

	num_join_invites++;
	crm_info("Completed local cluster membership");
	return;
	}

	/* create the ack/nack */
	if(safe_str_eq(join_state, CRMD_JOINSTATE_MEMBER)) {
	num_join_invites++;
	set_xml_property_copy(
	options, CRM_OP_JOINACK, XML_BOOLEAN_TRUE);

	} else {
	crm_info("NACK'ing join request from %s, state %s",
	join_to, join_state);

	set_xml_property_copy(
	options, CRM_OP_JOINACK, XML_BOOLEAN_FALSE);
	}

	/* send the CIB to the node */
	cib_copy = get_cib_copy();
	send_request(NULL, cib_copy, CRM_OP_REPLACE,
	join_to, CRM_SYSTEM_CRMD, NULL);
	free_xml(cib_copy);

	/* send the ack/nack to the node */
	send_request(options, NULL, CRM_OP_JOINACK,
	join_to, CRM_SYSTEM_CRMD, NULL);
	}

	void
	-initialize_join(void)
	+initialize_join(gboolean before)
	{
	/* clear out/reset a bunch of stuff */
	if(join_offers != NULL) {
	g_hash_table_destroy(join_offers);
	}
	if(join_requests != NULL) {
	g_hash_table_destroy(join_requests);
	}
	if(confirmed_nodes != NULL) {
	g_hash_table_destroy(confirmed_nodes);
	}

	- free_xml(our_generation);
	- our_generation = cib_get_generation();
	-
	- max_generation_from = NULL;
	- set_bit_inplace(fsa_input_register, R_HAVE_CIB);
	- clear_bit_inplace(fsa_input_register, R_CIB_ASKED);
	-
	+ if(before) {
	+ free_xml(our_generation);
	+ our_generation = cib_get_generation();
	+
	+ max_generation_from = NULL;
	+ set_bit_inplace(fsa_input_register, R_HAVE_CIB);
	+ clear_bit_inplace(fsa_input_register, R_CIB_ASKED);
	+ }
	+
	join_offers = g_hash_table_new(&g_str_hash, &g_str_equal);
	join_requests = g_hash_table_new(&g_str_hash, &g_str_equal);
	confirmed_nodes = g_hash_table_new(&g_str_hash, &g_str_equal);

	/* mark ourselves joined */
	g_hash_table_insert(join_requests, crm_strdup(fsa_our_uname),
	crm_strdup(CRMD_JOINSTATE_MEMBER));

	}

	+
	void
	join_send_offer(gpointer key, gpointer value, gpointer user_data)
	{
	const char *join_to = NULL;
	const oc_node_t member = (const oc_node_t)value;

	crm_debug("Sending %s offer", CRM_OP_WELCOME);
	if(member != NULL) {
	join_to = member->node_uname;
	}

	if(join_to == NULL) {
	crm_err("No recipient for welcome message");

	} else if(safe_str_eq(join_to, fsa_our_uname)) {
	crm_debug("Skipping %s msg for ourselves (%s)",
	CRM_OP_WELCOME, join_to);

	} else {
	/* send the welcome */
	crm_debug("Sending %s to %s", CRM_OP_WELCOME, join_to);

	send_request(NULL, NULL, CRM_OP_WELCOME,
	join_to, CRM_SYSTEM_CRMD, NULL);

	g_hash_table_insert(join_offers, crm_strdup(join_to),
	crm_strdup(CRMD_JOINSTATE_PENDING));

	}
	}

File Metadata

Mime Type: text/x-diff
Expires: Thu, Jun 26, 5:20 PM (16 h, 40 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1959221
Default Alt Text: (55 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions