diff --git a/crmd/fsa.c b/crmd/fsa.c index 9d340a9a83..99dfce2562 100644 --- a/crmd/fsa.c +++ b/crmd/fsa.c @@ -1,670 +1,672 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include char *fsa_our_dc = NULL; cib_t *fsa_cib_conn = NULL; char *fsa_our_dc_version = NULL; ll_lrm_t *fsa_lrm_conn; char *fsa_our_uuid = NULL; char *fsa_our_uname = NULL; #if SUPPORT_HEARTBEAT ll_cluster_t *fsa_cluster_conn; #endif fsa_timer_t *wait_timer = NULL; fsa_timer_t *recheck_timer = NULL; fsa_timer_t *election_trigger = NULL; fsa_timer_t *election_timeout = NULL; fsa_timer_t *transition_timer = NULL; fsa_timer_t *integration_timer = NULL; fsa_timer_t *finalization_timer = NULL; fsa_timer_t *shutdown_escalation_timer = NULL; volatile gboolean do_fsa_stall = FALSE; volatile long long fsa_input_register = 0; volatile long long fsa_actions = A_NOTHING; volatile enum crmd_fsa_state fsa_state = S_STARTING; extern uint highest_born_on; extern uint num_join_invites; extern GHashTable *welcomed_nodes; extern GHashTable *finalized_nodes; extern GHashTable *confirmed_nodes; extern GHashTable *integrated_nodes; extern void initialize_join(gboolean before); #define DOT_PREFIX "actions:trace: " #define do_dot_log(fmt, args...) crm_trace( fmt, ##args) long long do_state_transition(long long actions, enum crmd_fsa_state cur_state, enum crmd_fsa_state next_state, fsa_data_t * msg_data); void dump_rsc_info(void); void dump_rsc_info_callback(const xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data); void ghash_print_node(gpointer key, gpointer value, gpointer user_data); void s_crmd_fsa_actions(fsa_data_t * fsa_data); void log_fsa_input(fsa_data_t * stored_msg); void init_dotfile(void); void init_dotfile(void) { do_dot_log(DOT_PREFIX "digraph \"g\" {"); do_dot_log(DOT_PREFIX " size = \"30,30\""); do_dot_log(DOT_PREFIX " graph ["); do_dot_log(DOT_PREFIX " fontsize = \"12\""); do_dot_log(DOT_PREFIX " fontname = \"Times-Roman\""); do_dot_log(DOT_PREFIX " fontcolor = \"black\""); do_dot_log(DOT_PREFIX " bb = \"0,0,398.922306,478.927856\""); do_dot_log(DOT_PREFIX " color = \"black\""); do_dot_log(DOT_PREFIX " ]"); do_dot_log(DOT_PREFIX " node ["); do_dot_log(DOT_PREFIX " fontsize = \"12\""); do_dot_log(DOT_PREFIX " fontname = \"Times-Roman\""); do_dot_log(DOT_PREFIX " fontcolor = \"black\""); do_dot_log(DOT_PREFIX " shape = \"ellipse\""); do_dot_log(DOT_PREFIX " color = \"black\""); do_dot_log(DOT_PREFIX " ]"); do_dot_log(DOT_PREFIX " edge ["); do_dot_log(DOT_PREFIX " fontsize = \"12\""); do_dot_log(DOT_PREFIX " fontname = \"Times-Roman\""); do_dot_log(DOT_PREFIX " fontcolor = \"black\""); do_dot_log(DOT_PREFIX " color = \"black\""); do_dot_log(DOT_PREFIX " ]"); do_dot_log(DOT_PREFIX "// special nodes"); do_dot_log(DOT_PREFIX " \"S_PENDING\" "); do_dot_log(DOT_PREFIX " ["); do_dot_log(DOT_PREFIX " color = \"blue\""); do_dot_log(DOT_PREFIX " fontcolor = \"blue\""); do_dot_log(DOT_PREFIX " ]"); do_dot_log(DOT_PREFIX " \"S_TERMINATE\" "); do_dot_log(DOT_PREFIX " ["); do_dot_log(DOT_PREFIX " color = \"red\""); do_dot_log(DOT_PREFIX " fontcolor = \"red\""); do_dot_log(DOT_PREFIX " ]"); do_dot_log(DOT_PREFIX "// DC only nodes"); do_dot_log(DOT_PREFIX " \"S_INTEGRATION\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX " \"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX " \"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX " \"S_RELEASE_DC\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX " \"S_IDLE\" [ fontcolor = \"green\" ]"); } static void do_fsa_action(fsa_data_t * fsa_data, long long an_action, void (*function) (long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)) { int action_log_level = LOG_DEBUG; /* The calls to fsa_action2string() is expensive, * only make it if we will use the result */ if (an_action & A_MSG_ROUTE) { action_log_level = LOG_DEBUG_2; } fsa_actions &= ~an_action; do_crm_log(action_log_level, DOT_PREFIX "\t// %s", fsa_action2string(an_action)); function(an_action, fsa_data->fsa_cause, fsa_state, fsa_data->fsa_input, fsa_data); } static long long startup_actions = A_STARTUP | A_CIB_START | A_LRM_CONNECT | A_CCM_CONNECT | A_HA_CONNECT | A_READCONFIG | A_STARTED | A_CL_JOIN_QUERY; enum crmd_fsa_state s_crmd_fsa(enum crmd_fsa_cause cause) { fsa_data_t *fsa_data = NULL; long long register_copy = fsa_input_register; long long new_actions = A_NOTHING; enum crmd_fsa_state last_state = fsa_state; crm_trace("FSA invoked with Cause: %s\tState: %s", fsa_cause2string(cause), fsa_state2string(fsa_state)); do_fsa_stall = FALSE; if (is_message() == FALSE && fsa_actions != A_NOTHING) { /* fake the first message so we can get into the loop */ crm_malloc0(fsa_data, sizeof(fsa_data_t)); fsa_data->fsa_input = I_NULL; fsa_data->fsa_cause = C_FSA_INTERNAL; fsa_data->origin = __FUNCTION__; fsa_data->data_type = fsa_dt_none; fsa_message_queue = g_list_append(fsa_message_queue, fsa_data); fsa_data = NULL; } while (is_message() && do_fsa_stall == FALSE) { crm_trace("Checking messages (%d remaining)", g_list_length(fsa_message_queue)); fsa_data = get_message(); CRM_CHECK(fsa_data != NULL, continue); log_fsa_input(fsa_data); /* add any actions back to the queue */ fsa_actions |= fsa_data->actions; /* get the next batch of actions */ new_actions = crmd_fsa_actions[fsa_data->fsa_input][fsa_state]; fsa_actions |= new_actions; if (fsa_data->fsa_input != I_NULL && fsa_data->fsa_input != I_ROUTER) { crm_debug("Processing %s: [ state=%s cause=%s origin=%s ]", fsa_input2string(fsa_data->fsa_input), fsa_state2string(fsa_state), fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin); } #ifdef FSA_TRACE if (new_actions != A_NOTHING) { crm_trace("Adding FSA actions %.16llx for %s/%s", new_actions, fsa_input2string(fsa_data->fsa_input), fsa_state2string(fsa_state)); fsa_dump_actions(new_actions, "\tFSA scheduled"); } else if (fsa_data->fsa_input != I_NULL && new_actions == A_NOTHING) { crm_debug("No action specified for input,state (%s,%s)", fsa_input2string(fsa_data->fsa_input), fsa_state2string(fsa_state)); } if (fsa_data->actions != A_NOTHING) { crm_trace("Adding input actions %.16llx for %s/%s", new_actions, fsa_input2string(fsa_data->fsa_input), fsa_state2string(fsa_state)); fsa_dump_actions(fsa_data->actions, "\tInput scheduled"); } #endif /* logging : *before* the state is changed */ if (is_set(fsa_actions, A_ERROR)) { do_fsa_action(fsa_data, A_ERROR, do_log); } if (is_set(fsa_actions, A_WARN)) { do_fsa_action(fsa_data, A_WARN, do_log); } if (is_set(fsa_actions, A_LOG)) { do_fsa_action(fsa_data, A_LOG, do_log); } /* update state variables */ last_state = fsa_state; fsa_state = crmd_fsa_state[fsa_data->fsa_input][fsa_state]; /* * Remove certain actions during shutdown */ if (fsa_state == S_STOPPING || ((fsa_input_register & R_SHUTDOWN) == R_SHUTDOWN)) { clear_bit_inplace(fsa_actions, startup_actions); } /* * Hook for change of state. * Allows actions to be added or removed when entering a state */ if (last_state != fsa_state) { fsa_actions = do_state_transition(fsa_actions, last_state, fsa_state, fsa_data); } else { do_dot_log(DOT_PREFIX "\t// FSA input: State=%s \tCause=%s" " \tInput=%s \tOrigin=%s() \tid=%d", fsa_state2string(fsa_state), fsa_cause2string(fsa_data->fsa_cause), fsa_input2string(fsa_data->fsa_input), fsa_data->origin, fsa_data->id); } /* start doing things... */ s_crmd_fsa_actions(fsa_data); delete_fsa_input(fsa_data); fsa_data = NULL; } if (g_list_length(fsa_message_queue) > 0 || fsa_actions != A_NOTHING || do_fsa_stall) { crm_debug("Exiting the FSA: queue=%d, fsa_actions=0x%llx, stalled=%s", g_list_length(fsa_message_queue), fsa_actions, do_fsa_stall ? "true" : "false"); } else { crm_trace("Exiting the FSA"); } /* cleanup inputs? */ if (register_copy != fsa_input_register) { long long same = register_copy & fsa_input_register; fsa_dump_inputs(LOG_DEBUG, "Added input:", fsa_input_register ^ same); fsa_dump_inputs(LOG_DEBUG, "Removed input:", register_copy ^ same); } fsa_dump_queue(LOG_DEBUG); return fsa_state; } void s_crmd_fsa_actions(fsa_data_t * fsa_data) { /* * Process actions in order of priority but do only one * action at a time to avoid complicating the ordering. */ CRM_CHECK(fsa_data != NULL, return); while (fsa_actions != A_NOTHING && do_fsa_stall == FALSE) { /* regular action processing in order of action priority * * Make sure all actions that connect to required systems * are performed first */ if (fsa_actions & A_ERROR) { do_fsa_action(fsa_data, A_ERROR, do_log); } else if (fsa_actions & A_WARN) { do_fsa_action(fsa_data, A_WARN, do_log); } else if (fsa_actions & A_LOG) { do_fsa_action(fsa_data, A_LOG, do_log); /* get out of here NOW! before anything worse happens */ } else if (fsa_actions & A_EXIT_1) { do_fsa_action(fsa_data, A_EXIT_1, do_exit); + /* sub-system restart */ + } else if ((fsa_actions & O_LRM_RECONNECT) == O_LRM_RECONNECT) { + do_fsa_action(fsa_data, O_LRM_RECONNECT, do_lrm_control); + } else if ((fsa_actions & O_CIB_RESTART) == O_CIB_RESTART) { + do_fsa_action(fsa_data, O_CIB_RESTART, do_cib_control); + } else if ((fsa_actions & O_PE_RESTART) == O_PE_RESTART) { + do_fsa_action(fsa_data, O_PE_RESTART, do_pe_control); + } else if ((fsa_actions & O_TE_RESTART) == O_TE_RESTART) { + do_fsa_action(fsa_data, O_TE_RESTART, do_te_control); + /* essential start tasks */ } else if (fsa_actions & A_STARTUP) { do_fsa_action(fsa_data, A_STARTUP, do_startup); } else if (fsa_actions & A_CIB_START) { do_fsa_action(fsa_data, A_CIB_START, do_cib_control); } else if (fsa_actions & A_HA_CONNECT) { do_fsa_action(fsa_data, A_HA_CONNECT, do_ha_control); } else if (fsa_actions & A_READCONFIG) { do_fsa_action(fsa_data, A_READCONFIG, do_read_config); /* sub-system start/connect */ } else if (fsa_actions & A_LRM_CONNECT) { do_fsa_action(fsa_data, A_LRM_CONNECT, do_lrm_control); } else if (fsa_actions & A_CCM_CONNECT) { do_fsa_action(fsa_data, A_CCM_CONNECT, do_ccm_control); } else if (fsa_actions & A_TE_START) { do_fsa_action(fsa_data, A_TE_START, do_te_control); } else if (fsa_actions & A_PE_START) { do_fsa_action(fsa_data, A_PE_START, do_pe_control); - /* sub-system restart */ - } else if ((fsa_actions & O_CIB_RESTART) == O_CIB_RESTART) { - do_fsa_action(fsa_data, O_CIB_RESTART, do_cib_control); - } else if ((fsa_actions & O_PE_RESTART) == O_PE_RESTART) { - do_fsa_action(fsa_data, O_PE_RESTART, do_pe_control); - } else if ((fsa_actions & O_TE_RESTART) == O_TE_RESTART) { - do_fsa_action(fsa_data, O_TE_RESTART, do_te_control); - /* Timers */ /* else if(fsa_actions & O_DC_TIMER_RESTART) { do_fsa_action(fsa_data, O_DC_TIMER_RESTART, do_timer_control) */ ; } else if (fsa_actions & A_DC_TIMER_STOP) { do_fsa_action(fsa_data, A_DC_TIMER_STOP, do_timer_control); } else if (fsa_actions & A_INTEGRATE_TIMER_STOP) { do_fsa_action(fsa_data, A_INTEGRATE_TIMER_STOP, do_timer_control); } else if (fsa_actions & A_INTEGRATE_TIMER_START) { do_fsa_action(fsa_data, A_INTEGRATE_TIMER_START, do_timer_control); } else if (fsa_actions & A_FINALIZE_TIMER_STOP) { do_fsa_action(fsa_data, A_FINALIZE_TIMER_STOP, do_timer_control); } else if (fsa_actions & A_FINALIZE_TIMER_START) { do_fsa_action(fsa_data, A_FINALIZE_TIMER_START, do_timer_control); /* * Highest priority actions */ } else if (fsa_actions & A_MSG_ROUTE) { do_fsa_action(fsa_data, A_MSG_ROUTE, do_msg_route); } else if (fsa_actions & A_RECOVER) { do_fsa_action(fsa_data, A_RECOVER, do_recover); } else if (fsa_actions & A_CL_JOIN_RESULT) { do_fsa_action(fsa_data, A_CL_JOIN_RESULT, do_cl_join_finalize_respond); } else if (fsa_actions & A_CL_JOIN_REQUEST) { do_fsa_action(fsa_data, A_CL_JOIN_REQUEST, do_cl_join_offer_respond); } else if (fsa_actions & A_SHUTDOWN_REQ) { do_fsa_action(fsa_data, A_SHUTDOWN_REQ, do_shutdown_req); } else if (fsa_actions & A_ELECTION_VOTE) { do_fsa_action(fsa_data, A_ELECTION_VOTE, do_election_vote); } else if (fsa_actions & A_ELECTION_COUNT) { do_fsa_action(fsa_data, A_ELECTION_COUNT, do_election_count_vote); } else if (fsa_actions & A_LRM_EVENT) { do_fsa_action(fsa_data, A_LRM_EVENT, do_lrm_event); /* * High priority actions */ } else if (fsa_actions & A_STARTED) { do_fsa_action(fsa_data, A_STARTED, do_started); } else if (fsa_actions & A_CL_JOIN_QUERY) { do_fsa_action(fsa_data, A_CL_JOIN_QUERY, do_cl_join_query); } else if (fsa_actions & A_DC_TIMER_START) { do_fsa_action(fsa_data, A_DC_TIMER_START, do_timer_control); /* * Medium priority actions */ } else if (fsa_actions & A_DC_TAKEOVER) { do_fsa_action(fsa_data, A_DC_TAKEOVER, do_dc_takeover); } else if (fsa_actions & A_DC_RELEASE) { do_fsa_action(fsa_data, A_DC_RELEASE, do_dc_release); } else if (fsa_actions & A_DC_JOIN_FINAL) { do_fsa_action(fsa_data, A_DC_JOIN_FINAL, do_dc_join_final); } else if (fsa_actions & A_ELECTION_CHECK) { do_fsa_action(fsa_data, A_ELECTION_CHECK, do_election_check); } else if (fsa_actions & A_ELECTION_START) { do_fsa_action(fsa_data, A_ELECTION_START, do_election_vote); } else if (fsa_actions & A_TE_HALT) { do_fsa_action(fsa_data, A_TE_HALT, do_te_invoke); } else if (fsa_actions & A_TE_CANCEL) { do_fsa_action(fsa_data, A_TE_CANCEL, do_te_invoke); } else if (fsa_actions & A_DC_JOIN_OFFER_ALL) { do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ALL, do_dc_join_offer_all); } else if (fsa_actions & A_DC_JOIN_OFFER_ONE) { do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ONE, do_dc_join_offer_all); } else if (fsa_actions & A_DC_JOIN_PROCESS_REQ) { do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_REQ, do_dc_join_filter_offer); } else if (fsa_actions & A_DC_JOIN_PROCESS_ACK) { do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_ACK, do_dc_join_ack); /* * Low(er) priority actions * Make sure the CIB is always updated before invoking the * PE, and the PE before the TE */ } else if (fsa_actions & A_DC_JOIN_FINALIZE) { do_fsa_action(fsa_data, A_DC_JOIN_FINALIZE, do_dc_join_finalize); } else if (fsa_actions & A_LRM_INVOKE) { do_fsa_action(fsa_data, A_LRM_INVOKE, do_lrm_invoke); } else if (fsa_actions & A_PE_INVOKE) { do_fsa_action(fsa_data, A_PE_INVOKE, do_pe_invoke); } else if (fsa_actions & A_TE_INVOKE) { do_fsa_action(fsa_data, A_TE_INVOKE, do_te_invoke); } else if (fsa_actions & A_CL_JOIN_ANNOUNCE) { do_fsa_action(fsa_data, A_CL_JOIN_ANNOUNCE, do_cl_join_announce); /* sub-system stop */ } else if (fsa_actions & A_DC_RELEASED) { do_fsa_action(fsa_data, A_DC_RELEASED, do_dc_release); } else if (fsa_actions & A_PE_STOP) { do_fsa_action(fsa_data, A_PE_STOP, do_pe_control); } else if (fsa_actions & A_TE_STOP) { do_fsa_action(fsa_data, A_TE_STOP, do_te_control); } else if (fsa_actions & A_SHUTDOWN) { do_fsa_action(fsa_data, A_SHUTDOWN, do_shutdown); } else if (fsa_actions & A_LRM_DISCONNECT) { do_fsa_action(fsa_data, A_LRM_DISCONNECT, do_lrm_control); } else if (fsa_actions & A_CCM_DISCONNECT) { do_fsa_action(fsa_data, A_CCM_DISCONNECT, do_ccm_control); } else if (fsa_actions & A_HA_DISCONNECT) { do_fsa_action(fsa_data, A_HA_DISCONNECT, do_ha_control); } else if (fsa_actions & A_CIB_STOP) { do_fsa_action(fsa_data, A_CIB_STOP, do_cib_control); } else if (fsa_actions & A_STOP) { do_fsa_action(fsa_data, A_STOP, do_stop); /* exit gracefully */ } else if (fsa_actions & A_EXIT_0) { do_fsa_action(fsa_data, A_EXIT_0, do_exit); /* Error checking and reporting */ } else { crm_err("Action %s (0x%llx) not supported ", fsa_action2string(fsa_actions), fsa_actions); register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, fsa_data, NULL, __FUNCTION__); } } } void log_fsa_input(fsa_data_t * stored_msg) { crm_trace("Processing queued input %d", stored_msg->id); if (stored_msg->fsa_cause == C_CCM_CALLBACK) { crm_trace("FSA processing CCM callback from %s", stored_msg->origin); } else if (stored_msg->fsa_cause == C_LRM_OP_CALLBACK) { crm_trace("FSA processing LRM callback from %s", stored_msg->origin); } else if (stored_msg->data == NULL) { crm_trace("FSA processing input from %s", stored_msg->origin); } else { ha_msg_input_t *ha_input = fsa_typed_data_adv(stored_msg, fsa_dt_ha_msg, __FUNCTION__); crm_trace("FSA processing XML message from %s", stored_msg->origin); crm_log_xml_trace(ha_input->xml, "FSA message data"); } } long long do_state_transition(long long actions, enum crmd_fsa_state cur_state, enum crmd_fsa_state next_state, fsa_data_t * msg_data) { long long tmp = actions; gboolean clear_recovery_bit = TRUE; enum crmd_fsa_cause cause = msg_data->fsa_cause; enum crmd_fsa_input current_input = msg_data->fsa_input; const char *state_from = fsa_state2string(cur_state); const char *state_to = fsa_state2string(next_state); const char *input = fsa_input2string(current_input); CRM_LOG_ASSERT(cur_state != next_state); do_dot_log(DOT_PREFIX "\t%s -> %s [ label=%s cause=%s origin=%s ]", state_from, state_to, input, fsa_cause2string(cause), msg_data->origin); crm_notice("State transition %s -> %s [ input=%s cause=%s origin=%s ]", state_from, state_to, input, fsa_cause2string(cause), msg_data->origin); /* the last two clauses might cause trouble later */ if (election_timeout != NULL && next_state != S_ELECTION && cur_state != S_RELEASE_DC) { crm_timer_stop(election_timeout); /* } else { */ /* crm_timer_start(election_timeout); */ } #if 0 if ((fsa_input_register & R_SHUTDOWN)) { set_bit_inplace(tmp, A_DC_TIMER_STOP); } #endif if (next_state == S_INTEGRATION) { set_bit_inplace(tmp, A_INTEGRATE_TIMER_START); } else { set_bit_inplace(tmp, A_INTEGRATE_TIMER_STOP); } if (next_state == S_FINALIZE_JOIN) { set_bit_inplace(tmp, A_FINALIZE_TIMER_START); } else { set_bit_inplace(tmp, A_FINALIZE_TIMER_STOP); } if (next_state != S_PENDING) { set_bit_inplace(tmp, A_DC_TIMER_STOP); } if (next_state != S_ELECTION) { highest_born_on = 0; } if (next_state != S_IDLE) { crm_timer_stop(recheck_timer); } if (cur_state == S_FINALIZE_JOIN && next_state == S_POLICY_ENGINE) { populate_cib_nodes(FALSE); do_update_cib_nodes(TRUE, __FUNCTION__); } switch (next_state) { case S_PENDING: fsa_cib_conn->cmds->set_slave(fsa_cib_conn, cib_scope_local); /* fall through */ case S_ELECTION: crm_trace("Resetting our DC to NULL on transition to %s", fsa_state2string(next_state)); update_dc(NULL); break; case S_NOT_DC: if (is_set(fsa_input_register, R_SHUTDOWN)) { crm_info("(Re)Issuing shutdown request now" " that we have a new DC"); set_bit_inplace(tmp, A_SHUTDOWN_REQ); } CRM_LOG_ASSERT(fsa_our_dc != NULL); if (fsa_our_dc == NULL) { crm_err("Reached S_NOT_DC without a DC" " being recorded"); } break; case S_RECOVERY: clear_recovery_bit = FALSE; break; case S_FINALIZE_JOIN: CRM_LOG_ASSERT(AM_I_DC); if (cause == C_TIMER_POPPED) { crm_warn("Progressed to state %s after %s", fsa_state2string(next_state), fsa_cause2string(cause)); } if (g_hash_table_size(welcomed_nodes) > 0) { char *msg = crm_strdup(" Welcome reply not received from"); crm_warn("%u cluster nodes failed to respond" " to the join offer.", g_hash_table_size(welcomed_nodes)); g_hash_table_foreach(welcomed_nodes, ghash_print_node, msg); crm_free(msg); } else { crm_debug("All %d cluster nodes " "responded to the join offer.", g_hash_table_size(integrated_nodes)); } break; case S_POLICY_ENGINE: CRM_LOG_ASSERT(AM_I_DC); if (cause == C_TIMER_POPPED) { crm_info("Progressed to state %s after %s", fsa_state2string(next_state), fsa_cause2string(cause)); } if (g_hash_table_size(finalized_nodes) > 0) { char *msg = crm_strdup(" Confirm not received from"); crm_err("%u cluster nodes failed to confirm" " their join.", g_hash_table_size(finalized_nodes)); g_hash_table_foreach(finalized_nodes, ghash_print_node, msg); crm_free(msg); } else if (g_hash_table_size(confirmed_nodes) == crm_active_members()) { crm_debug("All %u cluster nodes are" " eligible to run resources.", crm_active_members()); } else if (g_hash_table_size(confirmed_nodes) > crm_active_members()) { crm_err("We have more confirmed nodes than our membership does: %d vs. %d", g_hash_table_size(confirmed_nodes), crm_active_members()); register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); } else if (saved_ccm_membership_id != crm_peer_seq) { crm_info("Membership changed: %llu -> %llu - join restart", saved_ccm_membership_id, crm_peer_seq); register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL); } else { crm_warn("Only %u of %u cluster " "nodes are eligible to run resources - continue %d", g_hash_table_size(confirmed_nodes), crm_active_members(), g_hash_table_size(welcomed_nodes)); } /* initialize_join(FALSE); */ break; case S_STOPPING: case S_TERMINATE: /* possibly redundant */ set_bit_inplace(fsa_input_register, R_SHUTDOWN); break; case S_IDLE: CRM_LOG_ASSERT(AM_I_DC); dump_rsc_info(); if (is_set(fsa_input_register, R_SHUTDOWN)) { crm_info("(Re)Issuing shutdown request now" " that we are the DC"); set_bit_inplace(tmp, A_SHUTDOWN_REQ); } if (recheck_timer->period_ms > 0) { crm_debug("Starting %s", get_timer_desc(recheck_timer)); crm_timer_start(recheck_timer); } break; default: break; } if (clear_recovery_bit && next_state != S_PENDING) { tmp &= ~A_RECOVER; } else if (clear_recovery_bit == FALSE) { tmp |= A_RECOVER; } if (tmp != actions) { /* fsa_dump_actions(actions ^ tmp, "New actions"); */ actions = tmp; } return actions; } void dump_rsc_info(void) { } void ghash_print_node(gpointer key, gpointer value, gpointer user_data) { const char *text = user_data; const char *uname = key; const char *value_s = value; crm_info("%s: %s %s", text, uname, value_s); } diff --git a/crmd/fsa_defines.h b/crmd/fsa_defines.h index b991863523..3b0869c37c 100644 --- a/crmd/fsa_defines.h +++ b/crmd/fsa_defines.h @@ -1,492 +1,493 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef FSA_DEFINES__H # define FSA_DEFINES__H /*====================================== * States the DC/CRMd can be in *======================================*/ enum crmd_fsa_state { S_IDLE = 0, /* Nothing happening */ S_ELECTION, /* Take part in the election algorithm as * described below */ S_INTEGRATION, /* integrate that status of new nodes (which is * all of them if we have just been elected DC) * to form a complete and up-to-date picture of * the CIB */ S_FINALIZE_JOIN, /* integrate that status of new nodes (which is * all of them if we have just been elected DC) * to form a complete and up-to-date picture of * the CIB */ S_NOT_DC, /* we are in crmd/slave mode */ S_POLICY_ENGINE, /* Determin the next stable state of the cluster */ S_RECOVERY, /* Something bad happened, check everything is ok * before continuing and attempt to recover if * required */ S_RELEASE_DC, /* we were the DC, but now we arent anymore, * possibly by our own request, and we should * release all unnecessary sub-systems, finish * any pending actions, do general cleanup and * unset anything that makes us think we are * special :) */ S_STARTING, /* we are just starting out */ S_PENDING, /* we are not a full/active member yet */ S_STOPPING, /* We are in the final stages of shutting down */ S_TERMINATE, /* We are going to shutdown, this is the equiv of * "Sending TERM signal to all processes" in Linux * and in worst case scenarios could be considered * a self STONITH */ S_TRANSITION_ENGINE, /* Attempt to make the calculated next stable * state of the cluster a reality */ S_HALT, /* Freeze - dont do anything * Something ad happened that needs the admin to fix * Wait for I_ELECTION */ /* ----------- Last input found in table is above ---------- */ S_ILLEGAL /* This is an illegal FSA state */ /* (must be last) */ }; # define MAXSTATE S_ILLEGAL /* A state diagram can be constructed from the dc_fsa.dot with the following command: dot -Tpng crmd_fsa.dot > crmd_fsa.png Description: Once we start and do some basic sanity checks, we go into the S_NOT_DC state and await instructions from the DC or input from the CCM which indicates the election algorithm needs to run. If the election algorithm is triggered we enter the S_ELECTION state from where we can either go back to the S_NOT_DC state or progress to the S_INTEGRATION state (or S_RELEASE_DC if we used to be the DC but arent anymore). The election algorithm has been adapted from http://www.cs.indiana.edu/cgi-bin/techreports/TRNNN.cgi?trnum=TR521 Loosly known as the Bully Algorithm, its major points are: - Election is initiated by any node (N) notices that the coordinator is no longer responding - Concurrent multiple elections are possible - Algorithm + N sends ELECTION messages to all nodes that occur earlier in the CCM's membership list. + If no one responds, N wins and becomes coordinator + N sends out COORDINATOR messages to all other nodes in the partition + If one of higher-ups answers, it takes over. N is done. Once the election is complete, if we are the DC, we enter the S_INTEGRATION state which is a DC-in-waiting style state. We are the DC, but we shouldnt do anything yet because we may not have an up-to-date picture of the cluster. There may of course be times when this fails, so we should go back to the S_RECOVERY stage and check everything is ok. We may also end up here if a new node came online, since each node is authorative on itself and we would want to incorporate its information into the CIB. Once we have the latest CIB, we then enter the S_POLICY_ENGINE state where invoke the Policy Engine. It is possible that between invoking the Policy Engine and recieving an answer, that we recieve more input. In this case we would discard the orginal result and invoke it again. Once we are satisfied with the output from the Policy Engine we enter S_TRANSITION_ENGINE and feed the Policy Engine's output to the Transition Engine who attempts to make the Policy Engine's calculation a reality. If the transition completes successfully, we enter S_IDLE, otherwise we go back to S_POLICY_ENGINE with the current unstable state and try again. Of course we may be asked to shutdown at any time, however we must progress to S_NOT_DC before doing so. Once we have handed over DC duties to another node, we can then shut down like everyone else, that is by asking the DC for permission and waiting it to take all our resources away. The case where we are the DC and the only node in the cluster is a special case and handled as an escalation which takes us to S_SHUTDOWN. Similarly if any other point in the shutdown fails or stalls, this is escalated and we end up in S_TERMINATE. At any point, the CRMd/DC can relay messages for its sub-systems, but outbound messages (from sub-systems) should probably be blocked until S_INTEGRATION (for the DC case) or the join protocol has completed (for the CRMd case) */ /*====================================== * * Inputs/Events/Stimuli to be given to the finite state machine * * Some of these a true events, and others a synthesised based on * the "register" (see below) and the contents or source of messages. * * At this point, my plan is to have a loop of some sort that keeps * going until recieving I_NULL * *======================================*/ enum crmd_fsa_input { /* 0 */ I_NULL, /* Nothing happened */ /* 1 */ I_CIB_OP, /* An update to the CIB occurred */ I_CIB_UPDATE, /* An update to the CIB occurred */ I_DC_TIMEOUT, /* We have lost communication with the DC */ I_ELECTION, /* Someone started an election */ I_PE_CALC, /* The Policy Engine needs to be invoked */ I_RELEASE_DC, /* The election completed and we were not * elected, but we were the DC beforehand */ I_ELECTION_DC, /* The election completed and we were (re-)elected * DC */ I_ERROR, /* Something bad happened (more serious than * I_FAIL) and may not have been due to the action * being performed. For example, we may have lost * our connection to the CIB. */ /* 9 */ I_FAIL, /* The action failed to complete successfully */ I_INTEGRATED, I_FINALIZED, I_NODE_JOIN, /* A node has entered the cluster */ I_NOT_DC, /* We are not and were not the DC before or after * the current operation or state */ I_RECOVERED, /* The recovery process completed successfully */ I_RELEASE_FAIL, /* We could not give up DC status for some reason */ I_RELEASE_SUCCESS, /* We are no longer the DC */ I_RESTART, /* The current set of actions needs to be * restarted */ I_TE_SUCCESS, /* Some non-resource, non-ccm action is required * of us, eg. ping */ /* 20 */ I_ROUTER, /* Do our job as router and forward this to the * right place */ I_SHUTDOWN, /* We are asking to shutdown */ I_STOP, /* We have been told to shutdown */ I_TERMINATE, /* Actually exit */ I_STARTUP, I_PE_SUCCESS, /* The action completed successfully */ I_JOIN_OFFER, /* The DC is offering membership */ I_JOIN_REQUEST, /* The client is requesting membership */ I_JOIN_RESULT, /* If not the DC: The result of a join request * Else: A client is responding with its local state info */ I_WAIT_FOR_EVENT, /* we may be waiting for an async task to "happen" * and until it does, we cant do anything else */ I_DC_HEARTBEAT, /* The DC is telling us that it is alive and well */ I_LRM_EVENT, /* 30 */ I_PENDING, I_HALT, /* ------------ Last input found in table is above ----------- */ I_ILLEGAL /* This is an illegal value for an FSA input */ /* (must be last) */ }; # define MAXINPUT I_ILLEGAL # define I_MESSAGE I_ROUTER /*====================================== * * actions * * Some of the actions below will always occur together for now, but I can * forsee that this may not always be the case. So I've spilt them up so * that if they ever do need to be called independantly in the future, it * wont be a problem. * * For example, separating A_LRM_CONNECT from A_STARTUP might be useful * if we ever try to recover from a faulty or disconnected LRM. * *======================================*/ /* Dont do anything */ # define A_NOTHING 0x0000000000000000ULL /* -- Startup actions -- */ /* Hook to perform any actions (other than starting the CIB, * connecting to HA or the CCM) that might be needed as part * of the startup. */ # define A_STARTUP 0x0000000000000001ULL /* Hook to perform any actions that might be needed as part * after startup is successful. */ # define A_STARTED 0x0000000000000002ULL /* Connect to Heartbeat */ # define A_HA_CONNECT 0x0000000000000004ULL # define A_HA_DISCONNECT 0x0000000000000008ULL # define A_INTEGRATE_TIMER_START 0x0000000000000010ULL # define A_INTEGRATE_TIMER_STOP 0x0000000000000020ULL # define A_FINALIZE_TIMER_START 0x0000000000000040ULL # define A_FINALIZE_TIMER_STOP 0x0000000000000080ULL /* -- Election actions -- */ # define A_DC_TIMER_START 0x0000000000000100ULL # define A_DC_TIMER_STOP 0x0000000000000200ULL # define A_ELECTION_COUNT 0x0000000000000400ULL # define A_ELECTION_VOTE 0x0000000000000800ULL # define A_ELECTION_START 0x0000000000001000ULL /* -- Message processing -- */ /* Process the queue of requests */ # define A_MSG_PROCESS 0x0000000000002000ULL /* Send the message to the correct recipient */ # define A_MSG_ROUTE 0x0000000000004000ULL /* Send a welcome message to new node(s) */ # define A_DC_JOIN_OFFER_ONE 0x0000000000008000ULL /* -- Server Join protocol actions -- */ /* Send a welcome message to all nodes */ # define A_DC_JOIN_OFFER_ALL 0x0000000000010000ULL /* Process the remote node's ack of our join message */ # define A_DC_JOIN_PROCESS_REQ 0x0000000000020000ULL /* Send out the reults of the Join phase */ # define A_DC_JOIN_FINALIZE 0x0000000000040000ULL /* Send out the reults of the Join phase */ # define A_DC_JOIN_PROCESS_ACK 0x0000000000080000ULL /* -- Client Join protocol actions -- */ # define A_CL_JOIN_QUERY 0x0000000000100000ULL # define A_CL_JOIN_ANNOUNCE 0x0000000000200000ULL /* Request membership to the DC list */ # define A_CL_JOIN_REQUEST 0x0000000000400000ULL /* Did the DC accept or reject the request */ # define A_CL_JOIN_RESULT 0x0000000000800000ULL /* -- Recovery, DC start/stop -- */ /* Something bad happened, try to recover */ # define A_RECOVER 0x0000000001000000ULL /* Hook to perform any actions (apart from starting, the TE, PE * and gathering the latest CIB) that might be necessary before * giving up the responsibilities of being the DC. */ # define A_DC_RELEASE 0x0000000002000000ULL /* */ # define A_DC_RELEASED 0x0000000004000000ULL /* Hook to perform any actions (apart from starting, the TE, PE * and gathering the latest CIB) that might be necessary before * taking over the responsibilities of being the DC. */ # define A_DC_TAKEOVER 0x0000000008000000ULL /* -- Shutdown actions -- */ # define A_SHUTDOWN 0x0000000010000000ULL # define A_STOP 0x0000000020000000ULL # define A_EXIT_0 0x0000000040000000ULL # define A_EXIT_1 0x0000000080000000ULL # define A_SHUTDOWN_REQ 0x0000000100000000ULL # define A_ELECTION_CHECK 0x0000000200000000ULL # define A_DC_JOIN_FINAL 0x0000000400000000ULL /* -- CCM actions -- */ # define A_CCM_CONNECT 0x0000001000000000ULL # define A_CCM_DISCONNECT 0x0000002000000000ULL /* -- CIB actions -- */ # define A_CIB_START 0x0000020000000000ULL # define A_CIB_STOP 0x0000040000000000ULL /* -- Transition Engine actions -- */ /* Attempt to reach the newly calculated cluster state. This is * only called once per transition (except if it is asked to * stop the transition or start a new one). * Once given a cluster state to reach, the TE will determin * tasks that can be performed in parallel, execute them, wait * for replies and then determin the next set until the new * state is reached or no further tasks can be taken. */ # define A_TE_INVOKE 0x0000100000000000ULL # define A_TE_START 0x0000200000000000ULL # define A_TE_STOP 0x0000400000000000ULL # define A_TE_CANCEL 0x0000800000000000ULL # define A_TE_HALT 0x0001000000000000ULL /* -- Policy Engine actions -- */ /* Calculate the next state for the cluster. This is only * invoked once per needed calculation. */ # define A_PE_INVOKE 0x0002000000000000ULL # define A_PE_START 0x0004000000000000ULL # define A_PE_STOP 0x0008000000000000ULL /* -- Misc actions -- */ /* Add a system generate "block" so that resources arent moved * to or are activly moved away from the affected node. This * way we can return quickly even if busy with other things. */ # define A_NODE_BLOCK 0x0010000000000000ULL /* Update our information in the local CIB */ # define A_UPDATE_NODESTATUS 0x0020000000000000ULL # define A_CIB_BUMPGEN 0x0040000000000000ULL # define A_READCONFIG 0x0080000000000000ULL /* -- LRM Actions -- */ /* Connect to the Local Resource Manager */ # define A_LRM_CONNECT 0x0100000000000000ULL /* Disconnect from the Local Resource Manager */ # define A_LRM_DISCONNECT 0x0200000000000000ULL # define A_LRM_INVOKE 0x0400000000000000ULL # define A_LRM_EVENT 0x0800000000000000ULL /* -- Logging actions -- */ # define A_LOG 0x1000000000000000ULL # define A_ERROR 0x2000000000000000ULL # define A_WARN 0x4000000000000000ULL # define O_EXIT (A_SHUTDOWN|A_STOP|A_CCM_DISCONNECT|A_LRM_DISCONNECT|A_HA_DISCONNECT|A_EXIT_0|A_CIB_STOP) # define O_RELEASE (A_DC_TIMER_STOP|A_DC_RELEASE|A_PE_STOP|A_TE_STOP|A_DC_RELEASED) # define O_PE_RESTART (A_PE_START|A_PE_STOP) # define O_TE_RESTART (A_TE_START|A_TE_STOP) # define O_CIB_RESTART (A_CIB_START|A_CIB_STOP) +# define O_LRM_RECONNECT (A_LRM_CONNECT|A_LRM_DISCONNECT) # define O_DC_TIMER_RESTART (A_DC_TIMER_STOP|A_DC_TIMER_START) /*====================================== * * "register" contents * * Things we may want to remember regardless of which state we are in. * * These also count as inputs for synthesizing I_* * *======================================*/ # define R_THE_DC 0x00000001ULL /* Are we the DC? */ # define R_STARTING 0x00000002ULL /* Are we starting up? */ # define R_SHUTDOWN 0x00000004ULL /* Are we trying to shut down? */ # define R_STAYDOWN 0x00000008ULL /* Should we restart? */ # define R_JOIN_OK 0x00000010ULL /* Have we completed the join process */ # define R_READ_CONFIG 0x00000040ULL # define R_INVOKE_PE 0x00000080ULL /* Does the PE needed to be invoked at the next appropriate point? */ # define R_CIB_CONNECTED 0x00000100ULL /* Is the CIB connected? */ # define R_PE_CONNECTED 0x00000200ULL /* Is the Policy Engine connected? */ # define R_TE_CONNECTED 0x00000400ULL /* Is the Transition Engine connected? */ # define R_LRM_CONNECTED 0x00000800ULL /* Is the Local Resource Manager connected? */ # define R_CIB_REQUIRED 0x00001000ULL /* Is the CIB required? */ # define R_PE_REQUIRED 0x00002000ULL /* Is the Policy Engine required? */ # define R_TE_REQUIRED 0x00004000ULL /* Is the Transition Engine required? */ # define R_ST_REQUIRED 0x00008000ULL /* Is the Stonith daemon required? */ # define R_CIB_DONE 0x00010000ULL /* Have we calculated the CIB? */ # define R_HAVE_CIB 0x00020000ULL /* Do we have an up-to-date CIB */ # define R_CIB_ASKED 0x00040000ULL /* Have we asked for an up-to-date CIB */ # define R_CCM_DATA 0x00100000ULL /* Have we got CCM data yet */ # define R_PEER_DATA 0x00200000ULL /* Have we got T_CL_STATUS data yet */ # define R_HA_DISCONNECTED 0x00400000ULL /* did we sign out of our own accord */ # define R_CCM_DISCONNECTED 0x00800000ULL /* did we sign out of our own accord */ # define R_REQ_PEND 0x01000000ULL /* Are there Requests waiting for processing? */ # define R_PE_PEND 0x02000000ULL /* Has the PE been invoked and we're awaiting a reply? */ # define R_TE_PEND 0x04000000ULL /* Has the TE been invoked and we're awaiting completion? */ # define R_RESP_PEND 0x08000000ULL /* Do we have clients waiting on a response? if so perhaps we shouldnt stop yet */ # define R_IN_TRANSITION 0x10000000ULL /* */ # define R_SENT_RSC_STOP 0x20000000ULL /* Have we sent a stop action to all * resources in preparation for * shutting down */ # define R_IN_RECOVERY 0x80000000ULL enum crmd_fsa_cause { C_UNKNOWN = 0, C_STARTUP, C_IPC_MESSAGE, C_HA_MESSAGE, C_CCM_CALLBACK, C_CRMD_STATUS_CALLBACK, C_LRM_OP_CALLBACK, C_LRM_MONITOR_CALLBACK, C_TIMER_POPPED, C_SHUTDOWN, C_HEARTBEAT_FAILED, C_SUBSYSTEM_CONNECT, C_HA_DISCONNECT, C_FSA_INTERNAL, C_ILLEGAL }; extern const char *fsa_input2string(enum crmd_fsa_input input); extern const char *fsa_state2string(enum crmd_fsa_state state); extern const char *fsa_cause2string(enum crmd_fsa_cause cause); extern const char *fsa_action2string(long long action); #endif diff --git a/crmd/fsa_matrix.h b/crmd/fsa_matrix.h index ab7b35ece9..a2030a30b4 100644 --- a/crmd/fsa_matrix.h +++ b/crmd/fsa_matrix.h @@ -1,1231 +1,1231 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef XML_FSA_MATRIX__H # define XML_FSA_MATRIX__H /* * The state transition table. The rows are inputs, and * the columns are states. */ const enum crmd_fsa_state crmd_fsa_state[MAXINPUT][MAXSTATE] = { /* Got an I_NULL */ { /* S_IDLE ==> */ S_IDLE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_FINALIZE_JOIN, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_POLICY_ENGINE, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_STARTING, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_TRANSITION_ENGINE, /* S_HALT ==> */ S_HALT, }, /* Got an I_CIB_OP */ { /* S_IDLE ==> */ S_IDLE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_FINALIZE_JOIN, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_POLICY_ENGINE, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_STARTING, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_TRANSITION_ENGINE, /* S_HALT ==> */ S_HALT, }, /* Got an I_CIB_UPDATE */ { /* S_IDLE ==> */ S_IDLE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_FINALIZE_JOIN, /* S_NOT_DC ==> */ S_RECOVERY, /* S_POLICY_ENGINE ==> */ S_POLICY_ENGINE, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_RECOVERY, /* S_PENDING ==> */ S_RECOVERY, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_TRANSITION_ENGINE, /* S_HALT ==> */ S_HALT, }, /* Got an I_DC_TIMEOUT */ { /* S_IDLE ==> */ S_RECOVERY, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_RECOVERY, /* S_FINALIZE_JOIN ==> */ S_RECOVERY, /* S_NOT_DC ==> */ S_ELECTION, /* S_POLICY_ENGINE ==> */ S_RECOVERY, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RECOVERY, /* S_STARTING ==> */ S_STARTING, /* S_PENDING ==> */ S_ELECTION, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_RECOVERY, /* S_HALT ==> */ S_ELECTION, }, /* Got an I_ELECTION */ { /* S_IDLE ==> */ S_ELECTION, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_ELECTION, /* S_FINALIZE_JOIN ==> */ S_ELECTION, /* S_NOT_DC ==> */ S_ELECTION, /* S_POLICY_ENGINE ==> */ S_ELECTION, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_STARTING, /* S_PENDING ==> */ S_ELECTION, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_ELECTION, /* S_HALT ==> */ S_HALT, }, /* Got an I_PE_CALC */ { /* S_IDLE ==> */ S_POLICY_ENGINE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_FINALIZE_JOIN, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_POLICY_ENGINE, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_RECOVERY, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_POLICY_ENGINE, /* S_HALT ==> */ S_HALT, }, /* Got an I_RELEASE_DC */ { /* S_IDLE ==> */ S_RELEASE_DC, /* S_ELECTION ==> */ S_RELEASE_DC, /* S_INTEGRATION ==> */ S_RELEASE_DC, /* S_FINALIZE_JOIN ==> */ S_RELEASE_DC, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_RELEASE_DC, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_RECOVERY, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_RELEASE_DC, /* S_HALT ==> */ S_HALT, }, /* Got an I_ELECTION_DC */ { /* S_IDLE ==> */ S_INTEGRATION, /* S_ELECTION ==> */ S_INTEGRATION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_INTEGRATION, /* S_NOT_DC ==> */ S_INTEGRATION, /* S_POLICY_ENGINE ==> */ S_INTEGRATION, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_RECOVERY, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_INTEGRATION, /* S_HALT ==> */ S_HALT, }, /* Got an I_ERROR */ { /* S_IDLE ==> */ S_RECOVERY, /* S_ELECTION ==> */ S_RECOVERY, /* S_INTEGRATION ==> */ S_RECOVERY, /* S_FINALIZE_JOIN ==> */ S_RECOVERY, /* S_NOT_DC ==> */ S_RECOVERY, /* S_POLICY_ENGINE ==> */ S_RECOVERY, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RECOVERY, /* S_STARTING ==> */ S_RECOVERY, /* S_PENDING ==> */ S_RECOVERY, /* S_STOPPING ==> */ S_TERMINATE, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_RECOVERY, /* S_HALT ==> */ S_RECOVERY, }, /* Got an I_FAIL */ { /* S_IDLE ==> */ S_RECOVERY, /* S_ELECTION ==> */ S_RELEASE_DC, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_INTEGRATION, /* S_NOT_DC ==> */ S_RECOVERY, /* S_POLICY_ENGINE ==> */ S_INTEGRATION, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_STOPPING, /* S_PENDING ==> */ S_STOPPING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_POLICY_ENGINE, /* S_HALT ==> */ S_RELEASE_DC, }, /* Got an I_INTEGRATED */ { /* S_IDLE ==> */ S_IDLE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_FINALIZE_JOIN, /* S_FINALIZE_JOIN ==> */ S_FINALIZE_JOIN, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_POLICY_ENGINE, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_RECOVERY, /* S_PENDING ==> */ S_RECOVERY, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_TRANSITION_ENGINE, /* S_HALT ==> */ S_HALT, }, /* Got an I_FINALIZED */ { /* S_IDLE ==> */ S_IDLE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_POLICY_ENGINE, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_POLICY_ENGINE, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_RECOVERY, /* S_PENDING ==> */ S_RECOVERY, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_TRANSITION_ENGINE, /* S_HALT ==> */ S_HALT, }, /* Got an I_NODE_JOIN */ { /* S_IDLE ==> */ S_INTEGRATION, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_INTEGRATION, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_INTEGRATION, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_RECOVERY, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_INTEGRATION, /* S_HALT ==> */ S_HALT, }, /* Got an I_NOT_DC */ { /* S_IDLE ==> */ S_RECOVERY, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_RECOVERY, /* S_FINALIZE_JOIN ==> */ S_RECOVERY, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_RECOVERY, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_RECOVERY, /* S_PENDING ==> */ S_NOT_DC, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_RECOVERY, /* S_HALT ==> */ S_HALT, }, /* Got an I_RECOVERED */ { /* S_IDLE ==> */ S_IDLE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_INTEGRATION, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_POLICY_ENGINE, /* S_RECOVERY ==> */ S_PENDING, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_STARTING, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_TRANSITION_ENGINE, /* S_HALT ==> */ S_HALT, }, /* Got an I_RELEASE_FAIL */ { /* S_IDLE ==> */ S_STOPPING, /* S_ELECTION ==> */ S_STOPPING, /* S_INTEGRATION ==> */ S_STOPPING, /* S_FINALIZE_JOIN ==> */ S_STOPPING, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_STOPPING, /* S_RECOVERY ==> */ S_STOPPING, /* S_RELEASE_DC ==> */ S_STOPPING, /* S_STARTING ==> */ S_RECOVERY, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_STOPPING, /* S_HALT ==> */ S_HALT, }, /* Got an I_RELEASE_SUCCESS */ { /* S_IDLE ==> */ S_RECOVERY, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_RECOVERY, /* S_FINALIZE_JOIN ==> */ S_RECOVERY, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_RECOVERY, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_PENDING, /* S_STARTING ==> */ S_RECOVERY, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_RECOVERY, /* S_HALT ==> */ S_HALT, }, /* Got an I_RESTART */ { /* S_IDLE ==> */ S_IDLE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_FINALIZE_JOIN, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_POLICY_ENGINE, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_STARTING, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_TRANSITION_ENGINE, /* S_HALT ==> */ S_HALT, }, /* Got an I_TE_SUCCESS */ { /* S_IDLE ==> */ S_IDLE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_FINALIZE_JOIN, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_POLICY_ENGINE, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_RECOVERY, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_IDLE, /* S_HALT ==> */ S_HALT, }, /* Got an I_ROUTER */ { /* S_IDLE ==> */ S_IDLE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_FINALIZE_JOIN, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_POLICY_ENGINE, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_STARTING, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_TRANSITION_ENGINE, /* S_HALT ==> */ S_HALT, }, /* Got an I_SHUTDOWN */ { /* S_IDLE ==> */ S_POLICY_ENGINE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_FINALIZE_JOIN, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_POLICY_ENGINE, /* S_RECOVERY ==> */ S_STOPPING, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_STOPPING, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_POLICY_ENGINE, /* S_HALT ==> */ S_ELECTION, }, /* Got an I_STOP */ { /* S_IDLE ==> */ S_STOPPING, /* S_ELECTION ==> */ S_STOPPING, /* S_INTEGRATION ==> */ S_STOPPING, /* S_FINALIZE_JOIN ==> */ S_STOPPING, /* S_NOT_DC ==> */ S_STOPPING, /* S_POLICY_ENGINE ==> */ S_STOPPING, /* S_RECOVERY ==> */ S_STOPPING, /* S_RELEASE_DC ==> */ S_STOPPING, /* S_STARTING ==> */ S_STOPPING, /* S_PENDING ==> */ S_STOPPING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_STOPPING, /* S_HALT ==> */ S_STOPPING, }, /* Got an I_TERMINATE */ { /* S_IDLE ==> */ S_TERMINATE, /* S_ELECTION ==> */ S_TERMINATE, /* S_INTEGRATION ==> */ S_TERMINATE, /* S_FINALIZE_JOIN ==> */ S_TERMINATE, /* S_NOT_DC ==> */ S_TERMINATE, /* S_POLICY_ENGINE ==> */ S_TERMINATE, /* S_RECOVERY ==> */ S_TERMINATE, /* S_RELEASE_DC ==> */ S_TERMINATE, /* S_STARTING ==> */ S_TERMINATE, /* S_PENDING ==> */ S_TERMINATE, /* S_STOPPING ==> */ S_TERMINATE, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_TERMINATE, /* S_HALT ==> */ S_TERMINATE, }, /* Got an I_STARTUP */ { /* S_IDLE ==> */ S_RECOVERY, /* S_ELECTION ==> */ S_RECOVERY, /* S_INTEGRATION ==> */ S_RECOVERY, /* S_FINALIZE_JOIN ==> */ S_RECOVERY, /* S_NOT_DC ==> */ S_RECOVERY, /* S_POLICY_ENGINE ==> */ S_RECOVERY, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_STARTING, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_RECOVERY, /* S_HALT ==> */ S_HALT, }, /* Got an I_PE_SUCCESS */ { /* S_IDLE ==> */ S_IDLE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_FINALIZE_JOIN, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_TRANSITION_ENGINE, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_RECOVERY, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_TRANSITION_ENGINE, /* S_HALT ==> */ S_HALT, }, /* Got an I_JOIN_OFFER */ { /* S_IDLE ==> */ S_IDLE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_FINALIZE_JOIN, /* S_NOT_DC ==> */ S_PENDING, /* S_POLICY_ENGINE ==> */ S_POLICY_ENGINE, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_STARTING, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_TRANSITION_ENGINE, /* S_HALT ==> */ S_HALT, }, /* Got an I_JOIN_REQUEST */ { /* S_IDLE ==> */ S_INTEGRATION, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_INTEGRATION, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_INTEGRATION, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_STARTING, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_INTEGRATION, /* S_HALT ==> */ S_HALT, }, /* Got an I_JOIN_RESULT */ { /* S_IDLE ==> */ S_INTEGRATION, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_FINALIZE_JOIN, /* S_NOT_DC ==> */ S_PENDING, /* S_POLICY_ENGINE ==> */ S_INTEGRATION, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_RECOVERY, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_INTEGRATION, /* S_HALT ==> */ S_HALT, }, /* Got an I_WAIT_FOR_EVENT */ { /* S_IDLE ==> */ S_IDLE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_FINALIZE_JOIN, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_POLICY_ENGINE, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_STARTING, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_TRANSITION_ENGINE, /* S_HALT ==> */ S_HALT, }, /* Got an I_DC_HEARTBEAT */ { /* S_IDLE ==> */ S_IDLE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_FINALIZE_JOIN, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_POLICY_ENGINE, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_STARTING, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_TRANSITION_ENGINE, /* S_HALT ==> */ S_HALT, }, /* Got an I_LRM_EVENT */ { /* S_IDLE ==> */ S_IDLE, /* S_ELECTION ==> */ S_ELECTION, /* S_INTEGRATION ==> */ S_INTEGRATION, /* S_FINALIZE_JOIN ==> */ S_FINALIZE_JOIN, /* S_NOT_DC ==> */ S_NOT_DC, /* S_POLICY_ENGINE ==> */ S_POLICY_ENGINE, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_STARTING, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_TRANSITION_ENGINE, /* S_HALT ==> */ S_HALT, }, /* Got an I_PENDING */ { /* S_IDLE ==> */ S_PENDING, /* S_ELECTION ==> */ S_PENDING, /* S_INTEGRATION ==> */ S_PENDING, /* S_FINALIZE_JOIN ==> */ S_PENDING, /* S_NOT_DC ==> */ S_PENDING, /* S_POLICY_ENGINE ==> */ S_PENDING, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_PENDING, /* S_PENDING ==> */ S_PENDING, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_PENDING, /* S_HALT ==> */ S_HALT, }, /* Got an I_HALT */ { /* S_IDLE ==> */ S_HALT, /* S_ELECTION ==> */ S_HALT, /* S_INTEGRATION ==> */ S_HALT, /* S_FINALIZE_JOIN ==> */ S_HALT, /* S_NOT_DC ==> */ S_HALT, /* S_POLICY_ENGINE ==> */ S_HALT, /* S_RECOVERY ==> */ S_RECOVERY, /* S_RELEASE_DC ==> */ S_RELEASE_DC, /* S_STARTING ==> */ S_STARTING, /* S_PENDING ==> */ S_HALT, /* S_STOPPING ==> */ S_STOPPING, /* S_TERMINATE ==> */ S_TERMINATE, /* S_TRANSITION_ENGINE ==> */ S_HALT, /* S_HALT ==> */ S_HALT, }, }; /* * The action table. Each entry is a set of actions to take or-ed * together. Like the state table, the rows are inputs, and * the columns are states. */ /* NOTE: In the fsa, the actions are extracted then state is updated. */ const long long crmd_fsa_actions[MAXINPUT][MAXSTATE] = { /* Got an I_NULL */ { /* S_IDLE ==> */ A_NOTHING, /* S_ELECTION ==> */ A_NOTHING, /* S_INTEGRATION ==> */ A_NOTHING, /* S_FINALIZE_JOIN ==> */ A_NOTHING, /* S_NOT_DC ==> */ A_NOTHING, /* S_POLICY_ENGINE ==> */ A_NOTHING, /* S_RECOVERY ==> */ A_NOTHING, /* S_RELEASE_DC ==> */ A_NOTHING, /* S_STARTING ==> */ A_NOTHING, /* S_PENDING ==> */ A_NOTHING, /* S_STOPPING ==> */ A_NOTHING, /* S_TERMINATE ==> */ A_NOTHING, /* S_TRANSITION_ENGINE ==> */ A_NOTHING, /* S_HALT ==> */ A_NOTHING, }, /* Got an I_CIB_OP */ { /* S_IDLE ==> */ A_ERROR, /* S_ELECTION ==> */ A_ERROR, /* S_INTEGRATION ==> */ A_ERROR, /* S_FINALIZE_JOIN ==> */ A_ERROR, /* S_NOT_DC ==> */ A_ERROR, /* S_POLICY_ENGINE ==> */ A_ERROR, /* S_RECOVERY ==> */ A_ERROR, /* S_RELEASE_DC ==> */ A_ERROR, /* S_STARTING ==> */ A_ERROR, /* S_PENDING ==> */ A_ERROR, /* S_STOPPING ==> */ A_ERROR, /* S_TERMINATE ==> */ A_ERROR, /* S_TRANSITION_ENGINE ==> */ A_ERROR, /* S_HALT ==> */ A_ERROR, }, /* Got an I_CIB_UPDATE */ { /* S_IDLE ==> */ A_LOG, /* S_ELECTION ==> */ A_LOG, /* S_INTEGRATION ==> */ A_WARN, /* S_FINALIZE_JOIN ==> */ A_WARN, /* S_NOT_DC ==> */ A_WARN, /* S_POLICY_ENGINE ==> */ A_LOG, /* S_RECOVERY ==> */ A_WARN, /* S_RELEASE_DC ==> */ A_WARN, /* S_STARTING ==> */ A_WARN, /* S_PENDING ==> */ A_WARN, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN, /* S_TRANSITION_ENGINE ==> */ A_LOG, /* S_HALT ==> */ A_WARN, }, /* Got an I_DC_TIMEOUT */ { /* S_IDLE ==> */ A_WARN, /* S_ELECTION ==> */ A_ELECTION_VOTE, /* S_INTEGRATION ==> */ A_WARN, /* S_FINALIZE_JOIN ==> */ A_WARN, /* S_NOT_DC ==> */ A_ELECTION_VOTE | A_WARN, /* S_POLICY_ENGINE ==> */ A_WARN, /* S_RECOVERY ==> */ A_NOTHING, /* S_RELEASE_DC ==> */ A_WARN, /* S_STARTING ==> */ A_WARN, /* S_PENDING ==> */ A_ELECTION_VOTE | A_WARN, /* S_STOPPING ==> */ A_NOTHING, /* S_TERMINATE ==> */ A_NOTHING, /* S_TRANSITION_ENGINE ==> */ A_TE_CANCEL | A_WARN, /* S_HALT ==> */ A_WARN, }, /* Got an I_ELECTION */ { /* S_IDLE ==> */ A_ELECTION_VOTE, /* S_ELECTION ==> */ A_ELECTION_VOTE, /* S_INTEGRATION ==> */ A_ELECTION_VOTE, /* S_FINALIZE_JOIN ==> */ A_ELECTION_VOTE, /* S_NOT_DC ==> */ A_ELECTION_VOTE, /* S_POLICY_ENGINE ==> */ A_ELECTION_VOTE, /* S_RECOVERY ==> */ A_LOG, /* S_RELEASE_DC ==> */ A_LOG, /* S_STARTING ==> */ A_WARN, /* S_PENDING ==> */ A_ELECTION_VOTE, /* S_STOPPING ==> */ A_LOG, /* S_TERMINATE ==> */ A_LOG, /* S_TRANSITION_ENGINE ==> */ A_ELECTION_VOTE, /* S_HALT ==> */ A_ELECTION_VOTE, }, /* Got an I_PE_CALC */ { /* S_IDLE ==> */ A_PE_INVOKE, /* S_ELECTION ==> */ A_NOTHING, /* S_INTEGRATION ==> */ A_NOTHING, /* S_FINALIZE_JOIN ==> */ A_NOTHING, /* S_NOT_DC ==> */ A_WARN, /* S_POLICY_ENGINE ==> */ A_PE_INVOKE, /* S_RECOVERY ==> */ A_NOTHING, /* S_RELEASE_DC ==> */ A_NOTHING, /* S_STARTING ==> */ A_ERROR, /* S_PENDING ==> */ A_WARN, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_ERROR, /* S_TRANSITION_ENGINE ==> */ A_PE_INVOKE, /* S_HALT ==> */ A_ERROR, }, /* Got an I_RELEASE_DC */ { /* S_IDLE ==> */ O_RELEASE, /* S_ELECTION ==> */ O_RELEASE, /* S_INTEGRATION ==> */ O_RELEASE | A_WARN, /* S_FINALIZE_JOIN ==> */ O_RELEASE | A_WARN, /* S_NOT_DC ==> */ A_WARN, /* S_POLICY_ENGINE ==> */ O_RELEASE | A_WARN, /* S_RECOVERY ==> */ O_RELEASE, /* S_RELEASE_DC ==> */ O_RELEASE | A_WARN, /* S_STARTING ==> */ A_ERROR, /* S_PENDING ==> */ A_WARN, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN, /* S_TRANSITION_ENGINE ==> */ O_RELEASE | A_WARN, /* S_HALT ==> */ A_WARN, }, /* Got an I_ELECTION_DC */ { /* S_IDLE ==> */ A_WARN | A_ELECTION_VOTE, /* S_ELECTION ==> */ A_LOG | A_DC_TAKEOVER | A_PE_START | A_TE_START | A_DC_JOIN_OFFER_ALL | A_DC_TIMER_STOP, /* S_INTEGRATION ==> */ A_WARN | A_ELECTION_VOTE | A_DC_JOIN_OFFER_ALL, /* S_FINALIZE_JOIN ==> */ A_WARN | A_ELECTION_VOTE | A_DC_JOIN_OFFER_ALL, /* S_NOT_DC ==> */ A_LOG | A_ELECTION_VOTE, /* S_POLICY_ENGINE ==> */ A_WARN | A_ELECTION_VOTE, /* S_RECOVERY ==> */ A_WARN, /* S_RELEASE_DC ==> */ A_WARN | A_ELECTION_VOTE, /* S_STARTING ==> */ A_LOG | A_WARN, /* S_PENDING ==> */ A_LOG | A_WARN, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN, /* S_TRANSITION_ENGINE ==> */ A_WARN | A_ELECTION_VOTE, /* S_HALT ==> */ A_WARN, }, /* Got an I_ERROR */ { /* S_IDLE ==> */ A_ERROR | A_RECOVER | O_RELEASE | A_ELECTION_START, /* S_ELECTION ==> */ A_ERROR | A_RECOVER | O_RELEASE, /* S_INTEGRATION ==> */ A_ERROR | A_RECOVER | O_RELEASE | A_ELECTION_START, /* S_FINALIZE_JOIN ==> */ A_ERROR | A_RECOVER | O_RELEASE | A_ELECTION_START, /* S_NOT_DC ==> */ A_ERROR | A_RECOVER, /* S_POLICY_ENGINE ==> */ A_ERROR | A_RECOVER | O_RELEASE | A_ELECTION_START, /* S_RECOVERY ==> */ A_ERROR | O_RELEASE, /* S_RELEASE_DC ==> */ A_ERROR | A_RECOVER, /* S_STARTING ==> */ A_ERROR | A_RECOVER, /* S_PENDING ==> */ A_ERROR | A_RECOVER, /* S_STOPPING ==> */ A_ERROR | A_EXIT_1, /* S_TERMINATE ==> */ A_ERROR | A_EXIT_1, /* S_TRANSITION_ENGINE ==> */ A_ERROR | A_RECOVER | O_RELEASE | A_ELECTION_START, /* S_HALT ==> */ A_ERROR | A_RECOVER | O_RELEASE | A_ELECTION_START, }, /* Got an I_FAIL */ { /* S_IDLE ==> */ A_WARN, /* S_ELECTION ==> */ A_WARN, /* S_INTEGRATION ==> */ A_WARN | A_DC_JOIN_OFFER_ALL, /* S_FINALIZE_JOIN ==> */ A_WARN | A_DC_JOIN_OFFER_ALL, /* S_NOT_DC ==> */ A_WARN, /* S_POLICY_ENGINE ==> */ A_WARN | A_DC_JOIN_OFFER_ALL | A_TE_CANCEL, /* S_RECOVERY ==> */ A_WARN | O_RELEASE, /* S_RELEASE_DC ==> */ A_WARN, /* S_STARTING ==> */ A_WARN, /* S_PENDING ==> */ A_WARN, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN | A_EXIT_1, - /* S_TRANSITION_ENGINE ==> */ A_WARN | O_TE_RESTART | A_RECOVER, + /* S_TRANSITION_ENGINE ==> */ A_WARN | O_LRM_RECONNECT, /* S_HALT ==> */ A_WARN, }, /* Got an I_INTEGRATED */ { /* S_IDLE ==> */ A_NOTHING, /* S_ELECTION ==> */ A_WARN, /* S_INTEGRATION ==> */ A_DC_JOIN_FINALIZE, /* S_FINALIZE_JOIN ==> */ A_WARN, /* S_NOT_DC ==> */ A_WARN, /* S_POLICY_ENGINE ==> */ A_NOTHING, /* S_RECOVERY ==> */ A_WARN, /* S_RELEASE_DC ==> */ A_WARN, /* S_STARTING ==> */ A_WARN, /* S_PENDING ==> */ A_WARN, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN, /* S_TRANSITION_ENGINE ==> */ A_NOTHING, /* S_HALT ==> */ A_WARN, }, /* Got an I_FINALIZED */ { /* S_IDLE ==> */ A_NOTHING, /* S_ELECTION ==> */ A_WARN, /* S_INTEGRATION ==> */ A_WARN, /* S_FINALIZE_JOIN ==> */ A_DC_JOIN_FINAL | A_TE_CANCEL, /* S_NOT_DC ==> */ A_WARN, /* S_POLICY_ENGINE ==> */ A_NOTHING, /* S_RECOVERY ==> */ A_WARN, /* S_RELEASE_DC ==> */ A_WARN, /* S_STARTING ==> */ A_WARN, /* S_PENDING ==> */ A_WARN, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN, /* S_TRANSITION_ENGINE ==> */ A_NOTHING, /* S_HALT ==> */ A_WARN, }, /* Got an I_NODE_JOIN */ { /* S_IDLE ==> */ A_TE_HALT | A_DC_JOIN_OFFER_ONE, /* S_ELECTION ==> */ A_WARN, /* S_INTEGRATION ==> */ A_DC_JOIN_OFFER_ONE, /* S_FINALIZE_JOIN ==> */ A_DC_JOIN_OFFER_ONE, /* S_NOT_DC ==> */ A_WARN, /* S_POLICY_ENGINE ==> */ A_DC_JOIN_OFFER_ONE, /* S_RECOVERY ==> */ A_WARN, /* S_RELEASE_DC ==> */ A_WARN, /* S_STARTING ==> */ A_WARN, /* S_PENDING ==> */ A_WARN, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN, /* S_TRANSITION_ENGINE ==> */ A_TE_HALT | A_DC_JOIN_OFFER_ONE, /* S_HALT ==> */ A_WARN, }, /* Got an I_NOT_DC */ { /* S_IDLE ==> */ A_WARN | O_RELEASE, /* S_ELECTION ==> */ A_ERROR | A_ELECTION_START | A_DC_TIMER_STOP, /* S_INTEGRATION ==> */ A_ERROR | O_RELEASE, /* S_FINALIZE_JOIN ==> */ A_ERROR | O_RELEASE, /* S_NOT_DC ==> */ A_LOG, /* S_POLICY_ENGINE ==> */ A_ERROR | O_RELEASE, /* S_RECOVERY ==> */ A_ERROR | O_RELEASE, /* S_RELEASE_DC ==> */ A_ERROR | O_RELEASE, /* S_STARTING ==> */ A_WARN, /* S_PENDING ==> */ A_LOG | A_DC_TIMER_STOP, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN, /* S_TRANSITION_ENGINE ==> */ A_ERROR | O_RELEASE, /* S_HALT ==> */ A_WARN, }, /* Got an I_RECOVERED */ { /* S_IDLE ==> */ A_WARN, /* S_ELECTION ==> */ A_ELECTION_VOTE, /* S_INTEGRATION ==> */ A_WARN, /* S_FINALIZE_JOIN ==> */ A_WARN, /* S_NOT_DC ==> */ A_WARN, /* S_POLICY_ENGINE ==> */ A_WARN, /* S_RECOVERY ==> */ A_LOG, /* S_RELEASE_DC ==> */ A_WARN, /* S_STARTING ==> */ A_WARN, /* S_PENDING ==> */ A_WARN, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN, /* S_TRANSITION_ENGINE ==> */ A_WARN, /* S_HALT ==> */ A_WARN, }, /* Got an I_RELEASE_FAIL */ { /* S_IDLE ==> */ A_WARN, /* S_ELECTION ==> */ A_WARN, /* S_INTEGRATION ==> */ A_WARN, /* S_FINALIZE_JOIN ==> */ A_WARN, /* S_NOT_DC ==> */ A_WARN, /* S_POLICY_ENGINE ==> */ A_NOTHING, /* S_RECOVERY ==> */ A_WARN | A_SHUTDOWN_REQ, /* S_RELEASE_DC ==> */ A_NOTHING, /* S_STARTING ==> */ A_WARN, /* S_PENDING ==> */ A_WARN, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN, /* S_TRANSITION_ENGINE ==> */ A_WARN, /* S_HALT ==> */ A_WARN, }, /* Got an I_RELEASE_SUCCESS */ { /* S_IDLE ==> */ A_WARN, /* S_ELECTION ==> */ A_WARN, /* S_INTEGRATION ==> */ A_WARN, /* S_FINALIZE_JOIN ==> */ A_WARN, /* S_NOT_DC ==> */ A_WARN, /* S_POLICY_ENGINE ==> */ A_WARN, /* S_RECOVERY ==> */ A_WARN, /* S_RELEASE_DC ==> */ A_LOG, /* S_STARTING ==> */ A_WARN, /* S_PENDING ==> */ A_LOG, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN, /* S_TRANSITION_ENGINE ==> */ A_WARN, /* S_HALT ==> */ A_WARN, }, /* Got an I_RESTART */ { /* S_IDLE ==> */ A_NOTHING, /* S_ELECTION ==> */ A_LOG | A_ELECTION_VOTE, /* S_INTEGRATION ==> */ A_LOG | A_DC_JOIN_OFFER_ALL, /* S_FINALIZE_JOIN ==> */ A_LOG | A_DC_JOIN_FINALIZE, /* S_NOT_DC ==> */ A_LOG | A_NOTHING, /* S_POLICY_ENGINE ==> */ A_LOG | A_PE_INVOKE, /* S_RECOVERY ==> */ A_LOG | A_RECOVER | O_RELEASE, /* S_RELEASE_DC ==> */ A_LOG | O_RELEASE, /* S_STARTING ==> */ A_LOG, /* S_PENDING ==> */ A_LOG, /* S_STOPPING ==> */ A_LOG, /* S_TERMINATE ==> */ A_LOG, /* S_TRANSITION_ENGINE ==> */ A_LOG | A_TE_INVOKE, /* S_HALT ==> */ A_WARN, }, /* Got an I_TE_SUCCESS */ { /* S_IDLE ==> */ A_LOG, /* S_ELECTION ==> */ A_WARN, /* S_INTEGRATION ==> */ A_WARN, /* S_FINALIZE_JOIN ==> */ A_WARN, /* S_NOT_DC ==> */ A_ERROR, /* S_POLICY_ENGINE ==> */ A_WARN, /* S_RECOVERY ==> */ A_RECOVER | A_WARN, /* S_RELEASE_DC ==> */ A_WARN, /* S_STARTING ==> */ A_ERROR, /* S_PENDING ==> */ A_ERROR, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN, /* S_TRANSITION_ENGINE ==> */ A_LOG, /* S_HALT ==> */ A_WARN, }, /* Got an I_ROUTER */ { /* S_IDLE ==> */ A_MSG_ROUTE, /* S_ELECTION ==> */ A_MSG_ROUTE, /* S_INTEGRATION ==> */ A_MSG_ROUTE, /* S_FINALIZE_JOIN ==> */ A_MSG_ROUTE, /* S_NOT_DC ==> */ A_MSG_ROUTE, /* S_POLICY_ENGINE ==> */ A_MSG_ROUTE, /* S_RECOVERY ==> */ A_MSG_ROUTE, /* S_RELEASE_DC ==> */ A_MSG_ROUTE, /* S_STARTING ==> */ A_MSG_ROUTE, /* S_PENDING ==> */ A_MSG_ROUTE, /* S_STOPPING ==> */ A_MSG_ROUTE, /* S_TERMINATE ==> */ A_MSG_ROUTE, /* S_TRANSITION_ENGINE ==> */ A_MSG_ROUTE, /* S_HALT ==> */ A_WARN | A_MSG_ROUTE, }, /* Got an I_SHUTDOWN */ { /* S_IDLE ==> */ A_LOG | A_SHUTDOWN_REQ, /* S_ELECTION ==> */ A_LOG | A_SHUTDOWN_REQ | A_ELECTION_VOTE, /* S_INTEGRATION ==> */ A_LOG | A_SHUTDOWN_REQ, /* S_FINALIZE_JOIN ==> */ A_LOG | A_SHUTDOWN_REQ, /* S_NOT_DC ==> */ A_SHUTDOWN_REQ, /* S_POLICY_ENGINE ==> */ A_LOG | A_SHUTDOWN_REQ, /* S_RECOVERY ==> */ A_WARN | O_EXIT | O_RELEASE, /* S_RELEASE_DC ==> */ A_WARN | A_SHUTDOWN_REQ, /* S_STARTING ==> */ A_WARN | O_EXIT, /* S_PENDING ==> */ A_SHUTDOWN_REQ, /* S_STOPPING ==> */ A_LOG, /* S_TERMINATE ==> */ A_LOG, /* S_TRANSITION_ENGINE ==> */ A_WARN | A_SHUTDOWN_REQ, /* S_HALT ==> */ A_WARN | A_ELECTION_START | A_SHUTDOWN_REQ, }, /* Got an I_STOP */ { /* S_IDLE ==> */ A_ERROR | O_RELEASE | O_EXIT, /* S_ELECTION ==> */ O_RELEASE | O_EXIT, /* S_INTEGRATION ==> */ A_WARN | O_RELEASE | O_EXIT, /* S_FINALIZE_JOIN ==> */ A_ERROR | O_RELEASE | O_EXIT, /* S_NOT_DC ==> */ O_EXIT, /* S_POLICY_ENGINE ==> */ A_WARN | O_RELEASE | O_EXIT, /* S_RECOVERY ==> */ A_ERROR | O_RELEASE | O_EXIT, /* S_RELEASE_DC ==> */ A_ERROR | O_RELEASE | O_EXIT, /* S_STARTING ==> */ O_EXIT, /* S_PENDING ==> */ O_EXIT, /* S_STOPPING ==> */ O_EXIT, /* S_TERMINATE ==> */ A_ERROR | A_EXIT_1, /* S_TRANSITION_ENGINE ==> */ A_LOG | O_RELEASE | O_EXIT, /* S_HALT ==> */ O_RELEASE | O_EXIT | A_WARN, }, /* Got an I_TERMINATE */ { /* S_IDLE ==> */ A_ERROR | O_EXIT, /* S_ELECTION ==> */ A_ERROR | O_EXIT, /* S_INTEGRATION ==> */ A_ERROR | O_EXIT, /* S_FINALIZE_JOIN ==> */ A_ERROR | O_EXIT, /* S_NOT_DC ==> */ A_ERROR | O_EXIT, /* S_POLICY_ENGINE ==> */ A_ERROR | O_EXIT, /* S_RECOVERY ==> */ A_ERROR | O_EXIT, /* S_RELEASE_DC ==> */ A_ERROR | O_EXIT, /* S_STARTING ==> */ O_EXIT, /* S_PENDING ==> */ A_ERROR | O_EXIT, /* S_STOPPING ==> */ O_EXIT, /* S_TERMINATE ==> */ O_EXIT, /* S_TRANSITION_ENGINE ==> */ A_ERROR | O_EXIT, /* S_HALT ==> */ A_ERROR | O_EXIT, }, /* Got an I_STARTUP */ { /* S_IDLE ==> */ A_WARN, /* S_ELECTION ==> */ A_WARN, /* S_INTEGRATION ==> */ A_WARN, /* S_FINALIZE_JOIN ==> */ A_WARN, /* S_NOT_DC ==> */ A_WARN, /* S_POLICY_ENGINE ==> */ A_WARN, /* S_RECOVERY ==> */ A_WARN, /* S_RELEASE_DC ==> */ A_WARN, /* S_STARTING ==> */ A_LOG | A_STARTUP | A_CIB_START | A_LRM_CONNECT | A_CCM_CONNECT | A_HA_CONNECT | A_READCONFIG | A_STARTED, /* S_PENDING ==> */ A_LOG, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN, /* S_TRANSITION_ENGINE ==> */ A_WARN, /* S_HALT ==> */ A_WARN, }, /* Got an I_PE_SUCCESS */ { /* S_IDLE ==> */ A_LOG, /* S_ELECTION ==> */ A_WARN, /* S_INTEGRATION ==> */ A_WARN, /* S_FINALIZE_JOIN ==> */ A_WARN, /* S_NOT_DC ==> */ A_NOTHING, /* S_POLICY_ENGINE ==> */ A_LOG | A_TE_INVOKE, /* S_RECOVERY ==> */ A_RECOVER | A_LOG, /* S_RELEASE_DC ==> */ A_LOG, /* S_STARTING ==> */ A_ERROR, /* S_PENDING ==> */ A_LOG, /* S_STOPPING ==> */ A_ERROR, /* S_TERMINATE ==> */ A_ERROR, /* S_TRANSITION_ENGINE ==> */ A_LOG, /* S_HALT ==> */ A_WARN, }, /* Got an I_JOIN_OFFER */ { /* S_IDLE ==> */ A_WARN | A_CL_JOIN_REQUEST, /* S_ELECTION ==> */ A_WARN | A_ELECTION_VOTE, /* S_INTEGRATION ==> */ A_CL_JOIN_REQUEST, /* S_FINALIZE_JOIN ==> */ A_CL_JOIN_REQUEST, /* S_NOT_DC ==> */ A_CL_JOIN_REQUEST | A_DC_TIMER_STOP, /* S_POLICY_ENGINE ==> */ A_WARN | A_CL_JOIN_REQUEST, /* S_RECOVERY ==> */ A_WARN | A_CL_JOIN_REQUEST | A_DC_TIMER_STOP, /* S_RELEASE_DC ==> */ A_WARN | A_CL_JOIN_REQUEST | A_DC_TIMER_STOP, /* S_STARTING ==> */ A_LOG, /* S_PENDING ==> */ A_CL_JOIN_REQUEST | A_DC_TIMER_STOP, /* S_STOPPING ==> */ A_LOG, /* S_TERMINATE ==> */ A_LOG, /* S_TRANSITION_ENGINE ==> */ A_WARN | A_CL_JOIN_REQUEST, /* S_HALT ==> */ A_WARN, }, /* Got an I_JOIN_REQUEST */ { /* S_IDLE ==> */ A_DC_JOIN_OFFER_ONE, /* S_ELECTION ==> */ A_WARN, /* S_INTEGRATION ==> */ A_DC_JOIN_PROCESS_REQ, /* S_FINALIZE_JOIN ==> */ A_DC_JOIN_OFFER_ONE, /* S_NOT_DC ==> */ A_WARN, /* S_POLICY_ENGINE ==> */ A_DC_JOIN_OFFER_ONE, /* S_RECOVERY ==> */ A_WARN, /* S_RELEASE_DC ==> */ A_WARN, /* S_STARTING ==> */ A_WARN, /* S_PENDING ==> */ A_WARN, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN, /* S_TRANSITION_ENGINE ==> */ A_DC_JOIN_OFFER_ONE, /* S_HALT ==> */ A_WARN, }, /* Got an I_JOIN_RESULT */ { /* S_IDLE ==> */ A_ERROR | A_TE_HALT | A_DC_JOIN_OFFER_ALL, /* S_ELECTION ==> */ A_LOG, /* S_INTEGRATION ==> */ A_LOG, /* S_FINALIZE_JOIN ==> */ A_CL_JOIN_RESULT | A_DC_JOIN_PROCESS_ACK, /* S_NOT_DC ==> */ A_ERROR | A_CL_JOIN_ANNOUNCE, /* S_POLICY_ENGINE ==> */ A_ERROR | A_TE_HALT | A_DC_JOIN_OFFER_ALL, /* S_RECOVERY ==> */ A_LOG, /* S_RELEASE_DC ==> */ A_LOG, /* S_STARTING ==> */ A_ERROR, /* S_PENDING ==> */ A_CL_JOIN_RESULT, /* S_STOPPING ==> */ A_ERROR, /* S_TERMINATE ==> */ A_ERROR, /* S_TRANSITION_ENGINE ==> */ A_ERROR | A_TE_HALT | A_DC_JOIN_OFFER_ALL, /* S_HALT ==> */ A_WARN, }, /* Got an I_WAIT_FOR_EVENT */ { /* S_IDLE ==> */ A_LOG, /* S_ELECTION ==> */ A_LOG, /* S_INTEGRATION ==> */ A_LOG, /* S_FINALIZE_JOIN ==> */ A_LOG, /* S_NOT_DC ==> */ A_LOG, /* S_POLICY_ENGINE ==> */ A_LOG, /* S_RECOVERY ==> */ A_LOG, /* S_RELEASE_DC ==> */ A_LOG, /* S_STARTING ==> */ A_LOG, /* S_PENDING ==> */ A_LOG, /* S_STOPPING ==> */ A_LOG, /* S_TERMINATE ==> */ A_LOG, /* S_TRANSITION_ENGINE ==> */ A_LOG, /* S_HALT ==> */ A_WARN, }, /* Got an I_DC_HEARTBEAT */ { /* S_IDLE ==> */ A_ERROR, /* S_ELECTION ==> */ A_WARN | A_ELECTION_VOTE, /* S_INTEGRATION ==> */ A_ERROR, /* S_FINALIZE_JOIN ==> */ A_ERROR, /* S_NOT_DC ==> */ A_NOTHING, /* S_POLICY_ENGINE ==> */ A_ERROR, /* S_RECOVERY ==> */ A_NOTHING, /* S_RELEASE_DC ==> */ A_LOG, /* S_STARTING ==> */ A_LOG, /* S_PENDING ==> */ A_LOG | A_CL_JOIN_ANNOUNCE, /* S_STOPPING ==> */ A_NOTHING, /* S_TERMINATE ==> */ A_NOTHING, /* S_TRANSITION_ENGINE ==> */ A_ERROR, /* S_HALT ==> */ A_WARN, }, /* Got an I_LRM_EVENT */ { /* S_IDLE ==> */ A_LRM_EVENT, /* S_ELECTION ==> */ A_LRM_EVENT, /* S_INTEGRATION ==> */ A_LRM_EVENT, /* S_FINALIZE_JOIN ==> */ A_LRM_EVENT, /* S_NOT_DC ==> */ A_LRM_EVENT, /* S_POLICY_ENGINE ==> */ A_LRM_EVENT, /* S_RECOVERY ==> */ A_LRM_EVENT, /* S_RELEASE_DC ==> */ A_LRM_EVENT, /* S_STARTING ==> */ A_LRM_EVENT, /* S_PENDING ==> */ A_LRM_EVENT, /* S_STOPPING ==> */ A_LRM_EVENT, /* S_TERMINATE ==> */ A_LRM_EVENT, /* S_TRANSITION_ENGINE ==> */ A_LRM_EVENT, /* S_HALT ==> */ A_WARN, }, /* For everyone ending up in S_PENDING, (re)start the DC timer and wait for I_JOIN_OFFER or I_NOT_DC */ /* Got an I_PENDING */ { /* S_IDLE ==> */ O_RELEASE | O_DC_TIMER_RESTART, /* S_ELECTION ==> */ O_RELEASE | O_DC_TIMER_RESTART, /* S_INTEGRATION ==> */ O_RELEASE | O_DC_TIMER_RESTART, /* S_FINALIZE_JOIN ==> */ O_RELEASE | O_DC_TIMER_RESTART, /* S_NOT_DC ==> */ A_LOG | O_DC_TIMER_RESTART, /* S_POLICY_ENGINE ==> */ O_RELEASE | O_DC_TIMER_RESTART, /* S_RECOVERY ==> */ A_WARN, /* S_RELEASE_DC ==> */ A_WARN | O_DC_TIMER_RESTART, /* S_STARTING ==> */ A_LOG | A_DC_TIMER_START | A_CL_JOIN_QUERY, /* S_PENDING ==> */ A_LOG | O_DC_TIMER_RESTART, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN, /* S_TRANSITION_ENGINE ==> */ O_RELEASE | O_DC_TIMER_RESTART, /* S_HALT ==> */ A_WARN, }, /* Got an I_HALT */ { /* S_IDLE ==> */ A_WARN, /* S_ELECTION ==> */ A_WARN, /* S_INTEGRATION ==> */ A_WARN, /* S_FINALIZE_JOIN ==> */ A_WARN, /* S_NOT_DC ==> */ A_WARN, /* S_POLICY_ENGINE ==> */ A_WARN, /* S_RECOVERY ==> */ A_WARN, /* S_RELEASE_DC ==> */ A_WARN, /* S_STARTING ==> */ A_WARN, /* S_PENDING ==> */ A_WARN, /* S_STOPPING ==> */ A_WARN, /* S_TERMINATE ==> */ A_WARN, /* S_TRANSITION_ENGINE ==> */ A_WARN, /* S_HALT ==> */ A_WARN, }, }; #endif diff --git a/crmd/lrm.c b/crmd/lrm.c index a3c9215639..d2698822d1 100644 --- a/crmd/lrm.c +++ b/crmd/lrm.c @@ -1,2081 +1,2093 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define START_DELAY_THRESHOLD 5 * 60 * 1000 typedef struct resource_history_s { char *id; lrm_rsc_t rsc; lrm_op_t *last; lrm_op_t *failed; GList *recurring_op_list; } rsc_history_t; struct recurring_op_s { char *rsc_id; char *op_key; int call_id; int interval; gboolean remove; gboolean cancelled; }; struct pending_deletion_op_s { char *rsc; ha_msg_input_t *input; }; struct delete_event_s { int rc; const char *rsc; }; GHashTable *resource_history = NULL; GHashTable *pending_ops = NULL; GHashTable *deletion_ops = NULL; GCHSource *lrm_source = NULL; int num_lrm_register_fails = 0; int max_lrm_register_fails = 30; gboolean populate_history_cache(void); gboolean process_lrm_event(lrm_op_t * op); gboolean is_rsc_active(const char *rsc_id); gboolean build_active_RAs(xmlNode * rsc_list); static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data); static int delete_rsc_status(const char *rsc_id, int call_options, const char *user_name); lrm_op_t *construct_op(xmlNode * rsc_op, const char *rsc_id, const char *operation); void do_lrm_rsc_op(lrm_rsc_t * rsc, const char *operation, xmlNode * msg, xmlNode * request); void send_direct_ack(const char *to_host, const char *to_sys, lrm_rsc_t * rsc, lrm_op_t * op, const char *rsc_id); void lrm_connection_destroy(gpointer user_data) { if (is_set(fsa_input_register, R_LRM_CONNECTED)) { crm_crit("LRM Connection failed"); register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL); clear_bit_inplace(fsa_input_register, R_LRM_CONNECTED); } else { crm_info("LRM Connection disconnected"); } lrm_source = NULL; } static void free_deletion_op(gpointer value) { struct pending_deletion_op_s *op = value; crm_free(op->rsc); delete_ha_msg_input(op->input); crm_free(op); } static void free_recurring_op(gpointer value) { struct recurring_op_s *op = (struct recurring_op_s *)value; crm_free(op->rsc_id); crm_free(op->op_key); crm_free(op); } static char * make_stop_id(const char *rsc, int call_id) { char *op_id = NULL; crm_malloc0(op_id, strlen(rsc) + 34); if (op_id != NULL) { snprintf(op_id, strlen(rsc) + 34, "%s:%d", rsc, call_id); } return op_id; } static void dup_attr(gpointer key, gpointer value, gpointer user_data) { g_hash_table_replace(user_data, crm_strdup(key), crm_strdup(value)); } static void history_cache_destroy(gpointer data) { rsc_history_t *entry = data; crm_free(entry->rsc.type); crm_free(entry->rsc.class); crm_free(entry->rsc.provider); free_lrm_op(entry->failed); free_lrm_op(entry->last); crm_free(entry->id); crm_free(entry); } static void update_history_cache(lrm_rsc_t * rsc, lrm_op_t * op) { int target_rc = 0; rsc_history_t *entry = NULL; #ifdef HAVE_LRM_OP_T_RSC_DELETED if (op->rsc_deleted) { crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type); delete_rsc_status(op->rsc_id, cib_quorum_override, NULL); return; } #endif if (safe_str_eq(op->op_type, RSC_NOTIFY)) { return; } crm_debug("Appending %s op to history for '%s'", op->op_type, op->rsc_id); entry = g_hash_table_lookup(resource_history, op->rsc_id); if (entry == NULL && rsc) { crm_malloc0(entry, sizeof(rsc_history_t)); entry->id = crm_strdup(op->rsc_id); g_hash_table_insert(resource_history, entry->id, entry); entry->rsc.id = entry->id; entry->rsc.type = crm_strdup(rsc->type); entry->rsc.class = crm_strdup(rsc->class); if (rsc->provider) { entry->rsc.provider = crm_strdup(rsc->provider); } else { entry->rsc.provider = NULL; } } else if (entry == NULL) { crm_info("Resource %s no longer exists, not updating cache", op->rsc_id); return; } target_rc = rsc_op_expected_rc(op); if (op->op_status == LRM_OP_CANCELLED) { crm_trace("Skipping %s_%s_%d rc=%d, status=%d", op->rsc_id, op->op_type, op->interval, op->rc, op->op_status); } else if (did_rsc_op_fail(op, target_rc)) { /* We must store failed monitors here * - otherwise the block below will cause them to be forgetten them when a stop happens */ if (entry->failed) { free_lrm_op(entry->failed); } entry->failed = copy_lrm_op(op); } else if (op->interval == 0) { if (entry->last) { free_lrm_op(entry->last); } entry->last = copy_lrm_op(op); } if (op->interval > 0) { crm_trace("Adding recurring op: %s_%s_%d", op->rsc_id, op->op_type, op->interval); entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, copy_lrm_op(op)); } else if (entry->recurring_op_list && safe_str_eq(op->op_type, RSC_STATUS) == FALSE) { GList *gIter = entry->recurring_op_list; crm_trace("Dropping %d recurring ops because of: %s_%s_%d", g_list_length(gIter), op->rsc_id, op->op_type, op->interval); for (; gIter != NULL; gIter = gIter->next) { free_lrm_op(gIter->data); } g_list_free(entry->recurring_op_list); entry->recurring_op_list = NULL; } } /* A_LRM_CONNECT */ void do_lrm_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { if (fsa_lrm_conn == NULL) { register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); return; } if (action & A_LRM_DISCONNECT) { if (verify_stopped(cur_state, LOG_INFO) == FALSE) { - crmd_fsa_stall(NULL); - return; + if(action == A_LRM_DISCONNECT) { + crmd_fsa_stall(NULL); + return; + } } if (is_set(fsa_input_register, R_LRM_CONNECTED)) { clear_bit_inplace(fsa_input_register, R_LRM_CONNECTED); + + if (lrm_source) { + G_main_del_IPC_Channel(lrm_source); + lrm_source = NULL; + } fsa_lrm_conn->lrm_ops->signoff(fsa_lrm_conn); crm_info("Disconnected from the LRM"); } - /* TODO: Clean up the hashtable */ + g_hash_table_destroy(resource_history); + resource_history = NULL; + g_hash_table_destroy(deletion_ops); + deletion_ops = NULL; + g_hash_table_destroy(pending_ops); + pending_ops = NULL; } if (action & A_LRM_CONNECT) { int ret = HA_OK; deletion_ops = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, free_deletion_op); pending_ops = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, free_recurring_op); resource_history = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, history_cache_destroy); if (ret == HA_OK) { crm_debug("Connecting to the LRM"); ret = fsa_lrm_conn->lrm_ops->signon(fsa_lrm_conn, CRM_SYSTEM_CRMD); } if (ret != HA_OK) { if (++num_lrm_register_fails < max_lrm_register_fails) { crm_warn("Failed to sign on to the LRM %d" " (%d max) times", num_lrm_register_fails, max_lrm_register_fails); crm_timer_start(wait_timer); crmd_fsa_stall(NULL); return; } } if (ret == HA_OK) { crm_trace("LRM: set_lrm_callback..."); ret = fsa_lrm_conn->lrm_ops->set_lrm_callback(fsa_lrm_conn, lrm_op_callback); if (ret != HA_OK) { crm_err("Failed to set LRM callbacks"); } } if (ret != HA_OK) { crm_err("Failed to sign on to the LRM %d" " (max) times", num_lrm_register_fails); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); return; } populate_history_cache(); /* TODO: create a destroy handler that causes * some recovery to happen */ lrm_source = G_main_add_IPC_Channel(G_PRIORITY_LOW, fsa_lrm_conn->lrm_ops->ipcchan(fsa_lrm_conn), FALSE, lrm_dispatch, fsa_lrm_conn, lrm_connection_destroy); set_bit_inplace(fsa_input_register, R_LRM_CONNECTED); crm_debug("LRM connection established"); } if (action & ~(A_LRM_CONNECT | A_LRM_DISCONNECT)) { crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__); } } static void ghash_print_pending(gpointer key, gpointer value, gpointer user_data) { const char *stop_id = key; int *log_level = user_data; struct recurring_op_s *pending = value; do_crm_log(*log_level, "Pending action: %s (%s)", stop_id, pending->op_key); } static void ghash_print_pending_for_rsc(gpointer key, gpointer value, gpointer user_data) { const char *stop_id = key; char *rsc = user_data; struct recurring_op_s *pending = value; if (safe_str_eq(rsc, pending->rsc_id)) { crm_notice("%sction %s (%s) incomplete at shutdown", pending->interval == 0 ? "A" : "Recurring a", stop_id, pending->op_key); } } static void ghash_count_pending(gpointer key, gpointer value, gpointer user_data) { int *counter = user_data; struct recurring_op_s *pending = value; if (pending->interval > 0) { /* Ignore recurring actions in the shutdown calculations */ return; } (*counter)++; } gboolean verify_stopped(enum crmd_fsa_state cur_state, int log_level) { int counter = 0; gboolean rc = TRUE; GHashTableIter gIter; rsc_history_t *entry = NULL; crm_debug("Checking for active resources before exit"); if (cur_state == S_TERMINATE) { log_level = LOG_ERR; } if (pending_ops) { if (is_set(fsa_input_register, R_LRM_CONNECTED)) { /* Only log/complain about non-recurring actions */ g_hash_table_foreach_remove(pending_ops, stop_recurring_actions, NULL); } g_hash_table_foreach(pending_ops, ghash_count_pending, &counter); } if (counter > 0) { rc = FALSE; do_crm_log(log_level, "%d pending LRM operations at shutdown%s", counter, cur_state == S_TERMINATE ? "" : "... waiting"); if (cur_state == S_TERMINATE || !is_set(fsa_input_register, R_SENT_RSC_STOP)) { g_hash_table_foreach(pending_ops, ghash_print_pending, &log_level); } goto bail; } if (resource_history == NULL) { goto bail; } g_hash_table_iter_init(&gIter, resource_history); while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) { if (is_rsc_active(entry->id) == FALSE) { continue; } crm_err("Resource %s was active at shutdown." " You may ignore this error if it is unmanaged.", entry->id); g_hash_table_foreach(pending_ops, ghash_print_pending_for_rsc, entry->id); } bail: set_bit_inplace(fsa_input_register, R_SENT_RSC_STOP); if (cur_state == S_TERMINATE) { rc = TRUE; } return rc; } static char * get_rsc_metadata(const char *type, const char *class, const char *provider) { char *metadata = NULL; CRM_CHECK(type != NULL, return NULL); CRM_CHECK(class != NULL, return NULL); if (provider == NULL) { provider = "heartbeat"; } crm_trace("Retreiving metadata for %s::%s:%s", type, class, provider); metadata = fsa_lrm_conn->lrm_ops->get_rsc_type_metadata(fsa_lrm_conn, class, type, provider); if (metadata) { /* copy the metadata because the LRM likes using * g_alloc instead of cl_malloc */ char *m_copy = crm_strdup(metadata); g_free(metadata); metadata = m_copy; } else { crm_warn("No metadata found for %s::%s:%s", type, class, provider); } return metadata; } typedef struct reload_data_s { char *key; char *metadata; time_t last_query; gboolean can_reload; GListPtr restart_list; } reload_data_t; static void g_hash_destroy_reload(gpointer data) { reload_data_t *reload = data; crm_free(reload->key); crm_free(reload->metadata); slist_basic_destroy(reload->restart_list); crm_free(reload); } GHashTable *reload_hash = NULL; static GListPtr get_rsc_restart_list(lrm_rsc_t * rsc, lrm_op_t * op) { int len = 0; char *key = NULL; char *copy = NULL; const char *value = NULL; const char *provider = NULL; xmlNode *param = NULL; xmlNode *params = NULL; xmlNode *actions = NULL; xmlNode *metadata = NULL; time_t now = time(NULL); reload_data_t *reload = NULL; if (reload_hash == NULL) { reload_hash = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, g_hash_destroy_reload); } provider = rsc->provider; if (provider == NULL) { provider = "heartbeat"; } len = strlen(rsc->type) + strlen(rsc->class) + strlen(provider) + 4; crm_malloc(key, len); snprintf(key, len, "%s::%s:%s", rsc->type, rsc->class, provider); reload = g_hash_table_lookup(reload_hash, key); if (reload && ((now - 9) > reload->last_query) && safe_str_eq(op->op_type, RSC_START)) { reload = NULL; /* re-query */ } if (reload == NULL) { xmlNode *action = NULL; crm_malloc0(reload, sizeof(reload_data_t)); g_hash_table_replace(reload_hash, key, reload); reload->last_query = now; reload->key = key; key = NULL; reload->metadata = get_rsc_metadata(rsc->type, rsc->class, provider); metadata = string2xml(reload->metadata); if (metadata == NULL) { crm_err("Metadata for %s::%s:%s is not valid XML", rsc->provider, rsc->class, rsc->type); goto cleanup; } actions = find_xml_node(metadata, "actions", TRUE); for (action = __xml_first_child(actions); action != NULL; action = __xml_next(action)) { if (crm_str_eq((const char *)action->name, "action", TRUE)) { value = crm_element_value(action, "name"); if (safe_str_eq("reload", value)) { reload->can_reload = TRUE; break; } } } if (reload->can_reload == FALSE) { goto cleanup; } params = find_xml_node(metadata, "parameters", TRUE); for (param = __xml_first_child(params); param != NULL; param = __xml_next(param)) { if (crm_str_eq((const char *)param->name, "parameter", TRUE)) { value = crm_element_value(param, "unique"); if (crm_is_true(value)) { value = crm_element_value(param, "name"); if (value == NULL) { crm_err("%s: NULL param", key); continue; } crm_debug("Attr %s is not reloadable", value); copy = crm_strdup(value); CRM_CHECK(copy != NULL, continue); reload->restart_list = g_list_append(reload->restart_list, copy); } } } } cleanup: crm_free(key); free_xml(metadata); return reload ? reload->restart_list : NULL; } static void append_restart_list(lrm_rsc_t * rsc, lrm_op_t * op, xmlNode * update, const char *version) { int len = 0; char *list = NULL; char *digest = NULL; const char *value = NULL; gboolean non_empty = FALSE; xmlNode *restart = NULL; GListPtr restart_list = NULL; GListPtr lpc = NULL; if (op->interval > 0) { /* monitors are not reloadable */ return; } else if (op->params == NULL) { crm_debug("%s has no parameters", ID(update)); return; } else if (rsc == NULL) { return; } else if (crm_str_eq(CRMD_ACTION_STOP, op->op_type, TRUE)) { /* Stopped resources don't need to be reloaded */ return; } else if (compare_version("1.0.8", version) > 0) { /* Caller version does not support reloads */ return; } restart_list = get_rsc_restart_list(rsc, op); if (restart_list == NULL) { /* Resource does not support reloads */ return; } restart = create_xml_node(NULL, XML_TAG_PARAMS); for (lpc = restart_list; lpc != NULL; lpc = lpc->next) { const char *param = (const char *)lpc->data; int start = len; CRM_CHECK(param != NULL, continue); value = g_hash_table_lookup(op->params, param); if (value != NULL) { non_empty = TRUE; crm_xml_add(restart, param, value); } len += strlen(param) + 2; crm_realloc(list, len + 1); sprintf(list + start, " %s ", param); } digest = calculate_operation_digest(restart, version); crm_xml_add(update, XML_LRM_ATTR_OP_RESTART, list); crm_xml_add(update, XML_LRM_ATTR_RESTART_DIGEST, digest); #if 0 crm_debug("%s: %s, %s", rsc->id, digest, list); if (non_empty) { crm_log_xml_debug(restart, "restart digest source"); } #endif free_xml(restart); crm_free(digest); crm_free(list); } static gboolean build_operation_update(xmlNode * parent, lrm_rsc_t * rsc, lrm_op_t * op, const char *src) { int target_rc = 0; xmlNode *xml_op = NULL; const char *caller_version = CRM_FEATURE_SET; if (op == NULL) { return FALSE; } else if (AM_I_DC) { } else if (fsa_our_dc_version != NULL) { caller_version = fsa_our_dc_version; } else if (op->params == NULL) { caller_version = fsa_our_dc_version; } else { /* there is a small risk in formerly mixed clusters that * it will be sub-optimal. * however with our upgrade policy, the update we send * should still be completely supported anyway */ caller_version = g_hash_table_lookup(op->params, XML_ATTR_CRM_VERSION); crm_warn("Falling back to operation originator version: %s", caller_version); } target_rc = rsc_op_expected_rc(op); xml_op = create_operation_update(parent, op, caller_version, target_rc, src, LOG_DEBUG); if (xml_op) { append_restart_list(rsc, op, xml_op, caller_version); } return TRUE; } gboolean is_rsc_active(const char *rsc_id) { rsc_history_t *entry = NULL; crm_trace("Processing lrm_rsc_t entry %s", rsc_id); entry = g_hash_table_lookup(resource_history, rsc_id); if (entry == NULL || entry->last == NULL) { return FALSE; } if (entry->last->rc == EXECRA_OK && safe_str_eq(entry->last->op_type, CRMD_ACTION_STOP)) { return FALSE; } else if (entry->last->rc == EXECRA_OK && safe_str_eq(entry->last->op_type, CRMD_ACTION_MIGRATE)) { /* a stricter check is too complex... * leave that to the PE */ return FALSE; } else if (entry->last->rc == EXECRA_NOT_RUNNING) { return FALSE; } else if (entry->last->interval == 0 && entry->last->rc == EXECRA_NOT_CONFIGURED) { /* Badly configured resources can't be reliably stopped */ return FALSE; } return TRUE; } gboolean build_active_RAs(xmlNode * rsc_list) { GHashTableIter iter; rsc_history_t *entry = NULL; g_hash_table_iter_init(&iter, resource_history); while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) { GList *gIter = NULL; xmlNode *xml_rsc = create_xml_node(rsc_list, XML_LRM_TAG_RESOURCE); crm_xml_add(xml_rsc, XML_ATTR_ID, entry->id); crm_xml_add(xml_rsc, XML_ATTR_TYPE, entry->rsc.type); crm_xml_add(xml_rsc, XML_AGENT_ATTR_CLASS, entry->rsc.class); crm_xml_add(xml_rsc, XML_AGENT_ATTR_PROVIDER, entry->rsc.provider); build_operation_update(xml_rsc, &(entry->rsc), entry->last, __FUNCTION__); build_operation_update(xml_rsc, &(entry->rsc), entry->failed, __FUNCTION__); for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) { build_operation_update(xml_rsc, &(entry->rsc), gIter->data, __FUNCTION__); } } return FALSE; } gboolean populate_history_cache(void) { GListPtr gIter = NULL; GListPtr gIter2 = NULL; GList *op_list = NULL; GList *rsc_list = NULL; state_flag_t cur_state = 0; rsc_list = fsa_lrm_conn->lrm_ops->get_all_rscs(fsa_lrm_conn); for (gIter = rsc_list; gIter != NULL; gIter = gIter->next) { char *rid = (char *)gIter->data; int max_call_id = -1; lrm_rsc_t *rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rid); if (rsc == NULL) { crm_err("NULL resource returned from the LRM: %s", rid); continue; } op_list = rsc->ops->get_cur_state(rsc, &cur_state); for (gIter2 = op_list; gIter2 != NULL; gIter2 = gIter2->next) { lrm_op_t *op = (lrm_op_t *) gIter2->data; if (max_call_id < op->call_id) { update_history_cache(rsc, op); } else if (max_call_id > op->call_id) { crm_err("Bad call_id in list=%d. Previous call_id=%d", op->call_id, max_call_id); } else { crm_warn("lrm->get_cur_state() returned" " duplicate entries for call_id=%d", op->call_id); } max_call_id = op->call_id; lrm_free_op(op); } g_list_free(op_list); lrm_free_rsc(rsc); free(rid); } g_list_free(rsc_list); return TRUE; } xmlNode * do_lrm_query(gboolean is_replace) { gboolean shut_down = FALSE; xmlNode *xml_result = NULL; xmlNode *xml_state = NULL; xmlNode *xml_data = NULL; xmlNode *rsc_list = NULL; const char *exp_state = CRMD_STATE_ACTIVE; if (is_set(fsa_input_register, R_SHUTDOWN)) { exp_state = CRMD_STATE_INACTIVE; shut_down = TRUE; } xml_state = create_node_state(fsa_our_uname, ACTIVESTATUS, XML_BOOLEAN_TRUE, ONLINESTATUS, CRMD_JOINSTATE_MEMBER, exp_state, !shut_down, __FUNCTION__); xml_data = create_xml_node(xml_state, XML_CIB_TAG_LRM); crm_xml_add(xml_data, XML_ATTR_ID, fsa_our_uuid); rsc_list = create_xml_node(xml_data, XML_LRM_TAG_RESOURCES); /* Build a list of active (not always running) resources */ build_active_RAs(rsc_list); xml_result = create_cib_fragment(xml_state, XML_CIB_TAG_STATUS); crm_log_xml_trace(xml_state, "Current state of the LRM"); free_xml(xml_state); return xml_result; } static void notify_deleted(ha_msg_input_t * input, const char *rsc_id, int rc) { lrm_op_t *op = NULL; const char *from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM); const char *from_host = crm_element_value(input->msg, F_CRM_HOST_FROM); crm_info("Notifying %s on %s that %s was%s deleted", from_sys, from_host, rsc_id, rc == HA_OK ? "" : " not"); op = construct_op(input->xml, rsc_id, CRMD_ACTION_DELETE); CRM_ASSERT(op != NULL); if (rc == HA_OK) { op->op_status = LRM_OP_DONE; op->rc = EXECRA_OK; } else { op->op_status = LRM_OP_ERROR; op->rc = EXECRA_UNKNOWN_ERROR; } send_direct_ack(from_host, from_sys, NULL, op, rsc_id); free_lrm_op(op); if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) { /* this isn't expected - trigger a new transition */ time_t now = time(NULL); char *now_s = crm_itoa(now); crm_debug("Triggering a refresh after %s deleted %s from the LRM", from_sys, rsc_id); update_attr(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL, "last-lrm-refresh", now_s, FALSE); crm_free(now_s); } } static gboolean lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data) { struct delete_event_s *event = user_data; struct pending_deletion_op_s *op = value; if (safe_str_eq(event->rsc, op->rsc)) { notify_deleted(op->input, event->rsc, event->rc); return TRUE; } return FALSE; } static gboolean lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data) { const char *rsc = user_data; struct recurring_op_s *pending = value; if (safe_str_eq(rsc, pending->rsc_id)) { crm_info("Removing op %s:%d for deleted resource %s", pending->op_key, pending->call_id, rsc); return TRUE; } return FALSE; } /* * Remove the rsc from the CIB * * Avoids refreshing the entire LRM section of this host */ #define rsc_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']" static int delete_rsc_status(const char *rsc_id, int call_options, const char *user_name) { char *rsc_xpath = NULL; int max = 0; int rc = cib_ok; CRM_CHECK(rsc_id != NULL, return cib_id_check); max = strlen(rsc_template) + strlen(rsc_id) + strlen(fsa_our_uname) + 1; crm_malloc0(rsc_xpath, max); snprintf(rsc_xpath, max, rsc_template, fsa_our_uname, rsc_id); rc = fsa_cib_conn->cmds->delegated_variant_op(fsa_cib_conn, CIB_OP_DELETE, NULL, rsc_xpath, NULL, NULL, call_options | cib_xpath, user_name); crm_free(rsc_xpath); return rc; } static void delete_rsc_entry(ha_msg_input_t * input, const char *rsc_id, GHashTableIter *rsc_gIter, int rc, const char *user_name) { struct delete_event_s event; CRM_CHECK(rsc_id != NULL, return); if (rc == HA_OK) { char *rsc_id_copy = crm_strdup(rsc_id); if (rsc_gIter) g_hash_table_iter_remove(rsc_gIter); else g_hash_table_remove(resource_history, rsc_id_copy); crm_debug("sync: Sending delete op for %s", rsc_id_copy); delete_rsc_status(rsc_id_copy, cib_quorum_override, user_name); g_hash_table_foreach_remove(pending_ops, lrm_remove_deleted_op, rsc_id_copy); crm_free(rsc_id_copy); } if (input) { notify_deleted(input, rsc_id, rc); } event.rc = rc; event.rsc = rsc_id; g_hash_table_foreach_remove(deletion_ops, lrm_remove_deleted_rsc, &event); } /* * Remove the op from the CIB * * Avoids refreshing the entire LRM section of this host */ #define op_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s']" #define op_call_template "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']/"XML_LRM_TAG_RSC_OP"[@id='%s' and @"XML_LRM_ATTR_CALLID"='%d']" static void delete_op_entry(lrm_op_t * op, const char *rsc_id, const char *key, int call_id) { xmlNode *xml_top = NULL; if (op != NULL) { xml_top = create_xml_node(NULL, XML_LRM_TAG_RSC_OP); crm_xml_add_int(xml_top, XML_LRM_ATTR_CALLID, op->call_id); crm_xml_add(xml_top, XML_ATTR_TRANSITION_KEY, op->user_data); if(op->interval > 0) { /* Avoid deleting last_failure too (if it was a result of this recurring op failing) */ crm_xml_add(xml_top, XML_ATTR_ID, op->user_data); } crm_debug("async: Sending delete op for %s_%s_%d (call=%d)", op->rsc_id, op->op_type, op->interval, op->call_id); fsa_cib_conn->cmds->delete(fsa_cib_conn, XML_CIB_TAG_STATUS, xml_top, cib_quorum_override); } else if (rsc_id != NULL && key != NULL) { int max = 0; char *op_xpath = NULL; if (call_id > 0) { max = strlen(op_call_template) + strlen(rsc_id) + strlen(fsa_our_uname) + strlen(key) + 10; crm_malloc0(op_xpath, max); snprintf(op_xpath, max, op_call_template, fsa_our_uname, rsc_id, key, call_id); } else { max = strlen(op_template) + strlen(rsc_id) + strlen(fsa_our_uname) + strlen(key) + 1; crm_malloc0(op_xpath, max); snprintf(op_xpath, max, op_template, fsa_our_uname, rsc_id, key); } crm_debug("sync: Sending delete op for %s (call=%d)", rsc_id, call_id); fsa_cib_conn->cmds->delete(fsa_cib_conn, op_xpath, NULL, cib_quorum_override | cib_xpath); crm_free(op_xpath); } else { crm_err("Not enough information to delete op entry: rsc=%p key=%p", rsc_id, key); return; } crm_log_xml_trace(xml_top, "op:cancel"); free_xml(xml_top); } void lrm_clear_last_failure(const char *rsc_id) { char *attr = NULL; GHashTableIter iter; rsc_history_t *entry = NULL; attr = generate_op_key(rsc_id, "last_failure", 0); delete_op_entry(NULL, rsc_id, attr, 0); crm_free(attr); if (!resource_history) { return; } g_hash_table_iter_init(&iter, resource_history); while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) { if (safe_str_eq(rsc_id, entry->id)) { free_lrm_op(entry->failed); entry->failed = NULL; } } } static gboolean cancel_op(lrm_rsc_t * rsc, const char *key, int op, gboolean remove) { int rc = HA_OK; struct recurring_op_s *pending = NULL; CRM_CHECK(op != 0, return FALSE); CRM_CHECK(rsc != NULL, return FALSE); if (key == NULL) { key = make_stop_id(rsc->id, op); } pending = g_hash_table_lookup(pending_ops, key); if (pending) { if (remove && pending->remove == FALSE) { pending->remove = TRUE; crm_debug("Scheduling %s for removal", key); } if (pending->cancelled) { crm_debug("Operation %s already cancelled", key); return TRUE; } pending->cancelled = TRUE; } else { crm_info("No pending op found for %s", key); } crm_debug("Cancelling op %d for %s (%s)", op, rsc->id, key); rc = rsc->ops->cancel_op(rsc, op); if (rc == HA_OK) { crm_debug("Op %d for %s (%s): cancelled", op, rsc->id, key); #ifdef HAVE_LRM_OP_T_RSC_DELETED } else if (rc == HA_RSCBUSY) { crm_debug("Op %d for %s (%s): cancelation pending", op, rsc->id, key); #endif } else { crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc->id, key); /* The caller needs to make sure the entry is * removed from the pending_ops list * * Usually by returning TRUE inside the worker function * supplied to g_hash_table_foreach_remove() * * Not removing the entry from pending_ops will block * the node from shutting down */ return FALSE; } return TRUE; } struct cancel_data { gboolean done; gboolean remove; const char *key; lrm_rsc_t *rsc; }; static gboolean cancel_action_by_key(gpointer key, gpointer value, gpointer user_data) { struct cancel_data *data = user_data; struct recurring_op_s *op = (struct recurring_op_s *)value; if (safe_str_eq(op->op_key, data->key)) { data->done = TRUE; if (cancel_op(data->rsc, key, op->call_id, data->remove) == FALSE) { return TRUE; } } return FALSE; } static gboolean cancel_op_key(lrm_rsc_t * rsc, const char *key, gboolean remove) { struct cancel_data data; CRM_CHECK(rsc != NULL, return FALSE); CRM_CHECK(key != NULL, return FALSE); data.key = key; data.rsc = rsc; data.done = FALSE; data.remove = remove; g_hash_table_foreach_remove(pending_ops, cancel_action_by_key, &data); return data.done; } static lrm_rsc_t * get_lrm_resource(xmlNode * resource, xmlNode * op_msg, gboolean do_create) { char rid[RID_LEN]; lrm_rsc_t *rsc = NULL; const char *short_id = ID(resource); const char *long_id = crm_element_value(resource, XML_ATTR_ID_LONG); crm_trace("Retrieving %s from the LRM.", short_id); CRM_CHECK(short_id != NULL, return NULL); if (rsc == NULL) { /* check if its already there (short name) */ strncpy(rid, short_id, RID_LEN); rid[RID_LEN - 1] = 0; rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rid); } if (rsc == NULL && long_id != NULL) { /* try the long name instead */ strncpy(rid, long_id, RID_LEN); rid[RID_LEN - 1] = 0; rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rid); } if (rsc == NULL && do_create) { /* add it to the LRM */ const char *type = crm_element_value(resource, XML_ATTR_TYPE); const char *class = crm_element_value(resource, XML_AGENT_ATTR_CLASS); const char *provider = crm_element_value(resource, XML_AGENT_ATTR_PROVIDER); GHashTable *params = xml2list(op_msg); CRM_CHECK(class != NULL, return NULL); CRM_CHECK(type != NULL, return NULL); crm_trace("Adding rsc %s before operation", short_id); strncpy(rid, short_id, RID_LEN); rid[RID_LEN - 1] = 0; if (g_hash_table_size(params) == 0) { crm_log_xml_warn(op_msg, "EmptyParams"); } if (params != NULL) { g_hash_table_remove(params, CRM_META "_op_target_rc"); } fsa_lrm_conn->lrm_ops->add_rsc(fsa_lrm_conn, rid, class, type, provider, params); rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, rid); g_hash_table_destroy(params); if (rsc == NULL) { fsa_data_t *msg_data = NULL; crm_err("Could not add resource %s to LRM", rid); register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL); } } return rsc; } static void delete_resource(const char *id, lrm_rsc_t * rsc, GHashTableIter *gIter, const char *sys, const char *host, const char *user, ha_msg_input_t * request) { int rc = HA_OK; crm_info("Removing resource %s for %s (%s) on %s", id, sys, user ? user : "internal", host); if (rsc) { rc = fsa_lrm_conn->lrm_ops->delete_rsc(fsa_lrm_conn, id); } if (rc == HA_OK) { crm_trace("Resource '%s' deleted", id); #ifdef HAVE_LRM_OP_T_RSC_DELETED } else if (rc == HA_RSCBUSY) { crm_info("Deletion of resource '%s' pending", id); if (request) { struct pending_deletion_op_s *op = NULL; char *ref = crm_element_value_copy(request->msg, XML_ATTR_REFERENCE); crm_malloc0(op, sizeof(struct pending_deletion_op_s)); op->rsc = crm_strdup(rsc->id); op->input = copy_ha_msg_input(request); g_hash_table_insert(deletion_ops, ref, op); } return; #endif } else { crm_warn("Deletion of resource '%s' for %s (%s) on %s failed: %d", id, sys, user ? user : "internal", host, rc); } delete_rsc_entry(request, id, gIter, rc, user); } /* A_LRM_INVOKE */ void do_lrm_invoke(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { gboolean done = FALSE; gboolean create_rsc = TRUE; const char *crm_op = NULL; const char *from_sys = NULL; const char *from_host = NULL; const char *operation = NULL; ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg); const char *user_name = NULL; #if ENABLE_ACL user_name = crm_element_value(input->msg, F_CRM_USER); crm_trace("LRM command from user '%s'", user_name); #endif crm_op = crm_element_value(input->msg, F_CRM_TASK); from_sys = crm_element_value(input->msg, F_CRM_SYS_FROM); if (safe_str_neq(from_sys, CRM_SYSTEM_TENGINE)) { from_host = crm_element_value(input->msg, F_CRM_HOST_FROM); } crm_trace("LRM command from: %s", from_sys); if (safe_str_eq(crm_op, CRM_OP_LRM_DELETE)) { operation = CRMD_ACTION_DELETE; } else if (safe_str_eq(operation, CRM_OP_LRM_REFRESH)) { crm_op = CRM_OP_LRM_REFRESH; } else if (safe_str_eq(crm_op, CRM_OP_LRM_FAIL)) { #if HAVE_STRUCT_LRM_OPS_FAIL_RSC int rc = HA_OK; lrm_op_t *op = NULL; lrm_rsc_t *rsc = NULL; xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE); CRM_CHECK(xml_rsc != NULL, return); op = construct_op(input->xml, ID(xml_rsc), "fail"); op->op_status = LRM_OP_ERROR; op->rc = EXECRA_UNKNOWN_ERROR; CRM_ASSERT(op != NULL); # if ENABLE_ACL if (user_name && is_privileged(user_name) == FALSE) { crm_err("%s does not have permission to fail %s", user_name, ID(xml_rsc)); send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc)); free_lrm_op(op); return; } # endif rsc = get_lrm_resource(xml_rsc, input->xml, create_rsc); if (rsc) { crm_info("Failing resource %s...", rsc->id); rc = fsa_lrm_conn->lrm_ops->fail_rsc(fsa_lrm_conn, rsc->id, 1, "do_lrm_invoke: Async failure"); if (rc != HA_OK) { crm_err("Could not initiate an asynchronous failure for %s (%d)", rsc->id, rc); } else { op->op_status = LRM_OP_DONE; op->rc = EXECRA_OK; } lrm_free_rsc(rsc); } else { crm_info("Cannot find/create resource in order to fail it..."); crm_log_xml_warn(input->msg, "bad input"); } send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc)); free_lrm_op(op); return; #else crm_info("Failing resource..."); operation = "fail"; #endif } else if (input->xml != NULL) { operation = crm_element_value(input->xml, XML_LRM_ATTR_TASK); } if (safe_str_eq(crm_op, CRM_OP_LRM_REFRESH)) { enum cib_errors rc = cib_ok; xmlNode *fragment = do_lrm_query(TRUE); crm_info("Forcing a local LRM refresh"); fsa_cib_update(XML_CIB_TAG_STATUS, fragment, cib_quorum_override, rc, user_name); free_xml(fragment); } else if (safe_str_eq(crm_op, CRM_OP_LRM_QUERY)) { xmlNode *data = do_lrm_query(FALSE); xmlNode *reply = create_reply(input->msg, data); if (relay_message(reply, TRUE) == FALSE) { crm_err("Unable to route reply"); crm_log_xml_err(reply, "reply"); } free_xml(reply); free_xml(data); } else if (safe_str_eq(operation, CRM_OP_PROBED)) { update_attrd(NULL, CRM_OP_PROBED, XML_BOOLEAN_TRUE, user_name); } else if (safe_str_eq(crm_op, CRM_OP_REPROBE)) { GHashTableIter gIter; rsc_history_t *entry = NULL; crm_notice("Forcing the status of all resources to be redetected"); g_hash_table_iter_init(&gIter, resource_history); while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) { lrm_rsc_t *rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, entry->id); delete_resource(entry->id, rsc, &gIter, from_sys, from_host, user_name, NULL); lrm_free_rsc(rsc); } /* Now delete the copy in the CIB */ erase_status_tag(fsa_our_uname, XML_CIB_TAG_LRM, cib_scope_local); /* And finally, _delete_ the value in attrd * Setting it to FALSE results in the PE sending us back here again */ update_attrd(NULL, CRM_OP_PROBED, NULL, user_name); } else if (operation != NULL) { lrm_rsc_t *rsc = NULL; xmlNode *params = NULL; xmlNode *xml_rsc = find_xml_node(input->xml, XML_CIB_TAG_RESOURCE, TRUE); CRM_CHECK(xml_rsc != NULL, return); /* only the first 16 chars are used by the LRM */ params = find_xml_node(input->xml, XML_TAG_ATTRS, TRUE); if (safe_str_eq(operation, CRMD_ACTION_DELETE)) { create_rsc = FALSE; } rsc = get_lrm_resource(xml_rsc, input->xml, create_rsc); if (rsc == NULL && create_rsc) { crm_err("Invalid resource definition"); crm_log_xml_warn(input->msg, "bad input"); } else if (rsc == NULL) { lrm_op_t *op = NULL; crm_notice("Not creating resource for a %s event: %s", operation, ID(input->xml)); delete_rsc_entry(input, ID(xml_rsc), NULL, HA_OK, user_name); op = construct_op(input->xml, ID(xml_rsc), operation); op->op_status = LRM_OP_DONE; op->rc = EXECRA_OK; CRM_ASSERT(op != NULL); send_direct_ack(from_host, from_sys, NULL, op, ID(xml_rsc)); free_lrm_op(op); } else if (safe_str_eq(operation, CRMD_ACTION_CANCEL)) { lrm_op_t *op = NULL; char *op_key = NULL; char *meta_key = NULL; int call = 0; const char *call_id = NULL; const char *op_task = NULL; const char *op_interval = NULL; CRM_CHECK(params != NULL, crm_log_xml_warn(input->xml, "Bad command"); return); meta_key = crm_meta_name(XML_LRM_ATTR_INTERVAL); op_interval = crm_element_value(params, meta_key); crm_free(meta_key); meta_key = crm_meta_name(XML_LRM_ATTR_TASK); op_task = crm_element_value(params, meta_key); crm_free(meta_key); meta_key = crm_meta_name(XML_LRM_ATTR_CALLID); call_id = crm_element_value(params, meta_key); crm_free(meta_key); CRM_CHECK(op_task != NULL, crm_log_xml_warn(input->xml, "Bad command"); return); CRM_CHECK(op_interval != NULL, crm_log_xml_warn(input->xml, "Bad command"); return); op = construct_op(input->xml, rsc->id, op_task); CRM_ASSERT(op != NULL); op_key = generate_op_key(rsc->id, op_task, crm_parse_int(op_interval, "0")); crm_debug("PE requested op %s (call=%s) be cancelled", op_key, call_id ? call_id : "NA"); call = crm_parse_int(call_id, "0"); if (call == 0) { /* the normal case when the PE cancels a recurring op */ done = cancel_op_key(rsc, op_key, TRUE); } else { /* the normal case when the PE cancels an orphan op */ done = cancel_op(rsc, NULL, call, TRUE); } if (done == FALSE) { crm_debug("Nothing known about operation %d for %s", call, op_key); delete_op_entry(NULL, rsc->id, op_key, call); /* needed?? surely not otherwise the cancel_op_(_key) wouldn't * have failed in the first place */ g_hash_table_remove(pending_ops, op_key); } op->rc = EXECRA_OK; op->op_status = LRM_OP_DONE; send_direct_ack(from_host, from_sys, rsc, op, rsc->id); crm_free(op_key); free_lrm_op(op); } else if (safe_str_eq(operation, CRMD_ACTION_DELETE)) { int cib_rc = cib_ok; CRM_ASSERT(rsc != NULL); cib_rc = delete_rsc_status(rsc->id, cib_dryrun | cib_sync_call, user_name); if (cib_rc != cib_ok) { lrm_op_t *op = NULL; crm_err ("Attempt of deleting resource status '%s' from CIB for %s (user=%s) on %s failed: (rc=%d) %s", rsc->id, from_sys, user_name ? user_name : "unknown", from_host, cib_rc, cib_error2string(cib_rc)); op = construct_op(input->xml, rsc->id, operation); op->op_status = LRM_OP_ERROR; if (cib_rc == cib_permission_denied) { op->rc = EXECRA_INSUFFICIENT_PRIV; } else { op->rc = EXECRA_UNKNOWN_ERROR; } send_direct_ack(from_host, from_sys, NULL, op, rsc->id); free_lrm_op(op); return; } delete_resource(rsc->id, rsc, NULL, from_sys, from_host, user_name, input); } else if (rsc != NULL) { do_lrm_rsc_op(rsc, operation, input->xml, input->msg); } lrm_free_rsc(rsc); } else { crm_err("Operation was neither a lrm_query, nor a rsc op. %s", crm_str(crm_op)); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } static void copy_meta_keys(gpointer key, gpointer value, gpointer user_data) { if (strstr(key, CRM_META "_") != NULL) { g_hash_table_insert(user_data, strdup((const char *)key), strdup((const char *)value)); } } lrm_op_t * construct_op(xmlNode * rsc_op, const char *rsc_id, const char *operation) { lrm_op_t *op = NULL; const char *op_delay = NULL; const char *op_timeout = NULL; const char *op_interval = NULL; GHashTable *params = NULL; const char *transition = NULL; CRM_LOG_ASSERT(rsc_id != NULL); crm_malloc0(op, sizeof(lrm_op_t)); op->op_type = crm_strdup(operation); op->op_status = LRM_OP_PENDING; op->rc = -1; op->rsc_id = crm_strdup(rsc_id); op->interval = 0; op->timeout = 0; op->start_delay = 0; op->copyparams = 0; op->app_name = crm_strdup(CRM_SYSTEM_CRMD); if (rsc_op == NULL) { CRM_LOG_ASSERT(safe_str_eq(CRMD_ACTION_STOP, operation)); op->user_data = NULL; op->user_data_len = 0; /* the stop_all_resources() case * by definition there is no DC (or they'd be shutting * us down). * So we should put our version here. */ op->params = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); g_hash_table_insert(op->params, crm_strdup(XML_ATTR_CRM_VERSION), crm_strdup(CRM_FEATURE_SET)); crm_trace("Constructed %s op for %s", operation, rsc_id); return op; } params = xml2list(rsc_op); g_hash_table_remove(params, CRM_META "_op_target_rc"); op_delay = crm_meta_value(params, XML_OP_ATTR_START_DELAY); op_timeout = crm_meta_value(params, XML_ATTR_TIMEOUT); op_interval = crm_meta_value(params, XML_LRM_ATTR_INTERVAL); op->interval = crm_parse_int(op_interval, "0"); op->timeout = crm_parse_int(op_timeout, "0"); op->start_delay = crm_parse_int(op_delay, "0"); if (safe_str_neq(operation, RSC_STOP)) { op->params = params; } else { /* Create a blank parameter list so that we stop the resource * with the old attributes, not the new ones */ const char *version = g_hash_table_lookup(params, XML_ATTR_CRM_VERSION); op->params = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); if (version) { g_hash_table_insert(op->params, crm_strdup(XML_ATTR_CRM_VERSION), crm_strdup(version)); } g_hash_table_foreach(params, copy_meta_keys, op->params); g_hash_table_destroy(params); params = NULL; } /* sanity */ if (op->interval < 0) { op->interval = 0; } if (op->timeout <= 0) { op->timeout = op->interval; } if (op->start_delay < 0) { op->start_delay = 0; } transition = crm_element_value(rsc_op, XML_ATTR_TRANSITION_KEY); CRM_CHECK(transition != NULL, return op); op->user_data = crm_strdup(transition); op->user_data_len = 1 + strlen(op->user_data); if (op->interval != 0) { if (safe_str_eq(operation, CRMD_ACTION_START) || safe_str_eq(operation, CRMD_ACTION_STOP)) { crm_err("Start and Stop actions cannot have an interval: %d", op->interval); op->interval = 0; } } /* reset the resource's parameters? */ if (op->interval == 0) { if (safe_str_eq(CRMD_ACTION_START, operation) || safe_str_eq(CRMD_ACTION_STATUS, operation)) { op->copyparams = 1; } } crm_trace("Constructed %s op for %s: interval=%d", operation, rsc_id, op->interval); return op; } void send_direct_ack(const char *to_host, const char *to_sys, lrm_rsc_t * rsc, lrm_op_t * op, const char *rsc_id) { xmlNode *reply = NULL; xmlNode *update, *iter; xmlNode *fragment; CRM_CHECK(op != NULL, return); if (op->rsc_id == NULL) { CRM_LOG_ASSERT(rsc_id != NULL); op->rsc_id = crm_strdup(rsc_id); } if (to_sys == NULL) { to_sys = CRM_SYSTEM_TENGINE; } update = create_node_state(fsa_our_uname, NULL, NULL, NULL, NULL, NULL, FALSE, __FUNCTION__); iter = create_xml_node(update, XML_CIB_TAG_LRM); crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid); iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES); iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE); crm_xml_add(iter, XML_ATTR_ID, op->rsc_id); build_operation_update(iter, rsc, op, __FUNCTION__); fragment = create_cib_fragment(update, XML_CIB_TAG_STATUS); reply = create_request(CRM_OP_INVOKE_LRM, fragment, to_host, to_sys, CRM_SYSTEM_LRMD, NULL); crm_log_xml_trace(update, "ACK Update"); crm_debug("ACK'ing resource op %s_%s_%d from %s: %s", op->rsc_id, op->op_type, op->interval, op->user_data, crm_element_value(reply, XML_ATTR_REFERENCE)); if (relay_message(reply, TRUE) == FALSE) { crm_log_xml_err(reply, "Unable to route reply"); } free_xml(fragment); free_xml(update); free_xml(reply); } static gboolean stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data) { lrm_rsc_t *rsc = user_data; struct recurring_op_s *op = (struct recurring_op_s *)value; if (op->interval != 0 && safe_str_eq(op->rsc_id, rsc->id)) { if (cancel_op(rsc, key, op->call_id, FALSE) == FALSE) { return TRUE; } } return FALSE; } static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data) { gboolean remove = FALSE; struct recurring_op_s *op = (struct recurring_op_s *)value; if (op->interval != 0) { lrm_rsc_t *rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, op->rsc_id); if (rsc) { remove = cancel_op(rsc, key, op->call_id, FALSE); lrm_free_rsc(rsc); } } return remove; } void do_lrm_rsc_op(lrm_rsc_t * rsc, const char *operation, xmlNode * msg, xmlNode * request) { int call_id = 0; char *op_id = NULL; lrm_op_t *op = NULL; fsa_data_t *msg_data = NULL; const char *transition = NULL; CRM_CHECK(rsc != NULL, return); if (msg != NULL) { transition = crm_element_value(msg, XML_ATTR_TRANSITION_KEY); if (transition == NULL) { crm_log_xml_err(msg, "Missing transition number"); } } op = construct_op(msg, rsc->id, operation); /* stop the monitor before stopping the resource */ if (crm_str_eq(operation, CRMD_ACTION_STOP, TRUE) || crm_str_eq(operation, CRMD_ACTION_DEMOTE, TRUE) || crm_str_eq(operation, CRMD_ACTION_PROMOTE, TRUE) || crm_str_eq(operation, CRMD_ACTION_MIGRATE, TRUE)) { g_hash_table_foreach_remove(pending_ops, stop_recurring_action_by_rsc, rsc); } /* now do the op */ crm_debug("Performing key=%s op=%s_%s_%d", transition, rsc->id, operation, op->interval); if (fsa_state != S_NOT_DC && fsa_state != S_POLICY_ENGINE && fsa_state != S_TRANSITION_ENGINE) { if (safe_str_neq(operation, "fail") && safe_str_neq(operation, CRMD_ACTION_STOP)) { crm_info("Discarding attempt to perform action %s on %s" " in state %s", operation, rsc->id, fsa_state2string(fsa_state)); op->rc = 99; op->op_status = LRM_OP_ERROR; send_direct_ack(NULL, NULL, rsc, op, rsc->id); free_lrm_op(op); crm_free(op_id); return; } } op_id = generate_op_key(rsc->id, op->op_type, op->interval); if (op->interval > 0) { /* cancel it so we can then restart it without conflict */ cancel_op_key(rsc, op_id, FALSE); op->target_rc = CHANGED; } else { op->target_rc = EVERYTIME; } call_id = rsc->ops->perform_op(rsc, op); if (call_id <= 0) { crm_err("Operation %s on %s failed: %d", operation, rsc->id, call_id); register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL); } else { /* record all operations so we can wait * for them to complete during shutdown */ char *call_id_s = make_stop_id(rsc->id, call_id); struct recurring_op_s *pending = NULL; crm_malloc0(pending, sizeof(struct recurring_op_s)); crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s); pending->call_id = call_id; pending->interval = op->interval; pending->op_key = crm_strdup(op_id); pending->rsc_id = crm_strdup(rsc->id); g_hash_table_replace(pending_ops, call_id_s, pending); if (op->interval > 0 && op->start_delay > START_DELAY_THRESHOLD) { char *uuid = NULL; int dummy = 0, target_rc = 0; crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id); decode_transition_key(op->user_data, &uuid, &dummy, &dummy, &target_rc); crm_free(uuid); op->rc = target_rc; op->op_status = LRM_OP_DONE; send_direct_ack(NULL, NULL, rsc, op, rsc->id); } } crm_free(op_id); free_lrm_op(op); return; } lrm_op_t * copy_lrm_op(const lrm_op_t * op) { lrm_op_t *op_copy = NULL; CRM_CHECK(op != NULL, return NULL); CRM_CHECK(op->rsc_id != NULL, return NULL); crm_malloc0(op_copy, sizeof(lrm_op_t)); op_copy->op_type = crm_strdup(op->op_type); /* input fields */ op_copy->params = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); if (op->params != NULL) { g_hash_table_foreach(op->params, dup_attr, op_copy->params); } op_copy->timeout = op->timeout; op_copy->interval = op->interval; op_copy->target_rc = op->target_rc; /* in the CRM, this is always a string */ if (op->user_data != NULL) { op_copy->user_data = crm_strdup(op->user_data); } /* output fields */ op_copy->op_status = op->op_status; op_copy->rc = op->rc; op_copy->call_id = op->call_id; op_copy->output = NULL; op_copy->rsc_id = crm_strdup(op->rsc_id); if (op->app_name != NULL) { op_copy->app_name = crm_strdup(op->app_name); } if (op->output != NULL) { op_copy->output = crm_strdup(op->output); } return op_copy; } lrm_rsc_t * copy_lrm_rsc(const lrm_rsc_t * rsc) { lrm_rsc_t *rsc_copy = NULL; if (rsc == NULL) { return NULL; } crm_malloc0(rsc_copy, sizeof(lrm_rsc_t)); rsc_copy->id = crm_strdup(rsc->id); rsc_copy->type = crm_strdup(rsc->type); rsc_copy->class = NULL; rsc_copy->provider = NULL; if (rsc->class != NULL) { rsc_copy->class = crm_strdup(rsc->class); } if (rsc->provider != NULL) { rsc_copy->provider = crm_strdup(rsc->provider); } /* GHashTable* params; */ rsc_copy->params = NULL; rsc_copy->ops = NULL; return rsc_copy; } static void cib_rsc_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) { switch (rc) { case cib_ok: case cib_diff_failed: case cib_diff_resync: crm_trace("Resource update %d complete: rc=%d", call_id, rc); break; default: crm_warn("Resource update %d failed: (rc=%d) %s", call_id, rc, cib_error2string(rc)); } } static int do_update_resource(lrm_rsc_t * rsc, lrm_op_t * op) { /* */ int rc = cib_ok; xmlNode *update, *iter = NULL; int call_opt = cib_quorum_override; CRM_CHECK(op != NULL, return 0); if (fsa_state == S_ELECTION || fsa_state == S_PENDING) { crm_info("Sending update to local CIB in state: %s", fsa_state2string(fsa_state)); call_opt |= cib_scope_local; } iter = create_xml_node(iter, XML_CIB_TAG_STATUS); update = iter; iter = create_xml_node(iter, XML_CIB_TAG_STATE); set_uuid(iter, XML_ATTR_UUID, fsa_our_uname); crm_xml_add(iter, XML_ATTR_UNAME, fsa_our_uname); crm_xml_add(iter, XML_ATTR_ORIGIN, __FUNCTION__); iter = create_xml_node(iter, XML_CIB_TAG_LRM); crm_xml_add(iter, XML_ATTR_ID, fsa_our_uuid); iter = create_xml_node(iter, XML_LRM_TAG_RESOURCES); iter = create_xml_node(iter, XML_LRM_TAG_RESOURCE); crm_xml_add(iter, XML_ATTR_ID, op->rsc_id); build_operation_update(iter, rsc, op, __FUNCTION__); if (rsc) { crm_xml_add(iter, XML_ATTR_TYPE, rsc->type); crm_xml_add(iter, XML_AGENT_ATTR_CLASS, rsc->class); crm_xml_add(iter, XML_AGENT_ATTR_PROVIDER, rsc->provider); CRM_CHECK(rsc->type != NULL, crm_err("Resource %s has no value for type", op->rsc_id)); CRM_CHECK(rsc->class != NULL, crm_err("Resource %s has no value for class", op->rsc_id)); } else { crm_warn("Resource %s no longer exists in the lrmd", op->rsc_id); goto cleanup; } /* make it an asyncronous call and be done with it * * Best case: * the resource state will be discovered during * the next signup or election. * * Bad case: * we are shutting down and there is no DC at the time, * but then why were we shutting down then anyway? * (probably because of an internal error) * * Worst case: * we get shot for having resources "running" when the really weren't * * the alternative however means blocking here for too long, which * isnt acceptable */ fsa_cib_update(XML_CIB_TAG_STATUS, update, call_opt, rc, NULL); /* the return code is a call number, not an error code */ crm_trace("Sent resource state update message: %d", rc); fsa_cib_conn->cmds->register_callback(fsa_cib_conn, rc, 60, FALSE, NULL, "cib_rsc_callback", cib_rsc_callback); cleanup: free_xml(update); return rc; } void do_lrm_event(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data) { CRM_CHECK(FALSE, return); } gboolean process_lrm_event(lrm_op_t * op) { char *op_id = NULL; char *op_key = NULL; int update_id = 0; int log_level = LOG_ERR; gboolean removed = FALSE; lrm_rsc_t *rsc = NULL; struct recurring_op_s *pending = NULL; CRM_CHECK(op != NULL, return FALSE); CRM_CHECK(op->rsc_id != NULL, return FALSE); op_key = generate_op_key(op->rsc_id, op->op_type, op->interval); switch (op->op_status) { case LRM_OP_ERROR: case LRM_OP_PENDING: case LRM_OP_NOTSUPPORTED: break; case LRM_OP_CANCELLED: log_level = LOG_INFO; break; case LRM_OP_DONE: log_level = LOG_INFO; break; case LRM_OP_TIMEOUT: log_level = LOG_DEBUG_3; crm_err("LRM operation %s (%d) %s (timeout=%dms)", op_key, op->call_id, op_status2text(op->op_status), op->timeout); break; default: crm_err("Mapping unknown status (%d) to ERROR", op->op_status); op->op_status = LRM_OP_ERROR; } if (op->op_status == LRM_OP_ERROR && (op->rc == EXECRA_RUNNING_MASTER || op->rc == EXECRA_NOT_RUNNING)) { /* Leave it up to the TE/PE to decide if this is an error */ op->op_status = LRM_OP_DONE; log_level = LOG_INFO; } op_id = make_stop_id(op->rsc_id, op->call_id); pending = g_hash_table_lookup(pending_ops, op_id); rsc = fsa_lrm_conn->lrm_ops->get_rsc(fsa_lrm_conn, op->rsc_id); if (op->op_status != LRM_OP_CANCELLED) { if (safe_str_eq(op->op_type, RSC_NOTIFY)) { /* Keep notify ops out of the CIB */ send_direct_ack(NULL, NULL, NULL, op, op->rsc_id); } else { update_id = do_update_resource(rsc, op); } if (op->interval != 0) { goto out; } } else if (op->interval == 0) { /* This will occur when "crm resource cleanup" is called while actions are in-flight */ crm_err("Op %s (call=%d): Cancelled", op_key, op->call_id); send_direct_ack(NULL, NULL, NULL, op, op->rsc_id); } else if (pending == NULL) { crm_err("Op %s (call=%d): No 'pending' entry", op_key, op->call_id); } else if (op->user_data == NULL) { crm_err("Op %s (call=%d): No user data", op_key, op->call_id); } else if (pending->remove) { delete_op_entry(op, op->rsc_id, op_key, op->call_id); } else { /* Before a stop is called, no need to direct ack */ crm_trace("Op %s (call=%d): no delete event required", op_key, op->call_id); } if (g_hash_table_remove(pending_ops, op_id)) { removed = TRUE; crm_trace("Op %s (call=%d, stop-id=%s): Confirmed", op_key, op->call_id, op_id); } out: if (op->op_status == LRM_OP_DONE) { do_crm_log(log_level, "LRM operation %s (call=%d, rc=%d, cib-update=%d, confirmed=%s) %s", op_key, op->call_id, op->rc, update_id, removed ? "true" : "false", execra_code2string(op->rc)); } else { do_crm_log(log_level, "LRM operation %s (call=%d, status=%d, cib-update=%d, confirmed=%s) %s", op_key, op->call_id, op->op_status, update_id, removed ? "true" : "false", op_status2text(op->op_status)); } if (op->rc != 0 && op->output != NULL) { crm_info("Result: %s", op->output); } else if (op->output != NULL) { crm_debug("Result: %s", op->output); } #ifdef HAVE_LRM_OP_T_RSC_DELETED if (op->rsc_deleted) { crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key); delete_rsc_entry(NULL, op->rsc_id, NULL, HA_OK, NULL); } #endif /* If a shutdown was escalated while operations were pending, * then the FSA will be stalled right now... allow it to continue */ mainloop_set_trigger(fsa_source); update_history_cache(rsc, op); lrm_free_rsc(rsc); crm_free(op_key); crm_free(op_id); return TRUE; } diff --git a/crmd/utils.c b/crmd/utils.c index 5eb4d3cbee..a76133600f 100644 --- a/crmd/utils.c +++ b/crmd/utils.c @@ -1,1208 +1,1211 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include <../tools/attrd.h> /* A_DC_TIMER_STOP, A_DC_TIMER_START, * A_FINALIZE_TIMER_STOP, A_FINALIZE_TIMER_START * A_INTEGRATE_TIMER_STOP, A_INTEGRATE_TIMER_START */ void do_timer_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { gboolean timer_op_ok = TRUE; if (action & A_DC_TIMER_STOP) { timer_op_ok = crm_timer_stop(election_trigger); } else if (action & A_FINALIZE_TIMER_STOP) { timer_op_ok = crm_timer_stop(finalization_timer); } else if (action & A_INTEGRATE_TIMER_STOP) { timer_op_ok = crm_timer_stop(integration_timer); /* } else if(action & A_ELECTION_TIMEOUT_STOP) { */ /* timer_op_ok = crm_timer_stop(election_timeout); */ } /* dont start a timer that wasnt already running */ if (action & A_DC_TIMER_START && timer_op_ok) { crm_timer_start(election_trigger); if (AM_I_DC) { /* there can be only one */ register_fsa_input(cause, I_ELECTION, NULL); } } else if (action & A_FINALIZE_TIMER_START) { crm_timer_start(finalization_timer); } else if (action & A_INTEGRATE_TIMER_START) { crm_timer_start(integration_timer); /* } else if(action & A_ELECTION_TIMEOUT_START) { */ /* crm_timer_start(election_timeout); */ } } const char * get_timer_desc(fsa_timer_t * timer) { if (timer == election_trigger) { return "Election Trigger"; } else if (timer == election_timeout) { return "Election Timeout"; } else if (timer == shutdown_escalation_timer) { return "Shutdown Escalation"; } else if (timer == integration_timer) { return "Integration Timer"; } else if (timer == finalization_timer) { return "Finalization Timer"; } else if (timer == transition_timer) { return "New Transition Timer"; } else if (timer == wait_timer) { return "Wait Timer"; } else if (timer == recheck_timer) { return "PEngine Recheck Timer"; } return "Unknown Timer"; } gboolean crm_timer_popped(gpointer data) { fsa_timer_t *timer = (fsa_timer_t *) data; if (timer == wait_timer || timer == recheck_timer || timer == transition_timer || timer == finalization_timer || timer == election_trigger) { crm_info("%s (%s) just popped (%dms)", get_timer_desc(timer), fsa_input2string(timer->fsa_input), timer->period_ms); } else { crm_err("%s (%s) just popped in state %s! (%dms)", get_timer_desc(timer), fsa_input2string(timer->fsa_input), fsa_state2string(fsa_state), timer->period_ms); } if (timer->repeat == FALSE) { crm_timer_stop(timer); /* make it _not_ go off again */ } if (timer->fsa_input == I_INTEGRATED) { crm_info("Welcomed: %d, Integrated: %d", g_hash_table_size(welcomed_nodes), g_hash_table_size(integrated_nodes)); if (g_hash_table_size(welcomed_nodes) == 0) { /* If we don't even have ourself, start again */ register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, NULL, __FUNCTION__); } else { register_fsa_input_before(C_TIMER_POPPED, timer->fsa_input, NULL); } } else if (timer == recheck_timer && fsa_state != S_IDLE) { crm_debug("Discarding %s event in state: %s", fsa_input2string(timer->fsa_input), fsa_state2string(fsa_state)); } else if (timer == finalization_timer && fsa_state != S_FINALIZE_JOIN) { crm_debug("Discarding %s event in state: %s", fsa_input2string(timer->fsa_input), fsa_state2string(fsa_state)); } else if (timer->fsa_input != I_NULL) { register_fsa_input(C_TIMER_POPPED, timer->fsa_input, NULL); } crm_trace("Triggering FSA: %s", __FUNCTION__); mainloop_set_trigger(fsa_source); return TRUE; } gboolean crm_timer_start(fsa_timer_t * timer) { const char *timer_desc = get_timer_desc(timer); if (timer->source_id == 0 && timer->period_ms > 0) { timer->source_id = g_timeout_add(timer->period_ms, timer->callback, (void *)timer); CRM_ASSERT(timer->source_id != 0); crm_debug("Started %s (%s:%dms), src=%d", timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms, timer->source_id); } else if (timer->period_ms < 0) { crm_err("Tried to start %s (%s:%dms) with a -ve period", timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms); } else { crm_debug("%s (%s:%dms) already running: src=%d", timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms, timer->source_id); return FALSE; } return TRUE; } gboolean crm_timer_stop(fsa_timer_t * timer) { const char *timer_desc = get_timer_desc(timer); if (timer == NULL) { crm_err("Attempted to stop NULL timer"); return FALSE; } else if (timer->source_id != 0) { crm_trace("Stopping %s (%s:%dms), src=%d", timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms, timer->source_id); g_source_remove(timer->source_id); timer->source_id = 0; } else { crm_trace("%s (%s:%dms) already stopped", timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms); return FALSE; } return TRUE; } const char * fsa_input2string(enum crmd_fsa_input input) { const char *inputAsText = NULL; switch (input) { case I_NULL: inputAsText = "I_NULL"; break; case I_CIB_OP: inputAsText = "I_CIB_OP (unused)"; break; case I_CIB_UPDATE: inputAsText = "I_CIB_UPDATE"; break; case I_DC_TIMEOUT: inputAsText = "I_DC_TIMEOUT"; break; case I_ELECTION: inputAsText = "I_ELECTION"; break; case I_PE_CALC: inputAsText = "I_PE_CALC"; break; case I_RELEASE_DC: inputAsText = "I_RELEASE_DC"; break; case I_ELECTION_DC: inputAsText = "I_ELECTION_DC"; break; case I_ERROR: inputAsText = "I_ERROR"; break; case I_FAIL: inputAsText = "I_FAIL"; break; case I_INTEGRATED: inputAsText = "I_INTEGRATED"; break; case I_FINALIZED: inputAsText = "I_FINALIZED"; break; case I_NODE_JOIN: inputAsText = "I_NODE_JOIN"; break; case I_JOIN_OFFER: inputAsText = "I_JOIN_OFFER"; break; case I_JOIN_REQUEST: inputAsText = "I_JOIN_REQUEST"; break; case I_JOIN_RESULT: inputAsText = "I_JOIN_RESULT"; break; case I_NOT_DC: inputAsText = "I_NOT_DC"; break; case I_RECOVERED: inputAsText = "I_RECOVERED"; break; case I_RELEASE_FAIL: inputAsText = "I_RELEASE_FAIL"; break; case I_RELEASE_SUCCESS: inputAsText = "I_RELEASE_SUCCESS"; break; case I_RESTART: inputAsText = "I_RESTART"; break; case I_PE_SUCCESS: inputAsText = "I_PE_SUCCESS"; break; case I_ROUTER: inputAsText = "I_ROUTER"; break; case I_SHUTDOWN: inputAsText = "I_SHUTDOWN"; break; case I_STARTUP: inputAsText = "I_STARTUP"; break; case I_TE_SUCCESS: inputAsText = "I_TE_SUCCESS"; break; case I_STOP: inputAsText = "I_STOP"; break; case I_DC_HEARTBEAT: inputAsText = "I_DC_HEARTBEAT"; break; case I_WAIT_FOR_EVENT: inputAsText = "I_WAIT_FOR_EVENT"; break; case I_LRM_EVENT: inputAsText = "I_LRM_EVENT"; break; case I_PENDING: inputAsText = "I_PENDING"; break; case I_HALT: inputAsText = "I_HALT"; break; case I_TERMINATE: inputAsText = "I_TERMINATE"; break; case I_ILLEGAL: inputAsText = "I_ILLEGAL"; break; } if (inputAsText == NULL) { crm_err("Input %d is unknown", input); inputAsText = ""; } return inputAsText; } const char * fsa_state2string(enum crmd_fsa_state state) { const char *stateAsText = NULL; switch (state) { case S_IDLE: stateAsText = "S_IDLE"; break; case S_ELECTION: stateAsText = "S_ELECTION"; break; case S_INTEGRATION: stateAsText = "S_INTEGRATION"; break; case S_FINALIZE_JOIN: stateAsText = "S_FINALIZE_JOIN"; break; case S_NOT_DC: stateAsText = "S_NOT_DC"; break; case S_POLICY_ENGINE: stateAsText = "S_POLICY_ENGINE"; break; case S_RECOVERY: stateAsText = "S_RECOVERY"; break; case S_RELEASE_DC: stateAsText = "S_RELEASE_DC"; break; case S_PENDING: stateAsText = "S_PENDING"; break; case S_STOPPING: stateAsText = "S_STOPPING"; break; case S_TERMINATE: stateAsText = "S_TERMINATE"; break; case S_TRANSITION_ENGINE: stateAsText = "S_TRANSITION_ENGINE"; break; case S_STARTING: stateAsText = "S_STARTING"; break; case S_HALT: stateAsText = "S_HALT"; break; case S_ILLEGAL: stateAsText = "S_ILLEGAL"; break; } if (stateAsText == NULL) { crm_err("State %d is unknown", state); stateAsText = ""; } return stateAsText; } const char * fsa_cause2string(enum crmd_fsa_cause cause) { const char *causeAsText = NULL; switch (cause) { case C_UNKNOWN: causeAsText = "C_UNKNOWN"; break; case C_STARTUP: causeAsText = "C_STARTUP"; break; case C_IPC_MESSAGE: causeAsText = "C_IPC_MESSAGE"; break; case C_HA_MESSAGE: causeAsText = "C_HA_MESSAGE"; break; case C_CCM_CALLBACK: causeAsText = "C_CCM_CALLBACK"; break; case C_TIMER_POPPED: causeAsText = "C_TIMER_POPPED"; break; case C_SHUTDOWN: causeAsText = "C_SHUTDOWN"; break; case C_HEARTBEAT_FAILED: causeAsText = "C_HEARTBEAT_FAILED"; break; case C_SUBSYSTEM_CONNECT: causeAsText = "C_SUBSYSTEM_CONNECT"; break; case C_LRM_OP_CALLBACK: causeAsText = "C_LRM_OP_CALLBACK"; break; case C_LRM_MONITOR_CALLBACK: causeAsText = "C_LRM_MONITOR_CALLBACK"; break; case C_CRMD_STATUS_CALLBACK: causeAsText = "C_CRMD_STATUS_CALLBACK"; break; case C_HA_DISCONNECT: causeAsText = "C_HA_DISCONNECT"; break; case C_FSA_INTERNAL: causeAsText = "C_FSA_INTERNAL"; break; case C_ILLEGAL: causeAsText = "C_ILLEGAL"; break; } if (causeAsText == NULL) { crm_err("Cause %d is unknown", cause); causeAsText = ""; } return causeAsText; } const char * fsa_action2string(long long action) { const char *actionAsText = NULL; switch (action) { case A_NOTHING: actionAsText = "A_NOTHING"; break; case A_ELECTION_START: actionAsText = "A_ELECTION_START"; break; case A_DC_JOIN_FINAL: actionAsText = "A_DC_JOIN_FINAL"; break; case A_READCONFIG: actionAsText = "A_READCONFIG"; break; case O_RELEASE: actionAsText = "O_RELEASE"; break; case A_STARTUP: actionAsText = "A_STARTUP"; break; case A_STARTED: actionAsText = "A_STARTED"; break; case A_HA_CONNECT: actionAsText = "A_HA_CONNECT"; break; case A_HA_DISCONNECT: actionAsText = "A_HA_DISCONNECT"; break; case A_LRM_CONNECT: actionAsText = "A_LRM_CONNECT"; break; case A_LRM_EVENT: actionAsText = "A_LRM_EVENT"; break; case A_LRM_INVOKE: actionAsText = "A_LRM_INVOKE"; break; case A_LRM_DISCONNECT: actionAsText = "A_LRM_DISCONNECT"; break; + case O_LRM_RECONNECT: + actionAsText = "O_LRM_RECONNECT"; + break; case A_CL_JOIN_QUERY: actionAsText = "A_CL_JOIN_QUERY"; break; case A_DC_TIMER_STOP: actionAsText = "A_DC_TIMER_STOP"; break; case A_DC_TIMER_START: actionAsText = "A_DC_TIMER_START"; break; case A_INTEGRATE_TIMER_START: actionAsText = "A_INTEGRATE_TIMER_START"; break; case A_INTEGRATE_TIMER_STOP: actionAsText = "A_INTEGRATE_TIMER_STOP"; break; case A_FINALIZE_TIMER_START: actionAsText = "A_FINALIZE_TIMER_START"; break; case A_FINALIZE_TIMER_STOP: actionAsText = "A_FINALIZE_TIMER_STOP"; break; case A_ELECTION_COUNT: actionAsText = "A_ELECTION_COUNT"; break; case A_ELECTION_VOTE: actionAsText = "A_ELECTION_VOTE"; break; case A_ELECTION_CHECK: actionAsText = "A_ELECTION_CHECK"; break; case A_CL_JOIN_ANNOUNCE: actionAsText = "A_CL_JOIN_ANNOUNCE"; break; case A_CL_JOIN_REQUEST: actionAsText = "A_CL_JOIN_REQUEST"; break; case A_CL_JOIN_RESULT: actionAsText = "A_CL_JOIN_RESULT"; break; case A_DC_JOIN_OFFER_ALL: actionAsText = "A_DC_JOIN_OFFER_ALL"; break; case A_DC_JOIN_OFFER_ONE: actionAsText = "A_DC_JOIN_OFFER_ONE"; break; case A_DC_JOIN_PROCESS_REQ: actionAsText = "A_DC_JOIN_PROCESS_REQ"; break; case A_DC_JOIN_PROCESS_ACK: actionAsText = "A_DC_JOIN_PROCESS_ACK"; break; case A_DC_JOIN_FINALIZE: actionAsText = "A_DC_JOIN_FINALIZE"; break; case A_MSG_PROCESS: actionAsText = "A_MSG_PROCESS"; break; case A_MSG_ROUTE: actionAsText = "A_MSG_ROUTE"; break; case A_RECOVER: actionAsText = "A_RECOVER"; break; case A_DC_RELEASE: actionAsText = "A_DC_RELEASE"; break; case A_DC_RELEASED: actionAsText = "A_DC_RELEASED"; break; case A_DC_TAKEOVER: actionAsText = "A_DC_TAKEOVER"; break; case A_SHUTDOWN: actionAsText = "A_SHUTDOWN"; break; case A_SHUTDOWN_REQ: actionAsText = "A_SHUTDOWN_REQ"; break; case A_STOP: actionAsText = "A_STOP "; break; case A_EXIT_0: actionAsText = "A_EXIT_0"; break; case A_EXIT_1: actionAsText = "A_EXIT_1"; break; case A_CCM_CONNECT: actionAsText = "A_CCM_CONNECT"; break; case A_CCM_DISCONNECT: actionAsText = "A_CCM_DISCONNECT"; break; case O_CIB_RESTART: actionAsText = "O_CIB_RESTART"; break; case A_CIB_START: actionAsText = "A_CIB_START"; break; case A_CIB_STOP: actionAsText = "A_CIB_STOP"; break; case A_TE_INVOKE: actionAsText = "A_TE_INVOKE"; break; case O_TE_RESTART: actionAsText = "O_TE_RESTART"; break; case A_TE_START: actionAsText = "A_TE_START"; break; case A_TE_STOP: actionAsText = "A_TE_STOP"; break; case A_TE_HALT: actionAsText = "A_TE_HALT"; break; case A_TE_CANCEL: actionAsText = "A_TE_CANCEL"; break; case A_PE_INVOKE: actionAsText = "A_PE_INVOKE"; break; case O_PE_RESTART: actionAsText = "O_PE_RESTART"; break; case A_PE_START: actionAsText = "A_PE_START"; break; case A_PE_STOP: actionAsText = "A_PE_STOP"; break; case A_NODE_BLOCK: actionAsText = "A_NODE_BLOCK"; break; case A_UPDATE_NODESTATUS: actionAsText = "A_UPDATE_NODESTATUS"; break; case A_LOG: actionAsText = "A_LOG "; break; case A_ERROR: actionAsText = "A_ERROR "; break; case A_WARN: actionAsText = "A_WARN "; break; /* Composite actions */ case A_DC_TIMER_START | A_CL_JOIN_QUERY: actionAsText = "A_DC_TIMER_START|A_CL_JOIN_QUERY"; break; } if (actionAsText == NULL) { crm_err("Action %.16llx is unknown", action); actionAsText = ""; } return actionAsText; } void fsa_dump_inputs(int log_level, const char *text, long long input_register) { if (input_register == A_NOTHING) { return; } if (text == NULL) { text = "Input register contents:"; } if (is_set(input_register, R_THE_DC)) { crm_trace( "%s %.16llx (R_THE_DC)", text, R_THE_DC); } if (is_set(input_register, R_STARTING)) { crm_trace( "%s %.16llx (R_STARTING)", text, R_STARTING); } if (is_set(input_register, R_SHUTDOWN)) { crm_trace( "%s %.16llx (R_SHUTDOWN)", text, R_SHUTDOWN); } if (is_set(input_register, R_STAYDOWN)) { crm_trace( "%s %.16llx (R_STAYDOWN)", text, R_STAYDOWN); } if (is_set(input_register, R_JOIN_OK)) { crm_trace( "%s %.16llx (R_JOIN_OK)", text, R_JOIN_OK); } if (is_set(input_register, R_READ_CONFIG)) { crm_trace( "%s %.16llx (R_READ_CONFIG)", text, R_READ_CONFIG); } if (is_set(input_register, R_INVOKE_PE)) { crm_trace( "%s %.16llx (R_INVOKE_PE)", text, R_INVOKE_PE); } if (is_set(input_register, R_CIB_CONNECTED)) { crm_trace( "%s %.16llx (R_CIB_CONNECTED)", text, R_CIB_CONNECTED); } if (is_set(input_register, R_PE_CONNECTED)) { crm_trace( "%s %.16llx (R_PE_CONNECTED)", text, R_PE_CONNECTED); } if (is_set(input_register, R_TE_CONNECTED)) { crm_trace( "%s %.16llx (R_TE_CONNECTED)", text, R_TE_CONNECTED); } if (is_set(input_register, R_LRM_CONNECTED)) { crm_trace( "%s %.16llx (R_LRM_CONNECTED)", text, R_LRM_CONNECTED); } if (is_set(input_register, R_CIB_REQUIRED)) { crm_trace( "%s %.16llx (R_CIB_REQUIRED)", text, R_CIB_REQUIRED); } if (is_set(input_register, R_PE_REQUIRED)) { crm_trace( "%s %.16llx (R_PE_REQUIRED)", text, R_PE_REQUIRED); } if (is_set(input_register, R_TE_REQUIRED)) { crm_trace( "%s %.16llx (R_TE_REQUIRED)", text, R_TE_REQUIRED); } if (is_set(input_register, R_REQ_PEND)) { crm_trace( "%s %.16llx (R_REQ_PEND)", text, R_REQ_PEND); } if (is_set(input_register, R_PE_PEND)) { crm_trace( "%s %.16llx (R_PE_PEND)", text, R_PE_PEND); } if (is_set(input_register, R_TE_PEND)) { crm_trace( "%s %.16llx (R_TE_PEND)", text, R_TE_PEND); } if (is_set(input_register, R_RESP_PEND)) { crm_trace( "%s %.16llx (R_RESP_PEND)", text, R_RESP_PEND); } if (is_set(input_register, R_CIB_DONE)) { crm_trace( "%s %.16llx (R_CIB_DONE)", text, R_CIB_DONE); } if (is_set(input_register, R_HAVE_CIB)) { crm_trace( "%s %.16llx (R_HAVE_CIB)", text, R_HAVE_CIB); } if (is_set(input_register, R_CIB_ASKED)) { crm_trace( "%s %.16llx (R_CIB_ASKED)", text, R_CIB_ASKED); } if (is_set(input_register, R_CCM_DATA)) { crm_trace( "%s %.16llx (R_CCM_DATA)", text, R_CCM_DATA); } if (is_set(input_register, R_PEER_DATA)) { crm_trace( "%s %.16llx (R_PEER_DATA)", text, R_PEER_DATA); } if (is_set(input_register, R_IN_RECOVERY)) { crm_trace( "%s %.16llx (R_IN_RECOVERY)", text, R_IN_RECOVERY); } } void fsa_dump_actions(long long action, const char *text) { if (is_set(action, A_READCONFIG)) { crm_trace( "Action %.16llx (A_READCONFIG) %s", A_READCONFIG, text); } if (is_set(action, A_STARTUP)) { crm_trace( "Action %.16llx (A_STARTUP) %s", A_STARTUP, text); } if (is_set(action, A_STARTED)) { crm_trace( "Action %.16llx (A_STARTED) %s", A_STARTED, text); } if (is_set(action, A_HA_CONNECT)) { crm_trace( "Action %.16llx (A_CONNECT) %s", A_HA_CONNECT, text); } if (is_set(action, A_HA_DISCONNECT)) { crm_trace( "Action %.16llx (A_DISCONNECT) %s", A_HA_DISCONNECT, text); } if (is_set(action, A_LRM_CONNECT)) { crm_trace( "Action %.16llx (A_LRM_CONNECT) %s", A_LRM_CONNECT, text); } if (is_set(action, A_LRM_EVENT)) { crm_trace( "Action %.16llx (A_LRM_EVENT) %s", A_LRM_EVENT, text); } if (is_set(action, A_LRM_INVOKE)) { crm_trace( "Action %.16llx (A_LRM_INVOKE) %s", A_LRM_INVOKE, text); } if (is_set(action, A_LRM_DISCONNECT)) { crm_trace( "Action %.16llx (A_LRM_DISCONNECT) %s", A_LRM_DISCONNECT, text); } if (is_set(action, A_DC_TIMER_STOP)) { crm_trace( "Action %.16llx (A_DC_TIMER_STOP) %s", A_DC_TIMER_STOP, text); } if (is_set(action, A_DC_TIMER_START)) { crm_trace( "Action %.16llx (A_DC_TIMER_START) %s", A_DC_TIMER_START, text); } if (is_set(action, A_INTEGRATE_TIMER_START)) { crm_trace( "Action %.16llx (A_INTEGRATE_TIMER_START) %s", A_INTEGRATE_TIMER_START, text); } if (is_set(action, A_INTEGRATE_TIMER_STOP)) { crm_trace( "Action %.16llx (A_INTEGRATE_TIMER_STOP) %s", A_INTEGRATE_TIMER_STOP, text); } if (is_set(action, A_FINALIZE_TIMER_START)) { crm_trace( "Action %.16llx (A_FINALIZE_TIMER_START) %s", A_FINALIZE_TIMER_START, text); } if (is_set(action, A_FINALIZE_TIMER_STOP)) { crm_trace( "Action %.16llx (A_FINALIZE_TIMER_STOP) %s", A_FINALIZE_TIMER_STOP, text); } if (is_set(action, A_ELECTION_COUNT)) { crm_trace( "Action %.16llx (A_ELECTION_COUNT) %s", A_ELECTION_COUNT, text); } if (is_set(action, A_ELECTION_VOTE)) { crm_trace( "Action %.16llx (A_ELECTION_VOTE) %s", A_ELECTION_VOTE, text); } if (is_set(action, A_ELECTION_CHECK)) { crm_trace( "Action %.16llx (A_ELECTION_CHECK) %s", A_ELECTION_CHECK, text); } if (is_set(action, A_CL_JOIN_ANNOUNCE)) { crm_trace( "Action %.16llx (A_CL_JOIN_ANNOUNCE) %s", A_CL_JOIN_ANNOUNCE, text); } if (is_set(action, A_CL_JOIN_REQUEST)) { crm_trace( "Action %.16llx (A_CL_JOIN_REQUEST) %s", A_CL_JOIN_REQUEST, text); } if (is_set(action, A_CL_JOIN_RESULT)) { crm_trace( "Action %.16llx (A_CL_JOIN_RESULT) %s", A_CL_JOIN_RESULT, text); } if (is_set(action, A_DC_JOIN_OFFER_ALL)) { crm_trace( "Action %.16llx (A_DC_JOIN_OFFER_ALL) %s", A_DC_JOIN_OFFER_ALL, text); } if (is_set(action, A_DC_JOIN_OFFER_ONE)) { crm_trace( "Action %.16llx (A_DC_JOIN_OFFER_ONE) %s", A_DC_JOIN_OFFER_ONE, text); } if (is_set(action, A_DC_JOIN_PROCESS_REQ)) { crm_trace( "Action %.16llx (A_DC_JOIN_PROCESS_REQ) %s", A_DC_JOIN_PROCESS_REQ, text); } if (is_set(action, A_DC_JOIN_PROCESS_ACK)) { crm_trace( "Action %.16llx (A_DC_JOIN_PROCESS_ACK) %s", A_DC_JOIN_PROCESS_ACK, text); } if (is_set(action, A_DC_JOIN_FINALIZE)) { crm_trace( "Action %.16llx (A_DC_JOIN_FINALIZE) %s", A_DC_JOIN_FINALIZE, text); } if (is_set(action, A_MSG_PROCESS)) { crm_trace( "Action %.16llx (A_MSG_PROCESS) %s", A_MSG_PROCESS, text); } if (is_set(action, A_MSG_ROUTE)) { crm_trace( "Action %.16llx (A_MSG_ROUTE) %s", A_MSG_ROUTE, text); } if (is_set(action, A_RECOVER)) { crm_trace( "Action %.16llx (A_RECOVER) %s", A_RECOVER, text); } if (is_set(action, A_DC_RELEASE)) { crm_trace( "Action %.16llx (A_DC_RELEASE) %s", A_DC_RELEASE, text); } if (is_set(action, A_DC_RELEASED)) { crm_trace( "Action %.16llx (A_DC_RELEASED) %s", A_DC_RELEASED, text); } if (is_set(action, A_DC_TAKEOVER)) { crm_trace( "Action %.16llx (A_DC_TAKEOVER) %s", A_DC_TAKEOVER, text); } if (is_set(action, A_SHUTDOWN)) { crm_trace( "Action %.16llx (A_SHUTDOWN) %s", A_SHUTDOWN, text); } if (is_set(action, A_SHUTDOWN_REQ)) { crm_trace( "Action %.16llx (A_SHUTDOWN_REQ) %s", A_SHUTDOWN_REQ, text); } if (is_set(action, A_STOP)) { crm_trace( "Action %.16llx (A_STOP ) %s", A_STOP, text); } if (is_set(action, A_EXIT_0)) { crm_trace( "Action %.16llx (A_EXIT_0) %s", A_EXIT_0, text); } if (is_set(action, A_EXIT_1)) { crm_trace( "Action %.16llx (A_EXIT_1) %s", A_EXIT_1, text); } if (is_set(action, A_CCM_CONNECT)) { crm_trace( "Action %.16llx (A_CCM_CONNECT) %s", A_CCM_CONNECT, text); } if (is_set(action, A_CCM_DISCONNECT)) { crm_trace( "Action %.16llx (A_CCM_DISCONNECT) %s", A_CCM_DISCONNECT, text); } if (is_set(action, A_CIB_START)) { crm_trace( "Action %.16llx (A_CIB_START) %s", A_CIB_START, text); } if (is_set(action, A_CIB_STOP)) { crm_trace( "Action %.16llx (A_CIB_STOP) %s", A_CIB_STOP, text); } if (is_set(action, A_TE_INVOKE)) { crm_trace( "Action %.16llx (A_TE_INVOKE) %s", A_TE_INVOKE, text); } if (is_set(action, A_TE_START)) { crm_trace( "Action %.16llx (A_TE_START) %s", A_TE_START, text); } if (is_set(action, A_TE_STOP)) { crm_trace( "Action %.16llx (A_TE_STOP) %s", A_TE_STOP, text); } if (is_set(action, A_TE_CANCEL)) { crm_trace( "Action %.16llx (A_TE_CANCEL) %s", A_TE_CANCEL, text); } if (is_set(action, A_PE_INVOKE)) { crm_trace( "Action %.16llx (A_PE_INVOKE) %s", A_PE_INVOKE, text); } if (is_set(action, A_PE_START)) { crm_trace( "Action %.16llx (A_PE_START) %s", A_PE_START, text); } if (is_set(action, A_PE_STOP)) { crm_trace( "Action %.16llx (A_PE_STOP) %s", A_PE_STOP, text); } if (is_set(action, A_NODE_BLOCK)) { crm_trace( "Action %.16llx (A_NODE_BLOCK) %s", A_NODE_BLOCK, text); } if (is_set(action, A_UPDATE_NODESTATUS)) { crm_trace( "Action %.16llx (A_UPDATE_NODESTATUS) %s", A_UPDATE_NODESTATUS, text); } if (is_set(action, A_LOG)) { crm_trace( "Action %.16llx (A_LOG ) %s", A_LOG, text); } if (is_set(action, A_ERROR)) { crm_trace( "Action %.16llx (A_ERROR ) %s", A_ERROR, text); } if (is_set(action, A_WARN)) { crm_trace( "Action %.16llx (A_WARN ) %s", A_WARN, text); } } void create_node_entry(const char *uuid, const char *uname, const char *type) { /* make sure a node entry exists for the new node * * this will add anyone except the first ever node in the cluster * since it will also be the DC which doesnt go through the * join process (with itself). We can include a special case * later if desired. */ xmlNode *tmp1 = create_xml_node(NULL, XML_CIB_TAG_NODE); crm_trace("Creating node entry for %s", uname); set_uuid(tmp1, XML_ATTR_UUID, uname); crm_xml_add(tmp1, XML_ATTR_UNAME, uname); crm_xml_add(tmp1, XML_ATTR_TYPE, type); fsa_cib_anon_update(XML_CIB_TAG_NODES, tmp1, cib_scope_local | cib_quorum_override | cib_can_create); free_xml(tmp1); } xmlNode * create_node_state(const char *uname, const char *ha_state, const char *ccm_state, const char *crmd_state, const char *join_state, const char *exp_state, gboolean clear_shutdown, const char *src) { xmlNode *node_state = create_xml_node(NULL, XML_CIB_TAG_STATE); crm_trace("%s Creating node state entry for %s", src, uname); set_uuid(node_state, XML_ATTR_UUID, uname); if (crm_element_value(node_state, XML_ATTR_UUID) == NULL) { crm_debug("Node %s is not a cluster member", uname); free_xml(node_state); return NULL; } crm_xml_add(node_state, XML_ATTR_UNAME, uname); crm_xml_add(node_state, XML_CIB_ATTR_HASTATE, ha_state); crm_xml_add(node_state, XML_CIB_ATTR_INCCM, ccm_state); crm_xml_add(node_state, XML_CIB_ATTR_CRMDSTATE, crmd_state); crm_xml_add(node_state, XML_CIB_ATTR_JOINSTATE, join_state); crm_xml_add(node_state, XML_CIB_ATTR_EXPSTATE, exp_state); crm_xml_add(node_state, XML_ATTR_ORIGIN, src); if (clear_shutdown) { crm_xml_add(node_state, XML_CIB_ATTR_SHUTDOWN, "0"); } crm_log_xml_trace(node_state, "created"); return node_state; } extern GHashTable *ipc_clients; void process_client_disconnect(crmd_client_t * curr_client) { struct crm_subsystem_s *the_subsystem = NULL; CRM_CHECK(curr_client != NULL, return); crm_trace("received HUP from %s", curr_client->table_key); if (curr_client->sub_sys == NULL) { crm_trace("Client hadn't registered with us yet"); } else if (strcasecmp(CRM_SYSTEM_PENGINE, curr_client->sub_sys) == 0) { the_subsystem = pe_subsystem; } else if (strcasecmp(CRM_SYSTEM_TENGINE, curr_client->sub_sys) == 0) { the_subsystem = te_subsystem; } else if (strcasecmp(CRM_SYSTEM_CIB, curr_client->sub_sys) == 0) { the_subsystem = cib_subsystem; } if (the_subsystem != NULL) { the_subsystem->ipc = NULL; the_subsystem->client = NULL; crm_info("Received HUP from %s:[%d]", the_subsystem->name, the_subsystem->pid); } else { /* else that was a transient client */ crm_trace("Received HUP from transient client"); } if (curr_client->table_key != NULL) { /* * Key is destroyed below as: * curr_client->table_key * Value is cleaned up by: * crmd_ipc_connection_destroy * which will also call: * G_main_del_IPC_Channel */ g_hash_table_remove(ipc_clients, curr_client->table_key); } } gboolean update_dc(xmlNode * msg) { char *last_dc = fsa_our_dc; const char *dc_version = NULL; const char *welcome_from = NULL; if (msg != NULL) { gboolean invalid = FALSE; dc_version = crm_element_value(msg, F_CRM_VERSION); welcome_from = crm_element_value(msg, F_CRM_HOST_FROM); CRM_CHECK(dc_version != NULL, return FALSE); CRM_CHECK(welcome_from != NULL, return FALSE); if (AM_I_DC && safe_str_neq(welcome_from, fsa_our_uname)) { invalid = TRUE; } else if (fsa_our_dc && safe_str_neq(welcome_from, fsa_our_dc)) { invalid = TRUE; } if (invalid) { CRM_CHECK(fsa_our_dc != NULL, crm_err("We have no DC")); if (AM_I_DC) { crm_err("Not updating DC to %s (%s): we are also a DC", welcome_from, dc_version); } else { crm_warn("New DC %s is not %s", welcome_from, fsa_our_dc); } register_fsa_action(A_CL_JOIN_QUERY | A_DC_TIMER_START); return FALSE; } } crm_free(fsa_our_dc_version); fsa_our_dc_version = NULL; fsa_our_dc = NULL; /* Free'd as last_dc */ if (welcome_from != NULL) { fsa_our_dc = crm_strdup(welcome_from); } if (dc_version != NULL) { fsa_our_dc_version = crm_strdup(dc_version); } if (safe_str_eq(fsa_our_dc, last_dc)) { /* do nothing */ } else if (fsa_our_dc != NULL) { crm_info("Set DC to %s (%s)", crm_str(fsa_our_dc), crm_str(fsa_our_dc_version)); } else if (last_dc != NULL) { crm_debug("Unset DC. Was %s", crm_str(last_dc)); } crm_free(last_dc); return TRUE; } #define STATUS_PATH_MAX 512 static void erase_xpath_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) { char *xpath = user_data; do_crm_log(rc == 0 ? LOG_DEBUG : LOG_NOTICE, "Deletion of \"%s\": %s (rc=%d)", xpath, cib_error2string(rc), rc); crm_free(xpath); } void erase_status_tag(const char *uname, const char *tag, int options) { int rc = cib_ok; char xpath[STATUS_PATH_MAX]; int cib_opts = cib_quorum_override | cib_xpath | options; if (fsa_cib_conn && uname) { snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", uname, tag); crm_info("Deleting xpath: %s", xpath); rc = fsa_cib_conn->cmds->delete(fsa_cib_conn, xpath, NULL, cib_opts); add_cib_op_callback(fsa_cib_conn, rc, FALSE, crm_strdup(xpath), erase_xpath_callback); } } static IPC_Channel *attrd = NULL; static gboolean attrd_dispatch(IPC_Channel * client, gpointer user_data) { xmlNode *msg = NULL; gboolean stay_connected = TRUE; while (IPC_ISRCONN(client)) { if (client->ops->is_message_pending(client) == 0) { break; } msg = xmlfromIPC(client, MAX_IPC_DELAY); if (msg != NULL) { crm_log_xml_err(msg, "attrd:msg"); free_xml(msg); } } if (client->ch_status != IPC_CONNECT) { stay_connected = FALSE; } return stay_connected; } static void attrd_connection_destroy(gpointer user_data) { crm_err("Lost connection to attrd"); attrd = NULL; return; } void update_attrd(const char *host, const char *name, const char *value, const char *user_name) { int retries = 0; gboolean rc = FALSE; do { rc = FALSE; if (attrd == NULL) { crm_info("Connecting to attrd..."); attrd = init_client_ipc_comms_nodispatch(T_ATTRD); if (attrd) { G_main_add_IPC_Channel(G_PRIORITY_LOW, attrd, FALSE, attrd_dispatch, NULL, attrd_connection_destroy); } } if (attrd != NULL) { rc = attrd_update_delegate(attrd, 'U', host, name, value, XML_CIB_TAG_STATUS, NULL, NULL, user_name); } else { crm_warn("Could not connect to %s", T_ATTRD); } if (rc == FALSE) { attrd = NULL; if (retries < 5) { retries++; sleep(retries); } } } while(rc == FALSE && retries < 5); if (rc == FALSE) { crm_err("Could not send %s %s %s (%d)", T_ATTRD, name ? "update" : "refresh", name?name:"", is_set(fsa_input_register, R_SHUTDOWN)); if(is_set(fsa_input_register, R_SHUTDOWN)) { register_fsa_input(C_FSA_INTERNAL, I_FAIL, NULL); } } else if(retries) { crm_debug("Needed %d retries to send %s %s %s", retries, T_ATTRD, name ? "update" : "refresh", name?name:""); } }