diff --git a/crm/crmd/callbacks.c b/crm/crmd/callbacks.c index f9e5af2927..533f86f810 100644 --- a/crm/crmd/callbacks.c +++ b/crm/crmd/callbacks.c @@ -1,614 +1,617 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include GHashTable *crmd_peer_state = NULL; crm_data_t *find_xml_in_hamessage(const HA_Message * msg); void crmd_ha_connection_destroy(gpointer user_data); /* From join_dc... */ extern gboolean check_join_state( enum crmd_fsa_state cur_state, const char *source); /* #define MAX_EMPTY_CALLBACKS 20 */ /* int empty_callbacks = 0; */ gboolean crmd_ha_msg_dispatch(IPC_Channel *channel, gpointer user_data) { int lpc = 0; ll_cluster_t *hb_cluster = (ll_cluster_t*)user_data; while(lpc < 2 && hb_cluster->llc_ops->msgready(hb_cluster)) { if(channel->ch_status != IPC_CONNECT) { /* there really is no point continuing */ break; } lpc++; /* invoke the callbacks but dont block */ hb_cluster->llc_ops->rcvmsg(hb_cluster, 0); } crm_devel("%d HA messages dispatched", lpc); G_main_set_trigger(fsa_source); if (channel && (channel->ch_status != IPC_CONNECT)) { crm_crit("Lost connection to heartbeat service."); return FALSE; } return TRUE; } void crmd_ha_msg_callback(const HA_Message * msg, void* private_data) { ha_msg_input_t *new_input = NULL; oc_node_t *from_node = NULL; const char *from = ha_msg_value(msg, F_ORIG); const char *seq = ha_msg_value(msg, F_SEQ); const char *op = ha_msg_value(msg, F_CRM_TASK); const char *sys_to = ha_msg_value(msg, F_CRM_SYS_TO); const char *sys_from = ha_msg_value(msg, F_CRM_SYS_FROM); CRM_DEV_ASSERT(from != NULL); if(fsa_membership_copy == NULL) { crm_debug("Ignoring HA messages until we are" " connected to the CCM (%s op from %s)", op, from); crm_log_message_adv( LOG_MSG, "HA[inbound]: Ignore (No CCM)", msg); return; } from_node = g_hash_table_lookup(fsa_membership_copy->members, from); if(from_node == NULL) { int level = LOG_DEBUG; if(safe_str_eq(op, CRM_OP_VOTE)) { level = LOG_WARNING; } else if(AM_I_DC && safe_str_eq(op, CRM_OP_JOIN_ANNOUNCE)) { level = LOG_WARNING; } else if(safe_str_eq(sys_from, CRM_SYSTEM_DC)) { level = LOG_WARNING; } do_crm_log(level, __FILE__, __FUNCTION__, "Ignoring HA message (op=%s) from %s: not in our" " membership list (size=%d)", op, from, g_hash_table_size(fsa_membership_copy->members)); crm_log_message_adv(LOG_MSG, "HA[inbound]: CCM Discard", msg); } else if(AM_I_DC && safe_str_eq(sys_from, CRM_SYSTEM_DC) && safe_str_neq(from, fsa_our_uname)) { crm_err("Another DC detected: %s (op=%s)", from, op); crm_log_message_adv( LOG_WARNING, "HA[inbound]: Duplicate DC", msg); new_input = new_ha_msg_input(msg); /* make sure the election happens NOW */ register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, new_input, __FUNCTION__); #if 0 /* still thinking about this one... * could create a timing issue if we dont notice the * election before a new DC is elected. */ } else if(fsa_our_dc != NULL && safe_str_eq(sys_from, CRM_SYSTEM_DC) && safe_str_neq(from, fsa_our_dc)) { crm_warn("Ignoring message from wrong DC: %s vs. %s ", from, fsa_our_dc); crm_log_message_adv(LOG_WARNING, "HA[inbound]: wrong DC", msg); #endif } else if(safe_str_eq(sys_to, CRM_SYSTEM_DC) && AM_I_DC == FALSE) { crm_verbose("Ignoring message for the DC [F_SEQ=%s]", seq); crm_log_message_adv(LOG_TRACE, "HA[inbound]: ignore", msg); return; } else if(safe_str_eq(from, fsa_our_uname) && safe_str_eq(op, CRM_OP_VOTE)) { crm_log_message_adv(LOG_TRACE, "HA[inbound]", msg); crm_verbose("Ignoring our own vote [F_SEQ=%s]: own vote", seq); return; } else if(AM_I_DC && safe_str_eq(op, CRM_OP_HBEAT)) { crm_verbose("Ignoring our own heartbeat [F_SEQ=%s]", seq); crm_log_message_adv(LOG_TRACE, "HA[inbound]: own heartbeat", msg); return; } else { crm_devel("Processing message"); crm_log_message_adv(LOG_MSG, "HA[inbound]", msg); new_input = new_ha_msg_input(msg); register_fsa_input(C_HA_MESSAGE, I_ROUTER, new_input); } #if 0 if(ha_msg_value(msg, XML_ATTR_REFERENCE) == NULL) { ha_msg_add(new_input->msg, XML_ATTR_REFERENCE, seq); } #endif delete_ha_msg_input(new_input); return; } /* * Apparently returning TRUE means "stay connected, keep doing stuff". * Returning FALSE means "we're all done, close the connection" */ gboolean crmd_ipc_msg_callback(IPC_Channel *client, gpointer user_data) { int lpc = 0; IPC_Message *msg = NULL; ha_msg_input_t *new_input = NULL; crmd_client_t *curr_client = (crmd_client_t*)user_data; gboolean stay_connected = TRUE; crm_verbose("Processing IPC message from %s", curr_client->table_key); while(lpc == 0 && client->ops->is_message_pending(client)) { if (client->ch_status != IPC_CONNECT) { /* The message which was pending for us is that * the IPC status is now IPC_DISCONNECT */ break; } if (client->ops->recv(client, &msg) != IPC_OK) { perror("Receive failure:"); crm_err("[%s] [receive failure]", curr_client->table_key); stay_connected = FALSE; break; } else if (msg == NULL) { crm_err("No message from %s this time", curr_client->table_key); continue; } lpc++; new_input = new_ipc_msg_input(msg); msg->msg_done(msg); crm_verbose("Processing msg from %s", curr_client->table_key); crm_log_message_adv(LOG_MSG, "CRMd[inbound]", new_input->msg); crmd_authorize_message(new_input, curr_client); delete_ha_msg_input(new_input); msg = NULL; new_input = NULL; } crm_verbose("Processed %d messages", lpc); if (client->ch_status != IPC_CONNECT) { stay_connected = FALSE; crm_verbose("received HUP from %s", curr_client->table_key); if (curr_client != NULL) { struct crm_subsystem_s *the_subsystem = NULL; if (curr_client->sub_sys == NULL) { crm_debug("Client hadn't registered with us yet"); } else if (strcmp(CRM_SYSTEM_PENGINE, curr_client->sub_sys) == 0) { the_subsystem = pe_subsystem; } else if (strcmp(CRM_SYSTEM_TENGINE, curr_client->sub_sys) == 0) { the_subsystem = te_subsystem; } else if (strcmp(CRM_SYSTEM_CIB, curr_client->sub_sys) == 0){ the_subsystem = cib_subsystem; } if(the_subsystem != NULL) { cleanup_subsystem(the_subsystem); } /* else that was a transient client */ if (curr_client->table_key != NULL) { /* * Key is destroyed below: * curr_client->table_key * Value is cleaned up by: * G_main_del_IPC_Channel */ g_hash_table_remove( ipc_clients, curr_client->table_key); } #if 0 if(curr_client->client_source != NULL) { gboolean det = G_main_del_IPC_Channel( curr_client->client_source); crm_verbose("crm_client was %s detached", det?"successfully":"not"); } #endif crm_free(curr_client->table_key); crm_free(curr_client->sub_sys); crm_free(curr_client->uuid); crm_free(curr_client); } } G_main_set_trigger(fsa_source); return stay_connected; } gboolean lrm_dispatch(IPC_Channel*src_not_used, gpointer user_data) { int num_msgs = 0; ll_lrm_t *lrm = (ll_lrm_t*)user_data; crm_devel("received callback"); num_msgs = lrm->lrm_ops->rcvmsg(lrm, FALSE); if(num_msgs < 1) { crm_err("lrm->lrm_ops->rcvmsg() failed, connection lost?"); clear_bit_inplace(fsa_input_register, R_LRM_CONNECTED); register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL); return FALSE; } return TRUE; } void lrm_op_callback(lrm_op_t* op) { CRM_DEV_ASSERT(op != NULL); if(crm_assert_failed) { return; } crm_debug("received callback: %s/%s (%s)", op->op_type, op->rsc_id, op_status2text(op->op_status)); /* Make sure the LRM events are received in order */ register_fsa_input_later(C_LRM_OP_CALLBACK, I_LRM_EVENT, op); } void crmd_ha_status_callback( const char *node, const char * status, void* private_data) { crm_data_t *update = NULL; crm_devel("received callback"); crm_notice("Status update: Node %s now has status [%s]",node,status); if(safe_str_neq(status, DEADSTATUS)) { crm_devel("nstatus callback was not for a dead node"); return; } /* this node is taost */ update = create_node_state( node, node, status, NULL, NULL, NULL, NULL); set_xml_property_copy( update, XML_CIB_ATTR_CLEAR_SHUTDOWN, XML_BOOLEAN_TRUE); update_local_cib(create_cib_fragment(update, NULL)); G_main_set_trigger(fsa_source); free_xml(update); } void crmd_client_status_callback(const char * node, const char * client, const char * status, void * private) { const char *join = NULL; const char *extra = NULL; crm_data_t * update = NULL; crm_devel("received callback"); if(safe_str_neq(client, CRM_SYSTEM_CRMD)) { return; } if(safe_str_eq(status, JOINSTATUS)){ status = ONLINESTATUS; extra = XML_CIB_ATTR_CLEAR_SHUTDOWN; } else if(safe_str_eq(status, LEAVESTATUS)){ status = OFFLINESTATUS; join = CRMD_JOINSTATE_DOWN; extra = XML_CIB_ATTR_CLEAR_SHUTDOWN; } set_bit_inplace(fsa_input_register, R_PEER_DATA); g_hash_table_replace( crmd_peer_state, crm_strdup(node), crm_strdup(status)); if(fsa_state == S_STARTING || fsa_state == S_STOPPING) { return; } crm_notice("Status update: Client %s/%s now has status [%s]", node, client, status); if(safe_str_eq(node, fsa_our_dc) && safe_str_eq(status, OFFLINESTATUS)) { /* did our DC leave us */ crm_info("Got client status callback - our DC is dead"); register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL); } else { crm_devel("Got client status callback"); update = create_node_state( node, node, NULL, NULL, status, join, NULL); set_xml_property_copy(update, extra, XML_BOOLEAN_TRUE); update_local_cib(create_cib_fragment(update, NULL)); free_xml(update); if(AM_I_DC && safe_str_eq(status, OFFLINESTATUS)) { g_hash_table_remove(confirmed_nodes, node); g_hash_table_remove(finalized_nodes, node); g_hash_table_remove(integrated_nodes, node); g_hash_table_remove(welcomed_nodes, node); check_join_state(fsa_state, __FUNCTION__); } } G_main_set_trigger(fsa_source); } void crmd_ha_connection_destroy(gpointer user_data) { crm_crit("Heartbeat has left us"); /* this is always an error */ /* feed this back into the FSA */ register_fsa_input(C_HA_DISCONNECT, I_ERROR, NULL); } gboolean crmd_client_connect(IPC_Channel *client_channel, gpointer user_data) { if (client_channel == NULL) { crm_err("Channel was NULL"); } else if (client_channel->ch_status == IPC_DISCONNECT) { crm_err("Channel was disconnected"); } else { crmd_client_t *blank_client = NULL; crm_devel("Channel connected"); crm_malloc0(blank_client, sizeof(crmd_client_t)); if (blank_client == NULL) { return FALSE; } client_channel->ops->set_recv_qlen(client_channel, 100); client_channel->ops->set_send_qlen(client_channel, 100); blank_client->client_channel = client_channel; blank_client->sub_sys = NULL; blank_client->uuid = NULL; blank_client->table_key = NULL; blank_client->client_source = G_main_add_IPC_Channel( G_PRIORITY_LOW, client_channel, FALSE, crmd_ipc_msg_callback, blank_client, default_ipc_connection_destroy); } return TRUE; } gboolean ccm_dispatch(int fd, gpointer user_data) { int rc = 0; oc_ev_t *ccm_token = (oc_ev_t*)user_data; gboolean was_error = FALSE; crm_devel("received callback"); rc = oc_ev_handle_event(ccm_token); if(rc != 0) { crm_err("CCM connection appears to have failed: rc=%d.", rc); register_fsa_input(C_CCM_CALLBACK, I_ERROR, NULL); was_error = TRUE; } G_main_set_trigger(fsa_source); return !was_error; } static gboolean fsa_have_quorum = FALSE; void crmd_ccm_msg_callback( oc_ed_t event, void *cookie, size_t size, const void *data) { int instance = -1; gboolean update_cache = FALSE; struct crmd_ccm_data_s *event_data = NULL; const oc_ev_membership_t *membership = data; gboolean update_quorum = FALSE; gboolean trigger_transition = FALSE; crm_devel("received callback"); if(data != NULL) { instance = membership->m_instance; } crm_info("Quorum %s after event=%s (id=%d)", ccm_have_quorum(event)?"(re)attained":"lost", ccm_event_name(event), instance); switch(event) { case OC_EV_MS_NEW_MEMBERSHIP: case OC_EV_MS_INVALID: update_cache = TRUE; update_quorum = TRUE; if(AM_I_DC && need_transition(fsa_state)) { trigger_transition = TRUE; } break; case OC_EV_MS_NOT_PRIMARY: #if UNTESTED if(AM_I_DC == FALSE) { break; } /* tell the TE to pretend it completed and stop */ /* side effect: we'll end up in S_IDLE */ register_fsa_action(A_TE_HALT, TRUE); #endif break; case OC_EV_MS_PRIMARY_RESTORED: if(AM_I_DC && need_transition(fsa_state)) { fsa_membership_copy->id = instance; trigger_transition = TRUE; } break; case OC_EV_MS_EVICTED: update_quorum = TRUE; register_fsa_input(C_FSA_INTERNAL, I_STOP, NULL); break; default: crm_err("Unknown CCM event: %d", event); } if(update_quorum && ccm_have_quorum(event) == FALSE) { /* did we just loose quorum? */ if(fsa_have_quorum && need_transition(fsa_state)) { crm_info("Quorum lost: triggering transition (%s)", ccm_event_name(event)); trigger_transition = TRUE; } fsa_have_quorum = FALSE; } else if(update_quorum) { crm_debug("Updating quorum after event %s", ccm_event_name(event)); fsa_have_quorum = TRUE; } if(update_cache) { crm_debug("Updating cache after event %s", ccm_event_name(event)); crm_malloc0(event_data, sizeof(struct crmd_ccm_data_s)); if(event_data == NULL) { return; } event_data->event = event; if(data != NULL) { event_data->oc = copy_ccm_oc_data(data); } register_fsa_input_adv( C_CCM_CALLBACK, I_CCM_EVENT, event_data, trigger_transition?A_TE_CANCEL:A_NOTHING, FALSE, __FUNCTION__); if (event_data->oc) { crm_free(event_data->oc); event_data->oc = NULL; } crm_free(event_data); } else if(trigger_transition) { crm_debug("Scheduling transition after event %s", ccm_event_name(event)); register_fsa_action(A_TE_CANCEL); } oc_ev_callback_done(cookie); return; } void crmd_cib_connection_destroy(gpointer user_data) { if(is_set(fsa_input_register, R_SHUTDOWN)) { crm_info("Connection to the CIB terminated..."); return; } /* eventually this will trigger a reconnect, not a shutdown */ crm_err("Connection to the CIB terminated..."); register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL); clear_bit_inplace(fsa_input_register, R_CIB_CONNECTED); return; } longclock_t fsa_start = 0; longclock_t fsa_stop = 0; longclock_t fsa_diff = 0; gboolean crm_fsa_trigger(gpointer user_data) { unsigned int fsa_diff_ms = 0; if(fsa_diff_max_ms > 0) { fsa_start = time_longclock(); } s_crmd_fsa(C_FSA_INTERNAL); if(fsa_diff_max_ms > 0) { fsa_stop = time_longclock(); fsa_diff = sub_longclock(fsa_stop, fsa_start); fsa_diff_ms = longclockto_ms(fsa_diff); - if(fsa_diff_ms < 0 || fsa_diff_ms > fsa_diff_max_ms) { + if(fsa_diff_ms > fsa_diff_max_ms) { crm_err("FSA took %dms to complete", fsa_diff_ms); + + } else if(fsa_diff_ms > fsa_diff_warn_ms) { + crm_warn("FSA took %dms to complete", fsa_diff_ms); } } return TRUE; } diff --git a/crm/crmd/fsa.c b/crm/crmd/fsa.c index 41c5deb9b1..5de2b9ebb3 100644 --- a/crm/crmd/fsa.c +++ b/crm/crmd/fsa.c @@ -1,701 +1,705 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include extern uint highest_born_on; extern int num_join_invites; extern GHashTable *welcomed_nodes; extern GHashTable *integrated_nodes; extern GHashTable *finalized_nodes; extern GHashTable *confirmed_nodes; extern void initialize_join(gboolean before); long long do_state_transition(long long actions, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_state next_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data); long long clear_flags(long long actions, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input); void dump_rsc_info(void); void dump_rsc_info_callback(const HA_Message *msg, int call_id, int rc, crm_data_t *output, void *user_data); void ghash_print_node(gpointer key, gpointer value, gpointer user_data); #define DOT_PREFIX "live.dot: " #define DOT_LOG LOG_DEBUG #define do_dot_log(fmt...) do_crm_log(DOT_LOG, NULL, NULL, fmt) #define do_dot_action(fmt...) do_crm_log(DOT_LOG+1, NULL, NULL, fmt) longclock_t action_start = 0; longclock_t action_stop = 0; longclock_t action_diff = 0; unsigned int action_diff_ms = 0; #define IF_FSA_ACTION(x,y) \ if(is_set(fsa_actions,x)) { \ last_action = x; \ fsa_actions = clear_bit(fsa_actions, x); \ crm_verbose("Invoking action %s (%.16llx)", \ fsa_action2string(x), x); \ if(action_diff_max_ms > 0) { \ action_start = time_longclock(); \ } \ next_input = y(x, cause, fsa_state, last_input, fsa_data); \ if(action_diff_max_ms > 0) { \ action_stop = time_longclock(); \ action_diff = sub_longclock(action_stop, action_start); \ action_diff_ms = longclockto_ms(action_diff); \ if(action_diff_ms > action_diff_max_ms) { \ crm_err("Action %s took %dms to complete", \ fsa_action2string(x), \ action_diff_ms); \ + } else if(action_diff_ms > action_diff_warn_ms) { \ + crm_warn("Action %s took %dms to complete", \ + fsa_action2string(x), \ + action_diff_ms); \ } \ } \ crm_verbose("Action complete: %s (%.16llx)", \ fsa_action2string(x), x); \ CRM_DEV_ASSERT(next_input == I_NULL); \ do_dot_action(DOT_PREFIX"\t// %s", fsa_action2string(x)); \ } /* #define ELSEIF_FSA_ACTION(x,y) else IF_FSA_ACTION(x,y) */ void init_dotfile(void); void init_dotfile(void) { do_dot_log(DOT_PREFIX"digraph \"g\" {"); do_dot_log(DOT_PREFIX" size = \"30,30\""); do_dot_log(DOT_PREFIX" graph ["); do_dot_log(DOT_PREFIX" fontsize = \"12\""); do_dot_log(DOT_PREFIX" fontname = \"Times-Roman\""); do_dot_log(DOT_PREFIX" fontcolor = \"black\""); do_dot_log(DOT_PREFIX" bb = \"0,0,398.922306,478.927856\""); do_dot_log(DOT_PREFIX" color = \"black\""); do_dot_log(DOT_PREFIX" ]"); do_dot_log(DOT_PREFIX" node ["); do_dot_log(DOT_PREFIX" fontsize = \"12\""); do_dot_log(DOT_PREFIX" fontname = \"Times-Roman\""); do_dot_log(DOT_PREFIX" fontcolor = \"black\""); do_dot_log(DOT_PREFIX" shape = \"ellipse\""); do_dot_log(DOT_PREFIX" color = \"black\""); do_dot_log(DOT_PREFIX" ]"); do_dot_log(DOT_PREFIX" edge ["); do_dot_log(DOT_PREFIX" fontsize = \"12\""); do_dot_log(DOT_PREFIX" fontname = \"Times-Roman\""); do_dot_log(DOT_PREFIX" fontcolor = \"black\""); do_dot_log(DOT_PREFIX" color = \"black\""); do_dot_log(DOT_PREFIX" ]"); do_dot_log(DOT_PREFIX"// special nodes"); do_dot_log(DOT_PREFIX" \"S_PENDING\" "); do_dot_log(DOT_PREFIX" ["); do_dot_log(DOT_PREFIX" color = \"blue\""); do_dot_log(DOT_PREFIX" fontcolor = \"blue\""); do_dot_log(DOT_PREFIX" ]"); do_dot_log(DOT_PREFIX" \"S_TERMINATE\" "); do_dot_log(DOT_PREFIX" ["); do_dot_log(DOT_PREFIX" color = \"red\""); do_dot_log(DOT_PREFIX" fontcolor = \"red\""); do_dot_log(DOT_PREFIX" ]"); do_dot_log(DOT_PREFIX"// DC only nodes"); do_dot_log(DOT_PREFIX" \"S_INTEGRATION\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX" \"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX" \"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX" \"S_RELEASE_DC\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX" \"S_IDLE\" [ fontcolor = \"green\" ]"); } volatile enum crmd_fsa_state fsa_state = S_STARTING; oc_node_list_t *fsa_membership_copy; ll_cluster_t *fsa_cluster_conn; ll_lrm_t *fsa_lrm_conn; volatile long long fsa_input_register; volatile long long fsa_actions = A_NOTHING; const char *fsa_our_uname; char *fsa_our_dc; cib_t *fsa_cib_conn = NULL; fsa_timer_t *election_trigger = NULL; /* */ fsa_timer_t *election_timeout = NULL; /* */ fsa_timer_t *shutdown_escalation_timer = NULL; /* */ fsa_timer_t *shutdown_timer = NULL; /* */ fsa_timer_t *integration_timer = NULL; fsa_timer_t *finalization_timer = NULL; fsa_timer_t *dc_heartbeat = NULL; fsa_timer_t *wait_timer = NULL; volatile gboolean do_fsa_stall = FALSE; enum crmd_fsa_state s_crmd_fsa(enum crmd_fsa_cause cause) { time_t now; fsa_data_t *fsa_data = NULL; long long register_copy = fsa_input_register; long long new_actions = A_NOTHING; long long last_action = A_NOTHING; enum crmd_fsa_input last_input = I_NULL; enum crmd_fsa_input cur_input = I_NULL; enum crmd_fsa_input next_input = I_NULL; enum crmd_fsa_state starting_state = fsa_state; enum crmd_fsa_state last_state = starting_state; enum crmd_fsa_state next_state = starting_state; crm_verbose("FSA invoked with Cause: %s\tState: %s", fsa_cause2string(cause), fsa_state2string(fsa_state)); /* * Process actions in order of priority but do only one * action at a time to avoid complicating the ordering. * * Actions may result in a new I_ event, these are added to * (not replace) existing actions before the next iteration. * */ do_fsa_stall = FALSE; while(next_input != I_NULL || fsa_actions != A_NOTHING || is_message()){ msg_queue_helper(); if(do_fsa_stall) { /* we may be waiting for an a-sync task to "happen" * and until it does, we cant do anything else */ crm_info("Wait until something else happens"); break; } else if((is_message() && fsa_data == NULL) || (is_message() && fsa_actions == A_NOTHING && next_input == I_NULL)) { fsa_data_t *stored_msg = NULL; crm_devel("Finished with current input..." " Checking messages (%d remaining)", g_list_length(fsa_message_queue)); next_input = I_NULL; stored_msg = get_message(); crm_debug("Processing queued input %d", stored_msg->id); if(stored_msg == NULL) { crm_crit("Invalid stored message"); exit(1); } delete_fsa_input(fsa_data); if(stored_msg->fsa_cause == C_CCM_CALLBACK) { crm_devel("FSA processing CCM callback from %s", stored_msg->origin); } else if(stored_msg->fsa_cause == C_LRM_OP_CALLBACK) { crm_devel("FSA processing LRM callback from %s", stored_msg->origin); } else if(stored_msg->data == NULL) { crm_devel("FSA processing input from %s", stored_msg->origin); } else { ha_msg_input_t *ha_input = fsa_typed_data_adv( stored_msg, fsa_dt_ha_msg, __FUNCTION__); crm_devel("FSA processing XML message from %s", stored_msg->origin); crm_log_message(LOG_MSG, ha_input->msg); crm_xml_devel(ha_input->xml, "FSA message data"); } fsa_data = stored_msg; /* set up the input */ next_input = fsa_data->fsa_input; /* add any actions back to the queue */ fsa_actions |= fsa_data->actions; /* update the cause */ cause = fsa_data->fsa_cause; fsa_dump_actions(fsa_data->actions, "\tadded back"); do_dot_log(DOT_PREFIX"\t// FSA input: State=%s \tCause=%s" " \tInput=%s \tOrigin=%s() \tid=%d", fsa_state2string(fsa_state), fsa_cause2string(fsa_data->fsa_cause), fsa_input2string(fsa_data->fsa_input), fsa_data->origin, fsa_data->id); } else if(fsa_data == NULL) { crm_malloc0(fsa_data, sizeof(fsa_data_t)); fsa_data->fsa_input = I_NULL; fsa_data->fsa_cause = cause; fsa_data->actions = A_NOTHING; fsa_data->origin = "s_crmd_fsa (enter)"; fsa_data->data = NULL; fsa_data->data_type = fsa_dt_none; if(fsa_data->origin == NULL) { crm_crit("Out of memory"); exit(1); } } /* update input variables */ cur_input = next_input; if(cur_input != I_NULL) { /* record the most recent non I_NULL input */ crm_devel("Updating last_input to %s", fsa_input2string(cur_input)); last_input = cur_input; } /* get the next batch of actions */ new_actions = crmd_fsa_actions[cur_input][fsa_state]; if(new_actions != A_NOTHING) { #ifdef FSA_TRACE crm_verbose("Adding actions %.16llx for %s/%s", new_actions, fsa_input2string(cur_input), fsa_state2string(fsa_state)); fsa_dump_actions(new_actions, "\tscheduled"); #endif fsa_actions |= new_actions; } if(fsa_data == NULL) { crm_err("No input for FSA.... terminating"); exit(1); } #ifdef FSA_TRACE crm_verbose("FSA while loop:\tState: %s, Cause: %s," " Input: %s, Origin=%s", fsa_state2string(crmd_fsa_state[cur_input][fsa_state]), fsa_cause2string(fsa_data->fsa_cause), fsa_input2string(cur_input), fsa_data->origin); #endif /* logging : *before* the state is changed */ IF_FSA_ACTION(A_ERROR, do_log) else IF_FSA_ACTION(A_WARN, do_log) else IF_FSA_ACTION(A_LOG, do_log) /* this is always run, some inputs/states may make various * actions irrelevant/invalid */ fsa_actions = clear_flags(fsa_actions, cause, fsa_state, cur_input); /* update state variables */ next_state = crmd_fsa_state[cur_input][fsa_state]; last_state = fsa_state; fsa_state = next_state; /* start doing things... */ /* * Hook for change of state. * Allows actions to be added or removed when entering a state */ if(last_state != fsa_state){ fsa_actions = do_state_transition( fsa_actions, cause, last_state, fsa_state, cur_input, fsa_data); } /* regular action processing in order of action priority * * Make sure all actions that connect to required systems * are performed first */ /* get out of here NOW! before anything worse happens */ IF_FSA_ACTION(A_EXIT_1, do_exit) else IF_FSA_ACTION(A_ERROR, do_log) else IF_FSA_ACTION(A_WARN, do_log) else IF_FSA_ACTION(A_LOG, do_log) /* essential start tasks */ else IF_FSA_ACTION(A_HA_CONNECT, do_ha_control) else IF_FSA_ACTION(A_STARTUP, do_startup) else IF_FSA_ACTION(A_CIB_START, do_cib_control) else IF_FSA_ACTION(A_READCONFIG, do_read_config) /* sub-system start/connect */ else IF_FSA_ACTION(A_LRM_CONNECT, do_lrm_control) else IF_FSA_ACTION(A_CCM_CONNECT, do_ccm_control) else IF_FSA_ACTION(A_TE_START, do_te_control) else IF_FSA_ACTION(A_PE_START, do_pe_control) /* sub-system restart */ else IF_FSA_ACTION(O_CIB_RESTART, do_cib_control) else IF_FSA_ACTION(O_PE_RESTART, do_pe_control) else IF_FSA_ACTION(O_TE_RESTART, do_te_control) /* Timers */ /* else IF_FSA_ACTION(O_DC_TIMER_RESTART, do_timer_control) */ else IF_FSA_ACTION(A_DC_TIMER_STOP, do_timer_control) else IF_FSA_ACTION(A_INTEGRATE_TIMER_STOP, do_timer_control) else IF_FSA_ACTION(A_INTEGRATE_TIMER_START, do_timer_control) else IF_FSA_ACTION(A_FINALIZE_TIMER_STOP, do_timer_control) else IF_FSA_ACTION(A_FINALIZE_TIMER_START, do_timer_control) /* * Highest priority actions */ else IF_FSA_ACTION(A_CIB_BUMPGEN, do_cib_invoke) else IF_FSA_ACTION(A_MSG_ROUTE, do_msg_route) else IF_FSA_ACTION(A_RECOVER, do_recover) else IF_FSA_ACTION(A_CL_JOIN_REQUEST, do_cl_join_request) else IF_FSA_ACTION(A_CL_JOIN_RESULT, do_cl_join_result) else IF_FSA_ACTION(A_SHUTDOWN_REQ, do_shutdown_req) else IF_FSA_ACTION(A_ELECTION_VOTE, do_election_vote) else IF_FSA_ACTION(A_ELECTION_COUNT, do_election_count_vote) /* * High priority actions * Update the cache first */ else IF_FSA_ACTION(A_CCM_UPDATE_CACHE, do_ccm_update_cache) else IF_FSA_ACTION(A_CCM_EVENT, do_ccm_event) else IF_FSA_ACTION(A_STARTED, do_started) else IF_FSA_ACTION(A_CL_JOIN_QUERY, do_cl_join_query) else IF_FSA_ACTION(A_DC_TIMER_START, do_timer_control) /* * Medium priority actions */ else IF_FSA_ACTION(A_DC_TAKEOVER, do_dc_takeover) else IF_FSA_ACTION(A_DC_RELEASE, do_dc_release) else IF_FSA_ACTION(A_ELECTION_START, do_election_vote) else IF_FSA_ACTION(A_DC_JOIN_OFFER_ALL, do_dc_join_offer_all) else IF_FSA_ACTION(A_DC_JOIN_OFFER_ONE, do_dc_join_offer_all) else IF_FSA_ACTION(A_DC_JOIN_PROCESS_REQ,do_dc_join_req) else IF_FSA_ACTION(A_DC_JOIN_PROCESS_ACK,do_dc_join_ack) /* * Low(er) priority actions * Make sure the CIB is always updated before invoking the * PE, and the PE before the TE */ else IF_FSA_ACTION(A_CIB_INVOKE_LOCAL, do_cib_invoke) else IF_FSA_ACTION(A_CIB_INVOKE, do_cib_invoke) else IF_FSA_ACTION(A_DC_JOIN_FINALIZE, do_dc_join_finalize) else IF_FSA_ACTION(A_LRM_INVOKE, do_lrm_invoke) else IF_FSA_ACTION(A_LRM_EVENT, do_lrm_event) else IF_FSA_ACTION(A_TE_HALT, do_te_invoke) else IF_FSA_ACTION(A_TE_CANCEL, do_te_invoke) else IF_FSA_ACTION(A_PE_INVOKE, do_pe_invoke) else IF_FSA_ACTION(A_TE_INVOKE, do_te_invoke) else IF_FSA_ACTION(A_CL_JOIN_ANNOUNCE, do_cl_join_announce) /* sub-system stop */ else IF_FSA_ACTION(A_DC_RELEASED, do_dc_release) else IF_FSA_ACTION(A_PE_STOP, do_pe_control) else IF_FSA_ACTION(A_TE_STOP, do_te_control) else IF_FSA_ACTION(A_SHUTDOWN, do_shutdown) else IF_FSA_ACTION(A_STOP, do_stop) else IF_FSA_ACTION(A_CCM_DISCONNECT, do_ccm_control) else IF_FSA_ACTION(A_LRM_DISCONNECT, do_lrm_control) else IF_FSA_ACTION(A_HA_DISCONNECT, do_ha_control) else IF_FSA_ACTION(A_CIB_STOP, do_cib_control) /* exit gracefully */ else IF_FSA_ACTION(A_EXIT_0, do_exit) /* else IF_FSA_ACTION(A_, do_) */ /* Error checking and reporting */ else if(cur_input != I_NULL && fsa_actions == A_NOTHING) { crm_debug( "No action specified for input,state (%s,%s)", fsa_input2string(cur_input), fsa_state2string(fsa_state)); next_input = I_NULL; } else if(cur_input == I_NULL && fsa_actions == A_NOTHING) { #ifdef FSA_TRACE crm_info("Nothing left to do..."); fsa_dump_actions(fsa_actions, "still here"); #endif break; } else { crm_err("Action %s (0x%llx) not supported ", fsa_action2string(fsa_actions), fsa_actions); next_input = I_ERROR; } } now = time(NULL); do_dot_log(DOT_PREFIX"\t// ### Exiting the FSA (%s%s): %s", fsa_state2string(fsa_state), do_fsa_stall?": paused":"", asctime(localtime(&now))); crm_debug("Exiting the FSA: is_message=%s, queue=%d, fsa_data=%p," " fsa_actions=0x%llx, next_input=%s, stalled=%s", is_message()?"true":"false", g_list_length(fsa_message_queue), fsa_data, fsa_actions, fsa_input2string(next_input), do_fsa_stall?"true":"false"); /* cleanup inputs? */ delete_fsa_input(fsa_data); if(register_copy != fsa_input_register) { fsa_dump_inputs(LOG_DEBUG, fsa_input_register); } fsa_dump_queue(LOG_VERBOSE); return fsa_state; } long long do_state_transition(long long actions, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_state next_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { gboolean clear_recovery_bit = TRUE; long long tmp = actions; const char *state_from = fsa_state2string(cur_state); const char *state_to = fsa_state2string(next_state); const char *input = fsa_input2string(current_input); time_t now = time(NULL); if(cur_state == next_state) { crm_err("%s called in state %s with no transtion", __FUNCTION__, state_from); return A_NOTHING; } do_dot_log(DOT_PREFIX"\t%s -> %s [ label=%s cause=%s origin=%s ] // %s", state_from, state_to, input, fsa_cause2string(cause), msg_data->origin, asctime(localtime(&now))); crm_info("State transition %s -> %s [ input=%s cause=%s origin=%s ]", state_from, state_to, input, fsa_cause2string(cause), msg_data->origin); /* the last two clauses might cause trouble later */ if(election_timeout != NULL && next_state != S_ELECTION && next_state != S_RELEASE_DC && next_state != S_PENDING) { crm_timer_stop(election_timeout); /* } else { */ /* crm_timer_start(election_timeout); */ } #if 0 if(is_set(fsa_input_register, R_SHUTDOWN)){ set_bit_inplace(tmp, A_DC_TIMER_STOP); } #endif if(next_state == S_INTEGRATION) { set_bit_inplace(tmp, A_INTEGRATE_TIMER_START); } else { set_bit_inplace(tmp, A_INTEGRATE_TIMER_STOP); } if(next_state == S_FINALIZE_JOIN) { set_bit_inplace(tmp, A_FINALIZE_TIMER_START); } else { set_bit_inplace(tmp, A_FINALIZE_TIMER_STOP); } if(next_state != S_PENDING) { set_bit_inplace(tmp, A_DC_TIMER_STOP); } if(next_state != S_ELECTION) { highest_born_on = 0; } switch(next_state) { case S_PENDING: case S_ELECTION: crm_info("Resetting our DC to NULL on transition to %s", fsa_state2string(next_state)); crm_free(fsa_our_dc); fsa_our_dc = NULL; break; case S_NOT_DC: if(is_set(fsa_input_register, R_SHUTDOWN)){ crm_info("(Re)Issuing shutdown request now" " that we have a new DC"); set_bit_inplace(tmp, A_SHUTDOWN_REQ); } CRM_DEV_ASSERT(fsa_our_dc != NULL); if(fsa_our_dc == NULL) { crm_err("Reached S_NOT_DC without a DC" " being recorded"); } break; case S_RECOVERY: clear_recovery_bit = FALSE; break; case S_FINALIZE_JOIN: if(cause == C_TIMER_POPPED) { crm_warn("Progressed to state %s after %s", fsa_state2string(next_state), fsa_cause2string(cause)); } if(g_hash_table_size(welcomed_nodes) > 0) { char *msg = crm_strdup( " Welcome reply not received from"); crm_warn("%u cluster nodes failed to respond" "to the join offer.", g_hash_table_size(welcomed_nodes)); g_hash_table_foreach( welcomed_nodes, ghash_print_node, msg); } else { crm_info("All %d cluster nodes " "responded to the join offer.", g_hash_table_size(integrated_nodes)); } break; case S_POLICY_ENGINE: if(cause == C_TIMER_POPPED) { crm_warn("Progressed to state %s after %s", fsa_state2string(next_state), fsa_cause2string(cause)); } if(g_hash_table_size(finalized_nodes) > 0) { char *msg = crm_strdup( " Confirm not received from"); crm_err("%u cluster nodes failed to confirm" " their join.", g_hash_table_size(finalized_nodes)); g_hash_table_foreach( finalized_nodes, ghash_print_node, msg); } else if(g_hash_table_size(confirmed_nodes) == fsa_membership_copy->members_size) { crm_info("All %u cluster nodes are" " eligable to run resources.", fsa_membership_copy->members_size); } else { crm_warn("Only %u of %u cluster " "nodes are eligable to run resources", g_hash_table_size(confirmed_nodes), fsa_membership_copy->members_size); } /* initialize_join(FALSE); */ break; case S_STOPPING: case S_TERMINATE: /* possibly redundant */ crm_timer_stop(shutdown_timer); set_bit_inplace(fsa_input_register, R_SHUTDOWN); break; case S_IDLE: dump_rsc_info(); break; default: break; } if(clear_recovery_bit && next_state != S_PENDING) { tmp = clear_bit(tmp, A_RECOVER); } else if(clear_recovery_bit == FALSE) { tmp = set_bit(tmp, A_RECOVER); } if(tmp != actions) { fsa_dump_actions(actions ^ tmp, "New actions"); actions = tmp; } return actions; } long long clear_flags(long long actions, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input) { long long saved_actions = actions; long long startup_actions = A_STARTUP|A_CIB_START|A_LRM_CONNECT|A_CCM_CONNECT|A_HA_CONNECT|A_READCONFIG|A_STARTED|A_CL_JOIN_QUERY; if(cur_state == S_STOPPING || is_set(fsa_input_register, R_SHUTDOWN)) { clear_bit_inplace(actions, startup_actions); } fsa_dump_actions(actions ^ saved_actions, "Cleared Actions"); return actions; } void dump_rsc_info(void) { } void ghash_print_node(gpointer key, gpointer value, gpointer user_data) { const char *text = user_data; const char *uname = key; crm_info("%s: %s", text, uname); } diff --git a/include/crm/crm.h b/include/crm/crm.h index 5d61080016..307dcfc539 100644 --- a/include/crm/crm.h +++ b/include/crm/crm.h @@ -1,282 +1,284 @@ -/* $Id: crm.h,v 1.58 2005/05/10 13:20:32 andrew Exp $ */ +/* $Id: crm.h,v 1.59 2005/05/12 11:16:40 andrew Exp $ */ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef CRM__H #define CRM__H #include #include #include #include #include #include #ifdef MCHECK #include #endif #include #ifndef CRM_DEV_BUILD # define CRM_DEV_BUILD 0 #endif -#define ipc_call_diff_max_ms 1000 -#define action_diff_max_ms 1000 -#define fsa_diff_max_ms 2000 +#define ipc_call_diff_max_ms 2000 +#define action_diff_warn_ms 5000 +#define action_diff_max_ms 15000 +#define fsa_diff_warn_ms 10000 +#define fsa_diff_max_ms 20000 #include #define CRM_ASSERT(expr) if((expr) == FALSE) { \ do_crm_log(LOG_CRIT, __FILE__, __PRETTY_FUNCTION__, \ "Triggered dev assert at %s:%d : %s", \ __FILE__, __LINE__, #expr); \ abort(); \ } extern gboolean crm_assert_failed; #define CRM_DEV_ASSERT(expr) crm_assert_failed = FALSE; \ if((expr) == FALSE) { \ crm_assert_failed = TRUE; \ do_crm_log(CRM_DEV_BUILD?LOG_CRIT:LOG_ERR, \ __FILE__, __PRETTY_FUNCTION__, \ "Triggered dev assert at %s:%d : %s", \ __FILE__, __LINE__, #expr); \ if(CRM_DEV_BUILD) { \ abort(); \ } \ } /* Clean these up at some point, some probably should be runtime options */ #define WORKING_DIR HA_VARLIBDIR"/heartbeat/crm" #define BIN_DIR HA_LIBDIR"/heartbeat" #define SOCKET_LEN 1024 #define APPNAME_LEN 256 #define LOG_DIR "/var/log" #define MAX_IPC_FAIL 5 #define CIB_FILENAME WORKING_DIR"/cib.xml" #define CIB_BACKUP WORKING_DIR"/cib_backup.xml" #define DEVEL_CIB_COPY 1 #define DEVEL_DIR "/tmp/crm" #define CRM_VERSION "0.7" #define MSG_LOG 1 #define DOT_FSA_ACTIONS 1 #define DOT_ALL_FSA_INPUTS 1 #define FSA_TRACE 1 /* #define USE_FAKE_LRM 1 */ #define INFINITY_S "INFINITY" #define MINUS_INFINITY_S "-INFINITY" #define INFINITY 1000000.0 /* Sub-systems */ #define CRM_SYSTEM_DC "dc" #define CRM_SYSTEM_DCIB "dcib" /* The master CIB */ #define CRM_SYSTEM_CIB "cib" #define CRM_SYSTEM_CRMD "crmd" #define CRM_SYSTEM_LRMD "lrmd" #define CRM_SYSTEM_PENGINE "pengine" #define CRM_SYSTEM_TENGINE "tengine" /* Valid operations */ #define CRM_OP_NOOP "noop" /* soon to be moved to cib.h */ #define CRM_OP_CIB_SLAVE "cib_slave" #define CRM_OP_CIB_SLAVEALL "cib_slave_all" #define CRM_OP_CIB_MASTER "cib_master" #define CRM_OP_CIB_SYNC "cib_sync" #define CRM_OP_CIB_ISMASTER "cib_ismaster" #define CRM_OP_CIB_BUMP "cib_bump" #define CRM_OP_CIB_QUERY "cib_query" #define CRM_OP_CIB_CREATE "cib_create" #define CRM_OP_CIB_UPDATE "cib_update" #define CRM_OP_CIB_DELETE "cib_delete" #define CRM_OP_CIB_ERASE "cib_erase" #define CRM_OP_CIB_REPLACE "cib_replace" #define CRM_OP_CIB_NOTIFY "cib_notify" #define CRM_OP_JOIN_ANNOUNCE "join_announce" #define CRM_OP_JOIN_OFFER "join_offer" #define CRM_OP_JOIN_REQUEST "join_request" #define CRM_OP_JOIN_ACKNAK "join_ack_nack" #define CRM_OP_JOIN_CONFIRM "join_confirm" #define CRM_OP_DIE "die_no_respawn" #define CRM_OP_RETRIVE_CIB "retrieve_cib" #define CRM_OP_PING "ping" #define CRM_OP_VOTE "vote" #define CRM_OP_HELLO "hello" #define CRM_OP_HBEAT "dc_beat" #define CRM_OP_PECALC "pe_calc" #define CRM_OP_ABORT "abort" #define CRM_OP_QUIT "quit" #define CRM_OP_LOCAL_SHUTDOWN "start_shutdown" #define CRM_OP_SHUTDOWN_REQ "req_shutdown" #define CRM_OP_SHUTDOWN "do_shutdown" #define CRM_OP_EVENTCC "event_cc" #define CRM_OP_TEABORT "te_abort" #define CRM_OP_TE_HALT "te_halt" #define CRM_OP_TECOMPLETE "te_complete" #define CRM_OP_TETIMEOUT "te_timeout" #define CRM_OP_TRANSITION "transition" #define CRM_OP_REGISTER "register" #define CRM_OP_DEBUG_UP "debug_inc" #define CRM_OP_DEBUG_DOWN "debug_dec" #define CRMD_STATE_ACTIVE "member" #define CRMD_STATE_INACTIVE "down" #define CRMD_JOINSTATE_DOWN "down" #define CRMD_JOINSTATE_PENDING "pending" #define CRMD_JOINSTATE_MEMBER "member" #define CRMD_RSCSTATE_START "start" #define CRMD_RSCSTATE_START_PENDING "starting" #define CRMD_RSCSTATE_START_OK "running" #define CRMD_RSCSTATE_START_FAIL "start_failed" #define CRMD_RSCSTATE_STOP "stop" #define CRMD_RSCSTATE_STOP_PENDING "stopping" #define CRMD_RSCSTATE_STOP_OK "stopped" #define CRMD_RSCSTATE_STOP_FAIL "stop_failed" #define CRMD_RSCSTATE_MON "monitor" #define CRMD_RSCSTATE_MON_PENDING CRMD_RSCSTATE_START_OK #define CRMD_RSCSTATE_MON_OK CRMD_RSCSTATE_START_OK #define CRMD_RSCSTATE_MON_FAIL "monitor_failed" /* #define CRMD_RSCSTATE_GENERIC "pending" */ #define CRMD_RSCSTATE_GENERIC_PENDING "pending" #define CRMD_RSCSTATE_GENERIC_OK "complete" #define CRMD_RSCSTATE_GENERIC_FAIL "pending_failed" typedef GList* GListPtr; #define crm_atoi(text, default) atoi(text?text:default) extern gboolean safe_str_eq(const char *a, const char *b); extern gboolean safe_str_neq(const char *a, const char *b); #define slist_iter(child, child_type, parent, counter, a) \ { \ GListPtr __crm_iter_head = parent; \ child_type *child = NULL; \ int counter = 0; \ for(; __crm_iter_head != NULL; counter++) { \ child = __crm_iter_head->data; \ __crm_iter_head = __crm_iter_head->next; \ { a; } \ } \ } #define safe_val3(def, t,u,v) (t?t->u?t->u->v:def:def) /* Developmental debug stuff */ #define LOG_VERBOSE LOG_DEBUG+1 #define LOG_DEV LOG_DEBUG+2 #define LOG_TRACE LOG_DEBUG+3 #define LOG_INSANE LOG_DEBUG+5 #define LOG_MSG LOG_DEV # define crm_crit(w...) do_crm_log(LOG_CRIT, __FILE__, __FUNCTION__, w) # define crm_err(w...) do_crm_log(LOG_ERR, __FILE__, __FUNCTION__, w) # define crm_warn(w...) do_crm_log(LOG_WARNING, __FILE__, __FUNCTION__, w) # define crm_notice(w...) do_crm_log(LOG_NOTICE, __FILE__, __FUNCTION__, w) # define crm_info(w...) do_crm_log(LOG_INFO, __FILE__, __FUNCTION__, w) #if 1 # define crm_debug(w...) do_crm_log(LOG_DEBUG, __FILE__, __FUNCTION__, w) # define crm_devel(w...) do_crm_log(LOG_DEV, __FILE__, __FUNCTION__, w) # define crm_verbose(w...) do_crm_log(LOG_VERBOSE, __FILE__, __FUNCTION__, w) # define crm_trace(w...) do_crm_log(LOG_TRACE, __FILE__, __FUNCTION__, w) # define crm_insane(w...) do_crm_log(LOG_INSANE, __FILE__, __FUNCTION__, w) #else # define crm_debug(w...) if(0) { do_crm_log(LOG_DEBUG, NULL, NULL, w); } # define crm_devel(w...) if(0) { do_crm_log(LOG_DEV, NULL, NULL, w); } # define crm_verbose(w...) if(0) { do_crm_log(LOG_VERBOSE, NULL, NULL, w); } # define crm_trace(w...) if(0) { do_crm_log(LOG_TRACE, NULL, NULL, w); } # define crm_insane(w...) if(0) { do_crm_log(LOG_INSANE, NULL, NULL, w); } #endif extern void crm_log_message_adv(int level, const char *alt_debugfile, const HA_Message *msg); #define crm_log_message(level, msg) crm_log_message_adv(level, NULL, msg) #define crm_do_action(level, actions) if(crm_log_level >= level) { \ actions; \ } #define crm_info_action(x) crm_do_action(LOG_INFO, x) #define crm_debug_action(x) crm_do_action(LOG_DEBUG, x) #define crm_verbose_action(x) crm_do_action(LOG_VERBOSE, x) #define crm_devel_action(x) crm_do_action(LOG_DEV, x) #define crm_log_xml(level, text, xml) if(crm_log_level >= level) { \ print_xml_formatted(level, __FUNCTION__, xml, text); \ } #define crm_xml_crit(xml, text) crm_log_xml(LOG_CRIT, text, xml) #define crm_xml_err(xml, text) crm_log_xml(LOG_ERR, text, xml) #define crm_xml_warn(xml, text) crm_log_xml(LOG_WARNING, text, xml) #define crm_xml_notice(xml, text) crm_log_xml(LOG_NOTICE, text, xml) #define crm_xml_info(xml, text) crm_log_xml(LOG_INFO, text, xml) #define crm_xml_debug(xml, text) crm_log_xml(LOG_DEBUG, text, xml) #define crm_xml_devel(xml, text) crm_log_xml(LOG_DEV, text, xml) #define crm_xml_verbose(xml, text) crm_log_xml(LOG_VERBOSE, text, xml) #define crm_xml_trace(xml, text) crm_log_xml(LOG_TRACE, text, xml) #define crm_xml_insane(xml, text) crm_log_xml(LOG_INSANE, text, xml) #define crm_malloc0(new_obj,length) \ { \ if(new_obj) { \ crm_err("Potential memory leak:" \ " %s at %s:%d not NULL before alloc.", \ #new_obj, __FILE__, __LINE__); \ if(CRM_DEV_BUILD) { abort(); } \ } \ new_obj = cl_malloc(length); \ if(new_obj == NULL) { \ crm_crit("Out of memory... exiting"); \ exit(1); \ } else { \ memset(new_obj, 0, length); \ } \ } /* for temporary backwards compatibility */ #define crm_malloc(new_obj,length) crm_malloc0(new_obj,length) #if 1 # define crm_free(x) if(x) { \ CRM_ASSERT(cl_is_allocated(x) == 1); \ cl_free(x); \ x=NULL; \ } #else # define crm_free(x) x=NULL #endif #define crm_str(x) (const char*)(x?x:"") #if 1 -# define crm_msg_del(msg) if(msg != NULL) { ha_msg_del(msg); } +# define crm_msg_del(msg) if(msg != NULL) { ha_msg_del(msg); msg = NULL; } #else # define crm_msg_del(msg) msg = NULL #endif #endif