diff --git a/crm/crmd/callbacks.c b/crm/crmd/callbacks.c index ad6613839a..30b95ebe1d 100644 --- a/crm/crmd/callbacks.c +++ b/crm/crmd/callbacks.c @@ -1,672 +1,677 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include GHashTable *crmd_peer_state = NULL; crm_data_t *find_xml_in_hamessage(const HA_Message * msg); void crmd_ha_connection_destroy(gpointer user_data); /* From join_dc... */ extern gboolean check_join_state( enum crmd_fsa_state cur_state, const char *source); /* #define MAX_EMPTY_CALLBACKS 20 */ /* int empty_callbacks = 0; */ #define trigger_fsa(source) crm_debug_3("Triggering FSA: %s", __FUNCTION__); \ G_main_set_trigger(source); gboolean crmd_ha_msg_dispatch(ll_cluster_t *cluster_conn, gpointer user_data) { IPC_Channel *channel = NULL; gboolean stay_connected = TRUE; crm_debug_3("Invoked"); if(cluster_conn != NULL) { channel = cluster_conn->llc_ops->ipcchan(cluster_conn); } CRM_CHECK(cluster_conn != NULL, ;); CRM_CHECK(channel != NULL, ;); if(channel != NULL && IPC_ISRCONN(channel)) { if(cluster_conn->llc_ops->msgready(cluster_conn) == 0) { crm_debug_2("no message ready yet"); } /* invoke the callbacks but dont block */ cluster_conn->llc_ops->rcvmsg(cluster_conn, 0); } if (channel == NULL || channel->ch_status != IPC_CONNECT) { if(is_set(fsa_input_register, R_HA_DISCONNECTED) == FALSE) { crm_crit("Lost connection to heartbeat service."); } else { crm_info("Lost connection to heartbeat service."); } trigger_fsa(fsa_source); stay_connected = FALSE; } return stay_connected; } void crmd_ha_connection_destroy(gpointer user_data) { crm_debug_3("Invoked"); if(is_set(fsa_input_register, R_HA_DISCONNECTED)) { /* we signed out, so this is expected */ crm_info("Heartbeat disconnection complete"); return; } crm_crit("Lost connection to heartbeat service!"); register_fsa_input(C_HA_DISCONNECT, I_ERROR, NULL); trigger_fsa(fsa_source); } void crmd_ha_msg_callback(HA_Message * msg, void* private_data) { int level = LOG_DEBUG; ha_msg_input_t *new_input = NULL; oc_node_t *from_node = NULL; const char *from = ha_msg_value(msg, F_ORIG); const char *seq = ha_msg_value(msg, F_SEQ); const char *op = ha_msg_value(msg, F_CRM_TASK); const char *sys_to = ha_msg_value(msg, F_CRM_SYS_TO); const char *sys_from = ha_msg_value(msg, F_CRM_SYS_FROM); CRM_DEV_ASSERT(from != NULL); crm_debug_2("HA[inbound]: %s from %s", op, from); if(fsa_membership_copy == NULL) { crm_debug("Ignoring HA messages until we are" " connected to the CCM (%s op from %s)", op, from); crm_log_message_adv( LOG_MSG, "HA[inbound]: Ignore (No CCM)", msg); return; } from_node = g_hash_table_lookup(fsa_membership_copy->members, from); if(from_node == NULL) { if(safe_str_eq(op, CRM_OP_VOTE)) { level = LOG_WARNING; } else if(AM_I_DC && safe_str_eq(op, CRM_OP_JOIN_ANNOUNCE)) { level = LOG_WARNING; } else if(safe_str_eq(sys_from, CRM_SYSTEM_DC)) { level = LOG_WARNING; } do_crm_log(level, "Ignoring HA message (op=%s) from %s: not in our" " membership list (size=%d)", op, from, g_hash_table_size(fsa_membership_copy->members)); crm_log_message_adv(LOG_MSG, "HA[inbound]: CCM Discard", msg); } else if(safe_str_eq(sys_to, CRM_SYSTEM_DC) && AM_I_DC == FALSE) { crm_debug_2("Ignoring message for the DC [F_SEQ=%s]", seq); return; } else if(safe_str_eq(sys_from, CRM_SYSTEM_DC)) { if(AM_I_DC && safe_str_neq(from, fsa_our_uname)) { crm_err("Another DC detected: %s (op=%s)", from, op); /* make sure the election happens NOW */ level = LOG_WARNING; if(fsa_state != S_ELECTION) { new_input = new_ha_msg_input(msg); register_fsa_error_adv( C_FSA_INTERNAL, I_ELECTION, NULL, new_input, __FUNCTION__); } #if 0 /* still thinking about this one... * could create a timing issue if we dont notice the * election before a new DC is elected. */ } else if(fsa_our_dc != NULL && safe_str_neq(from,fsa_our_dc)){ crm_warn("Ignoring message from wrong DC: %s vs. %s ", from, fsa_our_dc); return; #endif } else { crm_debug_2("Processing DC message from %s [F_SEQ=%s]", from, seq); } } if(new_input == NULL) { crm_log_message_adv(LOG_MSG, "HA[inbound]", msg); new_input = new_ha_msg_input(msg); route_message(C_HA_MESSAGE, new_input); } delete_ha_msg_input(new_input); trigger_fsa(fsa_source); return; } /* * Apparently returning TRUE means "stay connected, keep doing stuff". * Returning FALSE means "we're all done, close the connection" */ gboolean crmd_ipc_msg_callback(IPC_Channel *client, gpointer user_data) { int lpc = 0; HA_Message *msg = NULL; ha_msg_input_t *new_input = NULL; crmd_client_t *curr_client = (crmd_client_t*)user_data; gboolean stay_connected = TRUE; crm_debug_2("Invoked: %s", curr_client->table_key); while(IPC_ISRCONN(client)) { if(client->ops->is_message_pending(client) == 0) { break; } msg = msgfromIPC_noauth(client); if (msg == NULL) { crm_info("%s: no message this time", curr_client->table_key); continue; } lpc++; new_input = new_ha_msg_input(msg); crm_msg_del(msg); crm_debug_2("Processing msg from %s", curr_client->table_key); crm_log_message_adv(LOG_DEBUG_2, "CRMd[inbound]", new_input->msg); if(crmd_authorize_message(new_input, curr_client)) { route_message(C_IPC_MESSAGE, new_input); } delete_ha_msg_input(new_input); msg = NULL; new_input = NULL; if(client->ch_status != IPC_CONNECT) { break; } } crm_debug_2("Processed %d messages", lpc); if (client->ch_status != IPC_CONNECT) { stay_connected = FALSE; process_client_disconnect(curr_client); } trigger_fsa(fsa_source); return stay_connected; } extern GCHSource *lrm_source; gboolean lrm_dispatch(IPC_Channel *src_not_used, gpointer user_data) { /* ?? src == lrm_channel ?? */ ll_lrm_t *lrm = (ll_lrm_t*)user_data; IPC_Channel *lrm_channel = lrm->lrm_ops->ipcchan(lrm); crm_debug_3("Invoked"); lrm->lrm_ops->rcvmsg(lrm, FALSE); if(lrm_channel->ch_status != IPC_CONNECT) { if(is_set(fsa_input_register, R_LRM_CONNECTED)) { crm_crit("LRM Connection failed"); register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL); clear_bit_inplace(fsa_input_register, R_LRM_CONNECTED); } else { crm_info("LRM Connection disconnected"); } lrm_source = NULL; return FALSE; } return TRUE; } extern gboolean process_lrm_event(lrm_op_t *op); void lrm_op_callback(lrm_op_t* op) { CRM_CHECK(op != NULL, return); process_lrm_event(op); } void crmd_ha_status_callback( const char *node, const char * status, void* private_data) { crm_data_t *update = NULL; crm_notice("Status update: Node %s now has status [%s]",node,status); if(safe_str_eq(status, DEADSTATUS)) { /* this node is taost */ update = create_node_state( node, status, XML_BOOLEAN_NO, OFFLINESTATUS, CRMD_STATE_INACTIVE, NULL, TRUE, __FUNCTION__); if(update) { crm_xml_add(update, XML_CIB_ATTR_REPLACE, XML_TAG_TRANSIENT_NODEATTRS); } } else if(safe_str_eq(status, ACTIVESTATUS)) { update = create_node_state( node, status, NULL, NULL, NULL, NULL, FALSE, __FUNCTION__); } if(update != NULL) { /* this change should not be broadcast */ fsa_cib_anon_update( XML_CIB_TAG_STATUS, update, cib_inhibit_bcast|cib_scope_local|cib_quorum_override); trigger_fsa(fsa_source); free_xml(update); } else { crm_info("Ping node %s is %s", node, status); } } void crmd_client_status_callback(const char * node, const char * client, const char * status, void * private) { const char *join = NULL; crm_data_t *update = NULL; gboolean clear_shutdown = FALSE; crm_debug_3("Invoked"); if(safe_str_neq(client, CRM_SYSTEM_CRMD)) { return; } if(safe_str_eq(status, JOINSTATUS)){ status = ONLINESTATUS; clear_shutdown = TRUE; } else if(safe_str_eq(status, LEAVESTATUS)){ status = OFFLINESTATUS; join = CRMD_STATE_INACTIVE; /* clear_shutdown = TRUE; */ } set_bit_inplace(fsa_input_register, R_PEER_DATA); g_hash_table_replace( crmd_peer_state, crm_strdup(node), crm_strdup(status)); if(is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) { return; } else if(fsa_state == S_STOPPING) { return; } crm_notice("Status update: Client %s/%s now has status [%s]", node, client, status); if(safe_str_eq(status, ONLINESTATUS)) { /* remove the cached value in case it changed */ crm_info("Uncaching UUID for %s", node); unget_uuid(node); } if(safe_str_eq(node, fsa_our_dc) && safe_str_eq(status, OFFLINESTATUS)){ /* did our DC leave us */ crm_info("Got client status callback - our DC is dead"); register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL); } else { crm_debug_3("Got client status callback"); update = create_node_state( node, NULL, NULL, status, join, NULL, clear_shutdown, __FUNCTION__); if(clear_shutdown) { crm_xml_add(update, XML_CIB_ATTR_REPLACE, XML_TAG_TRANSIENT_NODEATTRS); } /* it is safe to keep these updates on the local node * each node updates their own CIB */ fsa_cib_anon_update( XML_CIB_TAG_STATUS, update, cib_inhibit_bcast|cib_scope_local|cib_quorum_override); free_xml(update); if(AM_I_DC && safe_str_eq(status, OFFLINESTATUS)) { g_hash_table_remove(confirmed_nodes, node); g_hash_table_remove(finalized_nodes, node); g_hash_table_remove(integrated_nodes, node); g_hash_table_remove(welcomed_nodes, node); check_join_state(fsa_state, __FUNCTION__); } } trigger_fsa(fsa_source); } -static void +void crmd_ipc_connection_destroy(gpointer user_data) { + GCHSource *source = NULL; crmd_client_t *client = user_data; if(client == NULL) { crm_debug_4("No client to delete"); return; } - - if(client->client_source != NULL) { - crm_debug_4("Deleting %s (%p) from mainloop", - client->uuid, client->client_source); - G_main_del_IPC_Channel(client->client_source); - client->client_source = NULL; - } - crm_debug_3("Freeing %s client", client->uuid); - crm_free(client->table_key); - crm_free(client->sub_sys); - crm_free(client->uuid); - crm_free(client); + source = client->client_source; + client->client_source = NULL; + crm_debug_3("Processing %s (%p)", client->uuid, source); + if(source != NULL) { + crm_debug_2("Deleting %s (%p) from mainloop", + client->uuid, source); + /* this will re-call crmd_ipc_connection_destroy() */ + G_main_del_IPC_Channel(source); + + } else { + crm_debug_2("Freeing %s client", client->uuid); + crm_free(client->table_key); + crm_free(client->sub_sys); + crm_free(client->uuid); + crm_free(client); + } return; } gboolean crmd_client_connect(IPC_Channel *client_channel, gpointer user_data) { crm_debug_3("Invoked"); if (client_channel == NULL) { crm_err("Channel was NULL"); } else if (client_channel->ch_status == IPC_DISCONNECT) { crm_err("Channel was disconnected"); } else { crmd_client_t *blank_client = NULL; crm_debug_3("Channel connected"); crm_malloc0(blank_client, sizeof(crmd_client_t)); if (blank_client == NULL) { return FALSE; } client_channel->ops->set_recv_qlen(client_channel, 100); client_channel->ops->set_send_qlen(client_channel, 100); blank_client->client_channel = client_channel; blank_client->sub_sys = NULL; blank_client->uuid = NULL; blank_client->table_key = NULL; blank_client->client_source = G_main_add_IPC_Channel( G_PRIORITY_LOW, client_channel, FALSE, crmd_ipc_msg_callback, blank_client, crmd_ipc_connection_destroy); } return TRUE; } gboolean ccm_dispatch(int fd, gpointer user_data) { int rc = 0; oc_ev_t *ccm_token = (oc_ev_t*)user_data; gboolean was_error = FALSE; crm_debug_3("Invoked"); rc = oc_ev_handle_event(ccm_token); if(rc != 0) { if(is_set(fsa_input_register, R_CCM_DISCONNECTED) == FALSE) { /* we signed out, so this is expected */ register_fsa_input(C_CCM_CALLBACK, I_ERROR, NULL); crm_err("CCM connection appears to have failed: rc=%d.", rc); } was_error = TRUE; } trigger_fsa(fsa_source); return !was_error; } static gboolean fsa_have_quorum = FALSE; void crmd_ccm_msg_callback( oc_ed_t event, void *cookie, size_t size, const void *data) { int instance = -1; gboolean update_cache = FALSE; struct crmd_ccm_data_s *event_data = NULL; const oc_ev_membership_t *membership = data; gboolean update_quorum = FALSE; gboolean trigger_transition = FALSE; crm_debug_3("Invoked"); if(data != NULL) { instance = membership->m_instance; } crm_info("Quorum %s after event=%s (id=%d)", ccm_have_quorum(event)?"(re)attained":"lost", ccm_event_name(event), instance); /* * OC_EV_MS_NEW_MEMBERSHIP: membership with quorum * OC_EV_MS_MS_INVALID: membership without quorum * OC_EV_MS_NOT_PRIMARY: previous membership no longer valid * OC_EV_MS_PRIMARY_RESTORED: previous membership restored * OC_EV_MS_EVICTED: the client is evicted from ccm. */ switch(event) { case OC_EV_MS_NEW_MEMBERSHIP: case OC_EV_MS_INVALID: update_cache = TRUE; update_quorum = TRUE; break; case OC_EV_MS_NOT_PRIMARY: #if UNTESTED if(AM_I_DC == FALSE) { break; } /* tell the TE to pretend it had completed and stop */ /* side effect: we'll end up in S_IDLE */ register_fsa_action(A_TE_HALT, TRUE); #endif break; case OC_EV_MS_PRIMARY_RESTORED: fsa_membership_copy->id = instance; if(AM_I_DC && need_transition(fsa_state)) { trigger_transition = TRUE; } break; case OC_EV_MS_EVICTED: update_quorum = TRUE; register_fsa_input(C_FSA_INTERNAL, I_STOP, NULL); crm_err("Shutting down after CCM event: %s", ccm_event_name(event)); break; default: crm_err("Unknown CCM event: %d", event); } if(update_quorum && ccm_have_quorum(event) == FALSE) { /* did we just loose quorum? */ if(fsa_have_quorum && need_transition(fsa_state)) { crm_info("Quorum lost: triggering transition (%s)", ccm_event_name(event)); trigger_transition = TRUE; } fsa_have_quorum = FALSE; } else if(update_quorum) { crm_debug_2("Updating quorum after event %s", ccm_event_name(event)); fsa_have_quorum = TRUE; } if(trigger_transition) { crm_debug_2("Scheduling transition after event %s", ccm_event_name(event)); /* make sure that when we query the CIB that it has * the changes that triggered the transition */ switch(event) { case OC_EV_MS_NEW_MEMBERSHIP: case OC_EV_MS_INVALID: case OC_EV_MS_PRIMARY_RESTORED: fsa_membership_copy->id = instance; break; default: break; } if(update_cache == FALSE) { /* a stand-alone transition */ register_fsa_action(A_TE_CANCEL); } } if(update_cache) { crm_debug_2("Updating cache after event %s", ccm_event_name(event)); crm_malloc0(event_data, sizeof(struct crmd_ccm_data_s)); if(event_data == NULL) { return; } event_data->event = event; if(data != NULL) { event_data->oc = copy_ccm_oc_data(data); } register_fsa_input_adv( C_CCM_CALLBACK, I_CCM_EVENT, event_data, trigger_transition?A_TE_CANCEL:A_NOTHING, TRUE, __FUNCTION__); if (event_data->oc) { crm_free(event_data->oc); event_data->oc = NULL; } crm_free(event_data); } oc_ev_callback_done(cookie); return; } void crmd_cib_connection_destroy(gpointer user_data) { crm_debug_3("Invoked"); trigger_fsa(fsa_source); if(is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) { crm_info("Connection to the CIB terminated..."); return; } /* eventually this will trigger a reconnect, not a shutdown */ crm_err("Connection to the CIB terminated..."); register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL); clear_bit_inplace(fsa_input_register, R_CIB_CONNECTED); return; } longclock_t fsa_start = 0; longclock_t fsa_stop = 0; longclock_t fsa_diff = 0; gboolean crm_fsa_trigger(gpointer user_data) { unsigned int fsa_diff_ms = 0; if(fsa_diff_max_ms > 0) { fsa_start = time_longclock(); } crm_debug_2("Invoked (queue len: %d)", g_list_length(fsa_message_queue)); s_crmd_fsa(C_FSA_INTERNAL); crm_debug_2("Exited (queue len: %d)", g_list_length(fsa_message_queue)); if(fsa_diff_max_ms > 0) { fsa_stop = time_longclock(); fsa_diff = sub_longclock(fsa_stop, fsa_start); fsa_diff_ms = longclockto_ms(fsa_diff); if(fsa_diff_ms > fsa_diff_max_ms) { crm_err("FSA took %dms to complete", fsa_diff_ms); } else if(fsa_diff_ms > fsa_diff_warn_ms) { crm_warn("FSA took %dms to complete", fsa_diff_ms); } } return TRUE; } diff --git a/crm/crmd/ccm.c b/crm/crmd/ccm.c index 224e82fcea..b63f5f504e 100644 --- a/crm/crmd/ccm.c +++ b/crm/crmd/ccm.c @@ -1,701 +1,676 @@ /* $Id: ccm.c,v 1.106 2006/07/18 06:17:32 andrew Exp $ */ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ /* put these first so that uuid_t is defined without conflicts */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include void oc_ev_special(const oc_ev_t *, oc_ev_class_t , int ); int register_with_ccm(ll_cluster_t *hb_cluster); void msg_ccm_join(const HA_Message *msg, void *foo); void crmd_ccm_msg_callback(oc_ed_t event, void *cookie, size_t size, const void *data); -gboolean ghash_node_clfree(gpointer key, gpointer value, gpointer user_data); void ghash_update_cib_node(gpointer key, gpointer value, gpointer user_data); #define CCM_EVENT_DETAIL 0 #define CCM_EVENT_DETAIL_PARTIAL 0 oc_ev_t *fsa_ev_token; int num_ccm_register_fails = 0; int max_ccm_register_fails = 30; /* A_CCM_CONNECT */ enum crmd_fsa_input do_ccm_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { int ret; int fsa_ev_fd; gboolean did_fail = FALSE; if(action & A_CCM_DISCONNECT){ set_bit_inplace(fsa_input_register, R_CCM_DISCONNECTED); oc_ev_unregister(fsa_ev_token); } if(action & A_CCM_CONNECT) { crm_debug_3("Registering with CCM"); clear_bit_inplace(fsa_input_register, R_CCM_DISCONNECTED); ret = oc_ev_register(&fsa_ev_token); if (ret != 0) { crm_warn("CCM registration failed"); did_fail = TRUE; } if(did_fail == FALSE) { crm_debug_3("Setting up CCM callbacks"); ret = oc_ev_set_callback(fsa_ev_token, OC_EV_MEMB_CLASS, crmd_ccm_msg_callback, NULL); if (ret != 0) { crm_warn("CCM callback not set"); did_fail = TRUE; } } if(did_fail == FALSE) { oc_ev_special(fsa_ev_token, OC_EV_MEMB_CLASS, 0/*don't care*/); crm_debug_3("Activating CCM token"); ret = oc_ev_activate(fsa_ev_token, &fsa_ev_fd); if (ret != 0){ crm_warn("CCM Activation failed"); did_fail = TRUE; } } if(did_fail) { num_ccm_register_fails++; oc_ev_unregister(fsa_ev_token); if(num_ccm_register_fails < max_ccm_register_fails) { crm_warn("CCM Connection failed" " %d times (%d max)", num_ccm_register_fails, max_ccm_register_fails); crm_timer_start(wait_timer); crmd_fsa_stall(NULL); return I_NULL; } else { crm_err("CCM Activation failed %d (max) times", num_ccm_register_fails); register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL); return I_NULL; } } crm_info("CCM connection established..." " waiting for first callback"); G_main_add_fd(G_PRIORITY_HIGH, fsa_ev_fd, FALSE, ccm_dispatch, fsa_ev_token, default_ipc_connection_destroy); } if(action & ~(A_CCM_CONNECT|A_CCM_DISCONNECT)) { crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__); } return I_NULL; } extern GHashTable *voted; /* A_CCM_EVENT */ enum crmd_fsa_input do_ccm_event(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { enum crmd_fsa_input return_input = I_NULL; oc_ed_t event; const oc_ev_membership_t *oc = NULL; struct crmd_ccm_data_s *ccm_data = fsa_typed_data(fsa_dt_ccm); if(ccm_data == NULL) { crm_err("No data provided to FSA function"); register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL); return I_NULL; } else if(msg_data->fsa_cause != C_CCM_CALLBACK) { crm_err("FSA function called in response to incorect input"); register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL); return I_NULL; } event = ccm_data->event; oc = ccm_data->oc; ccm_event_detail(oc, event); if (OC_EV_MS_EVICTED == event) { /* todo: drop back to S_PENDING instead */ /* get out... NOW! * * go via the error recovery process so that HA will * restart us if required */ register_fsa_error(cause, I_ERROR, msg_data->data); return I_NULL; } return return_input; } static void check_dead_member(const char *uname, GHashTable *members) { CRM_CHECK(uname != NULL, return); if(members != NULL && g_hash_table_lookup(members, uname) != NULL) { crm_err("%s didnt really leave the membership!", uname); return; } if(confirmed_nodes != NULL) { g_hash_table_remove(confirmed_nodes, uname); } if(finalized_nodes != NULL) { g_hash_table_remove(finalized_nodes, uname); } if(integrated_nodes != NULL) { g_hash_table_remove(integrated_nodes, uname); } if(voted != NULL) { g_hash_table_remove(voted, uname); } if(safe_str_eq(fsa_our_uname, uname)) { crm_err("We're not part of the cluster anymore"); } if(AM_I_DC == FALSE && safe_str_eq(uname, fsa_our_dc)) { crm_warn("Our DC node (%s) left the cluster", uname); register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); } } /* A_CCM_UPDATE_CACHE */ /* * Take the opportunity to update the node status in the CIB as well */ enum crmd_fsa_input do_ccm_update_cache(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { enum crmd_fsa_input next_input = I_NULL; unsigned int lpc; int offset; GHashTable *members = NULL; oc_ed_t event; const oc_ev_membership_t *oc = NULL; oc_node_list_t *tmp = NULL, *membership_copy = NULL; struct crmd_ccm_data_s *ccm_data = fsa_typed_data(fsa_dt_ccm); HA_Message *no_op = create_request( CRM_OP_NOOP, NULL, NULL, CRM_SYSTEM_CRMD, AM_I_DC?CRM_SYSTEM_DC:CRM_SYSTEM_CRMD, NULL); if(ccm_data == NULL) { crm_err("No data provided to FSA function"); register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL); send_msg_via_ha(fsa_cluster_conn, no_op); return I_NULL; } event = ccm_data->event; oc = ccm_data->oc; if(fsa_membership_copy != NULL && fsa_membership_copy->id >= oc->m_instance) { crm_debug("Ignoring superceeded CCM event %d (%s).", oc->m_instance, ccm_event_name(event)); return I_NULL; } crm_debug("Updating cache after CCM event %d (%s).", oc->m_instance, ccm_event_name(event)); crm_debug_2("instance=%d, nodes=%d, new=%d, lost=%d n_idx=%d, " "new_idx=%d, old_idx=%d", oc->m_instance, oc->m_n_member, oc->m_n_in, oc->m_n_out, oc->m_memb_idx, oc->m_in_idx, oc->m_out_idx); #define ALAN_DEBUG 1 #ifdef ALAN_DEBUG { /* * Size (Size + 2) / 2 * * 3 (3+2)/2 = 5 / 2 = 2 * 4 (4+2)/2 = 6 / 2 = 3 * 5 (5+2)/2 = 7 / 2 = 3 * 6 (6+2)/2 = 8 / 2 = 4 * 7 (7+2)/2 = 9 / 2 = 4 */ unsigned int clsize = (oc->m_out_idx - oc->m_n_member); unsigned int plsize = (clsize + 2)/2; gboolean plurality = (oc->m_n_member >= plsize); gboolean Q = ccm_have_quorum(event); if(clsize == 2) { if (!Q) { crm_err("2 nodes w/o quorum"); } } else if(Q && !plurality) { crm_err("Quorum w/o plurality (%d/%d nodes)", oc->m_n_member, clsize); } else if(plurality && !Q) { crm_err("Plurality w/o Quorum (%d/%d nodes)", oc->m_n_member, clsize); } else { crm_debug_2("Quorum(%s) and plurality (%d/%d) agree.", Q?"true":"false", oc->m_n_member, clsize); } } #endif crm_malloc0(membership_copy, sizeof(oc_node_list_t)); if(membership_copy == NULL) { crm_crit("Couldnt create membership copy - out of memory"); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); return I_NULL; } membership_copy->id = oc->m_instance; membership_copy->last_event = event; crm_debug_3("Copying members"); /*--*-- All Member Nodes --*--*/ offset = oc->m_memb_idx; membership_copy->members_size = oc->m_n_member; if(membership_copy->members_size > 0) { membership_copy->members = g_hash_table_new(g_str_hash, g_str_equal); members = membership_copy->members; for(lpc=0; lpc < membership_copy->members_size; lpc++) { oc_node_t *member = NULL; crm_debug_3("Copying member %d", lpc); crm_malloc0(member, sizeof(oc_node_t)); if(member == NULL) { continue; } member->node_id = oc->m_array[offset+lpc].node_id; member->node_born_on = oc->m_array[offset+lpc].node_born_on; member->node_uname = NULL; if(oc->m_array[offset+lpc].node_uname != NULL) { member->node_uname = crm_strdup(oc->m_array[offset+lpc].node_uname); } else { crm_err("Node %d had a NULL uname", member->node_id); } g_hash_table_insert( members, member->node_uname, member); } } else { membership_copy->members = NULL; } crm_debug_3("Copying new members"); /*--*-- New Member Nodes --*--*/ offset = oc->m_in_idx; membership_copy->new_members_size = oc->m_n_in; if(membership_copy->new_members_size > 0) { membership_copy->new_members = g_hash_table_new(g_str_hash, g_str_equal); members = membership_copy->new_members; for(lpc=0; lpc < membership_copy->new_members_size; lpc++) { oc_node_t *member = NULL; crm_malloc0(member, sizeof(oc_node_t)); if(member == NULL) { continue; } member->node_uname = NULL; member->node_id = oc->m_array[offset+lpc].node_id; member->node_born_on = oc->m_array[offset+lpc].node_born_on; if(oc->m_array[offset+lpc].node_uname != NULL) { member->node_uname = crm_strdup(oc->m_array[offset+lpc].node_uname); } else { crm_err("Node %d had a NULL uname", member->node_id); } g_hash_table_insert( members, member->node_uname, member); g_hash_table_insert(members, member->node_uname, member); } } else { membership_copy->new_members = NULL; } crm_debug_3("Copying dead members"); /*--*-- Recently Dead Member Nodes --*--*/ offset = oc->m_out_idx; membership_copy->dead_members_size = oc->m_n_out; if(membership_copy->dead_members_size > 0) { membership_copy->dead_members = g_hash_table_new(g_str_hash, g_str_equal); members = membership_copy->dead_members; for(lpc=0; lpc < membership_copy->dead_members_size; lpc++) { oc_node_t *member = NULL; crm_malloc0(member, sizeof(oc_node_t)); if(member == NULL) { continue; } member->node_id = oc->m_array[offset+lpc].node_id; member->node_born_on = oc->m_array[offset+lpc].node_born_on; member->node_uname = NULL; CRM_DEV_ASSERT(oc->m_array[offset+lpc].node_uname != NULL); if(oc->m_array[offset+lpc].node_uname == NULL) { continue; } member->node_uname = crm_strdup(oc->m_array[offset+lpc].node_uname); g_hash_table_insert(members, member->node_uname, member); check_dead_member( member->node_uname, membership_copy->members); } } else { membership_copy->dead_members = NULL; } tmp = fsa_membership_copy; fsa_membership_copy = membership_copy; crm_debug_2("Updated membership cache with %d (%d new, %d lost) members", g_hash_table_size(fsa_membership_copy->members), g_hash_table_size(fsa_membership_copy->new_members), g_hash_table_size(fsa_membership_copy->dead_members)); - - /* Free the old copy */ - if(tmp != NULL) { - if(tmp->members != NULL) - g_hash_table_foreach_remove( - tmp->members, ghash_node_clfree, NULL); - if(tmp->new_members != NULL) - g_hash_table_foreach_remove( - tmp->new_members, ghash_node_clfree, NULL); - if(tmp->dead_members != NULL) - g_hash_table_foreach_remove( - tmp->dead_members, ghash_node_clfree, NULL); - crm_free(tmp); - } - crm_debug_3("Free'd old copies"); + free_ccm_cache(tmp); + set_bit_inplace(fsa_input_register, R_CCM_DATA); if(cur_state != S_STOPPING) { crm_debug_3("Updating the CIB from CCM cache"); do_update_cib_nodes(FALSE, __FUNCTION__); } /* Membership changed, remind everyone we're here. * This will aid detection of duplicate DCs */ send_msg_via_ha(fsa_cluster_conn, no_op); return next_input; } void ccm_event_detail(const oc_ev_membership_t *oc, oc_ed_t event) { int lpc; gboolean member = FALSE; member = FALSE; crm_debug_2("-----------------------"); crm_info("%s: trans=%d, nodes=%d, new=%d, lost=%d n_idx=%d, " "new_idx=%d, old_idx=%d", ccm_event_name(event), oc->m_instance, oc->m_n_member, oc->m_n_in, oc->m_n_out, oc->m_memb_idx, oc->m_in_idx, oc->m_out_idx); #if !CCM_EVENT_DETAIL_PARTIAL for(lpc=0; lpc < oc->m_n_member; lpc++) { crm_info("\tCURRENT: %s [nodeid=%d, born=%d]", oc->m_array[oc->m_memb_idx+lpc].node_uname, oc->m_array[oc->m_memb_idx+lpc].node_id, oc->m_array[oc->m_memb_idx+lpc].node_born_on); if(safe_str_eq(fsa_our_uname, oc->m_array[oc->m_memb_idx+lpc].node_uname)) { member = TRUE; } } if (member == FALSE) { crm_warn("MY NODE IS NOT IN CCM THE MEMBERSHIP LIST"); } #endif for(lpc=0; lpc<(int)oc->m_n_in; lpc++) { crm_info("\tNEW: %s [nodeid=%d, born=%d]", oc->m_array[oc->m_in_idx+lpc].node_uname, oc->m_array[oc->m_in_idx+lpc].node_id, oc->m_array[oc->m_in_idx+lpc].node_born_on); } for(lpc=0; lpc<(int)oc->m_n_out; lpc++) { crm_info("\tLOST: %s [nodeid=%d, born=%d]", oc->m_array[oc->m_out_idx+lpc].node_uname, oc->m_array[oc->m_out_idx+lpc].node_id, oc->m_array[oc->m_out_idx+lpc].node_born_on); } crm_debug_2("-----------------------"); } int register_with_ccm(ll_cluster_t *hb_cluster) { return 0; } void msg_ccm_join(const HA_Message *msg, void *foo) { crm_debug_2("###### Received ccm_join message..."); if (msg != NULL) { crm_debug_2("[type=%s]", ha_msg_value(msg, F_TYPE)); crm_debug_2("[orig=%s]", ha_msg_value(msg, F_ORIG)); crm_debug_2("[to=%s]", ha_msg_value(msg, F_TO)); crm_debug_2("[status=%s]", ha_msg_value(msg, F_STATUS)); crm_debug_2("[info=%s]", ha_msg_value(msg, F_COMMENT)); crm_debug_2("[rsc_hold=%s]", ha_msg_value(msg, F_RESOURCES)); crm_debug_2("[stable=%s]", ha_msg_value(msg, F_ISSTABLE)); crm_debug_2("[rtype=%s]", ha_msg_value(msg, F_RTYPE)); crm_debug_2("[ts=%s]", ha_msg_value(msg, F_TIME)); crm_debug_2("[seq=%s]", ha_msg_value(msg, F_SEQ)); crm_debug_2("[generation=%s]", ha_msg_value(msg, F_HBGENERATION)); /* crm_debug_2("[=%s]", ha_msg_value(msg, F_)); */ } return; } struct update_data_s { const char *state; const char *caller; crm_data_t *updates; gboolean overwrite_join; }; static void ccm_node_update_complete(const HA_Message *msg, int call_id, int rc, crm_data_t *output, void *user_data) { fsa_data_t *msg_data = NULL; if(rc == cib_ok) { crm_debug("Node update %d complete", call_id); } else { crm_err("Node update %d failed", call_id); crm_log_message(LOG_DEBUG, msg); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } void do_update_cib_nodes(gboolean overwrite, const char *caller) { int call_id = 0; int call_options = cib_scope_local|cib_quorum_override; struct update_data_s update_data; crm_data_t *fragment = NULL; if(fsa_membership_copy == NULL) { /* We got a replace notification before being connected to * the CCM. * So there is no need to update the local CIB with our values * - since we have none. */ return; } fragment = create_xml_node(NULL, XML_CIB_TAG_STATUS); update_data.caller = caller; update_data.updates = fragment; update_data.overwrite_join = overwrite; if(overwrite == FALSE) { call_options = call_options|cib_inhibit_bcast; crm_debug_2("Inhibiting bcast for membership updates"); } /* dead nodes */ update_data.state = XML_BOOLEAN_NO; if(fsa_membership_copy->dead_members != NULL) { g_hash_table_foreach(fsa_membership_copy->dead_members, ghash_update_cib_node, &update_data); } /* live nodes */ update_data.state = XML_BOOLEAN_YES; if(fsa_membership_copy->members != NULL) { g_hash_table_foreach(fsa_membership_copy->members, ghash_update_cib_node, &update_data); } fsa_cib_update(XML_CIB_TAG_STATUS, fragment, call_options, call_id); add_cib_op_callback(call_id, FALSE, NULL, ccm_node_update_complete); free_xml(fragment); } void ghash_update_cib_node(gpointer key, gpointer value, gpointer user_data) { crm_data_t *tmp1 = NULL; const char *join = NULL; const char *peer_online = NULL; const char *node_uname = (const char*)key; struct update_data_s* data = (struct update_data_s*)user_data; crm_debug_2("Updating %s: %s (overwrite=%s)", node_uname, data->state, data->overwrite_join?"true":"false"); peer_online = g_hash_table_lookup(crmd_peer_state, node_uname); if(data->overwrite_join) { if(safe_str_neq(peer_online, ONLINESTATUS)) { join = CRMD_JOINSTATE_DOWN; } else { const char *peer_member = g_hash_table_lookup( confirmed_nodes, node_uname); if(peer_member != NULL) { join = CRMD_JOINSTATE_MEMBER; } else { join = CRMD_JOINSTATE_PENDING; } } } tmp1 = create_node_state(node_uname, NULL, data->state, peer_online, join, NULL, FALSE, data->caller); add_node_copy(data->updates, tmp1); free_xml(tmp1); } -gboolean -ghash_node_clfree(gpointer key, gpointer value, gpointer user_data) -{ - /* value->node_uname is free'd as "key" */ - if(key != NULL) { - crm_free(key); - } - if(value != NULL) { - crm_free(value); - } - return TRUE; -} + diff --git a/crm/crmd/cib.c b/crm/crmd/cib.c index 992cf6ec47..da4c329dc8 100644 --- a/crm/crmd/cib.c +++ b/crm/crmd/cib.c @@ -1,382 +1,380 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include /* for access */ #include #include #include /* for calls to open */ #include /* for calls to open */ #include /* for calls to open */ #include /* for getpwuid */ #include /* for initgroups */ #include /* for getrlimit */ #include /* for getrlimit */ #include #include #include #include #include #include #include #include struct crm_subsystem_s *cib_subsystem = NULL; int cib_retries = 0; static void revision_check_callback(const HA_Message *msg, int call_id, int rc, crm_data_t *output, void *user_data) { int cmp = -1; const char *revision = NULL; crm_data_t *generation = NULL; #if CRM_DEPRECATED_SINCE_2_0_4 if(safe_str_eq(crm_element_name(output), XML_TAG_CIB)) { generation = output; } else { generation = find_xml_node(output, XML_TAG_CIB, TRUE); } #else generation = output; CRM_DEV_ASSERT(safe_str_eq(crm_element_name(generation), XML_TAG_CIB)); #endif if(rc != cib_ok) { fsa_data_t *msg_data = NULL; register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); return; } crm_debug_3("Checking our feature revision is allowed: %s", CIB_FEATURE_SET); revision = crm_element_value(generation, XML_ATTR_CIB_REVISION); cmp = compare_version(revision, CIB_FEATURE_SET); if(cmp > 0) { crm_err("This build (%s) does not support the current" " resource configuration", VERSION); crm_err("We can support up to CIB feature set %s (current=%s)", CIB_FEATURE_SET, revision); crm_err("Shutting down the CRM"); /* go into a stall state */ register_fsa_error_adv( C_FSA_INTERNAL, I_SHUTDOWN, NULL, NULL, __FUNCTION__); return; } revision = crm_element_value(generation, XML_ATTR_CRM_VERSION); cmp = compare_version(revision, CRM_FEATURE_SET); if(cmp > 0) { crm_err("This build (%s) does not support the current" " resource configuration", VERSION); crm_err("We can support up to CRM feature set %s (current=%s)", revision, CRM_FEATURE_SET); crm_err("Shutting down the CRM"); /* go into a stall state */ register_fsa_error_adv( C_FSA_INTERNAL, I_SHUTDOWN, NULL, NULL, __FUNCTION__); return; } } extern void populate_cib_nodes( ll_cluster_t *hb_cluster, gboolean with_client_status); static void do_cib_replaced(const char *event, HA_Message *msg) { crm_debug("Updating the CIB after a replace"); populate_cib_nodes(fsa_cluster_conn, FALSE); do_update_cib_nodes(AM_I_DC, __FUNCTION__); if(AM_I_DC) { /* start the join process again so we get everyone's LRM status */ register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); } } /* A_CIB_STOP, A_CIB_START, A_CIB_RESTART, */ enum crmd_fsa_input do_cib_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { enum crmd_fsa_input result = I_NULL; struct crm_subsystem_s *this_subsys = cib_subsystem; long long stop_actions = A_CIB_STOP; long long start_actions = A_CIB_START; if(action & stop_actions) { crm_info("Disconnecting CIB"); clear_bit_inplace(fsa_input_register, R_CIB_CONNECTED); - if(fsa_cib_conn != NULL - && fsa_cib_conn->state != cib_disconnected) { + CRM_ASSERT(fsa_cib_conn != NULL); + if(fsa_cib_conn->state != cib_disconnected) { fsa_cib_conn->cmds->set_slave( fsa_cib_conn, cib_scope_local); fsa_cib_conn->cmds->signoff(fsa_cib_conn); } } if(action & start_actions) { int rc = cib_ok; - if(fsa_cib_conn == NULL) { - fsa_cib_conn = cib_new(); - } + CRM_ASSERT(fsa_cib_conn != NULL); if(cur_state == S_STOPPING) { crm_err("Ignoring request to start %s after shutdown", this_subsys->name); return I_NULL; } rc = fsa_cib_conn->cmds->signon( fsa_cib_conn, CRM_SYSTEM_CRMD, cib_command); if(rc != cib_ok) { /* a short wait that usually avoids stalling the FSA */ sleep(1); rc = fsa_cib_conn->cmds->signon( fsa_cib_conn, CRM_SYSTEM_CRMD, cib_command); } if(rc != cib_ok){ crm_debug("Could not connect to the CIB service"); } else if(cib_ok != fsa_cib_conn->cmds->set_connection_dnotify( fsa_cib_conn, crmd_cib_connection_destroy)) { crm_err("Could not set dnotify callback"); } else if(cib_ok != fsa_cib_conn->cmds->add_notify_callback( fsa_cib_conn, T_CIB_REPLACE_NOTIFY, do_cib_replaced)) { crm_err("Could not set CIB notification callback"); } else { set_bit_inplace( fsa_input_register, R_CIB_CONNECTED); } if(is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) { cib_retries++; crm_warn("Couldn't complete CIB registration %d" " times... pause and retry", cib_retries); if(cib_retries < 30) { crm_timer_start(wait_timer); crmd_fsa_stall(NULL); } else { crm_err("Could not complete CIB" " registration %d times..." " hard error", cib_retries); register_fsa_error( C_FSA_INTERNAL, I_ERROR, NULL); } } else { int call_id = 0; crm_info("CIB connection established"); call_id = fsa_cib_conn->cmds->query( fsa_cib_conn, NULL, NULL, cib_scope_local); add_cib_op_callback(call_id, FALSE, NULL, revision_check_callback); cib_retries = 0; } } return result; } /* A_CIB_INVOKE, A_CIB_BUMPGEN, A_UPDATE_NODESTATUS */ enum crmd_fsa_input do_cib_invoke(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { HA_Message *answer = NULL; enum crmd_fsa_input result = I_NULL; ha_msg_input_t *cib_msg = fsa_typed_data(fsa_dt_ha_msg); const char *sys_from = cl_get_string(cib_msg->msg, F_CRM_SYS_FROM); if(fsa_cib_conn->state == cib_disconnected) { if(cur_state != S_STOPPING) { crm_err("CIB is disconnected"); crm_log_message_adv(LOG_WARNING, "CIB Input", cib_msg->msg); return I_NULL; } crm_info("CIB is disconnected"); crm_log_message_adv(LOG_DEBUG, "CIB Input", cib_msg->msg); return I_NULL; } if(action & A_CIB_INVOKE) { if(safe_str_eq(sys_from, CRM_SYSTEM_CRMD)) { action = A_CIB_INVOKE_LOCAL; } else if(safe_str_eq(sys_from, CRM_SYSTEM_DC)) { action = A_CIB_INVOKE_LOCAL; } } if(action & A_CIB_INVOKE || action & A_CIB_INVOKE_LOCAL) { int call_options = 0; enum cib_errors rc = cib_ok; crm_data_t *cib_frag = NULL; const char *section = NULL; const char *op = cl_get_string(cib_msg->msg, F_CRM_TASK); section = cl_get_string(cib_msg->msg, F_CIB_SECTION); ha_msg_value_int(cib_msg->msg, F_CIB_CALLOPTS, &call_options); crm_log_message(LOG_MSG, cib_msg->msg); crm_log_xml_debug_3(cib_msg->xml, "[CIB update]"); if(op == NULL) { crm_err("Invalid CIB Message"); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); return I_NULL; } cib_frag = NULL; rc = fsa_cib_conn->cmds->variant_op( fsa_cib_conn, op, NULL, section, cib_msg->xml, &cib_frag, call_options); if(rc < cib_ok || (action & A_CIB_INVOKE)) { answer = create_reply(cib_msg->msg, cib_frag); ha_msg_add(answer,XML_ATTR_RESULT,cib_error2string(rc)); } if(action & A_CIB_INVOKE) { if(relay_message(answer, TRUE) == FALSE) { crm_err("Confused what to do with cib result"); crm_log_message(LOG_ERR, answer); crm_msg_del(answer); result = I_ERROR; } } else if(rc < cib_ok) { ha_msg_input_t *input = NULL; crm_err("Internal CRM/CIB command from %s() failed: %s", msg_data->origin, cib_error2string(rc)); crm_log_message_adv(LOG_WARNING, "CIB Input", cib_msg->msg); crm_log_message_adv(LOG_WARNING, "CIB Reply", answer); input = new_ha_msg_input(answer); register_fsa_input(C_FSA_INTERNAL, I_ERROR, input); crm_msg_del(answer); delete_ha_msg_input(input); } return result; } else { crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__); } return I_NULL; } /* frees fragment as part of delete_ha_msg_input() */ void update_local_cib_adv( crm_data_t *msg_data, gboolean do_now, const char *raised_from) { HA_Message *msg = NULL; ha_msg_input_t *fsa_input = NULL; int call_options = cib_quorum_override|cib_scope_local; CRM_DEV_ASSERT(msg_data != NULL); crm_malloc0(fsa_input, sizeof(ha_msg_input_t)); msg = create_request(CIB_OP_UPDATE, msg_data, NULL, CRM_SYSTEM_CIB, CRM_SYSTEM_CRMD, NULL); ha_msg_add(msg, F_CIB_SECTION, crm_element_value(msg_data, XML_ATTR_SECTION)); ha_msg_add_int(msg, F_CIB_CALLOPTS, call_options); ha_msg_add(msg, "call_origin", raised_from); fsa_input->msg = msg; fsa_input->xml = msg_data; if(AM_I_DC && crm_assert_failed) { /* register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); */ } if(do_now == FALSE) { crm_debug_3("Registering event with FSA"); register_fsa_input_adv(C_FSA_INTERNAL, I_CIB_OP, fsa_input, 0, FALSE, raised_from); } else { fsa_data_t *op_data = NULL; crm_debug_3("Invoking CIB handler directly"); crm_malloc0(op_data, sizeof(fsa_data_t)); op_data->fsa_cause = C_FSA_INTERNAL; op_data->fsa_input = I_CIB_OP; op_data->origin = raised_from; op_data->data = fsa_input; op_data->data_type = fsa_dt_ha_msg; do_cib_invoke(A_CIB_INVOKE_LOCAL, C_FSA_INTERNAL, fsa_state, I_CIB_OP, op_data); crm_free(op_data); crm_debug_3("CIB handler completed"); } crm_debug_3("deleting input"); crm_msg_del(fsa_input->msg); free_xml(fsa_input->xml); crm_free(fsa_input); crm_debug_3("deleted input"); } diff --git a/crm/crmd/control.c b/crm/crmd/control.c index 61f4c52947..b81d6f1615 100644 --- a/crm/crmd/control.c +++ b/crm/crmd/control.c @@ -1,803 +1,872 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +char *ipc_server = NULL; + extern void crmd_ha_connection_destroy(gpointer user_data); extern gboolean verify_stopped(gboolean force, int log_level); gboolean crm_shutdown(int nsig, gpointer unused); gboolean register_with_ha(ll_cluster_t *hb_cluster, const char *client_name); void populate_cib_nodes(ll_cluster_t *hb_cluster, gboolean with_client_status); GHashTable *ipc_clients = NULL; GTRIGSource *fsa_source = NULL; /* A_HA_CONNECT */ enum crmd_fsa_input do_ha_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { gboolean registered = FALSE; if(action & A_HA_DISCONNECT) { if(fsa_cluster_conn != NULL) { set_bit_inplace(fsa_input_register, R_HA_DISCONNECTED); fsa_cluster_conn->llc_ops->signoff( fsa_cluster_conn, FALSE); fsa_cluster_conn->llc_ops->delete(fsa_cluster_conn); fsa_cluster_conn = NULL; } crm_info("Disconnected from Heartbeat"); } if(action & A_HA_CONNECT) { if(fsa_cluster_conn == NULL) { fsa_cluster_conn = ll_cluster_new("heartbeat"); } /* make sure we are disconnected first */ fsa_cluster_conn->llc_ops->signoff(fsa_cluster_conn, FALSE); registered = register_with_ha( fsa_cluster_conn, crm_system_name); if(registered == FALSE) { register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL); return I_NULL; } clear_bit_inplace(fsa_input_register, R_HA_DISCONNECTED); crm_info("Connected to Heartbeat"); } if(action & ~(A_HA_CONNECT|A_HA_DISCONNECT)) { crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__); } return I_NULL; } /* A_SHUTDOWN */ enum crmd_fsa_input do_shutdown(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { int lpc = 0; gboolean continue_shutdown = TRUE; struct crm_subsystem_s *subsystems[] = { pe_subsystem, te_subsystem }; /* just in case */ set_bit_inplace(fsa_input_register, R_SHUTDOWN); for(lpc = 0; lpc < DIMOF(subsystems); lpc++) { struct crm_subsystem_s *a_subsystem = subsystems[lpc]; if(is_set(fsa_input_register, a_subsystem->flag_connected)) { crm_info("Terminating the %s", a_subsystem->name); if(stop_subsystem(a_subsystem, TRUE) == FALSE) { /* its gone... */ crm_err("Faking %s exit", a_subsystem->name); clear_bit_inplace(fsa_input_register, a_subsystem->flag_connected); } continue_shutdown = FALSE; } } if(continue_shutdown == FALSE) { crm_info("Waiting for subsystems to exit"); crmd_fsa_stall(NULL); } else { register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL); } return I_NULL; } /* A_SHUTDOWN_REQ */ enum crmd_fsa_input do_shutdown_req(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { HA_Message *msg = NULL; crm_info("Sending shutdown request to DC: %s", crm_str(fsa_our_dc)); msg = create_request( CRM_OP_SHUTDOWN_REQ, NULL, NULL, CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL); /* set_bit_inplace(fsa_input_register, R_STAYDOWN); */ if(send_request(msg, NULL) == FALSE) { if(AM_I_DC) { crm_info("Processing shutdown locally"); } else { register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } return I_NULL; } +extern char *max_generation_from; +extern crm_data_t *max_generation_xml; + +static void free_mem(void) +{ + int lpc = 0; + crm_debug("Stage %d", lpc++); + empty_uuid_cache(); + + crm_debug("Stage %d", lpc++); + free_ccm_cache(fsa_membership_copy); + + crm_free(cib_subsystem); + + crm_debug("Stage %d", lpc++); + crmd_ipc_connection_destroy(te_subsystem->client); + crm_free(te_subsystem); + + crm_debug("Stage %d", lpc++); + crmd_ipc_connection_destroy(pe_subsystem->client); + crm_free(pe_subsystem); + + crm_debug("Stage %d", lpc++); + if(integrated_nodes) { + g_hash_table_destroy(integrated_nodes); + } + if(finalized_nodes) { + g_hash_table_destroy(finalized_nodes); + } + if(confirmed_nodes) { + g_hash_table_destroy(confirmed_nodes); + } + + crm_debug("Stage %d", lpc++); + if(ipc_clients) { + g_hash_table_destroy(ipc_clients); + } + + crm_debug("Stage %d", lpc++); + cib_delete(fsa_cib_conn); + fsa_cib_conn = NULL; + + crm_debug("Stage %d", lpc++); + crm_free(integration_timer); + crm_free(finalization_timer); + crm_free(election_trigger); + crm_free(election_timeout); + crm_free(shutdown_escalation_timer); + crm_free(wait_timer); + crm_free(recheck_timer); + + crm_debug("Stage %d", lpc++); + crm_free(fsa_our_dc_version); + crm_free(fsa_our_uuid); + crm_free(fsa_our_dc); + crm_free(ipc_server); + + crm_free(max_generation_from); + free_xml(max_generation_xml); +} + /* A_EXIT_0, A_EXIT_1 */ enum crmd_fsa_input do_exit(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { int exit_code = 0; int log_level = LOG_INFO; const char *exit_type = "gracefully"; if(action & A_EXIT_1) { exit_code = 1; log_level = LOG_ERR; exit_type = "forcefully"; verify_stopped(TRUE, LOG_ERR); } do_crm_log(log_level, "Performing %s - %s exiting the CRMd", fsa_action2string(action), exit_type); if(is_set(fsa_input_register, R_IN_RECOVERY)) { crm_err("Could not recover from internal error"); exit_code = 2; } if(is_set(fsa_input_register, R_STAYDOWN)) { crm_warn("Inhibiting respawn by Heartbeat"); exit_code = 100; } + + free_mem(); crm_info("[%s] stopped (%d)", crm_system_name, exit_code); +#ifdef HA_MALLOC_TRACK + cl_malloc_dump_allocated(LOG_ERR, FALSE); +#endif cl_flush_logs(); exit(exit_code); return I_NULL; } - /* A_STARTUP */ enum crmd_fsa_input do_startup(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { int was_error = 0; int interval = 1; /* seconds between DC heartbeats */ crm_debug("Registering Signal Handlers"); G_main_add_SignalHandler( G_PRIORITY_HIGH, SIGTERM, crm_shutdown, NULL, NULL); fsa_source = G_main_add_TriggerHandler( G_PRIORITY_HIGH, crm_fsa_trigger, NULL, NULL); ipc_clients = g_hash_table_new(g_str_hash, g_str_equal); + crm_debug("Creating CIB object"); + fsa_cib_conn = cib_new(); + if(was_error == 0) { crm_debug("Init server comms"); + if(ipc_server == NULL) { + ipc_server = crm_strdup(CRM_SYSTEM_CRMD); + } was_error = init_server_ipc_comms( - crm_strdup(CRM_SYSTEM_CRMD), crmd_client_connect, + ipc_server, crmd_client_connect, default_ipc_connection_destroy); } - if(was_error == 0) { - crm_debug("Creating CIB object"); - fsa_cib_conn = cib_new(); - } - /* set up the timers */ crm_malloc0(integration_timer, sizeof(fsa_timer_t)); crm_malloc0(finalization_timer, sizeof(fsa_timer_t)); crm_malloc0(election_trigger, sizeof(fsa_timer_t)); crm_malloc0(election_timeout, sizeof(fsa_timer_t)); crm_malloc0(shutdown_escalation_timer, sizeof(fsa_timer_t)); crm_malloc0(wait_timer, sizeof(fsa_timer_t)); crm_malloc0(recheck_timer, sizeof(fsa_timer_t)); interval = interval * 1000; if(election_trigger != NULL) { election_trigger->source_id = 0; election_trigger->period_ms = -1; election_trigger->fsa_input = I_DC_TIMEOUT; election_trigger->callback = crm_timer_popped; election_trigger->repeat = FALSE; } else { was_error = TRUE; } if(election_timeout != NULL) { election_timeout->source_id = 0; election_timeout->period_ms = -1; election_timeout->fsa_input = I_ELECTION_DC; election_timeout->callback = crm_timer_popped; election_timeout->repeat = FALSE; } else { was_error = TRUE; } if(integration_timer != NULL) { integration_timer->source_id = 0; integration_timer->period_ms = -1; integration_timer->fsa_input = I_INTEGRATED; integration_timer->callback = crm_timer_popped; integration_timer->repeat = FALSE; } else { was_error = TRUE; } if(finalization_timer != NULL) { finalization_timer->source_id = 0; finalization_timer->period_ms = -1; finalization_timer->fsa_input = I_FINALIZED; finalization_timer->callback = crm_timer_popped; finalization_timer->repeat = FALSE; /* for possible enabling... a bug in the join protocol left * a slave in S_PENDING while we think its in S_NOT_DC * * raising I_FINALIZED put us into a transition loop which is * never resolved. * in this loop we continually send probes which the node * NACK's because its in S_PENDING * * if we have nodes where heartbeat is active but the * CRM is not... then this will be handled in the * integration phase */ finalization_timer->fsa_input = I_ELECTION; } else { was_error = TRUE; } if(shutdown_escalation_timer != NULL) { shutdown_escalation_timer->source_id = 0; shutdown_escalation_timer->period_ms = -1; shutdown_escalation_timer->fsa_input = I_STOP; shutdown_escalation_timer->callback = crm_timer_popped; shutdown_escalation_timer->repeat = FALSE; } else { was_error = TRUE; } if(wait_timer != NULL) { wait_timer->source_id = 0; wait_timer->period_ms = 500; wait_timer->fsa_input = I_NULL; wait_timer->callback = crm_timer_popped; wait_timer->repeat = FALSE; } else { was_error = TRUE; } if(recheck_timer != NULL) { recheck_timer->source_id = 0; recheck_timer->period_ms = -1; recheck_timer->fsa_input = I_PE_CALC; recheck_timer->callback = crm_timer_popped; recheck_timer->repeat = FALSE; } else { was_error = TRUE; } /* set up the sub systems */ crm_malloc0(cib_subsystem, sizeof(struct crm_subsystem_s)); crm_malloc0(te_subsystem, sizeof(struct crm_subsystem_s)); crm_malloc0(pe_subsystem, sizeof(struct crm_subsystem_s)); if(cib_subsystem != NULL) { cib_subsystem->pid = -1; cib_subsystem->path = BIN_DIR; cib_subsystem->name = CRM_SYSTEM_CIB; cib_subsystem->command = BIN_DIR"/"CRM_SYSTEM_CIB; cib_subsystem->args = "-VVc"; cib_subsystem->flag_connected = R_CIB_CONNECTED; cib_subsystem->flag_required = R_CIB_REQUIRED; } else { was_error = TRUE; } if(te_subsystem != NULL) { te_subsystem->pid = -1; te_subsystem->path = BIN_DIR; te_subsystem->name = CRM_SYSTEM_TENGINE; te_subsystem->command = BIN_DIR"/"CRM_SYSTEM_TENGINE; te_subsystem->args = NULL; te_subsystem->flag_connected = R_TE_CONNECTED; te_subsystem->flag_required = R_TE_REQUIRED; } else { was_error = TRUE; } if(pe_subsystem != NULL) { pe_subsystem->pid = -1; pe_subsystem->path = BIN_DIR; pe_subsystem->name = CRM_SYSTEM_PENGINE; pe_subsystem->command = BIN_DIR"/"CRM_SYSTEM_PENGINE; pe_subsystem->args = NULL; pe_subsystem->flag_connected = R_PE_CONNECTED; pe_subsystem->flag_required = R_PE_REQUIRED; } else { was_error = TRUE; } if(was_error) { register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } welcomed_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); integrated_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); finalized_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); confirmed_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); crmd_peer_state = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); set_sigchld_proctrack(G_PRIORITY_HIGH); return I_NULL; } /* A_STOP */ enum crmd_fsa_input do_stop(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { if(verify_stopped(FALSE, LOG_DEBUG) == FALSE) { crmd_fsa_stall(NULL); } return I_NULL; } /* A_STARTED */ enum crmd_fsa_input do_started(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { if(is_set(fsa_input_register, R_CCM_DATA) == FALSE) { crm_info("Delaying start, CCM (%.16llx) not connected", R_CCM_DATA); crmd_fsa_stall(NULL); return I_NULL; } else if(is_set(fsa_input_register, R_LRM_CONNECTED) == FALSE) { crm_info("Delaying start, LRM (%.16llx) not connected", R_LRM_CONNECTED); crmd_fsa_stall(NULL); return I_NULL; } else if(is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) { crm_info("Delaying start, CIB (%.16llx) not connected", R_CIB_CONNECTED); crmd_fsa_stall(NULL); return I_NULL; } else if(is_set(fsa_input_register, R_READ_CONFIG) == FALSE) { crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG); crmd_fsa_stall(NULL); return I_NULL; } else if(is_set(fsa_input_register, R_PEER_DATA) == FALSE) { HA_Message * msg = NULL; /* try reading from HA */ crm_info("Delaying start, Peer data (%.16llx) not recieved", R_PEER_DATA); crm_debug_3("Looking for a HA message"); msg = fsa_cluster_conn->llc_ops->readmsg(fsa_cluster_conn, 0); if(msg != NULL) { crm_debug_3("There was a HA message"); crm_msg_del(msg); } crm_timer_start(wait_timer); crmd_fsa_stall(NULL); return I_NULL; } crm_info("The local CRM is operational"); clear_bit_inplace(fsa_input_register, R_STARTING); register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL); return I_NULL; } /* A_RECOVER */ enum crmd_fsa_input do_recover(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { set_bit_inplace(fsa_input_register, R_IN_RECOVERY); crm_err("Action %s (%.16llx) not supported", fsa_action2string(action), action); register_fsa_input(C_FSA_INTERNAL, I_STOP, NULL); return I_NULL; } pe_cluster_option crmd_opts[] = { /* name, old-name, validate, default, description */ { XML_CONFIG_ATTR_DC_DEADTIME, NULL, "time", NULL, "10s", &check_time, "How long to wait for a response from other nodes during startup.", "The \"correct\" value will depend on the speed and load of your network." }, { XML_CONFIG_ATTR_RECHECK, NULL, "time", "Zero disables polling. Positive values are an interval in seconds (unless other SI units are specified. eg. 5min)", "0", &check_timer, "Polling interval for time based changes to options, resource parameters and constraints.", "The Cluster is primarily event driven, however the configuration can have elements that change based on time. To ensure these changes take effect, we can optionally poll the cluster's status for changes." }, { XML_CONFIG_ATTR_ELECTION_FAIL, NULL, "time", NULL, "2min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." }, { XML_CONFIG_ATTR_FORCE_QUIT, NULL, "time", NULL, "20min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." }, { "crmd-integration-timeout", NULL, "time", NULL, "3min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." }, { "crmd-finalization-timeout", NULL, "time", NULL, "10min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." }, }; void crmd_metadata(void) { config_metadata("CRM Daemon", "1.0", "CRM Daemon Options", "This is a fake resource that details the options that can be configured for the CRM Daemon.", crmd_opts, DIMOF(crmd_opts)); } static void verify_crmd_options(GHashTable *options) { verify_all_options(options, crmd_opts, DIMOF(crmd_opts)); } static const char * crmd_pref(GHashTable *options, const char *name) { return get_cluster_pref(options, crmd_opts, DIMOF(crmd_opts), name); } static void config_query_callback(const HA_Message *msg, int call_id, int rc, crm_data_t *output, void *user_data) { const char *value = NULL; GHashTable *config_hash = NULL; if(rc != cib_ok) { fsa_data_t *msg_data = NULL; crm_err("Local CIB query resulted in an error: %s", cib_error2string(rc)); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); if(rc == cib_bad_permissions || rc == cib_bad_digest || rc == cib_bad_config) { crm_err("The cluster is mis-configured - shutting down and staying down"); set_bit_inplace(fsa_input_register, R_STAYDOWN); } return; } crm_debug("Call %d : Parsing CIB options", call_id); config_hash = g_hash_table_new_full( g_str_hash,g_str_equal, g_hash_destroy_str,g_hash_destroy_str); unpack_instance_attributes( output, XML_CIB_TAG_PROPSET, NULL, config_hash, CIB_OPTIONS_FIRST, NULL); value = g_hash_table_lookup(config_hash, XML_CONFIG_ATTR_DC_DEADTIME); if(value == NULL) { /* apparently we're not allowed to free the result of getenv */ char *param_val = getenv(ENV_PREFIX "" KEY_INITDEAD); value = crmd_pref(config_hash, XML_CONFIG_ATTR_DC_DEADTIME); if(param_val != NULL) { int from_env = crm_get_msec(param_val) / 2; int from_defaults = crm_get_msec(value); if(from_env > from_defaults) { g_hash_table_replace( config_hash, crm_strdup(XML_CONFIG_ATTR_DC_DEADTIME), crm_strdup(param_val)); } } } verify_crmd_options(config_hash); value = crmd_pref(config_hash, XML_CONFIG_ATTR_DC_DEADTIME); election_trigger->period_ms = crm_get_msec(value); value = crmd_pref(config_hash, XML_CONFIG_ATTR_FORCE_QUIT); shutdown_escalation_timer->period_ms = crm_get_msec(value); value = crmd_pref(config_hash, XML_CONFIG_ATTR_ELECTION_FAIL); election_timeout->period_ms = crm_get_msec(value); value = crmd_pref(config_hash, XML_CONFIG_ATTR_RECHECK); recheck_timer->period_ms = crm_get_msec(value); value = crmd_pref(config_hash, "crmd-integration-timeout"); integration_timer->period_ms = crm_get_msec(value); value = crmd_pref(config_hash, "crmd-finalization-timeout"); finalization_timer->period_ms = crm_get_msec(value); set_bit_inplace(fsa_input_register, R_READ_CONFIG); crm_debug_3("Triggering FSA: %s", __FUNCTION__); G_main_set_trigger(fsa_source); g_hash_table_destroy(config_hash); } /* A_READCONFIG */ enum crmd_fsa_input do_read_config(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { int call_id = fsa_cib_conn->cmds->query( fsa_cib_conn, XML_CIB_TAG_CRMCONFIG, NULL, cib_scope_local); add_cib_op_callback(call_id, FALSE, NULL, config_query_callback); crm_debug_2("Querying the CIB... call %d", call_id); return I_NULL; } gboolean crm_shutdown(int nsig, gpointer unused) { if (crmd_mainloop != NULL && g_main_is_running(crmd_mainloop)) { if(is_set(fsa_input_register, R_SHUTDOWN)) { crm_err("Escalating the shutdown"); register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL); } else { crm_info("Requesting shutdown"); set_bit_inplace(fsa_input_register, R_SHUTDOWN); register_fsa_input(C_SHUTDOWN,I_SHUTDOWN,NULL); if(shutdown_escalation_timer->period_ms < 1) { GHashTable *config_hash = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); const char *value = crmd_pref( config_hash, XML_CONFIG_ATTR_FORCE_QUIT); int msec = crm_atoi(value, NULL); crm_info("Using default shutdown escalation: %dms", msec); shutdown_escalation_timer->period_ms = msec; g_hash_table_destroy(config_hash); } /* cant rely on this... */ crm_timer_start(shutdown_escalation_timer); } } else { crm_info("exit from shutdown"); exit(LSB_EXIT_OK); } return TRUE; } static void default_cib_update_callback(const HA_Message *msg, int call_id, int rc, crm_data_t *output, void *user_data) { if(rc != cib_ok) { fsa_data_t *msg_data = NULL; crm_err("CIB Update failed: %s", cib_error2string(rc)); crm_log_xml_warn(output, "update:failed"); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } void populate_cib_nodes(ll_cluster_t *hb_cluster, gboolean with_client_status) { int call_id = 0; const char *ha_node = NULL; crm_data_t *cib_node_list = NULL; /* Async get client status information in the cluster */ crm_debug_2("Invoked"); if(with_client_status) { crm_debug_3("Requesting an initial dump of CRMD client_status"); fsa_cluster_conn->llc_ops->client_status( fsa_cluster_conn, NULL, CRM_SYSTEM_CRMD, -1); } crm_info("Requesting the list of configured nodes"); fsa_cluster_conn->llc_ops->init_nodewalk(fsa_cluster_conn); cib_node_list = create_xml_node(NULL, XML_CIB_TAG_NODES); do { const char *ha_node_type = NULL; const char *ha_node_uuid = NULL; crm_data_t *cib_new_node = NULL; ha_node = fsa_cluster_conn->llc_ops->nextnode(fsa_cluster_conn); if(ha_node == NULL) { continue; } ha_node_type = fsa_cluster_conn->llc_ops->node_type( fsa_cluster_conn, ha_node); if(safe_str_neq(NORMALNODE, ha_node_type)) { crm_debug("Node %s: skipping '%s'", ha_node, ha_node_type); continue; } ha_node_uuid = get_uuid(fsa_cluster_conn, ha_node); if(ha_node_uuid == NULL) { crm_warn("Node %s: no uuid found", ha_node); continue; } crm_notice("Node: %s (uuid: %s)", ha_node, ha_node_uuid); cib_new_node = create_xml_node(cib_node_list, XML_CIB_TAG_NODE); crm_xml_add(cib_new_node, XML_ATTR_ID, ha_node_uuid); crm_xml_add(cib_new_node, XML_ATTR_UNAME, ha_node); crm_xml_add(cib_new_node, XML_ATTR_TYPE, ha_node_type); } while(ha_node != NULL); fsa_cluster_conn->llc_ops->end_nodewalk(fsa_cluster_conn); /* Now update the CIB with the list of nodes */ fsa_cib_update( XML_CIB_TAG_NODES, cib_node_list, cib_scope_local|cib_quorum_override|cib_inhibit_bcast, call_id); add_cib_op_callback(call_id, FALSE, NULL, default_cib_update_callback); free_xml(cib_node_list); crm_debug_2("Complete"); } gboolean register_with_ha(ll_cluster_t *hb_cluster, const char *client_name) { + const char *const_uuid = NULL; crm_debug("Signing in with Heartbeat"); if (hb_cluster->llc_ops->signon(hb_cluster, client_name)!= HA_OK) { crm_err("Cannot sign on with heartbeat: %s", hb_cluster->llc_ops->errmsg(hb_cluster)); return FALSE; } crm_debug_3("Be informed of CRM messages"); if (HA_OK != hb_cluster->llc_ops->set_msg_callback( hb_cluster, T_CRM, crmd_ha_msg_callback, hb_cluster)){ crm_err("Cannot set msg callback: %s", hb_cluster->llc_ops->errmsg(hb_cluster)); return FALSE; } crm_debug_3("Be informed of Node Status changes"); if (HA_OK != hb_cluster->llc_ops->set_nstatus_callback( hb_cluster, crmd_ha_status_callback, hb_cluster)){ crm_err("Cannot set nstatus callback: %s", hb_cluster->llc_ops->errmsg(hb_cluster)); return FALSE; } crm_debug_3("Be informed of CRM Client Status changes"); if (HA_OK != hb_cluster->llc_ops->set_cstatus_callback( hb_cluster, crmd_client_status_callback, hb_cluster)) { crm_err("Cannot set cstatus callback: %s", hb_cluster->llc_ops->errmsg(hb_cluster)); return FALSE; } crm_debug_3("Adding channel to mainloop"); G_main_add_ll_cluster( G_PRIORITY_HIGH, hb_cluster, FALSE, crmd_ha_msg_dispatch, hb_cluster /* userdata */, crmd_ha_connection_destroy); crm_debug_3("Finding our node name"); if ((fsa_our_uname = hb_cluster->llc_ops->get_mynodeid(hb_cluster)) == NULL) { crm_err("get_mynodeid() failed"); return FALSE; } crm_info("Hostname: %s", fsa_our_uname); crm_debug_3("Finding our node uuid"); - fsa_our_uuid = get_uuid(fsa_cluster_conn, fsa_our_uname); - if(fsa_our_uuid == NULL) { + const_uuid = get_uuid(fsa_cluster_conn, fsa_our_uname); + if(const_uuid == NULL) { crm_err("get_uuid_by_name() failed"); return FALSE; } /* copy it so that unget_uuid() doesn't trash the value on us */ - fsa_our_uuid = crm_strdup(fsa_our_uuid); + fsa_our_uuid = crm_strdup(const_uuid); crm_info("UUID: %s", fsa_our_uuid); populate_cib_nodes(hb_cluster, TRUE); return TRUE; } diff --git a/crm/crmd/crmd_callbacks.h b/crm/crmd/crmd_callbacks.h index e08a7c628f..f97e13965d 100644 --- a/crm/crmd/crmd_callbacks.h +++ b/crm/crmd/crmd_callbacks.h @@ -1,61 +1,63 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include extern void ccm_event_detail(const oc_ev_membership_t *oc, oc_ed_t event); extern gboolean ccm_dispatch(int fd, gpointer user_data); extern void crmd_ccm_msg_callback( oc_ed_t event, void *cookie, size_t size, const void *data); /* * Apparently returning TRUE means "stay connected, keep doing stuff". * Returning FALSE means "we're all done, close the connection" */ +extern void crmd_ipc_connection_destroy(gpointer user_data); + extern gboolean crmd_ha_msg_dispatch( ll_cluster_t *cluster_conn, gpointer user_data); extern void crmd_ha_msg_callback( HA_Message * msg, void* private_data); extern gboolean crmd_ipc_msg_callback( IPC_Channel *client, gpointer user_data); extern gboolean crmd_ipc_msg_callback( IPC_Channel *client, gpointer user_data); extern gboolean lrm_dispatch(IPC_Channel*src, gpointer user_data); extern void lrm_op_callback (lrm_op_t* op); extern void crmd_ha_status_callback( const char *node, const char * status, void* private_data); extern void crmd_client_status_callback( const char * node, const char * client, const char * status, void * private); extern void msg_ccm_join(const HA_Message *msg, void *foo); extern gboolean crmd_client_connect( IPC_Channel *newclient, gpointer user_data); extern void crmd_cib_connection_destroy(gpointer user_data); extern gboolean crm_fsa_trigger(gpointer user_data); diff --git a/crm/crmd/crmd_fsa.h b/crm/crmd/crmd_fsa.h index f763a81f4e..9ffa9e654a 100644 --- a/crm/crmd/crmd_fsa.h +++ b/crm/crmd/crmd_fsa.h @@ -1,158 +1,159 @@ /* $Id: crmd_fsa.h,v 1.50 2006/02/20 12:10:41 andrew Exp $ */ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef CRMD_FSA__H #define CRMD_FSA__H #include #include #include #include #include #include #include #include struct crmd_ccm_data_s { oc_ev_membership_t *oc; oc_ed_t event; }; struct oc_node_list_s { guint id; /* membership id */ oc_ed_t last_event; guint members_size; GHashTable *members; /* contents: oc_node_t * */ guint new_members_size; GHashTable *new_members; /* contents: oc_node_t * */ guint dead_members_size; GHashTable *dead_members; /* contents: oc_node_t * */ }; typedef struct oc_node_list_s oc_node_list_t; /* copy from struct client_child in heartbeat.h * * Plus a couple of other things */ struct crm_subsystem_s { pid_t pid; /* Process id of child process */ const char* name; /* executable name */ const char* path; /* Command location */ const char* command; /* Command with path */ const char* args; /* Command arguments */ - + crmd_client_t* client; /* Client connection object */ + gboolean sent_kill; IPC_Channel *ipc; /* How can we communicate with it */ long long flag_connected; /* */ long long flag_required; /* */ }; typedef struct fsa_timer_s fsa_timer_t; struct fsa_timer_s { guint source_id; /* timer source id */ int period_ms; /* timer period */ enum crmd_fsa_input fsa_input; gboolean (*callback)(gpointer data); gboolean repeat; }; enum fsa_data_type { fsa_dt_none, fsa_dt_ha_msg, fsa_dt_xml, fsa_dt_lrm, fsa_dt_ccm }; typedef struct fsa_data_s fsa_data_t; struct fsa_data_s { int id; enum crmd_fsa_input fsa_input; enum crmd_fsa_cause fsa_cause; long long actions; const char *origin; void *data; enum fsa_data_type data_type; }; extern enum crmd_fsa_state s_crmd_fsa(enum crmd_fsa_cause cause); /* Global FSA stuff */ extern volatile gboolean do_fsa_stall; extern volatile enum crmd_fsa_state fsa_state; extern volatile long long fsa_input_register; extern volatile long long fsa_actions; extern oc_node_list_t *fsa_membership_copy; extern ll_cluster_t *fsa_cluster_conn; extern ll_lrm_t *fsa_lrm_conn; extern cib_t *fsa_cib_conn; extern const char *fsa_our_uname; -extern const char *fsa_our_uuid; +extern char *fsa_our_uuid; extern char *fsa_pe_ref; /* the last invocation of the PE */ extern char *fsa_our_dc; extern char *fsa_our_dc_version; extern GListPtr fsa_message_queue; extern fsa_timer_t *election_trigger; /* */ extern fsa_timer_t *election_timeout; /* */ extern fsa_timer_t *shutdown_escalation_timer; /* */ extern fsa_timer_t *integration_timer; extern fsa_timer_t *finalization_timer; extern fsa_timer_t *wait_timer; extern fsa_timer_t *recheck_timer; extern GTRIGSource *fsa_source; extern struct crm_subsystem_s *cib_subsystem; extern struct crm_subsystem_s *te_subsystem; extern struct crm_subsystem_s *pe_subsystem; extern GHashTable *welcomed_nodes; extern GHashTable *integrated_nodes; extern GHashTable *finalized_nodes; extern GHashTable *confirmed_nodes; extern GHashTable *crmd_peer_state; /* these two should be moved elsewhere... */ extern void do_update_cib_nodes(gboolean overwrite, const char *caller); extern gboolean do_dc_heartbeat(gpointer data); gboolean add_cib_op_callback( int call_id, gboolean only_success, void *user_data, void (*callback)(const HA_Message*, int, int, crm_data_t*,void*)); #define AM_I_DC is_set(fsa_input_register, R_THE_DC) #define AM_I_OPERATIONAL (is_set(fsa_input_register, R_STARTING)==FALSE) #include #include #endif diff --git a/crm/crmd/crmd_utils.h b/crm/crmd/crmd_utils.h index 751485ac10..e9d1d1e799 100644 --- a/crm/crmd/crmd_utils.h +++ b/crm/crmd/crmd_utils.h @@ -1,93 +1,96 @@ /* $Id: crmd_utils.h,v 1.24 2006/04/18 10:59:46 andrew Exp $ */ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef CRMD_UTILS__H #define CRMD_UTILS__H #include #include #define CLIENT_EXIT_WAIT 30 +extern void free_ccm_cache(oc_node_list_t *tmp); +extern void delete_ccm_data(struct crmd_ccm_data_s *ccm_input); + extern void process_client_disconnect(crmd_client_t *curr_client); extern void update_local_cib_adv( crm_data_t *msg_data, gboolean do_now, const char *raised_from); #define update_local_cib(data) update_local_cib_adv(data, TRUE, __FUNCTION__) #define fsa_cib_update(section, data, options, call_id) \ if(fsa_cib_conn != NULL) { \ call_id = fsa_cib_conn->cmds->update( \ fsa_cib_conn, section, data, NULL, options); \ \ } else { \ crm_err("No CIB connection available"); \ } #define fsa_cib_anon_update(section, data, options) \ if(fsa_cib_conn != NULL) { \ fsa_cib_conn->cmds->update( \ fsa_cib_conn, section, data, NULL, options); \ \ } else { \ crm_err("No CIB connection available"); \ } extern long long toggle_bit (long long action_list, long long action); extern long long clear_bit (long long action_list, long long action); extern long long set_bit (long long action_list, long long action); #define set_bit_inplace(word, bit) word = set_bit(word, bit) #define clear_bit_inplace(word, bit) word = clear_bit(word, bit) #define toggle_bit_inplace(word, bit) word = toggle_bit(word, bit) extern gboolean is_set(long long action_list, long long action); extern gboolean is_set_any(long long action_list, long long action); extern gboolean crm_timer_stop (fsa_timer_t *timer); extern gboolean crm_timer_start(fsa_timer_t *timer); extern gboolean crm_timer_popped(gpointer data); extern crm_data_t *create_node_state( const char *uname, const char *ha_state, const char *ccm_state, const char *crmd_state, const char *join_state, const char *exp_state, gboolean clear_shutdown, const char *src); extern void create_node_entry( const char *uuid, const char *uname, const char *type); extern gboolean stop_subsystem ( struct crm_subsystem_s *centry, gboolean force_quit); extern gboolean start_subsystem(struct crm_subsystem_s *centry); extern lrm_op_t *copy_lrm_op(const lrm_op_t *op); extern lrm_rsc_t *copy_lrm_rsc(const lrm_rsc_t *rsc); extern struct crmd_ccm_data_s *copy_ccm_data( const struct crmd_ccm_data_s *ccm_input); extern oc_ev_membership_t *copy_ccm_oc_data(const oc_ev_membership_t *oc_in) ; extern void fsa_dump_actions(long long action, const char *text); extern void fsa_dump_inputs( int log_level, const char *text, long long input_register); extern gboolean need_transition(enum crmd_fsa_state state); extern void update_dc(HA_Message *msg, gboolean assert_same); #endif diff --git a/crm/crmd/fsa.c b/crm/crmd/fsa.c index bdb58f70c7..9528f33eac 100644 --- a/crm/crmd/fsa.c +++ b/crm/crmd/fsa.c @@ -1,797 +1,800 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include longclock_t action_start = 0; longclock_t action_stop = 0; longclock_t action_diff = 0; unsigned int action_diff_ms = 0; char *fsa_our_dc = NULL; cib_t *fsa_cib_conn = NULL; char *fsa_our_dc_version = NULL; ll_lrm_t *fsa_lrm_conn; ll_cluster_t *fsa_cluster_conn; oc_node_list_t *fsa_membership_copy; -const char *fsa_our_uuid = NULL; +char *fsa_our_uuid = NULL; const char *fsa_our_uname = NULL; fsa_timer_t *wait_timer = NULL; fsa_timer_t *recheck_timer = NULL; fsa_timer_t *election_trigger = NULL; fsa_timer_t *election_timeout = NULL; fsa_timer_t *integration_timer = NULL; fsa_timer_t *finalization_timer = NULL; fsa_timer_t *shutdown_escalation_timer = NULL; volatile gboolean do_fsa_stall = FALSE; volatile long long fsa_input_register = 0; volatile long long fsa_actions = A_NOTHING; volatile enum crmd_fsa_state fsa_state = S_STARTING; extern uint highest_born_on; extern uint num_join_invites; extern GHashTable *welcomed_nodes; extern GHashTable *finalized_nodes; extern GHashTable *confirmed_nodes; extern GHashTable *integrated_nodes; extern void initialize_join(gboolean before); #define DOT_PREFIX "actions:trace: " #define do_dot_log(fmt, args...) do_crm_log(LOG_DEBUG_2, fmt, ##args) long long do_state_transition(long long actions, enum crmd_fsa_state cur_state, enum crmd_fsa_state next_state, fsa_data_t *msg_data); long long clear_flags(long long actions, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input); void dump_rsc_info(void); void dump_rsc_info_callback(const HA_Message *msg, int call_id, int rc, crm_data_t *output, void *user_data); void ghash_print_node(gpointer key, gpointer value, gpointer user_data); void s_crmd_fsa_actions(fsa_data_t *fsa_data); void log_fsa_input(fsa_data_t *stored_msg); void init_dotfile(void); void init_dotfile(void) { do_dot_log(DOT_PREFIX"digraph \"g\" {"); do_dot_log(DOT_PREFIX" size = \"30,30\""); do_dot_log(DOT_PREFIX" graph ["); do_dot_log(DOT_PREFIX" fontsize = \"12\""); do_dot_log(DOT_PREFIX" fontname = \"Times-Roman\""); do_dot_log(DOT_PREFIX" fontcolor = \"black\""); do_dot_log(DOT_PREFIX" bb = \"0,0,398.922306,478.927856\""); do_dot_log(DOT_PREFIX" color = \"black\""); do_dot_log(DOT_PREFIX" ]"); do_dot_log(DOT_PREFIX" node ["); do_dot_log(DOT_PREFIX" fontsize = \"12\""); do_dot_log(DOT_PREFIX" fontname = \"Times-Roman\""); do_dot_log(DOT_PREFIX" fontcolor = \"black\""); do_dot_log(DOT_PREFIX" shape = \"ellipse\""); do_dot_log(DOT_PREFIX" color = \"black\""); do_dot_log(DOT_PREFIX" ]"); do_dot_log(DOT_PREFIX" edge ["); do_dot_log(DOT_PREFIX" fontsize = \"12\""); do_dot_log(DOT_PREFIX" fontname = \"Times-Roman\""); do_dot_log(DOT_PREFIX" fontcolor = \"black\""); do_dot_log(DOT_PREFIX" color = \"black\""); do_dot_log(DOT_PREFIX" ]"); do_dot_log(DOT_PREFIX"// special nodes"); do_dot_log(DOT_PREFIX" \"S_PENDING\" "); do_dot_log(DOT_PREFIX" ["); do_dot_log(DOT_PREFIX" color = \"blue\""); do_dot_log(DOT_PREFIX" fontcolor = \"blue\""); do_dot_log(DOT_PREFIX" ]"); do_dot_log(DOT_PREFIX" \"S_TERMINATE\" "); do_dot_log(DOT_PREFIX" ["); do_dot_log(DOT_PREFIX" color = \"red\""); do_dot_log(DOT_PREFIX" fontcolor = \"red\""); do_dot_log(DOT_PREFIX" ]"); do_dot_log(DOT_PREFIX"// DC only nodes"); do_dot_log(DOT_PREFIX" \"S_INTEGRATION\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX" \"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX" \"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX" \"S_RELEASE_DC\" [ fontcolor = \"green\" ]"); do_dot_log(DOT_PREFIX" \"S_IDLE\" [ fontcolor = \"green\" ]"); } static void do_fsa_action(fsa_data_t *fsa_data, long long an_action, enum crmd_fsa_input (*function)(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t *msg_data)) { int action_log_level = LOG_DEBUG; gboolean do_time_check = TRUE; enum crmd_fsa_input result = I_NULL; if(is_set(fsa_actions, an_action) == FALSE) { crm_err("Action %s (%.16llx) was not requestsed", fsa_action2string(an_action), an_action); return; } if(an_action & A_MSG_ROUTE) { action_log_level = LOG_DEBUG_2; } else if(an_action & A_CIB_START) { do_time_check = FALSE; } fsa_actions = clear_bit(fsa_actions, an_action); crm_debug_3("Invoking action %s (%.16llx)", fsa_action2string(an_action), an_action); if(do_time_check) { action_start = time_longclock(); } do_crm_log(action_log_level, DOT_PREFIX"\t// %s", fsa_action2string(an_action)); result = function(an_action, fsa_data->fsa_cause, fsa_state, fsa_data->fsa_input, fsa_data); CRM_DEV_ASSERT(result == I_NULL); crm_debug_3("Action complete: %s (%.16llx)", fsa_action2string(an_action), an_action); if(do_time_check) { action_stop = time_longclock(); action_diff = sub_longclock(action_stop, action_start); action_diff_ms = longclockto_ms(action_diff); if(action_diff_ms > action_diff_max_ms) { crm_err("Action %s took %dms to complete", fsa_action2string(an_action), action_diff_ms); } else if(action_diff_ms > action_diff_warn_ms) { crm_warn("Action %s took %dms to complete", fsa_action2string(an_action), action_diff_ms); } } } enum crmd_fsa_state s_crmd_fsa(enum crmd_fsa_cause cause) { fsa_data_t *fsa_data = NULL; long long register_copy = fsa_input_register; long long new_actions = A_NOTHING; enum crmd_fsa_state last_state = fsa_state; crm_debug_2("FSA invoked with Cause: %s\tState: %s", fsa_cause2string(cause), fsa_state2string(fsa_state)); do_fsa_stall = FALSE; if(is_message() == FALSE && fsa_actions != A_NOTHING) { /* fake the first message so we can get into the loop */ crm_malloc0(fsa_data, sizeof(fsa_data_t)); fsa_data->fsa_input = I_NULL; fsa_data->fsa_cause = C_FSA_INTERNAL; fsa_data->origin = __FUNCTION__; fsa_data->data_type = fsa_dt_none; fsa_message_queue = g_list_append(fsa_message_queue, fsa_data); fsa_data = NULL; } while(is_message() && do_fsa_stall == FALSE) { crm_debug_2("Checking messages (%d remaining)", g_list_length(fsa_message_queue)); fsa_data = get_message(); CRM_DEV_ASSERT(fsa_data != NULL); if(crm_assert_failed) { continue; } log_fsa_input(fsa_data); /* add any actions back to the queue */ fsa_actions |= fsa_data->actions; /* get the next batch of actions */ new_actions = crmd_fsa_actions[fsa_data->fsa_input][fsa_state]; fsa_actions |= new_actions; if(fsa_data->fsa_input != I_NULL && fsa_data->fsa_input != I_ROUTER) { crm_debug("Processing %s: [ state=%s cause=%s origin=%s ]", fsa_input2string(fsa_data->fsa_input), fsa_state2string(fsa_state), fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin); } /* if(fsa_actions & A_SHUTDOWN) { crm_log_level = LOG_DEBUG_2; } */ #ifdef FSA_TRACE if(new_actions != A_NOTHING) { crm_debug_2("Adding FSA actions %.16llx for %s/%s", new_actions, fsa_input2string(fsa_data->fsa_input), fsa_state2string(fsa_state)); fsa_dump_actions(new_actions, "\tFSA scheduled"); } else if(fsa_data->fsa_input != I_NULL && new_actions == A_NOTHING) { crm_debug("No action specified for input,state (%s,%s)", fsa_input2string(fsa_data->fsa_input), fsa_state2string(fsa_state)); } if(fsa_data->actions != A_NOTHING) { crm_debug_2("Adding input actions %.16llx for %s/%s", new_actions, fsa_input2string(fsa_data->fsa_input), fsa_state2string(fsa_state)); fsa_dump_actions(fsa_data->actions,"\tInput scheduled"); } #endif /* logging : *before* the state is changed */ if(is_set(fsa_actions, A_ERROR)) { do_fsa_action(fsa_data, A_ERROR, do_log); } if(is_set(fsa_actions, A_WARN)) { do_fsa_action(fsa_data, A_WARN, do_log); } if(is_set(fsa_actions, A_LOG)) { do_fsa_action(fsa_data, A_LOG, do_log); } /* update state variables */ last_state = fsa_state; fsa_state = crmd_fsa_state[fsa_data->fsa_input][fsa_state]; /* * Hook to allow actions to removed due to certain inputs */ fsa_actions = clear_flags( fsa_actions, cause, fsa_state, fsa_data->fsa_input); /* * Hook for change of state. * Allows actions to be added or removed when entering a state */ if(last_state != fsa_state){ fsa_actions = do_state_transition( fsa_actions, last_state, fsa_state, fsa_data); } else { do_dot_log(DOT_PREFIX"\t// FSA input: State=%s \tCause=%s" " \tInput=%s \tOrigin=%s() \tid=%d", fsa_state2string(fsa_state), fsa_cause2string(fsa_data->fsa_cause), fsa_input2string(fsa_data->fsa_input), fsa_data->origin, fsa_data->id); } /* start doing things... */ s_crmd_fsa_actions(fsa_data); delete_fsa_input(fsa_data); fsa_data = NULL; } if(g_list_length(fsa_message_queue) > 0 || fsa_actions != A_NOTHING || do_fsa_stall) { crm_debug("Exiting the FSA: queue=%d, fsa_actions=0x%llx, stalled=%s", g_list_length(fsa_message_queue), fsa_actions, do_fsa_stall?"true":"false"); } else { crm_debug_2("Exiting the FSA"); } /* cleanup inputs? */ if(register_copy != fsa_input_register) { long long same = register_copy & fsa_input_register; fsa_dump_inputs(LOG_DEBUG, "Added input:", fsa_input_register ^ same); fsa_dump_inputs(LOG_DEBUG, "Removed input:", register_copy ^ same); } #if 0 if(fsa_state != S_STARTING && g_list_length(fsa_message_queue) && is_ipc_empty(te_subsystem->ipc) && is_ipc_empty(pe_subsystem->ipc) && is_ipc_empty(fsa_cib_conn->cmds->channel(fsa_cib_conn) && is_ipc_empty(fsa_cluster_conn->llc_ops->ipcchan(fsa_cluster_conn)) ){ static gboolean mem_needs_init = TRUE; if(mem_needs_init) { crm_debug("Reached a stable point:" " reseting memory usage stats to zero"); crm_zero_mem_stats(NULL); mem_needs_init = FALSE; } else { crm_debug("Reached a stable point:" " checking memory usage"); crm_mem_stats(NULL); } } #endif fsa_dump_queue(LOG_DEBUG); return fsa_state; } void s_crmd_fsa_actions(fsa_data_t *fsa_data) { /* * Process actions in order of priority but do only one * action at a time to avoid complicating the ordering. */ while(fsa_actions != A_NOTHING && do_fsa_stall == FALSE) { msg_queue_helper(); CRM_DEV_ASSERT(fsa_data != NULL); if(crm_assert_failed) { return; } /* regular action processing in order of action priority * * Make sure all actions that connect to required systems * are performed first */ if(is_set(fsa_actions, A_ERROR)) { do_fsa_action(fsa_data, A_ERROR, do_log); } else if(is_set(fsa_actions, A_WARN)) { do_fsa_action(fsa_data, A_WARN, do_log); } else if(is_set(fsa_actions, A_LOG)) { do_fsa_action(fsa_data, A_LOG, do_log); /* get out of here NOW! before anything worse happens */ } else if(is_set(fsa_actions, A_EXIT_1)) { do_fsa_action(fsa_data, A_EXIT_1, do_exit); /* essential start tasks */ } else if(is_set(fsa_actions, A_STARTUP)) { do_fsa_action(fsa_data, A_STARTUP, do_startup); } else if(is_set(fsa_actions, A_CIB_START)) { do_fsa_action(fsa_data, A_CIB_START, do_cib_control); } else if(is_set(fsa_actions, A_HA_CONNECT)) { do_fsa_action(fsa_data, A_HA_CONNECT, do_ha_control); } else if(is_set(fsa_actions, A_READCONFIG)) { do_fsa_action(fsa_data, A_READCONFIG, do_read_config); /* sub-system start/connect */ } else if(is_set(fsa_actions, A_LRM_CONNECT)) { do_fsa_action(fsa_data, A_LRM_CONNECT, do_lrm_control); } else if(is_set(fsa_actions, A_CCM_CONNECT)) { do_fsa_action(fsa_data, A_CCM_CONNECT, do_ccm_control); } else if(is_set(fsa_actions, A_TE_START)) { do_fsa_action(fsa_data, A_TE_START, do_te_control); } else if(is_set(fsa_actions, A_PE_START)) { do_fsa_action(fsa_data, A_PE_START, do_pe_control); /* sub-system restart */ } else if(is_set(fsa_actions, O_CIB_RESTART)) { do_fsa_action(fsa_data, O_CIB_RESTART, do_cib_control); } else if(is_set(fsa_actions, O_PE_RESTART)) { do_fsa_action(fsa_data, O_PE_RESTART, do_pe_control); } else if(is_set(fsa_actions, O_TE_RESTART)) { do_fsa_action(fsa_data, O_TE_RESTART, do_te_control); /* Timers */ /* else if(is_set(fsa_actions, O_DC_TIMER_RESTART)) { do_fsa_action(fsa_data, O_DC_TIMER_RESTART, do_timer_control) */; } else if(is_set(fsa_actions, A_DC_TIMER_STOP)) { do_fsa_action(fsa_data, A_DC_TIMER_STOP, do_timer_control); } else if(is_set(fsa_actions, A_INTEGRATE_TIMER_STOP)) { do_fsa_action(fsa_data, A_INTEGRATE_TIMER_STOP, do_timer_control); } else if(is_set(fsa_actions, A_INTEGRATE_TIMER_START)) { do_fsa_action(fsa_data, A_INTEGRATE_TIMER_START,do_timer_control); } else if(is_set(fsa_actions, A_FINALIZE_TIMER_STOP)) { do_fsa_action(fsa_data, A_FINALIZE_TIMER_STOP, do_timer_control); } else if(is_set(fsa_actions, A_FINALIZE_TIMER_START)) { do_fsa_action(fsa_data, A_FINALIZE_TIMER_START, do_timer_control); /* * Highest priority actions */ } else if(is_set(fsa_actions, A_CIB_BUMPGEN)) { do_fsa_action(fsa_data, A_CIB_BUMPGEN, do_cib_invoke); } else if(is_set(fsa_actions, A_MSG_ROUTE)) { do_fsa_action(fsa_data, A_MSG_ROUTE, do_msg_route); } else if(is_set(fsa_actions, A_RECOVER)) { do_fsa_action(fsa_data, A_RECOVER, do_recover); } else if(is_set(fsa_actions, A_CL_JOIN_RESULT)) { do_fsa_action(fsa_data, A_CL_JOIN_RESULT, do_cl_join_finalize_respond); } else if(is_set(fsa_actions, A_CL_JOIN_REQUEST)) { do_fsa_action(fsa_data, A_CL_JOIN_REQUEST, do_cl_join_offer_respond); } else if(is_set(fsa_actions, A_SHUTDOWN_REQ)) { do_fsa_action(fsa_data, A_SHUTDOWN_REQ, do_shutdown_req); } else if(is_set(fsa_actions, A_ELECTION_VOTE)) { do_fsa_action(fsa_data, A_ELECTION_VOTE, do_election_vote); } else if(is_set(fsa_actions, A_ELECTION_COUNT)) { do_fsa_action(fsa_data, A_ELECTION_COUNT, do_election_count_vote); } else if(is_set(fsa_actions, A_LRM_EVENT)) { do_fsa_action(fsa_data, A_LRM_EVENT, do_lrm_event); /* * High priority actions * Update the cache first */ } else if(is_set(fsa_actions, A_CCM_UPDATE_CACHE)) { do_fsa_action(fsa_data, A_CCM_UPDATE_CACHE, do_ccm_update_cache); } else if(is_set(fsa_actions, A_CCM_EVENT)) { do_fsa_action(fsa_data, A_CCM_EVENT, do_ccm_event); } else if(is_set(fsa_actions, A_STARTED)) { do_fsa_action(fsa_data, A_STARTED, do_started); } else if(is_set(fsa_actions, A_CL_JOIN_QUERY)) { do_fsa_action(fsa_data, A_CL_JOIN_QUERY, do_cl_join_query); } else if(is_set(fsa_actions, A_DC_TIMER_START)) { do_fsa_action(fsa_data, A_DC_TIMER_START, do_timer_control); /* * Medium priority actions */ } else if(is_set(fsa_actions, A_DC_TAKEOVER)) { do_fsa_action(fsa_data, A_DC_TAKEOVER, do_dc_takeover); } else if(is_set(fsa_actions, A_DC_RELEASE)) { do_fsa_action(fsa_data, A_DC_RELEASE, do_dc_release); } else if(is_set(fsa_actions, A_ELECTION_CHECK)) { do_fsa_action(fsa_data, A_ELECTION_CHECK, do_election_check); } else if(is_set(fsa_actions, A_ELECTION_START)) { do_fsa_action(fsa_data, A_ELECTION_START, do_election_vote); } else if(is_set(fsa_actions, A_TE_HALT)) { do_fsa_action(fsa_data, A_TE_HALT, do_te_invoke); } else if(is_set(fsa_actions, A_TE_CANCEL)) { do_fsa_action(fsa_data, A_TE_CANCEL, do_te_invoke); } else if(is_set(fsa_actions, A_DC_JOIN_OFFER_ALL)) { do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ALL, do_dc_join_offer_all); } else if(is_set(fsa_actions, A_DC_JOIN_OFFER_ONE)) { do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ONE, do_dc_join_offer_all); } else if(is_set(fsa_actions, A_DC_JOIN_PROCESS_REQ)) { do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_REQ, do_dc_join_filter_offer); } else if(is_set(fsa_actions, A_DC_JOIN_PROCESS_ACK)) { do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_ACK, do_dc_join_ack); /* * Low(er) priority actions * Make sure the CIB is always updated before invoking the * PE, and the PE before the TE */ } else if(is_set(fsa_actions, A_CIB_INVOKE_LOCAL)) { do_fsa_action(fsa_data, A_CIB_INVOKE_LOCAL, do_cib_invoke); } else if(is_set(fsa_actions, A_CIB_INVOKE)) { do_fsa_action(fsa_data, A_CIB_INVOKE, do_cib_invoke); } else if(is_set(fsa_actions, A_DC_JOIN_FINALIZE)) { do_fsa_action(fsa_data, A_DC_JOIN_FINALIZE, do_dc_join_finalize); } else if(is_set(fsa_actions, A_LRM_INVOKE)) { do_fsa_action(fsa_data, A_LRM_INVOKE, do_lrm_invoke); } else if(is_set(fsa_actions, A_PE_INVOKE)) { do_fsa_action(fsa_data, A_PE_INVOKE, do_pe_invoke); } else if(is_set(fsa_actions, A_TE_INVOKE)) { do_fsa_action(fsa_data, A_TE_INVOKE, do_te_invoke); } else if(is_set(fsa_actions, A_CL_JOIN_ANNOUNCE)) { do_fsa_action(fsa_data, A_CL_JOIN_ANNOUNCE, do_cl_join_announce); /* sub-system stop */ } else if(is_set(fsa_actions, A_DC_RELEASED)) { do_fsa_action(fsa_data, A_DC_RELEASED, do_dc_release); } else if(is_set(fsa_actions, A_PE_STOP)) { do_fsa_action(fsa_data, A_PE_STOP, do_pe_control); } else if(is_set(fsa_actions, A_TE_STOP)) { do_fsa_action(fsa_data, A_TE_STOP, do_te_control); } else if(is_set(fsa_actions, A_SHUTDOWN)) { do_fsa_action(fsa_data, A_SHUTDOWN, do_shutdown); } else if(is_set(fsa_actions, A_STOP)) { do_fsa_action(fsa_data, A_STOP, do_stop); } else if(is_set(fsa_actions, A_CCM_DISCONNECT)) { do_fsa_action(fsa_data, A_CCM_DISCONNECT, do_ccm_control); } else if(is_set(fsa_actions, A_LRM_DISCONNECT)) { do_fsa_action(fsa_data, A_LRM_DISCONNECT, do_lrm_control); } else if(is_set(fsa_actions, A_HA_DISCONNECT)) { do_fsa_action(fsa_data, A_HA_DISCONNECT, do_ha_control); } else if(is_set(fsa_actions, A_CIB_STOP)) { do_fsa_action(fsa_data, A_CIB_STOP, do_cib_control); /* exit gracefully */ } else if(is_set(fsa_actions, A_EXIT_0)) { do_fsa_action(fsa_data, A_EXIT_0, do_exit); /* Error checking and reporting */ } else { crm_err("Action %s (0x%llx) not supported ", fsa_action2string(fsa_actions), fsa_actions); register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, fsa_data, NULL, __FUNCTION__); } } } void log_fsa_input(fsa_data_t *stored_msg) { crm_debug_2("Processing queued input %d", stored_msg->id); if(stored_msg->fsa_cause == C_CCM_CALLBACK) { crm_debug_3("FSA processing CCM callback from %s", stored_msg->origin); } else if(stored_msg->fsa_cause == C_LRM_OP_CALLBACK) { crm_debug_3("FSA processing LRM callback from %s", stored_msg->origin); } else if(stored_msg->data == NULL) { crm_debug_3("FSA processing input from %s", stored_msg->origin); } else { ha_msg_input_t *ha_input = fsa_typed_data_adv( stored_msg, fsa_dt_ha_msg, __FUNCTION__); crm_debug_3("FSA processing XML message from %s", stored_msg->origin); crm_log_message(LOG_MSG, ha_input->msg); crm_log_xml_debug_3(ha_input->xml, "FSA message data"); } } long long do_state_transition(long long actions, enum crmd_fsa_state cur_state, enum crmd_fsa_state next_state, fsa_data_t *msg_data) { long long tmp = actions; gboolean clear_recovery_bit = TRUE; enum crmd_fsa_cause cause = msg_data->fsa_cause; enum crmd_fsa_input current_input = msg_data->fsa_input; const char *state_from = fsa_state2string(cur_state); const char *state_to = fsa_state2string(next_state); const char *input = fsa_input2string(current_input); CRM_DEV_ASSERT(cur_state != next_state); do_dot_log(DOT_PREFIX"\t%s -> %s [ label=%s cause=%s origin=%s ]", state_from, state_to, input, fsa_cause2string(cause), msg_data->origin); crm_info("%s: State transition %s -> %s [ input=%s cause=%s origin=%s ]", fsa_our_uname, state_from, state_to, input, fsa_cause2string(cause), msg_data->origin); /* the last two clauses might cause trouble later */ if(election_timeout != NULL && next_state != S_ELECTION && cur_state != S_RELEASE_DC) { crm_timer_stop(election_timeout); /* } else { */ /* crm_timer_start(election_timeout); */ } #if 0 if(is_set(fsa_input_register, R_SHUTDOWN)){ set_bit_inplace(tmp, A_DC_TIMER_STOP); } #endif if(next_state == S_INTEGRATION) { set_bit_inplace(tmp, A_INTEGRATE_TIMER_START); } else { set_bit_inplace(tmp, A_INTEGRATE_TIMER_STOP); } if(next_state == S_FINALIZE_JOIN) { set_bit_inplace(tmp, A_FINALIZE_TIMER_START); } else { set_bit_inplace(tmp, A_FINALIZE_TIMER_STOP); } if(next_state != S_PENDING) { set_bit_inplace(tmp, A_DC_TIMER_STOP); } if(next_state != S_ELECTION) { highest_born_on = 0; } if(next_state != S_IDLE) { crm_timer_stop(recheck_timer); } switch(next_state) { case S_PENDING: fsa_cib_conn->cmds->set_slave(fsa_cib_conn, cib_scope_local); /* fall through */ case S_ELECTION: crm_debug_2("Resetting our DC to NULL on transition to %s", fsa_state2string(next_state)); update_dc(NULL, FALSE); break; case S_NOT_DC: if(is_set(fsa_input_register, R_SHUTDOWN)){ crm_info("(Re)Issuing shutdown request now" " that we have a new DC"); set_bit_inplace(tmp, A_SHUTDOWN_REQ); } CRM_DEV_ASSERT(fsa_our_dc != NULL); if(fsa_our_dc == NULL) { crm_err("Reached S_NOT_DC without a DC" " being recorded"); } break; case S_RECOVERY: clear_recovery_bit = FALSE; break; case S_FINALIZE_JOIN: CRM_DEV_ASSERT(AM_I_DC); if(cause == C_TIMER_POPPED) { crm_warn("Progressed to state %s after %s", fsa_state2string(next_state), fsa_cause2string(cause)); } if(g_hash_table_size(welcomed_nodes) > 0) { char *msg = crm_strdup( " Welcome reply not received from"); crm_warn("%u cluster nodes failed to respond" " to the join offer.", g_hash_table_size(welcomed_nodes)); g_hash_table_foreach( welcomed_nodes, ghash_print_node, msg); crm_free(msg); } else { crm_info("All %d cluster nodes " "responded to the join offer.", g_hash_table_size(integrated_nodes)); } break; case S_POLICY_ENGINE: CRM_DEV_ASSERT(AM_I_DC); if(cause == C_TIMER_POPPED) { crm_warn("Progressed to state %s after %s", fsa_state2string(next_state), fsa_cause2string(cause)); } if(g_hash_table_size(finalized_nodes) > 0) { char *msg = crm_strdup( " Confirm not received from"); crm_err("%u cluster nodes failed to confirm" " their join.", g_hash_table_size(finalized_nodes)); g_hash_table_foreach( finalized_nodes, ghash_print_node, msg); crm_free(msg); } else if(g_hash_table_size(confirmed_nodes) == fsa_membership_copy->members_size) { crm_info("All %u cluster nodes are" " eligable to run resources.", fsa_membership_copy->members_size); } else if(g_hash_table_size(confirmed_nodes) > fsa_membership_copy->members_size) { crm_err("We have more confirmed nodes than our membership does"); register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); } else { crm_warn("Only %u of %u cluster " "nodes are eligable to run resources", g_hash_table_size(confirmed_nodes), fsa_membership_copy->members_size); } /* initialize_join(FALSE); */ break; case S_STOPPING: case S_TERMINATE: /* possibly redundant */ set_bit_inplace(fsa_input_register, R_SHUTDOWN); break; case S_IDLE: +#ifdef HA_MALLOC_TRACK + cl_malloc_dump_allocated(LOG_NOTICE, FALSE); +#endif CRM_DEV_ASSERT(AM_I_DC); dump_rsc_info(); if(is_set(fsa_input_register, R_SHUTDOWN)){ crm_info("(Re)Issuing shutdown request now" " that we are the DC"); set_bit_inplace(tmp, A_SHUTDOWN_REQ); } if(recheck_timer->period_ms > 0) { crm_timer_start(recheck_timer); } break; default: break; } if(clear_recovery_bit && next_state != S_PENDING) { tmp = clear_bit(tmp, A_RECOVER); } else if(clear_recovery_bit == FALSE) { tmp = set_bit(tmp, A_RECOVER); } if(tmp != actions) { fsa_dump_actions(actions ^ tmp, "New actions"); actions = tmp; } return actions; } long long clear_flags(long long actions, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input) { long long saved_actions = actions; long long startup_actions = A_STARTUP|A_CIB_START|A_LRM_CONNECT|A_CCM_CONNECT|A_HA_CONNECT|A_READCONFIG|A_STARTED|A_CL_JOIN_QUERY; if(cur_state == S_STOPPING || is_set(fsa_input_register, R_SHUTDOWN)) { clear_bit_inplace(actions, startup_actions); } fsa_dump_actions(actions ^ saved_actions, "Cleared Actions"); return actions; } void dump_rsc_info(void) { } void ghash_print_node(gpointer key, gpointer value, gpointer user_data) { const char *text = user_data; const char *uname = key; const char *value_s = value; crm_info("%s: %s %s", text, uname, value_s); } diff --git a/crm/crmd/main.c b/crm/crmd/main.c index a529eadfc5..d84468e676 100644 --- a/crm/crmd/main.c +++ b/crm/crmd/main.c @@ -1,208 +1,211 @@ /* $Id: main.c,v 1.21 2006/08/14 09:06:31 andrew Exp $ */ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include const char* crm_system_name = SYS_NAME; #define OPTARGS "hV" void usage(const char* cmd, int exit_status); int init_start(void); void crmd_hamsg_callback(const HA_Message * msg, void* private_data); gboolean crmd_tickle_apphb(gpointer data); extern void init_dotfile(void); GMainLoop* crmd_mainloop = NULL; int main(int argc, char ** argv) { int flag; int argerr = 0; crm_log_init(crm_system_name); crm_info("CRM Hg Version: %s\n", HA_HG_VERSION); while ((flag = getopt(argc, argv, OPTARGS)) != EOF) { switch(flag) { case 'V': cl_log_enable_stderr(1); alter_debug(DEBUG_INC); break; case 'h': /* Help message */ usage(crm_system_name, LSB_EXIT_OK); break; default: ++argerr; break; } } if(argc - optind == 1 && safe_str_eq("metadata", argv[optind])) { crmd_metadata(); return 0; } else if(argc - optind == 1 && safe_str_eq("version", argv[optind])) { fprintf(stderr, "CRM Version: "); fprintf(stdout, "%s (%s)\n", VERSION, HA_HG_VERSION); return 0; } if (optind > argc) { ++argerr; } if (argerr) { usage(crm_system_name,LSB_EXIT_GENERIC); } /* read local config file */ crm_debug_3("Enabling coredumps"); if(cl_enable_coredumps(1) != 0) { crm_warn("Cannot enable coredumps"); } if(crm_is_writable(HA_VARLIBDIR"/heartbeat/pengine", NULL, HA_CCMUSER, HA_APIGROUP, FALSE) == FALSE) { fprintf(stderr,"ERROR: Bad permissions on " HA_VARLIBDIR"/heartbeat/pengine... See logs for details\n"); fflush(stderr); return 100; } return init_start(); } int init_start(void) { int exit_code = 0; enum crmd_fsa_state state; fsa_state = S_STARTING; fsa_input_register = 0; /* zero out the regester */ init_dotfile(); crm_info("Starting %s", crm_system_name); register_fsa_input(C_STARTUP, I_STARTUP, NULL); state = s_crmd_fsa(C_STARTUP); if (state == S_PENDING || state == S_STARTING) { /* Create the mainloop and run it... */ crmd_mainloop = g_main_new(FALSE); crm_info("Starting %s's mainloop", crm_system_name); #ifdef REALTIME_SUPPORT static int crm_realtime = 1; if (crm_realtime == 1){ cl_enable_realtime(); }else if (crm_realtime == 0){ cl_disable_realtime(); } cl_make_realtime(SCHED_RR, 5, 64, 64); #endif g_main_run(crmd_mainloop); return_to_orig_privs(); if(is_set(fsa_input_register, R_STAYDOWN)) { crm_info("Inhibiting respawn by Heartbeat"); exit_code = 100; } } else { crm_err("Startup of %s failed. Current state: %s", crm_system_name, fsa_state2string(state)); exit_code = 1; } crm_info("[%s] stopped (%d)", crm_system_name, exit_code); +#ifdef HA_MALLOC_TRACK + cl_malloc_dump_allocated(LOG_ERR, FALSE); +#endif return exit_code; } void usage(const char* cmd, int exit_status) { FILE* stream; stream = exit_status ? stderr : stdout; fprintf(stream, "usage: %s [-V] [-h|version|metadata]\n", cmd); fprintf(stream, "\t-h\t: this help message\n"); fprintf(stream, "\t-V\t: increase verbosity\n"); fprintf(stream, "\tmetadata\t: show configurable crmd options\n"); fprintf(stream, "\tversion\t\t: show version information and quit\n"); fflush(stream); exit(exit_status); } gboolean crmd_tickle_apphb(gpointer data) { char app_instance[APPNAME_LEN]; int rc = 0; sprintf(app_instance, "%s_%ld", crm_system_name, (long)getpid()); rc = apphb_hb(); if (rc < 0) { cl_perror("%s apphb_hb failure", app_instance); exit(3); } return TRUE; } diff --git a/crm/crmd/messages.c b/crm/crmd/messages.c index ce2f2635cd..94315460b5 100644 --- a/crm/crmd/messages.c +++ b/crm/crmd/messages.c @@ -1,1221 +1,1218 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include GListPtr fsa_message_queue = NULL; extern void crm_shutdown(int nsig); enum crmd_fsa_input handle_request(ha_msg_input_t *stored_msg); enum crmd_fsa_input handle_response(ha_msg_input_t *stored_msg); enum crmd_fsa_input handle_shutdown_request(HA_Message *stored_msg); ha_msg_input_t *copy_ha_msg_input(ha_msg_input_t *orig); gboolean ipc_queue_helper(gpointer key, gpointer value, gpointer user_data); #ifdef MSG_LOG # define ROUTER_RESULT(x) crm_debug_3("Router result: %s", x); \ crm_log_message_adv(LOG_MSG, "router.log", relay_message); #else # define ROUTER_RESULT(x) crm_debug_3("Router result: %s", x) #endif /* debug only, can wrap all it likes */ int last_data_id = 0; void register_fsa_error_adv( enum crmd_fsa_cause cause, enum crmd_fsa_input input, fsa_data_t *cur_data, void *new_data, const char *raised_from) { /* save the current actions if any */ if(fsa_actions != A_NOTHING) { register_fsa_input_adv( cur_data?cur_data->fsa_cause:C_FSA_INTERNAL, I_NULL, cur_data?cur_data->data:NULL, fsa_actions, TRUE, __FUNCTION__); } /* reset the action list */ fsa_actions = A_NOTHING; /* register the error */ register_fsa_input_adv( cause, input, new_data, A_NOTHING, TRUE, raised_from); } static gboolean last_was_vote = FALSE; int register_fsa_input_adv( enum crmd_fsa_cause cause, enum crmd_fsa_input input, void *data, long long with_actions, gboolean prepend, const char *raised_from) { unsigned old_len = g_list_length(fsa_message_queue); fsa_data_t *fsa_data = NULL; last_data_id++; crm_debug_2("%s raised FSA input %d (%s) (cause=%s) %s data", raised_from, last_data_id, fsa_input2string(input), fsa_cause2string(cause), data?"with":"without"); if(input == I_WAIT_FOR_EVENT) { do_fsa_stall = TRUE; crm_debug("Stalling the FSA pending further input: cause=%s", fsa_cause2string(cause)); if(old_len > 0) { crm_warn("%s stalled the FSA with pending inputs", raised_from); fsa_dump_queue(LOG_DEBUG); } if(data == NULL) { set_bit_inplace(fsa_actions, with_actions); with_actions = A_NOTHING; return 0; } crm_err("%s stalled the FSA with data - this may be broken", raised_from); } if(old_len == 0) { last_was_vote = FALSE; } if(input == I_NULL && with_actions == A_NOTHING /* && data == NULL */){ /* no point doing anything */ crm_err("Cannot add entry to queue: no input and no action"); return 0; } else if(data == NULL) { last_was_vote = FALSE; #if 0 } else if(last_was_vote && cause == C_HA_MESSAGE && input == I_ROUTER) { const char *op = cl_get_string( ((ha_msg_input_t*)data)->msg, F_CRM_TASK); if(safe_str_eq(op, CRM_OP_VOTE)) { /* It is always safe to treat N successive votes as * a single one * * If all the discarded votes are more "loosing" than * the first then the result is accurate * (win or loose). * * If any of the discarded votes are less "loosing" * than the first then we will cast our vote and the * eventual winner will vote us down again (which * even in the case that N=2, is no worse than if we * had not disarded the vote). */ crm_debug_2("Vote compression: %d", old_len); return 0; } #endif } else if (cause == C_HA_MESSAGE && input == I_ROUTER) { const char *op = cl_get_string( ((ha_msg_input_t*)data)->msg, F_CRM_TASK); if(safe_str_eq(op, CRM_OP_VOTE)) { last_was_vote = TRUE; crm_debug_3("Added vote: %d", old_len); } } else { last_was_vote = FALSE; } crm_malloc0(fsa_data, sizeof(fsa_data_t)); fsa_data->id = last_data_id; fsa_data->fsa_input = input; fsa_data->fsa_cause = cause; fsa_data->origin = raised_from; fsa_data->data = NULL; fsa_data->data_type = fsa_dt_none; fsa_data->actions = with_actions; if(with_actions != A_NOTHING) { crm_debug_3("Adding actions %.16llx to input", with_actions); } if(data != NULL) { switch(cause) { case C_FSA_INTERNAL: case C_CRMD_STATUS_CALLBACK: case C_IPC_MESSAGE: case C_HA_MESSAGE: crm_debug_3("Copying %s data from %s as a HA msg", fsa_cause2string(cause), raised_from); fsa_data->data = copy_ha_msg_input(data); fsa_data->data_type = fsa_dt_ha_msg; break; case C_LRM_OP_CALLBACK: crm_debug_3("Copying %s data from %s as lrm_op_t", fsa_cause2string(cause), raised_from); fsa_data->data = copy_lrm_op((lrm_op_t*)data); fsa_data->data_type = fsa_dt_lrm; break; case C_CCM_CALLBACK: crm_debug_3("Copying %s data from %s as CCM data", fsa_cause2string(cause), raised_from); fsa_data->data = copy_ccm_data(data); fsa_data->data_type = fsa_dt_ccm; break; case C_SUBSYSTEM_CONNECT: case C_LRM_MONITOR_CALLBACK: case C_TIMER_POPPED: case C_SHUTDOWN: case C_HEARTBEAT_FAILED: case C_HA_DISCONNECT: case C_ILLEGAL: case C_UNKNOWN: case C_STARTUP: crm_err("Copying %s data (from %s)" " not yet implemented", fsa_cause2string(cause), raised_from); exit(1); break; } crm_debug_4("%s data copied", fsa_cause2string(fsa_data->fsa_cause)); } /* make sure to free it properly later */ if(prepend) { crm_debug_2("Prepending input"); fsa_message_queue = g_list_prepend(fsa_message_queue, fsa_data); } else { fsa_message_queue = g_list_append(fsa_message_queue, fsa_data); } crm_debug_2("Queue len: %d", g_list_length(fsa_message_queue)); fsa_dump_queue(LOG_DEBUG_2); if(old_len == g_list_length(fsa_message_queue)){ crm_err("Couldnt add message to the queue"); } if(fsa_source) { crm_debug_3("Triggering FSA: %s", __FUNCTION__); G_main_set_trigger(fsa_source); } return last_data_id; } void fsa_dump_queue(int log_level) { if(log_level < (int)crm_log_level) { return; } slist_iter( data, fsa_data_t, fsa_message_queue, lpc, do_crm_log(log_level, "queue[%d(%d)]: input %s raised by %s()\t(cause=%s)", lpc, data->id, fsa_input2string(data->fsa_input), data->origin, fsa_cause2string(data->fsa_cause)); ); } ha_msg_input_t * copy_ha_msg_input(ha_msg_input_t *orig) { ha_msg_input_t *input_copy = NULL; crm_malloc0(input_copy, sizeof(ha_msg_input_t)); if(orig != NULL) { crm_debug_4("Copy msg"); input_copy->msg = ha_msg_copy(orig->msg); if(orig->xml != NULL) { crm_debug_4("Copy xml"); input_copy->xml = copy_xml(orig->xml); } } else { crm_debug_3("No message to copy"); } return input_copy; } void delete_fsa_input(fsa_data_t *fsa_data) { lrm_op_t *op = NULL; crm_data_t *foo = NULL; struct crmd_ccm_data_s *ccm_input = NULL; if(fsa_data == NULL) { return; } crm_debug_4("About to free %s data", fsa_cause2string(fsa_data->fsa_cause)); if(fsa_data->data != NULL) { switch(fsa_data->data_type) { case fsa_dt_ha_msg: delete_ha_msg_input(fsa_data->data); break; case fsa_dt_xml: foo = fsa_data->data; free_xml(foo); break; case fsa_dt_lrm: op = (lrm_op_t*)fsa_data->data; free_lrm_op(op); break; case fsa_dt_ccm: ccm_input = (struct crmd_ccm_data_s *) fsa_data->data; - - crm_free(ccm_input->oc); - crm_free(ccm_input); + delete_ccm_data(ccm_input); break; case fsa_dt_none: if(fsa_data->data != NULL) { crm_err("Dont know how to free %s data from %s", fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin); exit(1); } break; } crm_debug_4("%s data freed", fsa_cause2string(fsa_data->fsa_cause)); } crm_free(fsa_data); } /* returns the next message */ fsa_data_t * get_message(void) { fsa_data_t* message = g_list_nth_data(fsa_message_queue, 0); fsa_message_queue = g_list_remove(fsa_message_queue, message); crm_debug_2("Processing input %d", message->id); return message; } /* returns the current head of the FIFO queue */ gboolean is_message(void) { return (g_list_length(fsa_message_queue) > 0); } void * fsa_typed_data_adv( fsa_data_t *fsa_data, enum fsa_data_type a_type, const char *caller) { void *ret_val = NULL; if(fsa_data == NULL) { do_crm_log(LOG_ERR, "%s: No FSA data available", caller); } else if(fsa_data->data == NULL) { do_crm_log(LOG_ERR, "%s: No message data available", caller); } else if(fsa_data->data_type != a_type) { do_crm_log(LOG_CRIT, "%s: Message data was the wrong type! %d vs. requested=%d." " Origin: %s", caller, fsa_data->data_type, a_type, fsa_data->origin); CRM_ASSERT(fsa_data->data_type == a_type); } else { ret_val = fsa_data->data; } return ret_val; } /* A_MSG_ROUTE */ enum crmd_fsa_input do_msg_route(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg); route_message(msg_data->fsa_cause, input); return I_NULL; } void route_message(enum crmd_fsa_cause cause, ha_msg_input_t *input) { enum crmd_fsa_input result = I_NULL; CRM_CHECK(cause == C_IPC_MESSAGE || cause == C_HA_MESSAGE, return); /* try passing the buck first */ crm_debug_4("Attempting to route message"); if(relay_message(input->msg, cause==C_IPC_MESSAGE)) { crm_debug_4("Message routed..."); input->msg = NULL; return; } crm_debug_4("Message wasn't routed... try handling locally"); /* calculate defer */ result = handle_message(input); switch(result) { case I_NULL: crm_debug_4("Message processed"); break; case I_CIB_OP: break; case I_ROUTER: break; default: crm_debug_4("Defering local processing of message"); register_fsa_input_later(cause, result, input); result = I_NULL; break; } if(result != I_NULL) { /* add to the front of the queue */ register_fsa_input(cause, result, input); } } /* * This method frees msg */ gboolean send_request(HA_Message *msg, char **msg_reference) { gboolean was_sent = FALSE; /* crm_log_xml_debug_3(request, "Final request..."); */ if(msg_reference != NULL) { *msg_reference = crm_strdup( cl_get_string(msg, XML_ATTR_REFERENCE)); } was_sent = relay_message(msg, TRUE); if(was_sent == FALSE) { ha_msg_input_t *fsa_input = new_ha_msg_input(msg); register_fsa_input(C_IPC_MESSAGE, I_ROUTER, fsa_input); delete_ha_msg_input(fsa_input); crm_msg_del(msg); } return was_sent; } /* unless more processing is required, relay_message is freed */ gboolean relay_message(HA_Message *relay_message, gboolean originated_locally) { int is_for_dc = 0; int is_for_dcib = 0; int is_for_te = 0; int is_for_crm = 0; int is_for_cib = 0; int is_local = 0; gboolean processing_complete = FALSE; const char *host_to = cl_get_string(relay_message, F_CRM_HOST_TO); const char *sys_to = cl_get_string(relay_message, F_CRM_SYS_TO); const char *sys_from= cl_get_string(relay_message, F_CRM_SYS_FROM); const char *type = cl_get_string(relay_message, F_TYPE); const char *msg_error = NULL; crm_debug_3("Routing message %s", cl_get_string(relay_message, XML_ATTR_REFERENCE)); if(relay_message == NULL) { msg_error = "Cannot route empty message"; } else if(safe_str_eq(CRM_OP_HELLO, cl_get_string(relay_message, F_CRM_TASK))){ /* quietly ignore */ processing_complete = TRUE; } else if(safe_str_neq(type, T_CRM)) { msg_error = "Bad message type"; } else if(sys_to == NULL) { msg_error = "Bad message destination: no subsystem"; } if(msg_error != NULL) { processing_complete = TRUE; crm_err("%s", msg_error); crm_log_message(LOG_WARNING, relay_message); } if(processing_complete) { crm_msg_del(relay_message); return TRUE; } processing_complete = TRUE; is_for_dc = (strcasecmp(CRM_SYSTEM_DC, sys_to) == 0); is_for_dcib = (strcasecmp(CRM_SYSTEM_DCIB, sys_to) == 0); is_for_te = (strcasecmp(CRM_SYSTEM_TENGINE, sys_to) == 0); is_for_cib = (strcasecmp(CRM_SYSTEM_CIB, sys_to) == 0); is_for_crm = (strcasecmp(CRM_SYSTEM_CRMD, sys_to) == 0); is_local = 0; if(host_to == NULL || strlen(host_to) == 0) { if(is_for_dc || is_for_te) { is_local = 0; } else if(is_for_crm && originated_locally) { is_local = 0; } else { is_local = 1; } } else if(strcasecmp(fsa_our_uname, host_to) == 0) { is_local=1; } if(is_for_dc || is_for_dcib || is_for_te) { if(AM_I_DC && is_for_te) { ROUTER_RESULT("Message result: Local relay"); send_msg_via_ipc(relay_message, sys_to); } else if(AM_I_DC) { ROUTER_RESULT("Message result: DC/CRMd process"); processing_complete = FALSE; /* more to be done by caller */ } else if(originated_locally && safe_str_neq(sys_from, CRM_SYSTEM_PENGINE) && safe_str_neq(sys_from, CRM_SYSTEM_TENGINE)) { /* Neither the TE or PE should be sending messages * to DC's on other nodes * * By definition, if we are no longer the DC, then * the PE or TE's data should be discarded */ ROUTER_RESULT("Message result: External relay to DC"); send_msg_via_ha(fsa_cluster_conn, relay_message); } else { /* discard */ ROUTER_RESULT("Message result: Discard, not DC"); crm_msg_del(relay_message); } } else if(is_local && (is_for_crm || is_for_cib)) { ROUTER_RESULT("Message result: CRMd process"); processing_complete = FALSE; /* more to be done by caller */ } else if(is_local) { ROUTER_RESULT("Message result: Local relay"); send_msg_via_ipc(relay_message, sys_to); } else { ROUTER_RESULT("Message result: External relay"); send_msg_via_ha(fsa_cluster_conn, relay_message); } return processing_complete; } gboolean crmd_authorize_message(ha_msg_input_t *client_msg, crmd_client_t *curr_client) { /* check the best case first */ const char *sys_from = cl_get_string(client_msg->msg, F_CRM_SYS_FROM); char *uuid = NULL; char *client_name = NULL; char *major_version = NULL; char *minor_version = NULL; const char *filtered_from; gpointer table_key = NULL; gboolean auth_result = FALSE; struct crm_subsystem_s *the_subsystem = NULL; gboolean can_reply = FALSE; /* no-one has registered with this id */ const char *op = cl_get_string(client_msg->msg, F_CRM_TASK); if (safe_str_neq(CRM_OP_HELLO, op)) { if(sys_from == NULL) { crm_warn("Message [%s] was had no value for %s... discarding", cl_get_string(client_msg->msg, XML_ATTR_REFERENCE), F_CRM_SYS_FROM); return FALSE; } filtered_from = sys_from; /* The CIB can have two names on the DC */ if(strcasecmp(sys_from, CRM_SYSTEM_DCIB) == 0) filtered_from = CRM_SYSTEM_CIB; if (g_hash_table_lookup (ipc_clients, filtered_from) != NULL) { can_reply = TRUE; /* reply can be routed */ } crm_debug_2("Message reply can%s be routed from %s.", can_reply?"":" not", sys_from); if(can_reply == FALSE) { crm_warn("Message [%s] not authorized", cl_get_string(client_msg->msg, XML_ATTR_REFERENCE)); } return can_reply; } crm_debug_3("received client join msg"); crm_log_message(LOG_MSG, client_msg->msg); auth_result = process_hello_message( client_msg->xml, &uuid, &client_name, &major_version, &minor_version); if (auth_result == TRUE) { if(client_name == NULL || uuid == NULL) { crm_err("Bad client details (client_name=%s, uuid=%s)", crm_str(client_name), crm_str(uuid)); auth_result = FALSE; } } if (auth_result == TRUE) { /* check version */ int mav = atoi(major_version); int miv = atoi(minor_version); crm_debug_3("Checking client version number"); if (mav < 0 || miv < 0) { crm_err("Client version (%d:%d) is not acceptable", mav, miv); auth_result = FALSE; } crm_free(major_version); crm_free(minor_version); } - if (auth_result == TRUE) { + if (strcasecmp(CRM_SYSTEM_PENGINE, client_name) == 0) { + the_subsystem = pe_subsystem; + + } else if (strcasecmp(CRM_SYSTEM_TENGINE, client_name) == 0) { + the_subsystem = te_subsystem; + } + + if (auth_result == TRUE && the_subsystem != NULL) { /* if we already have one of those clients * only applies to te, pe etc. not admin clients */ + crm_debug_3("Checking if %s is required/already connected", + client_name); - if (strcasecmp(CRM_SYSTEM_PENGINE, client_name) == 0) { - the_subsystem = pe_subsystem; + table_key = (gpointer)crm_strdup(client_name); + + if(is_set(fsa_input_register, the_subsystem->flag_connected)) { + auth_result = FALSE; + crm_free(table_key); + table_key = NULL; + crm_warn("Bit\t%.16llx set in %.16llx", + the_subsystem->flag_connected, + fsa_input_register); + crm_err("Client %s is already connected", + client_name); - } else if (strcasecmp(CRM_SYSTEM_TENGINE, client_name) == 0) { - the_subsystem = te_subsystem; - } - - if (the_subsystem != NULL) { - /* do we already have one? */ - crm_debug_3("Checking if %s is required/already connected", - client_name); + } else if(FALSE == is_set(fsa_input_register, + the_subsystem->flag_required)) { + crm_warn("Bit\t%.16llx not set in %.16llx", + the_subsystem->flag_connected, + fsa_input_register); + crm_warn("Client %s joined but we dont need it", + client_name); + stop_subsystem(the_subsystem, TRUE); - if(is_set(fsa_input_register, - the_subsystem->flag_connected)) { - auth_result = FALSE; - crm_warn("Bit\t%.16llx set in %.16llx", - the_subsystem->flag_connected, - fsa_input_register); - crm_err("Client %s is already connected", - client_name); - - } else if(FALSE == is_set(fsa_input_register, - the_subsystem->flag_required)) { - auth_result = TRUE; - crm_warn("Bit\t%.16llx not set in %.16llx", - the_subsystem->flag_connected, - fsa_input_register); - crm_warn("Client %s joined but we dont need it", - client_name); - stop_subsystem(the_subsystem, TRUE); - - } else { - the_subsystem->ipc = - curr_client->client_channel; - set_bit_inplace(fsa_input_register, - the_subsystem->flag_connected); - } - } else { - table_key = (gpointer) - generate_hash_key(client_name, uuid); + the_subsystem->ipc = curr_client->client_channel; + set_bit_inplace(fsa_input_register, + the_subsystem->flag_connected); } + + } else { + table_key = (gpointer)generate_hash_key(client_name, uuid); } if (auth_result == TRUE) { - if(table_key == NULL) { - table_key = (gpointer)crm_strdup(client_name); - } crm_debug_2("Accepted client %s", crm_str(table_key)); curr_client->table_key = table_key; curr_client->sub_sys = crm_strdup(client_name); curr_client->uuid = crm_strdup(uuid); g_hash_table_insert (ipc_clients, table_key, curr_client->client_channel); send_hello_message(curr_client->client_channel, "n/a", CRM_SYSTEM_CRMD, "0", "1"); crm_debug_3("Updated client list with %s", crm_str(table_key)); crm_debug_3("Triggering FSA: %s", __FUNCTION__); G_main_set_trigger(fsa_source); + if(the_subsystem != NULL) { + CRM_CHECK(the_subsystem->client == NULL, + process_client_disconnect(the_subsystem->client)); + the_subsystem->client = curr_client; + } + } else { crm_warn("Rejected client logon request"); curr_client->client_channel->ch_status = IPC_DISC_PENDING; } if(uuid != NULL) crm_free(uuid); if(minor_version != NULL) crm_free(minor_version); if(major_version != NULL) crm_free(major_version); if(client_name != NULL) crm_free(client_name); /* hello messages should never be processed further */ return FALSE; } enum crmd_fsa_input handle_message(ha_msg_input_t *stored_msg) { enum crmd_fsa_input next_input = I_NULL; const char *type = NULL; if(stored_msg == NULL || stored_msg->msg == NULL) { crm_err("No message to handle"); return I_NULL; } type = cl_get_string(stored_msg->msg, F_CRM_MSG_TYPE); if(safe_str_eq(type, XML_ATTR_REQUEST)) { next_input = handle_request(stored_msg); } else if(safe_str_eq(type, XML_ATTR_RESPONSE)) { next_input = handle_response(stored_msg); } else { crm_err("Unknown message type: %s", type); } /* crm_debug_2("%s: Next input is %s", __FUNCTION__, */ /* fsa_input2string(next_input)); */ return next_input; } #define schedule_pe() \ { \ next_input = I_PE_CALC; \ if(fsa_pe_ref) { \ crm_debug("Cancelling %s...", fsa_pe_ref); \ crm_free(fsa_pe_ref); \ fsa_pe_ref = NULL; \ } \ } enum crmd_fsa_input handle_request(ha_msg_input_t *stored_msg) { HA_Message *msg = NULL; enum crmd_fsa_input next_input = I_NULL; const char *op = cl_get_string(stored_msg->msg, F_CRM_TASK); const char *sys_to = cl_get_string(stored_msg->msg, F_CRM_SYS_TO); const char *host_from = cl_get_string(stored_msg->msg, F_CRM_HOST_FROM); crm_debug_2("Received %s "XML_ATTR_REQUEST" from %s in state %s", op, host_from, fsa_state2string(fsa_state)); if(op == NULL) { crm_err("Bad message"); crm_log_message(LOG_ERR, stored_msg->msg); /*========== common actions ==========*/ } else if(strcasecmp(op, CRM_OP_NOOP) == 0) { crm_debug_2("no-op from %s", crm_str(host_from)); } else if(strcasecmp(op, CRM_OP_NOVOTE) == 0) { register_fsa_input_adv(C_HA_MESSAGE, I_NULL, stored_msg, A_ELECTION_COUNT|A_ELECTION_CHECK, FALSE, __FUNCTION__); } else if(strcasecmp(op, CRM_OP_VOTE) == 0) { /* count the vote and decide what to do after that */ register_fsa_input_adv(C_HA_MESSAGE, I_NULL, stored_msg, A_ELECTION_COUNT|A_ELECTION_CHECK, FALSE, __FUNCTION__); /* Sometimes we _must_ go into S_ELECTION */ if(fsa_state == S_HALT) { crm_debug("Forcing an election from S_HALT"); next_input = I_ELECTION; #if 0 } else if(AM_I_DC) { /* This is the old way of doing things but what is gained? */ next_input = I_ELECTION; #endif } } else if(strcasecmp(op, CRM_OP_LOCAL_SHUTDOWN) == 0) { crm_shutdown(SIGTERM); /*next_input = I_SHUTDOWN; */ next_input = I_NULL; } else if(strcasecmp(op, CRM_OP_PING) == 0) { /* eventually do some stuff to figure out * if we /are/ ok */ crm_data_t *ping = createPingAnswerFragment(sys_to, "ok"); crm_xml_add(ping, "crmd_state", fsa_state2string(fsa_state)); crm_info("Current ping state: %s", fsa_state2string(fsa_state)); msg = create_reply(stored_msg->msg, ping); if(relay_message(msg, TRUE) == FALSE) { crm_msg_del(msg); } /* probably better to do this via signals on the * local node */ } else if(strcasecmp(op, CRM_OP_DEBUG_UP) == 0) { alter_debug(DEBUG_INC); crm_info("Debug set to %d", get_crm_log_level()); } else if(strcasecmp(op, CRM_OP_DEBUG_DOWN) == 0) { alter_debug(DEBUG_DEC); crm_info("Debug set to %d", get_crm_log_level()); } else if(strcasecmp(op, CRM_OP_JOIN_OFFER) == 0) { next_input = I_JOIN_OFFER; crm_debug("Raising I_JOIN_OFFER: join-%s", cl_get_string(stored_msg->msg, F_CRM_JOIN_ID)); } else if(strcasecmp(op, CRM_OP_JOIN_ACKNAK) == 0) { next_input = I_JOIN_RESULT; crm_debug("Raising I_JOIN_RESULT: join-%s", cl_get_string(stored_msg->msg, F_CRM_JOIN_ID)); } else if(strcasecmp(op, CRM_OP_LRM_DELETE) == 0 || strcasecmp(op, CRM_OP_LRM_REFRESH) == 0 || strcasecmp(op, CRM_OP_REPROBE) == 0) { cl_msg_modstring(stored_msg->msg, F_CRM_SYS_TO, CRM_SYSTEM_LRMD); next_input = I_ROUTER; /* this functionality should only be enabled * if this is a development build */ } else if(CRM_DEV_BUILD && strcasecmp(op, CRM_OP_DIE) == 0/*constant condition*/) { crm_warn("Test-only code: Killing the CRM without mercy"); crm_warn("Inhibiting respawns"); exit(100); /*========== (NOT_DC)-Only Actions ==========*/ } else if(AM_I_DC == FALSE){ gboolean dc_match = safe_str_eq(host_from, fsa_our_dc); if(dc_match || fsa_our_dc == NULL) { if(strcasecmp(op, CRM_OP_HBEAT) == 0) { crm_debug_3("Received DC heartbeat from %s", host_from); next_input = I_DC_HEARTBEAT; } else if(fsa_our_dc == NULL) { crm_warn("CRMd discarding request: %s" " (DC: %s, from: %s)", op, crm_str(fsa_our_dc), host_from); crm_warn("Ignored Request"); crm_log_message(LOG_WARNING, stored_msg->msg); } else if(strcasecmp(op, CRM_OP_SHUTDOWN) == 0) { next_input = I_STOP; } else { crm_err("CRMd didnt expect request: %s", op); crm_log_message(LOG_ERR, stored_msg->msg); } } else { crm_warn("Discarding %s op from %s", op, host_from); } /*========== DC-Only Actions ==========*/ } else if(AM_I_DC) { const char *message = ha_msg_value(stored_msg->msg, "message"); /* setting "fsa_pe_ref = NULL" makes sure we ignore any * PE reply that might be pending or in the queue while * we ask the CIB for a more up-to-date copy */ if(safe_str_eq(op, CRM_OP_TEABORT)) { crm_debug("Transition cancelled: %s/%s", op, message); clear_bit_inplace(fsa_input_register, R_IN_TRANSITION); if(need_transition(fsa_state)) { schedule_pe(); } else { crm_debug("Filtering %s op in state %s", op, fsa_state2string(fsa_state)); } } else if(strcasecmp(op, CRM_OP_TECOMPLETE) == 0) { crm_debug("Transition complete: %s/%s", op, message); clear_bit_inplace(fsa_input_register, R_IN_TRANSITION); if(fsa_state == S_TRANSITION_ENGINE) { next_input = I_TE_SUCCESS; } else { crm_debug("Filtering %s op in state %s", op, fsa_state2string(fsa_state)); } } else if(strcasecmp(op, CRM_OP_JOIN_ANNOUNCE) == 0) { next_input = I_NODE_JOIN; } else if(strcasecmp(op, CRM_OP_JOIN_REQUEST) == 0) { next_input = I_JOIN_REQUEST; } else if(strcasecmp(op, CRM_OP_JOIN_CONFIRM) == 0) { next_input = I_JOIN_RESULT; } else if(strcasecmp(op, CRM_OP_SHUTDOWN) == 0) { gboolean dc_match = safe_str_eq(host_from, fsa_our_dc); if(dc_match) { crm_err("We didnt ask to be shut down yet our" " TE is telling us too." " Better get out now!"); next_input = I_TERMINATE; } else if(is_set(fsa_input_register, R_SHUTDOWN)) { crm_info("Shutting ourselves down (DC)"); next_input = I_STOP; } else if(fsa_state != S_STOPPING) { crm_err("Another node is asking us to shutdown" " but we think we're ok."); next_input = I_ELECTION; } } else if(strcasecmp(op, CRM_OP_SHUTDOWN_REQ) == 0) { /* a slave wants to shut down */ /* create cib fragment and add to message */ next_input = handle_shutdown_request(stored_msg->msg); } else { crm_err("Unexpected request (%s) sent to the DC", op); crm_log_message(LOG_ERR, stored_msg->msg); } } return next_input; } enum crmd_fsa_input handle_response(ha_msg_input_t *stored_msg) { enum crmd_fsa_input next_input = I_NULL; const char *op = cl_get_string(stored_msg->msg, F_CRM_TASK); const char *sys_from = cl_get_string(stored_msg->msg, F_CRM_SYS_FROM); const char *host_from = cl_get_string(stored_msg->msg, F_CRM_HOST_FROM); const char *msg_ref = cl_get_string(stored_msg->msg, XML_ATTR_REFERENCE); crm_debug_2("Received %s "XML_ATTR_RESPONSE" from %s in state %s", op, host_from, fsa_state2string(fsa_state)); if(op == NULL) { crm_err("Bad message"); crm_log_message(LOG_ERR, stored_msg->msg); } else if(AM_I_DC && strcasecmp(op, CRM_OP_PECALC) == 0) { crm_debug_2("Processing %s reply %s (fsa=%s)", sys_from, msg_ref, crm_str(fsa_pe_ref)); if(msg_ref != NULL && safe_str_eq(msg_ref, fsa_pe_ref)) { next_input = I_PE_SUCCESS; crm_debug_2("Completed: %s...", fsa_pe_ref); crm_free(fsa_pe_ref); fsa_pe_ref = NULL; } else { crm_debug_2("Skipping superceeded reply from %s", sys_from); } } else if(strcasecmp(op, CRM_OP_VOTE) == 0 || strcasecmp(op, CRM_OP_HBEAT) == 0 || strcasecmp(op, CRM_OP_SHUTDOWN_REQ) == 0 || strcasecmp(op, CRM_OP_SHUTDOWN) == 0) { crm_debug_2("Ignoring %s from %s in %s", op, host_from, fsa_state2string(fsa_state)); next_input = I_NULL; } else { crm_err("Unexpected response (op=%s) sent to the %s", op, AM_I_DC?"DC":"CRMd"); next_input = I_NULL; } return next_input; } enum crmd_fsa_input handle_shutdown_request(HA_Message *stored_msg) { /* handle here to avoid potential version issues * where the shutdown message/proceedure may have * been changed in later versions. * * This way the DC is always in control of the shutdown */ time_t now = time(NULL); crm_data_t *node_state = NULL; const char *host_from = cl_get_string(stored_msg, F_CRM_HOST_FROM); if(host_from == NULL) { /* we're shutting down and the DC */ host_from = fsa_our_uname; } crm_info("Creating shutdown request for %s",host_from); crm_log_message(LOG_MSG, stored_msg); node_state = create_node_state( host_from, NULL, NULL, NULL, NULL, CRMD_STATE_INACTIVE, FALSE, __FUNCTION__); crm_xml_add_int(node_state, XML_CIB_ATTR_SHUTDOWN, (int)now); fsa_cib_anon_update(XML_CIB_TAG_STATUS,node_state, cib_quorum_override); crm_log_xml_debug_2(node_state, "Shutdown update"); free_xml(node_state); /* will be picked up by the TE as long as its running */ if(need_transition(fsa_state) && is_set(fsa_input_register, R_TE_CONNECTED) == FALSE) { register_fsa_action(A_TE_CANCEL); } return I_NULL; } /* frees msg upon completion */ gboolean send_msg_via_ha(ll_cluster_t *hb_fd, HA_Message *msg) { int log_level = LOG_DEBUG_3; gboolean broadcast = FALSE; gboolean all_is_good = TRUE; const char *op = cl_get_string(msg, F_CRM_TASK); const char *sys_to = cl_get_string(msg, F_CRM_SYS_TO); const char *host_to = cl_get_string(msg, F_CRM_HOST_TO); if (msg == NULL) { crm_err("Attempt to send NULL Message via HA failed."); all_is_good = FALSE; } else { crm_debug_4("Relaying message to (%s) via HA", host_to); } if (all_is_good) { if (sys_to == NULL || strlen(sys_to) == 0) { crm_err("You did not specify a destination sub-system" " for this message."); all_is_good = FALSE; } } /* There are a number of messages may not need to be ordered. * At a later point perhaps we should detect them and send them * as unordered messages. */ if (all_is_good) { if (host_to == NULL || strlen(host_to) == 0 || safe_str_eq(sys_to, CRM_SYSTEM_DC)) { broadcast = TRUE; all_is_good = send_ha_message(hb_fd, msg, NULL, FALSE); } else { all_is_good = send_ha_message(hb_fd, msg, host_to, FALSE); } } if(all_is_good == FALSE) { log_level = LOG_WARNING; } if(log_level == LOG_WARNING || (safe_str_neq(op, CRM_OP_HBEAT))) { do_crm_log(log_level, "Sending %sHA message (ref=%s) to %s@%s %s.", broadcast?"broadcast ":"directed ", cl_get_string(msg, XML_ATTR_REFERENCE), crm_str(sys_to), host_to==NULL?"":host_to, all_is_good?"succeeded":"failed"); } crm_msg_del(msg); return all_is_good; } /* msg is deleted by the time this returns */ gboolean send_msg_via_ipc(HA_Message *msg, const char *sys) { gboolean send_ok = TRUE; IPC_Channel *client_channel; enum crmd_fsa_input next_input; crm_debug_4("relaying msg to sub_sys=%s via IPC", sys); client_channel = (IPC_Channel*)g_hash_table_lookup(ipc_clients, sys); if(cl_get_string(msg, F_CRM_HOST_FROM) == NULL) { ha_msg_add(msg, F_CRM_HOST_FROM, fsa_our_uname); } if (client_channel != NULL) { crm_debug_3("Sending message via channel %s.", sys); send_ok = send_ipc_message(client_channel, msg); } else if(sys != NULL && strcasecmp(sys, CRM_SYSTEM_CIB) == 0) { crm_err("Sub-system (%s) has been incorporated into the CRMd.", sys); crm_err("Change the way we handle this CIB message"); crm_log_message(LOG_ERR, msg); send_ok = FALSE; } else if(sys != NULL && strcasecmp(sys, CRM_SYSTEM_LRMD) == 0) { fsa_data_t *fsa_data = NULL; ha_msg_input_t *msg_copy = new_ha_msg_input(msg); crm_malloc0(fsa_data, sizeof(fsa_data_t)); fsa_data->fsa_input = I_MESSAGE; fsa_data->fsa_cause = C_IPC_MESSAGE; fsa_data->data = msg_copy; fsa_data->origin = __FUNCTION__; fsa_data->data_type = fsa_dt_ha_msg; #ifdef FSA_TRACE crm_debug_2("Invoking action %s (%.16llx)", fsa_action2string(A_LRM_INVOKE), A_LRM_INVOKE); #endif next_input = do_lrm_invoke(A_LRM_INVOKE, C_IPC_MESSAGE, fsa_state, I_MESSAGE, fsa_data); delete_ha_msg_input(msg_copy); crm_free(fsa_data); /* todo: feed this back in for anything != I_NULL */ #ifdef FSA_TRACE crm_debug_2("Result of action %s was %s", fsa_action2string(A_LRM_INVOKE), fsa_input2string(next_input)); #endif } else { crm_err("Unknown Sub-system (%s)... discarding message.", crm_str(sys)); send_ok = FALSE; } crm_msg_del(msg); return send_ok; } void msg_queue_helper(void) { IPC_Channel *ipc = NULL; if(fsa_cluster_conn != NULL) { ipc = fsa_cluster_conn->llc_ops->ipcchan( fsa_cluster_conn); } if(ipc != NULL) { ipc->ops->resume_io(ipc); } /* g_hash_table_foreach_remove(ipc_clients, ipc_queue_helper, NULL); */ } gboolean ipc_queue_helper(gpointer key, gpointer value, gpointer user_data) { crmd_client_t *ipc_client = value; if(ipc_client->client_channel != NULL) { ipc_client->client_channel->ops->is_message_pending(ipc_client->client_channel); } return FALSE; } diff --git a/crm/crmd/utils.c b/crm/crmd/utils.c index f8ecb097f5..cc96f0ca04 100644 --- a/crm/crmd/utils.c +++ b/crm/crmd/utils.c @@ -1,1395 +1,1460 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include void copy_ccm_node(oc_node_t a_node, oc_node_t *a_node_copy); /* A_DC_TIMER_STOP, A_DC_TIMER_START, * A_FINALIZE_TIMER_STOP, A_FINALIZE_TIMER_START * A_INTEGRATE_TIMER_STOP, A_INTEGRATE_TIMER_START */ enum crmd_fsa_input do_timer_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { gboolean timer_op_ok = TRUE; enum crmd_fsa_input result = I_NULL; if(action & A_DC_TIMER_STOP) { timer_op_ok = crm_timer_stop(election_trigger); } else if(action & A_FINALIZE_TIMER_STOP) { timer_op_ok = crm_timer_stop(finalization_timer); } else if(action & A_INTEGRATE_TIMER_STOP) { timer_op_ok = crm_timer_stop(integration_timer); /* } else if(action & A_ELECTION_TIMEOUT_STOP) { */ /* timer_op_ok = crm_timer_stop(election_timeout); */ } /* dont start a timer that wasnt already running */ if(action & A_DC_TIMER_START && timer_op_ok) { crm_timer_start(election_trigger); if(AM_I_DC) { /* there can be only one */ result = I_ELECTION; } } else if(action & A_FINALIZE_TIMER_START) { crm_timer_start(finalization_timer); } else if(action & A_INTEGRATE_TIMER_START) { crm_timer_start(integration_timer); /* } else if(action & A_ELECTION_TIMEOUT_START) { */ /* crm_timer_start(election_timeout); */ } return I_NULL; } static const char * get_timer_desc(fsa_timer_t *timer) { if(timer == election_trigger) { return "Election Trigger"; } else if(timer == election_timeout) { return "Election Timeout"; } else if(timer == shutdown_escalation_timer) { return "Shutdown Escalation"; } else if(timer == integration_timer) { return "Integration Timer"; } else if(timer == finalization_timer) { return "Finalization Timer"; } else if(timer == wait_timer) { return "Wait Timer"; } else if(timer == recheck_timer) { return "PEngine Recheck Timer"; } return "Unknown Timer"; } gboolean crm_timer_popped(gpointer data) { fsa_timer_t *timer = (fsa_timer_t *)data; if(timer == wait_timer || timer == recheck_timer || timer == finalization_timer || timer == election_trigger) { crm_info("%s (%s) just popped!", get_timer_desc(timer), fsa_input2string(timer->fsa_input)); } else { crm_err("%s (%s) just popped!", get_timer_desc(timer), fsa_input2string(timer->fsa_input)); } if(timer->repeat == FALSE) { crm_timer_stop(timer); /* make it _not_ go off again */ } if(timer->fsa_input == I_INTEGRATED) { register_fsa_input_before( C_TIMER_POPPED, timer->fsa_input, NULL); } else if(timer->fsa_input == I_PE_CALC && fsa_state != S_IDLE) { crm_debug("Discarding %s event in state: %s", fsa_input2string(timer->fsa_input), fsa_state2string(fsa_state)); } else if(timer->fsa_input == I_FINALIZED && fsa_state != S_FINALIZE_JOIN) { crm_debug("Discarding %s event in state: %s", fsa_input2string(timer->fsa_input), fsa_state2string(fsa_state)); } else if(timer->fsa_input != I_NULL) { register_fsa_input(C_TIMER_POPPED, timer->fsa_input, NULL); } crm_debug_3("Triggering FSA: %s", __FUNCTION__); G_main_set_trigger(fsa_source); return TRUE; } gboolean crm_timer_start(fsa_timer_t *timer) { const char *timer_desc = get_timer_desc(timer); if(timer->source_id == 0 && timer->period_ms > 0) { timer->source_id = Gmain_timeout_add( timer->period_ms, timer->callback, (void*)timer); CRM_ASSERT(timer->source_id != 0); crm_debug("Started %s (%s:%dms), src=%d", timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms, timer->source_id); } else if(timer->period_ms < 0) { crm_err("Tried to start %s (%s:%dms) with a -ve period", timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms); } else { crm_debug("%s (%s:%dms) already running: src=%d", timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms, timer->source_id); return FALSE; } return TRUE; } gboolean crm_timer_stop(fsa_timer_t *timer) { const char *timer_desc = get_timer_desc(timer); if(timer == NULL) { crm_err("Attempted to stop NULL timer"); return FALSE; } else if(timer->source_id != 0) { crm_debug("Stopping %s (%s:%dms), src=%d", timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms, timer->source_id); Gmain_timeout_remove(timer->source_id); timer->source_id = 0; } else { crm_debug_2("%s (%s:%dms) already stopped", timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms); return FALSE; } return TRUE; } long long toggle_bit(long long action_list, long long action) { crm_debug_5("Toggling bit %.16llx", action); action_list ^= action; crm_debug_5("Result %.16llx", action_list & action); return action_list; } long long clear_bit(long long action_list, long long action) { unsigned int level = LOG_DEBUG_5; do_crm_log(level, "Clearing bit\t%.16llx", action); /* ensure its set */ action_list |= action; /* then toggle */ action_list = action_list ^ action; return action_list; } long long set_bit(long long action_list, long long action) { unsigned int level = LOG_DEBUG_5; do_crm_log(level, "Setting bit\t%.16llx", action); action_list |= action; return action_list; } gboolean is_set(long long action_list, long long action) { crm_debug_5("Checking bit\t%.16llx in %.16llx", action, action_list); return ((action_list & action) == action); } gboolean is_set_any(long long action_list, long long action) { crm_debug_5("Checking bit\t%.16llx in %.16llx", action, action_list); return ((action_list & action) != 0); } const char * fsa_input2string(enum crmd_fsa_input input) { const char *inputAsText = NULL; switch(input){ case I_NULL: inputAsText = "I_NULL"; break; case I_CCM_EVENT: inputAsText = "I_CCM_EVENT"; break; case I_CIB_OP: inputAsText = "I_CIB_OP"; break; case I_CIB_UPDATE: inputAsText = "I_CIB_UPDATE"; break; case I_DC_TIMEOUT: inputAsText = "I_DC_TIMEOUT"; break; case I_ELECTION: inputAsText = "I_ELECTION"; break; case I_PE_CALC: inputAsText = "I_PE_CALC"; break; case I_RELEASE_DC: inputAsText = "I_RELEASE_DC"; break; case I_ELECTION_DC: inputAsText = "I_ELECTION_DC"; break; case I_ERROR: inputAsText = "I_ERROR"; break; case I_FAIL: inputAsText = "I_FAIL"; break; case I_INTEGRATED: inputAsText = "I_INTEGRATED"; break; case I_FINALIZED: inputAsText = "I_FINALIZED"; break; case I_NODE_JOIN: inputAsText = "I_NODE_JOIN"; break; case I_JOIN_OFFER: inputAsText = "I_JOIN_OFFER"; break; case I_JOIN_REQUEST: inputAsText = "I_JOIN_REQUEST"; break; case I_JOIN_RESULT: inputAsText = "I_JOIN_RESULT"; break; case I_NOT_DC: inputAsText = "I_NOT_DC"; break; case I_RECOVERED: inputAsText = "I_RECOVERED"; break; case I_RELEASE_FAIL: inputAsText = "I_RELEASE_FAIL"; break; case I_RELEASE_SUCCESS: inputAsText = "I_RELEASE_SUCCESS"; break; case I_RESTART: inputAsText = "I_RESTART"; break; case I_PE_SUCCESS: inputAsText = "I_PE_SUCCESS"; break; case I_ROUTER: inputAsText = "I_ROUTER"; break; case I_SHUTDOWN: inputAsText = "I_SHUTDOWN"; break; case I_STARTUP: inputAsText = "I_STARTUP"; break; case I_TE_SUCCESS: inputAsText = "I_TE_SUCCESS"; break; case I_STOP: inputAsText = "I_STOP"; break; case I_DC_HEARTBEAT: inputAsText = "I_DC_HEARTBEAT"; break; case I_WAIT_FOR_EVENT: inputAsText = "I_WAIT_FOR_EVENT"; break; case I_LRM_EVENT: inputAsText = "I_LRM_EVENT"; break; case I_PENDING: inputAsText = "I_PENDING"; break; case I_HALT: inputAsText = "I_HALT"; break; case I_TERMINATE: inputAsText = "I_TERMINATE"; break; case I_ILLEGAL: inputAsText = "I_ILLEGAL"; break; } if(inputAsText == NULL) { crm_err("Input %d is unknown", input); inputAsText = ""; } return inputAsText; } const char * fsa_state2string(enum crmd_fsa_state state) { const char *stateAsText = NULL; switch(state){ case S_IDLE: stateAsText = "S_IDLE"; break; case S_ELECTION: stateAsText = "S_ELECTION"; break; case S_INTEGRATION: stateAsText = "S_INTEGRATION"; break; case S_FINALIZE_JOIN: stateAsText = "S_FINALIZE_JOIN"; break; case S_NOT_DC: stateAsText = "S_NOT_DC"; break; case S_POLICY_ENGINE: stateAsText = "S_POLICY_ENGINE"; break; case S_RECOVERY: stateAsText = "S_RECOVERY"; break; case S_RELEASE_DC: stateAsText = "S_RELEASE_DC"; break; case S_PENDING: stateAsText = "S_PENDING"; break; case S_STOPPING: stateAsText = "S_STOPPING"; break; case S_TERMINATE: stateAsText = "S_TERMINATE"; break; case S_TRANSITION_ENGINE: stateAsText = "S_TRANSITION_ENGINE"; break; case S_STARTING: stateAsText = "S_STARTING"; break; case S_HALT: stateAsText = "S_HALT"; break; case S_ILLEGAL: stateAsText = "S_ILLEGAL"; break; } if(stateAsText == NULL) { crm_err("State %d is unknown", state); stateAsText = ""; } return stateAsText; } const char * fsa_cause2string(enum crmd_fsa_cause cause) { const char *causeAsText = NULL; switch(cause){ case C_UNKNOWN: causeAsText = "C_UNKNOWN"; break; case C_STARTUP: causeAsText = "C_STARTUP"; break; case C_IPC_MESSAGE: causeAsText = "C_IPC_MESSAGE"; break; case C_HA_MESSAGE: causeAsText = "C_HA_MESSAGE"; break; case C_CCM_CALLBACK: causeAsText = "C_CCM_CALLBACK"; break; case C_TIMER_POPPED: causeAsText = "C_TIMER_POPPED"; break; case C_SHUTDOWN: causeAsText = "C_SHUTDOWN"; break; case C_HEARTBEAT_FAILED: causeAsText = "C_HEARTBEAT_FAILED"; break; case C_SUBSYSTEM_CONNECT: causeAsText = "C_SUBSYSTEM_CONNECT"; break; case C_LRM_OP_CALLBACK: causeAsText = "C_LRM_OP_CALLBACK"; break; case C_LRM_MONITOR_CALLBACK: causeAsText = "C_LRM_MONITOR_CALLBACK"; break; case C_CRMD_STATUS_CALLBACK: causeAsText = "C_CRMD_STATUS_CALLBACK"; break; case C_HA_DISCONNECT: causeAsText = "C_HA_DISCONNECT"; break; case C_FSA_INTERNAL: causeAsText = "C_FSA_INTERNAL"; break; case C_ILLEGAL: causeAsText = "C_ILLEGAL"; break; } if(causeAsText == NULL) { crm_err("Cause %d is unknown", cause); causeAsText = ""; } return causeAsText; } const char * fsa_action2string(long long action) { const char *actionAsText = NULL; switch(action){ case A_NOTHING: actionAsText = "A_NOTHING"; break; case A_ELECTION_START: actionAsText = "A_ELECTION_START"; break; case A_READCONFIG: actionAsText = "A_READCONFIG"; break; case O_RELEASE: actionAsText = "O_RELEASE"; break; case A_STARTUP: actionAsText = "A_STARTUP"; break; case A_STARTED: actionAsText = "A_STARTED"; break; case A_HA_CONNECT: actionAsText = "A_HA_CONNECT"; break; case A_HA_DISCONNECT: actionAsText = "A_HA_DISCONNECT"; break; case A_LRM_CONNECT: actionAsText = "A_LRM_CONNECT"; break; case A_LRM_EVENT: actionAsText = "A_LRM_EVENT"; break; case A_LRM_INVOKE: actionAsText = "A_LRM_INVOKE"; break; case A_LRM_DISCONNECT: actionAsText = "A_LRM_DISCONNECT"; break; case A_CL_JOIN_QUERY: actionAsText = "A_CL_JOIN_QUERY"; break; case A_DC_TIMER_STOP: actionAsText = "A_DC_TIMER_STOP"; break; case A_DC_TIMER_START: actionAsText = "A_DC_TIMER_START"; break; case A_INTEGRATE_TIMER_START: actionAsText = "A_INTEGRATE_TIMER_START"; break; case A_INTEGRATE_TIMER_STOP: actionAsText = "A_INTEGRATE_TIMER_STOP"; break; case A_FINALIZE_TIMER_START: actionAsText = "A_FINALIZE_TIMER_START"; break; case A_FINALIZE_TIMER_STOP: actionAsText = "A_FINALIZE_TIMER_STOP"; break; case A_ELECTION_COUNT: actionAsText = "A_ELECTION_COUNT"; break; case A_ELECTION_VOTE: actionAsText = "A_ELECTION_VOTE"; break; case A_ELECTION_CHECK: actionAsText = "A_ELECTION_CHECK"; break; case A_CL_JOIN_ANNOUNCE: actionAsText = "A_CL_JOIN_ANNOUNCE"; break; case A_CL_JOIN_REQUEST: actionAsText = "A_CL_JOIN_REQUEST"; break; case A_CL_JOIN_RESULT: actionAsText = "A_CL_JOIN_RESULT"; break; case A_DC_JOIN_OFFER_ALL: actionAsText = "A_DC_JOIN_OFFER_ALL"; break; case A_DC_JOIN_OFFER_ONE: actionAsText = "A_DC_JOIN_OFFER_ONE"; break; case A_DC_JOIN_PROCESS_REQ: actionAsText = "A_DC_JOIN_PROCESS_REQ"; break; case A_DC_JOIN_PROCESS_ACK: actionAsText = "A_DC_JOIN_PROCESS_ACK"; break; case A_DC_JOIN_FINALIZE: actionAsText = "A_DC_JOIN_FINALIZE"; break; case A_MSG_PROCESS: actionAsText = "A_MSG_PROCESS"; break; case A_MSG_ROUTE: actionAsText = "A_MSG_ROUTE"; break; case A_RECOVER: actionAsText = "A_RECOVER"; break; case A_DC_RELEASE: actionAsText = "A_DC_RELEASE"; break; case A_DC_RELEASED: actionAsText = "A_DC_RELEASED"; break; case A_DC_TAKEOVER: actionAsText = "A_DC_TAKEOVER"; break; case A_SHUTDOWN: actionAsText = "A_SHUTDOWN"; break; case A_SHUTDOWN_REQ: actionAsText = "A_SHUTDOWN_REQ"; break; case A_STOP: actionAsText = "A_STOP "; break; case A_EXIT_0: actionAsText = "A_EXIT_0"; break; case A_EXIT_1: actionAsText = "A_EXIT_1"; break; case A_CCM_CONNECT: actionAsText = "A_CCM_CONNECT"; break; case A_CCM_DISCONNECT: actionAsText = "A_CCM_DISCONNECT"; break; case A_CCM_EVENT: actionAsText = "A_CCM_EVENT"; break; case A_CCM_UPDATE_CACHE: actionAsText = "A_CCM_UPDATE_CACHE"; break; case A_CIB_BUMPGEN: actionAsText = "A_CIB_BUMPGEN"; break; case A_CIB_INVOKE: actionAsText = "A_CIB_INVOKE"; break; case O_CIB_RESTART: actionAsText = "O_CIB_RESTART"; break; case A_CIB_START: actionAsText = "A_CIB_START"; break; case A_CIB_STOP: actionAsText = "A_CIB_STOP"; break; case A_TE_INVOKE: actionAsText = "A_TE_INVOKE"; break; case O_TE_RESTART: actionAsText = "O_TE_RESTART"; break; case A_TE_START: actionAsText = "A_TE_START"; break; case A_TE_STOP: actionAsText = "A_TE_STOP"; break; case A_TE_HALT: actionAsText = "A_TE_HALT"; break; case A_TE_CANCEL: actionAsText = "A_TE_CANCEL"; break; case A_PE_INVOKE: actionAsText = "A_PE_INVOKE"; break; case O_PE_RESTART: actionAsText = "O_PE_RESTART"; break; case A_PE_START: actionAsText = "A_PE_START"; break; case A_PE_STOP: actionAsText = "A_PE_STOP"; break; case A_NODE_BLOCK: actionAsText = "A_NODE_BLOCK"; break; case A_UPDATE_NODESTATUS: actionAsText = "A_UPDATE_NODESTATUS"; break; case A_LOG: actionAsText = "A_LOG "; break; case A_ERROR: actionAsText = "A_ERROR "; break; case A_WARN: actionAsText = "A_WARN "; break; } if(actionAsText == NULL) { crm_err("Action %.16llx is unknown", action); actionAsText = ""; } return actionAsText; } void fsa_dump_inputs(int log_level, const char *text, long long input_register) { if(input_register == A_NOTHING) { return; } if(text == NULL) { text = "Input register contents:"; } if(is_set(input_register, R_THE_DC)) { do_crm_log(log_level, "%s %.16llx (R_THE_DC)", text, R_THE_DC); } if(is_set(input_register, R_STARTING)) { do_crm_log(log_level, "%s %.16llx (R_STARTING)", text, R_STARTING); } if(is_set(input_register, R_SHUTDOWN)) { do_crm_log(log_level, "%s %.16llx (R_SHUTDOWN)", text, R_SHUTDOWN); } if(is_set(input_register, R_STAYDOWN)) { do_crm_log(log_level, "%s %.16llx (R_STAYDOWN)", text, R_STAYDOWN); } if(is_set(input_register, R_JOIN_OK)) { do_crm_log(log_level, "%s %.16llx (R_JOIN_OK)", text, R_JOIN_OK); } if(is_set(input_register, R_READ_CONFIG)) { do_crm_log(log_level, "%s %.16llx (R_READ_CONFIG)", text, R_READ_CONFIG); } if(is_set(input_register, R_INVOKE_PE)) { do_crm_log(log_level, "%s %.16llx (R_INVOKE_PE)", text, R_INVOKE_PE); } if(is_set(input_register, R_CIB_CONNECTED)) { do_crm_log(log_level, "%s %.16llx (R_CIB_CONNECTED)", text, R_CIB_CONNECTED); } if(is_set(input_register, R_PE_CONNECTED)) { do_crm_log(log_level, "%s %.16llx (R_PE_CONNECTED)", text, R_PE_CONNECTED); } if(is_set(input_register, R_TE_CONNECTED)) { do_crm_log(log_level, "%s %.16llx (R_TE_CONNECTED)", text, R_TE_CONNECTED); } if(is_set(input_register, R_LRM_CONNECTED)) { do_crm_log(log_level, "%s %.16llx (R_LRM_CONNECTED)", text, R_LRM_CONNECTED); } if(is_set(input_register, R_CIB_REQUIRED)) { do_crm_log(log_level, "%s %.16llx (R_CIB_REQUIRED)", text, R_CIB_REQUIRED); } if(is_set(input_register, R_PE_REQUIRED)) { do_crm_log(log_level, "%s %.16llx (R_PE_REQUIRED)", text, R_PE_REQUIRED); } if(is_set(input_register, R_TE_REQUIRED)) { do_crm_log(log_level, "%s %.16llx (R_TE_REQUIRED)", text, R_TE_REQUIRED); } if(is_set(input_register, R_REQ_PEND)) { do_crm_log(log_level, "%s %.16llx (R_REQ_PEND)", text, R_REQ_PEND); } if(is_set(input_register, R_PE_PEND)) { do_crm_log(log_level, "%s %.16llx (R_PE_PEND)", text, R_PE_PEND); } if(is_set(input_register, R_TE_PEND)) { do_crm_log(log_level, "%s %.16llx (R_TE_PEND)", text, R_TE_PEND); } if(is_set(input_register, R_RESP_PEND)) { do_crm_log(log_level, "%s %.16llx (R_RESP_PEND)", text, R_RESP_PEND); } if(is_set(input_register, R_CIB_DONE)) { do_crm_log(log_level, "%s %.16llx (R_CIB_DONE)", text, R_CIB_DONE); } if(is_set(input_register, R_HAVE_CIB)) { do_crm_log(log_level, "%s %.16llx (R_HAVE_CIB)", text, R_HAVE_CIB); } if(is_set(input_register, R_CIB_ASKED)) { do_crm_log(log_level, "%s %.16llx (R_CIB_ASKED)", text, R_CIB_ASKED); } if(is_set(input_register, R_CCM_DATA)) { do_crm_log(log_level, "%s %.16llx (R_CCM_DATA)", text, R_CCM_DATA); } if(is_set(input_register, R_PEER_DATA)) { do_crm_log(log_level, "%s %.16llx (R_PEER_DATA)", text, R_PEER_DATA); } if(is_set(input_register, R_IN_RECOVERY)) { do_crm_log(log_level, "%s %.16llx (R_IN_RECOVERY)", text, R_IN_RECOVERY); } } void fsa_dump_actions(long long action, const char *text) { int log_level = LOG_DEBUG_3; if(is_set(action, A_READCONFIG)) { do_crm_log(log_level, "Action %.16llx (A_READCONFIG) %s", A_READCONFIG, text); } if(is_set(action, A_STARTUP)) { do_crm_log(log_level, "Action %.16llx (A_STARTUP) %s", A_STARTUP, text); } if(is_set(action, A_STARTED)) { do_crm_log(log_level, "Action %.16llx (A_STARTED) %s", A_STARTED, text); } if(is_set(action, A_HA_CONNECT)) { do_crm_log(log_level, "Action %.16llx (A_CONNECT) %s", A_HA_CONNECT, text); } if(is_set(action, A_HA_DISCONNECT)) { do_crm_log(log_level, "Action %.16llx (A_DISCONNECT) %s", A_HA_DISCONNECT, text); } if(is_set(action, A_LRM_CONNECT)) { do_crm_log(log_level, "Action %.16llx (A_LRM_CONNECT) %s", A_LRM_CONNECT, text); } if(is_set(action, A_LRM_EVENT)) { do_crm_log(log_level, "Action %.16llx (A_LRM_EVENT) %s", A_LRM_EVENT, text); } if(is_set(action, A_LRM_INVOKE)) { do_crm_log(log_level, "Action %.16llx (A_LRM_INVOKE) %s", A_LRM_INVOKE, text); } if(is_set(action, A_LRM_DISCONNECT)) { do_crm_log(log_level, "Action %.16llx (A_LRM_DISCONNECT) %s", A_LRM_DISCONNECT, text); } if(is_set(action, A_DC_TIMER_STOP)) { do_crm_log(log_level, "Action %.16llx (A_DC_TIMER_STOP) %s", A_DC_TIMER_STOP, text); } if(is_set(action, A_DC_TIMER_START)) { do_crm_log(log_level, "Action %.16llx (A_DC_TIMER_START) %s", A_DC_TIMER_START, text); } if(is_set(action, A_INTEGRATE_TIMER_START)) { do_crm_log(log_level, "Action %.16llx (A_INTEGRATE_TIMER_START) %s", A_INTEGRATE_TIMER_START, text); } if(is_set(action, A_INTEGRATE_TIMER_STOP)) { do_crm_log(log_level, "Action %.16llx (A_INTEGRATE_TIMER_STOP) %s", A_INTEGRATE_TIMER_STOP, text); } if(is_set(action, A_FINALIZE_TIMER_START)) { do_crm_log(log_level, "Action %.16llx (A_FINALIZE_TIMER_START) %s", A_FINALIZE_TIMER_START, text); } if(is_set(action, A_FINALIZE_TIMER_STOP)) { do_crm_log(log_level, "Action %.16llx (A_FINALIZE_TIMER_STOP) %s", A_FINALIZE_TIMER_STOP, text); } if(is_set(action, A_ELECTION_COUNT)) { do_crm_log(log_level, "Action %.16llx (A_ELECTION_COUNT) %s", A_ELECTION_COUNT, text); } if(is_set(action, A_ELECTION_VOTE)) { do_crm_log(log_level, "Action %.16llx (A_ELECTION_VOTE) %s", A_ELECTION_VOTE, text); } if(is_set(action, A_ELECTION_CHECK)) { do_crm_log(log_level, "Action %.16llx (A_ELECTION_CHECK) %s", A_ELECTION_CHECK, text); } if(is_set(action, A_CL_JOIN_ANNOUNCE)) { do_crm_log(log_level, "Action %.16llx (A_CL_JOIN_ANNOUNCE) %s", A_CL_JOIN_ANNOUNCE, text); } if(is_set(action, A_CL_JOIN_REQUEST)) { do_crm_log(log_level, "Action %.16llx (A_CL_JOIN_REQUEST) %s", A_CL_JOIN_REQUEST, text); } if(is_set(action, A_CL_JOIN_RESULT)) { do_crm_log(log_level, "Action %.16llx (A_CL_JOIN_RESULT) %s", A_CL_JOIN_RESULT, text); } if(is_set(action, A_DC_JOIN_OFFER_ALL)) { do_crm_log(log_level, "Action %.16llx (A_DC_JOIN_OFFER_ALL) %s", A_DC_JOIN_OFFER_ALL, text); } if(is_set(action, A_DC_JOIN_OFFER_ONE)) { do_crm_log(log_level, "Action %.16llx (A_DC_JOIN_OFFER_ONE) %s", A_DC_JOIN_OFFER_ONE, text); } if(is_set(action, A_DC_JOIN_PROCESS_REQ)) { do_crm_log(log_level, "Action %.16llx (A_DC_JOIN_PROCESS_REQ) %s", A_DC_JOIN_PROCESS_REQ, text); } if(is_set(action, A_DC_JOIN_PROCESS_ACK)) { do_crm_log(log_level, "Action %.16llx (A_DC_JOIN_PROCESS_ACK) %s", A_DC_JOIN_PROCESS_ACK, text); } if(is_set(action, A_DC_JOIN_FINALIZE)) { do_crm_log(log_level, "Action %.16llx (A_DC_JOIN_FINALIZE) %s", A_DC_JOIN_FINALIZE, text); } if(is_set(action, A_MSG_PROCESS)) { do_crm_log(log_level, "Action %.16llx (A_MSG_PROCESS) %s", A_MSG_PROCESS, text); } if(is_set(action, A_MSG_ROUTE)) { do_crm_log(log_level, "Action %.16llx (A_MSG_ROUTE) %s", A_MSG_ROUTE, text); } if(is_set(action, A_RECOVER)) { do_crm_log(log_level, "Action %.16llx (A_RECOVER) %s", A_RECOVER, text); } if(is_set(action, A_DC_RELEASE)) { do_crm_log(log_level, "Action %.16llx (A_DC_RELEASE) %s", A_DC_RELEASE, text); } if(is_set(action, A_DC_RELEASED)) { do_crm_log(log_level, "Action %.16llx (A_DC_RELEASED) %s", A_DC_RELEASED, text); } if(is_set(action, A_DC_TAKEOVER)) { do_crm_log(log_level, "Action %.16llx (A_DC_TAKEOVER) %s", A_DC_TAKEOVER, text); } if(is_set(action, A_SHUTDOWN)) { do_crm_log(log_level, "Action %.16llx (A_SHUTDOWN) %s", A_SHUTDOWN, text); } if(is_set(action, A_SHUTDOWN_REQ)) { do_crm_log(log_level, "Action %.16llx (A_SHUTDOWN_REQ) %s", A_SHUTDOWN_REQ, text); } if(is_set(action, A_STOP)) { do_crm_log(log_level, "Action %.16llx (A_STOP ) %s", A_STOP , text); } if(is_set(action, A_EXIT_0)) { do_crm_log(log_level, "Action %.16llx (A_EXIT_0) %s", A_EXIT_0, text); } if(is_set(action, A_EXIT_1)) { do_crm_log(log_level, "Action %.16llx (A_EXIT_1) %s", A_EXIT_1, text); } if(is_set(action, A_CCM_CONNECT)) { do_crm_log(log_level, "Action %.16llx (A_CCM_CONNECT) %s", A_CCM_CONNECT, text); } if(is_set(action, A_CCM_DISCONNECT)) { do_crm_log(log_level, "Action %.16llx (A_CCM_DISCONNECT) %s", A_CCM_DISCONNECT, text); } if(is_set(action, A_CCM_EVENT)) { do_crm_log(log_level, "Action %.16llx (A_CCM_EVENT) %s", A_CCM_EVENT, text); } if(is_set(action, A_CCM_UPDATE_CACHE)) { do_crm_log(log_level, "Action %.16llx (A_CCM_UPDATE_CACHE) %s", A_CCM_UPDATE_CACHE, text); } if(is_set(action, A_CIB_BUMPGEN)) { do_crm_log(log_level, "Action %.16llx (A_CIB_BUMPGEN) %s", A_CIB_BUMPGEN, text); } if(is_set(action, A_CIB_INVOKE)) { do_crm_log(log_level, "Action %.16llx (A_CIB_INVOKE) %s", A_CIB_INVOKE, text); } if(is_set(action, A_CIB_START)) { do_crm_log(log_level, "Action %.16llx (A_CIB_START) %s", A_CIB_START, text); } if(is_set(action, A_CIB_STOP)) { do_crm_log(log_level, "Action %.16llx (A_CIB_STOP) %s", A_CIB_STOP, text); } if(is_set(action, A_TE_INVOKE)) { do_crm_log(log_level, "Action %.16llx (A_TE_INVOKE) %s", A_TE_INVOKE, text); } if(is_set(action, A_TE_START)) { do_crm_log(log_level, "Action %.16llx (A_TE_START) %s", A_TE_START, text); } if(is_set(action, A_TE_STOP)) { do_crm_log(log_level, "Action %.16llx (A_TE_STOP) %s", A_TE_STOP, text); } if(is_set(action, A_TE_CANCEL)) { do_crm_log(log_level, "Action %.16llx (A_TE_CANCEL) %s", A_TE_CANCEL, text); } if(is_set(action, A_PE_INVOKE)) { do_crm_log(log_level, "Action %.16llx (A_PE_INVOKE) %s", A_PE_INVOKE, text); } if(is_set(action, A_PE_START)) { do_crm_log(log_level, "Action %.16llx (A_PE_START) %s", A_PE_START, text); } if(is_set(action, A_PE_STOP)) { do_crm_log(log_level, "Action %.16llx (A_PE_STOP) %s", A_PE_STOP, text); } if(is_set(action, A_NODE_BLOCK)) { do_crm_log(log_level, "Action %.16llx (A_NODE_BLOCK) %s", A_NODE_BLOCK, text); } if(is_set(action, A_UPDATE_NODESTATUS)) { do_crm_log(log_level, "Action %.16llx (A_UPDATE_NODESTATUS) %s", A_UPDATE_NODESTATUS, text); } if(is_set(action, A_LOG)) { do_crm_log(log_level, "Action %.16llx (A_LOG ) %s", A_LOG, text); } if(is_set(action, A_ERROR)) { do_crm_log(log_level, "Action %.16llx (A_ERROR ) %s", A_ERROR, text); } if(is_set(action, A_WARN)) { do_crm_log(log_level, "Action %.16llx (A_WARN ) %s", A_WARN, text); } } void create_node_entry(const char *uuid, const char *uname, const char *type) { /* make sure a node entry exists for the new node * * this will add anyone except the first ever node in the cluster * since it will also be the DC which doesnt go through the * join process (with itself). We can include a special case * later if desired. */ crm_data_t *tmp1 = create_xml_node(NULL, XML_CIB_TAG_NODE); crm_debug_3("Creating node entry for %s", uname); set_uuid(fsa_cluster_conn, tmp1, XML_ATTR_UUID, uname); crm_xml_add(tmp1, XML_ATTR_UNAME, uname); crm_xml_add(tmp1, XML_ATTR_TYPE, type); fsa_cib_anon_update(XML_CIB_TAG_NODES, tmp1, cib_scope_local|cib_quorum_override); free_xml(tmp1); } - - - struct crmd_ccm_data_s * copy_ccm_data(const struct crmd_ccm_data_s *ccm_input) { const oc_ev_membership_t *oc_in = (const oc_ev_membership_t *)ccm_input->oc; struct crmd_ccm_data_s *ccm_input_copy = NULL; crm_malloc0(ccm_input_copy, sizeof(struct crmd_ccm_data_s)); ccm_input_copy->oc = copy_ccm_oc_data(oc_in); ccm_input_copy->event = ccm_input->event; return ccm_input_copy; } +void +delete_ccm_data(struct crmd_ccm_data_s *ccm_input) +{ + int lpc, offset; + oc_node_t *a_node = NULL; + oc_ev_membership_t *oc_in = ccm_input->oc; + offset = oc_in->m_memb_idx; + for(lpc = 0; lpc < oc_in->m_n_member; lpc++) { + a_node = &oc_in->m_array[lpc+offset]; + crm_free(a_node->node_uname); + } + + offset = oc_in->m_in_idx; + for(lpc = 0; lpc < oc_in->m_n_in; lpc++) { + a_node = &oc_in->m_array[lpc+offset]; + crm_free(a_node->node_uname); + } + + offset = oc_in->m_out_idx; + for(lpc = 0; lpc < oc_in->m_n_out; lpc++) { + a_node = &oc_in->m_array[lpc+offset]; + crm_free(a_node->node_uname); + } + crm_free(ccm_input->oc); + crm_free(ccm_input); +} + oc_ev_membership_t * copy_ccm_oc_data(const oc_ev_membership_t *oc_in) { unsigned lpc = 0; int size = 0; int offset = 0; unsigned num_nodes = 0; oc_ev_membership_t *oc_copy = NULL; if(oc_in->m_n_member > 0 && num_nodes < oc_in->m_n_member + oc_in->m_memb_idx) { num_nodes = oc_in->m_n_member + oc_in->m_memb_idx; crm_debug_3("Updated ccm nodes to %d - 1", num_nodes); } if(oc_in->m_n_in > 0 && num_nodes < oc_in->m_n_in + oc_in->m_in_idx) { num_nodes = oc_in->m_n_in + oc_in->m_in_idx; crm_debug_3("Updated ccm nodes to %d - 2", num_nodes); } if(oc_in->m_n_out > 0 && num_nodes < oc_in->m_n_out + oc_in->m_out_idx) { num_nodes = oc_in->m_n_out + oc_in->m_out_idx; crm_debug_3("Updated ccm nodes to %d - 3", num_nodes); } /* why 2*?? * ccm code does it like this so i guess its right... */ size = sizeof(oc_ev_membership_t) + sizeof(int) + 2*num_nodes*sizeof(oc_node_t); crm_debug_3("Copying %d ccm nodes", num_nodes); crm_malloc0(oc_copy, size); oc_copy->m_instance = oc_in->m_instance; oc_copy->m_n_member = oc_in->m_n_member; oc_copy->m_memb_idx = oc_in->m_memb_idx; oc_copy->m_n_out = oc_in->m_n_out; oc_copy->m_out_idx = oc_in->m_out_idx; oc_copy->m_n_in = oc_in->m_n_in; oc_copy->m_in_idx = oc_in->m_in_idx; crm_debug_3("instance=%d, nodes=%d (idx=%d), new=%d (idx=%d), lost=%d (idx=%d)", oc_in->m_instance, oc_in->m_n_member, oc_in->m_memb_idx, oc_in->m_n_in, oc_in->m_in_idx, oc_in->m_n_out, oc_in->m_out_idx); offset = oc_in->m_memb_idx; for(lpc = 0; lpc < oc_in->m_n_member; lpc++) { oc_node_t a_node = oc_in->m_array[lpc+offset]; oc_node_t *a_node_copy = &(oc_copy->m_array[lpc+offset]); crm_debug_3("Copying ccm member node %d", lpc); - copy_ccm_node(a_node, a_node_copy); - + copy_ccm_node(a_node, a_node_copy); } offset = oc_in->m_in_idx; for(lpc = 0; lpc < oc_in->m_n_in; lpc++) { oc_node_t a_node = oc_in->m_array[lpc+offset]; oc_node_t *a_node_copy = &(oc_copy->m_array[lpc+offset]); crm_debug_3("Copying ccm new node %d", lpc); copy_ccm_node(a_node, a_node_copy); - } offset = oc_in->m_out_idx; for(lpc = 0; lpc < oc_in->m_n_out; lpc++) { oc_node_t a_node = oc_in->m_array[lpc+offset]; oc_node_t *a_node_copy = &(oc_copy->m_array[lpc+offset]); crm_debug_3("Copying ccm lost node %d", lpc); copy_ccm_node(a_node, a_node_copy); } return oc_copy; } void copy_ccm_node(oc_node_t a_node, oc_node_t *a_node_copy) { crm_debug_3("Copying ccm node: id=%d, born=%d, uname=%s", a_node.node_id, a_node.node_born_on, a_node.node_uname); + if(a_node_copy->node_uname != NULL) { + crm_debug_2("%p (%s) already copied", a_node_copy, a_node_copy->node_uname); + return; + } + a_node_copy->node_id = a_node.node_id; - a_node_copy->node_born_on = a_node.node_born_on; + a_node_copy->node_born_on = a_node.node_born_on; a_node_copy->node_uname = NULL; if(a_node.node_uname != NULL) { - a_node_copy->node_uname = - crm_strdup(a_node.node_uname); + a_node_copy->node_uname = crm_strdup(a_node.node_uname); } else { - crm_err("Node Id %d had a NULL uname!", - a_node.node_id); + crm_err("Node Id %d had a NULL uname!", a_node.node_id); } - crm_debug_3("Copied ccm node: id=%d, born=%d, uname=%s", a_node_copy->node_id, a_node_copy->node_born_on, a_node_copy->node_uname); } +static gboolean +ghash_node_clfree(gpointer key, gpointer value, gpointer user_data) +{ + /* value->node_uname is free'd as "key" */ + if(key != NULL) { + crm_free(key); + } + if(value != NULL) { + crm_free(value); + } + return TRUE; +} + +void +free_ccm_cache(oc_node_list_t *tmp) +{ + /* Free the old copy */ + if(tmp == NULL) { + return; + } + + if(tmp->members != NULL) { + g_hash_table_foreach_remove( + tmp->members, ghash_node_clfree, NULL); + } + if(tmp->new_members != NULL) { + g_hash_table_foreach_remove( + tmp->new_members, ghash_node_clfree, NULL); + } + if(tmp->dead_members != NULL) { + g_hash_table_foreach_remove( + tmp->dead_members, ghash_node_clfree, NULL); + } + + crm_free(tmp); + crm_debug_3("Free'd old copies"); +} + crm_data_t* create_node_state( const char *uname, const char *ha_state, const char *ccm_state, const char *crmd_state, const char *join_state, const char *exp_state, gboolean clear_shutdown, const char *src) { crm_data_t *node_state = create_xml_node(NULL, XML_CIB_TAG_STATE); crm_debug_2("%s Creating node state entry for %s", src, uname); set_uuid(fsa_cluster_conn, node_state, XML_ATTR_UUID, uname); if(ID(node_state) == NULL) { crm_debug("Node %s is not a cluster member", uname); free_xml(node_state); return NULL; } crm_xml_add(node_state, XML_ATTR_UNAME, uname); crm_xml_add(node_state, XML_CIB_ATTR_HASTATE, ha_state); crm_xml_add(node_state, XML_CIB_ATTR_INCCM, ccm_state); crm_xml_add(node_state, XML_CIB_ATTR_CRMDSTATE, crmd_state); crm_xml_add(node_state, XML_CIB_ATTR_JOINSTATE, join_state); crm_xml_add(node_state, XML_CIB_ATTR_EXPSTATE, exp_state); crm_xml_add(node_state, XML_ATTR_ORIGIN, src); if(clear_shutdown) { crm_xml_add(node_state, XML_CIB_ATTR_SHUTDOWN, "0"); #if CRM_DEPRECATED_SINCE_2_0_3 crm_xml_add(node_state, "clear_shutdown", "true"); #endif /* crm_xml_add(node_state, */ /* XML_CIB_ATTR_REPLACE, XML_TAG_TRANSIENT_NODEATTRS); */ } crm_log_xml_debug_3(node_state, "created"); return node_state; } gboolean need_transition(enum crmd_fsa_state state) { if(state == S_POLICY_ENGINE || state == S_TRANSITION_ENGINE || state == S_IDLE) { return TRUE; } return FALSE; } extern GHashTable *ipc_clients; void process_client_disconnect(crmd_client_t *curr_client) { struct crm_subsystem_s *the_subsystem = NULL; CRM_CHECK(curr_client != NULL, return); crm_debug_2("received HUP from %s", curr_client->table_key); if (curr_client->sub_sys == NULL) { crm_debug_2("Client hadn't registered with us yet"); } else if (strcasecmp(CRM_SYSTEM_PENGINE, curr_client->sub_sys) == 0) { the_subsystem = pe_subsystem; } else if (strcasecmp(CRM_SYSTEM_TENGINE, curr_client->sub_sys) == 0) { the_subsystem = te_subsystem; } else if (strcasecmp(CRM_SYSTEM_CIB, curr_client->sub_sys) == 0){ the_subsystem = cib_subsystem; } if(the_subsystem != NULL) { the_subsystem->ipc = NULL; + the_subsystem->client = NULL; crm_info("Received HUP from %s:[%d]", the_subsystem->name, the_subsystem->pid); } else { /* else that was a transient client */ crm_debug("Received HUP from transient client"); } if (curr_client->table_key != NULL) { /* - * Key is destroyed below: + * Key is destroyed below as: * curr_client->table_key * Value is cleaned up by: + * crmd_ipc_connection_destroy + * via: * G_main_del_IPC_Channel */ g_hash_table_remove(ipc_clients, curr_client->table_key); } } void update_dc(HA_Message *msg, gboolean assert_same) { const char *dc_version = NULL; const char *welcome_from = NULL; if(msg != NULL) { dc_version = cl_get_string(msg, F_CRM_VERSION); welcome_from = cl_get_string(msg, F_CRM_HOST_FROM); CRM_CHECK(dc_version != NULL, return); CRM_CHECK(welcome_from != NULL, return); if(AM_I_DC) { CRM_CHECK(safe_str_eq(welcome_from, fsa_our_uname), return); } if(assert_same) { CRM_CHECK(fsa_our_dc != NULL, ;); CRM_CHECK(safe_str_eq(fsa_our_dc, welcome_from), ;); } } crm_free(fsa_our_dc); crm_free(fsa_our_dc_version); fsa_our_dc = NULL; fsa_our_dc_version = NULL; if(welcome_from != NULL) { fsa_our_dc = crm_strdup(welcome_from); } if(dc_version != NULL) { fsa_our_dc_version = crm_strdup(dc_version); } crm_info("Set DC to %s (%s)", crm_str(fsa_our_dc), crm_str(fsa_our_dc_version)); } diff --git a/lib/crm/cib/cib_client.c b/lib/crm/cib/cib_client.c index 8afacd9cfe..d9d64b0ab1 100755 --- a/lib/crm/cib/cib_client.c +++ b/lib/crm/cib/cib_client.c @@ -1,1521 +1,1522 @@ /* * Copyright (c) 2004 International Business Machines * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* short term hack to reduce callback messages */ typedef struct cib_native_opaque_s { IPC_Channel *command_channel; IPC_Channel *callback_channel; GCHSource *callback_source; } cib_native_opaque_t; GHashTable *cib_op_callback_table = NULL; gboolean verify_cib_cmds(cib_t *cib); int cib_client_set_op_callback( cib_t *cib, void (*callback)(const struct ha_msg *msg, int call_id, int rc, crm_data_t *output)); int cib_client_noop(cib_t *cib, int call_options); int cib_client_ping(cib_t *cib, crm_data_t **output_data, int call_options); int cib_client_query(cib_t *cib, const char *section, crm_data_t **output_data, int call_options); int cib_client_query_from(cib_t *cib, const char *host, const char *section, crm_data_t **output_data, int call_options); int cib_client_sync(cib_t *cib, const char *section, int call_options); int cib_client_sync_from( cib_t *cib, const char *host, const char *section, int call_options); int cib_client_is_master(cib_t *cib); int cib_client_set_slave(cib_t *cib, int call_options); int cib_client_set_slave_all(cib_t *cib, int call_options); int cib_client_set_master(cib_t *cib, int call_options); int cib_client_bump_epoch(cib_t *cib, int call_options); int cib_client_create(cib_t *cib, const char *section, crm_data_t *data, crm_data_t **output_data, int call_options); int cib_client_modify(cib_t *cib, const char *section, crm_data_t *data, crm_data_t **output_data, int call_options); int cib_client_update(cib_t *cib, const char *section, crm_data_t *data, crm_data_t **output_data, int call_options); int cib_client_replace(cib_t *cib, const char *section, crm_data_t *data, crm_data_t **output_data, int call_options); int cib_client_delete(cib_t *cib, const char *section, crm_data_t *data, crm_data_t **output_data, int call_options); int cib_client_delete_absolute( cib_t *cib, const char *section, crm_data_t *data, crm_data_t **output_data, int call_options); int cib_client_erase( cib_t *cib, crm_data_t **output_data, int call_options); int cib_client_quit(cib_t *cib, int call_options); int cib_client_add_notify_callback( cib_t *cib, const char *event, void (*callback)( const char *event, struct ha_msg *msg)); int cib_client_del_notify_callback( cib_t *cib, const char *event, void (*callback)( const char *event, struct ha_msg *msg)); gint ciblib_GCompareFunc(gconstpointer a, gconstpointer b); extern cib_t *cib_native_new(cib_t *cib); extern void cib_native_delete(cib_t *cib); static enum cib_variant configured_variant = cib_native; /* define of the api functions*/ cib_t* cib_new(void) { cib_t* new_cib = NULL; if(configured_variant != cib_native) { crm_err("Only the native CIB type is currently implemented"); return NULL; } if(cib_op_callback_table != NULL) { g_hash_table_destroy(cib_op_callback_table); cib_op_callback_table = NULL; } if(cib_op_callback_table == NULL) { cib_op_callback_table = g_hash_table_new_full( g_direct_hash, g_direct_equal, NULL, g_hash_destroy_str); } crm_malloc0(new_cib, sizeof(cib_t)); new_cib->call_id = 1; new_cib->type = cib_none; new_cib->state = cib_disconnected; new_cib->op_callback = NULL; new_cib->variant_opaque = NULL; new_cib->notify_list = NULL; /* the rest will get filled in by the variant constructor */ crm_malloc0(new_cib->cmds, sizeof(cib_api_operations_t)); new_cib->cmds->set_op_callback = cib_client_set_op_callback; new_cib->cmds->add_notify_callback = cib_client_add_notify_callback; new_cib->cmds->del_notify_callback = cib_client_del_notify_callback; new_cib->cmds->noop = cib_client_noop; new_cib->cmds->ping = cib_client_ping; new_cib->cmds->query = cib_client_query; new_cib->cmds->sync = cib_client_sync; new_cib->cmds->query_from = cib_client_query_from; new_cib->cmds->sync_from = cib_client_sync_from; new_cib->cmds->is_master = cib_client_is_master; new_cib->cmds->set_master = cib_client_set_master; new_cib->cmds->set_slave = cib_client_set_slave; new_cib->cmds->set_slave_all = cib_client_set_slave_all; new_cib->cmds->bump_epoch = cib_client_bump_epoch; new_cib->cmds->create = cib_client_create; new_cib->cmds->modify = cib_client_modify; new_cib->cmds->update = cib_client_update; new_cib->cmds->replace = cib_client_replace; new_cib->cmds->delete = cib_client_delete; new_cib->cmds->erase = cib_client_erase; new_cib->cmds->quit = cib_client_quit; new_cib->cmds->delete_absolute = cib_client_delete_absolute; cib_native_new(new_cib); if(verify_cib_cmds(new_cib) == FALSE) { return NULL; } return new_cib; } void cib_delete(cib_t *cib) { cib_native_delete(cib); g_hash_table_destroy(cib_op_callback_table); crm_free(cib->cmds); crm_free(cib); } int cib_client_set_op_callback( cib_t *cib, void (*callback)(const struct ha_msg *msg, int call_id, int rc, crm_data_t *output)) { if(callback == NULL) { crm_info("Un-Setting operation callback"); } else { crm_debug_3("Setting operation callback"); } cib->op_callback = callback; return cib_ok; } int cib_client_noop(cib_t *cib, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } return cib->cmds->variant_op( cib, CRM_OP_NOOP, NULL, NULL, NULL, NULL, call_options); } int cib_client_ping(cib_t *cib, crm_data_t **output_data, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } return cib->cmds->variant_op( cib, CRM_OP_PING, NULL,NULL,NULL, output_data, call_options); } int cib_client_query(cib_t *cib, const char *section, crm_data_t **output_data, int call_options) { return cib->cmds->query_from( cib, NULL, section, output_data, call_options); } int cib_client_query_from(cib_t *cib, const char *host, const char *section, crm_data_t **output_data, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } return cib->cmds->variant_op(cib, CIB_OP_QUERY, host, section, NULL, output_data, call_options); } int cib_client_is_master(cib_t *cib) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } return cib->cmds->variant_op( cib, CIB_OP_ISMASTER, NULL, NULL,NULL,NULL, cib_scope_local|cib_sync_call); } int cib_client_set_slave(cib_t *cib, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } return cib->cmds->variant_op( cib, CIB_OP_SLAVE, NULL,NULL,NULL,NULL, call_options); } int cib_client_set_slave_all(cib_t *cib, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } return cib->cmds->variant_op( cib, CIB_OP_SLAVEALL, NULL,NULL,NULL,NULL, call_options); } int cib_client_set_master(cib_t *cib, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } crm_debug_3("Adding cib_scope_local to options"); return cib->cmds->variant_op( cib, CIB_OP_MASTER, NULL,NULL,NULL,NULL, call_options|cib_scope_local); } int cib_client_bump_epoch(cib_t *cib, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } return cib->cmds->variant_op( cib, CIB_OP_BUMP, NULL, NULL, NULL, NULL, call_options); } int cib_client_sync(cib_t *cib, const char *section, int call_options) { return cib->cmds->sync_from(cib, NULL, section, call_options); } int cib_client_sync_from( cib_t *cib, const char *host, const char *section, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } return cib->cmds->variant_op( cib, CIB_OP_SYNC, host, section, NULL, NULL, call_options); } int cib_client_create(cib_t *cib, const char *section, crm_data_t *data, crm_data_t **output_data, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } return cib->cmds->variant_op(cib, CIB_OP_CREATE, NULL, section, data, output_data, call_options); } int cib_client_modify(cib_t *cib, const char *section, crm_data_t *data, crm_data_t **output_data, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } return cib->cmds->variant_op(cib, CIB_OP_MODIFY, NULL, section, data, output_data, call_options); } int cib_client_update(cib_t *cib, const char *section, crm_data_t *data, crm_data_t **output_data, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } return cib->cmds->variant_op(cib, CIB_OP_UPDATE, NULL, section, data, output_data, call_options); } int cib_client_replace(cib_t *cib, const char *section, crm_data_t *data, crm_data_t **output_data, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } else if(data == NULL) { return cib_missing_data; } return cib->cmds->variant_op(cib, CIB_OP_REPLACE, NULL, section, data, output_data, call_options); } int cib_client_delete(cib_t *cib, const char *section, crm_data_t *data, crm_data_t **output_data, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } return cib->cmds->variant_op(cib, CIB_OP_DELETE, NULL, section, data, output_data, call_options); } int cib_client_delete_absolute( cib_t *cib, const char *section, crm_data_t *data, crm_data_t **output_data, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } return cib->cmds->variant_op(cib, CIB_OP_DELETE_ALT, NULL, section, data, output_data, call_options); } int cib_client_erase( cib_t *cib, crm_data_t **output_data, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } return cib->cmds->variant_op(cib, CIB_OP_ERASE, NULL, NULL, NULL, output_data, call_options); } int cib_client_quit(cib_t *cib, int call_options) { if(cib == NULL) { return cib_missing; } else if(cib->state == cib_disconnected) { return cib_not_connected; } else if(cib->cmds->variant_op == NULL) { return cib_variant; } return cib->cmds->variant_op( cib, CRM_OP_QUIT, NULL, NULL, NULL, NULL, call_options); } int cib_client_add_notify_callback( cib_t *cib, const char *event, void (*callback)( const char *event, struct ha_msg *msg)) { GList *list_item = NULL; cib_notify_client_t *new_client = NULL; crm_debug_2("Adding callback for %s events (%d)", event, g_list_length(cib->notify_list)); crm_malloc0(new_client, sizeof(cib_notify_client_t)); new_client->event = event; new_client->callback = callback; list_item = g_list_find_custom( cib->notify_list, new_client, ciblib_GCompareFunc); if(list_item != NULL) { crm_warn("Callback already present"); - + crm_free(new_client); + } else { cib->notify_list = g_list_append( cib->notify_list, new_client); cib->cmds->register_callback(cib, event, 1); crm_debug_3("Callback added (%d)", g_list_length(cib->notify_list)); } return cib_ok; } int cib_client_del_notify_callback( cib_t *cib, const char *event, void (*callback)( const char *event, struct ha_msg *msg)) { GList *list_item = NULL; cib_notify_client_t *new_client = NULL; crm_debug("Removing callback for %s events", event); crm_malloc0(new_client, sizeof(cib_notify_client_t)); new_client->event = event; new_client->callback = callback; list_item = g_list_find_custom( cib->notify_list, new_client, ciblib_GCompareFunc); cib->cmds->register_callback(cib, event, 0); if(list_item != NULL) { cib_notify_client_t *list_client = list_item->data; cib->notify_list = g_list_remove(cib->notify_list, list_client); crm_free(list_client); crm_debug_3("Removed callback"); } else { crm_debug_3("Callback not present"); } crm_free(new_client); return cib_ok; } gint ciblib_GCompareFunc(gconstpointer a, gconstpointer b) { const cib_notify_client_t *a_client = a; const cib_notify_client_t *b_client = b; if(a_client->callback == b_client->callback && safe_str_neq(a_client->event, b_client->event)) { return 0; } else if(((long)a_client->callback) < ((long)b_client->callback)) { return -1; } return 1; } gboolean add_cib_op_callback( int call_id, gboolean only_success, void *user_data, void (*callback)(const HA_Message*, int, int, crm_data_t*,void*)) { cib_callback_client_t *blob = NULL; if(call_id < 0) { crm_warn("CIB call failed: %s", cib_error2string(call_id)); if(only_success == FALSE) { callback(NULL, call_id, call_id, NULL, user_data); } return FALSE; } crm_malloc0(blob, sizeof(cib_callback_client_t)); blob->only_success = only_success; blob->user_data = user_data; blob->callback = callback; g_hash_table_insert( cib_op_callback_table, GINT_TO_POINTER(call_id), blob); return TRUE; } void remove_cib_op_callback(int call_id, gboolean all_callbacks) { if(all_callbacks) { if(cib_op_callback_table != NULL) { g_hash_table_destroy(cib_op_callback_table); } cib_op_callback_table = g_hash_table_new_full( g_direct_hash, g_direct_equal, NULL, g_hash_destroy_str); } else { g_hash_table_remove( cib_op_callback_table, GINT_TO_POINTER(call_id)); } } int num_cib_op_callbacks(void) { if(cib_op_callback_table == NULL) { return 0; } return g_hash_table_size(cib_op_callback_table); } char * cib_pluralSection(const char *a_section) { char *a_section_parent = NULL; if (a_section == NULL) { a_section_parent = crm_strdup("all"); } else if(strcasecmp(a_section, XML_TAG_CIB) == 0) { a_section_parent = crm_strdup("all"); } else if(strcasecmp(a_section, XML_CIB_TAG_NODE) == 0) { a_section_parent = crm_strdup(XML_CIB_TAG_NODES); } else if(strcasecmp(a_section, XML_CIB_TAG_STATE) == 0) { a_section_parent = crm_strdup(XML_CIB_TAG_STATUS); } else if(strcasecmp(a_section, XML_CIB_TAG_CONSTRAINT) == 0) { a_section_parent = crm_strdup(XML_CIB_TAG_CONSTRAINTS); } else if(strcasecmp(a_section, XML_CONS_TAG_RSC_LOCATION) == 0) { a_section_parent = crm_strdup(XML_CIB_TAG_CONSTRAINTS); } else if(strcasecmp(a_section, XML_CONS_TAG_RSC_DEPEND) == 0) { a_section_parent = crm_strdup(XML_CIB_TAG_CONSTRAINTS); } else if(strcasecmp(a_section, XML_CONS_TAG_RSC_ORDER) == 0) { a_section_parent = crm_strdup(XML_CIB_TAG_CONSTRAINTS); } else if(strcasecmp(a_section, "resource") == 0) { a_section_parent = crm_strdup(XML_CIB_TAG_RESOURCES); } else if(strcasecmp(a_section, XML_CIB_TAG_RESOURCE) == 0) { a_section_parent = crm_strdup(XML_CIB_TAG_RESOURCES); } else if(strcasecmp(a_section, XML_CIB_TAG_GROUP) == 0) { a_section_parent = crm_strdup(XML_CIB_TAG_RESOURCES); } else if(strcasecmp(a_section, XML_CIB_TAG_INCARNATION) == 0) { a_section_parent = crm_strdup(XML_CIB_TAG_RESOURCES); } else if(strcasecmp(a_section, XML_CIB_TAG_NVPAIR) == 0) { a_section_parent = crm_strdup(XML_CIB_TAG_CRMCONFIG); } else if(strcasecmp(a_section, XML_TAG_ATTR_SETS) == 0) { a_section_parent = crm_strdup(XML_CIB_TAG_CRMCONFIG); } else { crm_err("Unknown section %s", a_section); a_section_parent = crm_strdup("all"); } crm_debug_2("Plural of %s is %s", crm_str(a_section), a_section_parent); return a_section_parent; } const char * cib_error2string(enum cib_errors return_code) { const char *error_msg = NULL; switch(return_code) { case cib_bad_permissions: error_msg = "bad permissions for the on-disk configuration. shutdown heartbeat and repair."; break; case cib_bad_digest: error_msg = "the on-disk configuration was manually altered. shutdown heartbeat and repair."; break; case cib_bad_config: error_msg = "the on-disk configuration is not valid"; break; case cib_msg_field_add: error_msg = "failed adding field to cib message"; break; case cib_id_check: error_msg = "missing id or id-collision detected"; break; case cib_operation: error_msg = "invalid operation"; break; case cib_create_msg: error_msg = "couldnt create cib message"; break; case cib_client_gone: error_msg = "client left before we could send reply"; break; case cib_not_connected: error_msg = "not connected"; break; case cib_not_authorized: error_msg = "not authorized"; break; case cib_send_failed: error_msg = "send failed"; break; case cib_reply_failed: error_msg = "reply failed"; break; case cib_return_code: error_msg = "no return code"; break; case cib_output_ptr: error_msg = "nowhere to store output"; break; case cib_output_data: error_msg = "corrupt output data"; break; case cib_connection: error_msg = "connection failed"; break; case cib_callback_register: error_msg = "couldnt register callback channel"; break; case cib_authentication: error_msg = ""; break; case cib_registration_msg: error_msg = "invalid registration msg"; break; case cib_callback_token: error_msg = "callback token not found"; break; case cib_missing: error_msg = "cib object missing"; break; case cib_variant: error_msg = "unknown/corrupt cib variant"; break; case CIBRES_MISSING_ID: error_msg = "The id field is missing"; break; case CIBRES_MISSING_TYPE: error_msg = "The type field is missing"; break; case CIBRES_MISSING_FIELD: error_msg = "A required field is missing"; break; case CIBRES_OBJTYPE_MISMATCH: error_msg = "CIBRES_OBJTYPE_MISMATCH"; break; case cib_EXISTS: error_msg = "The object already exists"; break; case cib_NOTEXISTS: error_msg = "The object/attribute does not exist"; break; case CIBRES_CORRUPT: error_msg = "The CIB is corrupt"; break; case cib_NOOBJECT: error_msg = "The update was empty"; break; case cib_NOPARENT: error_msg = "The parent object does not exist"; break; case cib_NODECOPY: error_msg = "Failed while copying update"; break; case CIBRES_OTHER: error_msg = "CIBRES_OTHER"; break; case cib_ok: error_msg = "ok"; break; case cib_unknown: error_msg = "Unknown error"; break; case cib_STALE: error_msg = "Discarded old update"; break; case cib_ACTIVATION: error_msg = "Activation Failed"; break; case cib_NOSECTION: error_msg = "Required section was missing"; break; case cib_NOTSUPPORTED: error_msg = "Supplied information is not supported"; break; case cib_not_master: error_msg = "Local service is not the master instance"; break; case cib_client_corrupt: error_msg = "Service client not valid"; break; case cib_remote_timeout: error_msg = "Remote node did not respond"; break; case cib_master_timeout: error_msg = "No master service is currently active"; break; case cib_revision_unsupported: error_msg = "The required CIB revision number is not supported"; break; case cib_revision_unknown: error_msg = "The CIB revision number could not be determined"; break; case cib_missing_data: error_msg = "Required data for this CIB API call not found"; break; case cib_no_quorum: error_msg = "Write requires quorum"; break; case cib_diff_failed: error_msg = "Application of an update diff failed"; break; case cib_diff_resync: error_msg = "Application of an update diff failed, requesting a full refresh"; break; case cib_bad_section: error_msg = "Invalid CIB section specified"; break; case cib_old_data: error_msg = "Update was older than existing configuration"; break; case cib_dtd_validation: error_msg = "Update does conform to the DTD in "HA_LIBDIR"/heartbeat/crm.dtd"; break; case cib_invalid_argument: error_msg = "Invalid argument"; break; } if(error_msg == NULL) { crm_err("Unknown CIB Error Code: %d", return_code); error_msg = ""; } return error_msg; } const char * cib_op2string(enum cib_update_op operation) { const char *operation_msg = NULL; switch(operation) { case 0: operation_msg = "none"; break; case 1: operation_msg = "add"; break; case 2: operation_msg = "modify"; break; case 3: operation_msg = "delete"; break; case CIB_UPDATE_OP_MAX: operation_msg = "invalid operation"; break; } if(operation_msg == NULL) { crm_err("Unknown CIB operation %d", operation); operation_msg = ""; } return operation_msg; } int cib_section2enum(const char *a_section) { if(a_section == NULL || strcasecmp(a_section, "all") == 0) { return cib_section_all; } else if(strcasecmp(a_section, XML_CIB_TAG_NODES) == 0) { return cib_section_nodes; } else if(strcasecmp(a_section, XML_CIB_TAG_STATUS) == 0) { return cib_section_status; } else if(strcasecmp(a_section, XML_CIB_TAG_CONSTRAINTS) == 0) { return cib_section_constraints; } else if(strcasecmp(a_section, XML_CIB_TAG_RESOURCES) == 0) { return cib_section_resources; } else if(strcasecmp(a_section, XML_CIB_TAG_CRMCONFIG) == 0) { return cib_section_crmconfig; } crm_err("Unknown CIB section: %s", a_section); return cib_section_none; } int cib_compare_generation(crm_data_t *left, crm_data_t *right) { int lpc = 0; const char *attributes[] = { XML_ATTR_GENERATION_ADMIN, XML_ATTR_GENERATION, XML_ATTR_NUMUPDATES, XML_ATTR_NUMPEERS }; crm_log_xml_debug_3(left, "left"); crm_log_xml_debug_3(right, "right"); for(lpc = 0; lpc < DIMOF(attributes); lpc++) { int int_elem_l = -1; int int_elem_r = -1; const char *elem_r = NULL; const char *elem_l = crm_element_value(left, attributes[lpc]); if(right != NULL) { elem_r = crm_element_value(right, attributes[lpc]); } if(elem_l != NULL) { int_elem_l = crm_parse_int(elem_l, NULL); } if(elem_r != NULL) { int_elem_r = crm_parse_int(elem_r, NULL); } if(int_elem_l < int_elem_r) { crm_debug_2("%s (%s < %s)", attributes[lpc], crm_str(elem_l), crm_str(elem_r)); return -1; } else if(int_elem_l > int_elem_r) { crm_debug_2("%s (%s > %s)", attributes[lpc], crm_str(elem_l), crm_str(elem_r)); return 1; } } return 0; } crm_data_t* get_cib_copy(cib_t *cib) { crm_data_t *xml_cib; #if CRM_DEPRECATED_SINCE_2_0_4 crm_data_t *xml_cib_copy; #endif int options = cib_scope_local|cib_sync_call; if(cib->cmds->query(cib, NULL, &xml_cib, options) != cib_ok) { crm_err("Couldnt retrieve the CIB"); return NULL; } else if(xml_cib == NULL) { crm_err("The CIB result was empty"); return NULL; } if(safe_str_eq(crm_element_name(xml_cib), XML_TAG_CIB)) { return xml_cib; #if CRM_DEPRECATED_SINCE_2_0_4 } else { xml_cib_copy = copy_xml( find_xml_node(xml_cib, XML_TAG_CIB, TRUE)); free_xml(xml_cib); return xml_cib_copy; #endif } free_xml(xml_cib); return NULL; } crm_data_t* cib_get_generation(cib_t *cib) { crm_data_t *the_cib = get_cib_copy(cib); crm_data_t *generation = create_xml_node( NULL, XML_CIB_TAG_GENERATION_TUPPLE); if(the_cib != NULL) { copy_in_properties(generation, the_cib); free_xml(the_cib); } return generation; } gboolean apply_cib_diff(crm_data_t *old, crm_data_t *diff, crm_data_t **new) { gboolean result = TRUE; const char *value = NULL; int this_updates = 0; int this_epoch = 0; int this_admin_epoch = 0; int diff_add_updates = 0; int diff_add_epoch = 0; int diff_add_admin_epoch = 0; int diff_del_updates = 0; int diff_del_epoch = 0; int diff_del_admin_epoch = 0; CRM_CHECK(diff != NULL, return FALSE); CRM_CHECK(old != NULL, return FALSE); value = crm_element_value(old, XML_ATTR_GENERATION_ADMIN); this_admin_epoch = crm_parse_int(value, "0"); crm_debug_3("%s=%d (%s)", XML_ATTR_GENERATION_ADMIN, this_admin_epoch, value); value = crm_element_value(old, XML_ATTR_GENERATION); this_epoch = crm_parse_int(value, "0"); crm_debug_3("%s=%d (%s)", XML_ATTR_GENERATION, this_epoch, value); value = crm_element_value(old, XML_ATTR_NUMUPDATES); this_updates = crm_parse_int(value, "0"); crm_debug_3("%s=%d (%s)", XML_ATTR_NUMUPDATES, this_updates, value); cib_diff_version_details( diff, &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates, &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates); value = NULL; if(result && diff_del_admin_epoch != this_admin_epoch) { value = XML_ATTR_GENERATION_ADMIN; result = FALSE; crm_debug_3("%s=%d", value, diff_del_admin_epoch); } else if(result && diff_del_epoch != this_epoch) { value = XML_ATTR_GENERATION; result = FALSE; crm_debug_3("%s=%d", value, diff_del_epoch); } else if(result && diff_del_updates != this_updates) { value = XML_ATTR_NUMUPDATES; result = FALSE; crm_debug_3("%s=%d", value, diff_del_updates); } if(result) { int len = 0; crm_data_t *tmp = NULL; crm_data_t *diff_copy = copy_xml(diff); tmp = find_xml_node(diff_copy, "diff-removed", TRUE); if(tmp != NULL) { len = tmp->nfields; cl_msg_remove(tmp, XML_ATTR_GENERATION_ADMIN); cl_msg_remove(tmp, XML_ATTR_GENERATION); cl_msg_remove(tmp, XML_ATTR_NUMUPDATES); } tmp = find_xml_node(diff_copy, "diff-added", TRUE); if(tmp != NULL) { len = tmp->nfields; cl_msg_remove(tmp, XML_ATTR_GENERATION_ADMIN); cl_msg_remove(tmp, XML_ATTR_GENERATION); cl_msg_remove(tmp, XML_ATTR_NUMUPDATES); } result = apply_xml_diff(old, diff_copy, new); free_xml(diff_copy); } else { crm_err("target and diff %s values didnt match", value); } return result; } crm_data_t * diff_cib_object(crm_data_t *old_cib, crm_data_t *new_cib, gboolean suppress) { crm_data_t *diff = diff_xml_object(old_cib, new_cib, suppress); crm_data_t *dest = NULL; crm_data_t *src = NULL; const char *name = NULL; const char *value = NULL; /* add complete version information */ src = old_cib; dest = find_xml_node(diff, "diff-removed", FALSE); if(src != NULL && dest != NULL) { name = XML_ATTR_GENERATION_ADMIN; value = crm_element_value(src, name); if(value == NULL) { value = "0"; } crm_xml_add(dest, name, value); name = XML_ATTR_GENERATION; value = crm_element_value(src, name); if(value == NULL) { value = "0"; } crm_xml_add(dest, name, value); name = XML_ATTR_NUMUPDATES; value = crm_element_value(src, name); if(value == NULL) { value = "0"; } crm_xml_add(dest, name, value); } src = new_cib; dest = find_xml_node(diff, "diff-added", FALSE); if(src != NULL && dest != NULL) { name = XML_ATTR_GENERATION_ADMIN; value = crm_element_value(src, name); if(value == NULL) { value = "0"; } crm_xml_add(dest, name, value); name = XML_ATTR_GENERATION; value = crm_element_value(src, name); if(value == NULL) { value = "0"; } crm_xml_add(dest, name, value); name = XML_ATTR_NUMUPDATES; value = crm_element_value(src, name); if(value == NULL) { value = "0"; } crm_xml_add(dest, name, value); } return diff; } void log_cib_diff(int log_level, crm_data_t *diff, const char *function) { int add_updates = 0; int add_epoch = 0; int add_admin_epoch = 0; int del_updates = 0; int del_epoch = 0; int del_admin_epoch = 0; if(diff == NULL) { return; } cib_diff_version_details( diff, &add_admin_epoch, &add_epoch, &add_updates, &del_admin_epoch, &del_epoch, &del_updates); if(add_updates != del_updates) { do_crm_log(log_level, "%s: Diff: --- %d.%d.%d", function, del_admin_epoch, del_epoch, del_updates); do_crm_log(log_level, "%s: Diff: +++ %d.%d.%d", function, add_admin_epoch, add_epoch, add_updates); } else if(diff != NULL) { do_crm_log(log_level, "%s: Local-only Change: %d.%d.%d", function, add_admin_epoch, add_epoch, add_updates); } log_xml_diff(log_level, diff, function); } gboolean cib_version_details( crm_data_t *cib, int *admin_epoch, int *epoch, int *updates) { const char *value = NULL; if(cib == NULL) { *admin_epoch = -1; *epoch = -1; *updates = -1; return FALSE; } else { value = crm_element_value(cib, XML_ATTR_GENERATION_ADMIN); *admin_epoch = crm_parse_int(value, "-1"); value = crm_element_value(cib, XML_ATTR_GENERATION); *epoch = crm_parse_int(value, "-1"); value = crm_element_value(cib, XML_ATTR_NUMUPDATES); *updates = crm_parse_int(value, "-1"); } return TRUE; } gboolean cib_diff_version_details( crm_data_t *diff, int *admin_epoch, int *epoch, int *updates, int *_admin_epoch, int *_epoch, int *_updates) { crm_data_t *tmp = NULL; tmp = find_xml_node(diff, "diff-added", FALSE); cib_version_details(tmp, admin_epoch, epoch, updates); tmp = find_xml_node(diff, "diff-removed", FALSE); cib_version_details(tmp, _admin_epoch, _epoch, _updates); return TRUE; } /* * The caller should never free the return value */ crm_data_t* get_object_root(const char *object_type, crm_data_t *the_root) { const char *node_stack[2]; crm_data_t *tmp_node = NULL; if(the_root == NULL) { crm_err("CIB root object was NULL"); return NULL; } node_stack[0] = XML_CIB_TAG_CONFIGURATION; node_stack[1] = object_type; if(object_type == NULL || strlen(object_type) == 0 || safe_str_eq(XML_CIB_TAG_SECTION_ALL, object_type) || safe_str_eq(XML_TAG_CIB, object_type)) { /* get the whole cib */ return the_root; } else if(strcasecmp(object_type, XML_CIB_TAG_STATUS) == 0) { /* these live in a different place */ tmp_node = find_xml_node(the_root, XML_CIB_TAG_STATUS, FALSE); node_stack[0] = object_type; node_stack[1] = NULL; } else { tmp_node = find_xml_node_nested(the_root, node_stack, 2); } if (tmp_node == NULL) { crm_debug_2("Section [%s [%s]] not present in %s", node_stack[0], node_stack[1]?node_stack[1]:"", crm_element_name(the_root)); } return tmp_node; } const char * get_crm_option(crm_data_t *cib, const char *name, gboolean do_warn) { const char * value = NULL; crm_data_t * a_default = NULL; crm_data_t * config = get_object_root(XML_CIB_TAG_CRMCONFIG, cib); if(config != NULL) { a_default = find_entity(config, XML_CIB_TAG_NVPAIR, name); } if(a_default == NULL) { if(do_warn) { crm_warn("Option %s not set", name); } return NULL; } value = crm_element_value(a_default, XML_NVPAIR_ATTR_VALUE); if(safe_str_eq(value, "")) { value = NULL; } return value; } crm_data_t* create_cib_fragment_adv( crm_data_t *update, const char *update_section, const char *source) { crm_data_t *cib = NULL; gboolean whole_cib = FALSE; crm_data_t *object_root = NULL; const char *update_name = NULL; /* crm_debug("Creating a blank fragment: %s", update_section); */ if(update == NULL && update_section == NULL) { crm_debug_3("Creating a blank fragment"); update = createEmptyCib(); crm_xml_add(cib, XML_ATTR_ORIGIN, source); return update; } else if(update == NULL) { crm_err("No update to create a fragment for"); return NULL; } else if(update_section == NULL) { update_section = cib_pluralSection(update_name); } if(safe_str_eq(crm_element_name(update), XML_TAG_CIB)) { whole_cib = TRUE; } if(whole_cib == FALSE) { cib = createEmptyCib(); crm_xml_add(cib, XML_ATTR_ORIGIN, source); object_root = get_object_root(update_section, cib); add_node_copy(object_root, update); } else { cib = copy_xml(update); crm_xml_add(cib, XML_ATTR_ORIGIN, source); } crm_debug_3("Verifying created fragment"); if(verifyCibXml(cib) == FALSE) { crm_err("Fragment creation failed"); crm_log_xml_err(cib, "[src]"); free_xml(cib); cib = NULL; } return cib; } /* * It is the callers responsibility to free both the new CIB (output) * and the new CIB (input) */ crm_data_t* createEmptyCib(void) { crm_data_t *cib_root = NULL, *config = NULL, *status = NULL; cib_root = create_xml_node(NULL, XML_TAG_CIB); config = create_xml_node(cib_root, XML_CIB_TAG_CONFIGURATION); status = create_xml_node(cib_root, XML_CIB_TAG_STATUS); /* crm_xml_add(cib_root, "version", "1"); */ crm_xml_add(cib_root, "generated", XML_BOOLEAN_TRUE); create_xml_node(config, XML_CIB_TAG_CRMCONFIG); create_xml_node(config, XML_CIB_TAG_NODES); create_xml_node(config, XML_CIB_TAG_RESOURCES); create_xml_node(config, XML_CIB_TAG_CONSTRAINTS); if (verifyCibXml(cib_root)) { return cib_root; } free_xml(cib_root); crm_crit("The generated CIB did not pass integrity testing!!" " All hope is lost."); return NULL; } gboolean verifyCibXml(crm_data_t *cib) { int lpc = 0; gboolean is_valid = TRUE; crm_data_t *tmp_node = NULL; const char *sections[] = { XML_CIB_TAG_NODES, XML_CIB_TAG_RESOURCES, XML_CIB_TAG_CONSTRAINTS, XML_CIB_TAG_STATUS, XML_CIB_TAG_CRMCONFIG }; if (cib == NULL) { crm_warn("CIB was empty."); return FALSE; } /* basic tests... are the standard section all there */ for(lpc = 0; lpc < DIMOF(sections); lpc++) { tmp_node = get_object_root(sections[lpc], cib); if (tmp_node == NULL) { crm_warn("Section %s is not present in the CIB", sections[lpc]); is_valid = FALSE; } } /* more integrity tests */ return is_valid; } gboolean verify_cib_cmds(cib_t *cib) { gboolean valid = TRUE; if(cib->cmds->variant_op == NULL) { crm_err("Operation variant_op not set"); valid = FALSE; } if(cib->cmds->signon == NULL) { crm_err("Operation signon not set"); valid = FALSE; } if(cib->cmds->signoff == NULL) { crm_err("Operation signoff not set"); valid = FALSE; } if(cib->cmds->free == NULL) { crm_err("Operation free not set"); valid = FALSE; } if(cib->cmds->set_op_callback == NULL) { crm_err("Operation set_op_callback not set"); valid = FALSE; } if(cib->cmds->add_notify_callback == NULL) { crm_err("Operation add_notify_callback not set"); valid = FALSE; } if(cib->cmds->del_notify_callback == NULL) { crm_err("Operation del_notify_callback not set"); valid = FALSE; } if(cib->cmds->set_connection_dnotify == NULL) { crm_err("Operation set_connection_dnotify not set"); valid = FALSE; } if(cib->cmds->channel == NULL) { crm_err("Operation channel not set"); valid = FALSE; } if(cib->cmds->inputfd == NULL) { crm_err("Operation inputfd not set"); valid = FALSE; } if(cib->cmds->noop == NULL) { crm_err("Operation noop not set"); valid = FALSE; } if(cib->cmds->ping == NULL) { crm_err("Operation ping not set"); valid = FALSE; } if(cib->cmds->query == NULL) { crm_err("Operation query not set"); valid = FALSE; } if(cib->cmds->query_from == NULL) { crm_err("Operation query_from not set"); valid = FALSE; } if(cib->cmds->is_master == NULL) { crm_err("Operation is_master not set"); valid = FALSE; } if(cib->cmds->set_master == NULL) { crm_err("Operation set_master not set"); valid = FALSE; } if(cib->cmds->set_slave == NULL) { crm_err("Operation set_slave not set"); valid = FALSE; } if(cib->cmds->set_slave_all == NULL) { crm_err("Operation set_slave_all not set"); valid = FALSE; } if(cib->cmds->sync == NULL) { crm_err("Operation sync not set"); valid = FALSE; } if(cib->cmds->sync_from == NULL) { crm_err("Operation sync_from not set"); valid = FALSE; } if(cib->cmds->bump_epoch == NULL) { crm_err("Operation bump_epoch not set"); valid = FALSE; } if(cib->cmds->create == NULL) { crm_err("Operation create not set"); valid = FALSE; } if(cib->cmds->modify == NULL) { crm_err("Operation modify not set"); valid = FALSE; } if(cib->cmds->replace == NULL) { crm_err("Operation replace not set"); valid = FALSE; } if(cib->cmds->delete == NULL) { crm_err("Operation delete not set"); valid = FALSE; } if(cib->cmds->erase == NULL) { crm_err("Operation erase not set"); valid = FALSE; } if(cib->cmds->quit == NULL) { crm_err("Operation quit not set"); valid = FALSE; } if(cib->cmds->msgready == NULL) { crm_err("Operation msgready not set"); valid = FALSE; } if(cib->cmds->rcvmsg == NULL) { crm_err("Operation rcvmsg not set"); valid = FALSE; } if(cib->cmds->dispatch == NULL) { crm_err("Operation dispatch not set"); valid = FALSE; } return valid; }