diff --git a/crm/cib/callbacks.c b/crm/cib/callbacks.c index cc9a74527c..2b354feed9 100644 --- a/crm/cib/callbacks.c +++ b/crm/cib/callbacks.c @@ -1,1943 +1,1935 @@ -/* $Id: callbacks.c,v 1.130 2006/07/06 10:55:09 andrew Exp $ */ +/* $Id: callbacks.c,v 1.131 2006/07/06 13:30:24 andrew Exp $ */ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include extern GMainLoop* mainloop; extern gboolean cib_shutdown_flag; extern gboolean stand_alone; extern enum cib_errors cib_update_counter( crm_data_t *xml_obj, const char *field, gboolean reset); extern void GHFunc_count_peers( gpointer key, gpointer value, gpointer user_data); extern enum cib_errors revision_check( crm_data_t *cib_update, crm_data_t *cib_copy, int flags); void initiate_exit(void); void terminate_ha_connection(const char *caller); gint cib_GCompareFunc(gconstpointer a, gconstpointer b); gboolean cib_msg_timeout(gpointer data); void cib_GHFunc(gpointer key, gpointer value, gpointer user_data); gboolean can_write(int flags); HA_Message *cib_msg_copy(HA_Message *msg, gboolean with_data); gboolean ccm_manual_check(gpointer data); void send_cib_replace(const HA_Message *sync_request, const char *host); void cib_process_request( HA_Message *request, gboolean privileged, gboolean force_synchronous, gboolean from_peer, cib_client_t *cib_client); gboolean syncd_once = FALSE; GHashTable *peer_hash = NULL; int next_client_id = 0; gboolean cib_is_master = FALSE; gboolean cib_have_quorum = FALSE; char * ccm_transition_id = NULL; GHashTable *client_list = NULL; GHashTable *ccm_membership = NULL; extern const char *cib_our_uname; extern ll_cluster_t *hb_conn; extern unsigned long cib_num_ops, cib_num_local, cib_num_updates, cib_num_fail; extern unsigned long cib_bad_connects, cib_num_timeouts; extern longclock_t cib_call_time; extern enum cib_errors cib_status; static HA_Message * cib_prepare_common(HA_Message *root, const char *section) { HA_Message *data = NULL; /* extract the CIB from the fragment */ if(root == NULL) { return NULL; } else if(safe_str_eq(crm_element_name(root), XML_TAG_FRAGMENT)) { data = find_xml_node(root, XML_TAG_CIB, TRUE); if(data != NULL) { crm_debug_3("Extracted CIB from "XML_TAG_FRAGMENT); } else { crm_log_xml_debug_4(root, "No CIB"); } } else { data = root; crm_log_xml_debug_4(root, "cib:input"); } /* grab the section specified for the command */ if(data != NULL && safe_str_eq(crm_element_name(data), XML_TAG_CIB)){ int rc = revision_check(data, the_cib, 0/* call_options */); if(rc == cib_ok) { data = get_object_root(section, data); if(data != NULL) { crm_debug_3("Extracted %s from CIB", section); } else { crm_log_xml_debug_4(root, "No Section"); } } else { crm_debug_2("Revision check failed"); } } return data; } static gboolean verify_section(const char *section) { if(section == NULL) { return TRUE; } else if(safe_str_eq(section, XML_TAG_CIB)) { return TRUE; } else if(safe_str_eq(section, XML_CIB_TAG_STATUS)) { return TRUE; } else if(safe_str_eq(section, XML_CIB_TAG_CRMCONFIG)) { return TRUE; } else if(safe_str_eq(section, XML_CIB_TAG_NODES)) { return TRUE; } else if(safe_str_eq(section, XML_CIB_TAG_RESOURCES)) { return TRUE; } else if(safe_str_eq(section, XML_CIB_TAG_CONSTRAINTS)) { return TRUE; } return FALSE; } static enum cib_errors cib_prepare_none(HA_Message *request, HA_Message **data, const char **section) { *data = NULL; *section = cl_get_string(request, F_CIB_SECTION); if(verify_section(*section) == FALSE) { return cib_bad_section; } return cib_ok; } static enum cib_errors cib_prepare_data(HA_Message *request, HA_Message **data, const char **section) { HA_Message *input_fragment = cl_get_struct(request, F_CIB_CALLDATA); *section = cl_get_string(request, F_CIB_SECTION); *data = cib_prepare_common(input_fragment, *section); if(verify_section(*section) == FALSE) { return cib_bad_section; } return cib_ok; } static enum cib_errors cib_prepare_sync(HA_Message *request, HA_Message **data, const char **section) { *section = cl_get_string(request, F_CIB_SECTION); *data = request; if(verify_section(*section) == FALSE) { return cib_bad_section; } return cib_ok; } static enum cib_errors cib_prepare_diff(HA_Message *request, HA_Message **data, const char **section) { HA_Message *input_fragment = cl_get_struct(request,F_CIB_UPDATE_DIFF); *section = NULL; if(input_fragment == NULL) { input_fragment = cl_get_struct(request, F_CIB_CALLDATA); } *data = cib_prepare_common(input_fragment, NULL); return cib_ok; } static enum cib_errors cib_cleanup_query(const char *op, HA_Message **data, HA_Message **output) { CRM_DEV_ASSERT(*data == NULL); return cib_ok; } static enum cib_errors cib_cleanup_output(const char *op, HA_Message **data, HA_Message **output) { free_xml(*output); return cib_ok; } static enum cib_errors cib_cleanup_none(const char *op, HA_Message **data, HA_Message **output) { CRM_DEV_ASSERT(*data == NULL); CRM_DEV_ASSERT(*output == NULL); return cib_ok; } static enum cib_errors cib_cleanup_sync(const char *op, HA_Message **data, HA_Message **output) { /* data is non-NULL but doesnt need to be free'd */ CRM_DEV_ASSERT(*output == NULL); return cib_ok; } /* typedef struct cib_operation_s { const char* operation; gboolean modifies_cib; gboolean needs_privileges; gboolean needs_quorum; enum cib_errors (*prepare)(HA_Message *, crm_data_t**, const char **); enum cib_errors (*cleanup)(crm_data_t**, crm_data_t**); enum cib_errors (*fn)( const char *, int, const char *, crm_data_t*, crm_data_t*, crm_data_t**, crm_data_t**); } cib_operation_t; */ /* technically bump does modify the cib... * but we want to split the "bump" from the "sync" */ cib_operation_t cib_server_ops[] = { {NULL, FALSE, FALSE, FALSE, cib_prepare_none, cib_cleanup_none, cib_process_default}, {CIB_OP_QUERY, FALSE, FALSE, FALSE, cib_prepare_none, cib_cleanup_query, cib_process_query}, {CIB_OP_MODIFY, TRUE, TRUE, TRUE, cib_prepare_data, cib_cleanup_output, cib_process_modify}, {CIB_OP_UPDATE, TRUE, TRUE, TRUE, cib_prepare_data, cib_cleanup_output, cib_process_change}, {CIB_OP_APPLY_DIFF,TRUE, TRUE, TRUE, cib_prepare_diff, cib_cleanup_sync, cib_process_diff}, {CIB_OP_SLAVE, FALSE, TRUE, FALSE, cib_prepare_none, cib_cleanup_none, cib_process_readwrite}, {CIB_OP_SLAVEALL, FALSE, TRUE, FALSE, cib_prepare_none, cib_cleanup_none, cib_process_readwrite}, {CIB_OP_SYNC_ONE, FALSE, TRUE, FALSE, cib_prepare_sync, cib_cleanup_sync, cib_process_sync_one}, {CIB_OP_MASTER, FALSE, TRUE, FALSE, cib_prepare_none, cib_cleanup_none, cib_process_readwrite}, {CIB_OP_ISMASTER, FALSE, TRUE, FALSE, cib_prepare_none, cib_cleanup_none, cib_process_readwrite}, {CIB_OP_BUMP, TRUE, TRUE, TRUE, cib_prepare_none, cib_cleanup_output, cib_process_bump}, {CIB_OP_REPLACE, TRUE, TRUE, TRUE, cib_prepare_data, cib_cleanup_output, cib_process_replace}, {CIB_OP_CREATE, TRUE, TRUE, TRUE, cib_prepare_data, cib_cleanup_output, cib_process_change}, {CIB_OP_DELETE, TRUE, TRUE, TRUE, cib_prepare_data, cib_cleanup_output, cib_process_delete}, {CIB_OP_DELETE_ALT,TRUE, TRUE, TRUE, cib_prepare_data, cib_cleanup_output, cib_process_change}, {CIB_OP_SYNC, FALSE, TRUE, FALSE, cib_prepare_sync, cib_cleanup_sync, cib_process_sync}, {CRM_OP_QUIT, FALSE, TRUE, FALSE, cib_prepare_none, cib_cleanup_none, cib_process_quit}, {CRM_OP_PING, FALSE, FALSE, FALSE, cib_prepare_none, cib_cleanup_output, cib_process_ping}, {CIB_OP_ERASE, TRUE, TRUE, TRUE, cib_prepare_none, cib_cleanup_output, cib_process_erase}, {CRM_OP_NOOP, FALSE, FALSE, FALSE, cib_prepare_none, cib_cleanup_none, cib_process_default}, {"cib_shutdown_req",FALSE, TRUE, FALSE, cib_prepare_sync, cib_cleanup_sync, cib_process_shutdown_req}, }; int send_via_callback_channel(HA_Message *msg, const char *token); enum cib_errors cib_process_command( HA_Message *request, HA_Message **reply, crm_data_t **cib_diff, gboolean privileged); gboolean cib_common_callback(IPC_Channel *channel, cib_client_t *cib_client, gboolean force_synchronous, gboolean privileged); enum cib_errors cib_get_operation_id(const HA_Message * msg, int *operation); gboolean cib_process_disconnect(IPC_Channel *channel, cib_client_t *cib_client); int num_clients = 0; static void cib_ipc_connection_destroy(gpointer user_data) { cib_client_t *cib_client = user_data; /* cib_process_disconnect */ if(cib_client == NULL) { crm_debug_4("Destroying %p", user_data); return; } if(cib_client->source != NULL) { crm_debug_4("Deleting %s (%p) from mainloop", cib_client->name, cib_client->source); G_main_del_IPC_Channel(cib_client->source); cib_client->source = NULL; } crm_debug_3("Destroying %s (%p)", cib_client->name, user_data); num_clients--; crm_debug("Num unfree'd clients: %d", num_clients); START_stat_free_op(); crm_free(cib_client->name); crm_free(cib_client->callback_id); crm_free(cib_client->id); crm_free(cib_client); END_stat_free_op(); crm_debug_4("Freed the cib client"); return; } static cib_client_t * cib_client_connect_common( IPC_Channel *channel, const char *channel_name, gboolean (*callback)(IPC_Channel *channel, gpointer user_data)) { gboolean can_connect = TRUE; cib_client_t *new_client = NULL; crm_debug_3("Connecting channel"); if (channel == NULL) { crm_err("Channel was NULL"); can_connect = FALSE; cib_bad_connects++; } else if (channel->ch_status != IPC_CONNECT) { crm_err("Channel was disconnected"); can_connect = FALSE; cib_bad_connects++; } else if(channel_name == NULL) { crm_err("user_data must contain channel name"); can_connect = FALSE; cib_bad_connects++; } else if(cib_shutdown_flag) { crm_info("Ignoring new client [%d] during shutdown", channel->farside_pid); return NULL; } else { START_stat_free_op(); crm_malloc0(new_client, sizeof(cib_client_t)); END_stat_free_op(); num_clients++; new_client->channel = channel; new_client->channel_name = channel_name; crm_debug_3("Created channel %p for channel %s", new_client, new_client->channel_name); channel->ops->set_recv_qlen(channel, 100); channel->ops->set_send_qlen(channel, 400); if(callback != NULL) { new_client->source = G_main_add_IPC_Channel( G_PRIORITY_DEFAULT, channel, FALSE, callback, new_client, cib_ipc_connection_destroy); } crm_debug_3("Channel %s connected for client %s", new_client->channel_name, new_client->id); } return new_client; } gboolean cib_client_connect_rw_synch(IPC_Channel *channel, gpointer user_data) { cl_mem_stats_t saved_stats; cib_client_t *new_client = NULL; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); new_client = cib_client_connect_common( channel, cib_channel_ro_synchronous, cib_rw_synchronous_callback); - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_ERR, __PRETTY_FUNCTION__, NULL, &saved_stats); if(new_client == NULL) { return FALSE; } return TRUE; } gboolean cib_client_connect_ro_synch(IPC_Channel *channel, gpointer user_data) { cl_mem_stats_t saved_stats; cib_client_t *new_client = NULL; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); new_client = cib_client_connect_common( channel, cib_channel_ro_synchronous, cib_ro_synchronous_callback); - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_ERR, __PRETTY_FUNCTION__, NULL, &saved_stats); if(new_client == NULL) { return FALSE; } return TRUE; } gboolean cib_client_connect_rw_ro(IPC_Channel *channel, gpointer user_data) { cl_uuid_t client_id; HA_Message *reg_msg = NULL; cib_client_t *new_client = NULL; char uuid_str[UU_UNPARSE_SIZEOF]; gboolean (*callback)(IPC_Channel *channel, gpointer user_data); cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); callback = cib_ro_callback; if(safe_str_eq(user_data, cib_channel_rw)) { callback = cib_rw_callback; } new_client = cib_client_connect_common( channel, callback==cib_ro_callback?cib_channel_ro:cib_channel_rw, callback); if(new_client == NULL) { return FALSE; } cl_uuid_generate(&client_id); cl_uuid_unparse(&client_id, uuid_str); START_stat_free_op(); CRM_CHECK(new_client->id == NULL, crm_free(new_client->id)); new_client->id = crm_strdup(uuid_str); END_stat_free_op(); cl_uuid_generate(&client_id); cl_uuid_unparse(&client_id, uuid_str); START_stat_free_op(); CRM_CHECK(new_client->callback_id == NULL, crm_free(new_client->callback_id)); new_client->callback_id = crm_strdup(uuid_str); END_stat_free_op(); /* make sure we can find ourselves later for sync calls * redirected to the master instance */ START_stat_free_op(); g_hash_table_insert( client_list, new_client->id, new_client); END_stat_free_op(); reg_msg = ha_msg_new(3); ha_msg_add(reg_msg, F_CIB_OPERATION, CRM_OP_REGISTER); ha_msg_add(reg_msg, F_CIB_CLIENTID, new_client->id); ha_msg_add( reg_msg, F_CIB_CALLBACK_TOKEN, new_client->callback_id); send_ipc_message(channel, reg_msg); crm_msg_del(reg_msg); - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_ERR, __PRETTY_FUNCTION__, NULL, &saved_stats); return TRUE; } gboolean cib_client_connect_null(IPC_Channel *channel, gpointer user_data) { cl_mem_stats_t saved_stats; cib_client_t *new_client = NULL; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); new_client = cib_client_connect_common( channel, cib_channel_callback, cib_null_callback); - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_ERR, __PRETTY_FUNCTION__, NULL, &saved_stats); if(new_client == NULL) { return FALSE; } return TRUE; } gboolean cib_rw_callback(IPC_Channel *channel, gpointer user_data) { gboolean result = FALSE; cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); result = cib_common_callback(channel, user_data, FALSE, TRUE); - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_WARNING, __PRETTY_FUNCTION__, NULL, &saved_stats); return result; } gboolean cib_ro_synchronous_callback(IPC_Channel *channel, gpointer user_data) { gboolean result = FALSE; cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); result = cib_common_callback(channel, user_data, TRUE, FALSE); - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_WARNING, __PRETTY_FUNCTION__, NULL, &saved_stats); return result; } gboolean cib_rw_synchronous_callback(IPC_Channel *channel, gpointer user_data) { gboolean result = FALSE; cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); result = cib_common_callback(channel, user_data, TRUE, TRUE); - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_WARNING, __PRETTY_FUNCTION__, NULL, &saved_stats); return result; } gboolean cib_ro_callback(IPC_Channel *channel, gpointer user_data) { gboolean result = FALSE; cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); result = cib_common_callback(channel, user_data, FALSE, FALSE); - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_WARNING, __PRETTY_FUNCTION__, NULL, &saved_stats); return result; } gboolean cib_null_callback(IPC_Channel *channel, gpointer user_data) { gboolean keep_connection = TRUE; HA_Message *op_request = NULL; HA_Message *registered = NULL; cib_client_t *cib_client = user_data; cib_client_t *hash_client = NULL; const char *type = NULL; const char *uuid_ticket = NULL; const char *client_name = NULL; gboolean register_failed = FALSE; cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); if(cib_client == NULL) { crm_err("Discarding IPC message from unknown source" " on callback channel."); return FALSE; } while(IPC_ISRCONN(channel)) { crm_msg_del(op_request); if(channel->ops->is_message_pending(channel) == 0) { break; } op_request = msgfromIPC_noauth(channel); if(op_request == NULL) { break; } type = cl_get_string(op_request, F_CIB_OPERATION); if(safe_str_eq(type, T_CIB_NOTIFY) ) { /* Update the notify filters for this client */ int on_off = 0; ha_msg_value_int( op_request, F_CIB_NOTIFY_ACTIVATE, &on_off); type = cl_get_string(op_request, F_CIB_NOTIFY_TYPE); crm_info("Setting %s callbacks for %s: %s", type, cib_client->name, on_off?"on":"off"); if(safe_str_eq(type, T_CIB_POST_NOTIFY)) { cib_client->post_notify = on_off; } else if(safe_str_eq(type, T_CIB_PRE_NOTIFY)) { cib_client->pre_notify = on_off; } else if(safe_str_eq(type, T_CIB_UPDATE_CONFIRM)) { cib_client->confirmations = on_off; } else if(safe_str_eq(type, T_CIB_DIFF_NOTIFY)) { cib_client->diffs = on_off; } else if(safe_str_eq(type, T_CIB_REPLACE_NOTIFY)) { cib_client->replace = on_off; } continue; } else if(safe_str_neq(type, CRM_OP_REGISTER) ) { crm_warn("Discarding IPC message from %s on callback channel", cib_client->id); continue; } uuid_ticket = cl_get_string(op_request, F_CIB_CALLBACK_TOKEN); client_name = cl_get_string(op_request, F_CIB_CLIENTNAME); CRM_DEV_ASSERT(uuid_ticket != NULL); if(crm_assert_failed) { register_failed = crm_assert_failed; } CRM_DEV_ASSERT(client_name != NULL); if(crm_assert_failed) { register_failed = crm_assert_failed; } if(register_failed == FALSE) { hash_client = g_hash_table_lookup(client_list, uuid_ticket); if(hash_client != NULL) { crm_err("Duplicate registration request..." " disconnecting"); register_failed = TRUE; } } if(register_failed) { crm_err("Registration request failed... disconnecting"); crm_msg_del(op_request); return FALSE; } START_stat_free_op(); CRM_CHECK(cib_client->id == NULL, crm_free(cib_client->id)); CRM_CHECK(cib_client->name == NULL, crm_free(cib_client->name)); cib_client->id = crm_strdup(uuid_ticket); cib_client->name = crm_strdup(client_name); g_hash_table_insert(client_list, cib_client->id, cib_client); END_stat_free_op(); crm_debug_2("Registered %s on %s channel", cib_client->id, cib_client->channel_name); if(safe_str_eq(cib_client->name, CRM_SYSTEM_TENGINE)) { /* The TE is _always_ interested in these * Enable now to avoid timing issues */ cib_client->diffs = TRUE; } registered = ha_msg_new(2); ha_msg_add(registered, F_CIB_OPERATION, CRM_OP_REGISTER); ha_msg_add(registered, F_CIB_CLIENTID, cib_client->id); send_ipc_message(channel, registered); crm_msg_del(registered); if(channel->ch_status == IPC_CONNECT) { break; } } crm_msg_del(op_request); if(channel->ch_status != IPC_CONNECT) { crm_debug_2("Client disconnected"); keep_connection = cib_process_disconnect(channel, cib_client); } - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_WARNING, __PRETTY_FUNCTION__, NULL, &saved_stats); return keep_connection; } void cib_common_callback_worker(HA_Message *op_request, cib_client_t *cib_client, gboolean force_synchronous, gboolean privileged); void cib_common_callback_worker(HA_Message *op_request, cib_client_t *cib_client, gboolean force_synchronous, gboolean privileged) { int rc = cib_ok; int call_type = 0; const char *op = NULL; longclock_t call_stop = 0; longclock_t call_start = 0; cl_mem_stats_t saved_stats; - cl_mem_stats_t *running_stats = NULL; - if(running_stats == NULL) { - START_stat_free_op(); - crm_malloc0(running_stats, sizeof(cl_mem_stats_t)); - END_stat_free_op(); - crm_save_mem_stats("running-stats", running_stats); - } crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); call_start = time_longclock(); cib_client->num_calls++; op = cl_get_string(op_request, F_CIB_OPERATION); rc = cib_get_operation_id(op_request, &call_type); if(rc != cib_ok) { crm_debug("Invalid operation %s from %s/%s", op, cib_client->name, cib_client->channel_name); } else { crm_debug_2("Processing %s operation from %s/%s", op, cib_client->name, cib_client->channel_name); } if(rc == cib_ok) { cib_process_request( op_request, force_synchronous, privileged, FALSE, cib_client); } call_stop = time_longclock(); cib_call_time += (call_stop - call_start); - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); - crm_diff_mem_stats(LOG_ERR, "running-cib-usage", running_stats); + crm_diff_mem_stats(LOG_ERR, LOG_WARNING, __PRETTY_FUNCTION__, NULL, &saved_stats); } gboolean cib_common_callback(IPC_Channel *channel, cib_client_t *cib_client, gboolean force_synchronous, gboolean privileged) { int lpc = 0; HA_Message *op_request = NULL; gboolean keep_channel = TRUE; cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); if(cib_client == NULL) { crm_err("Receieved call from unknown source. Discarding."); return FALSE; } START_stat_free_op(); if(cib_client->name == NULL) { cib_client->name = crm_itoa(channel->farside_pid); } if(cib_client->id == NULL) { cib_client->id = crm_strdup(cib_client->name); g_hash_table_insert(client_list, cib_client->id, cib_client); } END_stat_free_op(); crm_debug_2("Callback for %s on %s channel", cib_client->id, cib_client->channel_name); while(IPC_ISRCONN(channel)) { if(channel->ops->is_message_pending(channel) == 0) { break; } op_request = msgfromIPC_noauth(channel); if (op_request == NULL) { perror("Receive failure:"); break; } lpc++; crm_assert_failed = FALSE; crm_log_message_adv(LOG_MSG, "Client[inbound]", op_request); ha_msg_add(op_request, F_CIB_CLIENTID, cib_client->id); ha_msg_add(op_request, F_CIB_CLIENTNAME, cib_client->name); cib_common_callback_worker( op_request, cib_client, force_synchronous, privileged); crm_msg_del(op_request); if(channel->ch_status == IPC_CONNECT) { break; } } crm_debug_2("Processed %d messages", lpc); if(channel->ch_status != IPC_CONNECT) { crm_debug_2("Client disconnected"); keep_channel = cib_process_disconnect(channel, cib_client); } - crm_diff_mem_stats(LOG_DEBUG, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_WARNING, LOG_WARNING, __PRETTY_FUNCTION__, NULL, &saved_stats); return keep_channel; } static void do_local_notify(HA_Message *notify_src, const char *client_id, gboolean sync_reply, gboolean from_peer) { /* send callback to originating child */ cib_client_t *client_obj = NULL; HA_Message *client_reply = NULL; enum cib_errors local_rc = cib_ok; cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); crm_debug_2("Performing notification"); client_reply = cib_msg_copy(notify_src, TRUE); if(client_id != NULL) { client_obj = g_hash_table_lookup( client_list, client_id); } else { crm_debug("No client to sent the response to." " F_CIB_CLIENTID not set."); } crm_debug_3("Sending callback to request originator"); if(client_obj == NULL) { local_rc = cib_reply_failed; } else { const char *client_id = client_obj->callback_id; crm_debug_2("Sending %ssync response to %s %s", sync_reply?"":"an a-", client_obj->name, from_peer?"(originator of delegated request)":""); if(sync_reply) { client_id = client_obj->id; } local_rc = send_via_callback_channel(client_reply, client_id); } ha_msg_del(client_reply); if(local_rc != cib_ok) { crm_warn("%sSync reply to %s failed: %s", sync_reply?"":"A-", client_obj?client_obj->name:"", cib_error2string(local_rc)); } - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_ERR, __PRETTY_FUNCTION__, NULL, &saved_stats); } static void parse_local_options( cib_client_t *cib_client, int call_type, int call_options, const char *host, const char *op, gboolean *local_notify, gboolean *needs_reply, gboolean *process, gboolean *needs_forward) { if(cib_server_ops[call_type].modifies_cib && !(call_options & cib_inhibit_bcast)) { /* we need to send an update anyway */ *needs_reply = TRUE; } else { *needs_reply = FALSE; } if(host == NULL && (call_options & cib_scope_local)) { crm_debug("Processing locally scoped %s op from %s", op, cib_client->name); *local_notify = TRUE; } else if(host == NULL && cib_is_master) { crm_debug("Processing master %s op locally from %s", op, cib_client->name); *local_notify = TRUE; } else if(safe_str_eq(host, cib_our_uname)) { crm_debug("Processing locally addressed %s op from %s", op, cib_client->name); *local_notify = TRUE; } else { crm_debug("%s op from %s needs to be forwarded to %s", op, cib_client->name, host?host:"the master instance"); *needs_forward = TRUE; *process = FALSE; } } static gboolean parse_peer_options( int call_type, HA_Message *request, gboolean *local_notify, gboolean *needs_reply, gboolean *process, gboolean *needs_forward) { const char *op = cl_get_string(request, F_CIB_OPERATION); const char *originator = cl_get_string(request, F_ORIG); const char *host = cl_get_string(request, F_CIB_HOST); const char *reply_to = cl_get_string(request, F_CIB_ISREPLY); const char *update = cl_get_string(request, F_CIB_GLOBAL_UPDATE); const char *delegated = cl_get_string(request, F_CIB_DELEGATED); if(safe_str_eq(op, "cib_shutdown_req")) { if(reply_to != NULL) { crm_debug("Processing %s from %s", op, host); *needs_reply = FALSE; } else { crm_debug("Processing %s reply from %s", op, host); } return TRUE; } else if(crm_is_true(update) && safe_str_eq(reply_to, cib_our_uname)) { crm_debug("Processing global/peer update from %s" " that originated from us", originator); *needs_reply = FALSE; if(cl_get_string(request, F_CIB_CLIENTID) != NULL) { *local_notify = TRUE; } return TRUE; } else if(crm_is_true(update)) { crm_debug("Processing global/peer update from %s", originator); *needs_reply = FALSE; return TRUE; } else if(host != NULL && safe_str_eq(host, cib_our_uname)) { crm_debug("Processing request sent to us from %s", originator); return TRUE; } else if(delegated != NULL && cib_is_master == TRUE) { crm_debug("Processing request sent to master instance from %s", originator); return TRUE; } else if(reply_to != NULL && safe_str_eq(reply_to, cib_our_uname)) { crm_debug("Forward reply sent from %s to local clients", originator); *process = FALSE; *needs_reply = FALSE; *local_notify = TRUE; return TRUE; } else if(delegated != NULL) { crm_debug("Ignoring msg for master instance"); } else if(host != NULL) { /* this is for a specific instance and we're not it */ crm_debug("Ignoring msg for instance on %s", crm_str(host)); } else if(reply_to == NULL && cib_is_master == FALSE) { /* this is for the master instance and we're not it */ crm_debug("Ignoring reply to %s", crm_str(reply_to)); } else { crm_err("Nothing for us to do?"); crm_log_message_adv(LOG_ERR, "Peer[inbound]", request); } return FALSE; } static void forward_request(HA_Message *request, cib_client_t *cib_client, int call_options) { HA_Message *forward_msg = NULL; cl_mem_stats_t saved_stats; const char *op = cl_get_string(request, F_CIB_OPERATION); const char *host = cl_get_string(request, F_CIB_HOST); crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); forward_msg = cib_msg_copy(request, TRUE); ha_msg_add(forward_msg, F_CIB_DELEGATED, cib_our_uname); if(host != NULL) { crm_debug_2("Forwarding %s op to %s", op, host); send_ha_message(hb_conn, forward_msg, host, FALSE); } else { crm_debug_2("Forwarding %s op to master instance", op); send_ha_message(hb_conn, forward_msg, NULL, FALSE); } if(call_options & cib_discard_reply) { crm_debug_2("Client not interested in reply"); } else if(call_options & cib_sync_call) { /* keep track of the request so we can time it * out if required */ crm_err("Registering delegated call from %s", cib_client->id); cib_client->delegated_calls = g_list_append( cib_client->delegated_calls, forward_msg); forward_msg = NULL; } crm_msg_del(forward_msg); - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_ERR, __PRETTY_FUNCTION__, NULL, &saved_stats); } static void send_peer_reply( HA_Message *msg, crm_data_t *result_diff, const char *originator, gboolean broadcast) { cl_mem_stats_t saved_stats; HA_Message *reply_copy = NULL; CRM_ASSERT(msg != NULL); crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); reply_copy = cib_msg_copy(msg, TRUE); if(broadcast) { /* this (successful) call modified the CIB _and_ the * change needs to be broadcast... * send via HA to other nodes */ int diff_add_updates = 0; int diff_add_epoch = 0; int diff_add_admin_epoch = 0; int diff_del_updates = 0; int diff_del_epoch = 0; int diff_del_admin_epoch = 0; cib_diff_version_details( result_diff, &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates, &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates); crm_debug("Sending update diff %d.%d.%d -> %d.%d.%d", diff_del_admin_epoch,diff_del_epoch,diff_del_updates, diff_add_admin_epoch,diff_add_epoch,diff_add_updates); ha_msg_add(reply_copy, F_CIB_ISREPLY, originator); ha_msg_add(reply_copy, F_CIB_GLOBAL_UPDATE, XML_BOOLEAN_TRUE); ha_msg_mod(reply_copy, F_CIB_OPERATION, CIB_OP_APPLY_DIFF); add_message_xml(reply_copy, F_CIB_UPDATE_DIFF, result_diff); crm_log_message(LOG_DEBUG_3, reply_copy); send_ha_message(hb_conn, reply_copy, NULL, TRUE); } else if(originator != NULL) { /* send reply via HA to originating node */ crm_debug_2("Sending request result to originator only"); ha_msg_add(reply_copy, F_CIB_ISREPLY, originator); send_ha_message(hb_conn, reply_copy, originator, FALSE); } crm_msg_del(reply_copy); - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_WARNING, __PRETTY_FUNCTION__, NULL, &saved_stats); } void cib_process_request( HA_Message *request, gboolean force_synchronous, gboolean privileged, gboolean from_peer, cib_client_t *cib_client) { int call_type = 0; int call_options = 0; gboolean process = TRUE; gboolean needs_reply = TRUE; gboolean local_notify = FALSE; gboolean needs_forward = FALSE; crm_data_t *result_diff = NULL; enum cib_errors rc = cib_ok; HA_Message *op_reply = NULL; const char *op = cl_get_string(request, F_CIB_OPERATION); const char *originator = cl_get_string(request, F_ORIG); const char *host = cl_get_string(request, F_CIB_HOST); const char *update = cl_get_string(request, F_CIB_GLOBAL_UPDATE); cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); crm_debug_4("%s Processing msg %s", cib_our_uname, cl_get_string(request, F_SEQ)); cib_num_ops++; if(cib_num_ops == 0) { cib_num_fail = 0; cib_num_local = 0; cib_num_updates = 0; crm_info("Stats wrapped around"); } if(host != NULL && strlen(host) == 0) { host = NULL; } ha_msg_value_int(request, F_CIB_CALLOPTS, &call_options); crm_debug_4("Retrieved call options: %d", call_options); if(force_synchronous) { call_options |= cib_sync_call; } crm_debug_2("Processing %s message (%s) to %s...", from_peer?"peer":"local", from_peer?originator:cib_our_uname, host?host:"master"); rc = cib_get_operation_id(request, &call_type); if(cib_server_ops[call_type].modifies_cib) { cib_num_updates++; } if(rc != cib_ok) { /* TODO: construct error reply */ crm_err("Pre-processing of command failed: %s", cib_error2string(rc)); } else if(from_peer == FALSE) { parse_local_options(cib_client, call_type, call_options, host, op, &local_notify, &needs_reply, &process, &needs_forward); } else if(parse_peer_options(call_type, request, &local_notify, &needs_reply, &process, &needs_forward) == FALSE) { return; } crm_debug_3("Finished determining processing actions"); if(call_options & cib_discard_reply) { needs_reply = cib_server_ops[call_type].modifies_cib; local_notify = FALSE; } if(needs_forward && stand_alone == FALSE) { forward_request(request, cib_client, call_options); return; } if(process) { cib_num_local++; crm_debug_2("Performing local processing:" " op=%s origin=%s/%s,%s (update=%s)", cl_get_string(request, F_CIB_OPERATION), originator, cl_get_string(request, F_CIB_CLIENTID), cl_get_string(request, F_CIB_CALLID), update); rc = cib_process_command( request, &op_reply, &result_diff, privileged); crm_debug_2("Processing complete"); if(rc == cib_diff_resync || rc == cib_diff_failed || rc == cib_old_data) { crm_warn("%s operation failed: %s", crm_str(op), cib_error2string(rc)); } else if(rc != cib_ok) { cib_num_fail++; crm_err("%s operation failed: %s", crm_str(op), cib_error2string(rc)); crm_log_message_adv(LOG_DEBUG, "CIB[output]", op_reply); crm_log_message_adv(LOG_INFO, "Input message", request); } if(op_reply == NULL && (needs_reply || local_notify)) { crm_err("Unexpected NULL reply to message"); crm_log_message(LOG_ERR, request); needs_reply = FALSE; local_notify = FALSE; } } crm_debug_3("processing response cases"); if(local_notify) { const char *client_id = cl_get_string(request, F_CIB_CLIENTID); if(process == FALSE) { do_local_notify(request, client_id, call_options & cib_sync_call, from_peer); } else { do_local_notify(op_reply, client_id, call_options & cib_sync_call, from_peer); } } /* from now on we are the server */ if(needs_reply == FALSE || stand_alone) { /* nothing more to do... * this was a non-originating slave update */ crm_debug_2("Completed slave update"); } else if(rc == cib_ok && result_diff != NULL && !(call_options & cib_inhibit_bcast)) { CRM_DEV_ASSERT(cib_server_ops[call_type].modifies_cib == FALSE || result_diff != NULL || rc != cib_ok); send_peer_reply(request, result_diff, originator, TRUE); } else if((call_options & cib_discard_reply) == 0) { CRM_DEV_ASSERT(cib_server_ops[call_type].modifies_cib == FALSE || result_diff != NULL || rc != cib_ok); crm_debug("Directed reply to %s", originator); if(call_options & cib_inhibit_bcast) { crm_debug("Request not broadcast: inhibited"); } if(cib_server_ops[call_type].modifies_cib == FALSE) { crm_debug_2("Request not broadcast: R/O call"); } if(rc != cib_ok) { crm_warn("Request not broadcast: call failed: %s", cib_error2string(rc)); } if(from_peer) { send_peer_reply(op_reply, result_diff, originator, FALSE); } } crm_msg_del(op_reply); free_xml(result_diff); - if(crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats)) { - crm_log_message_adv(LOG_ERR,"IPC[leak]", request); + if(crm_diff_mem_stats(LOG_ERR, LOG_WARNING, __PRETTY_FUNCTION__, NULL, &saved_stats)) { + ;/* crm_log_message_adv(LOG_ERR,"IPC[leak]", request); */ } return; } static HA_Message * cib_construct_reply(HA_Message *request, HA_Message *output, int rc) { int lpc = 0; HA_Message *reply = NULL; const char *name = NULL; const char *value = NULL; const char *names[] = { F_CIB_OPERATION, F_CIB_CALLID, F_CIB_CLIENTID, F_CIB_CALLOPTS }; crm_debug_4("Creating a basic reply"); reply = ha_msg_new(8); ha_msg_add(reply, F_TYPE, T_CIB); for(lpc = 0; lpc < DIMOF(names); lpc++) { name = names[lpc]; value = cl_get_string(request, name); ha_msg_add(reply, name, value); } ha_msg_add_int(reply, F_CIB_RC, rc); if(output != NULL) { crm_debug_4("Attaching reply output"); add_message_xml(reply, F_CIB_CALLDATA, output); } return reply; } enum cib_errors cib_process_command(HA_Message *request, HA_Message **reply, crm_data_t **cib_diff, gboolean privileged) { crm_data_t *output = NULL; crm_data_t *input = NULL; crm_data_t *current_cib = the_cib; crm_data_t *result_cib = NULL; int call_type = 0; int call_options = 0; enum cib_errors rc = cib_ok; enum cib_errors rc2 = cib_ok; const char *op = NULL; const char *section = NULL; gboolean global_update = crm_is_true( cl_get_string(request, F_CIB_GLOBAL_UPDATE)); *reply = NULL; *cib_diff = NULL; /* Start processing the request... */ op = cl_get_string(request, F_CIB_OPERATION); ha_msg_value_int(request, F_CIB_CALLOPTS, &call_options); rc = cib_get_operation_id(request, &call_type); if(rc == cib_ok && cib_server_ops[call_type].needs_privileges && privileged == FALSE) { /* abort */ rc = cib_not_authorized; } if(cib_status != cib_ok) { *reply = cib_construct_reply(request, the_cib, cib_status); return cib_status; } if(rc == cib_ok && global_update == FALSE && cib_server_ops[call_type].needs_quorum && can_write(call_options) == FALSE) { rc = cib_no_quorum; } /* prevent NUMUPDATES from being incrimented - apply the change as-is */ if(global_update) { call_options |= cib_inhibit_bcast; call_options |= cib_force_diff; } rc2 = cib_server_ops[call_type].prepare(request, &input, §ion); if(rc == cib_ok) { rc = rc2; } if(rc != cib_ok) { crm_debug_2("Call setup failed"); } else if(cib_server_ops[call_type].modifies_cib) { if((call_options & cib_inhibit_notify) == 0) { cib_pre_notify( call_options, op, get_object_root(section, current_cib), input); } if(rc == cib_ok) { result_cib = copy_xml(current_cib); rc = cib_server_ops[call_type].fn( op, call_options, section, input, current_cib, &result_cib, &output); } if(rc == cib_ok) { CRM_DEV_ASSERT(result_cib != NULL); CRM_DEV_ASSERT(current_cib != result_cib); update_counters(__FILE__, __FUNCTION__, result_cib); if(section == NULL && cib_server_ops[call_type].fn == cib_process_replace) { /* skip */ } else if(cib_server_ops[call_type].fn == cib_process_change && (call_options & cib_inhibit_bcast)) { /* skip */ } else { cib_update_counter(result_cib, XML_ATTR_NUMUPDATES, FALSE); } if(do_id_check(result_cib, NULL, TRUE, FALSE)) { rc = cib_id_check; if(call_options & cib_force_diff) { crm_err("Global update introduces id collision!"); } } else { *cib_diff = diff_cib_object( current_cib, result_cib, FALSE); } } if(rc != cib_ok) { free_xml(result_cib); } else if(activateCibXml(result_cib, CIB_FILENAME) != 0){ crm_warn("Activation failed"); rc = cib_ACTIVATION; } if((call_options & cib_inhibit_notify) == 0) { const char *call_id = cl_get_string( request, F_CIB_CALLID); const char *client = cl_get_string( request, F_CIB_CLIENTNAME); cib_post_notify(call_options, op, input, rc, the_cib); cib_diff_notify(call_options, client, call_id, op, input, rc, *cib_diff); } log_xml_diff(rc==cib_ok?cib_diff_loglevel:cib_diff_loglevel+1, *cib_diff, "cib:diff"); } else { rc = cib_server_ops[call_type].fn( op, call_options, section, input, current_cib, &result_cib, &output); CRM_CHECK(result_cib == NULL, free_xml(result_cib)); } if((call_options & cib_discard_reply) == 0) { *reply = cib_construct_reply(request, output, rc); } if(call_type >= 0) { cib_server_ops[call_type].cleanup(op, &input, &output); } return rc; } int send_via_callback_channel(HA_Message *msg, const char *token) { cib_client_t *hash_client = NULL; GList *list_item = NULL; enum cib_errors rc = cib_ok; cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); crm_debug_3("Delivering msg %p to client %s", msg, token); if(token == NULL) { crm_err("No client id token, cant send message"); if(rc == cib_ok) { rc = cib_missing; } } else { /* A client that left before we could reply is not really * _our_ error. Warn instead. */ hash_client = g_hash_table_lookup(client_list, token); if(hash_client == NULL) { crm_warn("Cannot find client for token %s", token); rc = cib_client_gone; } else if(hash_client->channel == NULL) { crm_err("Cannot find channel for client %s", token); rc = cib_client_corrupt; } else if(hash_client->channel->ops->get_chan_status( hash_client->channel) == IPC_DISCONNECT) { crm_warn("Client %s has disconnected", token); rc = cib_client_gone; cib_num_timeouts++; } } /* this is a more important error so overwriting rc is warrented */ if(msg == NULL) { crm_err("No message to send"); rc = cib_reply_failed; } if(rc == cib_ok) { list_item = g_list_find_custom( hash_client->delegated_calls, msg, cib_GCompareFunc); } if(list_item != NULL) { /* remove it - no need to time it out */ HA_Message *orig_msg = list_item->data; crm_debug_3("Removing msg from delegated list"); hash_client->delegated_calls = g_list_remove( hash_client->delegated_calls, orig_msg); CRM_DEV_ASSERT(orig_msg != msg); crm_msg_del(orig_msg); } if(rc == cib_ok) { crm_debug_3("Delivering reply to client %s", token); if(send_ipc_message(hash_client->channel, msg) == FALSE) { crm_warn("Delivery of reply to client %s/%s failed", hash_client->name, token); rc = cib_reply_failed; } } - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_ERR, __PRETTY_FUNCTION__, NULL, &saved_stats); return rc; } gint cib_GCompareFunc(gconstpointer a, gconstpointer b) { const HA_Message *a_msg = a; const HA_Message *b_msg = b; int msg_a_id = 0; int msg_b_id = 0; ha_msg_value_int(a_msg, F_CIB_CALLID, &msg_a_id); ha_msg_value_int(b_msg, F_CIB_CALLID, &msg_b_id); if(msg_a_id == msg_b_id) { return 0; } else if(msg_a_id < msg_b_id) { return -1; } return 1; } gboolean cib_msg_timeout(gpointer data) { crm_debug_4("Checking if any clients have timed out messages"); /* g_hash_table_foreach(client_list, cib_GHFunc, NULL); */ return TRUE; } void cib_GHFunc(gpointer key, gpointer value, gpointer user_data) { int timeout = 0; /* 1 iteration == 10 seconds */ HA_Message *msg = NULL; HA_Message *reply = NULL; const char *host_to = NULL; cib_client_t *client = value; GListPtr list = client->delegated_calls; while(list != NULL) { msg = list->data; ha_msg_value_int(msg, F_CIB_TIMEOUT, &timeout); if(timeout <= 0) { list = list->next; continue; } else { int seen = 0; ha_msg_value_int(msg, F_CIB_SEENCOUNT, &seen); crm_debug_4("Timeout %d, seen %d", timeout, seen); if(seen < timeout) { crm_debug_4("Updating seen count for msg from client %s", client->id); seen += 10; ha_msg_mod_int(msg, F_CIB_SEENCOUNT, seen); list = list->next; continue; } } cib_num_timeouts++; host_to = cl_get_string(msg, F_CIB_HOST); crm_warn("Sending operation timeout msg to client %s", client->id); reply = ha_msg_new(4); ha_msg_add(reply, F_TYPE, T_CIB); ha_msg_add(reply, F_CIB_OPERATION, cl_get_string(msg, F_CIB_OPERATION)); ha_msg_add(reply, F_CIB_CALLID, cl_get_string(msg, F_CIB_CALLID)); if(host_to == NULL) { ha_msg_add_int(reply, F_CIB_RC, cib_master_timeout); } else { ha_msg_add_int(reply, F_CIB_RC, cib_remote_timeout); } send_ipc_message(client->channel, reply); list = list->next; client->delegated_calls = g_list_remove( client->delegated_calls, msg); crm_msg_del(msg); crm_msg_del(reply); } } gboolean cib_process_disconnect(IPC_Channel *channel, cib_client_t *cib_client) { cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); if (channel == NULL) { CRM_DEV_ASSERT(cib_client == NULL); } else if (cib_client == NULL) { crm_err("No client"); } else { CRM_DEV_ASSERT(channel->ch_status != IPC_CONNECT); crm_debug_2("Cleaning up after client disconnect: %s/%s/%s", crm_str(cib_client->name), cib_client->channel_name, cib_client->id); if(cib_client->id != NULL) { START_stat_free_op(); if(!g_hash_table_remove(client_list, cib_client->id)) { crm_err("Client %s not found in the hashtable", cib_client->name); } END_stat_free_op(); } } if(cib_shutdown_flag && g_hash_table_size(client_list) == 0) { crm_info("All clients disconnected..."); initiate_exit(); } - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_ERR, __PRETTY_FUNCTION__, NULL, &saved_stats); return FALSE; } gboolean cib_ha_dispatch(IPC_Channel *channel, gpointer user_data) { ll_cluster_t *hb_cluster = (ll_cluster_t*)user_data; cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); crm_debug_3("Invoked"); if(IPC_ISRCONN(channel)) { if(hb_cluster->llc_ops->msgready(hb_cluster) == 0) { crm_debug_2("no message ready yet"); } /* invoke the callbacks but dont block */ hb_cluster->llc_ops->rcvmsg(hb_cluster, 0); } - crm_diff_mem_stats(LOG_DEBUG, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_WARNING, LOG_WARNING, __PRETTY_FUNCTION__, NULL, &saved_stats); return (channel->ch_status == IPC_CONNECT); } void cib_peer_callback(HA_Message * msg, void* private_data) { int call_type = 0; int call_options = 0; const char *originator = cl_get_string(msg, F_ORIG); const char *seq = cl_get_string(msg, F_SEQ); const char *op = cl_get_string(msg, F_CIB_OPERATION); cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); crm_log_message_adv(LOG_MSG, "Peer[inbound]", msg); crm_debug_2("Peer %s message (%s) from %s", op, seq, originator); if(originator == NULL || safe_str_eq(originator, cib_our_uname)) { crm_debug("Discarding %s message %s from ourselves", op, seq); return; } else if(ccm_membership == NULL) { crm_info("Discarding %s message (%s) from %s:" " membership not established", op, seq, originator); return; } else if(g_hash_table_lookup(ccm_membership, originator) == NULL) { crm_warn("Discarding %s message (%s) from %s:" " not in our membership", op, seq, originator); return; } else if(cib_get_operation_id(msg, &call_type) != cib_ok) { crm_debug("Discarding %s message (%s) from %s:" " Invalid operation", op, seq, originator); return; } crm_debug_2("Processing %s msg (%s) from %s",op, seq, originator); ha_msg_value_int(msg, F_CIB_CALLOPTS, &call_options); crm_debug_4("Retrieved call options: %d", call_options); if(cl_get_string(msg, F_CIB_CLIENTNAME) == NULL) { ha_msg_add(msg, F_CIB_CLIENTNAME, originator); } cib_process_request(msg, FALSE, TRUE, TRUE, NULL); - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_ERR, __PRETTY_FUNCTION__, NULL, &saved_stats); return; } HA_Message * cib_msg_copy(HA_Message *msg, gboolean with_data) { int lpc = 0; const char *field = NULL; const char *value = NULL; const HA_Message *value_struct = NULL; static const char *field_list[] = { F_TYPE , F_CIB_CLIENTID , F_CIB_CALLOPTS , F_CIB_CALLID , F_CIB_OPERATION , F_CIB_ISREPLY , F_CIB_SECTION , F_CIB_HOST , F_CIB_RC , F_CIB_DELEGATED , F_CIB_OBJID , F_CIB_OBJTYPE , F_CIB_EXISTING , F_CIB_SEENCOUNT , F_CIB_TIMEOUT , F_CIB_CALLBACK_TOKEN , F_CIB_GLOBAL_UPDATE , F_CIB_CLIENTNAME , F_CIB_NOTIFY_TYPE , F_CIB_NOTIFY_ACTIVATE }; static const char *data_list[] = { F_CIB_CALLDATA , F_CIB_UPDATE , F_CIB_UPDATE_RESULT }; HA_Message *copy = NULL; copy = ha_msg_new(10); if(copy == NULL) { return copy; } for(lpc = 0; lpc < DIMOF(field_list); lpc++) { field = field_list[lpc]; value = cl_get_string(msg, field); if(value != NULL) { ha_msg_add(copy, field, value); } } for(lpc = 0; with_data && lpc < DIMOF(data_list); lpc++) { field = data_list[lpc]; value_struct = cl_get_struct(msg, field); if(value_struct != NULL) { add_message_xml(copy, field, value_struct); } } return copy; } enum cib_errors cib_get_operation_id(const HA_Message * msg, int *operation) { int lpc = 0; int max_msg_types = DIMOF(cib_server_ops); const char *op = cl_get_string(msg, F_CIB_OPERATION); for (lpc = 0; lpc < max_msg_types; lpc++) { if (safe_str_eq(op, cib_server_ops[lpc].operation)) { *operation = lpc; return cib_ok; } } crm_err("Operation %s is not valid", op); *operation = -1; return cib_operation; } void cib_client_status_callback(const char * node, const char * client, const char * status, void * private) { if(safe_str_eq(client, CRM_SYSTEM_CIB)) { crm_info("Status update: Client %s/%s now has status [%s]", node, client, status); g_hash_table_replace(peer_hash, crm_strdup(node), crm_strdup(status)); set_connected_peers(the_cib); } return; } extern oc_ev_t *cib_ev_token; gboolean ccm_manual_check(gpointer data) { int rc = 0; oc_ev_t *ccm_token = cib_ev_token; crm_debug("manual check"); rc = oc_ev_handle_event(ccm_token); if(0 == rc) { return TRUE; } else { crm_err("CCM connection appears to have failed: rc=%d.", rc); return FALSE; } } gboolean cib_ccm_dispatch(int fd, gpointer user_data) { int rc = 0; oc_ev_t *ccm_token = (oc_ev_t*)user_data; crm_debug_2("received callback"); rc = oc_ev_handle_event(ccm_token); if(0 == rc) { return TRUE; } else { crm_err("CCM connection appears to have failed: rc=%d.", rc); return FALSE; } } static void crm_ghash_clfree(gpointer data) { crm_free(data); } void cib_ccm_msg_callback( oc_ed_t event, void *cookie, size_t size, const void *data) { int instance = -1; gboolean update_id = FALSE; gboolean update_quorum = FALSE; const oc_ev_membership_t *membership = data; if(membership != NULL) { instance = membership->m_instance; } crm_debug("Process CCM event=%s (id=%d)", ccm_event_name(event), instance); switch(event) { case OC_EV_MS_NEW_MEMBERSHIP: case OC_EV_MS_INVALID: update_id = TRUE; update_quorum = TRUE; break; case OC_EV_MS_PRIMARY_RESTORED: update_id = TRUE; break; case OC_EV_MS_NOT_PRIMARY: crm_debug_2("Ignoring transitional CCM event: %s", ccm_event_name(event)); break; case OC_EV_MS_EVICTED: crm_err("Evicted from CCM: %s", ccm_event_name(event)); update_quorum = TRUE; break; default: crm_err("Unknown CCM event: %d", event); } if(update_id) { CRM_DEV_ASSERT(membership != NULL); if(crm_assert_failed) { return; } if(ccm_transition_id != NULL) { crm_free(ccm_transition_id); ccm_transition_id = NULL; } ccm_transition_id = crm_itoa(instance); set_transition(the_cib); } if(update_quorum) { unsigned int members = 0; int offset = 0; unsigned int lpc = 0; cib_have_quorum = ccm_have_quorum(event); if(cib_have_quorum) { crm_xml_add( the_cib,XML_ATTR_HAVE_QUORUM,XML_BOOLEAN_TRUE); } else { crm_xml_add( the_cib,XML_ATTR_HAVE_QUORUM,XML_BOOLEAN_FALSE); } crm_debug("Quorum %s after event=%s (id=%d)", cib_have_quorum?"(re)attained":"lost", ccm_event_name(event), instance); if(ccm_membership == NULL) { ccm_membership = g_hash_table_new_full( g_str_hash, g_str_equal, crm_ghash_clfree, NULL); } if(membership != NULL && membership->m_n_out != 0) { members = membership->m_n_out; offset = membership->m_out_idx; for(lpc = 0; lpc < members; lpc++) { oc_node_t a_node = membership->m_array[lpc+offset]; crm_info("LOST: %s", a_node.node_uname); g_hash_table_remove( ccm_membership, a_node.node_uname); } } if(membership != NULL && membership->m_n_member != 0) { members = membership->m_n_member; offset = membership->m_memb_idx; for(lpc = 0; lpc < members; lpc++) { oc_node_t a_node = membership->m_array[lpc+offset]; char *uname = crm_strdup(a_node.node_uname); crm_info("PEER: %s", uname); g_hash_table_replace( ccm_membership, uname, uname); } } } oc_ev_callback_done(cookie); set_connected_peers(the_cib); return; } gboolean can_write(int flags) { if(cib_have_quorum) { return TRUE; } else if((flags & cib_quorum_override) != 0) { return TRUE; } return FALSE; } static gboolean cib_force_exit(gpointer data) { crm_notice("Forcing exit!"); terminate_ha_connection(__FUNCTION__); return FALSE; } void initiate_exit(void) { int active = 0; HA_Message *leaving = NULL; g_hash_table_foreach(peer_hash, GHFunc_count_peers, &active); if(active < 2) { terminate_ha_connection(__FUNCTION__); return; } crm_info("Sending disconnect notification to %d peers...", active); leaving = ha_msg_new(3); ha_msg_add(leaving, F_TYPE, "cib"); ha_msg_add(leaving, F_CIB_OPERATION, "cib_shutdown_req"); send_ha_message(hb_conn, leaving, NULL, TRUE); crm_msg_del(leaving); Gmain_timeout_add(crm_get_msec("5s"), cib_force_exit, NULL); } void terminate_ha_connection(const char *caller) { if(hb_conn != NULL) { crm_info("%s: Disconnecting heartbeat", caller); hb_conn->llc_ops->signoff(hb_conn, FALSE); } else { crm_err("%s: No heartbeat connection", caller); exit(LSB_EXIT_OK); } } diff --git a/crm/cib/io.c b/crm/cib/io.c index f9ba29c7b0..ae14f8e179 100644 --- a/crm/cib/io.c +++ b/crm/cib/io.c @@ -1,740 +1,745 @@ -/* $Id: io.c,v 1.74 2006/07/06 10:55:09 andrew Exp $ */ +/* $Id: io.c,v 1.75 2006/07/06 13:30:24 andrew Exp $ */ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include const char * local_resource_path[] = { XML_CIB_TAG_STATUS, }; const char * resource_path[] = { XML_CIB_TAG_RESOURCES, }; const char * node_path[] = { XML_CIB_TAG_NODES, }; const char * constraint_path[] = { XML_CIB_TAG_CONSTRAINTS, }; gboolean initialized = FALSE; crm_data_t *the_cib = NULL; crm_data_t *node_search = NULL; crm_data_t *resource_search = NULL; crm_data_t *constraint_search = NULL; crm_data_t *status_search = NULL; extern gboolean cib_writes_enabled; extern char *ccm_transition_id; extern gboolean cib_have_quorum; extern GHashTable *peer_hash; extern GHashTable *ccm_membership; extern GTRIGSource *cib_writer; extern enum cib_errors cib_status; int set_connected_peers(crm_data_t *xml_obj); void GHFunc_count_peers(gpointer key, gpointer value, gpointer user_data); int write_cib_contents(gpointer p); #include #include #include static gboolean validate_cib_digest(crm_data_t *local_cib) { int s_res = -1; struct stat buf; char *digest = NULL; char *expected = NULL; gboolean passed = FALSE; FILE *expected_strm = NULL; int start = 0, length = 0, read_len = 0; if(local_cib != NULL) { digest = calculate_xml_digest(local_cib, FALSE); } s_res = stat(CIB_FILENAME ".sig", &buf); if (s_res != 0) { crm_warn("No on-disk digest present"); return TRUE; } expected_strm = fopen(CIB_FILENAME ".sig", "r"); start = ftell(expected_strm); fseek(expected_strm, 0L, SEEK_END); length = ftell(expected_strm); fseek(expected_strm, 0L, start); CRM_ASSERT(start == ftell(expected_strm)); crm_debug_3("Reading %d bytes from file", length); crm_malloc0(expected, (length+1)); read_len = fread(expected, 1, length, expected_strm); CRM_ASSERT(read_len == length); if(expected == NULL) { crm_err("On-disk digest is empty"); } else if(safe_str_eq(expected, digest)) { crm_debug("Digest comparision passed: %s", digest); passed = TRUE; } else { crm_err("Digest comparision failed: %s vs. %s", expected, digest); } crm_free(digest); crm_free(expected); return passed; } static int write_cib_digest(crm_data_t *local_cib, char *digest) { int rc = 0; FILE *digest_strm = fopen(CIB_FILENAME ".sig", "w"); char *local_digest = NULL; CRM_ASSERT(digest_strm != NULL); if(digest == NULL) { digest = calculate_xml_digest(local_cib, FALSE); CRM_ASSERT(digest != NULL); local_digest = digest; } rc = fprintf(digest_strm, "%s", digest); if(rc < 0) { cl_perror("Cannot write output to %s.sig", CIB_FILENAME); } fflush(digest_strm); fclose(digest_strm); crm_free(local_digest); return rc; } static gboolean validate_on_disk_cib(const char *filename, crm_data_t **on_disk_cib) { int s_res = -1; struct stat buf; FILE *cib_file = NULL; gboolean passed = TRUE; crm_data_t *root = NULL; if(filename != NULL) { s_res = stat(filename, &buf); } if (s_res == 0) { cib_file = fopen(filename, "r"); crm_debug_2("Reading cluster configuration from: %s", filename); root = file2xml(cib_file); fclose(cib_file); if(validate_cib_digest(root) == FALSE) { passed = FALSE; } } if(on_disk_cib != NULL) { *on_disk_cib = root; } else { free_xml(root); } return passed; } /* * It is the callers responsibility to free the output of this function */ crm_data_t* readCibXmlFile(const char *filename) { int s_res = -1; struct stat buf; gboolean valid = TRUE; const char *name = NULL; const char *value = NULL; crm_data_t *root = NULL; crm_data_t *status = NULL; struct passwd *cib_user = NULL; gboolean user_readwritable = FALSE; if(filename != NULL) { s_res = stat(filename, &buf); } if (s_res != 0) { return NULL; } cib_user = getpwnam(HA_CCMUSER); user_readwritable = (cib_user != NULL && buf.st_uid == cib_user->pw_uid && (buf.st_mode & (S_IRUSR|S_IWUSR))); if( S_ISREG(buf.st_mode) == FALSE ) { cib_status = cib_bad_permissions; crm_err("%s must be a regular file", filename); } else if( user_readwritable == FALSE ) { struct group *cib_grp = getgrnam(HA_APIGROUP); gboolean group_readwritable = ( cib_grp != NULL && buf.st_gid == cib_grp->gr_gid && (buf.st_mode & (S_IRGRP|S_IWGRP))); if( group_readwritable == FALSE ) { crm_err("%s must be owned and read/writeable by user %s," " or owned and read/writable by group %s", filename, HA_CCMUSER, HA_APIGROUP); cib_status = cib_bad_permissions; return NULL; } crm_warn("%s should be owned and read/writeable by user %s", filename, HA_CCMUSER); } crm_info("Reading cluster configuration from: %s", filename); if(validate_on_disk_cib(filename, &root) == FALSE) { valid = FALSE; crm_err("%s has been manually changed" " - if this was intended, please remove the md5 digest in %s.sig", filename, filename); cib_status = cib_bad_digest; return NULL; } /* Do this before DTD validation happens */ if(root != NULL) { /* strip out the status section if there is one */ status = find_xml_node(root, XML_CIB_TAG_STATUS, FALSE); if(status != NULL) { free_xml_from_parent(root, status); } create_xml_node(root, XML_CIB_TAG_STATUS); /* fill in some defaults */ name = XML_ATTR_GENERATION_ADMIN; value = crm_element_value(root, name); if(value == NULL) { crm_xml_add_int(root, name, 0); } name = XML_ATTR_GENERATION; value = crm_element_value(root, name); if(value == NULL) { crm_xml_add_int(root, name, 0); } name = XML_ATTR_NUMUPDATES; value = crm_element_value(root, name); if(value == NULL) { crm_xml_add_int(root, name, 0); } /* unset these and require the DC/CCM to update as needed */ crm_xml_add(root, XML_ATTR_HAVE_QUORUM, XML_BOOLEAN_FALSE); xml_remove_prop(root, XML_ATTR_DC_UUID); } crm_log_xml_info(root, "[on-disk]"); if(root != NULL) { const char *ignore_dtd = crm_element_value(root, "ignore_dtd"); gboolean dtd_ok = validate_with_dtd( root, TRUE, HA_LIBDIR"/heartbeat/crm.dtd"); if(dtd_ok == FALSE #if CRM_DEPRECATED_SINCE_2_0_4 && ignore_dtd != NULL #endif && crm_is_true(ignore_dtd) == FALSE) { crm_err("On disk CIB does not conform to the DTD"); valid = FALSE; } } if(root == NULL) { crm_crit("Parse ERROR reading %s.", filename); cib_status = cib_bad_config; return NULL; } else if(valid == FALSE) { crm_err("%s does not contain a valid configuration", filename); cib_status = cib_bad_config; } crm_xml_add(root, "generated", XML_BOOLEAN_FALSE); if(do_id_check(root, NULL, FALSE, FALSE)) { crm_crit("%s does not contain a vaild configuration.", filename); cib_status = cib_bad_config; } if (verifyCibXml(root) == FALSE) { crm_crit("%s does not contain a vaild configuration.", filename); cib_status = cib_bad_config; } return root; } /* * The caller should never free the return value */ crm_data_t* get_the_CIB(void) { return the_cib; } gboolean uninitializeCib(void) { crm_data_t *tmp_cib = the_cib; if(tmp_cib == NULL) { crm_err("The CIB has already been deallocated."); return FALSE; } initialized = FALSE; the_cib = NULL; node_search = NULL; resource_search = NULL; constraint_search = NULL; status_search = NULL; crm_err("Deallocating the CIB."); free_xml(tmp_cib); crm_err("The CIB has been deallocated."); return TRUE; } /* * This method will not free the old CIB pointer or the new one. * We rely on the caller to have saved a pointer to the old CIB * and to free the old/bad one depending on what is appropriate. */ gboolean initializeCib(crm_data_t *new_cib) { gboolean is_valid = TRUE; crm_data_t *tmp_node = NULL; if(new_cib == NULL) { return FALSE; } xml_validate(new_cib); tmp_node = get_object_root(XML_CIB_TAG_NODES, new_cib); if (tmp_node == NULL) { is_valid = FALSE; } tmp_node = get_object_root(XML_CIB_TAG_RESOURCES, new_cib); if (tmp_node == NULL) { is_valid = FALSE; } tmp_node = get_object_root(XML_CIB_TAG_CONSTRAINTS, new_cib); if (tmp_node == NULL) { is_valid = FALSE; } tmp_node = get_object_root(XML_CIB_TAG_CRMCONFIG, new_cib); if (tmp_node == NULL) { is_valid = FALSE; } tmp_node = get_object_root(XML_CIB_TAG_STATUS, new_cib); if (is_valid && tmp_node == NULL) { create_xml_node(new_cib, XML_CIB_TAG_STATUS); } if(is_valid == FALSE) { crm_warn("CIB Verification failed"); return FALSE; } update_counters(__FILE__, __FUNCTION__, new_cib); the_cib = new_cib; initialized = TRUE; return TRUE; } static int archive_file(const char *oldname, const char *newname, const char *ext) { /* move 'oldname' to 'newname' by creating a hard link to it * and then removing the original hard link */ int rc = 0; int res = 0; struct stat tmp; int s_res = 0; char *backup_file = NULL; static const char *back_ext = "bak"; /* calculate the backup name if required */ if(newname != NULL) { backup_file = crm_strdup(newname); } else { int max_name_len = 1024; crm_malloc0(backup_file, max_name_len); if (ext == NULL) { ext = back_ext; } snprintf(backup_file, max_name_len - 1, "%s.%s", oldname, ext); } if(backup_file == NULL || strlen(backup_file) == 0) { crm_err("%s backup filename was %s", newname == NULL?"calculated":"supplied", backup_file == NULL?"null":"empty"); rc = -4; } s_res = stat(backup_file, &tmp); /* unlink the old backup */ if (rc == 0 && s_res >= 0) { res = unlink(backup_file); if (res < 0) { cl_perror("Could not unlink %s", backup_file); rc = -1; } } s_res = stat(oldname, &tmp); /* copy */ if (rc == 0 && s_res >= 0) { res = link(oldname, backup_file); if (res < 0) { cl_perror("Could not create backup %s from %s", backup_file, oldname); rc = -2; } } /* unlink the original */ if (rc == 0 && s_res >= 0) { res = unlink(oldname); if (res < 0) { cl_perror("Could not unlink %s", oldname); rc = -3; } } crm_free(backup_file); return rc; } /* * This method will free the old CIB pointer on success and the new one * on failure. */ int activateCibXml(crm_data_t *new_cib, const char *ignored) { int error_code = cib_ok; crm_data_t *saved_cib = get_the_CIB(); const char *ignore_dtd = NULL; long new_bytes, new_allocs, new_frees; long old_bytes, old_allocs, old_frees; crm_log_xml_debug_4(new_cib, "Attempting to activate CIB"); CRM_ASSERT(new_cib != saved_cib); if(saved_cib != NULL) { crm_validate_data(saved_cib); } ignore_dtd = crm_element_value(new_cib, "ignore_dtd"); if( #if CRM_DEPRECATED_SINCE_2_0_4 ignore_dtd != NULL && #endif crm_is_true(ignore_dtd) == FALSE && validate_with_dtd( new_cib, TRUE, HA_LIBDIR"/heartbeat/crm.dtd") == FALSE) { error_code = cib_dtd_validation; crm_err("Ignoring invalid CIB"); } if(error_code == cib_ok && initializeCib(new_cib) == FALSE) { error_code = cib_ACTIVATION; crm_err("Ignoring invalid or NULL CIB"); } if(error_code != cib_ok) { if(saved_cib != NULL) { crm_warn("Reverting to last known CIB"); if (initializeCib(saved_cib) == FALSE) { /* oh we are so dead */ crm_crit("Couldn't re-initialize the old CIB!"); cl_flush_logs(); exit(1); } } else { crm_crit("Could not write out new CIB and no saved" " version to revert to"); } } else if(cib_writes_enabled && cib_status == cib_ok) { crm_debug_2("Triggering CIB write"); G_main_set_trigger(cib_writer); crm_xml_nbytes(new_cib, &new_bytes, &new_allocs, &new_frees); crm_xml_nbytes(saved_cib, &old_bytes, &old_allocs, &old_frees); if(new_bytes != old_bytes) { crm_info("CIB size is %ld bytes (was %ld)", new_bytes, old_bytes); - crm_adjust_mem_stats(new_bytes - old_bytes, new_allocs - old_allocs, - new_frees - old_frees); - } + crm_adjust_mem_stats(NULL, new_bytes - old_bytes, + new_allocs - old_allocs, new_frees - old_frees); + if(crm_running_stats != NULL) { + crm_adjust_mem_stats( + crm_running_stats, new_bytes - old_bytes, + new_allocs - old_allocs, new_frees - old_frees); + } + } } if(the_cib != saved_cib && the_cib != new_cib) { CRM_DEV_ASSERT(error_code != cib_ok); CRM_DEV_ASSERT(the_cib == NULL); } if(the_cib != new_cib) { free_xml(new_cib); CRM_DEV_ASSERT(error_code != cib_ok); } if(the_cib != saved_cib) { free_xml(saved_cib); } return error_code; } int write_cib_contents(gpointer p) { int rc = 0; char *digest = NULL; crm_data_t *cib_status_root = NULL; const char *digest_filename = CIB_FILENAME ".sig"; /* we can scribble on "the_cib" here and not affect the parent */ const char *epoch = crm_element_value(the_cib, XML_ATTR_GENERATION); const char *updates = crm_element_value(the_cib, XML_ATTR_NUMUPDATES); const char *admin_epoch = crm_element_value( the_cib, XML_ATTR_GENERATION_ADMIN); /* check the admin didnt modify it underneath us */ if(validate_on_disk_cib(CIB_FILENAME, NULL) == FALSE) { crm_err("%s was manually modified while Heartbeat was active!", CIB_FILENAME); exit(LSB_EXIT_GENERIC); } rc = archive_file(CIB_FILENAME, NULL, "last"); if(rc != 0) { crm_err("Could not make backup of the existing CIB: %d", rc); exit(LSB_EXIT_GENERIC); } rc = archive_file(digest_filename, NULL, "last"); if(rc != 0) { crm_warn("Could not make backup of the existing CIB digest: %d", rc); } /* Given that we discard the status section on startup * there is no point writing it out in the first place * since users just get confused by it * * Although, it does help me once in a while * * So delete the status section before we write it out */ cib_status_root = find_xml_node(the_cib, XML_CIB_TAG_STATUS, TRUE); CRM_DEV_ASSERT(cib_status_root != NULL); if(cib_status_root != NULL) { free_xml_from_parent(the_cib, cib_status_root); } rc = write_xml_file(the_cib, CIB_FILENAME, FALSE); if(rc <= 0) { crm_err("Changes couldn't be written to disk"); exit(LSB_EXIT_GENERIC); } digest = calculate_xml_digest(the_cib, FALSE); crm_info("Wrote version %s.%s.%s of the CIB to disk (digest: %s)", admin_epoch?admin_epoch:"0", epoch?epoch:"0", updates?updates:"0", digest); rc = write_cib_digest(the_cib, digest); if(rc <= 0) { crm_err("Digest couldn't be written to disk"); exit(LSB_EXIT_GENERIC); } #if 0 if(validate_on_disk_cib(CIB_FILENAME, NULL) == FALSE) { crm_err("wrote incorrect digest"); exit(LSB_EXIT_GENERIC); } #endif exit(LSB_EXIT_OK); return HA_OK; } gboolean set_transition(crm_data_t *xml_obj) { const char *current = crm_element_value( xml_obj, XML_ATTR_CCM_TRANSITION); if(safe_str_neq(current, ccm_transition_id)) { crm_debug("CCM transition: old=%s, new=%s", current, ccm_transition_id); crm_xml_add(xml_obj, XML_ATTR_CCM_TRANSITION,ccm_transition_id); return TRUE; } return FALSE; } gboolean set_connected_peers(crm_data_t *xml_obj) { int active = 0; int current = 0; char *peers_s = NULL; const char *current_s = crm_element_value(xml_obj, XML_ATTR_NUMPEERS); g_hash_table_foreach(peer_hash, GHFunc_count_peers, &active); current = crm_parse_int(current_s, "0"); if(current != active) { peers_s = crm_itoa(active); crm_xml_add(xml_obj, XML_ATTR_NUMPEERS, peers_s); crm_debug("We now have %s active peers", peers_s); crm_free(peers_s); return TRUE; } return FALSE; } gboolean update_quorum(crm_data_t *xml_obj) { const char *quorum_value = XML_BOOLEAN_FALSE; const char *current = crm_element_value(xml_obj, XML_ATTR_HAVE_QUORUM); if(cib_have_quorum) { quorum_value = XML_BOOLEAN_TRUE; } if(safe_str_neq(current, quorum_value)) { crm_debug("CCM quorum: old=%s, new=%s", current, quorum_value); crm_xml_add(xml_obj, XML_ATTR_HAVE_QUORUM, quorum_value); return TRUE; } return FALSE; } gboolean update_counters(const char *file, const char *fn, crm_data_t *xml_obj) { gboolean did_update = FALSE; did_update = did_update || update_quorum(xml_obj); did_update = did_update || set_transition(xml_obj); did_update = did_update || set_connected_peers(xml_obj); if(did_update) { do_crm_log(LOG_DEBUG, file, fn, "Counters updated"); } return did_update; } void GHFunc_count_peers(gpointer key, gpointer value, gpointer user_data) { int *active = user_data; if(safe_str_eq(value, ONLINESTATUS)) { (*active)++; } else if(safe_str_eq(value, JOINSTATUS)) { (*active)++; } } diff --git a/crm/cib/main.c b/crm/cib/main.c index bcbdb97784..7e1e59a127 100644 --- a/crm/cib/main.c +++ b/crm/cib/main.c @@ -1,503 +1,517 @@ -/* $Id: main.c,v 1.47 2006/07/06 09:30:28 andrew Exp $ */ +/* $Id: main.c,v 1.48 2006/07/06 13:30:24 andrew Exp $ */ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* #include */ #include /* #include */ #include #include #include #include #include #include #include #include #include #include gboolean cib_shutdown_flag = FALSE; gboolean stand_alone = FALSE; enum cib_errors cib_status = cib_ok; extern void oc_ev_special(const oc_ev_t *, oc_ev_class_t , int ); GMainLoop* mainloop = NULL; const char* crm_system_name = CRM_SYSTEM_CIB; char *cib_our_uname = NULL; oc_ev_t *cib_ev_token; gboolean cib_writes_enabled = TRUE; void usage(const char* cmd, int exit_status); int init_start(void); gboolean cib_register_ha(ll_cluster_t *hb_cluster, const char *client_name); gboolean cib_shutdown(int nsig, gpointer unused); void cib_ha_connection_destroy(gpointer user_data); gboolean startCib(const char *filename); extern gboolean cib_msg_timeout(gpointer data); extern int write_cib_contents(gpointer p); ll_cluster_t *hb_conn = NULL; GTRIGSource *cib_writer = NULL; #define OPTARGS "hVs" static void cib_diskwrite_complete(gpointer userdata, int status, int signo, int exitcode) { if(exitcode != LSB_EXIT_OK || signo != 0 || status != 0) { crm_err("Disk write failed: status=%d, signo=%d, exitcode=%d", status, signo, exitcode); if(cib_writes_enabled) { crm_err("Disabling disk writes after write failure"); cib_writes_enabled = FALSE; } } else { crm_debug_2("Disk write passed"); } } int main(int argc, char ** argv) { int flag; int argerr = 0; crm_log_init(crm_system_name); G_main_add_SignalHandler( G_PRIORITY_HIGH, SIGTERM, cib_shutdown, NULL, NULL); cib_writer = G_main_add_tempproc_trigger( G_PRIORITY_LOW, write_cib_contents, "write_cib_contents", NULL, NULL, NULL, cib_diskwrite_complete); EnableProcLogging(); set_sigchld_proctrack(G_PRIORITY_HIGH); client_list = g_hash_table_new(g_str_hash, g_str_equal); peer_hash = g_hash_table_new(g_str_hash, g_str_equal); while ((flag = getopt(argc, argv, OPTARGS)) != EOF) { switch(flag) { case 'V': alter_debug(DEBUG_INC); break; case 's': stand_alone = TRUE; cl_log_enable_stderr(1); break; case 'h': /* Help message */ usage(crm_system_name, LSB_EXIT_OK); break; default: ++argerr; break; } } if (optind > argc) { ++argerr; } if (argerr) { usage(crm_system_name,LSB_EXIT_GENERIC); } /* read local config file */ return init_start(); } unsigned long cib_num_ops = 0; -const char *cib_stat_interval = "10min"; +const char *cib_stat_interval = "10s"; unsigned long cib_num_local = 0, cib_num_updates = 0, cib_num_fail = 0; unsigned long cib_bad_connects = 0, cib_num_timeouts = 0; longclock_t cib_call_time = 0; gboolean cib_stats(gpointer data); gboolean cib_stats(gpointer data) { int local_log_level = LOG_DEBUG; static unsigned long last_stat = 0; unsigned int cib_calls_ms = 0; static unsigned long cib_stat_interval_ms = 0; + cl_mem_stats_t saved_stats; + if(crm_running_stats == NULL) { + START_stat_free_op(); + crm_malloc0(crm_running_stats, sizeof(cl_mem_stats_t)); + END_stat_free_op(); + crm_zero_mem_stats(crm_running_stats); + } + crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); + if(crm_diff_mem_stats(LOG_ERR, LOG_ERR, __PRETTY_FUNCTION__, NULL, crm_running_stats)) { + crm_err("Total alloc's %ld for %ld bytes", + crm_running_stats->numalloc, crm_running_stats->nbytes_alloc); + } + *crm_running_stats = saved_stats; + if(cib_stat_interval_ms == 0) { cib_stat_interval_ms = crm_get_msec(cib_stat_interval); } cib_calls_ms = longclockto_ms(cib_call_time); if((cib_num_ops - last_stat) > 0) { unsigned long calls_diff = cib_num_ops - last_stat; double stat_1 = (1000*cib_calls_ms)/calls_diff; local_log_level = LOG_INFO; crm_log_maybe(local_log_level, "Processed %lu operations" " (%.2fus average, %lu%% utilization) in the last %s", calls_diff, stat_1, (100*cib_calls_ms)/cib_stat_interval_ms, cib_stat_interval); } crm_log_maybe(local_log_level+1, "\tDetail: %lu operations (%ums total)" " (%lu local, %lu updates, %lu failures," " %lu timeouts, %lu bad connects)", cib_num_ops, cib_calls_ms, cib_num_local, cib_num_updates, cib_num_fail, cib_bad_connects, cib_num_timeouts); last_stat = cib_num_ops; cib_call_time = 0; return TRUE; } int init_start(void) { gboolean was_error = FALSE; if(stand_alone == FALSE) { hb_conn = ll_cluster_new("heartbeat"); if(cib_register_ha(hb_conn, CRM_SYSTEM_CIB) == FALSE) { crm_crit("Cannot sign in to heartbeat... terminating"); exit(1); } } if(startCib(CIB_FILENAME) == FALSE){ crm_crit("Cannot start CIB... terminating"); exit(1); } was_error = init_server_ipc_comms( crm_strdup(cib_channel_callback), cib_client_connect_null, default_ipc_connection_destroy); was_error = was_error || init_server_ipc_comms( crm_strdup(cib_channel_ro), cib_client_connect_rw_ro, default_ipc_connection_destroy); was_error = was_error || init_server_ipc_comms( crm_strdup(cib_channel_rw), cib_client_connect_rw_ro, default_ipc_connection_destroy); was_error = was_error || init_server_ipc_comms( crm_strdup(cib_channel_rw_synchronous), cib_client_connect_rw_synch, default_ipc_connection_destroy); was_error = was_error || init_server_ipc_comms( crm_strdup(cib_channel_ro_synchronous), cib_client_connect_ro_synch, default_ipc_connection_destroy); if(stand_alone) { if(was_error) { crm_err("Couldnt start"); return 1; } cib_is_master = TRUE; /* Create the mainloop and run it... */ mainloop = g_main_new(FALSE); crm_info("Starting %s mainloop", crm_system_name); /* Gmain_timeout_add(crm_get_msec("10s"), cib_msg_timeout, NULL); */ /* Gmain_timeout_add( */ /* crm_get_msec(cib_stat_interval), cib_stats, NULL); */ g_main_run(mainloop); return_to_orig_privs(); return 0; } if(was_error == FALSE) { crm_debug_3("Be informed of CRM Client Status changes"); if (HA_OK != hb_conn->llc_ops->set_cstatus_callback( hb_conn, cib_client_status_callback, hb_conn)) { crm_err("Cannot set cstatus callback: %s", hb_conn->llc_ops->errmsg(hb_conn)); was_error = TRUE; } else { crm_debug_3("Client Status callback set"); } } if(was_error == FALSE) { gboolean did_fail = TRUE; int num_ccm_fails = 0; int max_ccm_fails = 30; int ret; int cib_ev_fd; while(did_fail && was_error == FALSE) { did_fail = FALSE; crm_debug_3("Registering with CCM"); ret = oc_ev_register(&cib_ev_token); if (ret != 0) { crm_warn("CCM registration failed"); did_fail = TRUE; } if(did_fail == FALSE) { crm_debug_3("Setting up CCM callbacks"); ret = oc_ev_set_callback( cib_ev_token, OC_EV_MEMB_CLASS, cib_ccm_msg_callback, NULL); if (ret != 0) { crm_warn("CCM callback not set"); did_fail = TRUE; } } if(did_fail == FALSE) { oc_ev_special(cib_ev_token, OC_EV_MEMB_CLASS, 0); crm_debug_3("Activating CCM token"); ret = oc_ev_activate(cib_ev_token, &cib_ev_fd); if (ret != 0){ crm_warn("CCM Activation failed"); did_fail = TRUE; } } if(did_fail) { num_ccm_fails++; oc_ev_unregister(cib_ev_token); if(num_ccm_fails < max_ccm_fails){ crm_warn("CCM Connection failed" " %d times (%d max)", num_ccm_fails, max_ccm_fails); sleep(1); } else { crm_err("CCM Activation failed" " %d (max) times", num_ccm_fails); was_error = TRUE; } } } if(was_error == FALSE) { crm_debug_3("CCM Activation passed... all set to go!"); G_main_add_fd(G_PRIORITY_HIGH, cib_ev_fd, FALSE, cib_ccm_dispatch, cib_ev_token, default_ipc_connection_destroy); } } if(was_error == FALSE) { /* Async get client status information in the cluster */ crm_debug_3("Requesting an initial dump of CIB client_status"); hb_conn->llc_ops->client_status( hb_conn, NULL, CRM_SYSTEM_CIB, -1); /* Create the mainloop and run it... */ mainloop = g_main_new(FALSE); crm_info("Starting %s mainloop", crm_system_name); Gmain_timeout_add(crm_get_msec("10s"), cib_msg_timeout, NULL); Gmain_timeout_add( crm_get_msec(cib_stat_interval), cib_stats, NULL); g_main_run(mainloop); return_to_orig_privs(); } else { crm_err("Couldnt start all communication channels, exiting."); } return 0; } void usage(const char* cmd, int exit_status) { FILE* stream; stream = exit_status ? stderr : stdout; fprintf(stream, "usage: %s [-srkh]" "[-c configure file]\n", cmd); /* fprintf(stream, "\t-d\tsets debug level\n"); */ /* fprintf(stream, "\t-s\tgets daemon status\n"); */ /* fprintf(stream, "\t-r\trestarts daemon\n"); */ /* fprintf(stream, "\t-k\tstops daemon\n"); */ /* fprintf(stream, "\t-h\thelp message\n"); */ fflush(stream); exit(exit_status); } gboolean cib_register_ha(ll_cluster_t *hb_cluster, const char *client_name) { const char *uname = NULL; crm_info("Signing in with Heartbeat"); if (hb_cluster->llc_ops->signon(hb_cluster, client_name)!= HA_OK) { crm_err("Cannot sign on with heartbeat: %s", hb_cluster->llc_ops->errmsg(hb_cluster)); return FALSE; } crm_debug_3("Be informed of CIB messages"); if (HA_OK != hb_cluster->llc_ops->set_msg_callback( hb_cluster, T_CIB, cib_peer_callback, hb_cluster)){ crm_err("Cannot set msg callback: %s", hb_cluster->llc_ops->errmsg(hb_cluster)); return FALSE; } crm_debug_3("Finding our node name"); if ((uname = hb_cluster->llc_ops->get_mynodeid(hb_cluster)) == NULL) { crm_err("get_mynodeid() failed"); return FALSE; } cib_our_uname = crm_strdup(uname); crm_info("FSA Hostname: %s", cib_our_uname); crm_debug_3("Adding channel to mainloop"); G_main_add_IPC_Channel( G_PRIORITY_DEFAULT, hb_cluster->llc_ops->ipcchan(hb_cluster), FALSE, cib_ha_dispatch, hb_cluster /* userdata */, cib_ha_connection_destroy); return TRUE; } void cib_ha_connection_destroy(gpointer user_data) { if(cib_shutdown_flag) { crm_info("Heartbeat disconnection complete... exiting"); } else { crm_err("Heartbeat connection lost! Exiting."); } if (mainloop != NULL && g_main_is_running(mainloop)) { g_main_quit(mainloop); } else { exit(LSB_EXIT_OK); } } static void disconnect_cib_client(gpointer key, gpointer value, gpointer user_data) { cib_client_t *a_client = value; crm_debug_2("Processing client %s/%s... send=%d, recv=%d", a_client->name, a_client->channel_name, (int)a_client->channel->send_queue->current_qlen, (int)a_client->channel->recv_queue->current_qlen); if(a_client->channel->ch_status == IPC_CONNECT) { a_client->channel->ops->resume_io(a_client->channel); if(a_client->channel->send_queue->current_qlen != 0 || a_client->channel->recv_queue->current_qlen != 0) { crm_info("Flushed messages to/from %s/%s... send=%d, recv=%d", a_client->name, a_client->channel_name, (int)a_client->channel->send_queue->current_qlen, (int)a_client->channel->recv_queue->current_qlen); } } if(a_client->channel->ch_status == IPC_CONNECT) { crm_warn("Disconnecting %s/%s...", a_client->name, a_client->channel_name); a_client->channel->ops->disconnect(a_client->channel); } } extern gboolean cib_process_disconnect( IPC_Channel *channel, cib_client_t *cib_client); gboolean cib_shutdown(int nsig, gpointer unused) { if(cib_shutdown_flag == FALSE) { cib_shutdown_flag = TRUE; crm_debug("Disconnecting %d clients", g_hash_table_size(client_list)); g_hash_table_foreach(client_list, disconnect_cib_client, NULL); crm_info("Disconnected %d clients", g_hash_table_size(client_list)); cib_process_disconnect(NULL, NULL); } else { crm_info("Waiting for %d clients to disconnect...", g_hash_table_size(client_list)); } return TRUE; } gboolean startCib(const char *filename) { crm_data_t *cib = readCibXmlFile(filename); if(cib == NULL) { crm_warn("Cluster configuration not found: %s." " Creating an empty one.", filename); cib = createEmptyCib(); crm_xml_add(cib, XML_ATTR_GENERATION_ADMIN, "0"); crm_xml_add(cib, XML_ATTR_GENERATION, "0"); crm_xml_add(cib, XML_ATTR_NUMUPDATES, "0"); } if(activateCibXml(cib, filename) != 0) { return FALSE; } crm_info("CIB Initialization completed successfully"); return TRUE; } diff --git a/crm/cib/notify.c b/crm/cib/notify.c index 8b2d4244fb..122754aa38 100644 --- a/crm/cib/notify.c +++ b/crm/cib/notify.c @@ -1,428 +1,428 @@ -/* $Id: notify.c,v 1.37 2006/07/06 09:30:28 andrew Exp $ */ +/* $Id: notify.c,v 1.38 2006/07/06 13:30:24 andrew Exp $ */ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include extern GHashTable *client_list; int pending_updates = 0; void cib_notify_client(gpointer key, gpointer value, gpointer user_data); void attach_cib_generation(HA_Message *msg, const char *field, crm_data_t *a_cib); void do_cib_notify( int options, const char *op, crm_data_t *update, enum cib_errors result, crm_data_t *result_data, const char *msg_type); void cib_notify_client(gpointer key, gpointer value, gpointer user_data) { IPC_Channel *ipc_client = NULL; HA_Message *update_msg = user_data; cib_client_t *client = value; const char *type = NULL; gboolean is_pre = FALSE; gboolean is_post = FALSE; gboolean is_confirm = FALSE; gboolean is_replace = FALSE; gboolean is_diff = FALSE; gboolean do_send = FALSE; int qlen = 0; int max_qlen = 0; cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); CRM_DEV_ASSERT(client != NULL); CRM_DEV_ASSERT(update_msg != NULL); type = cl_get_string(update_msg, F_SUBTYPE); CRM_DEV_ASSERT(type != NULL); if(client == NULL) { crm_warn("Skipping NULL client"); return; } else if(client->channel == NULL) { crm_warn("Skipping client with NULL channel"); return; } else if(client->name == NULL) { crm_debug_2("Skipping unnammed client / comamnd channel"); return; } if(safe_str_eq(type, T_CIB_PRE_NOTIFY)) { is_pre = TRUE; } else if(safe_str_eq(type, T_CIB_POST_NOTIFY)) { is_post = TRUE; } else if(safe_str_eq(type, T_CIB_UPDATE_CONFIRM)) { is_confirm = TRUE; } else if(safe_str_eq(type, T_CIB_DIFF_NOTIFY)) { is_diff = TRUE; } else if(safe_str_eq(type, T_CIB_REPLACE_NOTIFY)) { is_replace = TRUE; } ipc_client = client->channel; qlen = ipc_client->send_queue->current_qlen; max_qlen = ipc_client->send_queue->max_qlen; #if 1 /* get_chan_status() causes memory to be allocated that isnt free'd * until the message is read (which messes up the memory stats) */ if(ipc_client->ops->get_chan_status(ipc_client) != IPC_CONNECT) { crm_debug_2("Skipping notification to disconnected" " client %s/%s", client->name, client->id); } else if(client->pre_notify && is_pre) { if(qlen < (int)(0.4 * max_qlen)) { do_send = TRUE; } else { crm_warn("Throttling pre-notifications due to" " high load: queue=%d (max=%d)", qlen, max_qlen); } } else if(client->post_notify && is_post) { if(qlen < (int)(0.7 * max_qlen)) { do_send = TRUE; } else { crm_warn("Throttling post-notifications due to" " extreme load: queue=%d (max=%d)", qlen, max_qlen); } /* these are critical */ } else #endif if(client->diffs && is_diff) { do_send = TRUE; } else if(client->confirmations && is_confirm) { do_send = TRUE; } else if(client->replace && is_replace) { do_send = TRUE; } if(do_send) { crm_debug_2("Notifying client %s/%s of %s update (queue=%d)", client->name, client->channel_name, type, qlen); if(ipc_client->send_queue->current_qlen >= ipc_client->send_queue->max_qlen) { /* We never want the CIB to exit because our client is slow */ crm_crit("%s-notification of client %s/%s failed - queue saturated", is_confirm?"Confirmation":is_post?"Post":"Pre", client->name, client->id); } else if(send_ipc_message(ipc_client, update_msg) == FALSE) { crm_warn("Notification of client %s/%s failed", client->name, client->id); } } else { crm_debug_3("Client %s/%s not interested in %s notifications", client->name, client->channel_name, type); } - crm_diff_mem_stats(LOG_DEBUG, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_DEBUG, LOG_DEBUG, __PRETTY_FUNCTION__, NULL, &saved_stats); } void cib_pre_notify( int options, const char *op, crm_data_t *existing, crm_data_t *update) { HA_Message *update_msg = NULL; const char *type = NULL; const char *id = NULL; cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); update_msg = ha_msg_new(6); if(update != NULL) { id = crm_element_value(update, XML_ATTR_ID); } ha_msg_add(update_msg, F_TYPE, T_CIB_NOTIFY); ha_msg_add(update_msg, F_SUBTYPE, T_CIB_PRE_NOTIFY); ha_msg_add(update_msg, F_CIB_OPERATION, op); if(id != NULL) { ha_msg_add(update_msg, F_CIB_OBJID, id); } if(update != NULL) { ha_msg_add(update_msg, F_CIB_OBJTYPE, crm_element_name(update)); } else if(existing != NULL) { ha_msg_add(update_msg, F_CIB_OBJTYPE, crm_element_name(existing)); } type = cl_get_string(update_msg, F_CIB_OBJTYPE); attach_cib_generation(update_msg, "cib_generation", the_cib); if(existing != NULL) { add_message_xml(update_msg, F_CIB_EXISTING, existing); } if(update != NULL) { add_message_xml(update_msg, F_CIB_UPDATE, update); } g_hash_table_foreach(client_list, cib_notify_client, update_msg); if(update == NULL) { crm_debug_2("Performing operation %s (on section=%s)", op, type); } else { crm_debug_2("Performing %s on <%s%s%s>", op, type, id?" id=":"", id?id:""); } crm_msg_del(update_msg); - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_ERR, __PRETTY_FUNCTION__, NULL, &saved_stats); } void cib_post_notify(int options, const char *op, crm_data_t *update, enum cib_errors result, crm_data_t *new_obj) { do_cib_notify( options, op, update, result, new_obj, T_CIB_UPDATE_CONFIRM); } void cib_diff_notify( int options, const char *client, const char *call_id, const char *op, crm_data_t *update, enum cib_errors result, crm_data_t *diff) { int add_updates = 0; int add_epoch = 0; int add_admin_epoch = 0; int del_updates = 0; int del_epoch = 0; int del_admin_epoch = 0; int log_level = LOG_INFO; cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); if(diff == NULL) { return; } if(result != cib_ok) { log_level = LOG_WARNING; } cib_diff_version_details( diff, &add_admin_epoch, &add_epoch, &add_updates, &del_admin_epoch, &del_epoch, &del_updates); if(add_updates != del_updates) { crm_log_maybe(log_level, "Update (client: %s%s%s): %d.%d.%d -> %d.%d.%d (%s)", client, call_id?", call:":"", call_id?call_id:"", del_admin_epoch, del_epoch, del_updates, add_admin_epoch, add_epoch, add_updates, cib_error2string(result)); } else if(diff != NULL) { crm_log_maybe(log_level, "Local-only Change (client:%s%s%s): %d.%d.%d (%s)", client, call_id?", call: ":"", call_id?call_id:"", add_admin_epoch, add_epoch, add_updates, cib_error2string(result)); } do_cib_notify(options, op, update, result, diff, T_CIB_DIFF_NOTIFY); - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_ERR, __PRETTY_FUNCTION__, NULL, &saved_stats); } void do_cib_notify( int options, const char *op, crm_data_t *update, enum cib_errors result, crm_data_t *result_data, const char *msg_type) { HA_Message *update_msg = NULL; const char *type = NULL; const char *id = NULL; update_msg = ha_msg_new(8); if(result_data != NULL) { id = crm_element_value(result_data, XML_ATTR_ID); } ha_msg_add(update_msg, F_TYPE, T_CIB_NOTIFY); ha_msg_add(update_msg, F_SUBTYPE, msg_type); ha_msg_add(update_msg, F_CIB_OPERATION, op); ha_msg_add_int(update_msg, F_CIB_RC, result); if(id != NULL) { ha_msg_add(update_msg, F_CIB_OBJID, id); } if(update != NULL) { crm_debug_4("Setting type to update->name: %s", crm_element_name(update)); ha_msg_add(update_msg, F_CIB_OBJTYPE, crm_element_name(update)); type = crm_element_name(update); } else if(result_data != NULL) { crm_debug_4("Setting type to new_obj->name: %s", crm_element_name(result_data)); ha_msg_add(update_msg, F_CIB_OBJTYPE, crm_element_name(result_data)); type = crm_element_name(result_data); } else { crm_debug_4("Not Setting type"); } attach_cib_generation(update_msg, "cib_generation", the_cib); if(update != NULL) { add_message_xml(update_msg, F_CIB_UPDATE, update); } if(result_data != NULL) { add_message_xml(update_msg, F_CIB_UPDATE_RESULT, result_data); } crm_debug_3("Notifying clients"); g_hash_table_foreach(client_list, cib_notify_client, update_msg); if(update == NULL) { if(result == cib_ok) { crm_debug_2("Operation %s (on section=%s) completed", op, crm_str(type)); } else { crm_warn("Operation %s (on section=%s) FAILED: (%d) %s", op, crm_str(type), result, cib_error2string(result)); } } else { if(result == cib_ok) { crm_debug_2("Completed %s of <%s %s%s>", op, crm_str(type), id?"id=":"", id?id:""); } else { crm_warn("%s of <%s %s%s> FAILED: %s", op,crm_str(type), id?"id=":"", id?id:"", cib_error2string(result)); } } crm_msg_del(update_msg); crm_debug_3("Notify complete"); } void attach_cib_generation(HA_Message *msg, const char *field, crm_data_t *a_cib) { crm_data_t *generation = create_xml_node( NULL, XML_CIB_TAG_GENERATION_TUPPLE); if(a_cib != NULL) { copy_in_properties(generation, a_cib); } add_message_xml(msg, field, generation); free_xml(generation); } void cib_replace_notify(crm_data_t *update, enum cib_errors result, crm_data_t *diff) { const char *origin = NULL; HA_Message *replace_msg = NULL; int add_updates = 0; int add_epoch = 0; int add_admin_epoch = 0; int del_updates = 0; int del_epoch = 0; int del_admin_epoch = 0; cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); if(diff == NULL) { return; } cib_diff_version_details( diff, &add_admin_epoch, &add_epoch, &add_updates, &del_admin_epoch, &del_epoch, &del_updates); origin = crm_element_value(update, F_CRM_ORIGIN); if(add_updates != del_updates) { crm_info("Replaced: %d.%d.%d -> %d.%d.%d from %s", del_admin_epoch, del_epoch, del_updates, add_admin_epoch, add_epoch, add_updates, origin); } else if(diff != NULL) { crm_info("Local-only Replace: %d.%d.%d from %s", add_admin_epoch, add_epoch, add_updates, origin); } replace_msg = ha_msg_new(8); ha_msg_add(replace_msg, F_TYPE, T_CIB_NOTIFY); ha_msg_add(replace_msg, F_SUBTYPE, T_CIB_REPLACE_NOTIFY); ha_msg_add(replace_msg, F_CIB_OPERATION, CIB_OP_REPLACE); ha_msg_add_int(replace_msg, F_CIB_RC, result); attach_cib_generation(replace_msg, "cib-replace-generation", update); crm_log_message_adv(LOG_DEBUG_2,"CIB Replaced", replace_msg); g_hash_table_foreach(client_list, cib_notify_client, replace_msg); crm_msg_del(replace_msg); - crm_diff_mem_stats(LOG_ERR, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_ERR, LOG_ERR, __PRETTY_FUNCTION__, NULL, &saved_stats); } diff --git a/crm/crmd/cib.c b/crm/crmd/cib.c index 2290f7a4e4..992cf6ec47 100644 --- a/crm/crmd/cib.c +++ b/crm/crmd/cib.c @@ -1,389 +1,382 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include /* for access */ #include #include #include /* for calls to open */ #include /* for calls to open */ #include /* for calls to open */ #include /* for getpwuid */ #include /* for initgroups */ #include /* for getrlimit */ #include /* for getrlimit */ #include #include #include #include #include #include #include #include struct crm_subsystem_s *cib_subsystem = NULL; int cib_retries = 0; static void revision_check_callback(const HA_Message *msg, int call_id, int rc, crm_data_t *output, void *user_data) { int cmp = -1; const char *revision = NULL; crm_data_t *generation = NULL; #if CRM_DEPRECATED_SINCE_2_0_4 if(safe_str_eq(crm_element_name(output), XML_TAG_CIB)) { generation = output; } else { generation = find_xml_node(output, XML_TAG_CIB, TRUE); } #else generation = output; CRM_DEV_ASSERT(safe_str_eq(crm_element_name(generation), XML_TAG_CIB)); #endif if(rc != cib_ok) { fsa_data_t *msg_data = NULL; register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); return; } crm_debug_3("Checking our feature revision is allowed: %s", CIB_FEATURE_SET); revision = crm_element_value(generation, XML_ATTR_CIB_REVISION); cmp = compare_version(revision, CIB_FEATURE_SET); if(cmp > 0) { crm_err("This build (%s) does not support the current" " resource configuration", VERSION); crm_err("We can support up to CIB feature set %s (current=%s)", CIB_FEATURE_SET, revision); crm_err("Shutting down the CRM"); /* go into a stall state */ register_fsa_error_adv( C_FSA_INTERNAL, I_SHUTDOWN, NULL, NULL, __FUNCTION__); return; } revision = crm_element_value(generation, XML_ATTR_CRM_VERSION); cmp = compare_version(revision, CRM_FEATURE_SET); if(cmp > 0) { crm_err("This build (%s) does not support the current" " resource configuration", VERSION); crm_err("We can support up to CRM feature set %s (current=%s)", revision, CRM_FEATURE_SET); crm_err("Shutting down the CRM"); /* go into a stall state */ register_fsa_error_adv( C_FSA_INTERNAL, I_SHUTDOWN, NULL, NULL, __FUNCTION__); return; } } extern void populate_cib_nodes( ll_cluster_t *hb_cluster, gboolean with_client_status); static void do_cib_replaced(const char *event, HA_Message *msg) { crm_debug("Updating the CIB after a replace"); -#ifdef BUG - /* enabling this seems to cause delays (in the order of 3 minutes) - * in messages being received by the node that runs this - * - * no idea why :( - */ populate_cib_nodes(fsa_cluster_conn, FALSE); -#endif do_update_cib_nodes(AM_I_DC, __FUNCTION__); if(AM_I_DC) { /* start the join process again so we get everyone's LRM status */ register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); } } /* A_CIB_STOP, A_CIB_START, A_CIB_RESTART, */ enum crmd_fsa_input do_cib_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { enum crmd_fsa_input result = I_NULL; struct crm_subsystem_s *this_subsys = cib_subsystem; long long stop_actions = A_CIB_STOP; long long start_actions = A_CIB_START; if(action & stop_actions) { crm_info("Disconnecting CIB"); clear_bit_inplace(fsa_input_register, R_CIB_CONNECTED); if(fsa_cib_conn != NULL && fsa_cib_conn->state != cib_disconnected) { fsa_cib_conn->cmds->set_slave( fsa_cib_conn, cib_scope_local); fsa_cib_conn->cmds->signoff(fsa_cib_conn); } } if(action & start_actions) { int rc = cib_ok; if(fsa_cib_conn == NULL) { fsa_cib_conn = cib_new(); } if(cur_state == S_STOPPING) { crm_err("Ignoring request to start %s after shutdown", this_subsys->name); return I_NULL; } rc = fsa_cib_conn->cmds->signon( fsa_cib_conn, CRM_SYSTEM_CRMD, cib_command); if(rc != cib_ok) { /* a short wait that usually avoids stalling the FSA */ sleep(1); rc = fsa_cib_conn->cmds->signon( fsa_cib_conn, CRM_SYSTEM_CRMD, cib_command); } if(rc != cib_ok){ crm_debug("Could not connect to the CIB service"); } else if(cib_ok != fsa_cib_conn->cmds->set_connection_dnotify( fsa_cib_conn, crmd_cib_connection_destroy)) { crm_err("Could not set dnotify callback"); } else if(cib_ok != fsa_cib_conn->cmds->add_notify_callback( fsa_cib_conn, T_CIB_REPLACE_NOTIFY, do_cib_replaced)) { crm_err("Could not set CIB notification callback"); } else { set_bit_inplace( fsa_input_register, R_CIB_CONNECTED); } if(is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) { cib_retries++; crm_warn("Couldn't complete CIB registration %d" " times... pause and retry", cib_retries); if(cib_retries < 30) { crm_timer_start(wait_timer); crmd_fsa_stall(NULL); } else { crm_err("Could not complete CIB" " registration %d times..." " hard error", cib_retries); register_fsa_error( C_FSA_INTERNAL, I_ERROR, NULL); } } else { int call_id = 0; crm_info("CIB connection established"); call_id = fsa_cib_conn->cmds->query( fsa_cib_conn, NULL, NULL, cib_scope_local); add_cib_op_callback(call_id, FALSE, NULL, revision_check_callback); cib_retries = 0; } } return result; } /* A_CIB_INVOKE, A_CIB_BUMPGEN, A_UPDATE_NODESTATUS */ enum crmd_fsa_input do_cib_invoke(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { HA_Message *answer = NULL; enum crmd_fsa_input result = I_NULL; ha_msg_input_t *cib_msg = fsa_typed_data(fsa_dt_ha_msg); const char *sys_from = cl_get_string(cib_msg->msg, F_CRM_SYS_FROM); if(fsa_cib_conn->state == cib_disconnected) { if(cur_state != S_STOPPING) { crm_err("CIB is disconnected"); crm_log_message_adv(LOG_WARNING, "CIB Input", cib_msg->msg); return I_NULL; } crm_info("CIB is disconnected"); crm_log_message_adv(LOG_DEBUG, "CIB Input", cib_msg->msg); return I_NULL; } if(action & A_CIB_INVOKE) { if(safe_str_eq(sys_from, CRM_SYSTEM_CRMD)) { action = A_CIB_INVOKE_LOCAL; } else if(safe_str_eq(sys_from, CRM_SYSTEM_DC)) { action = A_CIB_INVOKE_LOCAL; } } if(action & A_CIB_INVOKE || action & A_CIB_INVOKE_LOCAL) { int call_options = 0; enum cib_errors rc = cib_ok; crm_data_t *cib_frag = NULL; const char *section = NULL; const char *op = cl_get_string(cib_msg->msg, F_CRM_TASK); section = cl_get_string(cib_msg->msg, F_CIB_SECTION); ha_msg_value_int(cib_msg->msg, F_CIB_CALLOPTS, &call_options); crm_log_message(LOG_MSG, cib_msg->msg); crm_log_xml_debug_3(cib_msg->xml, "[CIB update]"); if(op == NULL) { crm_err("Invalid CIB Message"); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); return I_NULL; } cib_frag = NULL; rc = fsa_cib_conn->cmds->variant_op( fsa_cib_conn, op, NULL, section, cib_msg->xml, &cib_frag, call_options); if(rc < cib_ok || (action & A_CIB_INVOKE)) { answer = create_reply(cib_msg->msg, cib_frag); ha_msg_add(answer,XML_ATTR_RESULT,cib_error2string(rc)); } if(action & A_CIB_INVOKE) { if(relay_message(answer, TRUE) == FALSE) { crm_err("Confused what to do with cib result"); crm_log_message(LOG_ERR, answer); crm_msg_del(answer); result = I_ERROR; } } else if(rc < cib_ok) { ha_msg_input_t *input = NULL; crm_err("Internal CRM/CIB command from %s() failed: %s", msg_data->origin, cib_error2string(rc)); crm_log_message_adv(LOG_WARNING, "CIB Input", cib_msg->msg); crm_log_message_adv(LOG_WARNING, "CIB Reply", answer); input = new_ha_msg_input(answer); register_fsa_input(C_FSA_INTERNAL, I_ERROR, input); crm_msg_del(answer); delete_ha_msg_input(input); } return result; } else { crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__); } return I_NULL; } /* frees fragment as part of delete_ha_msg_input() */ void update_local_cib_adv( crm_data_t *msg_data, gboolean do_now, const char *raised_from) { HA_Message *msg = NULL; ha_msg_input_t *fsa_input = NULL; int call_options = cib_quorum_override|cib_scope_local; CRM_DEV_ASSERT(msg_data != NULL); crm_malloc0(fsa_input, sizeof(ha_msg_input_t)); msg = create_request(CIB_OP_UPDATE, msg_data, NULL, CRM_SYSTEM_CIB, CRM_SYSTEM_CRMD, NULL); ha_msg_add(msg, F_CIB_SECTION, crm_element_value(msg_data, XML_ATTR_SECTION)); ha_msg_add_int(msg, F_CIB_CALLOPTS, call_options); ha_msg_add(msg, "call_origin", raised_from); fsa_input->msg = msg; fsa_input->xml = msg_data; if(AM_I_DC && crm_assert_failed) { /* register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); */ } if(do_now == FALSE) { crm_debug_3("Registering event with FSA"); register_fsa_input_adv(C_FSA_INTERNAL, I_CIB_OP, fsa_input, 0, FALSE, raised_from); } else { fsa_data_t *op_data = NULL; crm_debug_3("Invoking CIB handler directly"); crm_malloc0(op_data, sizeof(fsa_data_t)); op_data->fsa_cause = C_FSA_INTERNAL; op_data->fsa_input = I_CIB_OP; op_data->origin = raised_from; op_data->data = fsa_input; op_data->data_type = fsa_dt_ha_msg; do_cib_invoke(A_CIB_INVOKE_LOCAL, C_FSA_INTERNAL, fsa_state, I_CIB_OP, op_data); crm_free(op_data); crm_debug_3("CIB handler completed"); } crm_debug_3("deleting input"); crm_msg_del(fsa_input->msg); free_xml(fsa_input->xml); crm_free(fsa_input); crm_debug_3("deleted input"); } diff --git a/include/crm/common/util.h b/include/crm/common/util.h index 8d57596a33..8814e5df7c 100644 --- a/include/crm/common/util.h +++ b/include/crm/common/util.h @@ -1,147 +1,152 @@ -/* $Id: util.h,v 1.37 2006/07/06 09:30:28 andrew Exp $ */ +/* $Id: util.h,v 1.38 2006/07/06 13:30:23 andrew Exp $ */ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef CRM_COMMON_UTIL__H #define CRM_COMMON_UTIL__H #include #include #include #include #include #define DEBUG_INC SIGUSR1 #define DEBUG_DEC SIGUSR2 extern unsigned int crm_log_level; extern gboolean crm_log_init(const char *entity); extern void do_crm_log(int log_level, const char *file, const char *function, const char *format, ...) G_GNUC_PRINTF(4,5); /* returns the old value */ extern unsigned int set_crm_log_level(unsigned int level); extern unsigned int get_crm_log_level(void); extern char *crm_itoa(int an_int); extern char *crm_strdup(const char *a); extern char *generate_hash_key(const char *crm_msg_reference, const char *sys); extern char *generate_hash_value(const char *src_node, const char *src_subsys); extern gboolean decode_hash_value(gpointer value, char **node, char **subsys); extern gboolean decodeNVpair(const char *srcstring, char separator, char **name, char **value); extern int compare_version(const char *version1, const char *version2); extern char *generateReference(const char *custom1, const char *custom2); extern void alter_debug(int nsig); extern void g_hash_destroy_str(gpointer data); extern const char *get_uuid(ll_cluster_t *hb, const char *uname); extern const char *get_uname(ll_cluster_t *hb, const char *uuid); extern void unget_uuid(const char *uname); extern void set_uuid( ll_cluster_t* hb, crm_data_t *node, const char *attr, const char *uname); extern gboolean crm_is_true(const char * s); extern int crm_str_to_boolean(const char * s, int * ret); extern long crm_get_msec(const char * input); extern gboolean ccm_have_quorum(oc_ed_t event); extern const char *ccm_event_name(oc_ed_t event); extern const char *op_status2text(op_status_t status); extern char *generate_op_key( const char *rsc_id, const char *op_type, int interval); extern gboolean parse_op_key( const char *key, char **rsc_id, char **op_type, int *interval); extern char *generate_notify_key( const char *rsc_id, const char *notify_type, const char *op_type); extern gboolean crm_mem_stats(volatile cl_mem_stats_t *mem_stats); extern void crm_zero_mem_stats(volatile cl_mem_stats_t *stats); extern void crm_save_mem_stats(const char *location, cl_mem_stats_t *saved_stats); -extern gboolean crm_diff_mem_stats(int log_level, const char *location, cl_mem_stats_t *saved_stats); +extern gboolean crm_diff_mem_stats( + int log_level_up, int log_level_down, const char *location, + volatile cl_mem_stats_t *stats, volatile cl_mem_stats_t *saved_stats); extern void crm_xml_nbytes(crm_data_t *xml, long *bytes, long *allocs, long *frees); -extern void crm_adjust_mem_stats(long bytes, long allocs, long frees); +extern void crm_adjust_mem_stats( + volatile cl_mem_stats_t *stats, long bytes, long allocs, long frees); extern char *generate_transition_magic_v202( const char *transition_key, int op_status); extern char *generate_transition_magic( const char *transition_key, int op_status, int op_rc); extern gboolean decode_transition_magic( const char *magic, char **uuid, int *transition_id, int *op_status, int *op_rc); extern char *generate_transition_key(int transition_id, const char *node); extern gboolean decode_transition_key( const char *key, char **uuid, int *transition_id); extern char *crm_concat(const char *prefix, const char *suffix, char join); extern gboolean decode_op_key( const char *key, char **rsc_id, char **op_type, int *interval); extern void filter_action_parameters(crm_data_t *param_set, const char *version); extern gboolean safe_str_eq(const char *a, const char *b); extern gboolean safe_str_neq(const char *a, const char *b); extern int crm_parse_int(const char *text, const char *default_text); extern int crm_int_helper(const char *text, char **end_text); #define crm_atoi(text, default_text) crm_parse_int(text, default_text) extern void crm_abort(const char *file, const char *function, int line, const char *condition, gboolean do_fork); extern char *generate_series_filename( const char *directory, const char *series, int sequence, gboolean bzip); extern int get_last_sequence(const char *directory, const char *series); extern void write_last_sequence( const char *directory, const char *series, int sequence, int max); extern void crm_make_daemon( const char *name, gboolean daemonize, const char *pidfile); +extern cl_mem_stats_t *crm_running_stats; + #endif diff --git a/lib/crm/common/ipc.c b/lib/crm/common/ipc.c index a70aa5222c..35861993a1 100644 --- a/lib/crm/common/ipc.c +++ b/lib/crm/common/ipc.c @@ -1,432 +1,432 @@ -/* $Id: ipc.c,v 1.24 2006/07/06 09:30:27 andrew Exp $ */ +/* $Id: ipc.c,v 1.25 2006/07/06 13:30:23 andrew Exp $ */ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include gboolean send_ha_message(ll_cluster_t *hb_conn, HA_Message *msg, const char *node, gboolean force_ordered) { gboolean all_is_good = TRUE; cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); if (msg == NULL) { crm_err("cant send NULL message"); all_is_good = FALSE; } else if(hb_conn == NULL) { crm_err("No heartbeat connection specified"); all_is_good = FALSE; } else if(hb_conn->llc_ops->chan_is_connected(hb_conn) == FALSE) { crm_err("Not connected to Heartbeat"); all_is_good = FALSE; } else if(node != NULL) { if(hb_conn->llc_ops->send_ordered_nodemsg( hb_conn, msg, node) != HA_OK) { all_is_good = FALSE; crm_err("Send failed"); } else { crm_debug_2("Message sent..."); } } else if(force_ordered) { if(hb_conn->llc_ops->send_ordered_clustermsg(hb_conn, msg) != HA_OK) { all_is_good = FALSE; crm_err("Broadcast Send failed"); } else { crm_debug_2("Broadcast message sent..."); } } else { if(hb_conn->llc_ops->sendclustermsg(hb_conn, msg) != HA_OK) { all_is_good = FALSE; crm_err("Broadcast Send failed"); } else { crm_debug_2("Broadcast message sent..."); } } if(all_is_good == FALSE && hb_conn != NULL) { IPC_Channel *ipc = NULL; IPC_Queue *send_q = NULL; if(hb_conn->llc_ops->chan_is_connected(hb_conn) != HA_OK) { ipc = hb_conn->llc_ops->ipcchan(hb_conn); } if(ipc != NULL) { ipc->ops->resume_io(ipc); send_q = ipc->send_queue; } if(send_q != NULL) { CRM_CHECK(send_q->current_qlen < send_q->max_qlen, ;); } } crm_log_message_adv(all_is_good?LOG_MSG:LOG_WARNING,"HA[outbound]",msg); - crm_diff_mem_stats(LOG_DEBUG, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_DEBUG, LOG_DEBUG, __PRETTY_FUNCTION__, NULL, &saved_stats); return all_is_good; } /* frees msg */ gboolean send_ipc_message(IPC_Channel *ipc_client, HA_Message *msg) { gboolean all_is_good = TRUE; int fail_level = LOG_WARNING; cl_mem_stats_t saved_stats; crm_save_mem_stats(__PRETTY_FUNCTION__, &saved_stats); if(ipc_client != NULL && ipc_client->conntype == IPC_CLIENT) { fail_level = LOG_ERR; } if (msg == NULL) { crm_err("cant send NULL message"); all_is_good = FALSE; } else if (ipc_client == NULL) { crm_err("cant send message without an IPC Channel"); all_is_good = FALSE; } else if(ipc_client->ops->get_chan_status(ipc_client) != IPC_CONNECT) { crm_log_maybe(fail_level, "IPC Channel to %d is not connected", (int)ipc_client->farside_pid); all_is_good = FALSE; } if(all_is_good && msg2ipcchan(msg, ipc_client) != HA_OK) { crm_log_maybe(fail_level, "Could not send IPC message to %d", (int)ipc_client->farside_pid); all_is_good = FALSE; if(ipc_client->ops->get_chan_status(ipc_client) != IPC_CONNECT) { crm_log_maybe(fail_level, "IPC Channel to %d is no longer connected", (int)ipc_client->farside_pid); } else if(ipc_client->conntype == IPC_CLIENT) { CRM_CHECK(ipc_client->send_queue->current_qlen < ipc_client->send_queue->max_qlen, ;); } } ipc_client->ops->resume_io(ipc_client); crm_log_message_adv(all_is_good?LOG_MSG:LOG_WARNING,"IPC[outbound]",msg); - crm_diff_mem_stats(LOG_DEBUG, __PRETTY_FUNCTION__, &saved_stats); + crm_diff_mem_stats(LOG_DEBUG, LOG_DEBUG, __PRETTY_FUNCTION__, NULL, &saved_stats); return all_is_good; } void default_ipc_connection_destroy(gpointer user_data) { return; } int init_server_ipc_comms( char *channel_name, gboolean (*channel_client_connect)(IPC_Channel *newclient,gpointer user_data), void (*channel_connection_destroy)(gpointer user_data)) { /* the clients wait channel is the other source of events. * This source delivers the clients connection events. * listen to this source at a relatively lower priority. */ char commpath[SOCKET_LEN]; IPC_WaitConnection *wait_ch; sprintf(commpath, CRM_SOCK_DIR "/%s", channel_name); wait_ch = wait_channel_init(commpath); if (wait_ch == NULL) { return 1; } G_main_add_IPC_WaitConnection( G_PRIORITY_LOW, wait_ch, NULL, FALSE, channel_client_connect, channel_name, channel_connection_destroy); crm_debug_3("Listening on: %s", commpath); return 0; } GCHSource* init_client_ipc_comms(const char *channel_name, gboolean (*dispatch)( IPC_Channel* source_data, gpointer user_data), void *client_data, IPC_Channel **ch) { IPC_Channel *a_ch = NULL; GCHSource *the_source = NULL; void *callback_data = client_data; a_ch = init_client_ipc_comms_nodispatch(channel_name); if(ch != NULL) { *ch = a_ch; if(callback_data == NULL) { callback_data = a_ch; } } if(a_ch == NULL) { crm_warn("Setup of client connection failed," " not adding channel to mainloop"); return NULL; } if(dispatch == NULL) { crm_warn("No dispatch method specified..." "maybe you meant init_client_ipc_comms_nodispatch()?"); } else { crm_debug_3("Adding dispatch method to channel"); the_source = G_main_add_IPC_Channel( G_PRIORITY_HIGH, a_ch, FALSE, dispatch, callback_data, default_ipc_connection_destroy); } return the_source; } IPC_Channel * init_client_ipc_comms_nodispatch(const char *channel_name) { IPC_Channel *ch; GHashTable *attrs; static char path[] = IPC_PATH_ATTR; char *commpath = NULL; int local_socket_len = 2; /* 2 = '/' + '\0' */ local_socket_len += strlen(channel_name); local_socket_len += strlen(CRM_SOCK_DIR); crm_malloc0(commpath, local_socket_len); if(commpath != NULL) { sprintf(commpath, CRM_SOCK_DIR "/%s", channel_name); commpath[local_socket_len - 1] = '\0'; crm_debug_3("Attempting to talk on: %s", commpath); } attrs = g_hash_table_new(g_str_hash,g_str_equal); g_hash_table_insert(attrs, path, commpath); ch = ipc_channel_constructor(IPC_ANYTYPE, attrs); g_hash_table_destroy(attrs); if (ch == NULL) { crm_err("Could not access channel on: %s", commpath); return NULL; } else if (ch->ops->initiate_connection(ch) != IPC_OK) { crm_debug("Could not init comms on: %s", commpath); ch->ops->destroy(ch); return NULL; } ch->ops->set_recv_qlen(ch, 100); ch->ops->set_send_qlen(ch, 100); /* ch->should_send_block = TRUE; */ crm_debug_3("Processing of %s complete", commpath); return ch; } IPC_WaitConnection * wait_channel_init(char daemonsocket[]) { IPC_WaitConnection *wait_ch; mode_t mask; char path[] = IPC_PATH_ATTR; GHashTable * attrs; attrs = g_hash_table_new(g_str_hash,g_str_equal); g_hash_table_insert(attrs, path, daemonsocket); mask = umask(0); wait_ch = ipc_wait_conn_constructor(IPC_ANYTYPE, attrs); if (wait_ch == NULL) { cl_perror("Can't create wait channel of type %s", IPC_ANYTYPE); exit(1); } mask = umask(mask); g_hash_table_destroy(attrs); return wait_ch; } longclock_t ipc_call_start = 0; longclock_t ipc_call_stop = 0; longclock_t ipc_call_diff = 0; gboolean subsystem_msg_dispatch(IPC_Channel *sender, void *user_data) { int lpc = 0; HA_Message *msg = NULL; ha_msg_input_t *new_input = NULL; gboolean all_is_well = TRUE; const char *sys_to; const char *task; while(IPC_ISRCONN(sender)) { gboolean process = FALSE; if(sender->ops->is_message_pending(sender) == 0) { break; } msg = msgfromIPC_noauth(sender); if (msg == NULL) { crm_err("No message from %d this time", sender->farside_pid); continue; } lpc++; new_input = new_ha_msg_input(msg); crm_msg_del(msg); msg = NULL; crm_log_message(LOG_MSG, new_input->msg); sys_to = cl_get_string(new_input->msg, F_CRM_SYS_TO); task = cl_get_string(new_input->msg, F_CRM_TASK); if(safe_str_eq(task, CRM_OP_HELLO)) { process = TRUE; } else if(sys_to == NULL) { crm_err("Value of %s was NULL!!", F_CRM_SYS_TO); } else if(task == NULL) { crm_err("Value of %s was NULL!!", F_CRM_TASK); } else { process = TRUE; } if(process){ gboolean (*process_function) (HA_Message *msg, crm_data_t *data, IPC_Channel *sender) = NULL; process_function = user_data; #ifdef MSG_LOG crm_log_message_adv( LOG_MSG, __FUNCTION__, new_input->msg); #endif if(ipc_call_diff_max_ms > 0) { ipc_call_start = time_longclock(); } if(FALSE == process_function( new_input->msg, new_input->xml, sender)) { crm_warn("Received a message destined for %s" " by mistake", sys_to); } if(ipc_call_diff_max_ms > 0) { unsigned int ipc_call_diff_ms = 0; ipc_call_stop = time_longclock(); ipc_call_diff = sub_longclock( ipc_call_stop, ipc_call_start); ipc_call_diff_ms = longclockto_ms( ipc_call_diff); if(ipc_call_diff_ms > ipc_call_diff_max_ms) { crm_err("%s took %dms to complete", sys_to, ipc_call_diff_ms); } } } else { #ifdef MSG_LOG crm_log_message_adv( LOG_ERR, NULL, new_input->msg); #endif } delete_ha_msg_input(new_input); new_input = NULL; if(sender->ch_status == IPC_CONNECT) { break; } } crm_debug_2("Processed %d messages", lpc); if (sender->ch_status != IPC_CONNECT) { crm_err("The server %d has left us: Shutting down...NOW", sender->farside_pid); exit(1); /* shutdown properly later */ return !all_is_well; } return all_is_well; } gboolean is_ipc_empty(IPC_Channel *ch) { if(ch == NULL) { return TRUE; } else if(ch->send_queue->current_qlen == 0 && ch->recv_queue->current_qlen == 0) { return TRUE; } return FALSE; } diff --git a/lib/crm/common/utils.c b/lib/crm/common/utils.c index 0cef32e0fc..ded1c8326c 100644 --- a/lib/crm/common/utils.c +++ b/lib/crm/common/utils.c @@ -1,1528 +1,1530 @@ -/* $Id: utils.c,v 1.60 2006/07/06 10:55:09 andrew Exp $ */ +/* $Id: utils.c,v 1.61 2006/07/06 13:30:24 andrew Exp $ */ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #ifndef _GNU_SOURCE # define _GNU_SOURCE #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef MAXLINE # define MAXLINE 512 #endif static uint ref_counter = 0; gboolean crm_assert_failed = FALSE; unsigned int crm_log_level = LOG_INFO; void crm_set_env_options(void); char * generateReference(const char *custom1, const char *custom2) { const char *local_cust1 = custom1; const char *local_cust2 = custom2; int reference_len = 4; char *since_epoch = NULL; reference_len += 20; /* too big */ reference_len += 40; /* too big */ if(local_cust1 == NULL) { local_cust1 = "_empty_"; } reference_len += strlen(local_cust1); if(local_cust2 == NULL) { local_cust2 = "_empty_"; } reference_len += strlen(local_cust2); crm_malloc0(since_epoch, reference_len); if(since_epoch != NULL) { sprintf(since_epoch, "%s-%s-%ld-%u", local_cust1, local_cust2, (unsigned long)time(NULL), ref_counter++); } return since_epoch; } gboolean decodeNVpair(const char *srcstring, char separator, char **name, char **value) { int lpc = 0; int len = 0; const char *temp = NULL; CRM_ASSERT(name != NULL && value != NULL); *name = NULL; *value = NULL; crm_debug_4("Attempting to decode: [%s]", srcstring); if (srcstring != NULL) { len = strlen(srcstring); while(lpc <= len) { if (srcstring[lpc] == separator) { crm_malloc0(*name, lpc+1); if(*name == NULL) { break; /* and return FALSE */ } strncpy(*name, srcstring, lpc); (*name)[lpc] = '\0'; /* this sucks but as the strtok manpage says.. * it *is* a bug */ len = len-lpc; len--; if(len <= 0) { *value = NULL; } else { crm_malloc0(*value, len+1); if(*value == NULL) { crm_free(*name); break; /* and return FALSE */ } temp = srcstring+lpc+1; strncpy(*value, temp, len); (*value)[len] = '\0'; } return TRUE; } lpc++; } } if(*name != NULL) { crm_free(*name); } *name = NULL; *value = NULL; return FALSE; } char * crm_concat(const char *prefix, const char *suffix, char join) { int len = 0; char *new_str = NULL; CRM_ASSERT(prefix != NULL); CRM_ASSERT(suffix != NULL); len = strlen(prefix) + strlen(suffix) + 2; crm_malloc0(new_str, (len)); sprintf(new_str, "%s%c%s", prefix, join, suffix); new_str[len-1] = 0; return new_str; } char * generate_hash_key(const char *crm_msg_reference, const char *sys) { char *hash_key = crm_concat(sys?sys:"none", crm_msg_reference, '_'); crm_debug_3("created hash key: (%s)", hash_key); return hash_key; } char * generate_hash_value(const char *src_node, const char *src_subsys) { char *hash_value = NULL; if (src_node == NULL || src_subsys == NULL) { return NULL; } if (strcasecmp(CRM_SYSTEM_DC, src_subsys) == 0) { hash_value = crm_strdup(src_subsys); if (!hash_value) { crm_err("memory allocation failed in " "generate_hash_value()"); } return hash_value; } hash_value = crm_concat(src_node, src_subsys, '_'); crm_info("created hash value: (%s)", hash_value); return hash_value; } gboolean decode_hash_value(gpointer value, char **node, char **subsys) { char *char_value = (char*)value; int value_len = strlen(char_value); CRM_CHECK(value != NULL, return FALSE); CRM_CHECK(node != NULL, return FALSE); CRM_CHECK(subsys != NULL, return FALSE); *node = NULL; *subsys = NULL; crm_info("Decoding hash value: (%s:%d)", char_value, value_len); if (strcasecmp(CRM_SYSTEM_DC, (char*)value) == 0) { *node = NULL; *subsys = (char*)crm_strdup(char_value); CRM_CHECK(*subsys != NULL, return FALSE); crm_info("Decoded value: (%s:%d)", *subsys, (int)strlen(*subsys)); return TRUE; } else if (decodeNVpair(char_value, '_', node, subsys)) { return TRUE; } crm_free(*node); crm_free(*subsys); *node = NULL; *subsys = NULL; return FALSE; } char * crm_itoa(int an_int) { int len = 32; char *buffer = NULL; crm_malloc0(buffer, (len+1)); if(buffer != NULL) { snprintf(buffer, len, "%d", an_int); } return buffer; } extern int LogToLoggingDaemon(int priority, const char * buf, int bstrlen, gboolean use_pri_str); gboolean crm_log_init(const char *entity) { /* const char *test = "Testing log daemon connection"; */ /* Redirect messages from glib functions to our handler */ /* cl_malloc_forced_for_glib(); */ g_log_set_handler(NULL, G_LOG_LEVEL_ERROR | G_LOG_LEVEL_CRITICAL | G_LOG_LEVEL_WARNING | G_LOG_LEVEL_MESSAGE | G_LOG_LEVEL_INFO | G_LOG_LEVEL_DEBUG | G_LOG_FLAG_RECURSION | G_LOG_FLAG_FATAL, cl_glib_msg_handler, NULL); /* and for good measure... - this enum is a bit field (!) */ g_log_set_always_fatal((GLogLevelFlags)0); /*value out of range*/ cl_log_set_entity(entity); cl_log_set_facility(LOG_LOCAL7); cl_set_corerootdir(HA_COREDIR); cl_cdtocoredir(); crm_set_env_options(); CL_SIGNAL(DEBUG_INC, alter_debug); CL_SIGNAL(DEBUG_DEC, alter_debug); return TRUE; } /* returns the old value */ unsigned int set_crm_log_level(unsigned int level) { unsigned int old = crm_log_level; while(crm_log_level < 100 && crm_log_level < level) { alter_debug(DEBUG_INC); } while(crm_log_level > 0 && crm_log_level > level) { alter_debug(DEBUG_DEC); } return old; } unsigned int get_crm_log_level(void) { return crm_log_level; } void crm_log_message_adv(int level, const char *prefix, const HA_Message *msg) { if((int)crm_log_level >= level) { do_crm_log(level, NULL, NULL, "#========= %s message start ==========#", prefix?prefix:""); if(level > LOG_DEBUG) { cl_log_message(LOG_DEBUG, msg); } else { cl_log_message(level, msg); } } } void do_crm_log(int log_level, const char *file, const char *function, const char *fmt, ...) { int log_as = log_level; gboolean do_log = FALSE; if(log_level <= (int)crm_log_level) { do_log = TRUE; if(log_level > LOG_INFO) { log_as = LOG_DEBUG; } } if(do_log) { va_list ap; int nbytes; char buf[MAXLINE]; va_start(ap, fmt); nbytes=vsnprintf(buf, MAXLINE, fmt, ap); va_end(ap); log_level -= LOG_INFO; if(log_level > 1) { if(function == NULL) { cl_log(log_as, "debug%d: %s", log_level, buf); } else { cl_log(log_as, "debug%d: %s:%s %s", log_level, function, file?file:"", buf); } } else { if(function == NULL) { cl_log(log_as, "%s", buf); } else { cl_log(log_as, "%s:%s %s", function, file?file:"", buf); } } if(nbytes > MAXLINE) { cl_log(LOG_WARNING, "Log from %s() was truncated", crm_str(function)); } } } int compare_version(const char *version1, const char *version2) { int lpc = 0; char *step1 = NULL, *step2 = NULL; char *rest1 = NULL, *rest2 = NULL; if(version1 == NULL && version2 == NULL) { return 0; } else if(version1 == NULL) { return -1; } else if(version2 == NULL) { return 1; } rest1 = crm_strdup(version1); rest2 = crm_strdup(version2); while(1) { int cmp = 0; int step1_i = 0; int step2_i = 0; char *tmp1 = NULL, *tmp2 = NULL; decodeNVpair(rest1, '.', &step1, &tmp1); decodeNVpair(rest2, '.', &step2, &tmp2); if(step1 != NULL) { step1_i = crm_parse_int(step1, NULL); } if(step2 != NULL) { step2_i = crm_parse_int(step2, NULL); } if(step1_i < step2_i){ cmp = -1; } else if (step1_i > step2_i){ cmp = 1; } crm_debug_4("compare[%d (%d)]: %d(%s) %d(%s)", lpc++, cmp, step1_i, crm_str(step1), step2_i, crm_str(step2)); crm_free(rest1); crm_free(rest2); rest1 = tmp1; rest2 = tmp2; if(step1 == NULL && step2 == NULL) { break; } crm_free(step1); crm_free(step2); if(cmp < 0) { crm_debug_3("%s < %s", version1, version2); return -1; } else if(cmp > 0) { crm_debug_3("%s > %s", version1, version2); return 1; } } crm_debug_3("%s == %s", version1, version2); return 0; } gboolean do_stderr = FALSE; void alter_debug(int nsig) { CL_SIGNAL(DEBUG_INC, alter_debug); CL_SIGNAL(DEBUG_DEC, alter_debug); switch(nsig) { case DEBUG_INC: if (crm_log_level < 100) { crm_log_level++; } break; case DEBUG_DEC: if (crm_log_level > 0) { crm_log_level--; } break; default: fprintf(stderr, "Unknown signal %d\n", nsig); cl_log(LOG_ERR, "Unknown signal %d", nsig); break; } } void g_hash_destroy_str(gpointer data) { crm_free(data); } int crm_int_helper(const char *text, char **end_text) { int atoi_result = -1; char *local_end_text = NULL; errno = 0; if(text != NULL) { if(end_text != NULL) { atoi_result = (int)strtol(text, end_text, 10); } else { atoi_result = (int)strtol(text, &local_end_text, 10); } /* CRM_CHECK(errno != EINVAL); */ if(errno == EINVAL) { crm_err("Conversion of %s failed", text); atoi_result = -1; } else { if(errno == ERANGE) { crm_err("Conversion of %s was clipped", text); } if(end_text == NULL && local_end_text[0] != '\0') { crm_err("Characters left over after parsing " "\"%s\": \"%s\"", text, local_end_text); } } } return atoi_result; } int crm_parse_int(const char *text, const char *default_text) { int atoi_result = -1; if(text != NULL) { atoi_result = crm_int_helper(text, NULL); if(errno == 0) { return atoi_result; } } if(default_text != NULL) { atoi_result = crm_int_helper(default_text, NULL); if(errno == 0) { return atoi_result; } } else { crm_err("No default conversion value supplied"); } return -1; } gboolean safe_str_eq(const char *a, const char *b) { if(a == b) { return TRUE; } else if(a == NULL || b == NULL) { return FALSE; } else if(strcasecmp(a, b) == 0) { return TRUE; } return FALSE; } gboolean safe_str_neq(const char *a, const char *b) { if(a == b) { return FALSE; } else if(a==NULL || b==NULL) { return TRUE; } else if(strcasecmp(a, b) == 0) { return FALSE; } return TRUE; } char * crm_strdup(const char *a) { char *ret = NULL; CRM_CHECK(a != NULL, return NULL); if(a != NULL) { ret = cl_strdup(a); } else { crm_warn("Cannot dup NULL string"); } return ret; } static GHashTable *crm_uuid_cache = NULL; static GHashTable *crm_uname_cache = NULL; void unget_uuid(const char *uname) { if(crm_uuid_cache == NULL) { return; } g_hash_table_remove(crm_uuid_cache, uname); } const char * get_uuid(ll_cluster_t *hb, const char *uname) { cl_uuid_t uuid_raw; char *uuid_calc = NULL; const char *unknown = "00000000-0000-0000-0000-000000000000"; if(crm_uuid_cache == NULL) { crm_uuid_cache = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); } CRM_CHECK(uname != NULL, return NULL); /* avoid blocking calls where possible */ uuid_calc = g_hash_table_lookup(crm_uuid_cache, uname); if(uuid_calc != NULL) { return uuid_calc; } if(hb->llc_ops->get_uuid_by_name(hb, uname, &uuid_raw) == HA_FAIL) { crm_err("get_uuid_by_name() call failed for host %s", uname); crm_free(uuid_calc); return NULL; } crm_malloc0(uuid_calc, 50); if(uuid_calc == NULL) { return NULL; } cl_uuid_unparse(&uuid_raw, uuid_calc); if(safe_str_eq(uuid_calc, unknown)) { crm_warn("Could not calculate UUID for %s", uname); crm_free(uuid_calc); return NULL; } g_hash_table_insert(crm_uuid_cache, crm_strdup(uname), uuid_calc); uuid_calc = g_hash_table_lookup(crm_uuid_cache, uname); return uuid_calc; } const char * get_uname(ll_cluster_t *hb, const char *uuid) { char *uname = NULL; if(crm_uuid_cache == NULL) { crm_uname_cache = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); } CRM_CHECK(uuid != NULL, return NULL); /* avoid blocking calls where possible */ uname = g_hash_table_lookup(crm_uname_cache, uuid); if(uname != NULL) { return uname; } if(uuid != NULL) { cl_uuid_t uuid_raw; char *uuid_copy = crm_strdup(uuid); cl_uuid_parse(uuid_copy, &uuid_raw); if(hb->llc_ops->get_name_by_uuid( hb, &uuid_raw, uname, 256) == HA_FAIL) { crm_err("Could not calculate UUID for %s", uname); uname = NULL; crm_free(uuid_copy); } else { g_hash_table_insert( crm_uuid_cache, uuid_copy, crm_strdup(uname)); uname = g_hash_table_lookup(crm_uname_cache, uuid); } return uname; } return NULL; } void set_uuid(ll_cluster_t *hb,crm_data_t *node,const char *attr,const char *uname) { const char *uuid_calc = get_uuid(hb, uname); crm_xml_add(node, attr, uuid_calc); return; } #define ENV_PREFIX "HA_" void crm_set_env_options(void) { char *param_val = NULL; const char *param_name = NULL; /* apparently we're not allowed to free the result of getenv */ param_name = ENV_PREFIX "" KEY_DEBUGLEVEL; param_val = getenv(param_name); if(param_val != NULL) { int debug_level = crm_parse_int(param_val, NULL); if(debug_level > 0 && (debug_level+LOG_INFO) > (int)crm_log_level) { set_crm_log_level(LOG_INFO + debug_level); } crm_debug("%s = %s", param_name, param_val); param_val = NULL; } param_name = ENV_PREFIX "" KEY_FACILITY; param_val = getenv(param_name); crm_debug("%s = %s", param_name, param_val); if(param_val != NULL) { int facility = cl_syslogfac_str2int(param_val); if(facility > 0) { cl_log_set_facility(facility); } param_val = NULL; } param_name = ENV_PREFIX "" KEY_LOGFILE; param_val = getenv(param_name); crm_debug("%s = %s", param_name, param_val); if(param_val != NULL) { if(safe_str_eq("/dev/null", param_val)) { param_val = NULL; } cl_log_set_logfile(param_val); param_val = NULL; } param_name = ENV_PREFIX "" KEY_DBGFILE; param_val = getenv(param_name); crm_debug("%s = %s", param_name, param_val); if(param_val != NULL) { if(safe_str_eq("/dev/null", param_val)) { param_val = NULL; } cl_log_set_debugfile(param_val); param_val = NULL; } param_name = ENV_PREFIX "" KEY_LOGDAEMON; param_val = getenv(param_name); crm_debug("%s = %s", param_name, param_val); if(param_val != NULL) { int uselogd; cl_str_to_boolean(param_val, &uselogd); cl_log_set_uselogd(uselogd); if(uselogd) { cl_set_logging_wqueue_maxlen(500); cl_log_set_logd_channel_source(NULL, NULL); } param_val = NULL; } param_name = ENV_PREFIX "" KEY_CONNINTVAL; param_val = getenv(param_name); crm_debug("%s = %s", param_name, param_val); if(param_val != NULL) { int logdtime; logdtime = crm_get_msec(param_val); cl_log_set_logdtime(logdtime); param_val = NULL; } inherit_compress(); } gboolean crm_is_true(const char * s) { gboolean ret = FALSE; if(s != NULL) { cl_str_to_boolean(s, &ret); } return ret; } int crm_str_to_boolean(const char * s, int * ret) { if(s == NULL) { return -1; } else if (strcasecmp(s, "true") == 0 || strcasecmp(s, "on") == 0 || strcasecmp(s, "yes") == 0 || strcasecmp(s, "y") == 0 || strcasecmp(s, "1") == 0){ *ret = TRUE; return 1; } else if (strcasecmp(s, "false") == 0 || strcasecmp(s, "off") == 0 || strcasecmp(s, "no") == 0 || strcasecmp(s, "n") == 0 || strcasecmp(s, "0") == 0){ *ret = FALSE; return 1; } return -1; } #ifndef NUMCHARS # define NUMCHARS "0123456789." #endif #ifndef WHITESPACE # define WHITESPACE " \t\n\r\f" #endif long crm_get_msec(const char * input) { const char * cp = input; const char * units; long multiplier = 1000; long divisor = 1; long ret = -1; double dret; if(input == NULL) { return 0; } cp += strspn(cp, WHITESPACE); units = cp + strspn(cp, NUMCHARS); units += strspn(units, WHITESPACE); if (strchr(NUMCHARS, *cp) == NULL) { return ret; } if (strncasecmp(units, "ms", 2) == 0 || strncasecmp(units, "msec", 4) == 0) { multiplier = 1; divisor = 1; }else if (strncasecmp(units, "us", 2) == 0 || strncasecmp(units, "usec", 4) == 0) { multiplier = 1; divisor = 1000; }else if (strncasecmp(units, "s", 1) == 0 || strncasecmp(units, "sec", 3) == 0) { multiplier = 1000; divisor = 1; }else if (strncasecmp(units, "m", 1) == 0 || strncasecmp(units, "min", 3) == 0) { multiplier = 60*1000; divisor = 1; }else if (*units != EOS && *units != '\n' && *units != '\r') { return ret; } dret = atof(cp); dret *= (double)multiplier; dret /= (double)divisor; dret += 0.5; ret = (long)dret; return(ret); } gboolean ccm_have_quorum(oc_ed_t event) { if(event==OC_EV_MS_NEW_MEMBERSHIP) { return TRUE; } return FALSE; } const char * ccm_event_name(oc_ed_t event) { if(event==OC_EV_MS_NEW_MEMBERSHIP) { return "NEW MEMBERSHIP"; } else if(event==OC_EV_MS_NOT_PRIMARY) { return "NOT PRIMARY"; } else if(event==OC_EV_MS_PRIMARY_RESTORED) { return "PRIMARY RESTORED"; } else if(event==OC_EV_MS_EVICTED) { return "EVICTED"; } else if(event==OC_EV_MS_INVALID) { return "INVALID"; } return "NO QUORUM MEMBERSHIP"; } const char * op_status2text(op_status_t status) { switch(status) { case LRM_OP_PENDING: return "pending"; break; case LRM_OP_DONE: return "complete"; break; case LRM_OP_ERROR: return "Error"; break; case LRM_OP_TIMEOUT: return "Timed Out"; break; case LRM_OP_NOTSUPPORTED: return "NOT SUPPORTED"; break; case LRM_OP_CANCELLED: return "Cancelled"; break; } CRM_CHECK(status >= LRM_OP_PENDING && status <= LRM_OP_CANCELLED, crm_err("Unknown status: %d", status)); return "UNKNOWN!"; } char * generate_op_key(const char *rsc_id, const char *op_type, int interval) { int len = 35; char *op_id = NULL; CRM_CHECK(rsc_id != NULL, return NULL); CRM_CHECK(op_type != NULL, return NULL); len += strlen(op_type); len += strlen(rsc_id); crm_malloc0(op_id, len); CRM_CHECK(op_id != NULL, return NULL); sprintf(op_id, "%s_%s_%d", rsc_id, op_type, interval); return op_id; } gboolean parse_op_key(const char *key, char **rsc_id, char **op_type, int *interval) { char *interval_s = NULL; char *key2 = NULL; if(decodeNVpair(key, '_', rsc_id, &key2) == FALSE) { crm_err("Couldn't find '_' in: %s", key); return FALSE; } if(decodeNVpair(key2, '_', op_type, &interval_s) == FALSE) { crm_err("Couldn't find '_' in: %s", key2); return FALSE; } *interval = crm_parse_int(interval_s, NULL); return TRUE; } char * generate_notify_key(const char *rsc_id, const char *notify_type, const char *op_type) { int len = 12; char *op_id = NULL; CRM_CHECK(rsc_id != NULL, return NULL); CRM_CHECK(op_type != NULL, return NULL); CRM_CHECK(notify_type != NULL, return NULL); len += strlen(op_type); len += strlen(rsc_id); len += strlen(notify_type); crm_malloc0(op_id, len); if(op_id != NULL) { sprintf(op_id, "%s_%s_notify_%s_0", rsc_id, notify_type, op_type); } return op_id; } char * generate_transition_magic_v202(const char *transition_key, int op_status) { int len = 80; char *fail_state = NULL; CRM_CHECK(transition_key != NULL, return NULL); len += strlen(transition_key); crm_malloc0(fail_state, len); if(fail_state != NULL) { snprintf(fail_state, len, "%d:%s", op_status,transition_key); } return fail_state; } char * generate_transition_magic(const char *transition_key, int op_status, int op_rc) { int len = 80; char *fail_state = NULL; CRM_CHECK(transition_key != NULL, return NULL); len += strlen(transition_key); crm_malloc0(fail_state, len); if(fail_state != NULL) { snprintf(fail_state, len, "%d:%d;%s", op_status, op_rc, transition_key); } return fail_state; } gboolean decode_transition_magic( const char *magic, char **uuid, int *transition_id, int *op_status, int *op_rc) { char *rc = NULL; char *key = NULL; char *magic2 = NULL; char *status = NULL; if(decodeNVpair(magic, ':', &status, &magic2) == FALSE) { crm_err("Couldn't find ':' in: %s", magic); return FALSE; } if(decodeNVpair(magic2, ';', &rc, &key) == FALSE) { crm_err("Couldn't find ';' in: %s", magic2); return FALSE; } CRM_CHECK(decode_transition_key(key, uuid, transition_id), return FALSE); *op_rc = crm_parse_int(rc, NULL); *op_status = crm_parse_int(status, NULL); crm_free(rc); crm_free(key); crm_free(magic2); crm_free(status); return TRUE; } char * generate_transition_key(int transition_id, const char *node) { int len = 40; char *fail_state = NULL; CRM_CHECK(node != NULL, return NULL); len += strlen(node); crm_malloc0(fail_state, len); if(fail_state != NULL) { snprintf(fail_state, len, "%d:%s", transition_id, node); } return fail_state; } gboolean decode_transition_key(const char *key, char **uuid, int *transition_id) { char *transition = NULL; if(decodeNVpair(key, ':', &transition, uuid) == FALSE) { crm_err("Couldn't find ':' in: %s", key); return FALSE; } *transition_id = crm_parse_int(transition, NULL); crm_free(transition); return TRUE; } void filter_action_parameters(crm_data_t *param_set, const char *version) { #if CRM_DEPRECATED_SINCE_2_0_5 const char *filter_205[] = { XML_ATTR_TE_TARGET_RC, XML_ATTR_LRM_PROBE, XML_RSC_ATTR_START, XML_RSC_ATTR_NOTIFY, XML_RSC_ATTR_UNIQUE, XML_RSC_ATTR_MANAGED, XML_RSC_ATTR_PRIORITY, XML_RSC_ATTR_MULTIPLE, XML_RSC_ATTR_STICKINESS, XML_RSC_ATTR_FAIL_STICKINESS, XML_RSC_ATTR_TARGET_ROLE, /* ignore clone fields */ XML_RSC_ATTR_INCARNATION, XML_RSC_ATTR_INCARNATION_MAX, XML_RSC_ATTR_INCARNATION_NODEMAX, XML_RSC_ATTR_MASTER_MAX, XML_RSC_ATTR_MASTER_NODEMAX, /* old field names */ "role", "crm_role", "te-target-rc", /* ignore notify fields */ "notify_stop_resource", "notify_stop_uname", "notify_start_resource", "notify_start_uname", "notify_active_resource", "notify_active_uname", "notify_inactive_resource", "notify_inactive_uname", "notify_promote_resource", "notify_promote_uname", "notify_demote_resource", "notify_demote_uname", "notify_master_resource", "notify_master_uname", "notify_slave_resource", "notify_slave_uname" }; #endif const char *attr_filter[] = { XML_ATTR_ID, XML_ATTR_CRM_VERSION, XML_LRM_ATTR_OP_DIGEST, }; gboolean do_delete = FALSE; int lpc = 0; static int meta_len = 0; if(meta_len == 0) { meta_len = strlen(CRM_META); } if(param_set == NULL) { return; } #if CRM_DEPRECATED_SINCE_2_0_5 if(version == NULL || compare_version("1.0.5", version)) { for(lpc = 0; lpc < DIMOF(filter_205); lpc++) { xml_remove_prop(param_set, filter_205[lpc]); } } #endif for(lpc = 0; lpc < DIMOF(attr_filter); lpc++) { xml_remove_prop(param_set, attr_filter[lpc]); } xml_prop_iter(param_set, prop_name, prop_value, do_delete = FALSE; if(strncasecmp(prop_name, CRM_META, meta_len) == 0) { do_delete = TRUE; } if(do_delete) { /* remove it */ xml_remove_prop(param_set, prop_name); /* unwind the counetr */ __counter--; } ); } gboolean crm_mem_stats(volatile cl_mem_stats_t *mem_stats) { volatile cl_mem_stats_t *active_stats = mem_stats; if(active_stats == NULL) { active_stats = cl_malloc_getstats(); } CRM_CHECK(active_stats != NULL, ;); #ifndef CRM_USE_MALLOC if(active_stats->numalloc > active_stats->numfree) { crm_warn("Potential memory leak detected:" " %lu alloc's vs. %lu free's (%lu)" " (%lu bytes not freed: req=%lu, alloc'd=%lu)", active_stats->numalloc, active_stats->numfree, active_stats->numalloc - active_stats->numfree, active_stats->nbytes_alloc, active_stats->nbytes_req, active_stats->mallocbytes); return TRUE; } else if(active_stats->numalloc < active_stats->numfree) { crm_debug("Process shrank: %lu alloc's vs. %lu free's (%lu)", active_stats->numalloc, active_stats->numfree, active_stats->numalloc - active_stats->numfree); } #endif return FALSE; } void crm_zero_mem_stats(volatile cl_mem_stats_t *stats) { - volatile cl_mem_stats_t *active_stats = NULL; - if(stats != NULL) { - cl_malloc_setstats(stats); - } - crm_debug("Resetting memory stats"); - active_stats = cl_malloc_getstats(); - active_stats->numalloc = 0; - active_stats->numfree = 0; - active_stats->numrealloc = 0; - active_stats->nbytes_req = 0; - active_stats->nbytes_alloc = 0; - active_stats->mallocbytes = 0; - active_stats->arena = 0; + if(stats == NULL) { + crm_debug("Resetting global memory stats"); + stats = cl_malloc_getstats(); + } + stats->numalloc = 0; + stats->numfree = 0; + stats->numrealloc = 0; + stats->nbytes_req = 0; + stats->nbytes_alloc = 0; + stats->mallocbytes = 0; + stats->arena = 0; } void crm_save_mem_stats(const char *location, cl_mem_stats_t *saved_stats) { volatile cl_mem_stats_t *stats = cl_malloc_getstats(); if(saved_stats == NULL) { return; } crm_debug_2("Saving memory stats: %s", location); *saved_stats = *stats; } void crm_xml_nbytes(crm_data_t *xml, long *bytes, long *allocs, long *frees) { crm_data_t *xml_copy = NULL; volatile cl_mem_stats_t *stats = cl_malloc_getstats(); if(xml == NULL) { *bytes = 0; *allocs = 0; *frees = 0; return; } *bytes = 0 - stats->nbytes_alloc; *allocs = 0 - stats->numalloc; *frees = 0 - stats->numfree; xml_copy = copy_xml(xml); *bytes += stats->nbytes_alloc; *allocs += stats->numalloc; *frees += stats->numfree; crm_debug_3("XML size: %ld bytes, %ld allocs, %ld frees", *bytes, *allocs, *frees); free_xml(xml_copy); } void -crm_adjust_mem_stats(long bytes, long allocs, long frees) -{ - volatile cl_mem_stats_t *stats = cl_malloc_getstats(); - +crm_adjust_mem_stats(volatile cl_mem_stats_t *stats, long bytes, long allocs, long frees) +{ if(bytes == 0&& allocs == 0 && frees == 0) { return; } + if(stats == NULL) { + stats = cl_malloc_getstats(); + } + stats->nbytes_alloc -= bytes; stats->numalloc -= allocs; stats->numfree -= frees; - crm_debug("Adjusting CIB Memory usage by: %10ld bytes, %5ld allocs, %5ld frees", + crm_debug("Adjusted CIB Memory usage by: %10ld bytes, %5ld allocs, %5ld frees", bytes, allocs, frees); } +cl_mem_stats_t *crm_running_stats = NULL; gboolean -crm_diff_mem_stats(int log_level, const char *location, cl_mem_stats_t *saved_stats) +crm_diff_mem_stats(int log_level_up, int log_level_down, const char *location, + volatile cl_mem_stats_t *stats, volatile cl_mem_stats_t *saved_stats) { - gboolean increase = TRUE; - gboolean was_change = FALSE; - gboolean reset_on_change = (log_level != LOG_ERR); - - volatile cl_mem_stats_t *stats = cl_malloc_getstats(); - - static long running_total = 0; - static long running_total_bytes = 0; + long delta_allocs = 0; + long delta_frees = 0; + long delta_bytes = 0; + long delta_req = 0; - long last_running_total = running_total; - long last_running_total_bytes = running_total_bytes; + gboolean increase = TRUE; + gboolean reset_on_change = (log_level_up != LOG_ERR); - long delta_allocs = stats->numalloc - saved_stats->numalloc; - long delta_frees = stats->numfree - saved_stats->numfree; - long delta_bytes = stats->nbytes_alloc - saved_stats->nbytes_alloc; - long delta_req = stats->nbytes_req - saved_stats->nbytes_req; /* long delta_malloc = stats->mallocbytes - saved_stats->mallocbytes; */ - - running_total += delta_allocs; - running_total -= delta_frees; - running_total_bytes += delta_bytes; - if(running_total != last_running_total) { - was_change = TRUE; - } - if(running_total_bytes != last_running_total_bytes) { - was_change = TRUE; + if(stats == NULL && saved_stats == NULL) { + crm_err("Comparision doesnt make sense"); + return FALSE; + + } else if(stats == NULL) { + stats = cl_malloc_getstats(); + + } else if(saved_stats == NULL) { + saved_stats = cl_malloc_getstats(); } - if(was_change == FALSE) { + delta_allocs = stats->numalloc - saved_stats->numalloc; + delta_frees = stats->numfree - saved_stats->numfree; + delta_bytes = stats->nbytes_alloc - saved_stats->nbytes_alloc; + delta_req = stats->nbytes_req - saved_stats->nbytes_req; + + if(delta_bytes == 0) { crm_debug("Memory usage constant at %s: %ld alloc's %ld free's", location, delta_allocs, delta_frees); return FALSE; } - if(saved_stats->nbytes_alloc > stats->nbytes_alloc) { + if(delta_bytes < 0) { increase = FALSE; + reset_on_change = (log_level_down != LOG_ERR); } - crm_log_maybe(increase?log_level:log_level+2, - "Running total %10ld (%10ld bytes)" - " Memory usage %s detected at %s:\t" + crm_log_maybe(increase?log_level_up:log_level_down, + "Memory usage %s detected at %s:\t" " %10ld alloc's vs. %10ld free's (%5ld change" - " %10ld bytes leaked: %10ld requested)", - running_total, running_total_bytes, + " %10ld bytes leaked)", increase?"increase":"decrease", location, delta_allocs, delta_frees, delta_allocs - delta_frees, - delta_bytes, delta_req); + delta_bytes); - if(reset_on_change && increase) { - *stats = *saved_stats; + if(reset_on_change) { + crm_debug("resetting %s stats", location); + *stats = *saved_stats; + if(crm_running_stats) { + crm_adjust_mem_stats(crm_running_stats, delta_bytes, delta_allocs, delta_frees); + } } - return increase; + return TRUE; } void crm_abort(const char *file, const char *function, int line, const char *assert_condition, gboolean do_fork) { int pid = 0; if(do_fork == FALSE) { do_crm_log(LOG_ERR, file, function, "Triggered fatal assert at %s:%d : %s", file, line, assert_condition); } else if(crm_log_level < LOG_DEBUG) { do_crm_log(LOG_ERR, file, function, "Triggered non-fatal assert at %s:%d : %s", file, line, assert_condition); return; } else { pid=fork(); } switch(pid) { case -1: crm_err("Cannot fork!"); return; default: /* Parent */ do_crm_log(LOG_ERR, file, function, "Forked child %d to record non-fatal assert at %s:%d : %s", pid, file, line, assert_condition); return; case 0: /* Child */ abort(); break; } } char * generate_series_filename( const char *directory, const char *series, int sequence, gboolean bzip) { int len = 40; char *filename = NULL; const char *ext = "raw"; CRM_CHECK(directory != NULL, return NULL); CRM_CHECK(series != NULL, return NULL); len += strlen(directory); len += strlen(series); crm_malloc0(filename, len); CRM_CHECK(filename != NULL, return NULL); if(bzip) { ext = "bz2"; } sprintf(filename, "%s/%s-%d.%s", directory, series, sequence, ext); return filename; } int get_last_sequence(const char *directory, const char *series) { FILE *file_strm = NULL; int start = 0, length = 0, read_len = 0; char *series_file = NULL; char *buffer = NULL; int seq = 0; int len = 36; CRM_CHECK(directory != NULL, return 0); CRM_CHECK(series != NULL, return 0); len += strlen(directory); len += strlen(series); crm_malloc0(series_file, len); CRM_CHECK(series_file != NULL, return 0); sprintf(series_file, "%s/%s.last", directory, series); file_strm = fopen(series_file, "r"); if(file_strm == NULL) { crm_debug("%s does not exist", series_file); crm_free(series_file); return 0; } /* see how big the file is */ start = ftell(file_strm); fseek(file_strm, 0L, SEEK_END); length = ftell(file_strm); fseek(file_strm, 0L, start); CRM_ASSERT(start == ftell(file_strm)); crm_debug_3("Reading %d bytes from file", length); crm_malloc0(buffer, (length+1)); read_len = fread(buffer, 1, length, file_strm); if(read_len != length) { crm_err("Calculated and read bytes differ: %d vs. %d", length, read_len); crm_free(buffer); buffer = NULL; } else if(length <= 0) { crm_info("%s was not valid", series_file); crm_free(buffer); buffer = NULL; } crm_free(series_file); seq = crm_parse_int(buffer, "0"); crm_free(buffer); fclose(file_strm); return seq; } void write_last_sequence( const char *directory, const char *series, int sequence, int max) { int rc = 0; int len = 36; char *buffer = NULL; FILE *file_strm = NULL; char *series_file = NULL; CRM_CHECK(directory != NULL, return); CRM_CHECK(series != NULL, return); if(max == 0) { return; } while(max > 0 && sequence > max) { sequence -= max; } buffer = crm_itoa(sequence); len += strlen(directory); len += strlen(series); crm_malloc0(series_file, len); CRM_CHECK(series_file != NULL, return); sprintf(series_file, "%s/%s.last", directory, series); file_strm = fopen(series_file, "w"); if(file_strm == NULL) { crm_err("%s does not exist", series_file); crm_free(series_file); return; } rc = fprintf(file_strm, "%s", buffer); if(rc < 0) { cl_perror("Cannot write output to %s", series_file); } fflush(file_strm); fclose(file_strm); crm_free(series_file); crm_free(buffer); } void crm_make_daemon(const char *name, gboolean daemonize, const char *pidfile) { long pid; const char *devnull = "/dev/null"; if(daemonize == FALSE) { return; } pid = fork(); if (pid < 0) { fprintf(stderr, "%s: could not start daemon\n", name); cl_perror("fork"); exit(LSB_EXIT_GENERIC); } else if (pid > 0) { exit(LSB_EXIT_OK); } if (cl_lock_pidfile(pidfile) < 0 ) { pid = cl_read_pidfile_no_checking(pidfile); crm_warn("%s: already running [pid %ld] (%s).\n", name, pid, pidfile); exit(LSB_EXIT_OK); } umask(022); close(FD_STDIN); (void)open(devnull, O_RDONLY); /* Stdin: fd 0 */ close(FD_STDOUT); (void)open(devnull, O_WRONLY); /* Stdout: fd 1 */ close(FD_STDERR); (void)open(devnull, O_WRONLY); /* Stderr: fd 2 */ }