diff --git a/exec/cfg.c b/exec/cfg.c index 5bf4c966..6c9e0394 100644 --- a/exec/cfg.c +++ b/exec/cfg.c @@ -1,1159 +1,1168 @@ /* * Copyright (c) 2005-2006 MontaVista Software, Inc. * Copyright (c) 2006-2018 Red Hat, Inc. * * All rights reserved. * * Author: Steven Dake (sdake@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the MontaVista Software, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include +#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "totemconfig.h" #include "service.h" #include "main.h" LOGSYS_DECLARE_SUBSYS ("CFG"); enum cfg_message_req_types { MESSAGE_REQ_EXEC_CFG_RINGREENABLE = 0, MESSAGE_REQ_EXEC_CFG_KILLNODE = 1, MESSAGE_REQ_EXEC_CFG_SHUTDOWN = 2, MESSAGE_REQ_EXEC_CFG_RELOAD_CONFIG = 3 }; #define DEFAULT_SHUTDOWN_TIMEOUT 5 static struct qb_list_head trackers_list; /* * Variables controlling a requested shutdown */ static corosync_timer_handle_t shutdown_timer; static struct cfg_info *shutdown_con; static uint32_t shutdown_flags; static int shutdown_yes; static int shutdown_no; static int shutdown_expected; struct cfg_info { struct qb_list_head list; void *conn; void *tracker_conn; enum {SHUTDOWN_REPLY_UNKNOWN, SHUTDOWN_REPLY_YES, SHUTDOWN_REPLY_NO} shutdown_reply; }; static void cfg_confchg_fn ( enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id); static char *cfg_exec_init_fn (struct corosync_api_v1 *corosync_api_v1); static struct corosync_api_v1 *api; static int cfg_lib_init_fn (void *conn); static int cfg_lib_exit_fn (void *conn); static void message_handler_req_exec_cfg_ringreenable ( const void *message, unsigned int nodeid); static void message_handler_req_exec_cfg_killnode ( const void *message, unsigned int nodeid); static void message_handler_req_exec_cfg_shutdown ( const void *message, unsigned int nodeid); static void message_handler_req_exec_cfg_reload_config ( const void *message, unsigned int nodeid); static void exec_cfg_killnode_endian_convert (void *msg); static void message_handler_req_lib_cfg_ringstatusget ( void *conn, const void *msg); static void message_handler_req_lib_cfg_ringreenable ( void *conn, const void *msg); static void message_handler_req_lib_cfg_killnode ( void *conn, const void *msg); static void message_handler_req_lib_cfg_tryshutdown ( void *conn, const void *msg); static void message_handler_req_lib_cfg_replytoshutdown ( void *conn, const void *msg); static void message_handler_req_lib_cfg_get_node_addrs ( void *conn, const void *msg); static void message_handler_req_lib_cfg_local_get ( void *conn, const void *msg); static void message_handler_req_lib_cfg_reload_config ( void *conn, const void *msg); static void message_handler_req_lib_cfg_reopen_log_files ( void *conn, const void *msg); /* * Service Handler Definition */ static struct corosync_lib_handler cfg_lib_engine[] = { { /* 0 */ .lib_handler_fn = message_handler_req_lib_cfg_ringstatusget, .flow_control = CS_LIB_FLOW_CONTROL_REQUIRED }, { /* 1 */ .lib_handler_fn = message_handler_req_lib_cfg_ringreenable, .flow_control = CS_LIB_FLOW_CONTROL_REQUIRED }, { /* 2 */ .lib_handler_fn = message_handler_req_lib_cfg_killnode, .flow_control = CS_LIB_FLOW_CONTROL_REQUIRED }, { /* 3 */ .lib_handler_fn = message_handler_req_lib_cfg_tryshutdown, .flow_control = CS_LIB_FLOW_CONTROL_REQUIRED }, { /* 4 */ .lib_handler_fn = message_handler_req_lib_cfg_replytoshutdown, .flow_control = CS_LIB_FLOW_CONTROL_REQUIRED }, { /* 5 */ .lib_handler_fn = message_handler_req_lib_cfg_get_node_addrs, .flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED }, { /* 6 */ .lib_handler_fn = message_handler_req_lib_cfg_local_get, .flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED }, { /* 7 */ .lib_handler_fn = message_handler_req_lib_cfg_reload_config, .flow_control = CS_LIB_FLOW_CONTROL_REQUIRED }, { /* 8 */ .lib_handler_fn = message_handler_req_lib_cfg_reopen_log_files, .flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED } }; static struct corosync_exec_handler cfg_exec_engine[] = { { /* 0 */ .exec_handler_fn = message_handler_req_exec_cfg_ringreenable, }, { /* 1 */ .exec_handler_fn = message_handler_req_exec_cfg_killnode, .exec_endian_convert_fn = exec_cfg_killnode_endian_convert }, { /* 2 */ .exec_handler_fn = message_handler_req_exec_cfg_shutdown, }, { /* 3 */ .exec_handler_fn = message_handler_req_exec_cfg_reload_config, } }; /* * Exports the interface for the service */ struct corosync_service_engine cfg_service_engine = { .name = "corosync configuration service", .id = CFG_SERVICE, .priority = 1, .private_data_size = sizeof(struct cfg_info), .flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED, .allow_inquorate = CS_LIB_ALLOW_INQUORATE, .lib_init_fn = cfg_lib_init_fn, .lib_exit_fn = cfg_lib_exit_fn, .lib_engine = cfg_lib_engine, .lib_engine_count = sizeof (cfg_lib_engine) / sizeof (struct corosync_lib_handler), .exec_init_fn = cfg_exec_init_fn, .exec_engine = cfg_exec_engine, .exec_engine_count = sizeof (cfg_exec_engine) / sizeof (struct corosync_exec_handler), .confchg_fn = cfg_confchg_fn }; struct corosync_service_engine *cfg_get_service_engine_ver0 (void) { return (&cfg_service_engine); } struct req_exec_cfg_ringreenable { struct qb_ipc_request_header header __attribute__((aligned(8))); mar_message_source_t source __attribute__((aligned(8))); }; struct req_exec_cfg_reload_config { struct qb_ipc_request_header header __attribute__((aligned(8))); mar_message_source_t source __attribute__((aligned(8))); }; struct req_exec_cfg_killnode { struct qb_ipc_request_header header __attribute__((aligned(8))); mar_uint32_t nodeid __attribute__((aligned(8))); mar_name_t reason __attribute__((aligned(8))); }; struct req_exec_cfg_shutdown { struct qb_ipc_request_header header __attribute__((aligned(8))); }; /* IMPL */ static char *cfg_exec_init_fn ( struct corosync_api_v1 *corosync_api_v1) { api = corosync_api_v1; qb_list_init(&trackers_list); return (NULL); } static void cfg_confchg_fn ( enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id) { } /* * Tell other nodes we are shutting down */ static int send_shutdown(void) { struct req_exec_cfg_shutdown req_exec_cfg_shutdown; struct iovec iovec; ENTER(); req_exec_cfg_shutdown.header.size = sizeof (struct req_exec_cfg_shutdown); req_exec_cfg_shutdown.header.id = SERVICE_ID_MAKE (CFG_SERVICE, MESSAGE_REQ_EXEC_CFG_SHUTDOWN); iovec.iov_base = (char *)&req_exec_cfg_shutdown; iovec.iov_len = sizeof (struct req_exec_cfg_shutdown); assert (api->totem_mcast (&iovec, 1, TOTEM_SAFE) == 0); LEAVE(); return 0; } static void send_test_shutdown(void *only_conn, void *exclude_conn, int status) { struct res_lib_cfg_testshutdown res_lib_cfg_testshutdown; struct qb_list_head *iter; ENTER(); res_lib_cfg_testshutdown.header.size = sizeof(struct res_lib_cfg_testshutdown); res_lib_cfg_testshutdown.header.id = MESSAGE_RES_CFG_TESTSHUTDOWN; res_lib_cfg_testshutdown.header.error = status; res_lib_cfg_testshutdown.flags = shutdown_flags; if (only_conn) { TRACE1("sending testshutdown to only %p", only_conn); api->ipc_dispatch_send(only_conn, &res_lib_cfg_testshutdown, sizeof(res_lib_cfg_testshutdown)); } else { qb_list_for_each(iter, &trackers_list) { struct cfg_info *ci = qb_list_entry(iter, struct cfg_info, list); if (ci->conn != exclude_conn) { TRACE1("sending testshutdown to %p", ci->tracker_conn); api->ipc_dispatch_send(ci->tracker_conn, &res_lib_cfg_testshutdown, sizeof(res_lib_cfg_testshutdown)); } } } LEAVE(); } static void check_shutdown_status(void) { ENTER(); /* * Shutdown client might have gone away */ if (!shutdown_con) { LEAVE(); return; } /* * All replies safely gathered in ? */ if (shutdown_yes + shutdown_no >= shutdown_expected) { struct res_lib_cfg_tryshutdown res_lib_cfg_tryshutdown; api->timer_delete(shutdown_timer); if (shutdown_yes >= shutdown_expected || shutdown_flags == CFG_SHUTDOWN_FLAG_REGARDLESS) { TRACE1("shutdown confirmed"); res_lib_cfg_tryshutdown.header.size = sizeof(struct res_lib_cfg_tryshutdown); res_lib_cfg_tryshutdown.header.id = MESSAGE_RES_CFG_TRYSHUTDOWN; res_lib_cfg_tryshutdown.header.error = CS_OK; /* * Tell originator that shutdown was confirmed */ api->ipc_response_send(shutdown_con->conn, &res_lib_cfg_tryshutdown, sizeof(res_lib_cfg_tryshutdown)); shutdown_con = NULL; /* * Tell other nodes we are going down */ send_shutdown(); } else { TRACE1("shutdown cancelled"); res_lib_cfg_tryshutdown.header.size = sizeof(struct res_lib_cfg_tryshutdown); res_lib_cfg_tryshutdown.header.id = MESSAGE_RES_CFG_TRYSHUTDOWN; res_lib_cfg_tryshutdown.header.error = CS_ERR_BUSY; /* * Tell originator that shutdown was cancelled */ api->ipc_response_send(shutdown_con->conn, &res_lib_cfg_tryshutdown, sizeof(res_lib_cfg_tryshutdown)); shutdown_con = NULL; } log_printf(LOGSYS_LEVEL_DEBUG, "shutdown decision is: (yes count: %d, no count: %d) flags=%x", shutdown_yes, shutdown_no, shutdown_flags); } LEAVE(); } /* * Not all nodes responded to the shutdown (in time) */ static void shutdown_timer_fn(void *arg) { ENTER(); /* * Mark undecideds as "NO" */ shutdown_no = shutdown_expected; check_shutdown_status(); send_test_shutdown(NULL, NULL, CS_ERR_TIMEOUT); LEAVE(); } static void remove_ci_from_shutdown(struct cfg_info *ci) { ENTER(); /* * If the controlling shutdown process has quit, then cancel the * shutdown session */ if (ci == shutdown_con) { shutdown_con = NULL; api->timer_delete(shutdown_timer); } if (!qb_list_empty(&ci->list)) { qb_list_del(&ci->list); qb_list_init(&ci->list); /* * Remove our option */ if (shutdown_con) { if (ci->shutdown_reply == SHUTDOWN_REPLY_YES) shutdown_yes--; if (ci->shutdown_reply == SHUTDOWN_REPLY_NO) shutdown_no--; } /* * If we are leaving, then that's an implicit YES to shutdown */ ci->shutdown_reply = SHUTDOWN_REPLY_YES; shutdown_yes++; check_shutdown_status(); } LEAVE(); } int cfg_lib_exit_fn (void *conn) { struct cfg_info *ci = (struct cfg_info *)api->ipc_private_data_get (conn); ENTER(); remove_ci_from_shutdown(ci); LEAVE(); return (0); } static int cfg_lib_init_fn (void *conn) { struct cfg_info *ci = (struct cfg_info *)api->ipc_private_data_get (conn); ENTER(); qb_list_init(&ci->list); LEAVE(); return (0); } /* * Executive message handlers */ static void message_handler_req_exec_cfg_ringreenable ( const void *message, unsigned int nodeid) { ENTER(); LEAVE(); } static void exec_cfg_killnode_endian_convert (void *msg) { struct req_exec_cfg_killnode *req_exec_cfg_killnode = (struct req_exec_cfg_killnode *)msg; ENTER(); swab_mar_name_t(&req_exec_cfg_killnode->reason); LEAVE(); } static void message_handler_req_exec_cfg_killnode ( const void *message, unsigned int nodeid) { const struct req_exec_cfg_killnode *req_exec_cfg_killnode = message; cs_name_t reason; ENTER(); log_printf(LOGSYS_LEVEL_DEBUG, "request to kill node " CS_PRI_NODE_ID " (us=" CS_PRI_NODE_ID ")", req_exec_cfg_killnode->nodeid, api->totem_nodeid_get()); if (req_exec_cfg_killnode->nodeid == api->totem_nodeid_get()) { marshall_from_mar_name_t(&reason, &req_exec_cfg_killnode->reason); log_printf(LOGSYS_LEVEL_NOTICE, "Killed by node " CS_PRI_NODE_ID " : %s", nodeid, reason.value); corosync_fatal_error(COROSYNC_FATAL_ERROR_EXIT); } LEAVE(); } /* * Self shutdown */ static void message_handler_req_exec_cfg_shutdown ( const void *message, unsigned int nodeid) { ENTER(); log_printf(LOGSYS_LEVEL_NOTICE, "Node " CS_PRI_NODE_ID " was shut down by sysadmin", nodeid); if (nodeid == api->totem_nodeid_get()) { api->shutdown_request(); } LEAVE(); } /* strcmp replacement that can handle NULLs */ static int nullcheck_strcmp(const char* left, const char *right) { if (!left && right) return -1; if (left && !right) return 1; if (!left && !right) return 0; return strcmp(left, right); } /* * If a key has changed value in the new file, then warn the user and remove it from the temp_map */ static void delete_and_notify_if_changed(icmap_map_t temp_map, const char *key_name) { if (!(icmap_key_value_eq(temp_map, key_name, icmap_get_global_map(), key_name))) { if (icmap_delete_r(temp_map, key_name) == CS_OK) { log_printf(LOGSYS_LEVEL_NOTICE, "Modified entry '%s' in corosync.conf cannot be changed at run-time", key_name); } } } /* * Remove any keys from the new config file that in the new corosync.conf but that * cannot be changed at run time. A log message will be issued for each * entry that the user wants to change but they cannot. * * Add more here as needed. */ static void remove_ro_entries(icmap_map_t temp_map) { delete_and_notify_if_changed(temp_map, "totem.secauth"); delete_and_notify_if_changed(temp_map, "totem.crypto_hash"); delete_and_notify_if_changed(temp_map, "totem.crypto_cipher"); delete_and_notify_if_changed(temp_map, "totem.keyfile"); delete_and_notify_if_changed(temp_map, "totem.key"); delete_and_notify_if_changed(temp_map, "totem.version"); delete_and_notify_if_changed(temp_map, "totem.threads"); delete_and_notify_if_changed(temp_map, "totem.ip_version"); delete_and_notify_if_changed(temp_map, "totem.rrp_mode"); delete_and_notify_if_changed(temp_map, "totem.netmtu"); delete_and_notify_if_changed(temp_map, "totem.interface.ringnumber"); delete_and_notify_if_changed(temp_map, "totem.interface.bindnetaddr"); delete_and_notify_if_changed(temp_map, "totem.interface.mcastaddr"); delete_and_notify_if_changed(temp_map, "totem.interface.broadcast"); delete_and_notify_if_changed(temp_map, "totem.interface.mcastport"); delete_and_notify_if_changed(temp_map, "totem.interface.ttl"); delete_and_notify_if_changed(temp_map, "totem.vsftype"); delete_and_notify_if_changed(temp_map, "totem.transport"); delete_and_notify_if_changed(temp_map, "totem.cluster_name"); delete_and_notify_if_changed(temp_map, "quorum.provider"); delete_and_notify_if_changed(temp_map, "system.move_to_root_cgroup"); delete_and_notify_if_changed(temp_map, "system.sched_rr"); delete_and_notify_if_changed(temp_map, "system.priority"); delete_and_notify_if_changed(temp_map, "system.qb_ipc_type"); delete_and_notify_if_changed(temp_map, "system.state_dir"); } /* * Remove entries that exist in the global map, but not in the temp_map, this will * cause delete notifications to be sent to any listeners. * * NOTE: This routine depends entirely on the keys returned by the iterators * being in alpha-sorted order. */ static void remove_deleted_entries(icmap_map_t temp_map, const char *prefix) { icmap_iter_t old_iter; icmap_iter_t new_iter; const char *old_key, *new_key; int ret; old_iter = icmap_iter_init(prefix); new_iter = icmap_iter_init_r(temp_map, prefix); old_key = icmap_iter_next(old_iter, NULL, NULL); new_key = icmap_iter_next(new_iter, NULL, NULL); while (old_key || new_key) { ret = nullcheck_strcmp(old_key, new_key); if ((ret < 0 && old_key) || !new_key) { /* * new_key is greater, a line (or more) has been deleted * Continue until old is >= new */ do { /* Remove it from icmap & send notifications */ icmap_delete(old_key); old_key = icmap_iter_next(old_iter, NULL, NULL); ret = nullcheck_strcmp(old_key, new_key); } while (ret < 0 && old_key); } else if ((ret > 0 && new_key) || !old_key) { /* * old_key is greater, a line (or more) has been added * Continue until new is >= old * * we don't need to do anything special with this like tell * icmap. That will happen when we copy the values over */ do { new_key = icmap_iter_next(new_iter, NULL, NULL); ret = nullcheck_strcmp(old_key, new_key); } while (ret > 0 && new_key); } if (ret == 0) { new_key = icmap_iter_next(new_iter, NULL, NULL); old_key = icmap_iter_next(old_iter, NULL, NULL); } } icmap_iter_finalize(new_iter); icmap_iter_finalize(old_iter); } /* * Reload configuration file */ static void message_handler_req_exec_cfg_reload_config ( const void *message, unsigned int nodeid) { const struct req_exec_cfg_reload_config *req_exec_cfg_reload_config = message; struct res_lib_cfg_reload_config res_lib_cfg_reload_config; struct totem_config new_config; icmap_map_t temp_map; const char *error_string; int res = CS_OK; ENTER(); log_printf(LOGSYS_LEVEL_NOTICE, "Config reload requested by node " CS_PRI_NODE_ID, nodeid); icmap_set_uint8("config.totemconfig_reload_in_progress", 1); /* * Set up a new hashtable as a staging area. */ if ((res = icmap_init_r(&temp_map)) != CS_OK) { log_printf(LOGSYS_LEVEL_ERROR, "Unable to create temporary icmap. config file reload cancelled\n"); goto reload_fini_nomap; } /* * Load new config into the temporary map */ res = coroparse_configparse(temp_map, &error_string); if (res == -1) { log_printf (LOGSYS_LEVEL_ERROR, "Unable to reload config file: %s", error_string); res = CS_ERR_INVALID_PARAM; goto reload_fini_nofree; } /* Signal start of the reload process */ icmap_set_uint8("config.reload_in_progress", 1); /* Detect deleted entries and remove them from the main icmap hashtable */ remove_deleted_entries(temp_map, "logging."); remove_deleted_entries(temp_map, "totem."); remove_deleted_entries(temp_map, "nodelist."); remove_deleted_entries(temp_map, "quorum."); remove_deleted_entries(temp_map, "uidgid.config."); remove_deleted_entries(temp_map, "nozzle."); /* Remove entries that cannot be changed */ remove_ro_entries(temp_map); /* Take a copy of the current setup so we can check what has changed */ memset(&new_config, 0, sizeof(new_config)); new_config.orig_interfaces = malloc (sizeof (struct totem_interface) * INTERFACE_MAX); assert(new_config.orig_interfaces != NULL); totempg_get_config(&new_config); new_config.interfaces = malloc (sizeof (struct totem_interface) * INTERFACE_MAX); assert(new_config.interfaces != NULL); memset(new_config.interfaces, 0, sizeof (struct totem_interface) * INTERFACE_MAX); + /* For UDP[U] the configuration on link0 is static (apart from the nodelist) and only read at + startup. So preserve it here */ + if ( (new_config.transport_number == TOTEM_TRANSPORT_UDP) || + (new_config.transport_number == TOTEM_TRANSPORT_UDPU)) { + memcpy(&new_config.interfaces[0], &new_config.orig_interfaces[0], + sizeof(struct totem_interface)); + } + /* Calculate new node and interface definitions */ if (totemconfig_configure_new_params(&new_config, temp_map, &error_string) == -1) { log_printf (LOGSYS_LEVEL_ERROR, "Cannot configure new interface definitions: %s\n", error_string); res = CS_ERR_INVALID_PARAM; goto reload_fini; } /* Read from temp_map into new_config */ totem_volatile_config_read(&new_config, temp_map, NULL); /* Validate dynamic parameters */ if (totem_volatile_config_validate(&new_config, temp_map, &error_string) == -1) { log_printf (LOGSYS_LEVEL_ERROR, "Configuration is not valid: %s\n", error_string); res = CS_ERR_INVALID_PARAM; goto reload_fini; } /* * Copy new keys into live config. */ if ( (res = icmap_copy_map(icmap_get_global_map(), temp_map)) != CS_OK) { log_printf (LOGSYS_LEVEL_ERROR, "Error making new config live. cmap database may be inconsistent\n"); /* Return res from icmap */ goto reload_fini; } /* Copy into live system */ totempg_put_config(&new_config); - totemconfig_commit_new_params(&new_config, temp_map); + free(new_config.interfaces); reload_fini: /* All done - let clients know */ icmap_set_int32("config.reload_status", res); icmap_set_uint8("config.totemconfig_reload_in_progress", 0); icmap_set_uint8("config.reload_in_progress", 0); /* Finished with the temporary storage */ free(new_config.orig_interfaces); reload_fini_nofree: icmap_fini_r(temp_map); reload_fini_nomap: /* All done, return result to the caller if it was on this system */ if (nodeid == api->totem_nodeid_get()) { res_lib_cfg_reload_config.header.size = sizeof(res_lib_cfg_reload_config); res_lib_cfg_reload_config.header.id = MESSAGE_RES_CFG_RELOAD_CONFIG; res_lib_cfg_reload_config.header.error = res; api->ipc_response_send(req_exec_cfg_reload_config->source.conn, &res_lib_cfg_reload_config, sizeof(res_lib_cfg_reload_config)); api->ipc_refcnt_dec(req_exec_cfg_reload_config->source.conn);; } LEAVE(); } /* * Library Interface Implementation */ static void message_handler_req_lib_cfg_ringstatusget ( void *conn, const void *msg) { struct res_lib_cfg_ringstatusget res_lib_cfg_ringstatusget; struct totem_ip_address interfaces[INTERFACE_MAX]; unsigned int iface_count; char **status; const char *totem_ip_string; char ifname[CFG_INTERFACE_NAME_MAX_LEN]; unsigned int iface_ids[INTERFACE_MAX]; unsigned int i; cs_error_t res = CS_OK; ENTER(); res_lib_cfg_ringstatusget.header.id = MESSAGE_RES_CFG_RINGSTATUSGET; res_lib_cfg_ringstatusget.header.size = sizeof (struct res_lib_cfg_ringstatusget); api->totem_ifaces_get ( api->totem_nodeid_get(), iface_ids, interfaces, INTERFACE_MAX, &status, &iface_count); assert(iface_count <= CFG_MAX_INTERFACES); res_lib_cfg_ringstatusget.interface_count = iface_count; for (i = 0; i < iface_count; i++) { totem_ip_string = (const char *)api->totem_ip_print (&interfaces[i]); if (!totem_ip_string) { totem_ip_string=""; } /* Allow for i/f number at the start */ if (strlen(totem_ip_string) >= CFG_INTERFACE_NAME_MAX_LEN-3) { log_printf(LOGSYS_LEVEL_ERROR, "String representation of interface %u is too long", i); res = CS_ERR_NAME_TOO_LONG; goto send_response; } snprintf(ifname, sizeof(ifname), "%d %s", iface_ids[i], totem_ip_string); if (strlen(status[i]) >= CFG_INTERFACE_STATUS_MAX_LEN) { log_printf(LOGSYS_LEVEL_ERROR, "Status string for interface %u is too long", i); res = CS_ERR_NAME_TOO_LONG; goto send_response; } strcpy ((char *)&res_lib_cfg_ringstatusget.interface_status[i], status[i]); strcpy ((char *)&res_lib_cfg_ringstatusget.interface_name[i], ifname); } send_response: res_lib_cfg_ringstatusget.header.error = res; api->ipc_response_send ( conn, &res_lib_cfg_ringstatusget, sizeof (struct res_lib_cfg_ringstatusget)); LEAVE(); } static void message_handler_req_lib_cfg_ringreenable ( void *conn, const void *msg) { struct res_lib_cfg_ringreenable res_lib_cfg_ringreenable; ENTER(); res_lib_cfg_ringreenable.header.id = MESSAGE_RES_CFG_RINGREENABLE; res_lib_cfg_ringreenable.header.size = sizeof (struct res_lib_cfg_ringreenable); res_lib_cfg_ringreenable.header.error = CS_ERR_NOT_SUPPORTED; api->ipc_response_send ( conn, &res_lib_cfg_ringreenable, sizeof (struct res_lib_cfg_ringreenable)); LEAVE(); } static void message_handler_req_lib_cfg_killnode ( void *conn, const void *msg) { const struct req_lib_cfg_killnode *req_lib_cfg_killnode = msg; struct res_lib_cfg_killnode res_lib_cfg_killnode; struct req_exec_cfg_killnode req_exec_cfg_killnode; struct iovec iovec; ENTER(); req_exec_cfg_killnode.header.size = sizeof (struct req_exec_cfg_killnode); req_exec_cfg_killnode.header.id = SERVICE_ID_MAKE (CFG_SERVICE, MESSAGE_REQ_EXEC_CFG_KILLNODE); req_exec_cfg_killnode.nodeid = req_lib_cfg_killnode->nodeid; marshall_to_mar_name_t(&req_exec_cfg_killnode.reason, &req_lib_cfg_killnode->reason); iovec.iov_base = (char *)&req_exec_cfg_killnode; iovec.iov_len = sizeof (struct req_exec_cfg_killnode); (void)api->totem_mcast (&iovec, 1, TOTEM_SAFE); res_lib_cfg_killnode.header.size = sizeof(struct res_lib_cfg_killnode); res_lib_cfg_killnode.header.id = MESSAGE_RES_CFG_KILLNODE; res_lib_cfg_killnode.header.error = CS_OK; api->ipc_response_send(conn, &res_lib_cfg_killnode, sizeof(res_lib_cfg_killnode)); LEAVE(); } static void message_handler_req_lib_cfg_tryshutdown ( void *conn, const void *msg) { struct cfg_info *ci = (struct cfg_info *)api->ipc_private_data_get (conn); const struct req_lib_cfg_tryshutdown *req_lib_cfg_tryshutdown = msg; struct qb_list_head *iter; ENTER(); if (req_lib_cfg_tryshutdown->flags == CFG_SHUTDOWN_FLAG_IMMEDIATE) { struct res_lib_cfg_tryshutdown res_lib_cfg_tryshutdown; /* * Tell other nodes */ send_shutdown(); res_lib_cfg_tryshutdown.header.size = sizeof(struct res_lib_cfg_tryshutdown); res_lib_cfg_tryshutdown.header.id = MESSAGE_RES_CFG_TRYSHUTDOWN; res_lib_cfg_tryshutdown.header.error = CS_OK; api->ipc_response_send(conn, &res_lib_cfg_tryshutdown, sizeof(res_lib_cfg_tryshutdown)); LEAVE(); return; } /* * Shutdown in progress, return an error */ if (shutdown_con) { struct res_lib_cfg_tryshutdown res_lib_cfg_tryshutdown; res_lib_cfg_tryshutdown.header.size = sizeof(struct res_lib_cfg_tryshutdown); res_lib_cfg_tryshutdown.header.id = MESSAGE_RES_CFG_TRYSHUTDOWN; res_lib_cfg_tryshutdown.header.error = CS_ERR_EXIST; api->ipc_response_send(conn, &res_lib_cfg_tryshutdown, sizeof(res_lib_cfg_tryshutdown)); LEAVE(); return; } ci->conn = conn; shutdown_con = (struct cfg_info *)api->ipc_private_data_get (conn); shutdown_flags = req_lib_cfg_tryshutdown->flags; shutdown_yes = 0; shutdown_no = 0; /* * Count the number of listeners */ shutdown_expected = 0; qb_list_for_each(iter, &trackers_list) { struct cfg_info *testci = qb_list_entry(iter, struct cfg_info, list); /* * It is assumed that we will allow shutdown */ if (testci != ci) { testci->shutdown_reply = SHUTDOWN_REPLY_UNKNOWN; shutdown_expected++; } } /* * If no-one is listening for events then we can just go down now */ if (shutdown_expected == 0) { struct res_lib_cfg_tryshutdown res_lib_cfg_tryshutdown; res_lib_cfg_tryshutdown.header.size = sizeof(struct res_lib_cfg_tryshutdown); res_lib_cfg_tryshutdown.header.id = MESSAGE_RES_CFG_TRYSHUTDOWN; res_lib_cfg_tryshutdown.header.error = CS_OK; /* * Tell originator that shutdown was confirmed */ api->ipc_response_send(conn, &res_lib_cfg_tryshutdown, sizeof(res_lib_cfg_tryshutdown)); send_shutdown(); LEAVE(); return; } else { unsigned int shutdown_timeout = DEFAULT_SHUTDOWN_TIMEOUT; /* * Look for a shutdown timeout in configuration map */ icmap_get_uint32("cfg.shutdown_timeout", &shutdown_timeout); /* * Start the timer. If we don't get a full set of replies before this goes * off we'll cancel the shutdown */ api->timer_add_duration((unsigned long long)shutdown_timeout*1000000000, NULL, shutdown_timer_fn, &shutdown_timer); /* * Tell the users we would like to shut down */ send_test_shutdown(NULL, conn, CS_OK); } /* * We don't sent a reply to the caller here. * We send it when we know if we can shut down or not */ LEAVE(); } static void message_handler_req_lib_cfg_replytoshutdown ( void *conn, const void *msg) { struct cfg_info *ci = (struct cfg_info *)api->ipc_private_data_get (conn); const struct req_lib_cfg_replytoshutdown *req_lib_cfg_replytoshutdown = msg; struct res_lib_cfg_replytoshutdown res_lib_cfg_replytoshutdown; int status = CS_OK; ENTER(); if (!shutdown_con) { status = CS_ERR_ACCESS; goto exit_fn; } if (req_lib_cfg_replytoshutdown->response) { shutdown_yes++; ci->shutdown_reply = SHUTDOWN_REPLY_YES; } else { shutdown_no++; ci->shutdown_reply = SHUTDOWN_REPLY_NO; } check_shutdown_status(); exit_fn: res_lib_cfg_replytoshutdown.header.error = status; res_lib_cfg_replytoshutdown.header.id = MESSAGE_RES_CFG_REPLYTOSHUTDOWN; res_lib_cfg_replytoshutdown.header.size = sizeof(res_lib_cfg_replytoshutdown); api->ipc_response_send(conn, &res_lib_cfg_replytoshutdown, sizeof(res_lib_cfg_replytoshutdown)); LEAVE(); } static void message_handler_req_lib_cfg_get_node_addrs (void *conn, const void *msg) { struct totem_ip_address node_ifs[INTERFACE_MAX]; unsigned int iface_ids[INTERFACE_MAX]; char buf[PIPE_BUF]; char **status; unsigned int num_interfaces = 0; struct sockaddr_storage *ss; int ret = CS_OK; int i; int live_addrs = 0; const struct req_lib_cfg_get_node_addrs *req_lib_cfg_get_node_addrs = msg; struct res_lib_cfg_get_node_addrs *res_lib_cfg_get_node_addrs = (struct res_lib_cfg_get_node_addrs *)buf; unsigned int nodeid = req_lib_cfg_get_node_addrs->nodeid; char *addr_buf; if (nodeid == 0) nodeid = api->totem_nodeid_get(); if (api->totem_ifaces_get(nodeid, iface_ids, node_ifs, INTERFACE_MAX, &status, &num_interfaces)) { ret = CS_ERR_EXIST; num_interfaces = 0; } res_lib_cfg_get_node_addrs->header.size = sizeof(struct res_lib_cfg_get_node_addrs) + (num_interfaces * TOTEMIP_ADDRLEN); res_lib_cfg_get_node_addrs->header.id = MESSAGE_RES_CFG_GET_NODE_ADDRS; res_lib_cfg_get_node_addrs->header.error = ret; if (num_interfaces) { res_lib_cfg_get_node_addrs->family = node_ifs[0].family; for (i = 0, addr_buf = (char *)res_lib_cfg_get_node_addrs->addrs; i < num_interfaces; i++) { ss = (struct sockaddr_storage *)&node_ifs[i].addr; if (ss->ss_family) { memcpy(addr_buf, node_ifs[i].addr, TOTEMIP_ADDRLEN); live_addrs++; addr_buf += TOTEMIP_ADDRLEN; } } res_lib_cfg_get_node_addrs->num_addrs = live_addrs; } else { res_lib_cfg_get_node_addrs->header.error = CS_ERR_NOT_EXIST; } api->ipc_response_send(conn, res_lib_cfg_get_node_addrs, res_lib_cfg_get_node_addrs->header.size); } static void message_handler_req_lib_cfg_local_get (void *conn, const void *msg) { struct res_lib_cfg_local_get res_lib_cfg_local_get; res_lib_cfg_local_get.header.size = sizeof(res_lib_cfg_local_get); res_lib_cfg_local_get.header.id = MESSAGE_RES_CFG_LOCAL_GET; res_lib_cfg_local_get.header.error = CS_OK; res_lib_cfg_local_get.local_nodeid = api->totem_nodeid_get (); api->ipc_response_send(conn, &res_lib_cfg_local_get, sizeof(res_lib_cfg_local_get)); } static void message_handler_req_lib_cfg_reload_config (void *conn, const void *msg) { struct req_exec_cfg_reload_config req_exec_cfg_reload_config; struct iovec iovec; ENTER(); req_exec_cfg_reload_config.header.size = sizeof (struct req_exec_cfg_reload_config); req_exec_cfg_reload_config.header.id = SERVICE_ID_MAKE (CFG_SERVICE, MESSAGE_REQ_EXEC_CFG_RELOAD_CONFIG); api->ipc_source_set (&req_exec_cfg_reload_config.source, conn); api->ipc_refcnt_inc(conn); iovec.iov_base = (char *)&req_exec_cfg_reload_config; iovec.iov_len = sizeof (struct req_exec_cfg_reload_config); assert (api->totem_mcast (&iovec, 1, TOTEM_SAFE) == 0); LEAVE(); } static void message_handler_req_lib_cfg_reopen_log_files (void *conn, const void *msg) { struct res_lib_cfg_reopen_log_files res_lib_cfg_reopen_log_files; cs_error_t res; ENTER(); log_printf(LOGSYS_LEVEL_DEBUG, "Reopening logging files\n"); res = logsys_reopen_log_files(); res_lib_cfg_reopen_log_files.header.size = sizeof(res_lib_cfg_reopen_log_files); res_lib_cfg_reopen_log_files.header.id = MESSAGE_RES_CFG_REOPEN_LOG_FILES; res_lib_cfg_reopen_log_files.header.error = res; api->ipc_response_send(conn, &res_lib_cfg_reopen_log_files, sizeof(res_lib_cfg_reopen_log_files)); LEAVE(); } diff --git a/exec/totemconfig.c b/exec/totemconfig.c index 90eb330d..b28d7bef 100644 --- a/exec/totemconfig.c +++ b/exec/totemconfig.c @@ -1,2337 +1,2341 @@ /* * Copyright (c) 2002-2005 MontaVista Software, Inc. * Copyright (c) 2006-2019 Red Hat, Inc. * * All rights reserved. * * Author: Steven Dake (sdake@redhat.com) * Jan Friesse (jfriesse@redhat.com) * Chrissie Caulfield (ccaulfie@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the MontaVista Software, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "util.h" #include "totemconfig.h" #define TOKEN_RETRANSMITS_BEFORE_LOSS_CONST 4 #define TOKEN_TIMEOUT 1000 #define TOKEN_WARNING 75 #define TOKEN_COEFFICIENT 650 #define JOIN_TIMEOUT 50 #define MERGE_TIMEOUT 200 #define DOWNCHECK_TIMEOUT 1000 #define FAIL_TO_RECV_CONST 2500 #define SEQNO_UNCHANGED_CONST 30 #define MINIMUM_TIMEOUT (int)(1000/HZ)*3 #define MINIMUM_TIMEOUT_HOLD (int)(MINIMUM_TIMEOUT * 0.8 - (1000/HZ)) #define MAX_NETWORK_DELAY 50 #define WINDOW_SIZE 50 #define MAX_MESSAGES 17 #define MISS_COUNT_CONST 5 #define BLOCK_UNLISTED_IPS 1 /* Currently all but PONG_COUNT match the defaults in libknet.h */ #define KNET_PING_INTERVAL 1000 #define KNET_PING_TIMEOUT 2000 #define KNET_PING_PRECISION 2048 #define KNET_PONG_COUNT 2 #define KNET_PMTUD_INTERVAL 30 #define KNET_DEFAULT_TRANSPORT KNET_TRANSPORT_UDP #define DEFAULT_PORT 5405 static char error_string_response[768]; static void add_totem_config_notification(struct totem_config *totem_config); static void *totem_get_param_by_name(struct totem_config *totem_config, const char *param_name) { if (strcmp(param_name, "totem.token") == 0) return &totem_config->token_timeout; if (strcmp(param_name, "totem.token_warning") == 0) return &totem_config->token_warning; if (strcmp(param_name, "totem.token_retransmit") == 0) return &totem_config->token_retransmit_timeout; if (strcmp(param_name, "totem.hold") == 0) return &totem_config->token_hold_timeout; if (strcmp(param_name, "totem.token_retransmits_before_loss_const") == 0) return &totem_config->token_retransmits_before_loss_const; if (strcmp(param_name, "totem.join") == 0) return &totem_config->join_timeout; if (strcmp(param_name, "totem.send_join") == 0) return &totem_config->send_join_timeout; if (strcmp(param_name, "totem.consensus") == 0) return &totem_config->consensus_timeout; if (strcmp(param_name, "totem.merge") == 0) return &totem_config->merge_timeout; if (strcmp(param_name, "totem.downcheck") == 0) return &totem_config->downcheck_timeout; if (strcmp(param_name, "totem.fail_recv_const") == 0) return &totem_config->fail_to_recv_const; if (strcmp(param_name, "totem.seqno_unchanged_const") == 0) return &totem_config->seqno_unchanged_const; if (strcmp(param_name, "totem.heartbeat_failures_allowed") == 0) return &totem_config->heartbeat_failures_allowed; if (strcmp(param_name, "totem.max_network_delay") == 0) return &totem_config->max_network_delay; if (strcmp(param_name, "totem.window_size") == 0) return &totem_config->window_size; if (strcmp(param_name, "totem.max_messages") == 0) return &totem_config->max_messages; if (strcmp(param_name, "totem.miss_count_const") == 0) return &totem_config->miss_count_const; if (strcmp(param_name, "totem.knet_pmtud_interval") == 0) return &totem_config->knet_pmtud_interval; if (strcmp(param_name, "totem.knet_compression_threshold") == 0) return &totem_config->knet_compression_threshold; if (strcmp(param_name, "totem.knet_compression_level") == 0) return &totem_config->knet_compression_level; if (strcmp(param_name, "totem.knet_compression_model") == 0) return &totem_config->knet_compression_model; if (strcmp(param_name, "totem.block_unlisted_ips") == 0) return &totem_config->block_unlisted_ips; return NULL; } /* * Read key_name from icmap. If key is not found or key_name == delete_key or if allow_zero is false * and readed value is zero, default value is used and stored into totem_config. */ static void totem_volatile_config_set_uint32_value (struct totem_config *totem_config, icmap_map_t map, const char *key_name, const char *deleted_key, unsigned int default_value, int allow_zero_value) { char runtime_key_name[ICMAP_KEYNAME_MAXLEN]; if (icmap_get_uint32_r(map, key_name, totem_get_param_by_name(totem_config, key_name)) != CS_OK || (deleted_key != NULL && strcmp(deleted_key, key_name) == 0) || (!allow_zero_value && *(uint32_t *)totem_get_param_by_name(totem_config, key_name) == 0)) { *(uint32_t *)totem_get_param_by_name(totem_config, key_name) = default_value; } /* * Store totem_config value to cmap runtime section */ if (strlen("runtime.config.") + strlen(key_name) >= ICMAP_KEYNAME_MAXLEN) { /* * This shouldn't happen */ return ; } strcpy(runtime_key_name, "runtime.config."); strcat(runtime_key_name, key_name); icmap_set_uint32_r(map, runtime_key_name, *(uint32_t *)totem_get_param_by_name(totem_config, key_name)); } static void totem_volatile_config_set_int32_value (struct totem_config *totem_config, icmap_map_t map, const char *key_name, const char *deleted_key, int default_value, int allow_zero_value) { char runtime_key_name[ICMAP_KEYNAME_MAXLEN]; if (icmap_get_int32_r(map, key_name, totem_get_param_by_name(totem_config, key_name)) != CS_OK || (deleted_key != NULL && strcmp(deleted_key, key_name) == 0) || (!allow_zero_value && *(int32_t *)totem_get_param_by_name(totem_config, key_name) == 0)) { *(int32_t *)totem_get_param_by_name(totem_config, key_name) = default_value; } /* * Store totem_config value to cmap runtime section */ if (strlen("runtime.config.") + strlen(key_name) >= ICMAP_KEYNAME_MAXLEN) { /* * This shouldn't happen */ return ; } strcpy(runtime_key_name, "runtime.config."); strcat(runtime_key_name, key_name); icmap_set_int32_r(map, runtime_key_name, *(int32_t *)totem_get_param_by_name(totem_config, key_name)); } static void totem_volatile_config_set_string_value (struct totem_config *totem_config, icmap_map_t map, const char *key_name, const char *deleted_key, const char *default_value) { char runtime_key_name[ICMAP_KEYNAME_MAXLEN]; - void **config_value; + void **config_value = NULL; /* compiler placation */ void *old_config_ptr; config_value = totem_get_param_by_name(totem_config, key_name); old_config_ptr = *config_value; - if (icmap_get_string_r(map, key_name, (char **)config_value) != CS_OK || + if (icmap_get_string(key_name, (char **)config_value) != CS_OK || (deleted_key != NULL && strcmp(deleted_key, key_name) == 0)) { /* Need to strdup() here so that the free() below works for a default and a configured value */ *config_value = strdup(default_value); } free(old_config_ptr); /* * Store totem_config value to cmap runtime section */ if (strlen("runtime.config.") + strlen(key_name) >= ICMAP_KEYNAME_MAXLEN) { /* * This shouldn't happen */ return ; } strcpy(runtime_key_name, "runtime.config."); strcat(runtime_key_name, key_name); (void)icmap_set_string_r(map, runtime_key_name, (char *)*config_value); } /* * Read string value stored in key_name from icmap, use it as a boolean (yes/no) type, convert it * to integer value (1/0) and store into totem_config. * * If key is not found or key_name == delete_key default value is used * and stored into totem_config. */ static void totem_volatile_config_set_boolean_value (struct totem_config *totem_config, icmap_map_t map, const char *key_name, const char *deleted_key, unsigned int default_value) { char runtime_key_name[ICMAP_KEYNAME_MAXLEN]; char *str; int val; str = NULL; val = default_value; if ((deleted_key != NULL && strcmp(deleted_key, key_name) == 0) || (icmap_get_string_r(map, key_name, &str) != CS_OK)) { /* * Do nothing. str is NULL (icmap_get_string ether not called or * not changed str). */ } else { if (strcmp(str, "yes") == 0) { val = 1; } else if (strcmp(str, "no") == 0) { val = 0; } free(str); } /* * Store totem_config value to cmap runtime section */ if (strlen("runtime.config.") + strlen(key_name) >= ICMAP_KEYNAME_MAXLEN) { /* * This shouldn't happen */ return ; } strcpy(runtime_key_name, "runtime.config."); strcat(runtime_key_name, key_name); *(uint32_t *)totem_get_param_by_name(totem_config, key_name) = val; icmap_set_uint32_r(map, runtime_key_name, val); } /* * Read and validate config values from cmap and store them into totem_config. If key doesn't exists, * default value is stored. deleted_key is name of key beeing processed by delete operation * from cmap. It is considered as non existing even if it can be read. Can be NULL. */ void totem_volatile_config_read (struct totem_config *totem_config, icmap_map_t temp_map, const char *deleted_key) { uint32_t u32; totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.token_retransmits_before_loss_const", deleted_key, TOKEN_RETRANSMITS_BEFORE_LOSS_CONST, 0); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.token", deleted_key, TOKEN_TIMEOUT, 0); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.token_warning", deleted_key, TOKEN_WARNING, 1); if (totem_config->interfaces[0].member_count > 2) { u32 = TOKEN_COEFFICIENT; icmap_get_uint32_r(temp_map, "totem.token_coefficient", &u32); totem_config->token_timeout += (totem_config->interfaces[0].member_count - 2) * u32; /* * Store totem_config value to cmap runtime section */ icmap_set_uint32_r(temp_map, "runtime.config.totem.token", totem_config->token_timeout); } totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.max_network_delay", deleted_key, MAX_NETWORK_DELAY, 0); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.window_size", deleted_key, WINDOW_SIZE, 0); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.max_messages", deleted_key, MAX_MESSAGES, 0); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.miss_count_const", deleted_key, MISS_COUNT_CONST, 0); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.knet_pmtud_interval", deleted_key, KNET_PMTUD_INTERVAL, 0); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.token_retransmit", deleted_key, (int)(totem_config->token_timeout / (totem_config->token_retransmits_before_loss_const + 0.2)), 0); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.hold", deleted_key, (int)(totem_config->token_retransmit_timeout * 0.8 - (1000/HZ)), 0); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.join", deleted_key, JOIN_TIMEOUT, 0); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.consensus", deleted_key, (int)(float)(1.2 * totem_config->token_timeout), 0); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.merge", deleted_key, MERGE_TIMEOUT, 0); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.downcheck", deleted_key, DOWNCHECK_TIMEOUT, 0); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.fail_recv_const", deleted_key, FAIL_TO_RECV_CONST, 0); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.seqno_unchanged_const", deleted_key, SEQNO_UNCHANGED_CONST, 0); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.send_join", deleted_key, 0, 1); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.heartbeat_failures_allowed", deleted_key, 0, 1); totem_volatile_config_set_uint32_value(totem_config, temp_map, "totem.knet_compression_threshold", deleted_key, 0, 1); totem_volatile_config_set_int32_value(totem_config, temp_map, "totem.knet_compression_level", deleted_key, 0, 1); totem_volatile_config_set_string_value(totem_config, temp_map, "totem.knet_compression_model", deleted_key, "none"); totem_volatile_config_set_boolean_value(totem_config, temp_map, "totem.block_unlisted_ips", deleted_key, BLOCK_UNLISTED_IPS); } int totem_volatile_config_validate ( struct totem_config *totem_config, icmap_map_t temp_map, const char **error_string) { + /* Static just to keep them off the stack */ static char local_error_reason[512]; + static char addr_str_buf[INET6_ADDRSTRLEN]; const char *error_reason = local_error_reason; char name_key[ICMAP_KEYNAME_MAXLEN]; char *name_str; - int i, num_configured, members; + int i, j, num_configured, members; uint32_t tmp_config_value; if (totem_config->max_network_delay < MINIMUM_TIMEOUT) { snprintf (local_error_reason, sizeof(local_error_reason), "The max_network_delay parameter (%d ms) may not be less than (%d ms).", totem_config->max_network_delay, MINIMUM_TIMEOUT); goto parse_error; } if (totem_config->token_timeout < MINIMUM_TIMEOUT) { snprintf (local_error_reason, sizeof(local_error_reason), "The token timeout parameter (%d ms) may not be less than (%d ms).", totem_config->token_timeout, MINIMUM_TIMEOUT); goto parse_error; } if (totem_config->token_warning > 100 || totem_config->token_warning < 0) { snprintf (local_error_reason, sizeof(local_error_reason), "The token warning parameter (%d%%) must be between 0 (disabled) and 100.", totem_config->token_warning); goto parse_error; } if (totem_config->token_retransmit_timeout < MINIMUM_TIMEOUT) { if (icmap_get_uint32_r(temp_map, "totem.token_retransmit", &tmp_config_value) == CS_OK) { snprintf (local_error_reason, sizeof(local_error_reason), "The token retransmit timeout parameter (%d ms) may not be less than (%d ms).", totem_config->token_retransmit_timeout, MINIMUM_TIMEOUT); goto parse_error; } else { snprintf (local_error_reason, sizeof(local_error_reason), "Not appropriate token or token_retransmits_before_loss_const value set"); goto parse_error; } } if (totem_config->token_hold_timeout < MINIMUM_TIMEOUT_HOLD) { snprintf (local_error_reason, sizeof(local_error_reason), "The token hold timeout parameter (%d ms) may not be less than (%d ms).", totem_config->token_hold_timeout, MINIMUM_TIMEOUT_HOLD); goto parse_error; } if (totem_config->join_timeout < MINIMUM_TIMEOUT) { snprintf (local_error_reason, sizeof(local_error_reason), "The join timeout parameter (%d ms) may not be less than (%d ms).", totem_config->join_timeout, MINIMUM_TIMEOUT); goto parse_error; } if (totem_config->consensus_timeout < MINIMUM_TIMEOUT) { snprintf (local_error_reason, sizeof(local_error_reason), "The consensus timeout parameter (%d ms) may not be less than (%d ms).", totem_config->consensus_timeout, MINIMUM_TIMEOUT); goto parse_error; } if (totem_config->consensus_timeout < totem_config->join_timeout) { snprintf (local_error_reason, sizeof(local_error_reason), "The consensus timeout parameter (%d ms) may not be less than join timeout (%d ms).", totem_config->consensus_timeout, totem_config->join_timeout); goto parse_error; } if (totem_config->merge_timeout < MINIMUM_TIMEOUT) { snprintf (local_error_reason, sizeof(local_error_reason), "The merge timeout parameter (%d ms) may not be less than (%d ms).", totem_config->merge_timeout, MINIMUM_TIMEOUT); goto parse_error; } if (totem_config->downcheck_timeout < MINIMUM_TIMEOUT) { snprintf (local_error_reason, sizeof(local_error_reason), "The downcheck timeout parameter (%d ms) may not be less than (%d ms).", totem_config->downcheck_timeout, MINIMUM_TIMEOUT); goto parse_error; } /* Check that we have nodelist 'name' if there is more than one link */ num_configured = 0; members = -1; for (i = 0; i < INTERFACE_MAX; i++) { if (totem_config->interfaces[i].configured) { if (num_configured == 0) { members = totem_config->interfaces[i].member_count; } num_configured++; } } if (num_configured > 1) { /* * This assert is here just to make compiler happy */ assert(members != -1); for (i=0; i < members; i++) { snprintf(name_key, sizeof(name_key), "nodelist.node.%d.name", i); if (icmap_get_string_r(temp_map, name_key, &name_str) != CS_OK) { snprintf (local_error_reason, sizeof(local_error_reason), "for a multi-link configuration, all nodes must have a 'name' attribute"); goto parse_error; } free(name_str); } for (i=0; i < INTERFACE_MAX; i++) { if (!totem_config->interfaces[i].configured) { continue; } if (totem_config->interfaces[i].member_count != members) { snprintf (local_error_reason, sizeof(local_error_reason), "Not all nodes have the same number of links"); goto parse_error; } } } + /* Verify that all nodes on the same link have the same IP family */ + for (i=0; i < INTERFACE_MAX; i++) { + for (j=1; jinterfaces[i].member_count; j++) { + if (totem_config->interfaces[i].configured) { + if (totem_config->interfaces[i].member_list[j].family != + totem_config->interfaces[i].member_list[0].family) { + memcpy(addr_str_buf, + totemip_print(&(totem_config->interfaces[i].member_list[j])), + sizeof(addr_str_buf)); + + snprintf (local_error_reason, sizeof(local_error_reason), + "Nodes for link %d have different IP families " + "(compared %s with %s)", i, + addr_str_buf, + totemip_print(&(totem_config->interfaces[i].member_list[0]))); + goto parse_error; + } + } + } + } + return 0; parse_error: snprintf (error_string_response, sizeof(error_string_response), "parse error in config: %s\n", error_reason); *error_string = error_string_response; return (-1); } static int totem_get_crypto(struct totem_config *totem_config, const char **error_string) { char *str; const char *tmp_cipher; const char *tmp_hash; const char *tmp_model; tmp_hash = "none"; tmp_cipher = "none"; tmp_model = "none"; if (icmap_get_string("totem.crypto_model", &str) == CS_OK) { if (strcmp(str, "nss") == 0) { tmp_model = "nss"; } if (strcmp(str, "openssl") == 0) { tmp_model = "openssl"; } free(str); } else { tmp_model = "nss"; } if (icmap_get_string("totem.secauth", &str) == CS_OK) { if (strcmp(str, "on") == 0) { tmp_cipher = "aes256"; tmp_hash = "sha256"; } free(str); } if (icmap_get_string("totem.crypto_cipher", &str) == CS_OK) { if (strcmp(str, "none") == 0) { tmp_cipher = "none"; } if (strcmp(str, "aes256") == 0) { tmp_cipher = "aes256"; } if (strcmp(str, "aes192") == 0) { tmp_cipher = "aes192"; } if (strcmp(str, "aes128") == 0) { tmp_cipher = "aes128"; } free(str); } if (icmap_get_string("totem.crypto_hash", &str) == CS_OK) { if (strcmp(str, "none") == 0) { tmp_hash = "none"; } if (strcmp(str, "md5") == 0) { tmp_hash = "md5"; } if (strcmp(str, "sha1") == 0) { tmp_hash = "sha1"; } if (strcmp(str, "sha256") == 0) { tmp_hash = "sha256"; } if (strcmp(str, "sha384") == 0) { tmp_hash = "sha384"; } if (strcmp(str, "sha512") == 0) { tmp_hash = "sha512"; } free(str); } if ((strcmp(tmp_cipher, "none") != 0) && (strcmp(tmp_hash, "none") == 0)) { *error_string = "crypto_cipher requires crypto_hash with value other than none"; return -1; } if (strcmp(tmp_model, "none") == 0) { *error_string = "crypto_model should be 'nss' or 'openssl'"; return -1; } free(totem_config->crypto_cipher_type); free(totem_config->crypto_hash_type); free(totem_config->crypto_model); totem_config->crypto_cipher_type = strdup(tmp_cipher); totem_config->crypto_hash_type = strdup(tmp_hash); totem_config->crypto_model = strdup(tmp_model); return 0; } static int nodelist_byname(icmap_map_t map, const char *find_name, int strip_domain) { icmap_iter_t iter; const char *iter_key; char name_str[ICMAP_KEYNAME_MAXLEN]; int res = 0; unsigned int node_pos; char *name; unsigned int namelen; iter = icmap_iter_init_r(map, "nodelist.node."); while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, name_str); if (res != 2) { continue; } /* ring0_addr is allowed as a fallback */ if (strcmp(name_str, "name") && strcmp(name_str, "ring0_addr")) { continue; } if (icmap_get_string_r(map, iter_key, &name) != CS_OK) { continue; } namelen = strlen(name); if (strip_domain) { char *dot; dot = strchr(name, '.'); if (dot) { namelen = name - dot - 1; } } if (strncmp(find_name, name, namelen) == 0 && strlen(find_name) == strlen(name)) { icmap_iter_finalize(iter); return node_pos; } } icmap_iter_finalize(iter); return -1; } /* Compare two addresses - only address part (sin_addr/sin6_addr) is checked */ static int ipaddr_equal(const struct sockaddr *addr1, const struct sockaddr *addr2) { int addrlen = 0; const void *addr1p, *addr2p; if (addr1->sa_family != addr2->sa_family) return 0; switch (addr1->sa_family) { case AF_INET: addrlen = sizeof(struct in_addr); addr1p = &((struct sockaddr_in *)addr1)->sin_addr; addr2p = &((struct sockaddr_in *)addr2)->sin_addr; break; case AF_INET6: addrlen = sizeof(struct in6_addr); addr1p = &((struct sockaddr_in6 *)addr1)->sin6_addr; addr2p = &((struct sockaddr_in6 *)addr2)->sin6_addr; break; default: assert(0); } return (memcmp(addr1p, addr2p, addrlen) == 0); } /* Finds the local node and returns its position in the nodelist. * Uses nodelist.local_node_pos as a cache to save effort */ static int find_local_node(icmap_map_t map, int use_cache) { char nodename2[PATH_MAX]; char name_str[ICMAP_KEYNAME_MAXLEN]; icmap_iter_t iter; const char *iter_key; unsigned int cached_pos; char *dot = NULL; const char *node; struct ifaddrs *ifa, *ifa_list; struct sockaddr *sa; int found = 0; int node_pos = -1; int res; struct utsname utsname; /* Check for cached value first */ if (use_cache) { if (icmap_get_uint32("nodelist.local_node_pos", &cached_pos) == CS_OK) { return cached_pos; } } res = uname(&utsname); if (res) { return -1; } node = utsname.nodename; /* 1. Exact match */ node_pos = nodelist_byname(map, node, 0); if (node_pos > -1) { found = 1; goto ret_found; } /* 2. Try to match with increasingly more * specific versions of it */ strcpy(nodename2, node); dot = strrchr(nodename2, '.'); while (dot) { *dot = '\0'; node_pos = nodelist_byname(map, nodename2, 0); if (node_pos > -1) { found = 1; goto ret_found; } dot = strrchr(nodename2, '.'); } node_pos = nodelist_byname(map, nodename2, 1); if (node_pos > -1) { found = 1; goto ret_found; } /* * The corosync.conf name may not be related to uname at all, * they may match a hostname on some network interface. */ if (getifaddrs(&ifa_list)) return -1; for (ifa = ifa_list; ifa; ifa = ifa->ifa_next) { socklen_t salen = 0; /* Restore this */ strcpy(nodename2, node); sa = ifa->ifa_addr; if (!sa) { continue; } if (sa->sa_family != AF_INET && sa->sa_family != AF_INET6) { continue; } if (sa->sa_family == AF_INET) { salen = sizeof(struct sockaddr_in); } if (sa->sa_family == AF_INET6) { salen = sizeof(struct sockaddr_in6); } if (getnameinfo(sa, salen, nodename2, sizeof(nodename2), NULL, 0, 0) == 0) { node_pos = nodelist_byname(map, nodename2, 0); if (node_pos > -1) { found = 1; goto out; } /* Truncate this name and try again */ dot = strchr(nodename2, '.'); if (dot) { *dot = '\0'; node_pos = nodelist_byname(map, nodename2, 0); if (node_pos > -1) { found = 1; goto out; } } } /* See if it's the IP address that's in corosync.conf */ if (getnameinfo(sa, sizeof(*sa), nodename2, sizeof(nodename2), NULL, 0, NI_NUMERICHOST)) continue; node_pos = nodelist_byname(map, nodename2, 0); if (node_pos > -1) { found = 1; goto out; } } out: if (found) { freeifaddrs(ifa_list); goto ret_found; } /* * This section covers the usecase where the nodename specified in cluster.conf * is an alias specified in /etc/hosts. For example: * hostname alias1 alias2 * and * the above calls use uname and getnameinfo does not return aliases. * here we take the name specified in cluster.conf, resolve it to an address * and then compare against all known local ip addresses. * if we have a match, we found our nodename. In theory this chunk of code * could replace all the checks above, but let's avoid any possible regressions * and use it as last. */ iter = icmap_iter_init_r(map, "nodelist.node."); while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { char *dbnodename = NULL; struct addrinfo hints; struct addrinfo *result = NULL, *rp = NULL; res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, name_str); if (res != 2) { continue; } /* 'ring0_addr' is allowed as a fallback, but 'name' will be found first * because the names are in alpha order. */ if (strcmp(name_str, "name") && strcmp(name_str, "ring0_addr")) { continue; } if (icmap_get_string_r(map, iter_key, &dbnodename) != CS_OK) { continue; } memset(&hints, 0, sizeof(struct addrinfo)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_DGRAM; hints.ai_flags = 0; hints.ai_protocol = IPPROTO_UDP; if (getaddrinfo(dbnodename, NULL, &hints, &result)) { continue; } for (rp = result; rp != NULL; rp = rp->ai_next) { for (ifa = ifa_list; ifa; ifa = ifa->ifa_next) { if (ifa->ifa_addr && ipaddr_equal(rp->ai_addr, ifa->ifa_addr)) { freeaddrinfo(result); found = 1; goto out2; } } } freeaddrinfo(result); } out2: icmap_iter_finalize(iter); freeifaddrs(ifa_list); ret_found: if (found) { res = icmap_set_uint32_r(map, "nodelist.local_node_pos", node_pos); } return node_pos; } static enum totem_ip_version_enum totem_config_get_ip_version(struct totem_config *totem_config) { enum totem_ip_version_enum res; char *str; res = TOTEM_IP_VERSION_6_4; if (totem_config->transport_number == TOTEM_TRANSPORT_UDP) { res = TOTEM_IP_VERSION_4; } if (icmap_get_string("totem.ip_version", &str) == CS_OK) { if (strcmp(str, "ipv4") == 0) { res = TOTEM_IP_VERSION_4; } if (strcmp(str, "ipv6") == 0) { res = TOTEM_IP_VERSION_6; } if (strcmp(str, "ipv6-4") == 0) { res = TOTEM_IP_VERSION_6_4; } if (strcmp(str, "ipv4-6") == 0) { res = TOTEM_IP_VERSION_4_6; } free(str); } return (res); } static uint16_t generate_cluster_id (const char *cluster_name) { int i; int value = 0; for (i = 0; i < strlen(cluster_name); i++) { value <<= 1; value += cluster_name[i]; } return (value & 0xFFFF); } static int get_cluster_mcast_addr ( const char *cluster_name, unsigned int linknumber, enum totem_ip_version_enum ip_version, struct totem_ip_address *res) { uint16_t clusterid; char addr[INET6_ADDRSTRLEN + 1]; int err; if (cluster_name == NULL) { return (-1); } clusterid = generate_cluster_id(cluster_name) + linknumber; memset (res, 0, sizeof(*res)); switch (ip_version) { case TOTEM_IP_VERSION_4: case TOTEM_IP_VERSION_4_6: snprintf(addr, sizeof(addr), "239.192.%d.%d", clusterid >> 8, clusterid % 0xFF); break; case TOTEM_IP_VERSION_6: case TOTEM_IP_VERSION_6_4: snprintf(addr, sizeof(addr), "ff15::%x", clusterid); break; default: /* * Unknown family */ return (-1); } err = totemip_parse (res, addr, ip_version); return (err); } static unsigned int generate_nodeid( struct totem_config *totem_config, char *addr) { unsigned int nodeid; struct totem_ip_address totemip; /* AF_INET hard-coded here because auto-generated nodeids are only for IPv4 */ if (totemip_parse(&totemip, addr, TOTEM_IP_VERSION_4) != 0) return -1; memcpy (&nodeid, &totemip.addr, sizeof (unsigned int)); #if __BYTE_ORDER == __LITTLE_ENDIAN nodeid = swab32 (nodeid); #endif if (totem_config->clear_node_high_bit) { nodeid &= 0x7FFFFFFF; } return nodeid; } static int check_for_duplicate_nodeids( struct totem_config *totem_config, const char **error_string) { icmap_iter_t iter; icmap_iter_t subiter; const char *iter_key; int res = 0; int retval = 0; char tmp_key[ICMAP_KEYNAME_MAXLEN]; char *ring0_addr=NULL; char *ring0_addr1=NULL; unsigned int node_pos; unsigned int node_pos1; unsigned int last_node_pos = -1; unsigned int nodeid; unsigned int nodeid1; int autogenerated; iter = icmap_iter_init("nodelist.node."); while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, tmp_key); if (res != 2) { continue; } /* * This relies on the fact the icmap keys are always returned in order * so all of the keys for a node will be grouped together. We're basically * just running the code below once for each node. */ if (last_node_pos == node_pos) { continue; } last_node_pos = node_pos; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.nodeid", node_pos); autogenerated = 0; /* Generated nodeids are only allowed for UDP/UDPU so ring0_addr is valid here */ if (icmap_get_uint32(tmp_key, &nodeid) != CS_OK) { snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring0_addr", node_pos); if (icmap_get_string(tmp_key, &ring0_addr) != CS_OK) { continue; } /* Generate nodeid so we can check that auto-generated nodeids don't clash either */ nodeid = generate_nodeid(totem_config, ring0_addr); if (nodeid == -1) { continue; } autogenerated = 1; } node_pos1 = 0; subiter = icmap_iter_init("nodelist.node."); while (((iter_key = icmap_iter_next(subiter, NULL, NULL)) != NULL) && (node_pos1 < node_pos)) { res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos1, tmp_key); if ((res != 2) || (node_pos1 >= node_pos)) { continue; } if (strcmp(tmp_key, "nodeid") != 0) { continue; } snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.nodeid", node_pos1); if (icmap_get_uint32(tmp_key, &nodeid1) != CS_OK) { snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring0_addr", node_pos1); if (icmap_get_string(tmp_key, &ring0_addr1) != CS_OK) { continue; } nodeid1 = generate_nodeid(totem_config, ring0_addr1); if (nodeid1 == -1) { continue; } } if (nodeid == nodeid1) { retval = -1; snprintf (error_string_response, sizeof(error_string_response), "Nodeid %u%s%s%s appears twice in corosync.conf", nodeid, autogenerated?"(autogenerated from ":"", autogenerated?ring0_addr:"", autogenerated?")":""); log_printf (LOGSYS_LEVEL_ERROR, error_string_response); *error_string = error_string_response; break; } } icmap_iter_finalize(subiter); } icmap_iter_finalize(iter); return retval; } /* * This needs to be done last of all. It would be nice to do it when reading the * interface params, but the totem params need to have them to be read first. We * need both, so this is a way round that circular dependancy. */ static void calc_knet_ping_timers(struct totem_config *totem_config) { char runtime_key_name[ICMAP_KEYNAME_MAXLEN]; int interface; for (interface = 0; interface < INTERFACE_MAX; interface++) { if (totem_config->interfaces[interface].configured) { if (!totem_config->interfaces[interface].knet_pong_count) { totem_config->interfaces[interface].knet_pong_count = KNET_PONG_COUNT; } if (!totem_config->interfaces[interface].knet_ping_timeout) { totem_config->interfaces[interface].knet_ping_timeout = totem_config->token_timeout / totem_config->interfaces[interface].knet_pong_count; } snprintf(runtime_key_name, sizeof(runtime_key_name), "runtime.config.totem.interface.%d.knet_ping_timeout", interface); icmap_set_uint32(runtime_key_name, totem_config->interfaces[interface].knet_ping_timeout); if (!totem_config->interfaces[interface].knet_ping_interval) { totem_config->interfaces[interface].knet_ping_interval = totem_config->token_timeout / (totem_config->interfaces[interface].knet_pong_count * 2); } snprintf(runtime_key_name, sizeof(runtime_key_name), "runtime.config.totem.interface.%d.knet_ping_interval", interface); icmap_set_uint32(runtime_key_name, totem_config->interfaces[interface].knet_ping_interval); } } } /* * Compute difference between two set of totem interface arrays and commit it. * set1 and set2 * are changed so for same ring, ip existing in both set1 and set2 are cleared * (set to 0), and ips which are only in set1 or set2 remains untouched. * totempg_node_add/remove is called. */ static void compute_and_set_totempg_interfaces(struct totem_interface *set1, struct totem_interface *set2) { int ring_no, set1_pos, set2_pos; struct totem_ip_address empty_ip_address; memset(&empty_ip_address, 0, sizeof(empty_ip_address)); for (ring_no = 0; ring_no < INTERFACE_MAX; ring_no++) { if (!set1[ring_no].configured && !set2[ring_no].configured) { continue; } for (set1_pos = 0; set1_pos < set1[ring_no].member_count; set1_pos++) { for (set2_pos = 0; set2_pos < set2[ring_no].member_count; set2_pos++) { /* * For current ring_no remove all set1 items existing * in set2 */ if (memcmp(&set1[ring_no].member_list[set1_pos], &set2[ring_no].member_list[set2_pos], sizeof(struct totem_ip_address)) == 0) { memset(&set1[ring_no].member_list[set1_pos], 0, sizeof(struct totem_ip_address)); memset(&set2[ring_no].member_list[set2_pos], 0, sizeof(struct totem_ip_address)); } } } } for (ring_no = 0; ring_no < INTERFACE_MAX; ring_no++) { for (set1_pos = 0; set1_pos < set1[ring_no].member_count; set1_pos++) { /* * All items which remain in set1 and don't exist in set2 any more * have to be removed. */ if (memcmp(&set1[ring_no].member_list[set1_pos], &empty_ip_address, sizeof(empty_ip_address)) != 0) { log_printf(LOGSYS_LEVEL_DEBUG, "removing dynamic member %s for ring %u", totemip_print(&set1[ring_no].member_list[set1_pos]), ring_no); totempg_member_remove(&set1[ring_no].member_list[set1_pos], ring_no); } } if (!set2[ring_no].configured) { continue; } for (set2_pos = 0; set2_pos < set2[ring_no].member_count; set2_pos++) { /* * All items which remain in set2 and don't exist in set1 are new nodes * and have to be added. */ if (memcmp(&set2[ring_no].member_list[set2_pos], &empty_ip_address, sizeof(empty_ip_address)) != 0) { log_printf(LOGSYS_LEVEL_DEBUG, "adding dynamic member %s for ring %u", totemip_print(&set2[ring_no].member_list[set2_pos]), ring_no); totempg_member_add(&set2[ring_no].member_list[set2_pos], ring_no); } } } } /* * Configure parameters for links */ static void configure_link_params(struct totem_config *totem_config, icmap_map_t map) { int i; char tmp_key[ICMAP_KEYNAME_MAXLEN]; char *addr_string; int err; int local_node_pos = find_local_node(map, 0); for (i = 0; iinterfaces[i].configured) { continue; } log_printf(LOGSYS_LEVEL_DEBUG, "Configuring link %d params\n", i); snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring%u_addr", local_node_pos, i); if (icmap_get_string_r(map, tmp_key, &addr_string) != CS_OK) { continue; } err = totemip_parse(&totem_config->interfaces[i].local_ip, addr_string, totem_config->ip_version); if (err != 0) { continue; } totem_config->interfaces[i].local_ip.nodeid = totem_config->node_id; /* In case this is a new link, fill in the defaults if there was no interface{} section for it */ if (!totem_config->interfaces[i].knet_link_priority) totem_config->interfaces[i].knet_link_priority = 1; /* knet_ping_interval & knet_ping_timeout are set later once we know all the other params */ if (!totem_config->interfaces[i].knet_ping_precision) totem_config->interfaces[i].knet_ping_precision = KNET_PING_PRECISION; if (!totem_config->interfaces[i].knet_pong_count) totem_config->interfaces[i].knet_pong_count = KNET_PONG_COUNT; if (!totem_config->interfaces[i].knet_transport) totem_config->interfaces[i].knet_transport = KNET_TRANSPORT_UDP; if (!totem_config->interfaces[i].ip_port) totem_config->interfaces[i].ip_port = DEFAULT_PORT + i; } } static void configure_totem_links(struct totem_config *totem_config, icmap_map_t map) { int i; for (i = 0; iinterfaces[i].configured) { continue; } log_printf(LOGSYS_LEVEL_INFO, "Configuring link %d\n", i); totempg_iface_set(&totem_config->interfaces[i].local_ip, totem_config->interfaces[i].ip_port, i); } } /* Check for differences in config that can't be done on-the-fly and print an error */ static int check_things_have_not_changed(struct totem_config *totem_config, const char **error_string) { int i,j,k; const char *ip_str; char addr_buf[INET6_ADDRSTRLEN]; int changed = 0; for (i = 0; iinterfaces[i].configured && totem_config->orig_interfaces[i].configured) { if (totem_config->interfaces[i].knet_transport != totem_config->orig_interfaces[i].knet_transport) { log_printf(LOGSYS_LEVEL_ERROR, "New config has different knet transport for link %d. Internal value was NOT changed.\n", i); changed = 1; } /* Check each nodeid in the new configuration and make sure its IP address on this link has not changed */ for (j=0; j < totem_config->interfaces[i].member_count; j++) { for (k=0; k < totem_config->orig_interfaces[i].member_count; k++) { if (totem_config->interfaces[i].member_list[j].nodeid == totem_config->orig_interfaces[i].member_list[k].nodeid) { /* Found our nodeid - check the IP address */ if (memcmp(&totem_config->interfaces[i].member_list[j], &totem_config->orig_interfaces[i].member_list[k], sizeof(struct totem_ip_address))) { ip_str = totemip_print(&totem_config->orig_interfaces[i].member_list[k]); /* if ip_str is NULL then the old address was invalid and is allowed to change */ if (ip_str) { strncpy(addr_buf, ip_str, sizeof(addr_buf)); addr_buf[sizeof(addr_buf) - 1] = '\0'; log_printf(LOGSYS_LEVEL_ERROR, "new config has different address for link %d (addr changed from %s to %s). Internal value was NOT changed.\n", i, addr_buf, totemip_print(&totem_config->interfaces[i].member_list[j])); changed = 1; } } } } } } } if (changed) { snprintf (error_string_response, sizeof(error_string_response), "To reconfigure an interface it must be deleted and recreated. A working interface needs to be available to corosync at all times"); *error_string = error_string_response; return -1; } return 0; } static int put_nodelist_members_to_config(struct totem_config *totem_config, icmap_map_t map, int reload, const char **error_string) { icmap_iter_t iter, iter2; const char *iter_key, *iter_key2; int res = 0; unsigned int node_pos; char tmp_key[ICMAP_KEYNAME_MAXLEN]; char tmp_key2[ICMAP_KEYNAME_MAXLEN]; char *node_addr_str; int member_count; unsigned int linknumber = 0; int i, j; int last_node_pos = -1; /* Clear out nodelist so we can put the new one in if needed */ for (i = 0; i < INTERFACE_MAX; i++) { for (j = 0; j < PROCESSOR_COUNT_MAX; j++) { memset(&totem_config->interfaces[i].member_list[j], 0, sizeof(struct totem_ip_address)); } totem_config->interfaces[i].member_count = 0; } iter = icmap_iter_init_r(map, "nodelist.node."); while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, tmp_key); if (res != 2) { continue; } /* If it's the same as the last node_pos then skip it */ if (node_pos == last_node_pos) { continue; } last_node_pos = node_pos; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.", node_pos); iter2 = icmap_iter_init_r(map, tmp_key); while ((iter_key2 = icmap_iter_next(iter2, NULL, NULL)) != NULL) { unsigned int nodeid; char *str; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.nodeid", node_pos); if (icmap_get_uint32_r(map, tmp_key, &nodeid) != CS_OK) { nodeid = 0; } res = sscanf(iter_key2, "nodelist.node.%u.ring%u%s", &node_pos, &linknumber, tmp_key2); if (res != 3 || strcmp(tmp_key2, "_addr") != 0) { continue; } if (icmap_get_string_r(map, iter_key2, &node_addr_str) != CS_OK) { continue; } /* Generate nodeids if they are not provided and transport is UDP/U */ if (!nodeid && (totem_config->transport_number == TOTEM_TRANSPORT_UDP || totem_config->transport_number == TOTEM_TRANSPORT_UDPU)) { snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring0_addr", node_pos); if (icmap_get_string_r(map, tmp_key, &str) == CS_OK) { nodeid = generate_nodeid(totem_config, str); if (nodeid == -1) { sprintf(error_string_response, "An IPV6 network requires that a node ID be specified " "for address '%s'.", node_addr_str); *error_string = error_string_response; free(str); return (-1); } log_printf(LOGSYS_LEVEL_DEBUG, "Generated nodeid = " CS_PRI_NODE_ID " for %s", nodeid, str); free(str); } } member_count = totem_config->interfaces[linknumber].member_count; res = totemip_parse(&totem_config->interfaces[linknumber].member_list[member_count], node_addr_str, totem_config->ip_version); if (res == 0) { totem_config->interfaces[linknumber].member_list[member_count].nodeid = nodeid; totem_config->interfaces[linknumber].member_count++; totem_config->interfaces[linknumber].configured = 1; } else { sprintf(error_string_response, "failed to parse node address '%s'\n", node_addr_str); *error_string = error_string_response; memset(&totem_config->interfaces[linknumber].member_list[member_count], 0, sizeof(struct totem_ip_address)); free(node_addr_str); icmap_iter_finalize(iter2); icmap_iter_finalize(iter); return -1; } free(node_addr_str); } icmap_iter_finalize(iter2); } icmap_iter_finalize(iter); configure_link_params(totem_config, map); if (reload) { log_printf(LOGSYS_LEVEL_DEBUG, "About to reconfigure links from nodelist.\n"); if (check_things_have_not_changed(totem_config, error_string) == -1) { return -1; } } return 0; } static void config_convert_nodelist_to_interface(icmap_map_t map, struct totem_config *totem_config) { int res = 0; int node_pos; char tmp_key[ICMAP_KEYNAME_MAXLEN]; char tmp_key2[ICMAP_KEYNAME_MAXLEN]; char *node_addr_str; unsigned int linknumber = 0; icmap_iter_t iter; const char *iter_key; node_pos = find_local_node(map, 1); if (node_pos > -1) { /* * We found node, so create interface section */ snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.", node_pos); iter = icmap_iter_init_r(map, tmp_key); while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { res = sscanf(iter_key, "nodelist.node.%u.ring%u%s", &node_pos, &linknumber, tmp_key2); if (res != 3 || strcmp(tmp_key2, "_addr") != 0) { continue ; } if (icmap_get_string_r(map, iter_key, &node_addr_str) != CS_OK) { continue; } snprintf(tmp_key2, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.bindnetaddr", linknumber); icmap_set_string_r(map, tmp_key2, node_addr_str); free(node_addr_str); } icmap_iter_finalize(iter); } } static int get_interface_params(struct totem_config *totem_config, icmap_map_t map, const char **error_string, uint64_t *warnings, int reload) { int res = 0; unsigned int linknumber = 0; int member_count = 0; int i; icmap_iter_t iter, member_iter; const char *iter_key; const char *member_iter_key; char linknumber_key[ICMAP_KEYNAME_MAXLEN]; char tmp_key[ICMAP_KEYNAME_MAXLEN]; uint8_t u8; uint32_t u32; char *str; char *cluster_name = NULL; enum totem_ip_version_enum tmp_ip_version = TOTEM_IP_VERSION_4; int ret = 0; if (reload) { for (i=0; iinterfaces[i].configured = 0; totem_config->interfaces[i].knet_ping_timeout = 0; totem_config->interfaces[i].knet_ping_interval = 0; totem_config->interfaces[i].knet_ping_precision = KNET_PING_PRECISION; totem_config->interfaces[i].knet_pong_count = KNET_PONG_COUNT; } } if (icmap_get_string_r(map, "totem.cluster_name", &cluster_name) != CS_OK) { cluster_name = NULL; } iter = icmap_iter_init_r(map, "totem.interface."); while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { res = sscanf(iter_key, "totem.interface.%[^.].%s", linknumber_key, tmp_key); if (res != 2) { continue; } if (strcmp(tmp_key, "bindnetaddr") != 0 && totem_config->transport_number == TOTEM_TRANSPORT_UDP) { continue; } member_count = 0; linknumber = atoi(linknumber_key); if (linknumber >= INTERFACE_MAX) { snprintf (error_string_response, sizeof(error_string_response), "parse error in config: interface ring number %u is bigger than allowed maximum %u\n", linknumber, INTERFACE_MAX - 1); *error_string = error_string_response; ret = -1; goto out; } /* These things are only valid for the initial read */ if (!reload) { /* * Get the bind net address */ snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.bindnetaddr", linknumber); if (icmap_get_string_r(map, tmp_key, &str) == CS_OK) { res = totemip_parse (&totem_config->interfaces[linknumber].bindnet, str, totem_config->ip_version); if (res) { sprintf(error_string_response, "failed to parse bindnet address '%s'\n", str); *error_string = error_string_response; free(str); ret = -1; goto out; } free(str); } /* * Get interface multicast address */ snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastaddr", linknumber); if (icmap_get_string_r(map, tmp_key, &str) == CS_OK) { res = totemip_parse (&totem_config->interfaces[linknumber].mcast_addr, str, totem_config->ip_version); if (res) { sprintf(error_string_response, "failed to parse mcast address '%s'\n", str); *error_string = error_string_response; free(str); ret = -1; goto out; } free(str); } else if (totem_config->transport_number == TOTEM_TRANSPORT_UDP) { /* * User not specified address -> autogenerate one from cluster_name key * (if available). Return code is intentionally ignored, because * udpu doesn't need mcastaddr and validity of mcastaddr for udp is * checked later anyway. */ if (totem_config->interfaces[0].bindnet.family == AF_INET) { tmp_ip_version = TOTEM_IP_VERSION_4; } else if (totem_config->interfaces[0].bindnet.family == AF_INET6) { tmp_ip_version = TOTEM_IP_VERSION_6; } (void)get_cluster_mcast_addr (cluster_name, linknumber, tmp_ip_version, &totem_config->interfaces[linknumber].mcast_addr); } snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.broadcast", linknumber); if (icmap_get_string(tmp_key, &str) == CS_OK) { if (strcmp (str, "yes") == 0) { totem_config->broadcast_use = 1; } free(str); } } /* These things are only valid for the initial read OR a newly-defined link */ if (!reload || (totem_config->interfaces[linknumber].configured == 0)) { /* * Get mcast port */ snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastport", linknumber); if (icmap_get_uint16_r(map, tmp_key, &totem_config->interfaces[linknumber].ip_port) != CS_OK) { if (totem_config->broadcast_use) { totem_config->interfaces[linknumber].ip_port = DEFAULT_PORT + (2 * linknumber); } else { totem_config->interfaces[linknumber].ip_port = DEFAULT_PORT + linknumber; } } /* * Get the TTL */ totem_config->interfaces[linknumber].ttl = 1; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.ttl", linknumber); if (icmap_get_uint8_r(map, tmp_key, &u8) == CS_OK) { totem_config->interfaces[linknumber].ttl = u8; } totem_config->interfaces[linknumber].knet_transport = KNET_DEFAULT_TRANSPORT; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_transport", linknumber); if (icmap_get_string_r(map, tmp_key, &str) == CS_OK) { if (strcmp(str, "sctp") == 0) { totem_config->interfaces[linknumber].knet_transport = KNET_TRANSPORT_SCTP; } else if (strcmp(str, "udp") == 0) { totem_config->interfaces[linknumber].knet_transport = KNET_TRANSPORT_UDP; } else { *error_string = "Unrecognised knet_transport. expected 'udp' or 'sctp'"; ret = -1; goto out; } } } totem_config->interfaces[linknumber].configured = 1; /* * Get the knet link params */ totem_config->interfaces[linknumber].knet_link_priority = 1; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_link_priority", linknumber); if (icmap_get_uint8_r(map, tmp_key, &u8) == CS_OK) { totem_config->interfaces[linknumber].knet_link_priority = u8; } totem_config->interfaces[linknumber].knet_ping_interval = 0; /* real default applied later */ snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_interval", linknumber); if (icmap_get_uint32_r(map, tmp_key, &u32) == CS_OK) { totem_config->interfaces[linknumber].knet_ping_interval = u32; } totem_config->interfaces[linknumber].knet_ping_timeout = 0; /* real default applied later */ snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_timeout", linknumber); if (icmap_get_uint32_r(map, tmp_key, &u32) == CS_OK) { totem_config->interfaces[linknumber].knet_ping_timeout = u32; } totem_config->interfaces[linknumber].knet_ping_precision = KNET_PING_PRECISION; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_precision", linknumber); if (icmap_get_uint32_r(map, tmp_key, &u32) == CS_OK) { totem_config->interfaces[linknumber].knet_ping_precision = u32; } totem_config->interfaces[linknumber].knet_pong_count = KNET_PONG_COUNT; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_pong_count", linknumber); if (icmap_get_uint32_r(map, tmp_key, &u32) == CS_OK) { totem_config->interfaces[linknumber].knet_pong_count = u32; } snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.member.", linknumber); member_iter = icmap_iter_init_r(map, tmp_key); while ((member_iter_key = icmap_iter_next(member_iter, NULL, NULL)) != NULL) { if (member_count == 0) { if (icmap_get_string("nodelist.node.0.ring0_addr", &str) == CS_OK) { free(str); *warnings |= TOTEM_CONFIG_WARNING_MEMBERS_IGNORED; break; } else { *warnings |= TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED; } } if (icmap_get_string_r(map, member_iter_key, &str) == CS_OK) { res = totemip_parse (&totem_config->interfaces[linknumber].member_list[member_count++], str, totem_config->ip_version); if (res) { sprintf(error_string_response, "failed to parse node address '%s'\n", str); *error_string = error_string_response; icmap_iter_finalize(member_iter); free(str); ret = -1; goto out; } free(str); } } icmap_iter_finalize(member_iter); totem_config->interfaces[linknumber].member_count = member_count; } out: icmap_iter_finalize(iter); free(cluster_name); return (ret); } extern int totem_config_read ( struct totem_config *totem_config, const char **error_string, uint64_t *warnings) { int res = 0; char *str, *ring0_addr_str; char tmp_key[ICMAP_KEYNAME_MAXLEN]; uint16_t u16; int i; int local_node_pos; int nodeid_set; *warnings = 0; memset (totem_config, 0, sizeof (struct totem_config)); totem_config->interfaces = malloc (sizeof (struct totem_interface) * INTERFACE_MAX); if (totem_config->interfaces == 0) { *error_string = "Out of memory trying to allocate ethernet interface storage area"; return -1; } totem_config->transport_number = TOTEM_TRANSPORT_KNET; if (icmap_get_string("totem.transport", &str) == CS_OK) { if (strcmp (str, "udpu") == 0) { totem_config->transport_number = TOTEM_TRANSPORT_UDPU; } if (strcmp (str, "udp") == 0) { totem_config->transport_number = TOTEM_TRANSPORT_UDP; } if (strcmp (str, "knet") == 0) { totem_config->transport_number = TOTEM_TRANSPORT_KNET; } free(str); } memset (totem_config->interfaces, 0, sizeof (struct totem_interface) * INTERFACE_MAX); strcpy (totem_config->link_mode, "passive"); icmap_get_uint32("totem.version", (uint32_t *)&totem_config->version); if (totem_get_crypto(totem_config, error_string) != 0) { return -1; } if (icmap_get_string("totem.link_mode", &str) == CS_OK) { if (strlen(str) >= TOTEM_LINK_MODE_BYTES) { *error_string = "totem.link_mode is too long"; free(str); return -1; } strcpy (totem_config->link_mode, str); free(str); } icmap_get_uint32("totem.nodeid", &totem_config->node_id); totem_config->clear_node_high_bit = 0; if (icmap_get_string("totem.clear_node_high_bit", &str) == CS_OK) { if (strcmp (str, "yes") == 0) { totem_config->clear_node_high_bit = 1; } free(str); } icmap_get_uint32("totem.threads", &totem_config->threads); icmap_get_uint32("totem.netmtu", &totem_config->net_mtu); totem_config->ip_version = totem_config_get_ip_version(totem_config); if (icmap_get_string("totem.interface.0.bindnetaddr", &str) != CS_OK) { /* * We were not able to find ring 0 bindnet addr. Try to use nodelist informations */ config_convert_nodelist_to_interface(icmap_get_global_map(), totem_config); } else { if (icmap_get_string("nodelist.node.0.ring0_addr", &ring0_addr_str) == CS_OK) { /* * Both bindnetaddr and ring0_addr are set. * Log warning information, and use nodelist instead */ *warnings |= TOTEM_CONFIG_BINDNETADDR_NODELIST_SET; config_convert_nodelist_to_interface(icmap_get_global_map(), totem_config); free(ring0_addr_str); } free(str); } /* * Broadcast option is global but set in interface section, * so reset before processing interfaces. */ totem_config->broadcast_use = 0; res = get_interface_params(totem_config, icmap_get_global_map(), error_string, warnings, 0); if (res < 0) { return res; } /* * Use broadcast is global, so if set, make sure to fill mcast addr correctly * broadcast is only supported for UDP so just do interface 0; */ if (totem_config->broadcast_use) { totemip_parse (&totem_config->interfaces[0].mcast_addr, "255.255.255.255", TOTEM_IP_VERSION_4); } /* * Store automatically generated items back to icmap only for UDP */ if (totem_config->transport_number == TOTEM_TRANSPORT_UDP) { for (i = 0; i < INTERFACE_MAX; i++) { if (!totem_config->interfaces[i].configured) { continue; } snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastaddr", i); if (icmap_get_string(tmp_key, &str) == CS_OK) { free(str); } else { str = (char *)totemip_print(&totem_config->interfaces[i].mcast_addr); icmap_set_string(tmp_key, str); } snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastport", i); if (icmap_get_uint16(tmp_key, &u16) != CS_OK) { icmap_set_uint16(tmp_key, totem_config->interfaces[i].ip_port); } } } /* * Check existence of nodelist */ if ((icmap_get_string("nodelist.node.0.name", &str) == CS_OK) || (icmap_get_string("nodelist.node.0.ring0_addr", &str) == CS_OK)) { free(str); /* * find local node */ local_node_pos = find_local_node(icmap_get_global_map(), 1); if (local_node_pos != -1) { snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.nodeid", local_node_pos); nodeid_set = (totem_config->node_id != 0); if (icmap_get_uint32(tmp_key, &totem_config->node_id) == CS_OK && nodeid_set) { *warnings |= TOTEM_CONFIG_WARNING_TOTEM_NODEID_IGNORED; } if ((totem_config->transport_number == TOTEM_TRANSPORT_KNET) && (!totem_config->node_id)) { *error_string = "Knet requires an explicit nodeid for the local node"; return -1; } if ((totem_config->transport_number == TOTEM_TRANSPORT_UDP || totem_config->transport_number == TOTEM_TRANSPORT_UDPU) && (!totem_config->node_id)) { snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring0_addr", local_node_pos); icmap_get_string(tmp_key, &str); totem_config->node_id = generate_nodeid(totem_config, str); if (totem_config->node_id == -1) { *error_string = "An IPV6 network requires that a node ID be specified"; free(str); return (-1); } totem_config->interfaces[0].member_list[local_node_pos].nodeid = totem_config->node_id; free(str); } /* Users must not change this */ icmap_set_ro_access("nodelist.local_node_pos", 0, 1); } if (put_nodelist_members_to_config(totem_config, icmap_get_global_map(), 0, error_string)) { return -1; } } /* * Get things that might change in the future (and can depend on totem_config->interfaces); */ totem_volatile_config_read(totem_config, icmap_get_global_map(), NULL); calc_knet_ping_timers(totem_config); /* This is now done in the totemknet interface callback */ /* configure_totem_links(totem_config, icmap_get_global_map()); */ add_totem_config_notification(totem_config); return 0; } int totem_config_validate ( struct totem_config *totem_config, const char **error_string) { static char local_error_reason[512]; char parse_error[512]; - static char addr_str_buf[INET6_ADDRSTRLEN]; const char *error_reason = local_error_reason; - int i,j; + int i; uint32_t u32; int num_configured = 0; unsigned int interface_max = INTERFACE_MAX; for (i = 0; i < INTERFACE_MAX; i++) { if (totem_config->interfaces[i].configured) { num_configured++; } } if (num_configured == 0) { error_reason = "No interfaces defined"; goto parse_error; } /* Check we found a local node name */ if (icmap_get_uint32("nodelist.local_node_pos", &u32) != CS_OK) { error_reason = "No valid name found for local host"; goto parse_error; } for (i = 0; i < INTERFACE_MAX; i++) { /* * Some error checking of parsed data to make sure its valid */ struct totem_ip_address null_addr; if (!totem_config->interfaces[i].configured) { continue; } memset (&null_addr, 0, sizeof (struct totem_ip_address)); if ((totem_config->transport_number == TOTEM_TRANSPORT_UDP) && memcmp (&totem_config->interfaces[i].mcast_addr, &null_addr, sizeof (struct totem_ip_address)) == 0) { error_reason = "No multicast address specified"; goto parse_error; } if (totem_config->interfaces[i].ip_port == 0) { error_reason = "No multicast port specified"; goto parse_error; } if (totem_config->interfaces[i].ttl > 255) { error_reason = "Invalid TTL (should be 0..255)"; goto parse_error; } if (totem_config->transport_number != TOTEM_TRANSPORT_UDP && totem_config->interfaces[i].ttl != 1) { error_reason = "Can only set ttl on multicast transport types"; goto parse_error; } if (totem_config->interfaces[i].knet_link_priority > 255) { error_reason = "Invalid link priority (should be 0..255)"; goto parse_error; } if (totem_config->transport_number != TOTEM_TRANSPORT_KNET && totem_config->interfaces[i].knet_link_priority != 1) { error_reason = "Can only set link priority on knet transport type"; goto parse_error; } if (totem_config->interfaces[i].mcast_addr.family == AF_INET6 && totem_config->node_id == 0) { error_reason = "An IPV6 network requires that a node ID be specified."; goto parse_error; } if (totem_config->broadcast_use == 0 && totem_config->transport_number == TOTEM_TRANSPORT_UDP) { if (totem_config->interfaces[i].mcast_addr.family != totem_config->interfaces[i].bindnet.family) { error_reason = "Multicast address family does not match bind address family"; goto parse_error; } if (totemip_is_mcast (&totem_config->interfaces[i].mcast_addr) != 0) { error_reason = "mcastaddr is not a correct multicast address."; goto parse_error; } } - /* Verify that all nodes on the same knet link have the same IP family */ - for (j=1; jinterfaces[i].member_count; j++) { - if (totem_config->interfaces[i].configured) { - if (totem_config->interfaces[i].member_list[j].family != - totem_config->interfaces[i].member_list[0].family) { - memcpy(addr_str_buf, - totemip_print(&(totem_config->interfaces[i].member_list[j])), - sizeof(addr_str_buf)); - - snprintf (local_error_reason, sizeof(local_error_reason), - "Nodes for link %d have different IP families " - "(compared %s with %s)", i, - addr_str_buf, - totemip_print(&(totem_config->interfaces[i].member_list[0]))); - goto parse_error; - } - } - } } if (totem_config->version != 2) { error_reason = "This totem parser can only parse version 2 configurations."; goto parse_error; } if (totem_volatile_config_validate(totem_config, icmap_get_global_map(), error_string) == -1) { return (-1); } if (check_for_duplicate_nodeids(totem_config, error_string) == -1) { return (-1); } /* * KNET Link values validation */ if (strcmp (totem_config->link_mode, "active") && strcmp (totem_config->link_mode, "rr") && strcmp (totem_config->link_mode, "passive")) { snprintf (local_error_reason, sizeof(local_error_reason), "The Knet link mode \"%s\" specified is invalid. It must be active, passive or rr.\n", totem_config->link_mode); goto parse_error; } /* Only Knet does multiple interfaces */ if (totem_config->transport_number != TOTEM_TRANSPORT_KNET) { interface_max = 1; } if (interface_max < num_configured) { snprintf (parse_error, sizeof(parse_error), "%d is too many configured interfaces for non-Knet transport.", num_configured); error_reason = parse_error; goto parse_error; } /* Only knet allows crypto */ if (totem_config->transport_number != TOTEM_TRANSPORT_KNET) { if ((strcmp(totem_config->crypto_cipher_type, "none") != 0) || (strcmp(totem_config->crypto_hash_type, "none") != 0)) { snprintf (parse_error, sizeof(parse_error), "crypto_cipher & crypto_hash are only valid for the Knet transport."); error_reason = parse_error; goto parse_error; } } if (totem_config->net_mtu == 0) { if (totem_config->transport_number == TOTEM_TRANSPORT_KNET) { totem_config->net_mtu = KNET_MAX_PACKET_SIZE; } else { totem_config->net_mtu = 1500; } } return 0; parse_error: snprintf (error_string_response, sizeof(error_string_response), "parse error in config: %s\n", error_reason); *error_string = error_string_response; return (-1); } static int read_keyfile ( const char *key_location, struct totem_config *totem_config, const char **error_string) { int fd; int res; int saved_errno; char error_str[100]; const char *error_ptr; fd = open (key_location, O_RDONLY); if (fd == -1) { error_ptr = qb_strerror_r(errno, error_str, sizeof(error_str)); snprintf (error_string_response, sizeof(error_string_response), "Could not open %s: %s\n", key_location, error_ptr); goto parse_error; } res = read (fd, totem_config->private_key, TOTEM_PRIVATE_KEY_LEN_MAX); saved_errno = errno; close (fd); if (res == -1) { error_ptr = qb_strerror_r (saved_errno, error_str, sizeof(error_str)); snprintf (error_string_response, sizeof(error_string_response), "Could not read %s: %s\n", key_location, error_ptr); goto parse_error; } if (res < TOTEM_PRIVATE_KEY_LEN_MIN) { snprintf (error_string_response, sizeof(error_string_response), "Could only read %d bits of minimum %u bits from %s.\n", res * 8, TOTEM_PRIVATE_KEY_LEN_MIN * 8, key_location); goto parse_error; } totem_config->private_key_len = res; return 0; parse_error: *error_string = error_string_response; return (-1); } int totem_config_keyread ( struct totem_config *totem_config, const char **error_string) { int got_key = 0; char *key_location = NULL; int res; size_t key_len; memset (totem_config->private_key, 0, sizeof(totem_config->private_key)); totem_config->private_key_len = 0; if (strcmp(totem_config->crypto_cipher_type, "none") == 0 && strcmp(totem_config->crypto_hash_type, "none") == 0) { return (0); } /* cmap may store the location of the key file */ if (icmap_get_string("totem.keyfile", &key_location) == CS_OK) { res = read_keyfile(key_location, totem_config, error_string); free(key_location); if (res) { goto key_error; } got_key = 1; } else { /* Or the key itself may be in the cmap */ if (icmap_get("totem.key", NULL, &key_len, NULL) == CS_OK) { if (key_len > sizeof(totem_config->private_key)) { sprintf(error_string_response, "key is too long"); goto key_error; } if (key_len < TOTEM_PRIVATE_KEY_LEN_MIN) { sprintf(error_string_response, "key is too short"); goto key_error; } if (icmap_get("totem.key", totem_config->private_key, &key_len, NULL) == CS_OK) { totem_config->private_key_len = key_len; got_key = 1; } else { sprintf(error_string_response, "can't load private key"); goto key_error; } } } /* In desperation we read the default filename */ if (!got_key) { res = read_keyfile(COROSYSCONFDIR "/authkey", totem_config, error_string); if (res) goto key_error; } return (0); key_error: *error_string = error_string_response; return (-1); } static void debug_dump_totem_config(const struct totem_config *totem_config) { log_printf(LOGSYS_LEVEL_DEBUG, "Token Timeout (%d ms) retransmit timeout (%d ms)", totem_config->token_timeout, totem_config->token_retransmit_timeout); if (totem_config->token_warning) { uint32_t token_warning_ms = totem_config->token_warning * totem_config->token_timeout / 100; log_printf(LOGSYS_LEVEL_DEBUG, "Token warning every %d ms (%d%% of Token Timeout)", token_warning_ms, totem_config->token_warning); if (token_warning_ms < totem_config->token_retransmit_timeout) log_printf (LOGSYS_LEVEL_DEBUG, "The token warning interval (%d ms) is less than the token retransmit timeout (%d ms) " "which can lead to spurious token warnings. Consider increasing the token_warning parameter.", token_warning_ms, totem_config->token_retransmit_timeout); } else log_printf(LOGSYS_LEVEL_DEBUG, "Token warnings disabled"); log_printf(LOGSYS_LEVEL_DEBUG, "token hold (%d ms) retransmits before loss (%d retrans)", totem_config->token_hold_timeout, totem_config->token_retransmits_before_loss_const); log_printf(LOGSYS_LEVEL_DEBUG, "join (%d ms) send_join (%d ms) consensus (%d ms) merge (%d ms)", totem_config->join_timeout, totem_config->send_join_timeout, totem_config->consensus_timeout, totem_config->merge_timeout); log_printf(LOGSYS_LEVEL_DEBUG, "downcheck (%d ms) fail to recv const (%d msgs)", totem_config->downcheck_timeout, totem_config->fail_to_recv_const); log_printf(LOGSYS_LEVEL_DEBUG, "seqno unchanged const (%d rotations) Maximum network MTU %d", totem_config->seqno_unchanged_const, totem_config->net_mtu); log_printf(LOGSYS_LEVEL_DEBUG, "window size per rotation (%d messages) maximum messages per rotation (%d messages)", totem_config->window_size, totem_config->max_messages); log_printf(LOGSYS_LEVEL_DEBUG, "missed count const (%d messages)", totem_config->miss_count_const); log_printf(LOGSYS_LEVEL_DEBUG, "heartbeat_failures_allowed (%d)", totem_config->heartbeat_failures_allowed); log_printf(LOGSYS_LEVEL_DEBUG, "max_network_delay (%d ms)", totem_config->max_network_delay); } static void totem_change_notify( int32_t event, const char *key_name, struct icmap_notify_value new_val, struct icmap_notify_value old_val, void *user_data) { struct totem_config *totem_config = (struct totem_config *)user_data; uint32_t *param; uint8_t reloading; const char *deleted_key = NULL; const char *error_string; /* * If a full reload is in progress then don't do anything until it's done and * can reconfigure it all atomically */ if (icmap_get_uint8("config.reload_in_progress", &reloading) == CS_OK && reloading) return; param = totem_get_param_by_name((struct totem_config *)user_data, key_name); /* * Process change only if changed key is found in totem_config (-> param is not NULL) * or for special key token_coefficient. token_coefficient key is not stored in * totem_config, but it is used for computation of token timeout. */ if (!param && strcmp(key_name, "totem.token_coefficient") != 0) return; /* * Values other than UINT32 are not supported, or needed (yet) */ switch (event) { case ICMAP_TRACK_DELETE: deleted_key = key_name; break; case ICMAP_TRACK_ADD: case ICMAP_TRACK_MODIFY: deleted_key = NULL; break; default: break; } totem_volatile_config_read (totem_config, icmap_get_global_map(), deleted_key); log_printf(LOGSYS_LEVEL_DEBUG, "Totem related config key changed. Dumping actual totem config."); debug_dump_totem_config(totem_config); if (totem_volatile_config_validate(totem_config, icmap_get_global_map(), &error_string) == -1) { log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string); /* * TODO: Consider corosync exit and/or load defaults for volatile * values. For now, log error seems to be enough */ } } int totemconfig_configure_new_params( struct totem_config *totem_config, icmap_map_t map, const char **error_string) { uint64_t warnings = 0LL; get_interface_params(totem_config, map, error_string, &warnings, 1); if (put_nodelist_members_to_config (totem_config, map, 1, error_string)) { return -1; } calc_knet_ping_timers(totem_config); log_printf(LOGSYS_LEVEL_DEBUG, "Configuration reloaded. Dumping actual totem config."); debug_dump_totem_config(totem_config); /* Reinstate the local_node_pos */ (void)find_local_node(map, 0); return 0; } void totemconfig_commit_new_params( struct totem_config *totem_config, icmap_map_t map) { struct totem_interface *new_interfaces = NULL; new_interfaces = malloc (sizeof (struct totem_interface) * INTERFACE_MAX); assert(new_interfaces != NULL); memcpy(new_interfaces, totem_config->interfaces, sizeof (struct totem_interface) * INTERFACE_MAX); /* Set link parameters including local_ip */ configure_totem_links(totem_config, map); /* Add & remove nodes */ compute_and_set_totempg_interfaces(totem_config->orig_interfaces, new_interfaces); /* Does basic global params (like compression) */ totempg_reconfigure(); free(new_interfaces); } static void add_totem_config_notification(struct totem_config *totem_config) { icmap_track_t icmap_track; icmap_track_add("totem.", ICMAP_TRACK_ADD | ICMAP_TRACK_DELETE | ICMAP_TRACK_MODIFY | ICMAP_TRACK_PREFIX, totem_change_notify, totem_config, &icmap_track); } diff --git a/exec/totempg.c b/exec/totempg.c index bd629bb1..96d69372 100644 --- a/exec/totempg.c +++ b/exec/totempg.c @@ -1,1605 +1,1608 @@ /* * Copyright (c) 2003-2005 MontaVista Software, Inc. * Copyright (c) 2005 OSDL. * Copyright (c) 2006-2012 Red Hat, Inc. * * All rights reserved. * * Author: Steven Dake (sdake@redhat.com) * Author: Mark Haverkamp (markh@osdl.org) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the MontaVista Software, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ /* * FRAGMENTATION AND PACKING ALGORITHM: * * Assemble the entire message into one buffer * if full fragment * store fragment into lengths list * for each full fragment * multicast fragment * set length and fragment fields of pg mesage * store remaining multicast into head of fragmentation data and set lens field * * If a message exceeds the maximum packet size allowed by the totem * single ring protocol, the protocol could lose forward progress. * Statically calculating the allowed data amount doesn't work because * the amount of data allowed depends on the number of fragments in * each message. In this implementation, the maximum fragment size * is dynamically calculated for each fragment added to the message. * It is possible for a message to be two bytes short of the maximum * packet size. This occurs when a message or collection of * messages + the mcast header + the lens are two bytes short of the * end of the packet. Since another len field consumes two bytes, the * len field would consume the rest of the packet without room for data. * * One optimization would be to forgo the final len field and determine * it from the size of the udp datagram. Then this condition would no * longer occur. */ /* * ASSEMBLY AND UNPACKING ALGORITHM: * * copy incoming packet into assembly data buffer indexed by current * location of end of fragment * * if not fragmented * deliver all messages in assembly data buffer * else * if msg_count > 1 and fragmented * deliver all messages except last message in assembly data buffer * copy last fragmented section to start of assembly data buffer * else * if msg_count = 1 and fragmented * do nothing * */ #include #ifdef HAVE_ALLOCA_H #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define LOGSYS_UTILS_ONLY 1 #include #include "util.h" #include "totemsrp.h" struct totempg_mcast_header { short version; short type; }; #if !(defined(__i386__) || defined(__x86_64__)) /* * Need align on architectures different then i386 or x86_64 */ #define TOTEMPG_NEED_ALIGN 1 #endif /* * totempg_mcast structure * * header: Identify the mcast. * fragmented: Set if this message continues into next message * continuation: Set if this message is a continuation from last message * msg_count Indicates how many packed messages are contained * in the mcast. * Also, the size of each packed message and the messages themselves are * appended to the end of this structure when sent. */ struct totempg_mcast { struct totempg_mcast_header header; unsigned char fragmented; unsigned char continuation; unsigned short msg_count; /* * short msg_len[msg_count]; */ /* * data for messages */ }; /* * Maximum packet size for totem pg messages */ #define TOTEMPG_PACKET_SIZE (totempg_totem_config->net_mtu - \ sizeof (struct totempg_mcast)) /* * Local variables used for packing small messages */ static unsigned short mcast_packed_msg_lens[FRAME_SIZE_MAX]; static int mcast_packed_msg_count = 0; static int totempg_reserved = 1; static unsigned int totempg_size_limit; static totem_queue_level_changed_fn totem_queue_level_changed = NULL; static uint32_t totempg_threaded_mode = 0; static void *totemsrp_context; /* * Function and data used to log messages */ static int totempg_log_level_security; static int totempg_log_level_error; static int totempg_log_level_warning; static int totempg_log_level_notice; static int totempg_log_level_debug; static int totempg_subsys_id; static void (*totempg_log_printf) ( int level, int subsys, const char *function, const char *file, int line, const char *format, ...) __attribute__((format(printf, 6, 7))); struct totem_config *totempg_totem_config; static totempg_stats_t totempg_stats; enum throw_away_mode { THROW_AWAY_INACTIVE, THROW_AWAY_ACTIVE }; struct assembly { unsigned int nodeid; unsigned char data[MESSAGE_SIZE_MAX+KNET_MAX_PACKET_SIZE]; int index; unsigned char last_frag_num; enum throw_away_mode throw_away_mode; struct qb_list_head list; }; static void assembly_deref (struct assembly *assembly); static int callback_token_received_fn (enum totem_callback_token_type type, const void *data); QB_LIST_DECLARE(assembly_list_inuse); /* * Free list is used both for transitional and operational assemblies */ QB_LIST_DECLARE(assembly_list_free); QB_LIST_DECLARE(assembly_list_inuse_trans); QB_LIST_DECLARE(totempg_groups_list); /* * Staging buffer for packed messages. Messages are staged in this buffer * before sending. Multiple messages may fit which cuts down on the * number of mcasts sent. If a message doesn't completely fit, then * the mcast header has a fragment bit set that says that there are more * data to follow. fragment_size is an index into the buffer. It indicates * the size of message data and where to place new message data. * fragment_contuation indicates whether the first packed message in * the buffer is a continuation of a previously packed fragment. */ static unsigned char *fragmentation_data; static int fragment_size = 0; static int fragment_continuation = 0; static int totempg_waiting_transack = 0; struct totempg_group_instance { void (*deliver_fn) ( unsigned int nodeid, const void *msg, unsigned int msg_len, int endian_conversion_required); void (*confchg_fn) ( enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id); struct totempg_group *groups; int groups_cnt; int32_t q_level; struct qb_list_head list; }; static unsigned char next_fragment = 1; static pthread_mutex_t totempg_mutex = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t callback_token_mutex = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t mcast_msg_mutex = PTHREAD_MUTEX_INITIALIZER; #define log_printf(level, format, args...) \ do { \ totempg_log_printf(level, \ totempg_subsys_id, \ __FUNCTION__, __FILE__, __LINE__, \ format, ##args); \ } while (0); static int msg_count_send_ok (int msg_count); static int byte_count_send_ok (int byte_count); static void totempg_waiting_trans_ack_cb (int waiting_trans_ack) { log_printf(LOG_DEBUG, "waiting_trans_ack changed to %u", waiting_trans_ack); totempg_waiting_transack = waiting_trans_ack; } static struct assembly *assembly_ref (unsigned int nodeid) { struct assembly *assembly; struct qb_list_head *list; struct qb_list_head *active_assembly_list_inuse; if (totempg_waiting_transack) { active_assembly_list_inuse = &assembly_list_inuse_trans; } else { active_assembly_list_inuse = &assembly_list_inuse; } /* * Search inuse list for node id and return assembly buffer if found */ qb_list_for_each(list, active_assembly_list_inuse) { assembly = qb_list_entry (list, struct assembly, list); if (nodeid == assembly->nodeid) { return (assembly); } } /* * Nothing found in inuse list get one from free list if available */ if (qb_list_empty (&assembly_list_free) == 0) { assembly = qb_list_first_entry (&assembly_list_free, struct assembly, list); qb_list_del (&assembly->list); qb_list_add (&assembly->list, active_assembly_list_inuse); assembly->nodeid = nodeid; assembly->index = 0; assembly->last_frag_num = 0; assembly->throw_away_mode = THROW_AWAY_INACTIVE; return (assembly); } /* * Nothing available in inuse or free list, so allocate a new one */ assembly = malloc (sizeof (struct assembly)); /* * TODO handle memory allocation failure here */ assert (assembly); assembly->nodeid = nodeid; assembly->data[0] = 0; assembly->index = 0; assembly->last_frag_num = 0; assembly->throw_away_mode = THROW_AWAY_INACTIVE; qb_list_init (&assembly->list); qb_list_add (&assembly->list, active_assembly_list_inuse); return (assembly); } static void assembly_deref (struct assembly *assembly) { qb_list_del (&assembly->list); qb_list_add (&assembly->list, &assembly_list_free); } static void assembly_deref_from_normal_and_trans (int nodeid) { int j; struct qb_list_head *list, *tmp_iter; struct qb_list_head *active_assembly_list_inuse; struct assembly *assembly; for (j = 0; j < 2; j++) { if (j == 0) { active_assembly_list_inuse = &assembly_list_inuse; } else { active_assembly_list_inuse = &assembly_list_inuse_trans; } qb_list_for_each_safe(list, tmp_iter, active_assembly_list_inuse) { assembly = qb_list_entry (list, struct assembly, list); if (nodeid == assembly->nodeid) { qb_list_del (&assembly->list); qb_list_add (&assembly->list, &assembly_list_free); } } } } static inline void app_confchg_fn ( enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id) { int i; struct totempg_group_instance *instance; struct qb_list_head *list; /* * For every leaving processor, add to free list * This also has the side effect of clearing out the dataset * In the leaving processor's assembly buffer. */ for (i = 0; i < left_list_entries; i++) { assembly_deref_from_normal_and_trans (left_list[i]); } qb_list_for_each(list, &totempg_groups_list) { instance = qb_list_entry (list, struct totempg_group_instance, list); if (instance->confchg_fn) { instance->confchg_fn ( configuration_type, member_list, member_list_entries, left_list, left_list_entries, joined_list, joined_list_entries, ring_id); } } } static inline void group_endian_convert ( void *msg, int msg_len) { unsigned short *group_len; int i; char *aligned_msg; #ifdef TOTEMPG_NEED_ALIGN /* * Align data structure for not i386 or x86_64 */ if ((size_t)msg % 4 != 0) { aligned_msg = alloca(msg_len); memcpy(aligned_msg, msg, msg_len); } else { aligned_msg = msg; } #else aligned_msg = msg; #endif group_len = (unsigned short *)aligned_msg; group_len[0] = swab16(group_len[0]); for (i = 1; i < group_len[0] + 1; i++) { group_len[i] = swab16(group_len[i]); } if (aligned_msg != msg) { memcpy(msg, aligned_msg, msg_len); } } static inline int group_matches ( struct iovec *iovec, unsigned int iov_len, struct totempg_group *groups_b, unsigned int group_b_cnt, unsigned int *adjust_iovec) { unsigned short *group_len; char *group_name; int i; int j; #ifdef TOTEMPG_NEED_ALIGN struct iovec iovec_aligned = { NULL, 0 }; #endif assert (iov_len == 1); #ifdef TOTEMPG_NEED_ALIGN /* * Align data structure for not i386 or x86_64 */ if ((size_t)iovec->iov_base % 4 != 0) { iovec_aligned.iov_base = alloca(iovec->iov_len); memcpy(iovec_aligned.iov_base, iovec->iov_base, iovec->iov_len); iovec_aligned.iov_len = iovec->iov_len; iovec = &iovec_aligned; } #endif group_len = (unsigned short *)iovec->iov_base; group_name = ((char *)iovec->iov_base) + sizeof (unsigned short) * (group_len[0] + 1); /* * Calculate amount to adjust the iovec by before delivering to app */ *adjust_iovec = sizeof (unsigned short) * (group_len[0] + 1); for (i = 1; i < group_len[0] + 1; i++) { *adjust_iovec += group_len[i]; } /* * Determine if this message should be delivered to this instance */ for (i = 1; i < group_len[0] + 1; i++) { for (j = 0; j < group_b_cnt; j++) { if ((group_len[i] == groups_b[j].group_len) && (memcmp (groups_b[j].group, group_name, group_len[i]) == 0)) { return (1); } } group_name += group_len[i]; } return (0); } static inline void app_deliver_fn ( unsigned int nodeid, void *msg, unsigned int msg_len, int endian_conversion_required) { struct totempg_group_instance *instance; struct iovec stripped_iovec; unsigned int adjust_iovec; struct iovec *iovec; struct qb_list_head *list; struct iovec aligned_iovec = { NULL, 0 }; if (endian_conversion_required) { group_endian_convert (msg, msg_len); } /* * TODO: segmentation/assembly need to be redesigned to provide aligned access * in all cases to avoid memory copies on non386 archs. Probably broke backwars * compatibility */ #ifdef TOTEMPG_NEED_ALIGN /* * Align data structure for not i386 or x86_64 */ aligned_iovec.iov_base = alloca(msg_len); aligned_iovec.iov_len = msg_len; memcpy(aligned_iovec.iov_base, msg, msg_len); #else aligned_iovec.iov_base = msg; aligned_iovec.iov_len = msg_len; #endif iovec = &aligned_iovec; qb_list_for_each(list, &totempg_groups_list) { instance = qb_list_entry (list, struct totempg_group_instance, list); if (group_matches (iovec, 1, instance->groups, instance->groups_cnt, &adjust_iovec)) { stripped_iovec.iov_len = iovec->iov_len - adjust_iovec; stripped_iovec.iov_base = (char *)iovec->iov_base + adjust_iovec; #ifdef TOTEMPG_NEED_ALIGN /* * Align data structure for not i386 or x86_64 */ if ((char *)iovec->iov_base + adjust_iovec % 4 != 0) { /* * Deal with misalignment */ stripped_iovec.iov_base = alloca (stripped_iovec.iov_len); memcpy (stripped_iovec.iov_base, (char *)iovec->iov_base + adjust_iovec, stripped_iovec.iov_len); } #endif instance->deliver_fn ( nodeid, stripped_iovec.iov_base, stripped_iovec.iov_len, endian_conversion_required); } } } static void totempg_confchg_fn ( enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id) { // TODO optimize this app_confchg_fn (configuration_type, member_list, member_list_entries, left_list, left_list_entries, joined_list, joined_list_entries, ring_id); } static void totempg_deliver_fn ( unsigned int nodeid, const void *msg, unsigned int msg_len, int endian_conversion_required) { struct totempg_mcast *mcast; unsigned short *msg_lens; int i; struct assembly *assembly; char header[FRAME_SIZE_MAX]; int msg_count; int continuation; int start; const char *data; int datasize; struct iovec iov_delv; size_t expected_msg_len; assembly = assembly_ref (nodeid); assert (assembly); if (msg_len < sizeof(struct totempg_mcast)) { log_printf(LOG_WARNING, "Message (totempg_mcast) received from node " CS_PRI_NODE_ID " is too short... Ignoring.", nodeid); return ; } /* * Assemble the header into one block of data and * assemble the packet contents into one block of data to simplify delivery */ mcast = (struct totempg_mcast *)msg; if (endian_conversion_required) { mcast->msg_count = swab16 (mcast->msg_count); } msg_count = mcast->msg_count; datasize = sizeof (struct totempg_mcast) + msg_count * sizeof (unsigned short); if (msg_len < datasize) { log_printf(LOG_WARNING, "Message (totempg_mcast datasize) received from node " CS_PRI_NODE_ID " is too short... Ignoring.", nodeid); return ; } memcpy (header, msg, datasize); data = msg; msg_lens = (unsigned short *) (header + sizeof (struct totempg_mcast)); expected_msg_len = datasize; for (i = 0; i < mcast->msg_count; i++) { if (endian_conversion_required) { msg_lens[i] = swab16 (msg_lens[i]); } expected_msg_len += msg_lens[i]; } if (msg_len != expected_msg_len) { log_printf(LOG_WARNING, "Message (totempg_mcast) received from node " CS_PRI_NODE_ID " doesn't have expected length of %zu (has %u) bytes... Ignoring.", nodeid, expected_msg_len, msg_len); return ; } assert((assembly->index+msg_len) < sizeof(assembly->data)); memcpy (&assembly->data[assembly->index], &data[datasize], msg_len - datasize); /* * If the last message in the buffer is a fragment, then we * can't deliver it. We'll first deliver the full messages * then adjust the assembly buffer so we can add the rest of the * fragment when it arrives. */ msg_count = mcast->fragmented ? mcast->msg_count - 1 : mcast->msg_count; continuation = mcast->continuation; iov_delv.iov_base = (void *)&assembly->data[0]; iov_delv.iov_len = assembly->index + msg_lens[0]; /* * Make sure that if this message is a continuation, that it * matches the sequence number of the previous fragment. * Also, if the first packed message is a continuation * of a previous message, but the assembly buffer * is empty, then we need to discard it since we can't * assemble a complete message. Likewise, if this message isn't a * continuation and the assembly buffer is empty, we have to discard * the continued message. */ start = 0; if (assembly->throw_away_mode == THROW_AWAY_ACTIVE) { /* Throw away the first msg block */ if (mcast->fragmented == 0 || mcast->fragmented == 1) { assembly->throw_away_mode = THROW_AWAY_INACTIVE; assembly->index += msg_lens[0]; iov_delv.iov_base = (void *)&assembly->data[assembly->index]; iov_delv.iov_len = msg_lens[1]; start = 1; } } else if (assembly->throw_away_mode == THROW_AWAY_INACTIVE) { if (continuation == assembly->last_frag_num) { assembly->last_frag_num = mcast->fragmented; for (i = start; i < msg_count; i++) { app_deliver_fn(nodeid, iov_delv.iov_base, iov_delv.iov_len, endian_conversion_required); assembly->index += msg_lens[i]; iov_delv.iov_base = (void *)&assembly->data[assembly->index]; if (i < (msg_count - 1)) { iov_delv.iov_len = msg_lens[i + 1]; } } } else { log_printf (LOG_DEBUG, "fragmented continuation %u is not equal to assembly last_frag_num %u", continuation, assembly->last_frag_num); assembly->throw_away_mode = THROW_AWAY_ACTIVE; } } if (mcast->fragmented == 0) { /* * End of messages, dereference assembly struct */ assembly->last_frag_num = 0; assembly->index = 0; assembly_deref (assembly); } else { /* * Message is fragmented, keep around assembly list */ if (mcast->msg_count > 1) { memmove (&assembly->data[0], &assembly->data[assembly->index], msg_lens[msg_count]); assembly->index = 0; } assembly->index += msg_lens[msg_count]; } } /* * Totem Process Group Abstraction * depends on poll abstraction, POSIX, IPV4 */ void *callback_token_received_handle; int callback_token_received_fn (enum totem_callback_token_type type, const void *data) { struct totempg_mcast mcast; struct iovec iovecs[3]; if (totempg_threaded_mode == 1) { pthread_mutex_lock (&mcast_msg_mutex); } if (mcast_packed_msg_count == 0) { if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&mcast_msg_mutex); } return (0); } if (totemsrp_avail(totemsrp_context) == 0) { if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&mcast_msg_mutex); } return (0); } mcast.header.version = 0; mcast.header.type = 0; mcast.fragmented = 0; /* * Was the first message in this buffer a continuation of a * fragmented message? */ mcast.continuation = fragment_continuation; fragment_continuation = 0; mcast.msg_count = mcast_packed_msg_count; iovecs[0].iov_base = (void *)&mcast; iovecs[0].iov_len = sizeof (struct totempg_mcast); iovecs[1].iov_base = (void *)mcast_packed_msg_lens; iovecs[1].iov_len = mcast_packed_msg_count * sizeof (unsigned short); iovecs[2].iov_base = (void *)&fragmentation_data[0]; iovecs[2].iov_len = fragment_size; (void)totemsrp_mcast (totemsrp_context, iovecs, 3, 0); mcast_packed_msg_count = 0; fragment_size = 0; if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&mcast_msg_mutex); } return (0); } /* * Initialize the totem process group abstraction */ int totempg_initialize ( qb_loop_t *poll_handle, struct totem_config *totem_config) { int res; totempg_totem_config = totem_config; totempg_log_level_security = totem_config->totem_logging_configuration.log_level_security; totempg_log_level_error = totem_config->totem_logging_configuration.log_level_error; totempg_log_level_warning = totem_config->totem_logging_configuration.log_level_warning; totempg_log_level_notice = totem_config->totem_logging_configuration.log_level_notice; totempg_log_level_debug = totem_config->totem_logging_configuration.log_level_debug; totempg_log_printf = totem_config->totem_logging_configuration.log_printf; totempg_subsys_id = totem_config->totem_logging_configuration.log_subsys_id; fragmentation_data = malloc (TOTEMPG_PACKET_SIZE); if (fragmentation_data == 0) { return (-1); } totemsrp_net_mtu_adjust (totem_config); res = totemsrp_initialize ( poll_handle, &totemsrp_context, totem_config, &totempg_stats, totempg_deliver_fn, totempg_confchg_fn, totempg_waiting_trans_ack_cb); if (res == -1) { goto error_exit; } totemsrp_callback_token_create ( totemsrp_context, &callback_token_received_handle, TOTEM_CALLBACK_TOKEN_RECEIVED, 0, callback_token_received_fn, 0); totempg_size_limit = (totemsrp_avail(totemsrp_context) - 1) * (totempg_totem_config->net_mtu - sizeof (struct totempg_mcast) - 16); qb_list_init (&totempg_groups_list); error_exit: return (res); } void totempg_finalize (void) { if (totempg_threaded_mode == 1) { pthread_mutex_lock (&totempg_mutex); } totemsrp_finalize (totemsrp_context); if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&totempg_mutex); } } /* * Multicast a message */ static int mcast_msg ( struct iovec *iovec_in, unsigned int iov_len, int guarantee) { int res = 0; struct totempg_mcast mcast; struct iovec iovecs[3]; struct iovec iovec[64]; int i; int dest, src; int max_packet_size = 0; int copy_len = 0; int copy_base = 0; int total_size = 0; if (totempg_threaded_mode == 1) { pthread_mutex_lock (&mcast_msg_mutex); } totemsrp_event_signal (totemsrp_context, TOTEM_EVENT_NEW_MSG, 1); /* * Remove zero length iovectors from the list */ assert (iov_len < 64); for (dest = 0, src = 0; src < iov_len; src++) { if (iovec_in[src].iov_len) { memcpy (&iovec[dest++], &iovec_in[src], sizeof (struct iovec)); } } iov_len = dest; max_packet_size = TOTEMPG_PACKET_SIZE - (sizeof (unsigned short) * (mcast_packed_msg_count + 1)); mcast_packed_msg_lens[mcast_packed_msg_count] = 0; /* * Check if we would overwrite new message queue */ for (i = 0; i < iov_len; i++) { total_size += iovec[i].iov_len; } if (byte_count_send_ok (total_size + sizeof(unsigned short) * (mcast_packed_msg_count)) == 0) { if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&mcast_msg_mutex); } return(-1); } memset(&mcast, 0, sizeof(mcast)); mcast.header.version = 0; for (i = 0; i < iov_len; ) { mcast.fragmented = 0; mcast.continuation = fragment_continuation; copy_len = iovec[i].iov_len - copy_base; /* * If it all fits with room left over, copy it in. * We need to leave at least sizeof(short) + 1 bytes in the * fragment_buffer on exit so that max_packet_size + fragment_size * doesn't exceed the size of the fragment_buffer on the next call. */ if ((iovec[i].iov_len + fragment_size) < (max_packet_size - sizeof (unsigned short))) { memcpy (&fragmentation_data[fragment_size], (char *)iovec[i].iov_base + copy_base, copy_len); fragment_size += copy_len; mcast_packed_msg_lens[mcast_packed_msg_count] += copy_len; next_fragment = 1; copy_len = 0; copy_base = 0; i++; continue; /* * If it just fits or is too big, then send out what fits. */ } else { unsigned char *data_ptr; copy_len = min(copy_len, max_packet_size - fragment_size); if( copy_len == max_packet_size ) data_ptr = (unsigned char *)iovec[i].iov_base + copy_base; else { data_ptr = fragmentation_data; } memcpy (&fragmentation_data[fragment_size], (unsigned char *)iovec[i].iov_base + copy_base, copy_len); mcast_packed_msg_lens[mcast_packed_msg_count] += copy_len; /* * if we're not on the last iovec or the iovec is too large to * fit, then indicate a fragment. This also means that the next * message will have the continuation of this one. */ if ((i < (iov_len - 1)) || ((copy_base + copy_len) < iovec[i].iov_len)) { if (!next_fragment) { next_fragment++; } fragment_continuation = next_fragment; mcast.fragmented = next_fragment++; assert(fragment_continuation != 0); assert(mcast.fragmented != 0); } else { fragment_continuation = 0; } /* * assemble the message and send it */ mcast.msg_count = ++mcast_packed_msg_count; iovecs[0].iov_base = (void *)&mcast; iovecs[0].iov_len = sizeof(struct totempg_mcast); iovecs[1].iov_base = (void *)mcast_packed_msg_lens; iovecs[1].iov_len = mcast_packed_msg_count * sizeof(unsigned short); iovecs[2].iov_base = (void *)data_ptr; iovecs[2].iov_len = fragment_size + copy_len; assert (totemsrp_avail(totemsrp_context) > 0); res = totemsrp_mcast (totemsrp_context, iovecs, 3, guarantee); if (res == -1) { goto error_exit; } /* * Recalculate counts and indexes for the next. */ mcast_packed_msg_lens[0] = 0; mcast_packed_msg_count = 0; fragment_size = 0; max_packet_size = TOTEMPG_PACKET_SIZE - (sizeof(unsigned short)); /* * If the iovec all fit, go to the next iovec */ if ((copy_base + copy_len) == iovec[i].iov_len) { copy_len = 0; copy_base = 0; i++; /* * Continue with the rest of the current iovec. */ } else { copy_base += copy_len; } } } /* * Bump only if we added message data. This may be zero if * the last buffer just fit into the fragmentation_data buffer * and we were at the last iovec. */ if (mcast_packed_msg_lens[mcast_packed_msg_count]) { mcast_packed_msg_count++; } error_exit: if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&mcast_msg_mutex); } return (res); } /* * Determine if a message of msg_size could be queued */ static int msg_count_send_ok ( int msg_count) { int avail = 0; avail = totemsrp_avail (totemsrp_context); totempg_stats.msg_queue_avail = avail; return ((avail - totempg_reserved) > msg_count); } static int byte_count_send_ok ( int byte_count) { unsigned int msg_count = 0; int avail = 0; avail = totemsrp_avail (totemsrp_context); msg_count = (byte_count / (totempg_totem_config->net_mtu - sizeof (struct totempg_mcast) - 16)) + 1; return (avail >= msg_count); } static int send_reserve ( int msg_size) { unsigned int msg_count = 0; msg_count = (msg_size / (totempg_totem_config->net_mtu - sizeof (struct totempg_mcast) - 16)) + 1; totempg_reserved += msg_count; totempg_stats.msg_reserved = totempg_reserved; return (msg_count); } static void send_release ( int msg_count) { totempg_reserved -= msg_count; totempg_stats.msg_reserved = totempg_reserved; } #ifndef HAVE_SMALL_MEMORY_FOOTPRINT #undef MESSAGE_QUEUE_MAX #define MESSAGE_QUEUE_MAX ((4 * MESSAGE_SIZE_MAX) / totempg_totem_config->net_mtu) #endif /* HAVE_SMALL_MEMORY_FOOTPRINT */ static uint32_t q_level_precent_used(void) { return (100 - (((totemsrp_avail(totemsrp_context) - totempg_reserved) * 100) / MESSAGE_QUEUE_MAX)); } int totempg_callback_token_create ( void **handle_out, enum totem_callback_token_type type, int delete, int (*callback_fn) (enum totem_callback_token_type type, const void *), const void *data) { unsigned int res; if (totempg_threaded_mode == 1) { pthread_mutex_lock (&callback_token_mutex); } res = totemsrp_callback_token_create (totemsrp_context, handle_out, type, delete, callback_fn, data); if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&callback_token_mutex); } return (res); } void totempg_callback_token_destroy ( void *handle_out) { if (totempg_threaded_mode == 1) { pthread_mutex_lock (&callback_token_mutex); } totemsrp_callback_token_destroy (totemsrp_context, handle_out); if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&callback_token_mutex); } } /* * vi: set autoindent tabstop=4 shiftwidth=4 : */ int totempg_groups_initialize ( void **totempg_groups_instance, void (*deliver_fn) ( unsigned int nodeid, const void *msg, unsigned int msg_len, int endian_conversion_required), void (*confchg_fn) ( enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id)) { struct totempg_group_instance *instance; if (totempg_threaded_mode == 1) { pthread_mutex_lock (&totempg_mutex); } instance = malloc (sizeof (struct totempg_group_instance)); if (instance == NULL) { goto error_exit; } instance->deliver_fn = deliver_fn; instance->confchg_fn = confchg_fn; instance->groups = 0; instance->groups_cnt = 0; instance->q_level = QB_LOOP_MED; qb_list_init (&instance->list); qb_list_add (&instance->list, &totempg_groups_list); if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&totempg_mutex); } *totempg_groups_instance = instance; return (0); error_exit: if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&totempg_mutex); } return (-1); } int totempg_groups_join ( void *totempg_groups_instance, const struct totempg_group *groups, size_t group_cnt) { struct totempg_group_instance *instance = (struct totempg_group_instance *)totempg_groups_instance; struct totempg_group *new_groups; int res = 0; if (totempg_threaded_mode == 1) { pthread_mutex_lock (&totempg_mutex); } new_groups = realloc (instance->groups, sizeof (struct totempg_group) * (instance->groups_cnt + group_cnt)); if (new_groups == 0) { res = -1; goto error_exit; } memcpy (&new_groups[instance->groups_cnt], groups, group_cnt * sizeof (struct totempg_group)); instance->groups = new_groups; instance->groups_cnt += group_cnt; error_exit: if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&totempg_mutex); } return (res); } int totempg_groups_leave ( void *totempg_groups_instance, const struct totempg_group *groups, size_t group_cnt) { if (totempg_threaded_mode == 1) { pthread_mutex_lock (&totempg_mutex); } if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&totempg_mutex); } return (0); } #define MAX_IOVECS_FROM_APP 32 #define MAX_GROUPS_PER_MSG 32 int totempg_groups_mcast_joined ( void *totempg_groups_instance, const struct iovec *iovec, unsigned int iov_len, int guarantee) { struct totempg_group_instance *instance = (struct totempg_group_instance *)totempg_groups_instance; unsigned short group_len[MAX_GROUPS_PER_MSG + 1]; struct iovec iovec_mcast[MAX_GROUPS_PER_MSG + 1 + MAX_IOVECS_FROM_APP]; int i; unsigned int res; if (totempg_threaded_mode == 1) { pthread_mutex_lock (&totempg_mutex); } /* * Build group_len structure and the iovec_mcast structure */ group_len[0] = instance->groups_cnt; for (i = 0; i < instance->groups_cnt; i++) { group_len[i + 1] = instance->groups[i].group_len; iovec_mcast[i + 1].iov_len = instance->groups[i].group_len; iovec_mcast[i + 1].iov_base = (void *) instance->groups[i].group; } iovec_mcast[0].iov_len = (instance->groups_cnt + 1) * sizeof (unsigned short); iovec_mcast[0].iov_base = group_len; for (i = 0; i < iov_len; i++) { iovec_mcast[i + instance->groups_cnt + 1].iov_len = iovec[i].iov_len; iovec_mcast[i + instance->groups_cnt + 1].iov_base = iovec[i].iov_base; } res = mcast_msg (iovec_mcast, iov_len + instance->groups_cnt + 1, guarantee); if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&totempg_mutex); } return (res); } static void check_q_level( void *totempg_groups_instance) { struct totempg_group_instance *instance = (struct totempg_group_instance *)totempg_groups_instance; int32_t old_level = instance->q_level; int32_t percent_used = q_level_precent_used(); if (percent_used >= 75 && instance->q_level != TOTEM_Q_LEVEL_CRITICAL) { instance->q_level = TOTEM_Q_LEVEL_CRITICAL; } else if (percent_used < 30 && instance->q_level != TOTEM_Q_LEVEL_LOW) { instance->q_level = TOTEM_Q_LEVEL_LOW; } else if (percent_used > 40 && percent_used < 50 && instance->q_level != TOTEM_Q_LEVEL_GOOD) { instance->q_level = TOTEM_Q_LEVEL_GOOD; } else if (percent_used > 60 && percent_used < 70 && instance->q_level != TOTEM_Q_LEVEL_HIGH) { instance->q_level = TOTEM_Q_LEVEL_HIGH; } if (totem_queue_level_changed && old_level != instance->q_level) { totem_queue_level_changed(instance->q_level); } } void totempg_check_q_level( void *totempg_groups_instance) { struct totempg_group_instance *instance = (struct totempg_group_instance *)totempg_groups_instance; check_q_level(instance); } int totempg_groups_joined_reserve ( void *totempg_groups_instance, const struct iovec *iovec, unsigned int iov_len) { struct totempg_group_instance *instance = (struct totempg_group_instance *)totempg_groups_instance; unsigned int size = 0; unsigned int i; unsigned int reserved = 0; if (totempg_threaded_mode == 1) { pthread_mutex_lock (&totempg_mutex); pthread_mutex_lock (&mcast_msg_mutex); } for (i = 0; i < instance->groups_cnt; i++) { size += instance->groups[i].group_len; } for (i = 0; i < iov_len; i++) { size += iovec[i].iov_len; } if (size >= totempg_size_limit) { reserved = -1; goto error_exit; } if (byte_count_send_ok (size)) { reserved = send_reserve (size); } else { reserved = 0; } error_exit: check_q_level(instance); if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&mcast_msg_mutex); pthread_mutex_unlock (&totempg_mutex); } return (reserved); } int totempg_groups_joined_release (int msg_count) { if (totempg_threaded_mode == 1) { pthread_mutex_lock (&totempg_mutex); pthread_mutex_lock (&mcast_msg_mutex); } send_release (msg_count); if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&mcast_msg_mutex); pthread_mutex_unlock (&totempg_mutex); } return 0; } int totempg_groups_mcast_groups ( void *totempg_groups_instance, int guarantee, const struct totempg_group *groups, size_t groups_cnt, const struct iovec *iovec, unsigned int iov_len) { unsigned short group_len[MAX_GROUPS_PER_MSG + 1]; struct iovec iovec_mcast[MAX_GROUPS_PER_MSG + 1 + MAX_IOVECS_FROM_APP]; int i; unsigned int res; if (totempg_threaded_mode == 1) { pthread_mutex_lock (&totempg_mutex); } /* * Build group_len structure and the iovec_mcast structure */ group_len[0] = groups_cnt; for (i = 0; i < groups_cnt; i++) { group_len[i + 1] = groups[i].group_len; iovec_mcast[i + 1].iov_len = groups[i].group_len; iovec_mcast[i + 1].iov_base = (void *) groups[i].group; } iovec_mcast[0].iov_len = (groups_cnt + 1) * sizeof (unsigned short); iovec_mcast[0].iov_base = group_len; for (i = 0; i < iov_len; i++) { iovec_mcast[i + groups_cnt + 1].iov_len = iovec[i].iov_len; iovec_mcast[i + groups_cnt + 1].iov_base = iovec[i].iov_base; } res = mcast_msg (iovec_mcast, iov_len + groups_cnt + 1, guarantee); if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&totempg_mutex); } return (res); } /* * Returns -1 if error, 0 if can't send, 1 if can send the message */ int totempg_groups_send_ok_groups ( void *totempg_groups_instance, const struct totempg_group *groups, size_t groups_cnt, const struct iovec *iovec, unsigned int iov_len) { unsigned int size = 0; unsigned int i; unsigned int res; if (totempg_threaded_mode == 1) { pthread_mutex_lock (&totempg_mutex); } for (i = 0; i < groups_cnt; i++) { size += groups[i].group_len; } for (i = 0; i < iov_len; i++) { size += iovec[i].iov_len; } res = msg_count_send_ok (size); if (totempg_threaded_mode == 1) { pthread_mutex_unlock (&totempg_mutex); } return (res); } int totempg_iface_set ( struct totem_ip_address *interface_addr, unsigned short ip_port, unsigned int iface_no) { int res; res = totemsrp_iface_set ( totemsrp_context, interface_addr, ip_port, iface_no); return (res); } int totempg_ifaces_get ( unsigned int nodeid, unsigned int *interface_id, struct totem_ip_address *interfaces, unsigned int interfaces_size, char ***status, unsigned int *iface_count) { int res; res = totemsrp_ifaces_get ( totemsrp_context, nodeid, interface_id, interfaces, interfaces_size, status, iface_count); return (res); } void totempg_event_signal (enum totem_event_type type, int value) { totemsrp_event_signal (totemsrp_context, type, value); } void* totempg_get_stats (void) { return &totempg_stats; } int totempg_crypto_set ( const char *cipher_type, const char *hash_type) { int res; res = totemsrp_crypto_set (totemsrp_context, cipher_type, hash_type); return (res); } #define ONE_IFACE_LEN 63 const char *totempg_ifaces_print (unsigned int nodeid) { static char iface_string[256 * INTERFACE_MAX]; char one_iface[ONE_IFACE_LEN+1]; struct totem_ip_address interfaces[INTERFACE_MAX]; unsigned int iface_count; unsigned int iface_ids[INTERFACE_MAX]; unsigned int i; int res; iface_string[0] = '\0'; res = totempg_ifaces_get (nodeid, iface_ids, interfaces, INTERFACE_MAX, NULL, &iface_count); if (res == -1) { return ("no interface found for nodeid"); } res = totempg_ifaces_get (nodeid, iface_ids, interfaces, INTERFACE_MAX, NULL, &iface_count); for (i = 0; i < iface_count; i++) { if (!interfaces[i].family) { continue; } snprintf (one_iface, ONE_IFACE_LEN, "r(%d) ip(%s) ", i, totemip_print (&interfaces[i])); strcat (iface_string, one_iface); } return (iface_string); } unsigned int totempg_my_nodeid_get (void) { return (totemsrp_my_nodeid_get(totemsrp_context)); } int totempg_my_family_get (void) { return (totemsrp_my_family_get(totemsrp_context)); } extern void totempg_service_ready_register ( void (*totem_service_ready) (void)) { totemsrp_service_ready_register (totemsrp_context, totem_service_ready); } void totempg_queue_level_register_callback (totem_queue_level_changed_fn fn) { totem_queue_level_changed = fn; } extern int totempg_member_add ( const struct totem_ip_address *member, int ring_no) { return totemsrp_member_add (totemsrp_context, member, ring_no); } extern int totempg_member_remove ( const struct totem_ip_address *member, int ring_no) { return totemsrp_member_remove (totemsrp_context, member, ring_no); } extern int totempg_reconfigure (void) { return totemsrp_reconfigure (totemsrp_context, totempg_totem_config); } extern void totempg_stats_clear (int flags) { if (flags & TOTEMPG_STATS_CLEAR_TOTEM) { totempg_stats.msg_reserved = 0; totempg_stats.msg_queue_avail = 0; } return totemsrp_stats_clear (totemsrp_context, flags); } void totempg_threaded_mode_enable (void) { totempg_threaded_mode = 1; totemsrp_threaded_mode_enable (totemsrp_context); } void totempg_trans_ack (void) { totemsrp_trans_ack (totemsrp_context); } void totempg_force_gather (void) { totemsrp_force_gather(totemsrp_context); } /* Assumes ->orig_interfaces is already allocated */ void totempg_get_config(struct totem_config *config) { struct totem_interface *temp_if = config->orig_interfaces; memcpy(config, totempg_totem_config, sizeof(struct totem_config)); config->orig_interfaces = temp_if; memcpy(config->orig_interfaces, totempg_totem_config->interfaces, sizeof(struct totem_interface) * INTERFACE_MAX); config->interfaces = NULL; } void totempg_put_config(struct totem_config *config) { - /* Free this before we overwite the pointer */ - free(totempg_totem_config->interfaces); + struct totem_interface *temp_if = totempg_totem_config->interfaces; + + /* Preseve the existing interfaces[] array as transports might have pointers saved */ + memcpy(totempg_totem_config->interfaces, config->interfaces, sizeof(struct totem_interface) * INTERFACE_MAX); memcpy(totempg_totem_config, config, sizeof(struct totem_config)); + totempg_totem_config->interfaces = temp_if; }