diff --git a/exec/coroparse.c b/exec/coroparse.c index 8ad2d345..ec9ff461 100644 --- a/exec/coroparse.c +++ b/exec/coroparse.c @@ -1,1659 +1,1661 @@ /* * Copyright (c) 2006-2018 Red Hat, Inc. * * All rights reserved. * * Author: Patrick Caulfield (pcaulfie@redhat.com) * Jan Friesse (jfriesse@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the MontaVista Software, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define LOGSYS_UTILS_ONLY 1 #include #include #include "main.h" #include "util.h" enum parser_cb_type { PARSER_CB_START, PARSER_CB_END, PARSER_CB_SECTION_START, PARSER_CB_SECTION_END, PARSER_CB_ITEM, }; enum main_cp_cb_data_state { MAIN_CP_CB_DATA_STATE_NORMAL, MAIN_CP_CB_DATA_STATE_TOTEM, MAIN_CP_CB_DATA_STATE_INTERFACE, MAIN_CP_CB_DATA_STATE_LOGGER_SUBSYS, MAIN_CP_CB_DATA_STATE_UIDGID, MAIN_CP_CB_DATA_STATE_LOGGING_DAEMON, MAIN_CP_CB_DATA_STATE_MEMBER, MAIN_CP_CB_DATA_STATE_QUORUM, MAIN_CP_CB_DATA_STATE_QDEVICE, MAIN_CP_CB_DATA_STATE_NODELIST, MAIN_CP_CB_DATA_STATE_NODELIST_NODE, MAIN_CP_CB_DATA_STATE_PLOAD, MAIN_CP_CB_DATA_STATE_SYSTEM, MAIN_CP_CB_DATA_STATE_RESOURCES, MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM, MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS, MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM_MEMUSED, MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS_MEMUSED }; typedef int (*parser_cb_f)(const char *path, char *key, char *value, enum main_cp_cb_data_state *state, enum parser_cb_type type, const char **error_string, icmap_map_t config_map, void *user_data); struct key_value_list_item { char *key; char *value; struct qb_list_head list; }; struct main_cp_cb_data { int linknumber; char *bindnetaddr; char *mcastaddr; char *broadcast; int mcastport; int ttl; int knet_link_priority; int knet_ping_interval; int knet_ping_timeout; int knet_ping_precision; int knet_pong_count; int knet_pmtud_interval; char *knet_transport; struct qb_list_head logger_subsys_items_head; char *subsys; char *logging_daemon_name; struct qb_list_head member_items_head; int node_number; }; static int read_config_file_into_icmap( const char **error_string, icmap_map_t config_map); static char error_string_response[512]; static int uid_determine (const char *req_user) { int pw_uid = 0; struct passwd passwd; struct passwd* pwdptr = &passwd; struct passwd* temp_pwd_pt; char *pwdbuffer; int pwdlinelen, rc; long int id; char *ep; id = strtol(req_user, &ep, 10); if (*req_user != '\0' && *ep == '\0' && id >= 0 && id <= UINT_MAX) { return (id); } pwdlinelen = sysconf (_SC_GETPW_R_SIZE_MAX); if (pwdlinelen == -1) { pwdlinelen = 256; } pwdbuffer = malloc (pwdlinelen); while ((rc = getpwnam_r (req_user, pwdptr, pwdbuffer, pwdlinelen, &temp_pwd_pt)) == ERANGE) { char *n; pwdlinelen *= 2; if (pwdlinelen <= 32678) { n = realloc (pwdbuffer, pwdlinelen); if (n != NULL) { pwdbuffer = n; continue; } } } if (rc != 0) { free (pwdbuffer); sprintf (error_string_response, "getpwnam_r(): %s", strerror(rc)); return (-1); } if (temp_pwd_pt == NULL) { free (pwdbuffer); sprintf (error_string_response, "The '%s' user is not found in /etc/passwd, please read the documentation.", req_user); return (-1); } pw_uid = passwd.pw_uid; free (pwdbuffer); return pw_uid; } static int gid_determine (const char *req_group) { int corosync_gid = 0; struct group group; struct group * grpptr = &group; struct group * temp_grp_pt; char *grpbuffer; int grplinelen, rc; long int id; char *ep; id = strtol(req_group, &ep, 10); if (*req_group != '\0' && *ep == '\0' && id >= 0 && id <= UINT_MAX) { return (id); } grplinelen = sysconf (_SC_GETGR_R_SIZE_MAX); if (grplinelen == -1) { grplinelen = 256; } grpbuffer = malloc (grplinelen); while ((rc = getgrnam_r (req_group, grpptr, grpbuffer, grplinelen, &temp_grp_pt)) == ERANGE) { char *n; grplinelen *= 2; if (grplinelen <= 32678) { n = realloc (grpbuffer, grplinelen); if (n != NULL) { grpbuffer = n; continue; } } } if (rc != 0) { free (grpbuffer); sprintf (error_string_response, "getgrnam_r(): %s", strerror(rc)); return (-1); } if (temp_grp_pt == NULL) { free (grpbuffer); sprintf (error_string_response, "The '%s' group is not found in /etc/group, please read the documentation.", req_group); return (-1); } corosync_gid = group.gr_gid; free (grpbuffer); return corosync_gid; } static char *strchr_rs (const char *haystack, int byte) { const char *end_address = strchr (haystack, byte); if (end_address) { end_address += 1; /* skip past { or = */ while (*end_address == ' ' || *end_address == '\t') end_address++; } return ((char *) end_address); } int coroparse_configparse (icmap_map_t config_map, const char **error_string) { if (read_config_file_into_icmap(error_string, config_map)) { return -1; } return 0; } static char *remove_whitespace(char *string, int remove_colon_and_brace) { char *start; char *end; start = string; while (*start == ' ' || *start == '\t') start++; end = start+(strlen(start))-1; while ((*end == ' ' || *end == '\t' || (remove_colon_and_brace && (*end == ':' || *end == '{'))) && end > start) end--; if (*end != '\0') *(end + 1) = '\0'; return start; } static int parse_section(FILE *fp, const char *fname, int *line_no, char *path, const char **error_string, int depth, enum main_cp_cb_data_state state, parser_cb_f parser_cb, icmap_map_t config_map, void *user_data) { char line[512]; int i; char *loc; int ignore_line; char new_keyname[ICMAP_KEYNAME_MAXLEN]; static char formated_err[384]; const char *tmp_error_string; if (strcmp(path, "") == 0) { parser_cb("", NULL, NULL, &state, PARSER_CB_START, error_string, config_map, user_data); } tmp_error_string = NULL; while (fgets (line, sizeof (line), fp)) { (*line_no)++; if (strlen(line) > 0) { /* * Check if complete line was read. Use feof to handle files * without ending \n at the end of the file */ if ((line[strlen(line) - 1] != '\n') && !feof(fp)) { tmp_error_string = "Line too long"; goto parse_error; } if (line[strlen(line) - 1] == '\n') line[strlen(line) - 1] = '\0'; if (strlen (line) > 0 && line[strlen(line) - 1] == '\r') line[strlen(line) - 1] = '\0'; } /* * Clear out white space and tabs */ for (i = strlen (line) - 1; i > -1; i--) { if (line[i] == '\t' || line[i] == ' ') { line[i] = '\0'; } else { break; } } ignore_line = 1; for (i = 0; i < strlen (line); i++) { if (line[i] != '\t' && line[i] != ' ') { if (line[i] != '#') ignore_line = 0; break; } } /* * Clear out comments and empty lines */ if (ignore_line) { continue; } /* New section ? */ if ((loc = strchr_rs (line, '{'))) { char *section; char *after_section; enum main_cp_cb_data_state newstate; *(loc-1) = '\0'; section = remove_whitespace(line, 1); after_section = remove_whitespace(loc, 0); if (strcmp(section, "") == 0) { tmp_error_string = "Missing section name before opening bracket '{'"; goto parse_error; } if (strcmp(after_section, "") != 0) { tmp_error_string = "Extra characters after opening bracket '{'"; goto parse_error; } if (strlen(path) + strlen(section) + 1 >= ICMAP_KEYNAME_MAXLEN) { tmp_error_string = "Start of section makes total cmap path too long"; goto parse_error; } strcpy(new_keyname, path); if (strcmp(path, "") != 0) { strcat(new_keyname, "."); } strcat(new_keyname, section); /* Only use the new state for items further down the stack */ newstate = state; if (!parser_cb(new_keyname, NULL, NULL, &newstate, PARSER_CB_SECTION_START, &tmp_error_string, config_map, user_data)) { goto parse_error; } if (parse_section(fp, fname, line_no, new_keyname, error_string, depth + 1, newstate, parser_cb, config_map, user_data)) return -1; continue ; } /* New key/value */ if ((loc = strchr_rs (line, ':'))) { char *key; char *value; *(loc-1) = '\0'; key = remove_whitespace(line, 1); value = remove_whitespace(loc, 0); if (strlen(path) + strlen(key) + 1 >= ICMAP_KEYNAME_MAXLEN) { tmp_error_string = "New key makes total cmap path too long"; goto parse_error; } strcpy(new_keyname, path); if (strcmp(path, "") != 0) { strcat(new_keyname, "."); } strcat(new_keyname, key); if (!parser_cb(new_keyname, key, value, &state, PARSER_CB_ITEM, &tmp_error_string, config_map, user_data)) { goto parse_error; } continue ; } if (strchr_rs (line, '}')) { char *trimmed_line; trimmed_line = remove_whitespace(line, 0); if (strcmp(trimmed_line, "}") != 0) { tmp_error_string = "Extra characters before or after closing bracket '}'"; goto parse_error; } if (depth == 0) { tmp_error_string = "Unexpected closing brace"; goto parse_error; } if (!parser_cb(path, NULL, NULL, &state, PARSER_CB_SECTION_END, &tmp_error_string, config_map, user_data)) { goto parse_error; } return 0; } /* * Line is not opening section, ending section or value -> error */ tmp_error_string = "Line is not opening or closing section or key value"; goto parse_error; } if (strcmp(path, "") != 0) { tmp_error_string = "Missing closing brace"; goto parse_error; } if (strcmp(path, "") == 0) { parser_cb("", NULL, NULL, &state, PARSER_CB_END, error_string, config_map, user_data); } return 0; parse_error: if (snprintf(formated_err, sizeof(formated_err), "parser error: %s:%u: %s", fname, *line_no, tmp_error_string) >= sizeof(formated_err)) { *error_string = "Can't format parser error message"; } else { *error_string = formated_err; } return -1; } static int safe_atoq_range(icmap_value_types_t value_type, long long int *min_val, long long int *max_val) { switch (value_type) { case ICMAP_VALUETYPE_INT8: *min_val = INT8_MIN; *max_val = INT8_MAX; break; case ICMAP_VALUETYPE_UINT8: *min_val = 0; *max_val = UINT8_MAX; break; case ICMAP_VALUETYPE_INT16: *min_val = INT16_MIN; *max_val = INT16_MAX; break; case ICMAP_VALUETYPE_UINT16: *min_val = 0; *max_val = UINT16_MAX; break; case ICMAP_VALUETYPE_INT32: *min_val = INT32_MIN; *max_val = INT32_MAX; break; case ICMAP_VALUETYPE_UINT32: *min_val = 0; *max_val = UINT32_MAX; break; default: return (-1); } return (0); } /* * Convert string str to long long int res. Type of result is target_type and currently only * ICMAP_VALUETYPE_[U]INT[8|16|32] is supported. * Return 0 on success, -1 on failure. */ static int safe_atoq(const char *str, long long int *res, icmap_value_types_t target_type) { long long int val; long long int min_val, max_val; char *endptr; errno = 0; val = strtoll(str, &endptr, 10); if (errno == ERANGE) { return (-1); } if (endptr == str) { return (-1); } if (*endptr != '\0') { return (-1); } if (safe_atoq_range(target_type, &min_val, &max_val) != 0) { return (-1); } if (val < min_val || val > max_val) { return (-1); } *res = val; return (0); } static int str_to_ull(const char *str, unsigned long long int *res) { unsigned long long int val; char *endptr; errno = 0; val = strtoull(str, &endptr, 10); if (errno == ERANGE) { return (-1); } if (endptr == str) { return (-1); } if (*endptr != '\0') { return (-1); } *res = val; return (0); } static int main_config_parser_cb(const char *path, char *key, char *value, enum main_cp_cb_data_state *state, enum parser_cb_type type, const char **error_string, icmap_map_t config_map, void *user_data) { int ii; long long int val; long long int min_val, max_val; icmap_value_types_t val_type = ICMAP_VALUETYPE_BINARY; unsigned long long int ull; int add_as_string; char key_name[ICMAP_KEYNAME_MAXLEN + 1]; static char formated_err[256]; struct main_cp_cb_data *data = (struct main_cp_cb_data *)user_data; struct key_value_list_item *kv_item; struct qb_list_head *iter, *tmp_iter; int uid, gid; cs_error_t cs_err; cs_err = CS_OK; /* * Formally this check is not needed because length is checked by parse_section */ if (strlen(path) >= sizeof(key_name)) { if (snprintf(formated_err, sizeof(formated_err), "Can't store path \"%s\" into key_name", path) >= sizeof(formated_err)) { *error_string = "Can't format path into key_name error message"; } else { *error_string = formated_err; } return (0); } /* * Key_name is used in atoi_error/icmap_set_error, but many of icmap_set* * are using path, so initialize key_name to valid value */ strncpy(key_name, path, sizeof(key_name)); switch (type) { case PARSER_CB_START: memset(data, 0, sizeof(struct main_cp_cb_data)); *state = MAIN_CP_CB_DATA_STATE_NORMAL; break; case PARSER_CB_END: break; case PARSER_CB_ITEM: add_as_string = 1; switch (*state) { case MAIN_CP_CB_DATA_STATE_NORMAL: break; case MAIN_CP_CB_DATA_STATE_PLOAD: if ((strcmp(path, "pload.count") == 0) || (strcmp(path, "pload.size") == 0)) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } if ((cs_err = icmap_set_uint32_r(config_map, path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } break; case MAIN_CP_CB_DATA_STATE_QUORUM: if ((strcmp(path, "quorum.expected_votes") == 0) || (strcmp(path, "quorum.votes") == 0) || (strcmp(path, "quorum.last_man_standing_window") == 0) || (strcmp(path, "quorum.leaving_timeout") == 0)) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } if ((cs_err = icmap_set_uint32_r(config_map, path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } if ((strcmp(path, "quorum.two_node") == 0) || (strcmp(path, "quorum.expected_votes_tracking") == 0) || (strcmp(path, "quorum.allow_downscale") == 0) || (strcmp(path, "quorum.wait_for_all") == 0) || (strcmp(path, "quorum.auto_tie_breaker") == 0) || (strcmp(path, "quorum.last_man_standing") == 0)) { val_type = ICMAP_VALUETYPE_UINT8; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } if ((cs_err = icmap_set_uint8_r(config_map, path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } break; case MAIN_CP_CB_DATA_STATE_QDEVICE: if ((strcmp(path, "quorum.device.timeout") == 0) || (strcmp(path, "quorum.device.sync_timeout") == 0) || (strcmp(path, "quorum.device.votes") == 0)) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } if ((cs_err = icmap_set_uint32_r(config_map, path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } if ((strcmp(path, "quorum.device.master_wins") == 0)) { val_type = ICMAP_VALUETYPE_UINT8; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } if ((cs_err = icmap_set_uint8_r(config_map, path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } break; case MAIN_CP_CB_DATA_STATE_TOTEM: if ((strcmp(path, "totem.version") == 0) || (strcmp(path, "totem.nodeid") == 0) || (strcmp(path, "totem.threads") == 0) || (strcmp(path, "totem.token") == 0) || (strcmp(path, "totem.token_coefficient") == 0) || (strcmp(path, "totem.token_retransmit") == 0) || (strcmp(path, "totem.token_warning") == 0) || (strcmp(path, "totem.hold") == 0) || (strcmp(path, "totem.token_retransmits_before_loss_const") == 0) || (strcmp(path, "totem.join") == 0) || (strcmp(path, "totem.send_join") == 0) || (strcmp(path, "totem.consensus") == 0) || (strcmp(path, "totem.merge") == 0) || (strcmp(path, "totem.downcheck") == 0) || (strcmp(path, "totem.fail_recv_const") == 0) || (strcmp(path, "totem.seqno_unchanged_const") == 0) || (strcmp(path, "totem.rrp_token_expired_timeout") == 0) || (strcmp(path, "totem.rrp_problem_count_timeout") == 0) || (strcmp(path, "totem.rrp_problem_count_threshold") == 0) || (strcmp(path, "totem.rrp_problem_count_mcast_threshold") == 0) || (strcmp(path, "totem.rrp_autorecovery_check_timeout") == 0) || (strcmp(path, "totem.heartbeat_failures_allowed") == 0) || (strcmp(path, "totem.max_network_delay") == 0) || (strcmp(path, "totem.window_size") == 0) || (strcmp(path, "totem.max_messages") == 0) || (strcmp(path, "totem.miss_count_const") == 0) || (strcmp(path, "totem.knet_pmtud_interval") == 0) || (strcmp(path, "totem.knet_compression_threshold") == 0) || (strcmp(path, "totem.netmtu") == 0)) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } if ((cs_err = icmap_set_uint32_r(config_map,path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } if (strcmp(path, "totem.knet_compression_level") == 0) { val_type = ICMAP_VALUETYPE_INT32; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } if ((cs_err = icmap_set_int32_r(config_map, path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } if (strcmp(path, "totem.config_version") == 0) { if (str_to_ull(value, &ull) != 0) { goto atoi_error; } if ((cs_err = icmap_set_uint64_r(config_map, path, ull)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } if (strcmp(path, "totem.ip_version") == 0) { if ((strcmp(value, "ipv4") != 0) && - (strcmp(value, "ipv6") != 0)) { + (strcmp(value, "ipv6") != 0) && + (strcmp(value, "ipv6-4") != 0) && + (strcmp(value, "ipv4-6") != 0)) { *error_string = "Invalid ip_version type"; return (0); } } if (strcmp(path, "totem.crypto_type") == 0) { if ((strcmp(value, "nss") != 0) && (strcmp(value, "aes256") != 0) && (strcmp(value, "aes192") != 0) && (strcmp(value, "aes128") != 0) && (strcmp(value, "3des") != 0)) { *error_string = "Invalid crypto type"; return (0); } } if (strcmp(path, "totem.crypto_cipher") == 0) { if ((strcmp(value, "none") != 0) && (strcmp(value, "aes256") != 0) && (strcmp(value, "aes192") != 0) && (strcmp(value, "aes128") != 0) && (strcmp(value, "3des") != 0)) { *error_string = "Invalid cipher type"; return (0); } } if (strcmp(path, "totem.crypto_hash") == 0) { if ((strcmp(value, "none") != 0) && (strcmp(value, "md5") != 0) && (strcmp(value, "sha1") != 0) && (strcmp(value, "sha256") != 0) && (strcmp(value, "sha384") != 0) && (strcmp(value, "sha512") != 0)) { *error_string = "Invalid hash type"; return (0); } } break; case MAIN_CP_CB_DATA_STATE_SYSTEM: if (strcmp(path, "system.qb_ipc_type") == 0) { if ((strcmp(value, "native") != 0) && (strcmp(value, "shm") != 0) && (strcmp(value, "socket") != 0)) { *error_string = "Invalid system.qb_ipc_type"; return (0); } } if (strcmp(path, "system.sched_rr") == 0) { if ((strcmp(value, "yes") != 0) && (strcmp(value, "no") != 0)) { *error_string = "Invalid system.sched_rr value"; return (0); } } if (strcmp(path, "system.move_to_root_cgroup") == 0) { if ((strcmp(value, "yes") != 0) && (strcmp(value, "no") != 0)) { *error_string = "Invalid system.move_to_root_cgroup"; return (0); } } break; case MAIN_CP_CB_DATA_STATE_INTERFACE: if (strcmp(path, "totem.interface.linknumber") == 0) { val_type = ICMAP_VALUETYPE_UINT8; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } data->linknumber = val; add_as_string = 0; } if (strcmp(path, "totem.interface.bindnetaddr") == 0) { data->bindnetaddr = strdup(value); add_as_string = 0; } if (strcmp(path, "totem.interface.mcastaddr") == 0) { data->mcastaddr = strdup(value); add_as_string = 0; } if (strcmp(path, "totem.interface.broadcast") == 0) { data->broadcast = strdup(value); add_as_string = 0; } if (strcmp(path, "totem.interface.mcastport") == 0) { val_type = ICMAP_VALUETYPE_UINT16; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } data->mcastport = val; add_as_string = 0; } if (strcmp(path, "totem.interface.ttl") == 0) { val_type = ICMAP_VALUETYPE_UINT8; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } data->ttl = val; add_as_string = 0; } if (strcmp(path, "totem.interface.knet_link_priority") == 0) { val_type = ICMAP_VALUETYPE_UINT8; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } data->knet_link_priority = val; add_as_string = 0; } if (strcmp(path, "totem.interface.knet_ping_interval") == 0) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } data->knet_ping_interval = val; add_as_string = 0; } if (strcmp(path, "totem.interface.knet_ping_timeout") == 0) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } data->knet_ping_timeout = val; add_as_string = 0; } if (strcmp(path, "totem.interface.knet_ping_precision") == 0) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } data->knet_ping_precision = val; add_as_string = 0; } if (strcmp(path, "totem.interface.knet_pong_count") == 0) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } data->knet_pong_count = val; add_as_string = 0; } if (strcmp(path, "totem.interface.knet_transport") == 0) { val_type = ICMAP_VALUETYPE_STRING; data->knet_transport = strdup(value); add_as_string = 0; } break; case MAIN_CP_CB_DATA_STATE_LOGGER_SUBSYS: if (strcmp(key, "subsys") == 0) { data->subsys = strdup(value); if (data->subsys == NULL) { *error_string = "Can't alloc memory"; return (0); } } else { kv_item = malloc(sizeof(*kv_item)); if (kv_item == NULL) { *error_string = "Can't alloc memory"; return (0); } memset(kv_item, 0, sizeof(*kv_item)); kv_item->key = strdup(key); kv_item->value = strdup(value); if (kv_item->key == NULL || kv_item->value == NULL) { free(kv_item); *error_string = "Can't alloc memory"; return (0); } qb_list_init(&kv_item->list); qb_list_add(&kv_item->list, &data->logger_subsys_items_head); } add_as_string = 0; break; case MAIN_CP_CB_DATA_STATE_LOGGING_DAEMON: if (strcmp(key, "subsys") == 0) { data->subsys = strdup(value); if (data->subsys == NULL) { *error_string = "Can't alloc memory"; return (0); } } else if (strcmp(key, "name") == 0) { data->logging_daemon_name = strdup(value); if (data->logging_daemon_name == NULL) { *error_string = "Can't alloc memory"; return (0); } } else { kv_item = malloc(sizeof(*kv_item)); if (kv_item == NULL) { *error_string = "Can't alloc memory"; return (0); } memset(kv_item, 0, sizeof(*kv_item)); kv_item->key = strdup(key); kv_item->value = strdup(value); if (kv_item->key == NULL || kv_item->value == NULL) { free(kv_item); *error_string = "Can't alloc memory"; return (0); } qb_list_init(&kv_item->list); qb_list_add(&kv_item->list, &data->logger_subsys_items_head); } add_as_string = 0; break; case MAIN_CP_CB_DATA_STATE_UIDGID: if (strcmp(key, "uid") == 0) { uid = uid_determine(value); if (uid == -1) { *error_string = error_string_response; return (0); } snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "uidgid.config.uid.%u", uid); if ((cs_err = icmap_set_uint8_r(config_map, key_name, 1)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } else if (strcmp(key, "gid") == 0) { gid = gid_determine(value); if (gid == -1) { *error_string = error_string_response; return (0); } snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "uidgid.config.gid.%u", gid); if ((cs_err = icmap_set_uint8_r(config_map, key_name, 1)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } else { *error_string = "uidgid: Only uid and gid are allowed items"; return (0); } break; case MAIN_CP_CB_DATA_STATE_MEMBER: if (strcmp(key, "memberaddr") != 0) { *error_string = "Only memberaddr is allowed in member section"; return (0); } kv_item = malloc(sizeof(*kv_item)); if (kv_item == NULL) { *error_string = "Can't alloc memory"; return (0); } memset(kv_item, 0, sizeof(*kv_item)); kv_item->key = strdup(key); kv_item->value = strdup(value); if (kv_item->key == NULL || kv_item->value == NULL) { free(kv_item); *error_string = "Can't alloc memory"; return (0); } qb_list_init(&kv_item->list); qb_list_add(&kv_item->list, &data->member_items_head); add_as_string = 0; break; case MAIN_CP_CB_DATA_STATE_NODELIST: break; case MAIN_CP_CB_DATA_STATE_NODELIST_NODE: snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.%s", data->node_number, key); if ((strcmp(key, "nodeid") == 0) || (strcmp(key, "quorum_votes") == 0)) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } if ((cs_err = icmap_set_uint32_r(config_map, key_name, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } if (add_as_string) { if ((cs_err = icmap_set_string_r(config_map, key_name, value)) != CS_OK) { goto icmap_set_error; }; add_as_string = 0; } break; case MAIN_CP_CB_DATA_STATE_RESOURCES: if (strcmp(key, "watchdog_timeout") == 0) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto atoi_error; } if ((cs_err = icmap_set_uint32_r(config_map,path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } break; case MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM: case MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM_MEMUSED: if (strcmp(key, "poll_period") == 0) { if (str_to_ull(value, &ull) != 0) { goto atoi_error; } if ((cs_err = icmap_set_uint64_r(config_map,path, ull)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } break; case MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS: case MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS_MEMUSED: if (strcmp(key, "poll_period") == 0) { if (str_to_ull(value, &ull) != 0) { goto atoi_error; } if ((cs_err = icmap_set_uint64_r(config_map,path, ull)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } break; } if (add_as_string) { if ((cs_err = icmap_set_string_r(config_map, path, value)) != CS_OK) { goto icmap_set_error; } } break; case PARSER_CB_SECTION_START: if (strcmp(path, "totem.interface") == 0) { *state = MAIN_CP_CB_DATA_STATE_INTERFACE; data->linknumber = 0; data->mcastport = -1; data->ttl = -1; data->knet_link_priority = -1; data->knet_ping_interval = -1; data->knet_ping_timeout = -1; data->knet_ping_precision = -1; data->knet_pong_count = -1; data->knet_transport = NULL; qb_list_init(&data->member_items_head); }; if (strcmp(path, "totem") == 0) { *state = MAIN_CP_CB_DATA_STATE_TOTEM; }; if (strcmp(path, "system") == 0) { *state = MAIN_CP_CB_DATA_STATE_SYSTEM; } if (strcmp(path, "logging.logger_subsys") == 0) { *state = MAIN_CP_CB_DATA_STATE_LOGGER_SUBSYS; qb_list_init(&data->logger_subsys_items_head); data->subsys = NULL; } if (strcmp(path, "logging.logging_daemon") == 0) { *state = MAIN_CP_CB_DATA_STATE_LOGGING_DAEMON; qb_list_init(&data->logger_subsys_items_head); data->subsys = NULL; data->logging_daemon_name = NULL; } if (strcmp(path, "uidgid") == 0) { *state = MAIN_CP_CB_DATA_STATE_UIDGID; } if (strcmp(path, "totem.interface.member") == 0) { *state = MAIN_CP_CB_DATA_STATE_MEMBER; } if (strcmp(path, "quorum") == 0) { *state = MAIN_CP_CB_DATA_STATE_QUORUM; } if (strcmp(path, "quorum.device") == 0) { *state = MAIN_CP_CB_DATA_STATE_QDEVICE; } if (strcmp(path, "nodelist") == 0) { *state = MAIN_CP_CB_DATA_STATE_NODELIST; data->node_number = 0; } if (strcmp(path, "nodelist.node") == 0) { *state = MAIN_CP_CB_DATA_STATE_NODELIST_NODE; } if (strcmp(path, "resources") == 0) { *state = MAIN_CP_CB_DATA_STATE_RESOURCES; } if (strcmp(path, "resources.system") == 0) { *state = MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM; } if (strcmp(path, "resources.system.memory_used") == 0) { *state = MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM_MEMUSED; } if (strcmp(path, "resources.process") == 0) { *state = MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS; } if (strcmp(path, "resources.process.memory_used") == 0) { *state = MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS_MEMUSED; } break; case PARSER_CB_SECTION_END: switch (*state) { case MAIN_CP_CB_DATA_STATE_INTERFACE: /* * Create new interface section */ if (data->bindnetaddr != NULL) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.bindnetaddr", data->linknumber); cs_err = icmap_set_string_r(config_map, key_name, data->bindnetaddr); free(data->bindnetaddr); data->bindnetaddr = NULL; if (cs_err != CS_OK) { goto icmap_set_error; } } if (data->mcastaddr != NULL) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastaddr", data->linknumber); cs_err = icmap_set_string_r(config_map, key_name, data->mcastaddr); free(data->mcastaddr); data->mcastaddr = NULL; if (cs_err != CS_OK) { goto icmap_set_error; } } if (data->broadcast != NULL) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.broadcast", data->linknumber); cs_err = icmap_set_string_r(config_map, key_name, data->broadcast); free(data->broadcast); data->broadcast = NULL; if (cs_err != CS_OK) { goto icmap_set_error; } } if (data->mcastport > -1) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastport", data->linknumber); if ((cs_err = icmap_set_uint16_r(config_map, key_name, data->mcastport)) != CS_OK) { goto icmap_set_error; } } if (data->ttl > -1) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.ttl", data->linknumber); if ((cs_err = icmap_set_uint8_r(config_map, key_name, data->ttl)) != CS_OK) { goto icmap_set_error; } } if (data->knet_link_priority > -1) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_link_priority", data->linknumber); if ((cs_err = icmap_set_uint8_r(config_map, key_name, data->knet_link_priority)) != CS_OK) { goto icmap_set_error; } } if (data->knet_ping_interval > -1) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_interval", data->linknumber); if ((cs_err = icmap_set_uint32_r(config_map, key_name, data->knet_ping_interval)) != CS_OK) { goto icmap_set_error; } } if (data->knet_ping_timeout > -1) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_timeout", data->linknumber); if ((cs_err = icmap_set_uint32_r(config_map, key_name, data->knet_ping_timeout)) != CS_OK) { goto icmap_set_error; } } if (data->knet_ping_precision > -1) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_precision", data->linknumber); if ((cs_err = icmap_set_uint32_r(config_map, key_name, data->knet_ping_precision)) != CS_OK) { goto icmap_set_error; } } if (data->knet_pong_count > -1) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_pong_count", data->linknumber); if ((cs_err = icmap_set_uint32_r(config_map, key_name, data->knet_pong_count)) != CS_OK) { goto icmap_set_error; } } if (data->knet_transport) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_transport", data->linknumber); cs_err = icmap_set_string_r(config_map, key_name, data->knet_transport); free(data->knet_transport); if (cs_err != CS_OK) { goto icmap_set_error; } } ii = 0; qb_list_for_each_safe(iter, tmp_iter, &(data->member_items_head)) { kv_item = qb_list_entry(iter, struct key_value_list_item, list); snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.member.%u", data->linknumber, ii); cs_err = icmap_set_string_r(config_map, key_name, kv_item->value); free(kv_item->value); free(kv_item->key); free(kv_item); ii++; if (cs_err != CS_OK) { goto icmap_set_error; } } break; case MAIN_CP_CB_DATA_STATE_LOGGER_SUBSYS: if (data->subsys == NULL) { *error_string = "No subsys key in logger_subsys directive"; return (0); } qb_list_for_each_safe(iter, tmp_iter, &(data->logger_subsys_items_head)) { kv_item = qb_list_entry(iter, struct key_value_list_item, list); snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logger_subsys.%s.%s", data->subsys, kv_item->key); cs_err = icmap_set_string_r(config_map, key_name, kv_item->value); free(kv_item->value); free(kv_item->key); free(kv_item); if (cs_err != CS_OK) { goto icmap_set_error; } } snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logger_subsys.%s.subsys", data->subsys); cs_err = icmap_set_string_r(config_map, key_name, data->subsys); free(data->subsys); if (cs_err != CS_OK) { goto icmap_set_error; } break; case MAIN_CP_CB_DATA_STATE_LOGGING_DAEMON: if (data->logging_daemon_name == NULL) { *error_string = "No name key in logging_daemon directive"; return (0); } qb_list_for_each_safe(iter, tmp_iter, &(data->logger_subsys_items_head)) { kv_item = qb_list_entry(iter, struct key_value_list_item, list); if (data->subsys == NULL) { if (strcmp(data->logging_daemon_name, "corosync") == 0) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.%s", kv_item->key); } else { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logging_daemon.%s.%s", data->logging_daemon_name, kv_item->key); } } else { if (strcmp(data->logging_daemon_name, "corosync") == 0) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logger_subsys.%s.%s", data->subsys, kv_item->key); } else { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logging_daemon.%s.%s.%s", data->logging_daemon_name, data->subsys, kv_item->key); } } cs_err = icmap_set_string_r(config_map, key_name, kv_item->value); free(kv_item->value); free(kv_item->key); free(kv_item); if (cs_err != CS_OK) { goto icmap_set_error; } } if (data->subsys == NULL) { if (strcmp(data->logging_daemon_name, "corosync") != 0) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logging_daemon.%s.name", data->logging_daemon_name); cs_err = icmap_set_string_r(config_map, key_name, data->logging_daemon_name); } } else { if (strcmp(data->logging_daemon_name, "corosync") == 0) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logger_subsys.%s.subsys", data->subsys); cs_err = icmap_set_string_r(config_map, key_name, data->subsys); } else { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logging_daemon.%s.%s.subsys", data->logging_daemon_name, data->subsys); cs_err = icmap_set_string_r(config_map, key_name, data->subsys); if (cs_err != CS_OK) { free(data->subsys); free(data->logging_daemon_name); goto icmap_set_error; } snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logging_daemon.%s.%s.name", data->logging_daemon_name, data->subsys); cs_err = icmap_set_string_r(config_map, key_name, data->logging_daemon_name); } } free(data->subsys); free(data->logging_daemon_name); if (cs_err != CS_OK) { goto icmap_set_error; } break; case MAIN_CP_CB_DATA_STATE_NODELIST_NODE: data->node_number++; break; case MAIN_CP_CB_DATA_STATE_NORMAL: case MAIN_CP_CB_DATA_STATE_PLOAD: case MAIN_CP_CB_DATA_STATE_UIDGID: case MAIN_CP_CB_DATA_STATE_MEMBER: case MAIN_CP_CB_DATA_STATE_QUORUM: case MAIN_CP_CB_DATA_STATE_QDEVICE: case MAIN_CP_CB_DATA_STATE_NODELIST: case MAIN_CP_CB_DATA_STATE_TOTEM: case MAIN_CP_CB_DATA_STATE_SYSTEM: break; case MAIN_CP_CB_DATA_STATE_RESOURCES: *state = MAIN_CP_CB_DATA_STATE_NORMAL; break; case MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM: *state = MAIN_CP_CB_DATA_STATE_RESOURCES; break; case MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM_MEMUSED: *state = MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM; break; case MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS: *state = MAIN_CP_CB_DATA_STATE_RESOURCES; break; case MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS_MEMUSED: *state = MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS; break; } break; } return (1); atoi_error: min_val = max_val = 0; /* * This is really assert, because developer ether doesn't set val_type correctly or * we've got here after some nasty memory overwrite */ assert(safe_atoq_range(val_type, &min_val, &max_val) == 0); if (snprintf(formated_err, sizeof(formated_err), "Value of key \"%s\" is expected to be integer in range (%lld..%lld), but \"%s\" was given", key_name, min_val, max_val, value) >= sizeof(formated_err)) { *error_string = "Can't format parser error message"; } else { *error_string = formated_err; } return (0); icmap_set_error: if (snprintf(formated_err, sizeof(formated_err), "Can't store key \"%s\" into icmap, returned error is %s", key_name, cs_strerror(cs_err)) >= sizeof(formated_err)) { *error_string = "Can't format parser error message"; } else { *error_string = formated_err; } return (0); } static int uidgid_config_parser_cb(const char *path, char *key, char *value, enum main_cp_cb_data_state *state, enum parser_cb_type type, const char **error_string, icmap_map_t config_map, void *user_data) { char key_name[ICMAP_KEYNAME_MAXLEN]; int uid, gid; static char formated_err[256]; cs_error_t cs_err; switch (type) { case PARSER_CB_START: break; case PARSER_CB_END: break; case PARSER_CB_ITEM: if (strcmp(path, "uidgid.uid") == 0) { uid = uid_determine(value); if (uid == -1) { *error_string = error_string_response; return (0); } snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "uidgid.config.uid.%u", uid); if ((cs_err = icmap_set_uint8_r(config_map, key_name, 1)) != CS_OK) { goto icmap_set_error; } } else if (strcmp(path, "uidgid.gid") == 0) { gid = gid_determine(value); if (gid == -1) { *error_string = error_string_response; return (0); } snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "uidgid.config.gid.%u", gid); if ((cs_err = icmap_set_uint8_r(config_map, key_name, 1)) != CS_OK) { goto icmap_set_error; } } else { *error_string = "uidgid: Only uid and gid are allowed items"; return (0); } break; case PARSER_CB_SECTION_START: if (strcmp(path, "uidgid") != 0) { *error_string = "uidgid: Can't add subsection different than uidgid"; return (0); }; break; case PARSER_CB_SECTION_END: break; } return (1); icmap_set_error: if (snprintf(formated_err, sizeof(formated_err), "Can't store key \"%s\" into icmap, returned error is %s", key_name, cs_strerror(cs_err)) >= sizeof(formated_err)) { *error_string = "Can't format parser error message"; } else { *error_string = formated_err; } return (0); } static int read_uidgid_files_into_icmap( const char **error_string, icmap_map_t config_map) { FILE *fp; char *dirname_res; DIR *dp; struct dirent *dirent; char filename[PATH_MAX + FILENAME_MAX + 1]; char uidgid_dirname[PATH_MAX + FILENAME_MAX + 1]; int res = 0; struct stat stat_buf; enum main_cp_cb_data_state state = MAIN_CP_CB_DATA_STATE_NORMAL; char key_name[ICMAP_KEYNAME_MAXLEN]; int line_no; /* * Build uidgid directory based on corosync.conf file location */ res = snprintf(filename, sizeof(filename), "%s", corosync_get_config_file()); if (res >= sizeof(filename)) { *error_string = "uidgid.d path too long"; return (-1); } dirname_res = dirname(filename); res = snprintf(uidgid_dirname, sizeof(uidgid_dirname), "%s/%s", dirname_res, "uidgid.d"); if (res >= sizeof(uidgid_dirname)) { *error_string = "uidgid.d path too long"; return (-1); } dp = opendir (uidgid_dirname); if (dp == NULL) return 0; for (dirent = readdir(dp); dirent != NULL; dirent = readdir(dp)) { res = snprintf(filename, sizeof (filename), "%s/%s", uidgid_dirname, dirent->d_name); if (res >= sizeof(filename)) { res = -1; *error_string = "uidgid.d dirname path too long"; goto error_exit; } res = stat (filename, &stat_buf); if (res == 0 && S_ISREG(stat_buf.st_mode)) { fp = fopen (filename, "r"); if (fp == NULL) continue; key_name[0] = 0; line_no = 0; res = parse_section(fp, filename, &line_no, key_name, error_string, 0, state, uidgid_config_parser_cb, config_map, NULL); fclose (fp); if (res != 0) { goto error_exit; } } } error_exit: closedir(dp); return res; } /* Read config file and load into icmap */ static int read_config_file_into_icmap( const char **error_string, icmap_map_t config_map) { FILE *fp; const char *filename; char *error_reason = error_string_response; int res; char key_name[ICMAP_KEYNAME_MAXLEN]; struct main_cp_cb_data data; enum main_cp_cb_data_state state = MAIN_CP_CB_DATA_STATE_NORMAL; int line_no; filename = corosync_get_config_file(); fp = fopen (filename, "r"); if (fp == NULL) { char error_str[100]; const char *error_ptr = qb_strerror_r(errno, error_str, sizeof(error_str)); snprintf (error_reason, sizeof(error_string_response), "Can't read file %s reason = (%s)", filename, error_ptr); *error_string = error_reason; return -1; } key_name[0] = 0; line_no = 0; res = parse_section(fp, filename, &line_no, key_name, error_string, 0, state, main_config_parser_cb, config_map, &data); fclose(fp); if (res == 0) { res = read_uidgid_files_into_icmap(error_string, config_map); } if (res == 0) { snprintf (error_reason, sizeof(error_string_response), "Successfully read main configuration file '%s'.", filename); *error_string = error_reason; } return res; } diff --git a/exec/main.c b/exec/main.c index 0e5e8e2d..9c422694 100644 --- a/exec/main.c +++ b/exec/main.c @@ -1,1590 +1,1586 @@ /* * Copyright (c) 2002-2006 MontaVista Software, Inc. * Copyright (c) 2006-2018 Red Hat, Inc. * * All rights reserved. * * Author: Steven Dake (sdake@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the MontaVista Software, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ /** * \mainpage Corosync * * This is the doxygen generated developer documentation for the Corosync * project. For more information about Corosync, please see the project * web site, corosync.org. * * \section license License * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the MontaVista Software, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HAVE_LIBSYSTEMD #include #endif #include #include #include #include #include #include #include #include #include #include #include #include "quorum.h" #include "totemsrp.h" #include "logconfig.h" #include "totemconfig.h" #include "main.h" #include "sync.h" #include "timer.h" #include "util.h" #include "apidef.h" #include "service.h" #include "schedwrk.h" #include "ipcs_stats.h" #include "stats.h" #ifdef HAVE_SMALL_MEMORY_FOOTPRINT #define IPC_LOGSYS_SIZE 1024*64 #else #define IPC_LOGSYS_SIZE 8192*128 #endif /* * LibQB adds default "*" syslog filter so we have to set syslog_priority as low * as possible so filters applied later in _logsys_config_apply_per_file takes * effect. */ LOGSYS_DECLARE_SYSTEM ("corosync", LOGSYS_MODE_OUTPUT_STDERR | LOGSYS_MODE_OUTPUT_SYSLOG, LOG_DAEMON, LOG_EMERG); LOGSYS_DECLARE_SUBSYS ("MAIN"); #define SERVER_BACKLOG 5 static int sched_priority = 0; static unsigned int service_count = 32; static struct totem_logging_configuration totem_logging_configuration; static struct corosync_api_v1 *api = NULL; static int sync_in_process = 1; static qb_loop_t *corosync_poll_handle; struct sched_param global_sched_param; static corosync_timer_handle_t corosync_stats_timer_handle; static const char *corosync_lock_file = LOCALSTATEDIR"/run/corosync.pid"; -static int ip_version = AF_INET; - static char corosync_config_file[PATH_MAX + 1] = COROSYSCONFDIR "/corosync.conf"; qb_loop_t *cs_poll_handle_get (void) { return (corosync_poll_handle); } int cs_poll_dispatch_add (qb_loop_t * handle, int fd, int events, void *data, int (*dispatch_fn) (int fd, int revents, void *data)) { return qb_loop_poll_add(handle, QB_LOOP_MED, fd, events, data, dispatch_fn); } int cs_poll_dispatch_delete(qb_loop_t * handle, int fd) { return qb_loop_poll_del(handle, fd); } void corosync_state_dump (void) { int i; for (i = 0; i < SERVICES_COUNT_MAX; i++) { if (corosync_service[i] && corosync_service[i]->exec_dump_fn) { corosync_service[i]->exec_dump_fn (); } } } const char *corosync_get_config_file(void) { return (corosync_config_file); } static void corosync_blackbox_write_to_file (void) { char fname[PATH_MAX]; char fdata_fname[PATH_MAX]; char time_str[PATH_MAX]; struct tm cur_time_tm; time_t cur_time_t; ssize_t res; cur_time_t = time(NULL); localtime_r(&cur_time_t, &cur_time_tm); strftime(time_str, PATH_MAX, "%Y-%m-%dT%H:%M:%S", &cur_time_tm); if (snprintf(fname, PATH_MAX, "%s/fdata-%s-%lld", get_run_dir(), time_str, (long long int)getpid()) >= PATH_MAX) { log_printf(LOGSYS_LEVEL_ERROR, "Can't snprintf blackbox file name"); return ; } if ((res = qb_log_blackbox_write_to_file(fname)) < 0) { LOGSYS_PERROR(-res, LOGSYS_LEVEL_ERROR, "Can't store blackbox file"); return ; } snprintf(fdata_fname, sizeof(fdata_fname), "%s/fdata", get_run_dir()); unlink(fdata_fname); if (symlink(fname, fdata_fname) == -1) { log_printf(LOGSYS_LEVEL_ERROR, "Can't create symlink to '%s' for corosync blackbox file '%s'", fname, fdata_fname); } } static void unlink_all_completed (void) { api->timer_delete (corosync_stats_timer_handle); qb_loop_stop (corosync_poll_handle); icmap_fini(); } void corosync_shutdown_request (void) { corosync_service_unlink_all (api, unlink_all_completed); } static int32_t sig_diag_handler (int num, void *data) { corosync_state_dump (); return 0; } static int32_t sig_exit_handler (int num, void *data) { log_printf(LOGSYS_LEVEL_NOTICE, "Node was shut down by a signal"); corosync_service_unlink_all (api, unlink_all_completed); return 0; } static void sigsegv_handler (int num) { (void)signal (num, SIG_DFL); corosync_blackbox_write_to_file (); qb_log_fini(); raise (num); } #define LOCALHOST_IP inet_addr("127.0.0.1") static void *corosync_group_handle; static struct totempg_group corosync_group = { .group = "a", .group_len = 1 }; static void serialize_lock (void) { } static void serialize_unlock (void) { } static void corosync_sync_completed (void) { log_printf (LOGSYS_LEVEL_NOTICE, "Completed service synchronization, ready to provide service."); sync_in_process = 0; cs_ipcs_sync_state_changed(sync_in_process); cs_ipc_allow_connections(1); /* * Inform totem to start using new message queue again */ totempg_trans_ack(); #ifdef HAVE_LIBSYSTEMD sd_notify (0, "READY=1"); #endif } static int corosync_sync_callbacks_retrieve ( int service_id, struct sync_callbacks *callbacks) { if (corosync_service[service_id] == NULL) { return (-1); } if (callbacks == NULL) { return (0); } callbacks->name = corosync_service[service_id]->name; callbacks->sync_init = corosync_service[service_id]->sync_init; callbacks->sync_process = corosync_service[service_id]->sync_process; callbacks->sync_activate = corosync_service[service_id]->sync_activate; callbacks->sync_abort = corosync_service[service_id]->sync_abort; return (0); } static struct memb_ring_id corosync_ring_id; static void member_object_joined (unsigned int nodeid) { char member_ip[ICMAP_KEYNAME_MAXLEN]; char member_join_count[ICMAP_KEYNAME_MAXLEN]; char member_status[ICMAP_KEYNAME_MAXLEN]; snprintf(member_ip, ICMAP_KEYNAME_MAXLEN, "runtime.members.%u.ip", nodeid); snprintf(member_join_count, ICMAP_KEYNAME_MAXLEN, "runtime.members.%u.join_count", nodeid); snprintf(member_status, ICMAP_KEYNAME_MAXLEN, "runtime.members.%u.status", nodeid); if (icmap_get(member_ip, NULL, NULL, NULL) == CS_OK) { icmap_inc(member_join_count); icmap_set_string(member_status, "joined"); } else { icmap_set_string(member_ip, (char*)api->totem_ifaces_print (nodeid)); icmap_set_uint32(member_join_count, 1); icmap_set_string(member_status, "joined"); } log_printf (LOGSYS_LEVEL_DEBUG, "Member joined: %s", api->totem_ifaces_print (nodeid)); } static void member_object_left (unsigned int nodeid) { char member_status[ICMAP_KEYNAME_MAXLEN]; snprintf(member_status, ICMAP_KEYNAME_MAXLEN, "runtime.members.%u.status", nodeid); icmap_set_string(member_status, "left"); log_printf (LOGSYS_LEVEL_DEBUG, "Member left: %s", api->totem_ifaces_print (nodeid)); } static void confchg_fn ( enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id) { int i; int abort_activate = 0; if (sync_in_process == 1) { abort_activate = 1; } sync_in_process = 1; cs_ipcs_sync_state_changed(sync_in_process); memcpy (&corosync_ring_id, ring_id, sizeof (struct memb_ring_id)); for (i = 0; i < left_list_entries; i++) { member_object_left (left_list[i]); } for (i = 0; i < joined_list_entries; i++) { member_object_joined (joined_list[i]); } /* * Call configuration change for all services */ for (i = 0; i < service_count; i++) { if (corosync_service[i] && corosync_service[i]->confchg_fn) { corosync_service[i]->confchg_fn (configuration_type, member_list, member_list_entries, left_list, left_list_entries, joined_list, joined_list_entries, ring_id); } } if (abort_activate) { sync_abort (); } if (configuration_type == TOTEM_CONFIGURATION_TRANSITIONAL) { sync_save_transitional (member_list, member_list_entries, ring_id); } if (configuration_type == TOTEM_CONFIGURATION_REGULAR) { sync_start (member_list, member_list_entries, ring_id); } } static void priv_drop (void) { return; /* TODO: we are still not dropping privs */ } static void corosync_tty_detach (void) { int devnull; /* * Disconnect from TTY if this is not a debug run */ switch (fork ()) { case -1: corosync_exit_error (COROSYNC_DONE_FORK); break; case 0: /* * child which is disconnected, run this process */ break; default: exit (0); break; } /* Create new session */ (void)setsid(); /* * Map stdin/out/err to /dev/null. */ devnull = open("/dev/null", O_RDWR); if (devnull == -1) { corosync_exit_error (COROSYNC_DONE_STD_TO_NULL_REDIR); } if (dup2(devnull, 0) < 0 || dup2(devnull, 1) < 0 || dup2(devnull, 2) < 0) { close(devnull); corosync_exit_error (COROSYNC_DONE_STD_TO_NULL_REDIR); } close(devnull); } static void corosync_mlockall (void) { int res; struct rlimit rlimit; rlimit.rlim_cur = RLIM_INFINITY; rlimit.rlim_max = RLIM_INFINITY; #ifndef RLIMIT_MEMLOCK #define RLIMIT_MEMLOCK RLIMIT_VMEM #endif setrlimit (RLIMIT_MEMLOCK, &rlimit); res = mlockall (MCL_CURRENT | MCL_FUTURE); if (res == -1) { LOGSYS_PERROR (errno, LOGSYS_LEVEL_WARNING, "Could not lock memory of service to avoid page faults"); }; } static void corosync_totem_stats_updater (void *data) { totempg_stats_t * stats; uint32_t total_mtt_rx_token; uint32_t total_backlog_calc; uint32_t total_token_holdtime; int t, prev; int32_t token_count; const char *cstr; stats = api->totem_get_stats(); stats->srp->firewall_enabled_or_nic_failure = stats->srp->continuous_gather > MAX_NO_CONT_GATHER ? 1 : 0; if (stats->srp->continuous_gather > MAX_NO_CONT_GATHER || stats->srp->continuous_sendmsg_failures > MAX_NO_CONT_SENDMSG_FAILURES) { cstr = ""; if (stats->srp->continuous_sendmsg_failures > MAX_NO_CONT_SENDMSG_FAILURES) { cstr = "number of multicast sendmsg failures is above threshold"; } if (stats->srp->continuous_gather > MAX_NO_CONT_GATHER) { cstr = "totem is continuously in gather state"; } log_printf (LOGSYS_LEVEL_WARNING, "Totem is unable to form a cluster because of an " "operating system or network fault (reason: %s). The most common " "cause of this message is that the local firewall is " "configured improperly.", cstr); stats->srp->firewall_enabled_or_nic_failure = 1; } else { stats->srp->firewall_enabled_or_nic_failure = 0; } total_mtt_rx_token = 0; total_token_holdtime = 0; total_backlog_calc = 0; token_count = 0; t = stats->srp->latest_token; while (1) { if (t == 0) prev = TOTEM_TOKEN_STATS_MAX - 1; else prev = t - 1; if (prev == stats->srp->earliest_token) break; /* if tx == 0, then dropped token (not ours) */ if (stats->srp->token[t].tx != 0 || (stats->srp->token[t].rx - stats->srp->token[prev].rx) > 0 ) { total_mtt_rx_token += (stats->srp->token[t].rx - stats->srp->token[prev].rx); total_token_holdtime += (stats->srp->token[t].tx - stats->srp->token[t].rx); total_backlog_calc += stats->srp->token[t].backlog_calc; token_count++; } t = prev; } if (token_count) { stats->srp->mtt_rx_token = (total_mtt_rx_token / token_count); stats->srp->avg_token_workload = (total_token_holdtime / token_count); stats->srp->avg_backlog_calc = (total_backlog_calc / token_count); } stats->srp->time_since_token_last_received = qb_util_nano_current_get () / QB_TIME_NS_IN_MSEC - stats->srp->token[stats->srp->latest_token].rx; stats_trigger_trackers(); api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL, corosync_totem_stats_updater, &corosync_stats_timer_handle); } static void corosync_totem_stats_init (void) { /* start stats timer */ api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL, corosync_totem_stats_updater, &corosync_stats_timer_handle); } static void deliver_fn ( unsigned int nodeid, const void *msg, unsigned int msg_len, int endian_conversion_required) { const struct qb_ipc_request_header *header; int32_t service; int32_t fn_id; uint32_t id; header = msg; if (endian_conversion_required) { id = swab32 (header->id); } else { id = header->id; } /* * Call the proper executive handler */ service = id >> 16; fn_id = id & 0xffff; if (!corosync_service[service]) { return; } if (fn_id >= corosync_service[service]->exec_engine_count) { log_printf(LOGSYS_LEVEL_WARNING, "discarded unknown message %d for service %d (max id %d)", fn_id, service, corosync_service[service]->exec_engine_count); return; } icmap_fast_inc(service_stats_rx[service][fn_id]); if (endian_conversion_required) { assert(corosync_service[service]->exec_engine[fn_id].exec_endian_convert_fn != NULL); corosync_service[service]->exec_engine[fn_id].exec_endian_convert_fn ((void *)msg); } corosync_service[service]->exec_engine[fn_id].exec_handler_fn (msg, nodeid); } int main_mcast ( const struct iovec *iovec, unsigned int iov_len, unsigned int guarantee) { const struct qb_ipc_request_header *req = iovec->iov_base; int32_t service; int32_t fn_id; service = req->id >> 16; fn_id = req->id & 0xffff; if (corosync_service[service]) { icmap_fast_inc(service_stats_tx[service][fn_id]); } return (totempg_groups_mcast_joined (corosync_group_handle, iovec, iov_len, guarantee)); } static void corosync_ring_id_create_or_load ( struct memb_ring_id *memb_ring_id, unsigned int nodeid) { int fd; int res = 0; char filename[PATH_MAX]; snprintf (filename, sizeof(filename), "%s/ringid_%u", get_run_dir(), nodeid); fd = open (filename, O_RDONLY, 0700); /* * If file can be opened and read, read the ring id */ if (fd != -1) { res = read (fd, &memb_ring_id->seq, sizeof (uint64_t)); close (fd); } /* * If file could not be opened or read, create a new ring id */ if ((fd == -1) || (res != sizeof (uint64_t))) { memb_ring_id->seq = 0; umask(0); fd = open (filename, O_CREAT|O_RDWR, 0700); if (fd != -1) { res = write (fd, &memb_ring_id->seq, sizeof (uint64_t)); close (fd); if (res == -1) { LOGSYS_PERROR (errno, LOGSYS_LEVEL_ERROR, "Couldn't write ringid file '%s'", filename); corosync_exit_error (COROSYNC_DONE_STORE_RINGID); } } else { LOGSYS_PERROR (errno, LOGSYS_LEVEL_ERROR, "Couldn't create ringid file '%s'", filename); corosync_exit_error (COROSYNC_DONE_STORE_RINGID); } } memb_ring_id->rep = nodeid; } static void corosync_ring_id_store ( const struct memb_ring_id *memb_ring_id, unsigned int nodeid) { char filename[PATH_MAX]; int fd; int res; snprintf (filename, sizeof(filename), "%s/ringid_%u", get_run_dir(), nodeid); fd = open (filename, O_WRONLY, 0700); if (fd == -1) { fd = open (filename, O_CREAT|O_RDWR, 0700); } if (fd == -1) { LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "Couldn't store new ring id %llx to stable storage", memb_ring_id->seq); corosync_exit_error (COROSYNC_DONE_STORE_RINGID); } log_printf (LOGSYS_LEVEL_DEBUG, "Storing new sequence id for ring %llx", memb_ring_id->seq); res = write (fd, &memb_ring_id->seq, sizeof(memb_ring_id->seq)); close (fd); if (res != sizeof(memb_ring_id->seq)) { LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "Couldn't store new ring id %llx to stable storage", memb_ring_id->seq); corosync_exit_error (COROSYNC_DONE_STORE_RINGID); } } static qb_loop_timer_handle recheck_the_q_level_timer; void corosync_recheck_the_q_level(void *data) { totempg_check_q_level(corosync_group_handle); if (cs_ipcs_q_level_get() == TOTEM_Q_LEVEL_CRITICAL) { qb_loop_timer_add(cs_poll_handle_get(), QB_LOOP_MED, 1*QB_TIME_NS_IN_MSEC, NULL, corosync_recheck_the_q_level, &recheck_the_q_level_timer); } } struct sending_allowed_private_data_struct { int reserved_msgs; }; int corosync_sending_allowed ( unsigned int service, unsigned int id, const void *msg, void *sending_allowed_private_data) { struct sending_allowed_private_data_struct *pd = (struct sending_allowed_private_data_struct *)sending_allowed_private_data; struct iovec reserve_iovec; struct qb_ipc_request_header *header = (struct qb_ipc_request_header *)msg; int sending_allowed; reserve_iovec.iov_base = (char *)header; reserve_iovec.iov_len = header->size; pd->reserved_msgs = totempg_groups_joined_reserve ( corosync_group_handle, &reserve_iovec, 1); if (pd->reserved_msgs == -1) { return -EINVAL; } /* Message ID out of range */ if (id >= corosync_service[service]->lib_engine_count) { return -EINVAL; } sending_allowed = QB_FALSE; if (corosync_quorum_is_quorate() == 1 || corosync_service[service]->allow_inquorate == CS_LIB_ALLOW_INQUORATE) { // we are quorate // now check flow control if (corosync_service[service]->lib_engine[id].flow_control == CS_LIB_FLOW_CONTROL_NOT_REQUIRED) { sending_allowed = QB_TRUE; } else if (pd->reserved_msgs && sync_in_process == 0) { sending_allowed = QB_TRUE; } else if (pd->reserved_msgs == 0) { return -ENOBUFS; } else /* (sync_in_process) */ { return -EINPROGRESS; } } else { return -EHOSTUNREACH; } return (sending_allowed); } void corosync_sending_allowed_release (void *sending_allowed_private_data) { struct sending_allowed_private_data_struct *pd = (struct sending_allowed_private_data_struct *)sending_allowed_private_data; if (pd->reserved_msgs == -1) { return; } totempg_groups_joined_release (pd->reserved_msgs); } int message_source_is_local (const mar_message_source_t *source) { int ret = 0; assert (source != NULL); if (source->nodeid == totempg_my_nodeid_get ()) { ret = 1; } return ret; } void message_source_set ( mar_message_source_t *source, void *conn) { assert ((source != NULL) && (conn != NULL)); memset (source, 0, sizeof (mar_message_source_t)); source->nodeid = totempg_my_nodeid_get (); source->conn = conn; } struct scheduler_pause_timeout_data { struct totem_config *totem_config; qb_loop_timer_handle handle; unsigned long long tv_prev; unsigned long long max_tv_diff; }; static void timer_function_scheduler_timeout (void *data) { struct scheduler_pause_timeout_data *timeout_data = (struct scheduler_pause_timeout_data *)data; unsigned long long tv_current; unsigned long long tv_diff; tv_current = qb_util_nano_current_get (); if (timeout_data->tv_prev == 0) { /* * Initial call -> just pretent everything is ok */ timeout_data->tv_prev = tv_current; timeout_data->max_tv_diff = 0; } tv_diff = tv_current - timeout_data->tv_prev; timeout_data->tv_prev = tv_current; if (tv_diff > timeout_data->max_tv_diff) { log_printf (LOGSYS_LEVEL_WARNING, "Corosync main process was not scheduled for %0.4f ms " "(threshold is %0.4f ms). Consider token timeout increase.", (float)tv_diff / QB_TIME_NS_IN_MSEC, (float)timeout_data->max_tv_diff / QB_TIME_NS_IN_MSEC); } /* * Set next threshold, because token_timeout can change */ timeout_data->max_tv_diff = timeout_data->totem_config->token_timeout * QB_TIME_NS_IN_MSEC * 0.8; qb_loop_timer_add (corosync_poll_handle, QB_LOOP_MED, timeout_data->totem_config->token_timeout * QB_TIME_NS_IN_MSEC / 3, timeout_data, timer_function_scheduler_timeout, &timeout_data->handle); } static int corosync_set_rr_scheduler (void) { int ret_val = 0; #if defined(HAVE_PTHREAD_SETSCHEDPARAM) && defined(HAVE_SCHED_GET_PRIORITY_MAX) && defined(HAVE_SCHED_SETSCHEDULER) int res; sched_priority = sched_get_priority_max (SCHED_RR); if (sched_priority != -1) { global_sched_param.sched_priority = sched_priority; res = sched_setscheduler (0, SCHED_RR, &global_sched_param); if (res == -1) { LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "Could not set SCHED_RR at priority %d", global_sched_param.sched_priority); global_sched_param.sched_priority = 0; #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET qb_log_thread_priority_set (SCHED_OTHER, 0); #endif ret_val = -1; } else { /* * Turn on SCHED_RR in logsys system */ #ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET res = qb_log_thread_priority_set (SCHED_RR, sched_priority); #else res = -1; #endif if (res == -1) { log_printf (LOGSYS_LEVEL_ERROR, "Could not set logsys thread priority." " Can't continue because of priority inversions."); corosync_exit_error (COROSYNC_DONE_LOGSETUP); } } } else { LOGSYS_PERROR (errno, LOGSYS_LEVEL_WARNING, "Could not get maximum scheduler priority"); sched_priority = 0; ret_val = -1; } #else log_printf(LOGSYS_LEVEL_WARNING, "The Platform is missing process priority setting features. Leaving at default."); ret_val = -1; #endif return (ret_val); } /* The basename man page contains scary warnings about thread-safety and portability, hence this */ static const char *corosync_basename(const char *file_name) { char *base; base = strrchr (file_name, '/'); if (base) { return base + 1; } return file_name; } static void _logsys_log_printf(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format, ...) __attribute__((format(printf, 6, 7))); static void _logsys_log_printf(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format, ...) { va_list ap; va_start(ap, format); qb_log_from_external_source_va(function_name, corosync_basename(file_name), format, level, file_line, subsys, ap); va_end(ap); } static void fplay_key_change_notify_fn ( int32_t event, const char *key_name, struct icmap_notify_value new_val, struct icmap_notify_value old_val, void *user_data) { if (strcmp(key_name, "runtime.blackbox.dump_flight_data") == 0) { fprintf(stderr,"Writetofile\n"); corosync_blackbox_write_to_file (); } if (strcmp(key_name, "runtime.blackbox.dump_state") == 0) { fprintf(stderr,"statefump\n"); corosync_state_dump (); } } static void corosync_fplay_control_init (void) { icmap_track_t track = NULL; icmap_set_string("runtime.blackbox.dump_flight_data", "no"); icmap_set_string("runtime.blackbox.dump_state", "no"); icmap_track_add("runtime.blackbox.dump_flight_data", ICMAP_TRACK_ADD | ICMAP_TRACK_DELETE | ICMAP_TRACK_MODIFY, fplay_key_change_notify_fn, NULL, &track); icmap_track_add("runtime.blackbox.dump_state", ICMAP_TRACK_ADD | ICMAP_TRACK_DELETE | ICMAP_TRACK_MODIFY, fplay_key_change_notify_fn, NULL, &track); } static void force_gather_notify_fn( int32_t event, const char *key_name, struct icmap_notify_value new_val, struct icmap_notify_value old_val, void *user_data) { char *key_val; if (icmap_get_string(key_name, &key_val) == CS_OK && strcmp(key_val, "no") == 0) goto out; icmap_set_string("runtime.force_gather", "no"); if (strcmp(key_name, "runtime.force_gather") == 0) { log_printf(LOGSYS_LEVEL_ERROR, "Forcing into GATHER state\n"); totempg_force_gather(); } out: free(key_val); } static void corosync_force_gather_init (void) { icmap_track_t track = NULL; icmap_set_string("runtime.force_gather", "no"); icmap_track_add("runtime.force_gather", ICMAP_TRACK_ADD | ICMAP_TRACK_DELETE | ICMAP_TRACK_MODIFY, force_gather_notify_fn, NULL, &track); } /* * Set RO flag for keys, which ether doesn't make sense to change by user (statistic) * or which when changed are not reflected by runtime (totem.crypto_cipher, ...). * * Also some RO keys cannot be determined in this stage, so they are set later in * other functions (like nodelist.local_node_pos, ...) */ static void set_icmap_ro_keys_flag (void) { /* * Set RO flag for all keys of internal configuration and runtime statistics */ icmap_set_ro_access("internal_configuration.", CS_TRUE, CS_TRUE); icmap_set_ro_access("runtime.services.", CS_TRUE, CS_TRUE); icmap_set_ro_access("runtime.config.", CS_TRUE, CS_TRUE); icmap_set_ro_access("runtime.totem.", CS_TRUE, CS_TRUE); icmap_set_ro_access("uidgid.config.", CS_TRUE, CS_TRUE); icmap_set_ro_access("system.", CS_TRUE, CS_TRUE); icmap_set_ro_access("nodelist.", CS_TRUE, CS_TRUE); /* * Set RO flag for constrete keys of configuration which can't be changed * during runtime */ icmap_set_ro_access("totem.crypto_cipher", CS_FALSE, CS_TRUE); icmap_set_ro_access("totem.crypto_hash", CS_FALSE, CS_TRUE); icmap_set_ro_access("totem.secauth", CS_FALSE, CS_TRUE); icmap_set_ro_access("totem.ip_version", CS_FALSE, CS_TRUE); icmap_set_ro_access("totem.rrp_mode", CS_FALSE, CS_TRUE); icmap_set_ro_access("totem.transport", CS_FALSE, CS_TRUE); icmap_set_ro_access("totem.cluster_name", CS_FALSE, CS_TRUE); icmap_set_ro_access("totem.netmtu", CS_FALSE, CS_TRUE); icmap_set_ro_access("totem.threads", CS_FALSE, CS_TRUE); icmap_set_ro_access("totem.version", CS_FALSE, CS_TRUE); icmap_set_ro_access("totem.nodeid", CS_FALSE, CS_TRUE); icmap_set_ro_access("totem.clear_node_high_bit", CS_FALSE, CS_TRUE); icmap_set_ro_access("config.reload_in_progress", CS_FALSE, CS_TRUE); icmap_set_ro_access("config.totemconfig_reload_in_progress", CS_FALSE, CS_TRUE); } static void main_service_ready (void) { int res; /* * This must occur after totempg is initialized because "this_ip" must be set */ res = corosync_service_defaults_link_and_init (api); if (res == -1) { log_printf (LOGSYS_LEVEL_ERROR, "Could not initialize default services"); corosync_exit_error (COROSYNC_DONE_INIT_SERVICES); } cs_ipcs_init(); corosync_totem_stats_init (); corosync_fplay_control_init (); corosync_force_gather_init (); sync_init ( corosync_sync_callbacks_retrieve, corosync_sync_completed); } static enum e_corosync_done corosync_flock (const char *lockfile, pid_t pid) { struct flock lock; enum e_corosync_done err; char pid_s[17]; int fd_flag; int lf; err = COROSYNC_DONE_EXIT; lf = open (lockfile, O_WRONLY | O_CREAT, 0640); if (lf == -1) { log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't create lock file."); return (COROSYNC_DONE_ACQUIRE_LOCK); } retry_fcntl: lock.l_type = F_WRLCK; lock.l_start = 0; lock.l_whence = SEEK_SET; lock.l_len = 0; if (fcntl (lf, F_SETLK, &lock) == -1) { switch (errno) { case EINTR: goto retry_fcntl; break; case EAGAIN: case EACCES: log_printf (LOGSYS_LEVEL_ERROR, "Another Corosync instance is already running."); err = COROSYNC_DONE_ALREADY_RUNNING; goto error_close; break; default: log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't acquire lock. Error was %s", strerror(errno)); err = COROSYNC_DONE_ACQUIRE_LOCK; goto error_close; break; } } if (ftruncate (lf, 0) == -1) { log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't truncate lock file. Error was %s", strerror (errno)); err = COROSYNC_DONE_ACQUIRE_LOCK; goto error_close_unlink; } memset (pid_s, 0, sizeof (pid_s)); snprintf (pid_s, sizeof (pid_s) - 1, "%u\n", pid); retry_write: if (write (lf, pid_s, strlen (pid_s)) != strlen (pid_s)) { if (errno == EINTR) { goto retry_write; } else { log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't write pid to lock file. " "Error was %s", strerror (errno)); err = COROSYNC_DONE_ACQUIRE_LOCK; goto error_close_unlink; } } if ((fd_flag = fcntl (lf, F_GETFD, 0)) == -1) { log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't get close-on-exec flag from lock file. " "Error was %s", strerror (errno)); err = COROSYNC_DONE_ACQUIRE_LOCK; goto error_close_unlink; } fd_flag |= FD_CLOEXEC; if (fcntl (lf, F_SETFD, fd_flag) == -1) { log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't set close-on-exec flag to lock file. " "Error was %s", strerror (errno)); err = COROSYNC_DONE_ACQUIRE_LOCK; goto error_close_unlink; } return (err); error_close_unlink: unlink (lockfile); error_close: close (lf); return (err); } static int corosync_move_to_root_cgroup(void) { FILE *f; int res = -1; /* * /sys/fs/cgroup is hardcoded, because most of Linux distributions are now * using systemd and systemd uses hardcoded path of cgroup mount point. * * This feature is expected to be removed as soon as systemd gets support * for managing RT configuration. */ f = fopen("/sys/fs/cgroup/cpu/cpu.rt_runtime_us", "rt"); if (f == NULL) { log_printf(LOGSYS_LEVEL_DEBUG, "cpu.rt_runtime_us doesn't exists -> " "system without cgroup or with disabled CONFIG_RT_GROUP_SCHED"); res = 0; goto exit_res; } (void)fclose(f); f = fopen("/sys/fs/cgroup/cpu/tasks", "w"); if (f == NULL) { log_printf(LOGSYS_LEVEL_WARNING, "Can't open cgroups tasks file for writing"); goto exit_res; } if (fprintf(f, "%jd\n", (intmax_t)getpid()) <= 0) { log_printf(LOGSYS_LEVEL_WARNING, "Can't write corosync pid into cgroups tasks file"); goto close_and_exit_res; } close_and_exit_res: if (fclose(f) != 0) { log_printf(LOGSYS_LEVEL_WARNING, "Can't close cgroups tasks file"); goto exit_res; } exit_res: return (res); } int main (int argc, char **argv, char **envp) { const char *error_string; struct totem_config totem_config; int res, ch; int background, sched_rr, prio, testonly, move_to_root_cgroup; struct stat stat_out; enum e_corosync_done flock_err; uint64_t totem_config_warnings; struct scheduler_pause_timeout_data scheduler_pause_timeout_data; long int tmpli; char *ep; char *tmp_str; int log_subsys_id_totem; /* default configuration */ background = 1; testonly = 0; while ((ch = getopt (argc, argv, "c:ftv")) != EOF) { switch (ch) { case 'c': res = snprintf(corosync_config_file, sizeof(corosync_config_file), "%s", optarg); if (res >= sizeof(corosync_config_file)) { fprintf (stderr, "Config file path too long.\n"); syslog (LOGSYS_LEVEL_ERROR, "Config file path too long."); logsys_system_fini(); return EXIT_FAILURE; } break; case 'f': background = 0; break; case 't': testonly = 1; break; case 'v': printf ("Corosync Cluster Engine, version '%s'\n", VERSION); printf ("Copyright (c) 2006-2018 Red Hat, Inc.\n"); logsys_system_fini(); return EXIT_SUCCESS; break; default: fprintf(stderr, \ "usage:\n"\ " -c : Corosync config file path.\n"\ " -f : Start application in foreground.\n"\ " -t : Test configuration and exit.\n"\ " -v : Display version and SVN revision of Corosync and exit.\n"); logsys_system_fini(); return EXIT_FAILURE; } } /* * Other signals are registered later via qb_loop_signal_add */ (void)signal (SIGSEGV, sigsegv_handler); (void)signal (SIGABRT, sigsegv_handler); #if MSG_NOSIGNAL != 0 (void)signal (SIGPIPE, SIG_IGN); #endif if (icmap_init() != CS_OK) { fprintf (stderr, "Corosync Executive couldn't initialize configuration component.\n"); syslog (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't initialize configuration component."); corosync_exit_error (COROSYNC_DONE_ICMAP); } set_icmap_ro_keys_flag(); /* * Initialize the corosync_api_v1 definition */ api = apidef_get (); res = coroparse_configparse(icmap_get_global_map(), &error_string); if (res == -1) { /* * Logsys can't log properly at this early stage, and we need to get this message out * */ fprintf (stderr, "%s\n", error_string); syslog (LOGSYS_LEVEL_ERROR, "%s", error_string); corosync_exit_error (COROSYNC_DONE_MAINCONFIGREAD); } if (stats_map_init(api) != CS_OK) { fprintf (stderr, "Corosync Executive couldn't initialize statistics component.\n"); syslog (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't initialize statistics component."); corosync_exit_error (COROSYNC_DONE_STATS); } res = corosync_log_config_read (&error_string); if (res == -1) { /* * if we are here, we _must_ flush the logsys queue * and try to inform that we couldn't read the config. * this is a desperate attempt before certain death * and there is no guarantee that we can print to stderr * nor that logsys is sending the messages where we expect. */ log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string); fprintf(stderr, "%s", error_string); syslog (LOGSYS_LEVEL_ERROR, "%s", error_string); corosync_exit_error (COROSYNC_DONE_LOGCONFIGREAD); } if (!testonly) { log_printf (LOGSYS_LEVEL_NOTICE, "Corosync Cluster Engine ('%s'): started and ready to provide service.", VERSION); log_printf (LOGSYS_LEVEL_INFO, "Corosync built-in features:" PACKAGE_FEATURES ""); } /* * Create totem logsys subsys before totem_config_read so log functions can be used */ log_subsys_id_totem = _logsys_subsys_create("TOTEM", "totem," "totemip.c,totemconfig.c,totemcrypto.c,totemsrp.c," "totempg.c,totemudp.c,totemudpu.c,totemnet.c,totemknet.c"); /* * Make sure required directory is present */ res = stat (get_run_dir(), &stat_out); if ((res == -1) || (res == 0 && !S_ISDIR(stat_out.st_mode))) { log_printf (LOGSYS_LEVEL_ERROR, "Required directory not present %s. Please create it.", get_run_dir()); corosync_exit_error (COROSYNC_DONE_DIR_NOT_PRESENT); } res = chdir(get_run_dir()); if (res == -1) { log_printf (LOGSYS_LEVEL_ERROR, "Cannot chdir to run directory %s. " "Please make sure it has correct context and rights.", get_run_dir()); corosync_exit_error (COROSYNC_DONE_DIR_NOT_PRESENT); } res = totem_config_read (&totem_config, &error_string, &totem_config_warnings); if (res == -1) { log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string); corosync_exit_error (COROSYNC_DONE_MAINCONFIGREAD); } if (totem_config_warnings & TOTEM_CONFIG_WARNING_MEMBERS_IGNORED) { log_printf (LOGSYS_LEVEL_WARNING, "member section is used together with nodelist. Members ignored."); } if (totem_config_warnings & TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED) { log_printf (LOGSYS_LEVEL_WARNING, "member section is deprecated."); } if (totem_config_warnings & TOTEM_CONFIG_WARNING_TOTEM_NODEID_IGNORED) { log_printf (LOGSYS_LEVEL_WARNING, "nodeid appears both in totem section and nodelist. Nodelist one is used."); } if (totem_config_warnings & TOTEM_CONFIG_BINDNETADDR_NODELIST_SET) { log_printf (LOGSYS_LEVEL_WARNING, "interface section bindnetaddr is used together with nodelist. " "Nodelist one is going to be used."); } if (totem_config_warnings != 0) { log_printf (LOGSYS_LEVEL_WARNING, "Please migrate config file to nodelist."); } res = totem_config_keyread (&totem_config, &error_string); if (res == -1) { log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string); corosync_exit_error (COROSYNC_DONE_MAINCONFIGREAD); } res = totem_config_validate (&totem_config, &error_string); if (res == -1) { log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string); corosync_exit_error (COROSYNC_DONE_MAINCONFIGREAD); } if (testonly) { corosync_exit_error (COROSYNC_DONE_EXIT); } move_to_root_cgroup = 1; if (icmap_get_string("system.move_to_root_cgroup", &tmp_str) == CS_OK) { if (strcmp(tmp_str, "yes") != 0) { move_to_root_cgroup = 0; } free(tmp_str); } /* * Try to move corosync into root cpu cgroup. Failure is not fatal and * error is deliberately ignored. */ if (move_to_root_cgroup) { (void)corosync_move_to_root_cgroup(); } sched_rr = 1; if (icmap_get_string("system.sched_rr", &tmp_str) == CS_OK) { if (strcmp(tmp_str, "yes") != 0) { sched_rr = 0; } free(tmp_str); } prio = 0; if (icmap_get_string("system.priority", &tmp_str) == CS_OK) { if (strcmp(tmp_str, "max") == 0) { prio = INT_MIN; } else if (strcmp(tmp_str, "min") == 0) { prio = INT_MAX; } else { errno = 0; tmpli = strtol(tmp_str, &ep, 10); if (errno != 0 || *ep != '\0' || tmpli > INT_MAX || tmpli < INT_MIN) { log_printf (LOGSYS_LEVEL_ERROR, "Priority value %s is invalid", tmp_str); corosync_exit_error (COROSYNC_DONE_MAINCONFIGREAD); } prio = tmpli; } free(tmp_str); } /* * Set round robin realtime scheduling with priority 99 */ if (sched_rr) { if (corosync_set_rr_scheduler () != 0) { prio = INT_MIN; } else { prio = 0; } } if (prio != 0) { if (setpriority(PRIO_PGRP, 0, prio) != 0) { LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "Could not set priority %d", prio); } } - ip_version = totem_config.ip_version; - totem_config.totem_memb_ring_id_create_or_load = corosync_ring_id_create_or_load; totem_config.totem_memb_ring_id_store = corosync_ring_id_store; totem_config.totem_logging_configuration = totem_logging_configuration; totem_config.totem_logging_configuration.log_subsys_id = log_subsys_id_totem; totem_config.totem_logging_configuration.log_level_security = LOGSYS_LEVEL_WARNING; totem_config.totem_logging_configuration.log_level_error = LOGSYS_LEVEL_ERROR; totem_config.totem_logging_configuration.log_level_warning = LOGSYS_LEVEL_WARNING; totem_config.totem_logging_configuration.log_level_notice = LOGSYS_LEVEL_NOTICE; totem_config.totem_logging_configuration.log_level_debug = LOGSYS_LEVEL_DEBUG; totem_config.totem_logging_configuration.log_level_trace = LOGSYS_LEVEL_TRACE; totem_config.totem_logging_configuration.log_printf = _logsys_log_printf; logsys_config_apply(); /* * Now we are fully initialized. */ if (background) { logsys_blackbox_prefork(); corosync_tty_detach (); logsys_blackbox_postfork(); log_printf (LOGSYS_LEVEL_DEBUG, "Corosync TTY detached"); } /* * Lock all memory to avoid page faults which may interrupt * application healthchecking */ corosync_mlockall (); corosync_poll_handle = qb_loop_create (); memset(&scheduler_pause_timeout_data, 0, sizeof(scheduler_pause_timeout_data)); scheduler_pause_timeout_data.totem_config = &totem_config; timer_function_scheduler_timeout (&scheduler_pause_timeout_data); qb_loop_signal_add(corosync_poll_handle, QB_LOOP_LOW, SIGUSR2, NULL, sig_diag_handler, NULL); qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH, SIGINT, NULL, sig_exit_handler, NULL); qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH, SIGQUIT, NULL, sig_exit_handler, NULL); qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH, SIGTERM, NULL, sig_exit_handler, NULL); if (logsys_thread_start() != 0) { log_printf (LOGSYS_LEVEL_ERROR, "Can't initialize log thread"); corosync_exit_error (COROSYNC_DONE_LOGCONFIGREAD); } if ((flock_err = corosync_flock (corosync_lock_file, getpid ())) != COROSYNC_DONE_EXIT) { corosync_exit_error (flock_err); } /* * if totempg_initialize doesn't have root priveleges, it cannot * bind to a specific interface. This only matters if * there is more then one interface in a system, so * in this case, only a warning is printed */ /* * Join multicast group and setup delivery * and configuration change functions */ if (totempg_initialize ( corosync_poll_handle, &totem_config) != 0) { log_printf (LOGSYS_LEVEL_ERROR, "Can't initialize TOTEM layer"); corosync_exit_error (COROSYNC_DONE_FATAL_ERR); } totempg_service_ready_register ( main_service_ready); totempg_groups_initialize ( &corosync_group_handle, deliver_fn, confchg_fn); totempg_groups_join ( corosync_group_handle, &corosync_group, 1); /* * Drop root privleges to user 'corosync' * TODO: Don't really need full root capabilities; * needed capabilities are: * CAP_NET_RAW (bindtodevice) * CAP_SYS_NICE (setscheduler) * CAP_IPC_LOCK (mlockall) */ priv_drop (); schedwrk_init ( serialize_lock, serialize_unlock); /* * Start main processing loop */ qb_loop_run (corosync_poll_handle); /* * Exit was requested */ totempg_finalize (); /* * free the loop resources */ qb_loop_destroy (corosync_poll_handle); /* * free up the icmap */ /* * Remove pid lock file */ unlink (corosync_lock_file); corosync_exit_error (COROSYNC_DONE_EXIT); return EXIT_SUCCESS; } diff --git a/exec/totemconfig.c b/exec/totemconfig.c index e85ad7bd..a19db59a 100644 --- a/exec/totemconfig.c +++ b/exec/totemconfig.c @@ -1,2212 +1,2221 @@ /* * Copyright (c) 2002-2005 MontaVista Software, Inc. * Copyright (c) 2006-2018 Red Hat, Inc. * * All rights reserved. * * Author: Steven Dake (sdake@redhat.com) * Jan Friesse (jfriesse@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the MontaVista Software, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "util.h" #include "totemconfig.h" #define TOKEN_RETRANSMITS_BEFORE_LOSS_CONST 4 #define TOKEN_TIMEOUT 1000 #define TOKEN_WARNING 75 #define TOKEN_COEFFICIENT 650 #define JOIN_TIMEOUT 50 #define MERGE_TIMEOUT 200 #define DOWNCHECK_TIMEOUT 1000 #define FAIL_TO_RECV_CONST 2500 #define SEQNO_UNCHANGED_CONST 30 #define MINIMUM_TIMEOUT (int)(1000/HZ)*3 #define MAX_NETWORK_DELAY 50 #define WINDOW_SIZE 50 #define MAX_MESSAGES 17 #define MISS_COUNT_CONST 5 /* These currently match the defaults in libknet.h */ #define KNET_PING_INTERVAL 1000 #define KNET_PING_TIMEOUT 2000 #define KNET_PING_PRECISION 2048 #define KNET_PONG_COUNT 2 #define KNET_PMTUD_INTERVAL 30 #define KNET_DEFAULT_TRANSPORT KNET_TRANSPORT_UDP #define DEFAULT_PORT 5405 static char error_string_response[768]; static void add_totem_config_notification(struct totem_config *totem_config); static void *totem_get_param_by_name(struct totem_config *totem_config, const char *param_name) { if (strcmp(param_name, "totem.token") == 0) return &totem_config->token_timeout; if (strcmp(param_name, "totem.token_warning") == 0) return &totem_config->token_warning; if (strcmp(param_name, "totem.token_retransmit") == 0) return &totem_config->token_retransmit_timeout; if (strcmp(param_name, "totem.hold") == 0) return &totem_config->token_hold_timeout; if (strcmp(param_name, "totem.token_retransmits_before_loss_const") == 0) return &totem_config->token_retransmits_before_loss_const; if (strcmp(param_name, "totem.join") == 0) return &totem_config->join_timeout; if (strcmp(param_name, "totem.send_join") == 0) return &totem_config->send_join_timeout; if (strcmp(param_name, "totem.consensus") == 0) return &totem_config->consensus_timeout; if (strcmp(param_name, "totem.merge") == 0) return &totem_config->merge_timeout; if (strcmp(param_name, "totem.downcheck") == 0) return &totem_config->downcheck_timeout; if (strcmp(param_name, "totem.fail_recv_const") == 0) return &totem_config->fail_to_recv_const; if (strcmp(param_name, "totem.seqno_unchanged_const") == 0) return &totem_config->seqno_unchanged_const; if (strcmp(param_name, "totem.heartbeat_failures_allowed") == 0) return &totem_config->heartbeat_failures_allowed; if (strcmp(param_name, "totem.max_network_delay") == 0) return &totem_config->max_network_delay; if (strcmp(param_name, "totem.window_size") == 0) return &totem_config->window_size; if (strcmp(param_name, "totem.max_messages") == 0) return &totem_config->max_messages; if (strcmp(param_name, "totem.miss_count_const") == 0) return &totem_config->miss_count_const; if (strcmp(param_name, "totem.knet_pmtud_interval") == 0) return &totem_config->knet_pmtud_interval; if (strcmp(param_name, "totem.knet_compression_threshold") == 0) return &totem_config->knet_compression_threshold; if (strcmp(param_name, "totem.knet_compression_level") == 0) return &totem_config->knet_compression_level; if (strcmp(param_name, "totem.knet_compression_model") == 0) return &totem_config->knet_compression_model; return NULL; } /* * Read key_name from icmap. If key is not found or key_name == delete_key or if allow_zero is false * and readed value is zero, default value is used and stored into totem_config. */ static void totem_volatile_config_set_uint32_value (struct totem_config *totem_config, const char *key_name, const char *deleted_key, unsigned int default_value, int allow_zero_value) { char runtime_key_name[ICMAP_KEYNAME_MAXLEN]; if (icmap_get_uint32(key_name, totem_get_param_by_name(totem_config, key_name)) != CS_OK || (deleted_key != NULL && strcmp(deleted_key, key_name) == 0) || (!allow_zero_value && *(uint32_t *)totem_get_param_by_name(totem_config, key_name) == 0)) { *(uint32_t *)totem_get_param_by_name(totem_config, key_name) = default_value; } /* * Store totem_config value to cmap runtime section */ if (strlen("runtime.config.") + strlen(key_name) >= ICMAP_KEYNAME_MAXLEN) { /* * This shouldn't happen */ return ; } strcpy(runtime_key_name, "runtime.config."); strcat(runtime_key_name, key_name); icmap_set_uint32(runtime_key_name, *(uint32_t *)totem_get_param_by_name(totem_config, key_name)); } static void totem_volatile_config_set_int32_value (struct totem_config *totem_config, const char *key_name, const char *deleted_key, int default_value, int allow_zero_value) { char runtime_key_name[ICMAP_KEYNAME_MAXLEN]; if (icmap_get_int32(key_name, totem_get_param_by_name(totem_config, key_name)) != CS_OK || (deleted_key != NULL && strcmp(deleted_key, key_name) == 0) || (!allow_zero_value && *(int32_t *)totem_get_param_by_name(totem_config, key_name) == 0)) { *(int32_t *)totem_get_param_by_name(totem_config, key_name) = default_value; } /* * Store totem_config value to cmap runtime section */ if (strlen("runtime.config.") + strlen(key_name) >= ICMAP_KEYNAME_MAXLEN) { /* * This shouldn't happen */ return ; } strcpy(runtime_key_name, "runtime.config."); strcat(runtime_key_name, key_name); icmap_set_int32(runtime_key_name, *(int32_t *)totem_get_param_by_name(totem_config, key_name)); } static void totem_volatile_config_set_string_value (struct totem_config *totem_config, const char *key_name, const char *deleted_key, const char *default_value) { char runtime_key_name[ICMAP_KEYNAME_MAXLEN]; void **config_value; void *old_config_ptr; config_value = totem_get_param_by_name(totem_config, key_name); old_config_ptr = *config_value; if (icmap_get_string(key_name, totem_get_param_by_name(totem_config, key_name)) != CS_OK || (deleted_key != NULL && strcmp(deleted_key, key_name) == 0)) { /* Need to strdup() here so that the free() below works for a default and a configured value */ *config_value = strdup(default_value); } free(old_config_ptr); /* * Store totem_config value to cmap runtime section */ if (strlen("runtime.config.") + strlen(key_name) >= ICMAP_KEYNAME_MAXLEN) { /* * This shouldn't happen */ return ; } strcpy(runtime_key_name, "runtime.config."); strcat(runtime_key_name, key_name); icmap_set_string(runtime_key_name, (char *)*config_value); } /* * Read and validate config values from cmap and store them into totem_config. If key doesn't exists, * default value is stored. deleted_key is name of key beeing processed by delete operation * from cmap. It is considered as non existing even if it can be read. Can be NULL. */ static void totem_volatile_config_read (struct totem_config *totem_config, const char *deleted_key) { uint32_t u32; totem_volatile_config_set_uint32_value(totem_config, "totem.token_retransmits_before_loss_const", deleted_key, TOKEN_RETRANSMITS_BEFORE_LOSS_CONST, 0); totem_volatile_config_set_uint32_value(totem_config, "totem.token", deleted_key, TOKEN_TIMEOUT, 0); totem_volatile_config_set_uint32_value(totem_config, "totem.token_warning", deleted_key, TOKEN_WARNING, 1); if (totem_config->interfaces[0].member_count > 2) { u32 = TOKEN_COEFFICIENT; icmap_get_uint32("totem.token_coefficient", &u32); totem_config->token_timeout += (totem_config->interfaces[0].member_count - 2) * u32; /* * Store totem_config value to cmap runtime section */ icmap_set_uint32("runtime.config.totem.token", totem_config->token_timeout); } totem_volatile_config_set_uint32_value(totem_config, "totem.max_network_delay", deleted_key, MAX_NETWORK_DELAY, 0); totem_volatile_config_set_uint32_value(totem_config, "totem.window_size", deleted_key, WINDOW_SIZE, 0); totem_volatile_config_set_uint32_value(totem_config, "totem.max_messages", deleted_key, MAX_MESSAGES, 0); totem_volatile_config_set_uint32_value(totem_config, "totem.miss_count_const", deleted_key, MISS_COUNT_CONST, 0); totem_volatile_config_set_uint32_value(totem_config, "totem.knet_pmtud_interval", deleted_key, KNET_PMTUD_INTERVAL, 0); totem_volatile_config_set_uint32_value(totem_config, "totem.token_retransmit", deleted_key, (int)(totem_config->token_timeout / (totem_config->token_retransmits_before_loss_const + 0.2)), 0); totem_volatile_config_set_uint32_value(totem_config, "totem.hold", deleted_key, (int)(totem_config->token_retransmit_timeout * 0.8 - (1000/HZ)), 0); totem_volatile_config_set_uint32_value(totem_config, "totem.join", deleted_key, JOIN_TIMEOUT, 0); totem_volatile_config_set_uint32_value(totem_config, "totem.consensus", deleted_key, (int)(float)(1.2 * totem_config->token_timeout), 0); totem_volatile_config_set_uint32_value(totem_config, "totem.merge", deleted_key, MERGE_TIMEOUT, 0); totem_volatile_config_set_uint32_value(totem_config, "totem.downcheck", deleted_key, DOWNCHECK_TIMEOUT, 0); totem_volatile_config_set_uint32_value(totem_config, "totem.fail_recv_const", deleted_key, FAIL_TO_RECV_CONST, 0); totem_volatile_config_set_uint32_value(totem_config, "totem.seqno_unchanged_const", deleted_key, SEQNO_UNCHANGED_CONST, 0); totem_volatile_config_set_uint32_value(totem_config, "totem.send_join", deleted_key, 0, 1); totem_volatile_config_set_uint32_value(totem_config, "totem.heartbeat_failures_allowed", deleted_key, 0, 1); totem_volatile_config_set_uint32_value(totem_config, "totem.knet_compression_threshold", deleted_key, 0, 1); totem_volatile_config_set_int32_value(totem_config, "totem.knet_compression_level", deleted_key, 0, 1); totem_volatile_config_set_string_value(totem_config, "totem.knet_compression_model", deleted_key, "none"); } static int totem_volatile_config_validate ( struct totem_config *totem_config, const char **error_string) { static char local_error_reason[512]; const char *error_reason = local_error_reason; char name_key[ICMAP_KEYNAME_MAXLEN]; char *name_str; int i, num_configured, members; if (totem_config->max_network_delay < MINIMUM_TIMEOUT) { snprintf (local_error_reason, sizeof(local_error_reason), "The max_network_delay parameter (%d ms) may not be less than (%d ms).", totem_config->max_network_delay, MINIMUM_TIMEOUT); goto parse_error; } if (totem_config->token_timeout < MINIMUM_TIMEOUT) { snprintf (local_error_reason, sizeof(local_error_reason), "The token timeout parameter (%d ms) may not be less than (%d ms).", totem_config->token_timeout, MINIMUM_TIMEOUT); goto parse_error; } if (totem_config->token_warning > 100 || totem_config->token_warning < 0) { snprintf (local_error_reason, sizeof(local_error_reason), "The token warning parameter (%d%%) must be between 0 (disabled) and 100.", totem_config->token_warning); goto parse_error; } if (totem_config->token_retransmit_timeout < MINIMUM_TIMEOUT) { snprintf (local_error_reason, sizeof(local_error_reason), "The token retransmit timeout parameter (%d ms) may not be less than (%d ms).", totem_config->token_retransmit_timeout, MINIMUM_TIMEOUT); goto parse_error; } if (totem_config->token_hold_timeout < MINIMUM_TIMEOUT) { snprintf (local_error_reason, sizeof(local_error_reason), "The token hold timeout parameter (%d ms) may not be less than (%d ms).", totem_config->token_hold_timeout, MINIMUM_TIMEOUT); goto parse_error; } if (totem_config->join_timeout < MINIMUM_TIMEOUT) { snprintf (local_error_reason, sizeof(local_error_reason), "The join timeout parameter (%d ms) may not be less than (%d ms).", totem_config->join_timeout, MINIMUM_TIMEOUT); goto parse_error; } if (totem_config->consensus_timeout < MINIMUM_TIMEOUT) { snprintf (local_error_reason, sizeof(local_error_reason), "The consensus timeout parameter (%d ms) may not be less than (%d ms).", totem_config->consensus_timeout, MINIMUM_TIMEOUT); goto parse_error; } if (totem_config->consensus_timeout < totem_config->join_timeout) { snprintf (local_error_reason, sizeof(local_error_reason), "The consensus timeout parameter (%d ms) may not be less than join timeout (%d ms).", totem_config->consensus_timeout, totem_config->join_timeout); goto parse_error; } if (totem_config->merge_timeout < MINIMUM_TIMEOUT) { snprintf (local_error_reason, sizeof(local_error_reason), "The merge timeout parameter (%d ms) may not be less than (%d ms).", totem_config->merge_timeout, MINIMUM_TIMEOUT); goto parse_error; } if (totem_config->downcheck_timeout < MINIMUM_TIMEOUT) { snprintf (local_error_reason, sizeof(local_error_reason), "The downcheck timeout parameter (%d ms) may not be less than (%d ms).", totem_config->downcheck_timeout, MINIMUM_TIMEOUT); goto parse_error; } /* Check that we have nodelist 'name' if there is more than one link */ num_configured = 0; for (i = 0; i < INTERFACE_MAX; i++) { if (totem_config->interfaces[i].configured) { num_configured++; } } if (num_configured > 1) { members = totem_config->interfaces[0].member_count; for (i=0; iinterfaces[0].member_count; i++) { snprintf(name_key, sizeof(name_key), "nodelist.node.%d.name", i); if (icmap_get_string(name_key, &name_str) != CS_OK) { snprintf (local_error_reason, sizeof(local_error_reason), "for a multi-link configuration, all nodes must have a 'name' attribute"); goto parse_error; } } for (i=0; iinterfaces[i].member_count != members) { snprintf (local_error_reason, sizeof(local_error_reason), "Not all nodes have the same number of links"); goto parse_error; } } } return 0; parse_error: snprintf (error_string_response, sizeof(error_string_response), "parse error in config: %s\n", error_reason); *error_string = error_string_response; return (-1); } static int totem_get_crypto(struct totem_config *totem_config, const char **error_string) { char *str; const char *tmp_cipher; const char *tmp_hash; const char *tmp_model; tmp_hash = "none"; tmp_cipher = "none"; tmp_model = "none"; if (icmap_get_string("totem.crypto_model", &str) == CS_OK) { if (strcmp(str, "nss") == 0) { tmp_model = "nss"; } if (strcmp(str, "openssl") == 0) { tmp_model = "openssl"; } free(str); } else { tmp_model = "nss"; } if (icmap_get_string("totem.crypto_cipher", &str) == CS_OK) { if (strcmp(str, "none") == 0) { tmp_cipher = "none"; } if (strcmp(str, "aes256") == 0) { tmp_cipher = "aes256"; } if (strcmp(str, "aes192") == 0) { tmp_cipher = "aes192"; } if (strcmp(str, "aes128") == 0) { tmp_cipher = "aes128"; } if (strcmp(str, "3des") == 0) { tmp_cipher = "3des"; } free(str); } if (icmap_get_string("totem.crypto_hash", &str) == CS_OK) { if (strcmp(str, "none") == 0) { tmp_hash = "none"; } if (strcmp(str, "md5") == 0) { tmp_hash = "md5"; } if (strcmp(str, "sha1") == 0) { tmp_hash = "sha1"; } if (strcmp(str, "sha256") == 0) { tmp_hash = "sha256"; } if (strcmp(str, "sha384") == 0) { tmp_hash = "sha384"; } if (strcmp(str, "sha512") == 0) { tmp_hash = "sha512"; } free(str); } if ((strcmp(tmp_cipher, "none") != 0) && (strcmp(tmp_hash, "none") == 0)) { *error_string = "crypto_cipher requires crypto_hash with value other than none"; return -1; } if (strcmp(tmp_model, "none") == 0) { *error_string = "crypto_model should be 'nss' or 'openssl'"; return -1; } free(totem_config->crypto_cipher_type); free(totem_config->crypto_hash_type); free(totem_config->crypto_model); totem_config->crypto_cipher_type = strdup(tmp_cipher); totem_config->crypto_hash_type = strdup(tmp_hash); totem_config->crypto_model = strdup(tmp_model); return 0; } static int nodelist_byname(const char *find_name, int strip_domain) { icmap_iter_t iter; const char *iter_key; char name_str[ICMAP_KEYNAME_MAXLEN]; int res = 0; unsigned int node_pos; char *name; unsigned int namelen; iter = icmap_iter_init("nodelist.node."); while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, name_str); if (res != 2) { continue; } /* ring0_addr is allowed as a fallback */ if (strcmp(name_str, "name") && strcmp(name_str, "ring0_addr")) { continue; } if (icmap_get_string(iter_key, &name) != CS_OK) { continue; } namelen = strlen(name); if (strip_domain) { char *dot; dot = strchr(name, '.'); if (dot) { namelen = name - dot - 1; } } if (strncmp(find_name, name, namelen) == 0 && strlen(find_name) == strlen(name)) { icmap_iter_finalize(iter); return node_pos; } } icmap_iter_finalize(iter); return -1; } /* Compare two addresses */ static int ipaddr_equal(struct sockaddr_storage *addr1, struct sockaddr_storage *addr2) { int addrlen = 0; if (addr1->ss_family != addr2->ss_family) return 0; if (addr1->ss_family == AF_INET) { addrlen = sizeof(struct sockaddr_in); } if (addr1->ss_family == AF_INET6) { addrlen = sizeof(struct sockaddr_in6); } assert(addrlen); if (memcmp(addr1, addr2, addrlen) == 0) return 1; else return 0; } /* Finds the local node and returns its position in the nodelist. * Uses nodelist.local_node_pos as a cache to save effort */ static int find_local_node(int use_cache) { char nodename2[PATH_MAX]; char name_str[ICMAP_KEYNAME_MAXLEN]; icmap_iter_t iter; const char *iter_key; unsigned int cached_pos; char *dot = NULL; const char *node; struct ifaddrs *ifa, *ifa_list; struct sockaddr *sa; int found = 0; int node_pos = -1; int res; struct utsname utsname; /* Check for cached value first */ if (use_cache) { if (icmap_get_uint32("nodelist.local_node_pos", &cached_pos) == CS_OK) { return cached_pos; } } res = uname(&utsname); if (res) { return -1; } node = utsname.nodename; /* 1. Exact match */ node_pos = nodelist_byname(node, 0); if (node_pos > -1) { found = 1; goto ret_found; } /* 2. Try to match with increasingly more * specific versions of it */ strcpy(nodename2, node); dot = strrchr(nodename2, '.'); while (dot) { *dot = '\0'; node_pos = nodelist_byname(nodename2, 0); if (node_pos > -1) { found = 1; goto ret_found; } dot = strrchr(nodename2, '.'); } node_pos = nodelist_byname(nodename2, 1); if (node_pos > -1) { found = 1; goto ret_found; } /* * The corosync.conf name may not be related to uname at all, * they may match a hostname on some network interface. */ if (getifaddrs(&ifa_list)) return -1; for (ifa = ifa_list; ifa; ifa = ifa->ifa_next) { socklen_t salen = 0; /* Restore this */ strcpy(nodename2, node); sa = ifa->ifa_addr; if (!sa) { continue; } if (sa->sa_family != AF_INET && sa->sa_family != AF_INET6) { continue; } if (sa->sa_family == AF_INET) { salen = sizeof(struct sockaddr_in); } if (sa->sa_family == AF_INET6) { salen = sizeof(struct sockaddr_in6); } if (getnameinfo(sa, salen, nodename2, sizeof(nodename2), NULL, 0, 0) == 0) { node_pos = nodelist_byname(nodename2, 0); if (node_pos > -1) { found = 1; goto out; } /* Truncate this name and try again */ dot = strchr(nodename2, '.'); if (dot) { *dot = '\0'; node_pos = nodelist_byname(nodename2, 0); if (node_pos > -1) { found = 1; goto out; } } } /* See if it's the IP address that's in corosync.conf */ if (getnameinfo(sa, sizeof(*sa), nodename2, sizeof(nodename2), NULL, 0, NI_NUMERICHOST)) continue; node_pos = nodelist_byname(nodename2, 0); if (node_pos > -1) { found = 1; goto out; } } out: if (found) { freeifaddrs(ifa_list); goto ret_found; } /* * This section covers the usecase where the nodename specified in cluster.conf * is an alias specified in /etc/hosts. For example: * hostname alias1 alias2 * and * the above calls use uname and getnameinfo does not return aliases. * here we take the name specified in cluster.conf, resolve it to an address * and then compare against all known local ip addresses. * if we have a match, we found our nodename. In theory this chunk of code * could replace all the checks above, but let's avoid any possible regressions * and use it as last. */ iter = icmap_iter_init("nodelist.node."); while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { char *dbnodename = NULL; struct addrinfo hints; struct addrinfo *result = NULL, *rp = NULL; res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, name_str); if (res != 2) { continue; } /* 'ring0_addr' is allowed as a fallback, but 'name' will be found first * because the names are in alpha order. */ if (strcmp(name_str, "name") && strcmp(name_str, "ring0_addr")) { continue; } if (icmap_get_string(iter_key, &dbnodename) != CS_OK) { continue; } memset(&hints, 0, sizeof(struct addrinfo)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_DGRAM; hints.ai_flags = 0; hints.ai_protocol = IPPROTO_UDP; if (getaddrinfo(dbnodename, NULL, &hints, &result)) { continue; } for (rp = result; rp != NULL; rp = rp->ai_next) { for (ifa = ifa_list; ifa; ifa = ifa->ifa_next) { if (ifa->ifa_addr && ipaddr_equal((struct sockaddr_storage *)rp->ai_addr, (struct sockaddr_storage *)ifa->ifa_addr)) { freeaddrinfo(result); found = 1; goto out2; } } } freeaddrinfo(result); } out2: icmap_iter_finalize(iter); freeifaddrs(ifa_list); ret_found: if (found) { res = icmap_set_uint32("nodelist.local_node_pos", node_pos); } return node_pos; } -static int totem_config_get_ip_version(struct totem_config *totem_config) +static enum totem_ip_version_enum totem_config_get_ip_version(struct totem_config *totem_config) { - int res; + enum totem_ip_version_enum res; char *str; - res = AF_INET; - if (totem_config->transport_number == TOTEM_TRANSPORT_KNET) { - res = AF_UNSPEC; - } else { - if (icmap_get_string("totem.ip_version", &str) == CS_OK) { - if (strcmp(str, "ipv4") == 0) { - res = AF_INET; - } - if (strcmp(str, "ipv6") == 0) { - res = AF_INET6; - } - free(str); + res = TOTEM_IP_VERSION_6_4; + + if (totem_config->transport_number == TOTEM_TRANSPORT_UDP) { + res = TOTEM_IP_VERSION_4; + } + + if (icmap_get_string("totem.ip_version", &str) == CS_OK) { + if (strcmp(str, "ipv4") == 0) { + res = TOTEM_IP_VERSION_4; + } + if (strcmp(str, "ipv6") == 0) { + res = TOTEM_IP_VERSION_6; + } + if (strcmp(str, "ipv6-4") == 0) { + res = TOTEM_IP_VERSION_6_4; } + if (strcmp(str, "ipv4-6") == 0) { + res = TOTEM_IP_VERSION_4_6; + } + free(str); } return (res); } static uint16_t generate_cluster_id (const char *cluster_name) { int i; int value = 0; for (i = 0; i < strlen(cluster_name); i++) { value <<= 1; value += cluster_name[i]; } return (value & 0xFFFF); } static int get_cluster_mcast_addr ( const char *cluster_name, unsigned int linknumber, - int ip_version, + enum totem_ip_version_enum ip_version, struct totem_ip_address *res) { uint16_t clusterid; char addr[INET6_ADDRSTRLEN + 1]; int err; if (cluster_name == NULL) { return (-1); } clusterid = generate_cluster_id(cluster_name) + linknumber; memset (res, 0, sizeof(*res)); switch (ip_version) { - case AF_INET: + case TOTEM_IP_VERSION_4: + case TOTEM_IP_VERSION_4_6: snprintf(addr, sizeof(addr), "239.192.%d.%d", clusterid >> 8, clusterid % 0xFF); break; - case AF_INET6: + case TOTEM_IP_VERSION_6: + case TOTEM_IP_VERSION_6_4: snprintf(addr, sizeof(addr), "ff15::%x", clusterid); break; default: /* * Unknown family */ return (-1); } err = totemip_parse (res, addr, ip_version); return (err); } static unsigned int generate_nodeid( struct totem_config *totem_config, char *addr) { unsigned int nodeid; struct totem_ip_address totemip; /* AF_INET hard-coded here because auto-generated nodeids are only for IPv4 */ - if (totemip_parse(&totemip, addr, AF_INET) != 0) + if (totemip_parse(&totemip, addr, TOTEM_IP_VERSION_4) != 0) return -1; memcpy (&nodeid, &totemip.addr, sizeof (unsigned int)); #if __BYTE_ORDER == __LITTLE_ENDIAN nodeid = swab32 (nodeid); #endif if (totem_config->clear_node_high_bit) { nodeid &= 0x7FFFFFFF; } return nodeid; } static int check_for_duplicate_nodeids( struct totem_config *totem_config, const char **error_string) { icmap_iter_t iter; icmap_iter_t subiter; const char *iter_key; int res = 0; int retval = 0; char tmp_key[ICMAP_KEYNAME_MAXLEN]; char *ring0_addr=NULL; char *ring0_addr1=NULL; unsigned int node_pos; unsigned int node_pos1; unsigned int last_node_pos = -1; unsigned int nodeid; unsigned int nodeid1; int autogenerated; iter = icmap_iter_init("nodelist.node."); while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, tmp_key); if (res != 2) { continue; } /* * This relies on the fact the icmap keys are always returned in order * so all of the keys for a node will be grouped together. We're basically * just running the code below once for each node. */ if (last_node_pos == node_pos) { continue; } last_node_pos = node_pos; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.nodeid", node_pos); autogenerated = 0; /* Generated nodeids are only allowed for UDP/UDPU so ring0_addr is valid here */ if (icmap_get_uint32(tmp_key, &nodeid) != CS_OK) { snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring0_addr", node_pos); if (icmap_get_string(tmp_key, &ring0_addr) != CS_OK) { continue; } /* Generate nodeid so we can check that auto-generated nodeids don't clash either */ nodeid = generate_nodeid(totem_config, ring0_addr); if (nodeid == -1) { continue; } autogenerated = 1; } node_pos1 = 0; subiter = icmap_iter_init("nodelist.node."); while (((iter_key = icmap_iter_next(subiter, NULL, NULL)) != NULL) && (node_pos1 < node_pos)) { res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos1, tmp_key); if ((res != 2) || (node_pos1 >= node_pos)) { continue; } if (strcmp(tmp_key, "nodeid") != 0) { continue; } snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.nodeid", node_pos1); if (icmap_get_uint32(tmp_key, &nodeid1) != CS_OK) { snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring0_addr", node_pos1); if (icmap_get_string(tmp_key, &ring0_addr1) != CS_OK) { continue; } nodeid1 = generate_nodeid(totem_config, ring0_addr1); if (nodeid1 == -1) { continue; } } if (nodeid == nodeid1) { retval = -1; snprintf (error_string_response, sizeof(error_string_response), "Nodeid %u%s%s%s appears twice in corosync.conf", nodeid, autogenerated?"(autogenerated from ":"", autogenerated?ring0_addr:"", autogenerated?")":""); log_printf (LOGSYS_LEVEL_ERROR, error_string_response); *error_string = error_string_response; break; } } icmap_iter_finalize(subiter); } icmap_iter_finalize(iter); return retval; } /* * This needs to be done last of all. It would be nice to do it when reading the * interface params, but the totem params need to have them to be read first. We * need both, so this is a way round that circular dependancy. */ static void calc_knet_ping_timers(struct totem_config *totem_config) { char runtime_key_name[ICMAP_KEYNAME_MAXLEN]; int interface; for (interface = 0; interface < INTERFACE_MAX; interface++) { if (totem_config->interfaces[interface].configured) { if (!totem_config->interfaces[interface].knet_pong_count) { totem_config->interfaces[interface].knet_pong_count = KNET_PONG_COUNT; } if (!totem_config->interfaces[interface].knet_ping_timeout) { totem_config->interfaces[interface].knet_ping_timeout = totem_config->token_timeout / totem_config->interfaces[interface].knet_pong_count; } snprintf(runtime_key_name, sizeof(runtime_key_name), "runtime.config.totem.interface.%d.knet_ping_timeout", interface); icmap_set_uint32(runtime_key_name, totem_config->interfaces[interface].knet_ping_timeout); if (!totem_config->interfaces[interface].knet_ping_interval) { totem_config->interfaces[interface].knet_ping_interval = totem_config->token_timeout / (totem_config->interfaces[interface].knet_pong_count * 2); } snprintf(runtime_key_name, sizeof(runtime_key_name), "runtime.config.totem.interface.%d.knet_ping_interval", interface); icmap_set_uint32(runtime_key_name, totem_config->interfaces[interface].knet_ping_interval); } } } /* * Compute difference between two set of totem interface arrays. set1 and set2 * are changed so for same ring, ip existing in both set1 and set2 are cleared * (set to 0), and ips which are only in set1 or set2 remains untouched. * totempg_node_add/remove is called. */ static void compute_interfaces_diff(struct totem_interface *set1, struct totem_interface *set2) { int ring_no, set1_pos, set2_pos; struct totem_ip_address empty_ip_address; memset(&empty_ip_address, 0, sizeof(empty_ip_address)); for (ring_no = 0; ring_no < INTERFACE_MAX; ring_no++) { if (!set1[ring_no].configured && !set2[ring_no].configured) { continue; } for (set1_pos = 0; set1_pos < set1[ring_no].member_count; set1_pos++) { for (set2_pos = 0; set2_pos < set2[ring_no].member_count; set2_pos++) { /* * For current ring_no remove all set1 items existing * in set2 */ if (memcmp(&set1[ring_no].member_list[set1_pos], &set2[ring_no].member_list[set2_pos], sizeof(struct totem_ip_address)) == 0) { memset(&set1[ring_no].member_list[set1_pos], 0, sizeof(struct totem_ip_address)); memset(&set2[ring_no].member_list[set2_pos], 0, sizeof(struct totem_ip_address)); } } } } for (ring_no = 0; ring_no < INTERFACE_MAX; ring_no++) { for (set1_pos = 0; set1_pos < set1[ring_no].member_count; set1_pos++) { /* * All items which remained in set1 doesn't exists in set2 any longer so * node has to be removed. */ if (memcmp(&set1[ring_no].member_list[set1_pos], &empty_ip_address, sizeof(empty_ip_address)) != 0) { log_printf(LOGSYS_LEVEL_DEBUG, "removing dynamic member %s for ring %u", totemip_print(&set1[ring_no].member_list[set1_pos]), ring_no); totempg_member_remove(&set1[ring_no].member_list[set1_pos], ring_no); } } if (!set2[ring_no].configured) { continue; } for (set2_pos = 0; set2_pos < set2[ring_no].member_count; set2_pos++) { /* * All items which remained in set2 doesn't existed in set1 so this is no node * and has to be added. */ if (memcmp(&set2[ring_no].member_list[set2_pos], &empty_ip_address, sizeof(empty_ip_address)) != 0) { log_printf(LOGSYS_LEVEL_DEBUG, "adding dynamic member %s for ring %u", totemip_print(&set2[ring_no].member_list[set2_pos]), ring_no); totempg_member_add(&set2[ring_no].member_list[set2_pos], ring_no); } } } } /* * Reconfigure links in totempg. Sets new local IP address and adds params for new links. */ static void reconfigure_links(struct totem_config *totem_config) { int i; char tmp_key[ICMAP_KEYNAME_MAXLEN]; char *addr_string; struct totem_ip_address local_ip; int err; int local_node_pos = find_local_node(0); for (i = 0; iinterfaces[i].configured) { continue; } log_printf(LOGSYS_LEVEL_INFO, "Configuring link %d\n", i); snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring%u_addr", local_node_pos, i); if (icmap_get_string(tmp_key, &addr_string) != CS_OK) { continue; } - err = totemip_parse(&local_ip, addr_string, AF_UNSPEC); + err = totemip_parse(&local_ip, addr_string, totem_config->ip_version); if (err != 0) { continue; } local_ip.nodeid = totem_config->node_id; /* In case this is a new link, fill in the defaults if there was no interface{} section for it */ if (!totem_config->interfaces[i].knet_link_priority) totem_config->interfaces[i].knet_link_priority = 1; /* knet_ping_interval & knet_ping_timeout are set later once we know all the other params */ if (!totem_config->interfaces[i].knet_ping_precision) totem_config->interfaces[i].knet_ping_precision = KNET_PING_PRECISION; if (!totem_config->interfaces[i].knet_pong_count) totem_config->interfaces[i].knet_pong_count = KNET_PONG_COUNT; if (!totem_config->interfaces[i].knet_transport) totem_config->interfaces[i].knet_transport = KNET_TRANSPORT_UDP; if (!totem_config->interfaces[i].ip_port) totem_config->interfaces[i].ip_port = DEFAULT_PORT + i; totempg_iface_set(&local_ip, totem_config->interfaces[i].ip_port, i); } } /* Check for differences in config that can't be done on-the-fly and print an error */ static void check_things_have_not_changed(struct totem_config *totem_config) { int i,j; const char *ip_str; char addr_buf[INET6_ADDRSTRLEN]; int changed = 0; for (i = 0; iinterfaces[i].configured) { if (totem_config->interfaces[i].knet_transport != totem_config->orig_interfaces[i].knet_transport) { log_printf(LOGSYS_LEVEL_ERROR, "New config has different knet transport for link %d. Internal value was NOT changed.\n", i); changed = 1; } for (j=0; j < min(totem_config->interfaces[i].member_count, totem_config->orig_interfaces[i].member_count); j++) { if (memcmp(&totem_config->interfaces[i].member_list[j], &totem_config->orig_interfaces[i].member_list[j], sizeof(struct totem_ip_address))) { ip_str = totemip_print(&totem_config->orig_interfaces[i].member_list[j]); /* if ip_str is NULL then the old address was invalid and is allowed to change */ if (ip_str) { strncpy(addr_buf, ip_str, sizeof(addr_buf)); addr_buf[sizeof(addr_buf) - 1] = '\0'; log_printf(LOGSYS_LEVEL_ERROR, "new config has different address for link %d (addr changed from %s to %s). Internal value was NOT changed.\n", i, addr_buf, totemip_print(&totem_config->interfaces[i].member_list[j])); changed = 1; } } } } } if (changed) { log_printf(LOGSYS_LEVEL_ERROR, "To reconfigure an interface it must be deleted and recreated. A working interface needs to be available to corosync at all times"); } } static int put_nodelist_members_to_config(struct totem_config *totem_config, int reload, const char **error_string) { icmap_iter_t iter, iter2; const char *iter_key, *iter_key2; int res = 0; unsigned int node_pos; char tmp_key[ICMAP_KEYNAME_MAXLEN]; char tmp_key2[ICMAP_KEYNAME_MAXLEN]; char *node_addr_str; int member_count; unsigned int linknumber = 0; int i, j; int last_node_pos = -1; struct totem_interface *new_interfaces = NULL; if (reload) { /* * We need to compute diff only for reload. Also for initial configuration * not all totem structures are initialized so corosync will crash during * member_add/remove */ new_interfaces = malloc (sizeof (struct totem_interface) * INTERFACE_MAX); assert(new_interfaces != NULL); } /* Clear out nodelist so we can put the new one in if needed */ for (i = 0; i < INTERFACE_MAX; i++) { for (j = 0; j < PROCESSOR_COUNT_MAX; j++) { memset(&totem_config->interfaces[i].member_list[j], 0, sizeof(struct totem_ip_address)); } totem_config->interfaces[i].member_count = 0; } iter = icmap_iter_init("nodelist.node."); while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, tmp_key); if (res != 2) { continue; } /* If it's the same as the last node_pos then skip it */ if (node_pos == last_node_pos) { continue; } last_node_pos = node_pos; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.", node_pos); iter2 = icmap_iter_init(tmp_key); while ((iter_key2 = icmap_iter_next(iter2, NULL, NULL)) != NULL) { unsigned int nodeid; char *str; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.nodeid", node_pos); if (icmap_get_uint32(tmp_key, &nodeid) != CS_OK) { nodeid = 0; } res = sscanf(iter_key2, "nodelist.node.%u.ring%u%s", &node_pos, &linknumber, tmp_key2); if (res != 3 || strcmp(tmp_key2, "_addr") != 0) { continue; } if (icmap_get_string(iter_key2, &node_addr_str) != CS_OK) { continue; } /* Generate nodeids if they are not provided and transport is UDP/U */ if (!nodeid && (totem_config->transport_number == TOTEM_TRANSPORT_UDP || totem_config->transport_number == TOTEM_TRANSPORT_UDPU)) { snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring0_addr", node_pos); if (icmap_get_string(tmp_key, &str) == CS_OK) { nodeid = generate_nodeid(totem_config, str); log_printf(LOGSYS_LEVEL_DEBUG, "Generated nodeid = 0x%x for %s", nodeid, str); free(str); } } member_count = totem_config->interfaces[linknumber].member_count; res = totemip_parse(&totem_config->interfaces[linknumber].member_list[member_count], node_addr_str, totem_config->ip_version); if (res == 0) { totem_config->interfaces[linknumber].member_list[member_count].nodeid = nodeid; totem_config->interfaces[linknumber].member_count++; totem_config->interfaces[linknumber].configured = 1; } else { sprintf(error_string_response, "failed to parse node address '%s'\n", node_addr_str); *error_string = error_string_response; memset(&totem_config->interfaces[linknumber].member_list[member_count], 0, sizeof(struct totem_ip_address)); free(node_addr_str); icmap_iter_finalize(iter2); icmap_iter_finalize(iter); return -1; } free(node_addr_str); } icmap_iter_finalize(iter2); } icmap_iter_finalize(iter); if (reload) { log_printf(LOGSYS_LEVEL_DEBUG, "About to reconfigure links from nodelist.\n"); reconfigure_links(totem_config); memcpy(new_interfaces, totem_config->interfaces, sizeof (struct totem_interface) * INTERFACE_MAX); check_things_have_not_changed(totem_config); compute_interfaces_diff(totem_config->orig_interfaces, new_interfaces); free(new_interfaces); } return 0; } static void config_convert_nodelist_to_interface(struct totem_config *totem_config) { int res = 0; int node_pos; char tmp_key[ICMAP_KEYNAME_MAXLEN]; char tmp_key2[ICMAP_KEYNAME_MAXLEN]; char *node_addr_str; unsigned int linknumber = 0; icmap_iter_t iter; const char *iter_key; node_pos = find_local_node(1); if (node_pos > -1) { /* * We found node, so create interface section */ snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.", node_pos); iter = icmap_iter_init(tmp_key); while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { res = sscanf(iter_key, "nodelist.node.%u.ring%u%s", &node_pos, &linknumber, tmp_key2); if (res != 3 || strcmp(tmp_key2, "_addr") != 0) { continue ; } if (icmap_get_string(iter_key, &node_addr_str) != CS_OK) { continue; } snprintf(tmp_key2, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.bindnetaddr", linknumber); icmap_set_string(tmp_key2, node_addr_str); free(node_addr_str); } icmap_iter_finalize(iter); } } static int get_interface_params(struct totem_config *totem_config, const char **error_string, uint64_t *warnings, int reload) { int res = 0; unsigned int linknumber = 0; int member_count = 0; int i; icmap_iter_t iter, member_iter; const char *iter_key; const char *member_iter_key; char linknumber_key[ICMAP_KEYNAME_MAXLEN]; char tmp_key[ICMAP_KEYNAME_MAXLEN]; uint8_t u8; uint32_t u32; char *str; char *cluster_name = NULL; if (reload) { for (i=0; iinterfaces[i].configured = 0; totem_config->interfaces[i].knet_ping_timeout = 0; totem_config->interfaces[i].knet_ping_interval = 0; totem_config->interfaces[i].knet_ping_precision = KNET_PING_PRECISION; totem_config->interfaces[i].knet_pong_count = KNET_PONG_COUNT; } } if (icmap_get_string("totem.cluster_name", &cluster_name) != CS_OK) { cluster_name = NULL; } iter = icmap_iter_init("totem.interface."); while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { res = sscanf(iter_key, "totem.interface.%[^.].%s", linknumber_key, tmp_key); if (res != 2) { continue; } if (strcmp(tmp_key, "bindnetaddr") != 0 && totem_config->transport_number == TOTEM_TRANSPORT_UDP) { continue; } member_count = 0; linknumber = atoi(linknumber_key); if (linknumber >= INTERFACE_MAX) { free(cluster_name); snprintf (error_string_response, sizeof(error_string_response), "parse error in config: interface ring number %u is bigger than allowed maximum %u\n", linknumber, INTERFACE_MAX - 1); *error_string = error_string_response; return -1; } /* These things are only valid for the initial read */ if (!reload) { /* * Get the bind net address */ snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.bindnetaddr", linknumber); if (icmap_get_string(tmp_key, &str) == CS_OK) { res = totemip_parse (&totem_config->interfaces[linknumber].bindnet, str, totem_config->ip_version); if (res) { sprintf(error_string_response, "failed to parse bindnet address '%s'\n", str); *error_string = error_string_response; free(str); return -1; } free(str); } /* * Get interface multicast address */ snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastaddr", linknumber); if (icmap_get_string(tmp_key, &str) == CS_OK) { res = totemip_parse (&totem_config->interfaces[linknumber].mcast_addr, str, totem_config->ip_version); if (res) { sprintf(error_string_response, "failed to parse mcast address '%s'\n", str); *error_string = error_string_response; free(str); return -1; } free(str); - } else { + } else if (totem_config->transport_number == TOTEM_TRANSPORT_UDP) { /* * User not specified address -> autogenerate one from cluster_name key * (if available). Return code is intentionally ignored, because * udpu doesn't need mcastaddr and validity of mcastaddr for udp is * checked later anyway. */ (void)get_cluster_mcast_addr (cluster_name, linknumber, totem_config->ip_version, &totem_config->interfaces[linknumber].mcast_addr); } snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.broadcast", linknumber); if (icmap_get_string(tmp_key, &str) == CS_OK) { if (strcmp (str, "yes") == 0) { totem_config->broadcast_use = 1; } free(str); } } /* These things are only valid for the initial read OR a newly-defined link */ if (!reload || (totem_config->interfaces[linknumber].configured == 0)) { /* * Get mcast port */ snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastport", linknumber); if (icmap_get_uint16(tmp_key, &totem_config->interfaces[linknumber].ip_port) != CS_OK) { if (totem_config->broadcast_use) { totem_config->interfaces[linknumber].ip_port = DEFAULT_PORT + (2 * linknumber); } else { totem_config->interfaces[linknumber].ip_port = DEFAULT_PORT + linknumber; } } /* * Get the TTL */ totem_config->interfaces[linknumber].ttl = 1; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.ttl", linknumber); if (icmap_get_uint8(tmp_key, &u8) == CS_OK) { totem_config->interfaces[linknumber].ttl = u8; } totem_config->interfaces[linknumber].knet_transport = KNET_DEFAULT_TRANSPORT; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_transport", linknumber); if (icmap_get_string(tmp_key, &str) == CS_OK) { if (strcmp(str, "sctp") == 0) { totem_config->interfaces[linknumber].knet_transport = KNET_TRANSPORT_SCTP; } else if (strcmp(str, "udp") == 0) { totem_config->interfaces[linknumber].knet_transport = KNET_TRANSPORT_UDP; } else { *error_string = "Unrecognised knet_transport. expected 'udp' or 'sctp'"; return -1; } } } totem_config->interfaces[linknumber].configured = 1; /* * Get the knet link params */ totem_config->interfaces[linknumber].knet_link_priority = 1; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_link_priority", linknumber); if (icmap_get_uint8(tmp_key, &u8) == CS_OK) { totem_config->interfaces[linknumber].knet_link_priority = u8; } totem_config->interfaces[linknumber].knet_ping_interval = 0; /* real default applied later */ snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_interval", linknumber); if (icmap_get_uint32(tmp_key, &u32) == CS_OK) { totem_config->interfaces[linknumber].knet_ping_interval = u32; } totem_config->interfaces[linknumber].knet_ping_timeout = 0; /* real default applied later */ snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_timeout", linknumber); if (icmap_get_uint32(tmp_key, &u32) == CS_OK) { totem_config->interfaces[linknumber].knet_ping_timeout = u32; } totem_config->interfaces[linknumber].knet_ping_precision = KNET_PING_PRECISION; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_precision", linknumber); if (icmap_get_uint32(tmp_key, &u32) == CS_OK) { totem_config->interfaces[linknumber].knet_ping_precision = u32; } totem_config->interfaces[linknumber].knet_pong_count = KNET_PONG_COUNT; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_pong_count", linknumber); if (icmap_get_uint32(tmp_key, &u32) == CS_OK) { totem_config->interfaces[linknumber].knet_pong_count = u32; } snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.member.", linknumber); member_iter = icmap_iter_init(tmp_key); while ((member_iter_key = icmap_iter_next(member_iter, NULL, NULL)) != NULL) { if (member_count == 0) { if (icmap_get_string("nodelist.node.0.ring0_addr", &str) == CS_OK) { free(str); *warnings |= TOTEM_CONFIG_WARNING_MEMBERS_IGNORED; break; } else { *warnings |= TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED; } } if (icmap_get_string(member_iter_key, &str) == CS_OK) { res = totemip_parse (&totem_config->interfaces[linknumber].member_list[member_count++], str, totem_config->ip_version); if (res) { sprintf(error_string_response, "failed to parse node address '%s'\n", str); *error_string = error_string_response; icmap_iter_finalize(member_iter); icmap_iter_finalize(iter); free(str); return -1; } free(str); } } icmap_iter_finalize(member_iter); totem_config->interfaces[linknumber].member_count = member_count; } icmap_iter_finalize(iter); return 0; } extern int totem_config_read ( struct totem_config *totem_config, const char **error_string, uint64_t *warnings) { int res = 0; char *str, *ring0_addr_str; char tmp_key[ICMAP_KEYNAME_MAXLEN]; uint16_t u16; int i; int local_node_pos; int nodeid_set; *warnings = 0; memset (totem_config, 0, sizeof (struct totem_config)); totem_config->interfaces = malloc (sizeof (struct totem_interface) * INTERFACE_MAX); if (totem_config->interfaces == 0) { *error_string = "Out of memory trying to allocate ethernet interface storage area"; return -1; } totem_config->transport_number = TOTEM_TRANSPORT_KNET; if (icmap_get_string("totem.transport", &str) == CS_OK) { if (strcmp (str, "udpu") == 0) { totem_config->transport_number = TOTEM_TRANSPORT_UDPU; } if (strcmp (str, "udp") == 0) { totem_config->transport_number = TOTEM_TRANSPORT_UDP; } if (strcmp (str, "knet") == 0) { totem_config->transport_number = TOTEM_TRANSPORT_KNET; } free(str); } memset (totem_config->interfaces, 0, sizeof (struct totem_interface) * INTERFACE_MAX); strcpy (totem_config->link_mode, "passive"); icmap_get_uint32("totem.version", (uint32_t *)&totem_config->version); if (totem_get_crypto(totem_config, error_string) != 0) { return -1; } if (icmap_get_string("totem.link_mode", &str) == CS_OK) { if (strlen(str) >= TOTEM_LINK_MODE_BYTES) { *error_string = "totem.link_mode is too long"; free(str); return -1; } strcpy (totem_config->link_mode, str); free(str); } icmap_get_uint32("totem.nodeid", &totem_config->node_id); totem_config->clear_node_high_bit = 0; if (icmap_get_string("totem.clear_node_high_bit", &str) == CS_OK) { if (strcmp (str, "yes") == 0) { totem_config->clear_node_high_bit = 1; } free(str); } icmap_get_uint32("totem.threads", &totem_config->threads); icmap_get_uint32("totem.netmtu", &totem_config->net_mtu); totem_config->ip_version = totem_config_get_ip_version(totem_config); if (icmap_get_string("totem.interface.0.bindnetaddr", &str) != CS_OK) { /* * We were not able to find ring 0 bindnet addr. Try to use nodelist informations */ config_convert_nodelist_to_interface(totem_config); } else { if (icmap_get_string("nodelist.node.0.ring0_addr", &ring0_addr_str) == CS_OK) { /* * Both bindnetaddr and ring0_addr are set. * Log warning information, and use nodelist instead */ *warnings |= TOTEM_CONFIG_BINDNETADDR_NODELIST_SET; config_convert_nodelist_to_interface(totem_config); free(ring0_addr_str); } free(str); } /* * Broadcast option is global but set in interface section, * so reset before processing interfaces. */ totem_config->broadcast_use = 0; res = get_interface_params(totem_config, error_string, warnings, 0); if (res < 0) { return res; } /* * Use broadcast is global, so if set, make sure to fill mcast addr correctly * broadcast is only supported for UDP so just do interface 0; */ if (totem_config->broadcast_use) { totemip_parse (&totem_config->interfaces[0].mcast_addr, - "255.255.255.255", 0); + "255.255.255.255", TOTEM_IP_VERSION_4); } /* * Store automatically generated items back to icmap only for UDP */ if (totem_config->transport_number == TOTEM_TRANSPORT_UDP) { for (i = 0; i < INTERFACE_MAX; i++) { if (!totem_config->interfaces[i].configured) { continue; } snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastaddr", i); if (icmap_get_string(tmp_key, &str) == CS_OK) { free(str); } else { str = (char *)totemip_print(&totem_config->interfaces[i].mcast_addr); icmap_set_string(tmp_key, str); } snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastport", i); if (icmap_get_uint16(tmp_key, &u16) != CS_OK) { icmap_set_uint16(tmp_key, totem_config->interfaces[i].ip_port); } } } /* * Check existence of nodelist */ if ((icmap_get_string("nodelist.node.0.name", &str) == CS_OK) || (icmap_get_string("nodelist.node.0.ring0_addr", &str) == CS_OK)) { free(str); /* * find local node */ local_node_pos = find_local_node(1); if (local_node_pos != -1) { snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.nodeid", local_node_pos); nodeid_set = (totem_config->node_id != 0); if (icmap_get_uint32(tmp_key, &totem_config->node_id) == CS_OK && nodeid_set) { *warnings |= TOTEM_CONFIG_WARNING_TOTEM_NODEID_IGNORED; } if ((totem_config->transport_number == TOTEM_TRANSPORT_KNET) && (!totem_config->node_id)) { *error_string = "With knet, you must specify nodeid for current node"; return -1; } if ((totem_config->transport_number == TOTEM_TRANSPORT_UDP || totem_config->transport_number == TOTEM_TRANSPORT_UDPU) && (!totem_config->node_id)) { snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.ring0_addr", local_node_pos); icmap_get_string(tmp_key, &str); totem_config->node_id = generate_nodeid(totem_config, str); totem_config->interfaces[0].member_list[local_node_pos].nodeid = totem_config->node_id; free(str); } /* Users must not change this */ icmap_set_ro_access("nodelist.local_node_pos", 0, 1); } if (put_nodelist_members_to_config(totem_config, 0, error_string)) { return -1; } } /* * Get things that might change in the future (and can depend on totem_config->interfaces); */ totem_volatile_config_read(totem_config, NULL); calc_knet_ping_timers(totem_config); icmap_set_uint8("config.totemconfig_reload_in_progress", 0); add_totem_config_notification(totem_config); return 0; } int totem_config_validate ( struct totem_config *totem_config, const char **error_string) { static char local_error_reason[512]; char parse_error[512]; static char addr_str_buf[INET6_ADDRSTRLEN]; const char *error_reason = local_error_reason; int i,j; uint32_t u32; int num_configured = 0; unsigned int interface_max = INTERFACE_MAX; for (i = 0; i < INTERFACE_MAX; i++) { if (totem_config->interfaces[i].configured) { num_configured++; } } if (num_configured == 0) { error_reason = "No interfaces defined"; goto parse_error; } /* Check we found a local node name */ if (icmap_get_uint32("nodelist.local_node_pos", &u32) != CS_OK) { error_reason = "No valid name found for local host"; goto parse_error; } for (i = 0; i < INTERFACE_MAX; i++) { /* * Some error checking of parsed data to make sure its valid */ struct totem_ip_address null_addr; if (!totem_config->interfaces[i].configured) { continue; } memset (&null_addr, 0, sizeof (struct totem_ip_address)); if ((totem_config->transport_number == TOTEM_TRANSPORT_UDP) && memcmp (&totem_config->interfaces[i].mcast_addr, &null_addr, sizeof (struct totem_ip_address)) == 0) { error_reason = "No multicast address specified"; goto parse_error; } if (totem_config->interfaces[i].ip_port == 0) { error_reason = "No multicast port specified"; goto parse_error; } if (totem_config->interfaces[i].ttl > 255) { error_reason = "Invalid TTL (should be 0..255)"; goto parse_error; } if (totem_config->transport_number != TOTEM_TRANSPORT_UDP && totem_config->interfaces[i].ttl != 1) { error_reason = "Can only set ttl on multicast transport types"; goto parse_error; } if (totem_config->interfaces[i].knet_link_priority > 255) { error_reason = "Invalid link priority (should be 0..255)"; goto parse_error; } if (totem_config->transport_number != TOTEM_TRANSPORT_KNET && totem_config->interfaces[i].knet_link_priority != 1) { error_reason = "Can only set link priority on knet transport type"; goto parse_error; } if (totem_config->interfaces[i].mcast_addr.family == AF_INET6 && totem_config->node_id == 0) { error_reason = "An IPV6 network requires that a node ID be specified."; goto parse_error; } if (totem_config->broadcast_use == 0 && totem_config->transport_number == TOTEM_TRANSPORT_UDP) { if (totem_config->interfaces[i].mcast_addr.family != totem_config->interfaces[i].bindnet.family) { error_reason = "Multicast address family does not match bind address family"; goto parse_error; } if (totemip_is_mcast (&totem_config->interfaces[i].mcast_addr) != 0) { error_reason = "mcastaddr is not a correct multicast address."; goto parse_error; } } /* Verify that all nodes on the same knet link have the same IP family */ for (j=1; jinterfaces[i].member_count; j++) { if (totem_config->interfaces[i].configured) { if (totem_config->interfaces[i].member_list[j].family != totem_config->interfaces[i].member_list[0].family) { memcpy(addr_str_buf, totemip_print(&(totem_config->interfaces[i].member_list[j])), sizeof(addr_str_buf)); snprintf (local_error_reason, sizeof(local_error_reason), "Nodes for link %d have different IP families " "(compared %s with %s)", i, addr_str_buf, totemip_print(&(totem_config->interfaces[i].member_list[0]))); goto parse_error; } } } } if (totem_config->version != 2) { error_reason = "This totem parser can only parse version 2 configurations."; goto parse_error; } if (totem_volatile_config_validate(totem_config, error_string) == -1) { return (-1); } if (check_for_duplicate_nodeids(totem_config, error_string) == -1) { return (-1); } /* * KNET Link values validation */ if (strcmp (totem_config->link_mode, "active") && strcmp (totem_config->link_mode, "rr") && strcmp (totem_config->link_mode, "passive")) { snprintf (local_error_reason, sizeof(local_error_reason), "The Knet link mode \"%s\" specified is invalid. It must be active, passive or rr.\n", totem_config->link_mode); goto parse_error; } /* Only Knet does multiple interfaces */ if (totem_config->transport_number != TOTEM_TRANSPORT_KNET) { interface_max = 1; } if (interface_max < num_configured) { snprintf (parse_error, sizeof(parse_error), "%d is too many configured interfaces for non-Knet transport.", num_configured); error_reason = parse_error; goto parse_error; } /* Only knet allows crypto */ if (totem_config->transport_number != TOTEM_TRANSPORT_KNET) { if ((strcmp(totem_config->crypto_cipher_type, "none") != 0) || (strcmp(totem_config->crypto_hash_type, "none") != 0)) { snprintf (parse_error, sizeof(parse_error), "crypto_cipher & crypto_hash are only valid for the Knet transport."); error_reason = parse_error; goto parse_error; } } if (totem_config->net_mtu == 0) { if (totem_config->transport_number == TOTEM_TRANSPORT_KNET) { totem_config->net_mtu = KNET_MAX_PACKET_SIZE; } else { totem_config->net_mtu = 1500; } } return 0; parse_error: snprintf (error_string_response, sizeof(error_string_response), "parse error in config: %s\n", error_reason); *error_string = error_string_response; return (-1); } static int read_keyfile ( const char *key_location, struct totem_config *totem_config, const char **error_string) { int fd; int res; int saved_errno; char error_str[100]; const char *error_ptr; fd = open (key_location, O_RDONLY); if (fd == -1) { error_ptr = qb_strerror_r(errno, error_str, sizeof(error_str)); snprintf (error_string_response, sizeof(error_string_response), "Could not open %s: %s\n", key_location, error_ptr); goto parse_error; } res = read (fd, totem_config->private_key, TOTEM_PRIVATE_KEY_LEN_MAX); saved_errno = errno; close (fd); if (res == -1) { error_ptr = qb_strerror_r (saved_errno, error_str, sizeof(error_str)); snprintf (error_string_response, sizeof(error_string_response), "Could not read %s: %s\n", key_location, error_ptr); goto parse_error; } if (res < TOTEM_PRIVATE_KEY_LEN_MIN) { snprintf (error_string_response, sizeof(error_string_response), "Could only read %d bits of minimum %u bits from %s.\n", res * 8, TOTEM_PRIVATE_KEY_LEN_MIN * 8, key_location); goto parse_error; } totem_config->private_key_len = res; return 0; parse_error: *error_string = error_string_response; return (-1); } int totem_config_keyread ( struct totem_config *totem_config, const char **error_string) { int got_key = 0; char *key_location = NULL; int res; size_t key_len; memset (totem_config->private_key, 0, sizeof(totem_config->private_key)); totem_config->private_key_len = 0; if (strcmp(totem_config->crypto_cipher_type, "none") == 0 && strcmp(totem_config->crypto_hash_type, "none") == 0) { return (0); } /* cmap may store the location of the key file */ if (icmap_get_string("totem.keyfile", &key_location) == CS_OK) { res = read_keyfile(key_location, totem_config, error_string); free(key_location); if (res) { goto key_error; } got_key = 1; } else { /* Or the key itself may be in the cmap */ if (icmap_get("totem.key", NULL, &key_len, NULL) == CS_OK) { if (key_len > sizeof(totem_config->private_key)) { sprintf(error_string_response, "key is too long"); goto key_error; } if (key_len < TOTEM_PRIVATE_KEY_LEN_MIN) { sprintf(error_string_response, "key is too short"); goto key_error; } if (icmap_get("totem.key", totem_config->private_key, &key_len, NULL) == CS_OK) { totem_config->private_key_len = key_len; got_key = 1; } else { sprintf(error_string_response, "can't load private key"); goto key_error; } } } /* In desperation we read the default filename */ if (!got_key) { res = read_keyfile(COROSYSCONFDIR "/authkey", totem_config, error_string); if (res) goto key_error; } return (0); key_error: *error_string = error_string_response; return (-1); } static void debug_dump_totem_config(const struct totem_config *totem_config) { log_printf(LOGSYS_LEVEL_DEBUG, "Token Timeout (%d ms) retransmit timeout (%d ms)", totem_config->token_timeout, totem_config->token_retransmit_timeout); if (totem_config->token_warning) { uint32_t token_warning_ms = totem_config->token_warning * totem_config->token_timeout / 100; log_printf(LOGSYS_LEVEL_DEBUG, "Token warning every %d ms (%d%% of Token Timeout)", token_warning_ms, totem_config->token_warning); if (token_warning_ms < totem_config->token_retransmit_timeout) log_printf (LOGSYS_LEVEL_DEBUG, "The token warning interval (%d ms) is less than the token retransmit timeout (%d ms) " "which can lead to spurious token warnings. Consider increasing the token_warning parameter.", token_warning_ms, totem_config->token_retransmit_timeout); } else log_printf(LOGSYS_LEVEL_DEBUG, "Token warnings disabled"); log_printf(LOGSYS_LEVEL_DEBUG, "token hold (%d ms) retransmits before loss (%d retrans)", totem_config->token_hold_timeout, totem_config->token_retransmits_before_loss_const); log_printf(LOGSYS_LEVEL_DEBUG, "join (%d ms) send_join (%d ms) consensus (%d ms) merge (%d ms)", totem_config->join_timeout, totem_config->send_join_timeout, totem_config->consensus_timeout, totem_config->merge_timeout); log_printf(LOGSYS_LEVEL_DEBUG, "downcheck (%d ms) fail to recv const (%d msgs)", totem_config->downcheck_timeout, totem_config->fail_to_recv_const); log_printf(LOGSYS_LEVEL_DEBUG, "seqno unchanged const (%d rotations) Maximum network MTU %d", totem_config->seqno_unchanged_const, totem_config->net_mtu); log_printf(LOGSYS_LEVEL_DEBUG, "window size per rotation (%d messages) maximum messages per rotation (%d messages)", totem_config->window_size, totem_config->max_messages); log_printf(LOGSYS_LEVEL_DEBUG, "missed count const (%d messages)", totem_config->miss_count_const); log_printf(LOGSYS_LEVEL_DEBUG, "heartbeat_failures_allowed (%d)", totem_config->heartbeat_failures_allowed); log_printf(LOGSYS_LEVEL_DEBUG, "max_network_delay (%d ms)", totem_config->max_network_delay); } static void totem_change_notify( int32_t event, const char *key_name, struct icmap_notify_value new_val, struct icmap_notify_value old_val, void *user_data) { struct totem_config *totem_config = (struct totem_config *)user_data; uint32_t *param; uint8_t reloading; const char *deleted_key = NULL; const char *error_string; /* * If a full reload is in progress then don't do anything until it's done and * can reconfigure it all atomically */ if (icmap_get_uint8("config.reload_in_progress", &reloading) == CS_OK && reloading) return; param = totem_get_param_by_name((struct totem_config *)user_data, key_name); /* * Process change only if changed key is found in totem_config (-> param is not NULL) * or for special key token_coefficient. token_coefficient key is not stored in * totem_config, but it is used for computation of token timeout. */ if (!param && strcmp(key_name, "totem.token_coefficient") != 0) return; /* * Values other than UINT32 are not supported, or needed (yet) */ switch (event) { case ICMAP_TRACK_DELETE: deleted_key = key_name; break; case ICMAP_TRACK_ADD: case ICMAP_TRACK_MODIFY: deleted_key = NULL; break; default: break; } totem_volatile_config_read (totem_config, deleted_key); log_printf(LOGSYS_LEVEL_DEBUG, "Totem related config key changed. Dumping actual totem config."); debug_dump_totem_config(totem_config); if (totem_volatile_config_validate(totem_config, &error_string) == -1) { log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string); /* * TODO: Consider corosync exit and/or load defaults for volatile * values. For now, log error seems to be enough */ } } static void totem_reload_notify( int32_t event, const char *key_name, struct icmap_notify_value new_val, struct icmap_notify_value old_val, void *user_data) { struct totem_config *totem_config = (struct totem_config *)user_data; const char *error_string; uint64_t warnings; /* Reload has completed */ if (*(uint8_t *)new_val.data == 0) { totem_config->orig_interfaces = malloc (sizeof (struct totem_interface) * INTERFACE_MAX); assert(totem_config->orig_interfaces != NULL); memcpy(totem_config->orig_interfaces, totem_config->interfaces, sizeof (struct totem_interface) * INTERFACE_MAX); get_interface_params(totem_config, &error_string, &warnings, 1); if (put_nodelist_members_to_config (totem_config, 1, &error_string)) { log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string); } totem_volatile_config_read (totem_config, NULL); calc_knet_ping_timers(totem_config); log_printf(LOGSYS_LEVEL_DEBUG, "Configuration reloaded. Dumping actual totem config."); debug_dump_totem_config(totem_config); if (totem_volatile_config_validate(totem_config, &error_string) == -1) { log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string); /* * TODO: Consider corosync exit and/or load defaults for volatile * values. For now, log error seems to be enough */ } /* Reinstate the local_node_pos */ (void)find_local_node(0); /* Reconfigure network params as appropriate */ totempg_reconfigure(); free(totem_config->orig_interfaces); icmap_set_uint8("config.totemconfig_reload_in_progress", 0); } else { icmap_set_uint8("config.totemconfig_reload_in_progress", 1); } } static void add_totem_config_notification(struct totem_config *totem_config) { icmap_track_t icmap_track; icmap_track_add("totem.", ICMAP_TRACK_ADD | ICMAP_TRACK_DELETE | ICMAP_TRACK_MODIFY | ICMAP_TRACK_PREFIX, totem_change_notify, totem_config, &icmap_track); icmap_track_add("config.reload_in_progress", ICMAP_TRACK_ADD | ICMAP_TRACK_MODIFY, totem_reload_notify, totem_config, &icmap_track); } diff --git a/exec/totemip.c b/exec/totemip.c index afbda7ce..9d96e1ba 100644 --- a/exec/totemip.c +++ b/exec/totemip.c @@ -1,545 +1,561 @@ /* * Copyright (c) 2005-2011 Red Hat, Inc. * * All rights reserved. * * Author: Patrick Caulfield (pcaulfie@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the MontaVista Software, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ /* IPv4/6 abstraction */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define LOCALHOST_IPV4 "127.0.0.1" #define LOCALHOST_IPV6 "::1" #define NETLINK_BUFSIZE 16384 #ifdef SO_NOSIGPIPE void totemip_nosigpipe(int s) { int on = 1; setsockopt(s, SOL_SOCKET, SO_NOSIGPIPE, (void *)&on, sizeof(on)); } #endif /* Compare two addresses */ int totemip_equal(const struct totem_ip_address *addr1, const struct totem_ip_address *addr2) { int addrlen = 0; if (addr1->family != addr2->family) return 0; if (addr1->family == AF_INET) { addrlen = sizeof(struct in_addr); } if (addr1->family == AF_INET6) { addrlen = sizeof(struct in6_addr); } assert(addrlen); if (memcmp(addr1->addr, addr2->addr, addrlen) == 0) return 1; else return 0; } /* Copy a totem_ip_address */ void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2) { memcpy(addr1, addr2, sizeof(struct totem_ip_address)); } void totemip_copy_endian_convert(struct totem_ip_address *addr1, const struct totem_ip_address *addr2) { addr1->nodeid = swab32(addr2->nodeid); addr1->family = swab16(addr2->family); memcpy(addr1->addr, addr2->addr, TOTEMIP_ADDRLEN); } /* * Multicast address range is 224.0.0.0 to 239.255.255.255 this * translates to the first 4 bits == 1110 (0xE). * http://en.wikipedia.org/wiki/Multicast_address */ int32_t totemip_is_mcast(struct totem_ip_address *ip_addr) { uint32_t addr = 0; memcpy (&addr, ip_addr->addr, sizeof (uint32_t)); if (ip_addr->family == AF_INET) { addr = ntohl(addr); if ((addr >> 28) != 0xE) { return -1; } } return 0; } /* For sorting etc. params are void * for qsort's benefit */ int totemip_compare(const void *a, const void *b) { int i; const struct totem_ip_address *totemip_a = (const struct totem_ip_address *)a; const struct totem_ip_address *totemip_b = (const struct totem_ip_address *)b; struct in_addr ipv4_a1; struct in_addr ipv4_a2; struct in6_addr ipv6_a1; struct in6_addr ipv6_a2; unsigned short family; /* * Use memcpy to align since totem_ip_address is unaligned on various archs */ memcpy (&family, &totemip_a->family, sizeof (unsigned short)); if (family == AF_INET) { memcpy (&ipv4_a1, totemip_a->addr, sizeof (struct in_addr)); memcpy (&ipv4_a2, totemip_b->addr, sizeof (struct in_addr)); if (ipv4_a1.s_addr == ipv4_a2.s_addr) { return (0); } if (htonl(ipv4_a1.s_addr) < htonl(ipv4_a2.s_addr)) { return -1; } else { return +1; } } else if (family == AF_INET6) { /* * We can only compare 8 bits at time for portability reasons */ memcpy (&ipv6_a1, totemip_a->addr, sizeof (struct in6_addr)); memcpy (&ipv6_a2, totemip_b->addr, sizeof (struct in6_addr)); for (i = 0; i < 16; i++) { int res = ipv6_a1.s6_addr[i] - ipv6_a2.s6_addr[i]; if (res) { return res; } } return 0; } else { /* * Family not set, should be! */ assert (0); } return 0; } /* Build a localhost totem_ip_address */ int totemip_localhost(int family, struct totem_ip_address *localhost) { const char *addr_text; memset (localhost, 0, sizeof (struct totem_ip_address)); if (family == AF_INET) { addr_text = LOCALHOST_IPV4; if (inet_pton(family, addr_text, (char *)&localhost->nodeid) <= 0) { return -1; } } else { addr_text = LOCALHOST_IPV6; } if (inet_pton(family, addr_text, (char *)localhost->addr) <= 0) return -1; localhost->family = family; return 0; } int totemip_localhost_check(const struct totem_ip_address *addr) { struct totem_ip_address localhost; if (totemip_localhost(addr->family, &localhost)) return 0; return totemip_equal(addr, &localhost); } const char *totemip_sa_print(const struct sockaddr *sa) { static char buf[INET6_ADDRSTRLEN]; buf[0] = 0; switch (sa->sa_family) { case AF_INET: inet_ntop(sa->sa_family, &((struct sockaddr_in *)(sa))->sin_addr, buf, INET6_ADDRSTRLEN); break; case AF_INET6: inet_ntop(sa->sa_family, &((struct sockaddr_in6 *)(sa))->sin6_addr, buf, INET6_ADDRSTRLEN); break; default: return (NULL); } return (buf); } const char *totemip_print(const struct totem_ip_address *addr) { static char buf[INET6_ADDRSTRLEN]; return (inet_ntop(addr->family, addr->addr, buf, sizeof(buf))); } /* Make a totem_ip_address into a usable sockaddr_storage */ int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen) { int ret = -1; if (ip_addr->family == AF_INET) { struct sockaddr_in *sin = (struct sockaddr_in *)saddr; memset(sin, 0, sizeof(struct sockaddr_in)); #ifdef HAVE_SOCK_SIN_LEN sin->sin_len = sizeof(struct sockaddr_in); #endif sin->sin_family = ip_addr->family; sin->sin_port = ntohs(port); memcpy(&sin->sin_addr, ip_addr->addr, sizeof(struct in_addr)); *addrlen = sizeof(struct sockaddr_in); ret = 0; } if (ip_addr->family == AF_INET6) { struct sockaddr_in6 *sin = (struct sockaddr_in6 *)saddr; memset(sin, 0, sizeof(struct sockaddr_in6)); #ifdef HAVE_SOCK_SIN6_LEN sin->sin6_len = sizeof(struct sockaddr_in6); #endif sin->sin6_family = ip_addr->family; sin->sin6_port = ntohs(port); sin->sin6_scope_id = 2; memcpy(&sin->sin6_addr, ip_addr->addr, sizeof(struct in6_addr)); *addrlen = sizeof(struct sockaddr_in6); ret = 0; } return ret; } -/* Converts an address string string into a totem_ip_address. - family can be AF_INET, AF_INET6 or 0 ("for "don't care") -*/ -int totemip_parse(struct totem_ip_address *totemip, const char *addr, int family) +/* + * Converts an address string string into a totem_ip_address. ip_version enum + * defines order. + */ +int totemip_parse(struct totem_ip_address *totemip, const char *addr, + enum totem_ip_version_enum ip_version) { struct addrinfo *ainfo; struct addrinfo ahints; struct sockaddr_in *sa; struct sockaddr_in6 *sa6; int ret; int debug_ip_family; + int ai_family1, ai_family2; memset(&ahints, 0, sizeof(ahints)); ahints.ai_socktype = SOCK_DGRAM; ahints.ai_protocol = IPPROTO_UDP; - ahints.ai_family = family; - /* If no address family specified then try IPv6 first then IPv4 */ - if (family == AF_UNSPEC) { - ahints.ai_family = AF_INET6; - ret = getaddrinfo(addr, NULL, &ahints, &ainfo); - if (ret) { - ahints.ai_family = AF_INET; - ret = getaddrinfo(addr, NULL, &ahints, &ainfo); - } - } else { + ai_family1 = ai_family2 = -1; + + switch (ip_version) { + case TOTEM_IP_VERSION_4: + ai_family1 = AF_INET; + break; + case TOTEM_IP_VERSION_6: + ai_family1 = AF_INET6; + break; + case TOTEM_IP_VERSION_4_6: + ai_family1 = AF_INET; + ai_family2 = AF_INET6; + break; + case TOTEM_IP_VERSION_6_4: + ai_family1 = AF_INET6; + ai_family2 = AF_INET; + break; + } + + ahints.ai_family = ai_family1; + ret = getaddrinfo(addr, NULL, &ahints, &ainfo); + if (ret && ai_family2 != -1) { + ahints.ai_family = ai_family2; ret = getaddrinfo(addr, NULL, &ahints, &ainfo); } debug_ip_family = 4; if (ahints.ai_family == AF_INET6) { debug_ip_family = 6; } if (ret) { log_printf (LOGSYS_LEVEL_DEBUG, "totemip_parse: IPv%u address of %s not resolvable", debug_ip_family, addr); return -1; } sa = (struct sockaddr_in *)ainfo->ai_addr; sa6 = (struct sockaddr_in6 *)ainfo->ai_addr; totemip->family = ainfo->ai_family; if (ainfo->ai_family == AF_INET) memcpy(totemip->addr, &sa->sin_addr, sizeof(struct in_addr)); else memcpy(totemip->addr, &sa6->sin6_addr, sizeof(struct in6_addr)); log_printf (LOGSYS_LEVEL_DEBUG, "totemip_parse: IPv%u address of %s resolved as %s", debug_ip_family, addr, totemip_print(totemip)); freeaddrinfo(ainfo); return 0; } /* Make a sockaddr_* into a totem_ip_address */ int totemip_sockaddr_to_totemip_convert(const struct sockaddr_storage *saddr, struct totem_ip_address *ip_addr) { int ret = -1; ip_addr->family = saddr->ss_family; ip_addr->nodeid = 0; if (saddr->ss_family == AF_INET) { const struct sockaddr_in *sin = (const struct sockaddr_in *)saddr; memcpy(ip_addr->addr, &sin->sin_addr, sizeof(struct in_addr)); ret = 0; } if (saddr->ss_family == AF_INET6) { const struct sockaddr_in6 *sin = (const struct sockaddr_in6 *)saddr; memcpy(ip_addr->addr, &sin->sin6_addr, sizeof(struct in6_addr)); ret = 0; } return ret; } int totemip_getifaddrs(struct qb_list_head *addrs) { struct ifaddrs *ifap, *ifa; struct totem_ip_if_address *if_addr; if (getifaddrs(&ifap) != 0) return (-1); qb_list_init(addrs); for (ifa = ifap; ifa; ifa = ifa->ifa_next) { if (ifa->ifa_addr == NULL || ifa->ifa_netmask == NULL) continue ; if ((ifa->ifa_addr->sa_family != AF_INET && ifa->ifa_addr->sa_family != AF_INET6) || (ifa->ifa_netmask->sa_family != AF_INET && ifa->ifa_netmask->sa_family != AF_INET6 && ifa->ifa_netmask->sa_family != 0)) continue ; if (ifa->ifa_netmask->sa_family == 0) { ifa->ifa_netmask->sa_family = ifa->ifa_addr->sa_family; } if_addr = malloc(sizeof(struct totem_ip_if_address)); if (if_addr == NULL) { goto error_free_ifaddrs; } qb_list_init(&if_addr->list); memset(if_addr, 0, sizeof(struct totem_ip_if_address)); if_addr->interface_up = ifa->ifa_flags & IFF_UP; if_addr->interface_num = if_nametoindex(ifa->ifa_name); if_addr->name = strdup(ifa->ifa_name); if (if_addr->name == NULL) { goto error_free_addr; } if (totemip_sockaddr_to_totemip_convert((const struct sockaddr_storage *)ifa->ifa_addr, &if_addr->ip_addr) == -1) { goto error_free_addr_name; } if (totemip_sockaddr_to_totemip_convert((const struct sockaddr_storage *)ifa->ifa_netmask, &if_addr->mask_addr) == -1) { goto error_free_addr_name; } qb_list_add_tail(&if_addr->list, addrs); } freeifaddrs(ifap); return (0); error_free_addr_name: free(if_addr->name); error_free_addr: free(if_addr); error_free_ifaddrs: totemip_freeifaddrs(addrs); freeifaddrs(ifap); return (-1); } void totemip_freeifaddrs(struct qb_list_head *addrs) { struct totem_ip_if_address *if_addr; struct qb_list_head *list, *tmp_iter; qb_list_for_each_safe(list, tmp_iter, addrs) { if_addr = qb_list_entry(list, struct totem_ip_if_address, list); free(if_addr->name); qb_list_del(&if_addr->list); free(if_addr); } qb_list_init(addrs); } int totemip_iface_check(struct totem_ip_address *bindnet, struct totem_ip_address *boundto, int *interface_up, int *interface_num, int mask_high_bit) { struct qb_list_head addrs; struct qb_list_head *list; struct totem_ip_if_address *if_addr; struct totem_ip_address bn_netaddr, if_netaddr; socklen_t addr_len; socklen_t si; int res = -1; int exact_match_found = 0; int net_match_found = 0; *interface_up = 0; *interface_num = 0; if (totemip_getifaddrs(&addrs) == -1) { return (-1); } qb_list_for_each(list, &addrs) { if_addr = qb_list_entry(list, struct totem_ip_if_address, list); if (bindnet->family != if_addr->ip_addr.family) continue ; addr_len = 0; switch (bindnet->family) { case AF_INET: addr_len = sizeof(struct in_addr); break; case AF_INET6: addr_len = sizeof(struct in6_addr); break; } if (addr_len == 0) continue ; totemip_copy(&bn_netaddr, bindnet); totemip_copy(&if_netaddr, &if_addr->ip_addr); if (totemip_equal(&bn_netaddr, &if_netaddr)) { exact_match_found = 1; } for (si = 0; si < addr_len; si++) { bn_netaddr.addr[si] = bn_netaddr.addr[si] & if_addr->mask_addr.addr[si]; if_netaddr.addr[si] = if_netaddr.addr[si] & if_addr->mask_addr.addr[si]; } if (exact_match_found || (!net_match_found && totemip_equal(&bn_netaddr, &if_netaddr))) { totemip_copy(boundto, &if_addr->ip_addr); boundto->nodeid = bindnet->nodeid; *interface_up = if_addr->interface_up; *interface_num = if_addr->interface_num; net_match_found = 1; res = 0; if (exact_match_found) { goto finished; } } } finished: totemip_freeifaddrs(&addrs); return (res); } #define TOTEMIP_UDP_HEADER_SIZE 8 #define TOTEMIP_IPV4_HEADER_SIZE 20 #define TOTEMIP_IPV6_HEADER_SIZE 40 size_t totemip_udpip_header_size(int family) { size_t header_size; header_size = 0; switch (family) { case AF_INET: header_size = TOTEMIP_UDP_HEADER_SIZE + TOTEMIP_IPV4_HEADER_SIZE; break; case AF_INET6: header_size = TOTEMIP_UDP_HEADER_SIZE + TOTEMIP_IPV6_HEADER_SIZE; break; } return (header_size); } diff --git a/include/corosync/totem/totem.h b/include/corosync/totem/totem.h index a087119c..d9d8e466 100644 --- a/include/corosync/totem/totem.h +++ b/include/corosync/totem/totem.h @@ -1,262 +1,262 @@ /* * Copyright (c) 2005 MontaVista Software, Inc. * Copyright (c) 2006-2012 Red Hat, Inc. * * Author: Steven Dake (sdake@redhat.com) * * All rights reserved. * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the MontaVista Software, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef TOTEM_H_DEFINED #define TOTEM_H_DEFINED #include "totemip.h" #include #include #include #ifdef HAVE_SMALL_MEMORY_FOOTPRINT #define PROCESSOR_COUNT_MAX 16 #define MESSAGE_SIZE_MAX 1024*64 #define MESSAGE_QUEUE_MAX 512 #else #define PROCESSOR_COUNT_MAX 384 #define MESSAGE_SIZE_MAX 1024*1024 /* (1MB) */ #define MESSAGE_QUEUE_MAX ((4 * MESSAGE_SIZE_MAX) / totem_config->net_mtu) #endif /* HAVE_SMALL_MEMORY_FOOTPRINT */ #define FRAME_SIZE_MAX KNET_MAX_PACKET_SIZE /* * Estimation of required buffer size for totemudp and totemudpu - it should be at least * sizeof(memb_join) + PROCESSOR_MAX * 2 * sizeof(srp_addr)) * if we want to support PROCESSOR_MAX nodes, but because we don't have * srp_addr and memb_join, we have to use estimation. * TODO: Consider moving srp_addr/memb_join into totem headers instead of totemsrp.c */ #define UDP_RECEIVE_FRAME_SIZE_MAX (PROCESSOR_COUNT_MAX * (INTERFACE_MAX * 2 * sizeof(struct totem_ip_address)) + 1024) #define TRANSMITS_ALLOWED 16 #define SEND_THREADS_MAX 16 /* This must be <= KNET_MAX_LINK */ #define INTERFACE_MAX 8 #define BIND_MAX_RETRIES 10 #define BIND_RETRIES_INTERVAL 100 /** * Maximum number of continuous gather states */ #define MAX_NO_CONT_GATHER 3 /* * Maximum number of continuous failures get from sendmsg call */ #define MAX_NO_CONT_SENDMSG_FAILURES 30 struct totem_interface { struct totem_ip_address bindnet; struct totem_ip_address boundto; struct totem_ip_address mcast_addr; uint16_t ip_port; uint16_t ttl; uint8_t configured; int member_count; int knet_link_priority; int knet_ping_interval; int knet_ping_timeout; int knet_ping_precision; int knet_pong_count; int knet_transport; struct totem_ip_address member_list[PROCESSOR_COUNT_MAX]; }; struct totem_logging_configuration { void (*log_printf) ( int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format, ...) __attribute__((format(printf, 6, 7))); int log_level_security; int log_level_error; int log_level_warning; int log_level_notice; int log_level_debug; int log_level_trace; int log_subsys_id; }; /* * COrosync TOtem. Also used as an endian_detector. */ #define TOTEM_MH_MAGIC 0xC070 #define TOTEM_MH_VERSION 0x03 struct totem_message_header { unsigned short magic; char version; char type; char encapsulated; unsigned int nodeid; unsigned int target_nodeid; } __attribute__((packed)); enum { TOTEM_PRIVATE_KEY_LEN_MIN = KNET_MIN_KEY_LEN, TOTEM_PRIVATE_KEY_LEN_MAX = KNET_MAX_KEY_LEN }; enum { TOTEM_LINK_MODE_BYTES = 64 }; typedef enum { TOTEM_TRANSPORT_UDP = 0, TOTEM_TRANSPORT_UDPU = 1, TOTEM_TRANSPORT_KNET = 2 } totem_transport_t; #define MEMB_RING_ID struct memb_ring_id { unsigned int rep; unsigned long long seq; } __attribute__((packed)); struct totem_config { int version; /* * network */ struct totem_interface *interfaces; struct totem_interface *orig_interfaces; /* for reload */ unsigned int node_id; unsigned int clear_node_high_bit; unsigned int knet_pmtud_interval; /* * key information */ unsigned char private_key[TOTEM_PRIVATE_KEY_LEN_MAX]; unsigned int private_key_len; /* * Totem configuration parameters */ unsigned int token_timeout; unsigned int token_warning; unsigned int token_retransmit_timeout; unsigned int token_hold_timeout; unsigned int token_retransmits_before_loss_const; unsigned int join_timeout; unsigned int send_join_timeout; unsigned int consensus_timeout; unsigned int merge_timeout; unsigned int downcheck_timeout; unsigned int fail_to_recv_const; unsigned int seqno_unchanged_const; char link_mode[TOTEM_LINK_MODE_BYTES]; struct totem_logging_configuration totem_logging_configuration; unsigned int net_mtu; unsigned int threads; unsigned int heartbeat_failures_allowed; unsigned int max_network_delay; unsigned int window_size; unsigned int max_messages; const char *vsf_type; unsigned int broadcast_use; char *crypto_model; char *crypto_cipher_type; char *crypto_hash_type; char *knet_compression_model; uint32_t knet_compression_threshold; int knet_compression_level; totem_transport_t transport_number; unsigned int miss_count_const; - int ip_version; + enum totem_ip_version_enum ip_version; void (*totem_memb_ring_id_create_or_load) ( struct memb_ring_id *memb_ring_id, unsigned int nodeid); void (*totem_memb_ring_id_store) ( const struct memb_ring_id *memb_ring_id, unsigned int nodeid); }; #define TOTEM_CONFIGURATION_TYPE enum totem_configuration_type { TOTEM_CONFIGURATION_REGULAR, TOTEM_CONFIGURATION_TRANSITIONAL }; #define TOTEM_CALLBACK_TOKEN_TYPE enum totem_callback_token_type { TOTEM_CALLBACK_TOKEN_RECEIVED = 1, TOTEM_CALLBACK_TOKEN_SENT = 2 }; enum totem_event_type { TOTEM_EVENT_DELIVERY_CONGESTED, TOTEM_EVENT_NEW_MSG, }; #endif /* TOTEM_H_DEFINED */ diff --git a/include/corosync/totem/totemip.h b/include/corosync/totem/totemip.h index 4c9f42e4..de224617 100644 --- a/include/corosync/totem/totemip.h +++ b/include/corosync/totem/totemip.h @@ -1,124 +1,131 @@ /* * Copyright (c) 2005-2010 Red Hat, Inc. * * All rights reserved. * * Author: Patrick Caulfield (pcaulfie@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the MontaVista Software, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ /* IPv4/6 abstraction */ #ifndef TOTEMIP_H_DEFINED #define TOTEMIP_H_DEFINED #include #include #include #include #include #ifdef __cplusplus extern "C" { #endif #ifdef SO_NOSIGPIPE #ifndef MSG_NOSIGNAL #define MSG_NOSIGNAL 0 #endif void totemip_nosigpipe(int s); #else #define totemip_nosigpipe(s) #endif #define TOTEMIP_ADDRLEN (sizeof(struct in6_addr)) /* These are the things that get passed around */ #define TOTEM_IP_ADDRESS struct totem_ip_address { unsigned int nodeid; unsigned short family; unsigned char addr[TOTEMIP_ADDRLEN]; } __attribute__((packed)); +enum totem_ip_version_enum { + TOTEM_IP_VERSION_4, /* Use only AF_INET */ + TOTEM_IP_VERSION_6, /* Use only AF_INET6 */ + TOTEM_IP_VERSION_4_6, /* Use AF_INET and if it fails, use AF_INET6 */ + TOTEM_IP_VERSION_6_4 /* Use AF_INET6 and if it fails, use AF_INET */ +}; + struct totem_ip_if_address { struct totem_ip_address ip_addr; struct totem_ip_address mask_addr; int interface_up; int interface_num; char *name; struct qb_list_head list; }; extern int totemip_equal(const struct totem_ip_address *addr1, const struct totem_ip_address *addr2); extern int totemip_compare(const void *a, const void *b); extern int totemip_is_mcast(struct totem_ip_address *addr); extern void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2); extern void totemip_copy_endian_convert(struct totem_ip_address *addr1, const struct totem_ip_address *addr2); int totemip_localhost(int family, struct totem_ip_address *localhost); extern int totemip_localhost_check(const struct totem_ip_address *addr); extern const char *totemip_print(const struct totem_ip_address *addr); extern const char *totemip_sa_print(const struct sockaddr *sa); extern int totemip_sockaddr_to_totemip_convert(const struct sockaddr_storage *saddr, struct totem_ip_address *ip_addr); extern int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen); extern int totemip_parse(struct totem_ip_address *totemip, const char *addr, - int family); + enum totem_ip_version_enum ip_version); extern int totemip_iface_check(struct totem_ip_address *bindnet, struct totem_ip_address *boundto, int *interface_up, int *interface_num, int mask_high_bit); extern int totemip_getifaddrs(struct qb_list_head *addrs); extern void totemip_freeifaddrs(struct qb_list_head *addrs); /* These two simulate a zero in_addr by clearing the family field */ static inline void totemip_zero_set(struct totem_ip_address *addr) { addr->family = 0; } static inline int totemip_zero_check(const struct totem_ip_address *addr) { return (addr->family == 0); } extern size_t totemip_udpip_header_size(int family); #ifdef __cplusplus } #endif #endif diff --git a/man/corosync.conf.5 b/man/corosync.conf.5 index 2b5f743e..1b8b7cdc 100644 --- a/man/corosync.conf.5 +++ b/man/corosync.conf.5 @@ -1,898 +1,908 @@ .\"/* .\" * Copyright (c) 2005 MontaVista Software, Inc. .\" * Copyright (c) 2006-2018 Red Hat, Inc. .\" * .\" * All rights reserved. .\" * .\" * Author: Steven Dake (sdake@redhat.com) .\" * .\" * This software licensed under BSD license, the text of which follows: .\" * .\" * Redistribution and use in source and binary forms, with or without .\" * modification, are permitted provided that the following conditions are met: .\" * .\" * - Redistributions of source code must retain the above copyright notice, .\" * this list of conditions and the following disclaimer. .\" * - Redistributions in binary form must reproduce the above copyright notice, .\" * this list of conditions and the following disclaimer in the documentation .\" * and/or other materials provided with the distribution. .\" * - Neither the name of the MontaVista Software, Inc. nor the names of its .\" * contributors may be used to endorse or promote products derived from this .\" * software without specific prior written permission. .\" * .\" * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" .\" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE .\" * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR .\" * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF .\" * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS .\" * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN .\" * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) .\" * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF .\" * THE POSSIBILITY OF SUCH DAMAGE. .\" */ .TH COROSYNC_CONF 5 2018-11-13 "corosync Man Page" "Corosync Cluster Engine Programmer's Manual" .SH NAME corosync.conf - corosync executive configuration file .SH SYNOPSIS /etc/corosync/corosync.conf .SH DESCRIPTION The corosync.conf instructs the corosync executive about various parameters needed to control the corosync executive. Empty lines and lines starting with # character are ignored. The configuration file consists of bracketed top level directives. The possible directive choices are: .TP totem { } This top level directive contains configuration options for the totem protocol. .TP logging { } This top level directive contains configuration options for logging. .TP quorum { } This top level directive contains configuration options for quorum. .TP nodelist { } This top level directive contains configuration options for nodes in cluster. .TP system { } This top level directive contains configuration options related to system. .TP resources { } This top level directive contains configuration options for resources. .PP The .B interface sub-directive of totem is optional for UDP and knet transports. For knet, multiple interface subsections define parameters for each knet link on the system. For UDPU an interface section is not needed and it is recommended that the nodelist is used to define cluster nodes. .TP linknumber This specifies the link number for the interface. When using the knet protocol, each interface should specify separate link numbers to uniquely identify to the membership protocol which interface to use for which link. The linknumber must start at 0. For UDP the only supported linknumber is 0. .TP knet_link_priority This specifies the priority for the link when knet is used in 'passive' mode. (see link_mode below) .TP knet_ping_interval This specifies the interval between knet link pings. knet_ping_interval and knet_ping_timeout are a pair, if one is specified the other should be too, otherwise one will be calculated from the token timeout and one will be taken from the config file. (default is token timeout / (knet_pong_count*2)) .TP knet_ping_timeout If no ping is received within this time, the knet link is declared dead. knet_ping_interval and knet_ping_timeout are a pair, if one is specified the other should be too, otherwise one will be calculated from the token timeout and one will be taken from the config file. (default is token timeout / knet_pong_count) .TP knet_ping_precision How many values of latency are used to calculate the average link latency. (default 2048 samples) .TP knet_pong_count How many valid ping/pongs before a link is marked UP. (default 5) .TP knet_transport Which IP transport knet should use. valid values are "sctp" or "udp". (default: udp) .TP bindnetaddr (udp only) This specifies the network address the corosync executive should bind to when using udp. bindnetaddr (udp only) should be an IP address configured on the system, or a network address. For example, if the local interface is 192.168.5.92 with netmask 255.255.255.0, you should set bindnetaddr to 192.168.5.92 or 192.168.5.0. If the local interface is 192.168.5.92 with netmask 255.255.255.192, set bindnetaddr to 192.168.5.92 or 192.168.5.64, and so forth. This may also be an IPV6 address, in which case IPV6 networking will be used. In this case, the exact address must be specified and there is no automatic selection of the network interface within a specific subnet as with IPv4. If IPv6 networking is used, the nodeid field in nodelist must be specified. .TP broadcast (udp only) This is optional and can be set to yes. If it is set to yes, the broadcast address will be used for communication. If this option is set, mcastaddr should not be set. .TP mcastaddr (udp only) This is the multicast address used by corosync executive. The default should work for most networks, but the network administrator should be queried about a multicast address to use. Avoid 224.x.x.x because this is a "config" multicast address. This may also be an IPV6 multicast address, in which case IPV6 networking will be used. If IPv6 networking is used, the nodeid field in nodelist must be specified. It's not necessary to use this option if cluster_name option is used. If both options are used, mcastaddr has higher priority. .TP mcastport (udp only) This specifies the UDP port number. It is possible to use the same multicast address on a network with the corosync services configured for different UDP ports. Please note corosync uses two UDP ports mcastport (for mcast receives) and mcastport - 1 (for mcast sends). If you have multiple clusters on the same network using the same mcastaddr please configure the mcastports with a gap. .TP ttl (udp only) This specifies the Time To Live (TTL). If you run your cluster on a routed network then the default of "1" will be too small. This option provides a way to increase this up to 255. The valid range is 0..255. .PP .PP Within the .B totem directive, there are seven configuration options of which one is required, five are optional, and one is required when IPV6 is configured in the interface subdirective. The required directive controls the version of the totem configuration. The optional option unless using IPV6 directive controls identification of the processor. The optional options control secrecy and authentication, the network mode of operation and maximum network MTU field. .TP version This specifies the version of the configuration file. Currently the only valid version for this directive is 2. .PP clear_node_high_bit This configuration option is optional and is only relevant when no nodeid is specified. Some corosync clients require a signed 32 bit nodeid that is greater than zero however by default corosync uses all 32 bits of the IPv4 address space when generating a nodeid. Set this option to yes to force the high bit to be zero and therefore ensure the nodeid is a positive signed 32 bit integer. WARNING: Cluster behavior is undefined if this option is enabled on only a subset of the cluster (for example during a rolling upgrade). .TP crypto_model This specifies which cryptographic library should be used by knet. Options are nss and openssl. The default is nss. .TP crypto_hash This specifies which HMAC authentication should be used to authenticate all messages. Valid values are none (no authentication), md5, sha1, sha256, sha384 and sha512. Encrypted transmission is only supported for the knet transport. The default is none. .TP crypto_cipher This specifies which cipher should be used to encrypt all messages. Valid values are none (no encryption), aes256, aes192, aes128 and 3des. Enabling crypto_cipher, requires also enabling of crypto_hash. Encrypted transmission is only supported for the knet transport. The default is none. .TP keyfile This specifies the fully qualified path to the shared key used to authenticate and encrypt data used within the Totem protocol. The default is /etc/corosync/authkey. .TP key Shared key stored in configuration instead of authkey file. This option has lower precedence than keyfile option so it's used only when keyfile is not specified. Using this option is not recommended for security reasons. .TP link_mode This specifies the Kronosnet mode, which may be passive, active, or rr (round-robin). .B passive: the active link with the lowest priority will be used. If one or more links share the same priority the one with the lowest link ID will be used. .B active: All active links will be used simultaneously to send traffic. link priority is ignored. .B rr: Round-Robin policy. Each packet will be sent to the next active link in order. If only one interface directive is specified, passive is automatically chosen. The maximum number of interface directives that is allowed with Kronosnet is 8. For other transports it is 1. .TP netmtu This specifies the network maximum transmit unit. To set this value beyond 1500, the regular frame MTU, requires ethernet devices that support large, or also called jumbo, frames. If any device in the network doesn't support large frames, the protocol will not operate properly. The hosts must also have their mtu size set from 1500 to whatever frame size is specified here. Please note while some NICs or switches claim large frame support, they support 9000 MTU as the maximum frame size including the IP header. Setting the netmtu and host MTUs to 9000 will cause totem to use the full 9000 bytes of the frame. Then Linux will add a 18 byte header moving the full frame size to 9018. As a result some hardware will not operate properly with this size of data. A netmtu of 8982 seems to work for the few large frame devices that have been tested. Some manufacturers claim large frame support when in fact they support frame sizes of 4500 bytes. When sending multicast traffic, if the network frequently reconfigures, chances are that some device in the network doesn't support large frames. Choose hardware carefully if intending to use large frame support. The default is 1500. .TP transport This directive controls the transport mechanism used. The default is knet. The transport type can also be set to udpu or udp. Only knet allows crypto or multiple interfaces per node. .TP cluster_name This specifies the name of cluster and it's used for automatic generating of multicast address. .TP config_version This specifies version of config file. This is converted to unsigned 64-bit int. By default it's 0. Option is used to prevent joining old nodes with not up-to-date configuration. If value is not 0, and node is going for first time (only for first time, join after split doesn't follow this rules) from single-node membership to multiple nodes membership, other nodes config_versions are collected. If current node config_version is not equal to highest of collected versions, corosync is terminated. .TP ip_version -For udp or udpu, this specifies version of IP to use for communication. -The value can be one of ipv4 or ipv6. Default (if unspecified) is ipv4. -This does not apply to knet where both ipv4 and ipv6 address can be used, +This specifies version of IP to ask DNS resolver for. +The value can be one of +.B ipv4 +(look only for an IPv4 address) +, +.B ipv6 +(check only IPv6 address) +, +.B ipv4-6 +(first check IPv4 address, if that fails then look for an IPv6 address) and +.B ipv6-4 +(first check IPv6 address, if that fails then look for an IPv4 address). + +Default (if unspecified) is ipv6-4 for knet and udpu transports and ipv4 for udp. + +Knet transport allows to have a both ipv4 and ipv6 address, provided they are consistent on each link. Within the .B totem directive, there are several configuration options which are used to control the operation of the protocol. It is generally not recommended to change any of these values without proper guidance and sufficient testing. Some networks may require larger values if suffering from frequent reconfigurations. Some applications may require faster failure detection times which can be achieved by reducing the token timeout. .TP token This timeout is used directly or as a base for real token timeout calculation (explained in .B token_coefficient section). Token timeout specifies in milliseconds until a token loss is declared after not receiving a token. This is the time spent detecting a failure of a processor in the current configuration. Reforming a new configuration takes about 50 milliseconds in addition to this timeout. For real token timeout used by totem it's possible to read cmap value of .B runtime.config.totem.token key. The default is 1000 milliseconds. .TP token_warning Specifies the interval between warnings that the token has not been received. The value is a percentage of the token timeout and can be set to 0 to disable warnings. The default is 75%. .TP token_coefficient This value is used only when .B nodelist section is specified and contains at least 3 nodes. If so, real token timeout is then computed as token + (number_of_nodes - 2) * token_coefficient. This allows cluster to scale without manually changing token timeout every time new node is added. This value can be set to 0 resulting in effective removal of this feature. The default is 650 milliseconds. .TP token_retransmit This timeout specifies in milliseconds after how long before receiving a token the token is retransmitted. This will be automatically calculated if token is modified. It is not recommended to alter this value without guidance from the corosync community. The default is 238 milliseconds. .TP knet_compression_model The (optional) type of compression used by Kronosnet. The values available depend on the build and also avaialable libraries. Typically zlib and lz4 will be available but bzip2 and others could also be allowed. The default is 'none' .TP knet_compression_threshold Tells knet to NOT compress any packets that are smaller than the value indicated. Default 100 bytes. Set to 0 to reset to the default. Set to 1 to compress everything. .TP knet_compression_level Many compression libraries allow tuning of compression parameters. For example 0 or 1 ... 9 are commonly used to determine the level of compression. This value is passed unmodified to the compression library so it is recommended to consult the library's documentation for more detailed information. .TP hold This timeout specifies in milliseconds how long the token should be held by the representative when the protocol is under low utilization. It is not recommended to alter this value without guidance from the corosync community. The default is 180 milliseconds. .TP token_retransmits_before_loss_const This value identifies how many token retransmits should be attempted before forming a new configuration. If this value is set, retransmit and hold will be automatically calculated from retransmits_before_loss and token. The default is 4 retransmissions. .TP join This timeout specifies in milliseconds how long to wait for join messages in the membership protocol. The default is 50 milliseconds. .TP send_join This timeout specifies in milliseconds an upper range between 0 and send_join to wait before sending a join message. For configurations with less than 32 nodes, this parameter is not necessary. For larger rings, this parameter is necessary to ensure the NIC is not overflowed with join messages on formation of a new ring. A reasonable value for large rings (128 nodes) would be 80msec. Other timer values must also change if this value is changed. Seek advice from the corosync mailing list if trying to run larger configurations. The default is 0 milliseconds. .TP consensus This timeout specifies in milliseconds how long to wait for consensus to be achieved before starting a new round of membership configuration. The minimum value for consensus must be 1.2 * token. This value will be automatically calculated at 1.2 * token if the user doesn't specify a consensus value. For two node clusters, a consensus larger than the join timeout but less than token is safe. For three node or larger clusters, consensus should be larger than token. There is an increasing risk of odd membership changes, which still guarantee virtual synchrony, as node count grows if consensus is less than token. The default is 1200 milliseconds. .TP merge This timeout specifies in milliseconds how long to wait before checking for a partition when no multicast traffic is being sent. If multicast traffic is being sent, the merge detection happens automatically as a function of the protocol. The default is 200 milliseconds. .TP downcheck This timeout specifies in milliseconds how long to wait before checking that a network interface is back up after it has been downed. The default is 1000 milliseconds. .TP fail_recv_const This constant specifies how many rotations of the token without receiving any of the messages when messages should be received may occur before a new configuration is formed. The default is 2500 failures to receive a message. .TP seqno_unchanged_const This constant specifies how many rotations of the token without any multicast traffic should occur before the hold timer is started. The default is 30 rotations. .TP heartbeat_failures_allowed [HeartBeating mechanism] Configures the optional HeartBeating mechanism for faster failure detection. Keep in mind that engaging this mechanism in lossy networks could cause faulty loss declaration as the mechanism relies on the network for heartbeating. So as a rule of thumb use this mechanism if you require improved failure in low to medium utilized networks. This constant specifies the number of heartbeat failures the system should tolerate before declaring heartbeat failure e.g 3. Also if this value is not set or is 0 then the heartbeat mechanism is not engaged in the system and token rotation is the method of failure detection The default is 0 (disabled). .TP max_network_delay [HeartBeating mechanism] This constant specifies in milliseconds the approximate delay that your network takes to transport one packet from one machine to another. This value is to be set by system engineers and please don't change if not sure as this effects the failure detection mechanism using heartbeat. The default is 50 milliseconds. .TP window_size This constant specifies the maximum number of messages that may be sent on one token rotation. If all processors perform equally well, this value could be large (300), which would introduce higher latency from origination to delivery for very large rings. To reduce latency in large rings(16+), the defaults are a safe compromise. If 1 or more slow processor(s) are present among fast processors, window_size should be no larger than 256000 / netmtu to avoid overflow of the kernel receive buffers. The user is notified of this by the display of a retransmit list in the notification logs. There is no loss of data, but performance is reduced when these errors occur. The default is 50 messages. .TP max_messages This constant specifies the maximum number of messages that may be sent by one processor on receipt of the token. The max_messages parameter is limited to 256000 / netmtu to prevent overflow of the kernel transmit buffers. The default is 17 messages. .TP miss_count_const This constant defines the maximum number of times on receipt of a token a message is checked for retransmission before a retransmission occurs. This parameter is useful to modify for switches that delay multicast packets compared to unicast packets. The default setting works well for nearly all modern switches. The default is 5 messages. .TP knet_pmtud_interval How often the knet PMTUd runs to look for network MTU changes. Value in seconds, default: 30 .PP Within the .B logging directive, there are several configuration options which are all optional. .PP The following 3 options are valid only for the top level logging directive: .TP timestamp This specifies that a timestamp is placed on all log messages. It can be one of off (no timestamp), on (second precision timestamp) or hires (millisecond precision timestamp - only when supported by LibQB). The default is hires (or on if hires is not supported). .TP fileline This specifies that file and line should be printed. The default is off. .TP function_name This specifies that the code function name should be printed. The default is off. .TP blackbox This specifies that blackbox functionality should be enabled. The default is on. .PP The following options are valid both for top level logging directive and they can be overridden in logger_subsys entries. .TP to_stderr .TP to_logfile .TP to_syslog These specify the destination of logging output. Any combination of these options may be specified. Valid options are .B yes and .B no. The default is syslog and stderr. Please note, if you are using to_logfile and want to rotate the file, use logrotate(8) with the option .B copytruncate. eg. .ne 18 .RS .nf .ft CW /var/log/corosync.log { missingok compress notifempty daily rotate 7 copytruncate } .ft .fi .RE .TP logfile If the .B to_logfile directive is set to .B yes , this option specifies the pathname of the log file. No default. .TP logfile_priority This specifies the logfile priority for this particular subsystem. Ignored if debug is on. Possible values are: alert, crit, debug (same as debug = on), emerg, err, info, notice, warning. The default is: info. .TP syslog_facility This specifies the syslog facility type that will be used for any messages sent to syslog. options are daemon, local0, local1, local2, local3, local4, local5, local6 & local7. The default is daemon. .TP syslog_priority This specifies the syslog level for this particular subsystem. Ignored if debug is on. Possible values are: alert, crit, debug (same as debug = on), emerg, err, info, notice, warning. The default is: info. .TP debug This specifies whether debug output is logged for this particular logger. Also can contain value trace, what is highest level of debug information. The default is off. .PP Within the .B logging directive, logger_subsys directives are optional. .PP Within the .B logger_subsys sub-directive, all of the above logging configuration options are valid and can be used to override the default settings. The subsys entry, described below, is mandatory to identify the subsystem. .TP subsys This specifies the subsystem identity (name) for which logging is specified. This is the name used by a service in the log_init() call. E.g. 'CPG'. This directive is required. .PP Within the .B quorum directive it is possible to specify the quorum algorithm to use with the .TP provider directive. At the time of writing only corosync_votequorum is supported. See votequorum(5) for configuration options. .PP Within the .B nodelist directive it is possible to specify specific information about nodes in cluster. Directive can contain only .B node sub-directive, which specifies every node that should be a member of the membership, and where non-default options are needed. Every node must have at least ring0_addr field filled. Every node that should be a member of the membership must be specified. Possible options are: .TP ringX_addr This specifies IP or network hostname address of the particular node. X is a link number. .TP nodeid This configuration option is required for each node for Kronosnet mode. It is a 32 bit value specifying the node identifier delivered to the cluster membership service. The node identifier value of zero is reserved and should not be used. If knet is set, this field must be set. .TP name This option is used mainly with knet transport to identify local node. It's also used by client software (pacemaker). Algorithm for identifying local node is following: .RS .IP 1. Looks up $HOSTNAME in the nodelist .IP 2. If this fails strip the domain name from $HOSTNAME and looks up that in the nodelist .IP 3. If this fails look in the nodelist for a fully-qualified name whose short version matches the short version of $HOSTNAME .IP 4. If all this fails then search the interfaces list for an address that matches a name in the nodelist .RE .PP Within the .B system directive it is possible to specify system options. Possible options are: .TP qb_ipc_type This specifies type of IPC to use. Can be one of native (default), shm and socket. Native means one of shm or socket, depending on what is supported by OS. On systems with support for both, SHM is selected. SHM is generally faster, but need to allocate ring buffer file in /dev/shm. .TP sched_rr Should be set to yes (default) if corosync should try to set round robin realtime scheduling with maximal priority to itself. When setting of scheduler fails, fallback to set maximal priority. .TP priority Set priority of corosync process. Valid only when sched_rr is set to no. Can be ether numeric value with similar meaning as .BR nice (1) or .B max / .B min meaning maximal / minimal priority (so minimal / maximal nice value). .TP move_to_root_cgroup Should be set to yes (default) if corosync should try to move itself to root cgroup. This feature is available only for systems with cgroups with RT sched enabled (Linux with CONFIG_RT_GROUP_SCHED kernel option). .TP run_dir Existing directory where corosync should chdir into. Corosync stores important state files and blackboxes there. The default is /var/lib/corosync. .PP Within the .B resources directive it is possible to specify options for resources. Possible option is: .TP watchdog_device (Valid only if Corosync was compiled with watchdog support.) .br Watchdog device to use, for example /dev/watchdog. If unset, empty or "off", no watchdog is used. .IP In a cluster with properly configured power fencing a watchdog provides no additional value. On the other hand, slow watchdog communication may incur multi-second delays in the Corosync main loop, potentially breaking down membership. IPMI watchdogs are particularly notorious in this regard: read about kipmid_max_busy_us in IPMI.txt in the Linux kernel documentation. .SH "TO ADD A NEW NODE TO THE CLUSTER" For example to add a node with address 10.24.38.108 with nodeid 3. The node has the name NEW (in DNS or /etc/hosts) and is not currently running corosync. The current corosync.conf nodelist looks like this: .PP .nf .RS nodelist { node { nodeid: 1 ring0_addr: 10.24.38.101 name: node1 } node { nodeid: 2 ring0_addr: 10.24.38.102 name: node2 } } .RE .fi .PP Add a new entry for the node below the existing nodes. Node entries don't have to be in nodeid order, but it will help keep you sane. So the nodelist now looks like this: .PP .nf .RS nodelist { node { nodeid: 1 ring0_addr: 10.24.38.101 name: node1 } node { nodeid: 2 ring0_addr: 10.24.38.102 name: node2 } node { nodeid: 3 ring0_addr: 10.24.38.108 name: NEW } } .RE .fi .PP .PP This file must then be copied onto all three nodes - the existing two nodes, and the new one. On one of the existing corosync nodes, tell corosync to re-read the updated config file into memory: .PP .nf .RS corosync-cfgtool -R .RE .fi .PP This command only needs to be run on one node in the cluster. You may then start corosync on the NEW node and it should join the cluster. If this doesn't work as expected then check the communications between all three nodes is working, and check the syslog files on all nodes for more information. It's important to note that the key bit of information about a node failing to join might be on a different node than you expect. .SH "TO REMOVE A NODE FROM THE CLUSTER" This is the reverse procedure to 'Adding a node' above. First you need to shut down the node you will be removing from the cluster. .PP .nf .RS corosync-cfgtool -H .RE .fi .PP Then delete the nodelist stanza from corosync.conf and finally update corosync on the remaining nodes by running .PP .nf .RS corosync-cfgtool -R .RE .fi .TP on one of them. .SH "ADDRESS RESOLUTION" -corosync resolves ringX_addr names/IP addresses using the getaddrinfo(3) call in two passes. -First it will check for an IPv6 address, if that fails then it will look for an IPv4 address. +corosync resolves ringX_addr names/IP addresses using the getaddrinfo(3) call with respect +of totem.ip_version setting. getaddrinfo() function uses a sophisticated algorithm to sort node addresses into a preferred order and corosync always chooses the first address in that list of the required family. As such it is essential that your DNS or /etc/hosts files are correctly configured so that all addresses for ringX appear on the same network (or are reachable with minimal hops) and over the same IP protocol. If this is not the case then some nodes might not be able to join the cluster. It is possible to override the search order used by getaddrinfo() using the configuration file /etc/gai.conf(5) if necessary, but this is not recommended. If there is any doubt about the order of addresses returned from getaddrinfo() then it might be simpler to use IP addresses (v4 or v6) in the ringX_addr field. -For UDP/UDPU it is possible to force the IP family using totem.ip_version. This is ignored for knet as links -can have different IP versions. - .SH "FILES" .TP /etc/corosync/corosync.conf The corosync executive configuration file. .SH "SEE ALSO" .BR corosync_overview (7), .BR votequorum (5), .BR corosync-qdevice (8), .BR logrotate (8) .BR getaddrinfo (3) .BR gai.conf (5) .PP