diff --git a/exec/coroparse.c b/exec/coroparse.c index dfea609a..0bea5cc6 100644 --- a/exec/coroparse.c +++ b/exec/coroparse.c @@ -1,1885 +1,1887 @@ /* * Copyright (c) 2006-2022 Red Hat, Inc. * * All rights reserved. * * Author: Patrick Caulfield (pcaulfie@redhat.com) * Jan Friesse (jfriesse@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the MontaVista Software, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define LOGSYS_UTILS_ONLY 1 #include #include #include "main.h" #include "util.h" enum parser_cb_type { PARSER_CB_START, PARSER_CB_END, PARSER_CB_SECTION_START, PARSER_CB_SECTION_END, PARSER_CB_ITEM, PARSER_CB_CLEANUP, }; enum main_cp_cb_data_state { MAIN_CP_CB_DATA_STATE_NORMAL, MAIN_CP_CB_DATA_STATE_TOTEM, MAIN_CP_CB_DATA_STATE_INTERFACE, MAIN_CP_CB_DATA_STATE_LOGGER_SUBSYS, MAIN_CP_CB_DATA_STATE_UIDGID, MAIN_CP_CB_DATA_STATE_LOGGING_DAEMON, MAIN_CP_CB_DATA_STATE_MEMBER, MAIN_CP_CB_DATA_STATE_QUORUM, MAIN_CP_CB_DATA_STATE_QDEVICE, MAIN_CP_CB_DATA_STATE_NODELIST, MAIN_CP_CB_DATA_STATE_NODELIST_NODE, MAIN_CP_CB_DATA_STATE_PLOAD, MAIN_CP_CB_DATA_STATE_SYSTEM, MAIN_CP_CB_DATA_STATE_RESOURCES, MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM, MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS, MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM_MEMUSED, MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS_MEMUSED }; typedef int (*parser_cb_f)(const char *path, char *key, char *value, enum main_cp_cb_data_state *state, enum parser_cb_type type, const char **error_string, icmap_map_t config_map, void *user_data); struct key_value_list_item { char *key; char *value; struct qb_list_head list; }; struct main_cp_cb_data { int linknumber; char *bindnetaddr; char *mcastaddr; char *broadcast; int mcastport; int ttl; int knet_link_priority; int knet_ping_interval; int knet_ping_timeout; int knet_ping_precision; int knet_pong_count; int knet_pmtud_interval; unsigned int knet_mtu; char *knet_transport; struct qb_list_head logger_subsys_items_head; char *subsys; char *logging_daemon_name; struct qb_list_head member_items_head; int node_number; }; static int read_config_file_into_icmap( const char **error_string, icmap_map_t config_map); static char error_string_response[512]; static int uid_determine (const char *req_user) { int pw_uid = 0; struct passwd passwd; struct passwd* pwdptr = &passwd; struct passwd* temp_pwd_pt; char *pwdbuffer; int pwdlinelen, rc; long int id; char *ep; id = strtol(req_user, &ep, 10); if (*req_user != '\0' && *ep == '\0' && id >= 0 && id <= UINT_MAX) { return (id); } pwdlinelen = sysconf (_SC_GETPW_R_SIZE_MAX); if (pwdlinelen == -1) { pwdlinelen = 256; } pwdbuffer = malloc (pwdlinelen); while ((rc = getpwnam_r (req_user, pwdptr, pwdbuffer, pwdlinelen, &temp_pwd_pt)) == ERANGE) { char *n; pwdlinelen *= 2; if (pwdlinelen <= 32678) { n = realloc (pwdbuffer, pwdlinelen); if (n != NULL) { pwdbuffer = n; continue; } } } if (rc != 0) { free (pwdbuffer); sprintf (error_string_response, "getpwnam_r(): %s", strerror(rc)); return (-1); } if (temp_pwd_pt == NULL) { free (pwdbuffer); sprintf (error_string_response, "The '%s' user is not found in /etc/passwd, please read the documentation.", req_user); return (-1); } pw_uid = passwd.pw_uid; free (pwdbuffer); return pw_uid; } static int gid_determine (const char *req_group) { int corosync_gid = 0; struct group group; struct group * grpptr = &group; struct group * temp_grp_pt; char *grpbuffer; int grplinelen, rc; long int id; char *ep; id = strtol(req_group, &ep, 10); if (*req_group != '\0' && *ep == '\0' && id >= 0 && id <= UINT_MAX) { return (id); } grplinelen = sysconf (_SC_GETGR_R_SIZE_MAX); if (grplinelen == -1) { grplinelen = 256; } grpbuffer = malloc (grplinelen); while ((rc = getgrnam_r (req_group, grpptr, grpbuffer, grplinelen, &temp_grp_pt)) == ERANGE) { char *n; grplinelen *= 2; if (grplinelen <= 32678) { n = realloc (grpbuffer, grplinelen); if (n != NULL) { grpbuffer = n; continue; } } } if (rc != 0) { free (grpbuffer); sprintf (error_string_response, "getgrnam_r(): %s", strerror(rc)); return (-1); } if (temp_grp_pt == NULL) { free (grpbuffer); sprintf (error_string_response, "The '%s' group is not found in /etc/group, please read the documentation.", req_group); return (-1); } corosync_gid = group.gr_gid; free (grpbuffer); return corosync_gid; } static char *strchr_rs (const char *haystack, int byte) { const char *end_address = strchr (haystack, byte); if (end_address) { end_address += 1; /* skip past { or = */ while (*end_address == ' ' || *end_address == '\t' || (unsigned char)*end_address == 0xA0) end_address++; } return ((char *) end_address); } int coroparse_configparse (icmap_map_t config_map, const char **error_string) { if (read_config_file_into_icmap(error_string, config_map)) { return -1; } return 0; } static char *remove_whitespace(char *string, int remove_colon_and_brace) { char *start; char *end; start = string; if (*start == '\0') return start; while (*start == ' ' || *start == '\t' || (unsigned char)*start == 0xA0) start++; end = start+(strlen(start))-1; while ((*end == ' ' || *end == '\t' || (unsigned char)*end == 0xA0 || (remove_colon_and_brace && (*end == ':' || *end == '{'))) && end > start) end--; if (*end != '\0') *(end + 1) = '\0'; return start; } static int parse_section(FILE *fp, const char *fname, int *line_no, const char *path, const char **error_string, int depth, enum main_cp_cb_data_state state, parser_cb_f parser_cb, icmap_map_t config_map, void *user_data) { char line[512]; int i; char *loc; int ignore_line; char new_keyname[ICMAP_KEYNAME_MAXLEN]; static char formated_err[384]; const char *tmp_error_string; if (strcmp(path, "") == 0) { parser_cb("", NULL, NULL, &state, PARSER_CB_START, error_string, config_map, user_data); } tmp_error_string = NULL; while (fgets (line, sizeof (line), fp)) { (*line_no)++; if (strlen(line) > 0) { /* * Check if complete line was read. Use feof to handle files * without ending \n at the end of the file */ if ((line[strlen(line) - 1] != '\n') && !feof(fp)) { tmp_error_string = "Line too long"; goto parse_error; } if (line[strlen(line) - 1] == '\n') line[strlen(line) - 1] = '\0'; if (strlen (line) > 0 && line[strlen(line) - 1] == '\r') line[strlen(line) - 1] = '\0'; } /* * Clear out white space and tabs */ for (i = strlen (line) - 1; i > -1; i--) { if (line[i] == '\t' || line[i] == ' ' || (unsigned char)line[i] == 0xA0) { line[i] = '\0'; } else { break; } } ignore_line = 1; for (i = 0; i < strlen (line); i++) { if (line[i] != '\t' && line[i] != ' ' && (unsigned char)line[i] != 0xA0) { if (line[i] != '#') ignore_line = 0; break; } } /* * Clear out comments and empty lines */ if (ignore_line) { continue; } /* New section ? */ if ((loc = strchr_rs (line, '{'))) { char *section; char *after_section; enum main_cp_cb_data_state newstate; *(loc-1) = '\0'; section = remove_whitespace(line, 1); after_section = remove_whitespace(loc, 0); if (strcmp(section, "") == 0) { tmp_error_string = "Missing section name before opening bracket '{'"; goto parse_error; } if (strcmp(after_section, "") != 0) { tmp_error_string = "Extra characters after opening bracket '{'"; goto parse_error; } if (strlen(path) + strlen(section) + 1 >= ICMAP_KEYNAME_MAXLEN) { tmp_error_string = "Start of section makes total cmap path too long"; goto parse_error; } strcpy(new_keyname, path); if (strcmp(path, "") != 0) { strcat(new_keyname, "."); } strcat(new_keyname, section); /* Only use the new state for items further down the stack */ newstate = state; if (!parser_cb(new_keyname, NULL, NULL, &newstate, PARSER_CB_SECTION_START, &tmp_error_string, config_map, user_data)) { goto parse_error; } if (parse_section(fp, fname, line_no, new_keyname, error_string, depth + 1, newstate, parser_cb, config_map, user_data)) return -1; continue ; } /* New key/value */ if ((loc = strchr_rs (line, ':'))) { char *key; char *value; *(loc-1) = '\0'; key = remove_whitespace(line, 1); value = remove_whitespace(loc, 0); if (strlen(key) == 0) { tmp_error_string = "Key name can't be empty"; goto parse_error; } if (strlen(path) + strlen(key) + 1 >= ICMAP_KEYNAME_MAXLEN) { tmp_error_string = "New key makes total cmap path too long"; goto parse_error; } strcpy(new_keyname, path); if (strcmp(path, "") != 0) { strcat(new_keyname, "."); } strcat(new_keyname, key); if (!parser_cb(new_keyname, key, value, &state, PARSER_CB_ITEM, &tmp_error_string, config_map, user_data)) { goto parse_error; } continue ; } if (strchr_rs (line, '}')) { char *trimmed_line; trimmed_line = remove_whitespace(line, 0); if (strcmp(trimmed_line, "}") != 0) { tmp_error_string = "Extra characters before or after closing bracket '}'"; goto parse_error; } if (depth == 0) { tmp_error_string = "Unexpected closing brace"; goto parse_error; } if (!parser_cb(path, NULL, NULL, &state, PARSER_CB_SECTION_END, &tmp_error_string, config_map, user_data)) { goto parse_error; } return 0; } /* * Line is not opening section, ending section or value -> error */ tmp_error_string = "Line is not opening or closing section or key value"; goto parse_error; } if (strcmp(path, "") != 0) { tmp_error_string = "Missing closing brace"; goto parse_error; } if (strcmp(path, "") == 0) { parser_cb("", NULL, NULL, &state, PARSER_CB_END, error_string, config_map, user_data); parser_cb("", NULL, NULL, &state, PARSER_CB_CLEANUP, error_string, config_map, user_data); } return 0; parse_error: if (snprintf(formated_err, sizeof(formated_err), "parser error: %s:%u: %s", fname, *line_no, tmp_error_string) >= sizeof(formated_err)) { *error_string = "Can't format parser error message"; } else { *error_string = formated_err; } parser_cb("", NULL, NULL, &state, PARSER_CB_CLEANUP, error_string, config_map, user_data); return -1; } static int safe_atoq_range(icmap_value_types_t value_type, long long int *min_val, long long int *max_val) { switch (value_type) { case ICMAP_VALUETYPE_INT8: *min_val = INT8_MIN; *max_val = INT8_MAX; break; case ICMAP_VALUETYPE_UINT8: *min_val = 0; *max_val = UINT8_MAX; break; case ICMAP_VALUETYPE_INT16: *min_val = INT16_MIN; *max_val = INT16_MAX; break; case ICMAP_VALUETYPE_UINT16: *min_val = 0; *max_val = UINT16_MAX; break; case ICMAP_VALUETYPE_INT32: *min_val = INT32_MIN; *max_val = INT32_MAX; break; case ICMAP_VALUETYPE_UINT32: *min_val = 0; *max_val = UINT32_MAX; break; default: return (-1); } return (0); } /* * Convert string str to long long int res. Type of result is target_type and currently only * ICMAP_VALUETYPE_[U]INT[8|16|32] is supported. * Return 0 on success, -1 on failure. */ static int safe_atoq(const char *str, long long int *res, icmap_value_types_t target_type) { long long int val; long long int min_val, max_val; char *endptr; errno = 0; val = strtoll(str, &endptr, 10); if (errno == ERANGE) { return (-1); } if (endptr == str) { return (-1); } if (*endptr != '\0') { return (-1); } if (safe_atoq_range(target_type, &min_val, &max_val) != 0) { return (-1); } if (val < min_val || val > max_val) { return (-1); } *res = val; return (0); } static int str_to_ull(const char *str, unsigned long long int *res) { unsigned long long int val; char *endptr; errno = 0; val = strtoull(str, &endptr, 10); if (errno == ERANGE) { return (-1); } if (endptr == str) { return (-1); } if (*endptr != '\0') { return (-1); } *res = val; return (0); } static int get_dscp_value(char *str, int *dscp) { struct dscp_name {const char *name; int dscp;} names[] = { {"cs0", 0}, {"cs1", 8}, {"cs2", 16}, {"cs3", 24}, {"cs4", 32}, {"cs5", 40}, {"cs6", 48}, {"cs7", 56}, {"af11", 10}, {"af12", 12}, {"af13", 14}, {"af21", 18}, {"af22", 20}, {"af23", 22}, {"af31", 26}, {"af32", 28}, {"af33", 30}, {"af41", 34}, {"af42", 36}, {"af43", 38}, {"ef", 46}, {NULL, 0} }; struct dscp_name *n; long val; char *end; /* * allow dscp symbolical names according to * https://www.iana.org/assignments/dscp-registry/dscp-registry.xhtml */ for (n = names; n->name; n++) { if (strcmp(str, n->name) == 0) { *dscp = n->dscp; return 0; } } /* allow dscp as number (decimal, hex, octal) */ errno = 0; val = strtol(str, &end, 0); if (errno == 0 && *end == '\0' && val >= 0 && val <= 63) { *dscp = val; return 0; } return -1; } static int handle_crypto_model(const char *val, const char **error_string) { if (util_is_valid_knet_crypto_model(val, NULL, 0, "Invalid crypto model. Should be ", error_string) == 1) { return (0); } else { return (-1); } } static int handle_compress_model(const char *val, const char **error_string) { if (util_is_valid_knet_compress_model(val, NULL, 0, "Invalid compression model. Should be ", error_string) == 1) { return (0); } else { return (-1); } } static int main_config_parser_cb(const char *path, char *key, char *value, enum main_cp_cb_data_state *state, enum parser_cb_type type, const char **error_string, icmap_map_t config_map, void *user_data) { int ii; long long int val; long long int min_val, max_val; icmap_value_types_t val_type = ICMAP_VALUETYPE_BINARY; unsigned long long int ull; int add_as_string; char key_name[ICMAP_KEYNAME_MAXLEN + 1]; static char formated_err[256]; struct main_cp_cb_data *data = (struct main_cp_cb_data *)user_data; struct key_value_list_item *kv_item; struct qb_list_head *iter, *tmp_iter; int uid, gid, dscp; cs_error_t cs_err; const char *path_prefix; cs_err = CS_OK; /* * Formally this check is not needed because length is checked by parse_section */ if (strlen(path) >= sizeof(key_name)) { if (snprintf(formated_err, sizeof(formated_err), "Can't store path \"%s\" into key_name", path) >= sizeof(formated_err)) { *error_string = "Can't format path into key_name error message"; } else { *error_string = formated_err; } return (0); } /* * Key_name is used in atoi_error/icmap_set_error, but many of icmap_set* * are using path, so initialize key_name to valid value */ strncpy(key_name, path, sizeof(key_name) - 1); switch (type) { case PARSER_CB_START: memset(data, 0, sizeof(struct main_cp_cb_data)); qb_list_init(&data->logger_subsys_items_head); qb_list_init(&data->member_items_head); *state = MAIN_CP_CB_DATA_STATE_NORMAL; break; case PARSER_CB_END: break; case PARSER_CB_CLEANUP: free(data->bindnetaddr); free(data->mcastaddr); free(data->broadcast); free(data->knet_transport); qb_list_for_each_safe(iter, tmp_iter, &(data->logger_subsys_items_head)) { kv_item = qb_list_entry(iter, struct key_value_list_item, list); qb_list_del(&kv_item->list); free(kv_item->value); free(kv_item->key); free(kv_item); } free(data->subsys); free(data->logging_daemon_name); qb_list_for_each_safe(iter, tmp_iter, &(data->member_items_head)) { kv_item = qb_list_entry(iter, struct key_value_list_item, list); qb_list_del(&kv_item->list); free(kv_item->value); free(kv_item->key); free(kv_item); } break; case PARSER_CB_ITEM: add_as_string = 1; switch (*state) { case MAIN_CP_CB_DATA_STATE_NORMAL: break; case MAIN_CP_CB_DATA_STATE_PLOAD: if ((strcmp(path, "pload.count") == 0) || (strcmp(path, "pload.size") == 0)) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } if ((cs_err = icmap_set_uint32_r(config_map, path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } break; case MAIN_CP_CB_DATA_STATE_QUORUM: if ((strcmp(path, "quorum.expected_votes") == 0) || (strcmp(path, "quorum.votes") == 0) || (strcmp(path, "quorum.last_man_standing_window") == 0) || (strcmp(path, "quorum.leaving_timeout") == 0)) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } if ((cs_err = icmap_set_uint32_r(config_map, path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } if ((strcmp(path, "quorum.two_node") == 0) || (strcmp(path, "quorum.expected_votes_tracking") == 0) || (strcmp(path, "quorum.allow_downscale") == 0) || (strcmp(path, "quorum.wait_for_all") == 0) || (strcmp(path, "quorum.auto_tie_breaker") == 0) || - (strcmp(path, "quorum.last_man_standing") == 0)) { + (strcmp(path, "quorum.last_man_standing") == 0) || + (strcmp(path, "quorum.site_aware") == 0)) { val_type = ICMAP_VALUETYPE_UINT8; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } if ((cs_err = icmap_set_uint8_r(config_map, path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } break; case MAIN_CP_CB_DATA_STATE_QDEVICE: if ((strcmp(path, "quorum.device.timeout") == 0) || (strcmp(path, "quorum.device.sync_timeout") == 0) || (strcmp(path, "quorum.device.votes") == 0)) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } if ((cs_err = icmap_set_uint32_r(config_map, path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } if ((strcmp(path, "quorum.device.master_wins") == 0)) { val_type = ICMAP_VALUETYPE_UINT8; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } if ((cs_err = icmap_set_uint8_r(config_map, path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } break; case MAIN_CP_CB_DATA_STATE_TOTEM: if ((strcmp(path, "totem.version") == 0) || (strcmp(path, "totem.nodeid") == 0) || (strcmp(path, "totem.threads") == 0) || (strcmp(path, "totem.token") == 0) || (strcmp(path, "totem.token_coefficient") == 0) || (strcmp(path, "totem.token_retransmit") == 0) || (strcmp(path, "totem.token_warning") == 0) || (strcmp(path, "totem.hold") == 0) || (strcmp(path, "totem.token_retransmits_before_loss_const") == 0) || (strcmp(path, "totem.join") == 0) || (strcmp(path, "totem.send_join") == 0) || (strcmp(path, "totem.consensus") == 0) || (strcmp(path, "totem.merge") == 0) || (strcmp(path, "totem.downcheck") == 0) || (strcmp(path, "totem.fail_recv_const") == 0) || (strcmp(path, "totem.seqno_unchanged_const") == 0) || (strcmp(path, "totem.heartbeat_failures_allowed") == 0) || (strcmp(path, "totem.max_network_delay") == 0) || (strcmp(path, "totem.window_size") == 0) || (strcmp(path, "totem.max_messages") == 0) || (strcmp(path, "totem.miss_count_const") == 0) || (strcmp(path, "totem.knet_pmtud_interval") == 0) || (strcmp(path, "totem.knet_mtu") == 0) || (strcmp(path, "totem.knet_compression_threshold") == 0) || (strcmp(path, "totem.netmtu") == 0)) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } if ((cs_err = icmap_set_uint32_r(config_map,path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } if (strcmp(path, "totem.knet_compression_level") == 0) { val_type = ICMAP_VALUETYPE_INT32; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } if ((cs_err = icmap_set_int32_r(config_map, path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } if (strcmp(path, "totem.config_version") == 0) { if (str_to_ull(value, &ull) != 0) { goto str_to_ull_error; } if ((cs_err = icmap_set_uint64_r(config_map, path, ull)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } if (strcmp(path, "totem.ip_version") == 0) { if ((strcmp(value, "ipv4") != 0) && (strcmp(value, "ipv6") != 0) && (strcmp(value, "ipv6-4") != 0) && (strcmp(value, "ipv4-6") != 0)) { *error_string = "Invalid ip_version type"; return (0); } } if (strcmp(path, "totem.crypto_model") == 0) { if (handle_crypto_model(value, error_string) != 0) { return (0); } } if (strcmp(path, "totem.crypto_cipher") == 0) { if ((strcmp(value, "none") != 0) && (strcmp(value, "aes256") != 0) && (strcmp(value, "aes192") != 0) && (strcmp(value, "aes128") != 0)) { *error_string = "Invalid cipher type. " "Should be none, aes256, aes192 or aes128"; return (0); } } if (strcmp(path, "totem.crypto_hash") == 0) { if ((strcmp(value, "none") != 0) && (strcmp(value, "md5") != 0) && (strcmp(value, "sha1") != 0) && (strcmp(value, "sha256") != 0) && (strcmp(value, "sha384") != 0) && (strcmp(value, "sha512") != 0)) { *error_string = "Invalid hash type. " "Should be none, md5, sha1, sha256, sha384 or sha512"; return (0); } } if (strcmp(path, "totem.knet_compression_model") == 0) { if (handle_compress_model(value, error_string) != 0) { return (0); } } if (strcmp(path, "totem.ip_dscp") == 0) { if (get_dscp_value(value, &dscp) != 0) { goto str_to_dscp_error; } if ((cs_err = icmap_set_uint8_r(config_map, path, dscp)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } break; case MAIN_CP_CB_DATA_STATE_SYSTEM: if (strcmp(path, "system.qb_ipc_type") == 0) { if ((strcmp(value, "native") != 0) && (strcmp(value, "shm") != 0) && (strcmp(value, "socket") != 0)) { *error_string = "Invalid system.qb_ipc_type"; return (0); } } if (strcmp(path, "system.sched_rr") == 0) { if ((strcmp(value, "yes") != 0) && (strcmp(value, "no") != 0)) { *error_string = "Invalid system.sched_rr value"; return (0); } } if (strcmp(path, "system.move_to_root_cgroup") == 0) { if ((strcmp(value, "yes") != 0) && (strcmp(value, "no") != 0) && (strcmp(value, "auto") != 0)) { *error_string = "Invalid system.move_to_root_cgroup"; return (0); } } if (strcmp(path, "system.allow_knet_handle_fallback") == 0) { if ((strcmp(value, "yes") != 0) && (strcmp(value, "no") != 0)) { *error_string = "Invalid system.allow_knet_handle_fallback"; return (0); } } break; case MAIN_CP_CB_DATA_STATE_INTERFACE: if (strcmp(path, "totem.interface.linknumber") == 0) { val_type = ICMAP_VALUETYPE_UINT8; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } data->linknumber = val; add_as_string = 0; } if (strcmp(path, "totem.interface.bindnetaddr") == 0) { free(data->bindnetaddr); data->bindnetaddr = strdup(value); add_as_string = 0; } if (strcmp(path, "totem.interface.mcastaddr") == 0) { free(data->mcastaddr); data->mcastaddr = strdup(value); add_as_string = 0; } if (strcmp(path, "totem.interface.broadcast") == 0) { free(data->broadcast); data->broadcast = strdup(value); add_as_string = 0; } if (strcmp(path, "totem.interface.mcastport") == 0) { val_type = ICMAP_VALUETYPE_UINT16; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } data->mcastport = val; add_as_string = 0; } if (strcmp(path, "totem.interface.ttl") == 0) { val_type = ICMAP_VALUETYPE_UINT8; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } data->ttl = val; add_as_string = 0; } if (strcmp(path, "totem.interface.knet_link_priority") == 0) { val_type = ICMAP_VALUETYPE_UINT8; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } data->knet_link_priority = val; add_as_string = 0; } if (strcmp(path, "totem.interface.knet_ping_interval") == 0) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } data->knet_ping_interval = val; add_as_string = 0; } if (strcmp(path, "totem.interface.knet_ping_timeout") == 0) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } data->knet_ping_timeout = val; add_as_string = 0; } if (strcmp(path, "totem.interface.knet_ping_precision") == 0) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } data->knet_ping_precision = val; add_as_string = 0; } if (strcmp(path, "totem.interface.knet_pong_count") == 0) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } data->knet_pong_count = val; add_as_string = 0; } if (strcmp(path, "totem.interface.knet_transport") == 0) { free(data->knet_transport); data->knet_transport = strdup(value); add_as_string = 0; } break; case MAIN_CP_CB_DATA_STATE_LOGGER_SUBSYS: if (strcmp(path, "logging.logger_subsys.subsys") == 0) { free(data->subsys); data->subsys = strdup(value); if (data->subsys == NULL) { *error_string = "Can't alloc memory"; return (0); } } else { path_prefix = "logging.logger_subsys."; if (strlen(path) < strlen(path_prefix) || strncmp(path, path_prefix, strlen(path_prefix)) != 0) { *error_string = "Internal error - incorrect path prefix for logger subsys state"; return (0); } kv_item = malloc(sizeof(*kv_item)); if (kv_item == NULL) { *error_string = "Can't alloc memory"; return (0); } memset(kv_item, 0, sizeof(*kv_item)); kv_item->key = strdup(path + strlen(path_prefix)); kv_item->value = strdup(value); if (kv_item->key == NULL || kv_item->value == NULL) { free(kv_item->key); free(kv_item->value); free(kv_item); *error_string = "Can't alloc memory"; return (0); } qb_list_init(&kv_item->list); qb_list_add(&kv_item->list, &data->logger_subsys_items_head); } add_as_string = 0; break; case MAIN_CP_CB_DATA_STATE_LOGGING_DAEMON: if (strcmp(path, "logging.logging_daemon.subsys") == 0) { free(data->subsys); data->subsys = strdup(value); if (data->subsys == NULL) { *error_string = "Can't alloc memory"; return (0); } } else if (strcmp(path, "logging.logging_daemon.name") == 0) { free(data->logging_daemon_name); data->logging_daemon_name = strdup(value); if (data->logging_daemon_name == NULL) { *error_string = "Can't alloc memory"; return (0); } } else { path_prefix = "logging.logging_daemon."; if (strlen(path) < strlen(path_prefix) || strncmp(path, path_prefix, strlen(path_prefix)) != 0) { *error_string = "Internal error - incorrect path prefix for logging daemon state"; return (0); } kv_item = malloc(sizeof(*kv_item)); if (kv_item == NULL) { *error_string = "Can't alloc memory"; return (0); } memset(kv_item, 0, sizeof(*kv_item)); kv_item->key = strdup(path + strlen(path_prefix)); kv_item->value = strdup(value); if (kv_item->key == NULL || kv_item->value == NULL) { free(kv_item->key); free(kv_item->value); free(kv_item); *error_string = "Can't alloc memory"; return (0); } qb_list_init(&kv_item->list); qb_list_add(&kv_item->list, &data->logger_subsys_items_head); } add_as_string = 0; break; case MAIN_CP_CB_DATA_STATE_UIDGID: if (strcmp(path, "uidgid.uid") == 0) { uid = uid_determine(value); if (uid == -1) { *error_string = error_string_response; return (0); } snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "uidgid.config.uid.%u", uid); if ((cs_err = icmap_set_uint8_r(config_map, key_name, 1)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } else if (strcmp(path, "uidgid.gid") == 0) { gid = gid_determine(value); if (gid == -1) { *error_string = error_string_response; return (0); } snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "uidgid.config.gid.%u", gid); if ((cs_err = icmap_set_uint8_r(config_map, key_name, 1)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } else { *error_string = "uidgid: Only uid and gid are allowed items"; return (0); } break; case MAIN_CP_CB_DATA_STATE_MEMBER: if (strcmp(path, "totem.interface.member.memberaddr") != 0) { *error_string = "Only memberaddr is allowed in member section"; return (0); } kv_item = malloc(sizeof(*kv_item)); if (kv_item == NULL) { *error_string = "Can't alloc memory"; return (0); } memset(kv_item, 0, sizeof(*kv_item)); kv_item->key = strdup(key); kv_item->value = strdup(value); if (kv_item->key == NULL || kv_item->value == NULL) { free(kv_item->key); free(kv_item->value); free(kv_item); *error_string = "Can't alloc memory"; return (0); } qb_list_init(&kv_item->list); qb_list_add(&kv_item->list, &data->member_items_head); add_as_string = 0; break; case MAIN_CP_CB_DATA_STATE_NODELIST: break; case MAIN_CP_CB_DATA_STATE_NODELIST_NODE: path_prefix = "nodelist.node."; if (strlen(path) < strlen(path_prefix) || strncmp(path, path_prefix, strlen(path_prefix)) != 0) { *error_string = "Internal error - incorrect path prefix for nodelist node state"; return (0); } snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.%s", data->node_number, path + strlen(path_prefix)); if ((strcmp(path, "nodelist.node.nodeid") == 0) || - (strcmp(path, "nodelist.node.quorum_votes") == 0)) { + (strcmp(path, "nodelist.node.quorum_votes") == 0) || + (strcmp(path, "nodelist.node.site") == 0)) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } if ((cs_err = icmap_set_uint32_r(config_map, key_name, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } if (add_as_string) { if ((cs_err = icmap_set_string_r(config_map, key_name, value)) != CS_OK) { goto icmap_set_error; }; add_as_string = 0; } break; case MAIN_CP_CB_DATA_STATE_RESOURCES: if (strcmp(key, "watchdog_timeout") == 0) { val_type = ICMAP_VALUETYPE_UINT32; if (safe_atoq(value, &val, val_type) != 0) { goto safe_atoq_error; } if ((cs_err = icmap_set_uint32_r(config_map,path, val)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } break; case MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM: case MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM_MEMUSED: if (strcmp(key, "poll_period") == 0) { if (str_to_ull(value, &ull) != 0) { goto str_to_ull_error; } if ((cs_err = icmap_set_uint64_r(config_map,path, ull)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } break; case MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS: case MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS_MEMUSED: if (strcmp(key, "poll_period") == 0) { if (str_to_ull(value, &ull) != 0) { goto str_to_ull_error; } if ((cs_err = icmap_set_uint64_r(config_map,path, ull)) != CS_OK) { goto icmap_set_error; } add_as_string = 0; } break; } if (add_as_string) { if ((cs_err = icmap_set_string_r(config_map, path, value)) != CS_OK) { goto icmap_set_error; } } break; case PARSER_CB_SECTION_START: if (strcmp(path, "totem.interface") == 0) { *state = MAIN_CP_CB_DATA_STATE_INTERFACE; data->linknumber = 0; data->mcastport = -1; data->ttl = -1; data->knet_link_priority = -1; data->knet_ping_interval = -1; data->knet_ping_timeout = -1; data->knet_ping_precision = -1; data->knet_pong_count = -1; data->knet_transport = NULL; qb_list_init(&data->member_items_head); }; if (strcmp(path, "totem") == 0) { *state = MAIN_CP_CB_DATA_STATE_TOTEM; }; if (strcmp(path, "system") == 0) { *state = MAIN_CP_CB_DATA_STATE_SYSTEM; } if (strcmp(path, "logging.logger_subsys") == 0) { *state = MAIN_CP_CB_DATA_STATE_LOGGER_SUBSYS; qb_list_init(&data->logger_subsys_items_head); data->subsys = NULL; } if (strcmp(path, "logging.logging_daemon") == 0) { *state = MAIN_CP_CB_DATA_STATE_LOGGING_DAEMON; qb_list_init(&data->logger_subsys_items_head); data->subsys = NULL; data->logging_daemon_name = NULL; } if (strcmp(path, "uidgid") == 0) { *state = MAIN_CP_CB_DATA_STATE_UIDGID; } if (strcmp(path, "totem.interface.member") == 0) { *state = MAIN_CP_CB_DATA_STATE_MEMBER; } if (strcmp(path, "quorum") == 0) { *state = MAIN_CP_CB_DATA_STATE_QUORUM; } if (strcmp(path, "quorum.device") == 0) { *state = MAIN_CP_CB_DATA_STATE_QDEVICE; } if (strcmp(path, "nodelist") == 0) { *state = MAIN_CP_CB_DATA_STATE_NODELIST; data->node_number = 0; } if (strcmp(path, "nodelist.node") == 0) { *state = MAIN_CP_CB_DATA_STATE_NODELIST_NODE; } if (strcmp(path, "resources") == 0) { *state = MAIN_CP_CB_DATA_STATE_RESOURCES; } if (strcmp(path, "resources.system") == 0) { *state = MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM; } if (strcmp(path, "resources.system.memory_used") == 0) { *state = MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM_MEMUSED; } if (strcmp(path, "resources.process") == 0) { *state = MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS; } if (strcmp(path, "resources.process.memory_used") == 0) { *state = MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS_MEMUSED; } if (*state == MAIN_CP_CB_DATA_STATE_UIDGID && strcmp(path, "uidgid") != 0) { *error_string = "Subsections are not allowed within uidgid section"; return (0); }; if (*state == MAIN_CP_CB_DATA_STATE_MEMBER && strcmp(path, "totem.interface.member") != 0) { *error_string = "Subsections are not allowed within totem.interface.member section"; return (0); }; break; case PARSER_CB_SECTION_END: switch (*state) { case MAIN_CP_CB_DATA_STATE_INTERFACE: if (strcmp(path, "totem.interface") != 0) { /* * Process only end of totem.interface section, not subsections */ break; } /* * Create new interface section */ if (data->bindnetaddr != NULL) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.bindnetaddr", data->linknumber); cs_err = icmap_set_string_r(config_map, key_name, data->bindnetaddr); free(data->bindnetaddr); data->bindnetaddr = NULL; if (cs_err != CS_OK) { goto icmap_set_error; } } if (data->mcastaddr != NULL) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastaddr", data->linknumber); cs_err = icmap_set_string_r(config_map, key_name, data->mcastaddr); free(data->mcastaddr); data->mcastaddr = NULL; if (cs_err != CS_OK) { goto icmap_set_error; } } if (data->broadcast != NULL) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.broadcast", data->linknumber); cs_err = icmap_set_string_r(config_map, key_name, data->broadcast); free(data->broadcast); data->broadcast = NULL; if (cs_err != CS_OK) { goto icmap_set_error; } } if (data->mcastport > -1) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.mcastport", data->linknumber); if ((cs_err = icmap_set_uint16_r(config_map, key_name, data->mcastport)) != CS_OK) { goto icmap_set_error; } } if (data->ttl > -1) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.ttl", data->linknumber); if ((cs_err = icmap_set_uint8_r(config_map, key_name, data->ttl)) != CS_OK) { goto icmap_set_error; } } if (data->knet_link_priority > -1) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_link_priority", data->linknumber); if ((cs_err = icmap_set_uint8_r(config_map, key_name, data->knet_link_priority)) != CS_OK) { goto icmap_set_error; } } if (data->knet_ping_interval > -1) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_interval", data->linknumber); if ((cs_err = icmap_set_uint32_r(config_map, key_name, data->knet_ping_interval)) != CS_OK) { goto icmap_set_error; } } if (data->knet_ping_timeout > -1) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_timeout", data->linknumber); if ((cs_err = icmap_set_uint32_r(config_map, key_name, data->knet_ping_timeout)) != CS_OK) { goto icmap_set_error; } } if (data->knet_ping_precision > -1) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_ping_precision", data->linknumber); if ((cs_err = icmap_set_uint32_r(config_map, key_name, data->knet_ping_precision)) != CS_OK) { goto icmap_set_error; } } if (data->knet_pong_count > -1) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_pong_count", data->linknumber); if ((cs_err = icmap_set_uint32_r(config_map, key_name, data->knet_pong_count)) != CS_OK) { goto icmap_set_error; } } if (data->knet_transport) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.knet_transport", data->linknumber); cs_err = icmap_set_string_r(config_map, key_name, data->knet_transport); free(data->knet_transport); data->knet_transport = NULL; if (cs_err != CS_OK) { goto icmap_set_error; } } ii = 0; qb_list_for_each_safe(iter, tmp_iter, &(data->member_items_head)) { kv_item = qb_list_entry(iter, struct key_value_list_item, list); qb_list_del(&kv_item->list); snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "totem.interface.%u.member.%u", data->linknumber, ii); cs_err = icmap_set_string_r(config_map, key_name, kv_item->value); free(kv_item->value); free(kv_item->key); free(kv_item); ii++; if (cs_err != CS_OK) { goto icmap_set_error; } } qb_list_init(&data->member_items_head); break; case MAIN_CP_CB_DATA_STATE_LOGGER_SUBSYS: if (strcmp(path, "logging.logger_subsys") != 0) { /* * Process only end of logging.logger_subsys section, not subsections */ break; } if (data->subsys == NULL) { *error_string = "No subsys key in logger_subsys directive"; return (0); } qb_list_for_each_safe(iter, tmp_iter, &(data->logger_subsys_items_head)) { kv_item = qb_list_entry(iter, struct key_value_list_item, list); qb_list_del(&kv_item->list); snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logger_subsys.%s.%s", data->subsys, kv_item->key); cs_err = icmap_set_string_r(config_map, key_name, kv_item->value); free(kv_item->value); free(kv_item->key); free(kv_item); if (cs_err != CS_OK) { goto icmap_set_error; } } snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logger_subsys.%s.subsys", data->subsys); cs_err = icmap_set_string_r(config_map, key_name, data->subsys); qb_list_init(&data->logger_subsys_items_head); free(data->subsys); data->subsys = NULL; if (cs_err != CS_OK) { goto icmap_set_error; } break; case MAIN_CP_CB_DATA_STATE_LOGGING_DAEMON: if (strcmp(path, "logging.logging_daemon") != 0) { /* * Process only end of logging.logging_daemon section, not subsections */ break; } if (data->logging_daemon_name == NULL) { *error_string = "No name key in logging_daemon directive"; return (0); } qb_list_for_each_safe(iter, tmp_iter, &(data->logger_subsys_items_head)) { kv_item = qb_list_entry(iter, struct key_value_list_item, list); qb_list_del(&kv_item->list); if (data->subsys == NULL) { if (strcmp(data->logging_daemon_name, "corosync") == 0) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.%s", kv_item->key); } else { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logging_daemon.%s.%s", data->logging_daemon_name, kv_item->key); } } else { if (strcmp(data->logging_daemon_name, "corosync") == 0) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logger_subsys.%s.%s", data->subsys, kv_item->key); } else { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logging_daemon.%s.%s.%s", data->logging_daemon_name, data->subsys, kv_item->key); } } cs_err = icmap_set_string_r(config_map, key_name, kv_item->value); free(kv_item->value); free(kv_item->key); free(kv_item); if (cs_err != CS_OK) { goto icmap_set_error; } } if (data->subsys == NULL) { if (strcmp(data->logging_daemon_name, "corosync") != 0) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logging_daemon.%s.name", data->logging_daemon_name); cs_err = icmap_set_string_r(config_map, key_name, data->logging_daemon_name); } } else { if (strcmp(data->logging_daemon_name, "corosync") == 0) { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logger_subsys.%s.subsys", data->subsys); cs_err = icmap_set_string_r(config_map, key_name, data->subsys); } else { snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logging_daemon.%s.%s.subsys", data->logging_daemon_name, data->subsys); cs_err = icmap_set_string_r(config_map, key_name, data->subsys); if (cs_err != CS_OK) { goto icmap_set_error; } snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "logging.logging_daemon.%s.%s.name", data->logging_daemon_name, data->subsys); cs_err = icmap_set_string_r(config_map, key_name, data->logging_daemon_name); } } qb_list_init(&data->logger_subsys_items_head); free(data->subsys); data->subsys = NULL; free(data->logging_daemon_name); data->logging_daemon_name = NULL; if (cs_err != CS_OK) { goto icmap_set_error; } break; case MAIN_CP_CB_DATA_STATE_NODELIST_NODE: if (strcmp(path, "nodelist.node") != 0) { /* * Process only end of nodelist.node section, not subsections */ break; } data->node_number++; break; case MAIN_CP_CB_DATA_STATE_NORMAL: case MAIN_CP_CB_DATA_STATE_PLOAD: case MAIN_CP_CB_DATA_STATE_UIDGID: case MAIN_CP_CB_DATA_STATE_MEMBER: case MAIN_CP_CB_DATA_STATE_QUORUM: case MAIN_CP_CB_DATA_STATE_QDEVICE: case MAIN_CP_CB_DATA_STATE_NODELIST: case MAIN_CP_CB_DATA_STATE_TOTEM: case MAIN_CP_CB_DATA_STATE_SYSTEM: case MAIN_CP_CB_DATA_STATE_RESOURCES: case MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM: case MAIN_CP_CB_DATA_STATE_RESOURCES_SYSTEM_MEMUSED: case MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS: case MAIN_CP_CB_DATA_STATE_RESOURCES_PROCESS_MEMUSED: break; } break; } return (1); safe_atoq_error: /* * For integers supported by safe_atoq display range */ min_val = max_val = 0; /* * This is really assert, because developer ether doesn't set val_type correctly or * we've got here after some nasty memory overwrite */ assert(safe_atoq_range(val_type, &min_val, &max_val) == 0); if (snprintf(formated_err, sizeof(formated_err), "Value of key \"%s\" is expected to be integer in range (%lld..%lld), but \"%s\" was given", key_name, min_val, max_val, value) >= sizeof(formated_err)) { *error_string = "Can't format parser error message"; } else { *error_string = formated_err; } return (0); str_to_ull_error: /* * For integers not supported by safe_atoq (64-bit int) */ if (snprintf(formated_err, sizeof(formated_err), "Value of key \"%s\" is expected to be unsigned integer, but \"%s\" was given", key_name, value) >= sizeof(formated_err)) { *error_string = "Can't format parser error message"; } else { *error_string = formated_err; } return (0); str_to_dscp_error: if (snprintf(formated_err, sizeof(formated_err), "Value of key \"%s\" is expected to be number (0..63) or symbolical dscp value, " "but \"%s\" was given", key_name, value) >= sizeof(formated_err)) { *error_string = "Can't format parser error message"; } else { *error_string = formated_err; } return (0); icmap_set_error: if (snprintf(formated_err, sizeof(formated_err), "Can't store key \"%s\" into icmap, returned error is %s", key_name, cs_strerror(cs_err)) >= sizeof(formated_err)) { *error_string = "Can't format parser error message"; } else { *error_string = formated_err; } return (0); } static int uidgid_config_parser_cb(const char *path, char *key, char *value, enum main_cp_cb_data_state *state, enum parser_cb_type type, const char **error_string, icmap_map_t config_map, void *user_data) { char key_name[ICMAP_KEYNAME_MAXLEN]; int uid, gid; static char formated_err[256]; cs_error_t cs_err; switch (type) { case PARSER_CB_START: break; case PARSER_CB_END: break; case PARSER_CB_CLEANUP: break; case PARSER_CB_ITEM: if (strcmp(path, "uidgid.uid") == 0) { uid = uid_determine(value); if (uid == -1) { *error_string = error_string_response; return (0); } snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "uidgid.config.uid.%u", uid); if ((cs_err = icmap_set_uint8_r(config_map, key_name, 1)) != CS_OK) { goto icmap_set_error; } } else if (strcmp(path, "uidgid.gid") == 0) { gid = gid_determine(value); if (gid == -1) { *error_string = error_string_response; return (0); } snprintf(key_name, ICMAP_KEYNAME_MAXLEN, "uidgid.config.gid.%u", gid); if ((cs_err = icmap_set_uint8_r(config_map, key_name, 1)) != CS_OK) { goto icmap_set_error; } } else { *error_string = "uidgid: Only uid and gid are allowed items"; return (0); } break; case PARSER_CB_SECTION_START: if (strcmp(path, "uidgid") != 0) { *error_string = "uidgid: Can't add subsection different than uidgid"; return (0); }; break; case PARSER_CB_SECTION_END: break; } return (1); icmap_set_error: if (snprintf(formated_err, sizeof(formated_err), "Can't store key \"%s\" into icmap, returned error is %s", key_name, cs_strerror(cs_err)) >= sizeof(formated_err)) { *error_string = "Can't format parser error message"; } else { *error_string = formated_err; } return (0); } static int read_uidgid_files_into_icmap( const char **error_string, icmap_map_t config_map) { FILE *fp; char *dirname_res; DIR *dp; struct dirent *dirent; char filename[PATH_MAX + FILENAME_MAX + 1]; char uidgid_dirname[PATH_MAX + FILENAME_MAX + 1]; int res = 0; struct stat stat_buf; enum main_cp_cb_data_state state = MAIN_CP_CB_DATA_STATE_NORMAL; char key_name[ICMAP_KEYNAME_MAXLEN]; int line_no; /* * Build uidgid directory based on corosync.conf file location */ res = snprintf(filename, sizeof(filename), "%s", corosync_get_config_file()); if (res >= sizeof(filename)) { *error_string = "uidgid.d path too long"; return (-1); } dirname_res = dirname(filename); res = snprintf(uidgid_dirname, sizeof(uidgid_dirname), "%s/%s", dirname_res, "uidgid.d"); if (res >= sizeof(uidgid_dirname)) { *error_string = "uidgid.d path too long"; return (-1); } dp = opendir (uidgid_dirname); if (dp == NULL) return 0; for (dirent = readdir(dp); dirent != NULL; dirent = readdir(dp)) { res = snprintf(filename, sizeof (filename), "%s/%s", uidgid_dirname, dirent->d_name); if (res >= sizeof(filename)) { res = -1; *error_string = "uidgid.d dirname path too long"; goto error_exit; } // coverity[TOCTOU:SUPPRESS] not really problem res = stat (filename, &stat_buf); if (res == 0 && S_ISREG(stat_buf.st_mode)) { fp = fopen (filename, "r"); if (fp == NULL) continue; key_name[0] = 0; line_no = 0; res = parse_section(fp, filename, &line_no, key_name, error_string, 0, state, uidgid_config_parser_cb, config_map, NULL); fclose (fp); if (res != 0) { goto error_exit; } } } error_exit: closedir(dp); return res; } /* Read config file and load into icmap */ static int read_config_file_into_icmap( const char **error_string, icmap_map_t config_map) { FILE *fp; const char *filename; char *error_reason = error_string_response; int res; char key_name[ICMAP_KEYNAME_MAXLEN]; struct main_cp_cb_data data; enum main_cp_cb_data_state state = MAIN_CP_CB_DATA_STATE_NORMAL; int line_no; filename = corosync_get_config_file(); fp = fopen (filename, "r"); if (fp == NULL) { char error_str[100]; const char *error_ptr = qb_strerror_r(errno, error_str, sizeof(error_str)); snprintf (error_reason, sizeof(error_string_response), "Can't read file %s: %s", filename, error_ptr); *error_string = error_reason; return -1; } key_name[0] = 0; line_no = 0; res = parse_section(fp, filename, &line_no, key_name, error_string, 0, state, main_config_parser_cb, config_map, &data); fclose(fp); if (res == 0) { res = read_uidgid_files_into_icmap(error_string, config_map); } if (res == 0) { snprintf (error_reason, sizeof(error_string_response), "Successfully read main configuration file '%s'.", filename); *error_string = error_reason; } return res; } diff --git a/exec/votequorum.c b/exec/votequorum.c index 7c6ed3be..e5444528 100644 --- a/exec/votequorum.c +++ b/exec/votequorum.c @@ -1,3082 +1,3286 @@ /* * Copyright (c) 2009-2020 Red Hat, Inc. * * All rights reserved. * * Authors: Christine Caulfield (ccaulfie@redhat.com) * Fabio M. Di Nitto (fdinitto@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the MontaVista Software, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include "quorum.h" #include #include #include #include #include #include #include "service.h" #include "util.h" LOGSYS_DECLARE_SUBSYS ("VOTEQ"); /* * interface with corosync */ static struct corosync_api_v1 *corosync_api; /* * votequorum global config vars */ static char qdevice_name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]; static struct cluster_node *qdevice = NULL; static unsigned int qdevice_timeout = VOTEQUORUM_QDEVICE_DEFAULT_TIMEOUT; static unsigned int qdevice_sync_timeout = VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT; static uint8_t qdevice_can_operate = 1; static void *qdevice_reg_conn = NULL; static uint8_t qdevice_master_wins = 0; static uint8_t two_node = 0; static uint8_t wait_for_all = 0; static uint8_t wait_for_all_status = 0; static uint8_t wait_for_all_autoset = 0; /* Wait for all is not set explicitly and follows two_node */ static enum {ATB_NONE, ATB_LOWEST, ATB_HIGHEST, ATB_LIST} auto_tie_breaker = ATB_NONE, initial_auto_tie_breaker = ATB_NONE; static int lowest_node_id = -1; static int highest_node_id = -1; #define DEFAULT_LMS_WIN 10000 static uint8_t last_man_standing = 0; static uint32_t last_man_standing_window = DEFAULT_LMS_WIN; static uint8_t allow_downscale = 0; static uint32_t ev_barrier = 0; static uint8_t ev_tracking = 0; static uint32_t ev_tracking_barrier = 0; static int ev_tracking_fd = -1; +/* + * site-aware quorum + */ +#define VOTEQUORUM_MAX_SITES 256 +#define VOTEQUORUM_SITE_UNASSIGNED 0xFFFFFFFF + +static uint8_t site_aware = 0; +static uint32_t site_expected_nodes[VOTEQUORUM_MAX_SITES]; +static uint32_t num_sites = 0; + /* * votequorum_exec defines/structs/forward definitions */ struct req_exec_quorum_nodeinfo { struct qb_ipc_request_header header __attribute__((aligned(8))); uint32_t nodeid; uint32_t votes; uint32_t expected_votes; uint32_t flags; } __attribute__((packed)); struct req_exec_quorum_reconfigure { struct qb_ipc_request_header header __attribute__((aligned(8))); uint32_t nodeid; uint32_t value; uint8_t param; uint8_t _pad0; uint8_t _pad1; uint8_t _pad2; } __attribute__((packed)); struct req_exec_quorum_qdevice_reg { struct qb_ipc_request_header header __attribute__((aligned(8))); uint32_t operation; char qdevice_name[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]; } __attribute__((packed)); struct req_exec_quorum_qdevice_reconfigure { struct qb_ipc_request_header header __attribute__((aligned(8))); char oldname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]; char newname[VOTEQUORUM_QDEVICE_MAX_NAME_LEN]; } __attribute__((packed)); /* * votequorum_exec onwire version (via totem) */ #include "votequorum.h" /* * votequorum_exec onwire messages (via totem) */ #define MESSAGE_REQ_EXEC_VOTEQUORUM_NODEINFO 0 #define MESSAGE_REQ_EXEC_VOTEQUORUM_RECONFIGURE 1 #define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_REG 2 #define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_RECONFIGURE 3 static void votequorum_exec_send_expectedvotes_notification(void); static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context); static int votequorum_exec_send_nodelist_notification(void *conn, uint64_t context); #define VOTEQUORUM_RECONFIG_PARAM_EXPECTED_VOTES 1 #define VOTEQUORUM_RECONFIG_PARAM_NODE_VOTES 2 #define VOTEQUORUM_RECONFIG_PARAM_CANCEL_WFA 3 static int votequorum_exec_send_reconfigure(uint8_t param, unsigned int nodeid, uint32_t value); /* * used by req_exec_quorum_qdevice_reg */ #define VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER 0 #define VOTEQUORUM_QDEVICE_OPERATION_REGISTER 1 /* * votequorum internal node status/view */ #define NODE_FLAGS_QUORATE 1 #define NODE_FLAGS_LEAVING 2 #define NODE_FLAGS_WFASTATUS 4 #define NODE_FLAGS_FIRST 8 #define NODE_FLAGS_QDEVICE_REGISTERED 16 #define NODE_FLAGS_QDEVICE_ALIVE 32 #define NODE_FLAGS_QDEVICE_CAST_VOTE 64 #define NODE_FLAGS_QDEVICE_MASTER_WINS 128 typedef enum { NODESTATE_MEMBER=1, NODESTATE_DEAD, NODESTATE_LEAVING } nodestate_t; struct cluster_node { int node_id; nodestate_t state; uint32_t votes; uint32_t expected_votes; uint32_t flags; + uint32_t site_id; struct qb_list_head list; }; /* * votequorum internal quorum status */ static uint8_t quorum; static uint8_t cluster_is_quorate; /* * votequorum membership data */ static struct cluster_node *us; static struct qb_list_head cluster_members_list; static unsigned int quorum_members[PROCESSOR_COUNT_MAX]; static unsigned int previous_quorum_members[PROCESSOR_COUNT_MAX]; static unsigned int atb_nodelist[PROCESSOR_COUNT_MAX]; static int quorum_members_entries = 0; static int previous_quorum_members_entries = 0; static int atb_nodelist_entries = 0; static struct memb_ring_id quorum_ringid; /* * pre allocate all cluster_nodes + one for qdevice */ static struct cluster_node cluster_nodes[PROCESSOR_COUNT_MAX+2]; static int cluster_nodes_entries = 0; /* * votequorum tracking */ struct quorum_pd { unsigned char track_flags; int tracking_enabled; uint64_t tracking_context; struct qb_list_head list; void *conn; }; static struct qb_list_head trackers_list; /* * votequorum timers */ static corosync_timer_handle_t qdevice_timer; static int qdevice_timer_set = 0; static corosync_timer_handle_t last_man_standing_timer; static int last_man_standing_timer_set = 0; static int sync_nodeinfo_sent = 0; static int sync_wait_for_poll_or_timeout = 0; /* * Service Interfaces required by service_message_handler struct */ static int sync_in_progress = 0; static void votequorum_sync_init ( const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id); static int votequorum_sync_process (void); static void votequorum_sync_activate (void); static void votequorum_sync_abort (void); static quorum_set_quorate_fn_t quorum_callback; /* * votequorum_exec handler and definitions */ static char *votequorum_exec_init_fn (struct corosync_api_v1 *api); static int votequorum_exec_exit_fn (void); static int votequorum_exec_send_nodeinfo(uint32_t nodeid); static void message_handler_req_exec_votequorum_nodeinfo ( const void *message, unsigned int nodeid); static void exec_votequorum_nodeinfo_endian_convert (void *message); static void message_handler_req_exec_votequorum_reconfigure ( const void *message, unsigned int nodeid); static void exec_votequorum_reconfigure_endian_convert (void *message); static void message_handler_req_exec_votequorum_qdevice_reg ( const void *message, unsigned int nodeid); static void exec_votequorum_qdevice_reg_endian_convert (void *message); static void message_handler_req_exec_votequorum_qdevice_reconfigure ( const void *message, unsigned int nodeid); static void exec_votequorum_qdevice_reconfigure_endian_convert (void *message); static struct corosync_exec_handler votequorum_exec_engine[] = { { /* 0 */ .exec_handler_fn = message_handler_req_exec_votequorum_nodeinfo, .exec_endian_convert_fn = exec_votequorum_nodeinfo_endian_convert }, { /* 1 */ .exec_handler_fn = message_handler_req_exec_votequorum_reconfigure, .exec_endian_convert_fn = exec_votequorum_reconfigure_endian_convert }, { /* 2 */ .exec_handler_fn = message_handler_req_exec_votequorum_qdevice_reg, .exec_endian_convert_fn = exec_votequorum_qdevice_reg_endian_convert }, { /* 3 */ .exec_handler_fn = message_handler_req_exec_votequorum_qdevice_reconfigure, .exec_endian_convert_fn = exec_votequorum_qdevice_reconfigure_endian_convert }, }; /* * Library Handler and Functions Definitions */ static int quorum_lib_init_fn (void *conn); static int quorum_lib_exit_fn (void *conn); static void qdevice_timer_fn(void *arg); static void message_handler_req_lib_votequorum_getinfo (void *conn, const void *message); static void message_handler_req_lib_votequorum_setexpected (void *conn, const void *message); static void message_handler_req_lib_votequorum_setvotes (void *conn, const void *message); static void message_handler_req_lib_votequorum_trackstart (void *conn, const void *message); static void message_handler_req_lib_votequorum_trackstop (void *conn, const void *message); static void message_handler_req_lib_votequorum_qdevice_register (void *conn, const void *message); static void message_handler_req_lib_votequorum_qdevice_unregister (void *conn, const void *message); static void message_handler_req_lib_votequorum_qdevice_update (void *conn, const void *message); static void message_handler_req_lib_votequorum_qdevice_poll (void *conn, const void *message); static void message_handler_req_lib_votequorum_qdevice_master_wins (void *conn, const void *message); static struct corosync_lib_handler quorum_lib_service[] = { { /* 0 */ .lib_handler_fn = message_handler_req_lib_votequorum_getinfo, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED }, { /* 1 */ .lib_handler_fn = message_handler_req_lib_votequorum_setexpected, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED }, { /* 2 */ .lib_handler_fn = message_handler_req_lib_votequorum_setvotes, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED }, { /* 3 */ .lib_handler_fn = message_handler_req_lib_votequorum_trackstart, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED }, { /* 4 */ .lib_handler_fn = message_handler_req_lib_votequorum_trackstop, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED }, { /* 5 */ .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_register, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED }, { /* 6 */ .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_unregister, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED }, { /* 7 */ .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_update, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED }, { /* 8 */ .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_poll, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED }, { /* 9 */ .lib_handler_fn = message_handler_req_lib_votequorum_qdevice_master_wins, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED } }; static struct corosync_service_engine votequorum_service_engine = { .name = "corosync vote quorum service v1.0", .id = VOTEQUORUM_SERVICE, .priority = 2, .private_data_size = sizeof (struct quorum_pd), .allow_inquorate = CS_LIB_ALLOW_INQUORATE, .flow_control = COROSYNC_LIB_FLOW_CONTROL_REQUIRED, .lib_init_fn = quorum_lib_init_fn, .lib_exit_fn = quorum_lib_exit_fn, .lib_engine = quorum_lib_service, .lib_engine_count = sizeof (quorum_lib_service) / sizeof (struct corosync_lib_handler), .exec_init_fn = votequorum_exec_init_fn, .exec_exit_fn = votequorum_exec_exit_fn, .exec_engine = votequorum_exec_engine, .exec_engine_count = sizeof (votequorum_exec_engine) / sizeof (struct corosync_exec_handler), .sync_init = votequorum_sync_init, .sync_process = votequorum_sync_process, .sync_activate = votequorum_sync_activate, .sync_abort = votequorum_sync_abort }; struct corosync_service_engine *votequorum_get_service_engine_ver0 (void) { return (&votequorum_service_engine); } static struct default_service votequorum_service[] = { { .name = "corosync_votequorum", .ver = 0, .loader = votequorum_get_service_engine_ver0 }, }; /* * common/utility macros/functions */ #define max(a,b) (((a) > (b)) ? (a) : (b)) static void node_add_ordered(struct cluster_node *newnode) { struct cluster_node *node = NULL; struct qb_list_head *tmp; ENTER(); qb_list_for_each(tmp, &cluster_members_list) { node = qb_list_entry(tmp, struct cluster_node, list); if (newnode->node_id < node->node_id) { break; } } if (!node) { qb_list_add(&newnode->list, &cluster_members_list); } else { qb_list_add_tail(&newnode->list, &node->list); } LEAVE(); } static struct cluster_node *allocate_node(unsigned int nodeid) { struct cluster_node *cl = NULL; struct qb_list_head *tmp; ENTER(); if (cluster_nodes_entries <= PROCESSOR_COUNT_MAX + 1) { cl = (struct cluster_node *)&cluster_nodes[cluster_nodes_entries]; cluster_nodes_entries++; } else { qb_list_for_each(tmp, &cluster_members_list) { cl = qb_list_entry(tmp, struct cluster_node, list); if (cl->state == NODESTATE_DEAD) { break; } } /* * this should never happen */ if (!cl) { log_printf(LOGSYS_LEVEL_CRIT, "Unable to find memory for node " CS_PRI_NODE_ID " data!!", nodeid); goto out; } qb_list_del(tmp); } memset(cl, 0, sizeof(struct cluster_node)); cl->node_id = nodeid; if (nodeid != VOTEQUORUM_QDEVICE_NODEID) { node_add_ordered(cl); } out: LEAVE(); return cl; } static struct cluster_node *find_node_by_nodeid(unsigned int nodeid) { struct cluster_node *node; struct qb_list_head *tmp; ENTER(); if (nodeid == us->node_id) { LEAVE(); return us; } if (nodeid == VOTEQUORUM_QDEVICE_NODEID) { LEAVE(); return qdevice; } qb_list_for_each(tmp, &cluster_members_list) { node = qb_list_entry(tmp, struct cluster_node, list); if (node->node_id == nodeid) { LEAVE(); return node; } } LEAVE(); return NULL; } static void get_lowest_node_id(void) { struct cluster_node *node = NULL; struct qb_list_head *tmp; ENTER(); lowest_node_id = us->node_id; qb_list_for_each(tmp, &cluster_members_list) { node = qb_list_entry(tmp, struct cluster_node, list); if ((node->state == NODESTATE_MEMBER) && (node->node_id < lowest_node_id)) { lowest_node_id = node->node_id; } } log_printf(LOGSYS_LEVEL_DEBUG, "lowest node id: " CS_PRI_NODE_ID " us: " CS_PRI_NODE_ID, lowest_node_id, us->node_id); icmap_set_uint32("runtime.votequorum.lowest_node_id", lowest_node_id); LEAVE(); } static void get_highest_node_id(void) { struct cluster_node *node = NULL; struct qb_list_head *tmp; ENTER(); highest_node_id = us->node_id; qb_list_for_each(tmp, &cluster_members_list) { node = qb_list_entry(tmp, struct cluster_node, list); if ((node->state == NODESTATE_MEMBER) && (node->node_id > highest_node_id)) { highest_node_id = node->node_id; } } log_printf(LOGSYS_LEVEL_DEBUG, "highest node id: " CS_PRI_NODE_ID " us: " CS_PRI_NODE_ID, highest_node_id, us->node_id); icmap_set_uint32("runtime.votequorum.highest_node_id", highest_node_id); LEAVE(); } static int check_low_node_id_partition(void) { struct cluster_node *node = NULL; struct qb_list_head *tmp; int found = 0; ENTER(); qb_list_for_each(tmp, &cluster_members_list) { node = qb_list_entry(tmp, struct cluster_node, list); if ((node->state == NODESTATE_MEMBER) && (node->node_id == lowest_node_id)) { found = 1; } } LEAVE(); return found; } static int check_high_node_id_partition(void) { struct cluster_node *node = NULL; struct qb_list_head *tmp; int found = 0; ENTER(); qb_list_for_each(tmp, &cluster_members_list) { node = qb_list_entry(tmp, struct cluster_node, list); if ((node->state == NODESTATE_MEMBER) && (node->node_id == highest_node_id)) { found = 1; } } LEAVE(); return found; } static int is_in_nodelist(int nodeid, unsigned int *members, int entries) { int i; ENTER(); for (i=0; istate == NODESTATE_MEMBER) && (node->flags & NODE_FLAGS_QDEVICE_MASTER_WINS) && (node->flags & NODE_FLAGS_QDEVICE_CAST_VOTE)) { found = 1; } } LEAVE(); return found; } static void decode_flags(uint32_t flags) { ENTER(); log_printf(LOGSYS_LEVEL_DEBUG, "flags: quorate: %s Leaving: %s WFA Status: %s First: %s Qdevice: %s QdeviceAlive: %s QdeviceCastVote: %s QdeviceMasterWins: %s", (flags & NODE_FLAGS_QUORATE)?"Yes":"No", (flags & NODE_FLAGS_LEAVING)?"Yes":"No", (flags & NODE_FLAGS_WFASTATUS)?"Yes":"No", (flags & NODE_FLAGS_FIRST)?"Yes":"No", (flags & NODE_FLAGS_QDEVICE_REGISTERED)?"Yes":"No", (flags & NODE_FLAGS_QDEVICE_ALIVE)?"Yes":"No", (flags & NODE_FLAGS_QDEVICE_CAST_VOTE)?"Yes":"No", (flags & NODE_FLAGS_QDEVICE_MASTER_WINS)?"Yes":"No"); LEAVE(); } /* * load/save are copied almost pristine from totemsrp,c */ static int load_ev_tracking_barrier(void) { int res = 0; char filename[PATH_MAX]; ENTER(); snprintf(filename, sizeof(filename) - 1, "%s/ev_tracking", get_state_dir()); ev_tracking_fd = open(filename, O_RDWR, 0700); if (ev_tracking_fd != -1) { res = read (ev_tracking_fd, &ev_tracking_barrier, sizeof(uint32_t)); close(ev_tracking_fd); if (res == sizeof (uint32_t)) { LEAVE(); return 0; } } ev_tracking_barrier = 0; umask(0); ev_tracking_fd = open (filename, O_CREAT|O_RDWR, 0700); if (ev_tracking_fd != -1) { res = write (ev_tracking_fd, &ev_tracking_barrier, sizeof (uint32_t)); if ((res == -1) || (res != sizeof (uint32_t))) { log_printf(LOGSYS_LEVEL_WARNING, "Unable to write to %s", filename); } close(ev_tracking_fd); LEAVE(); return 0; } log_printf(LOGSYS_LEVEL_WARNING, "Unable to create %s file", filename); LEAVE(); return -1; } static void update_wait_for_all_status(uint8_t wfa_status) { ENTER(); wait_for_all_status = wfa_status; if (wait_for_all_status) { us->flags |= NODE_FLAGS_WFASTATUS; } else { us->flags &= ~NODE_FLAGS_WFASTATUS; } icmap_set_uint8("runtime.votequorum.wait_for_all_status", wait_for_all_status); LEAVE(); } static void update_two_node(void) { ENTER(); icmap_set_uint8("runtime.votequorum.two_node", two_node); LEAVE(); } +static void update_site_aware(void) +{ + uint32_t site; + char key[ICMAP_KEYNAME_MAXLEN]; + + ENTER(); + + icmap_set_uint8("runtime.votequorum.site_aware", site_aware); + icmap_set_uint32("runtime.votequorum.num_sites", num_sites); + + /* Export site membership info to runtime */ + for (site = 0; site < num_sites; site++) { + snprintf(key, ICMAP_KEYNAME_MAXLEN, + "runtime.votequorum.site.%u.expected_nodes", site); + icmap_set_uint32(key, site_expected_nodes[site]); + } + + LEAVE(); +} + static void update_ev_barrier(uint32_t expected_votes) { ENTER(); ev_barrier = expected_votes; icmap_set_uint32("runtime.votequorum.ev_barrier", ev_barrier); LEAVE(); } static void update_qdevice_can_operate(uint8_t status) { ENTER(); qdevice_can_operate = status; icmap_set_uint8("runtime.votequorum.qdevice_can_operate", qdevice_can_operate); LEAVE(); } static void update_qdevice_master_wins(uint8_t allow) { ENTER(); qdevice_master_wins = allow; icmap_set_uint8("runtime.votequorum.qdevice_master_wins", qdevice_master_wins); LEAVE(); } static void update_ev_tracking_barrier(uint32_t ev_t_barrier) { int res; ENTER(); ev_tracking_barrier = ev_t_barrier; icmap_set_uint32("runtime.votequorum.ev_tracking_barrier", ev_tracking_barrier); if (lseek (ev_tracking_fd, 0, SEEK_SET) != 0) { log_printf(LOGSYS_LEVEL_WARNING, "Unable to update ev_tracking_barrier on disk data!!!"); LEAVE(); return; } res = write (ev_tracking_fd, &ev_tracking_barrier, sizeof (uint32_t)); if (res != sizeof (uint32_t)) { log_printf(LOGSYS_LEVEL_WARNING, "Unable to update ev_tracking_barrier on disk data!!!"); } #ifdef HAVE_FDATASYNC fdatasync(ev_tracking_fd); #else fsync(ev_tracking_fd); #endif LEAVE(); } /* * quorum calculation core bits */ +static int partition_has_complete_site(void) +{ + uint32_t site_current_nodes[VOTEQUORUM_MAX_SITES] = {0}; + struct qb_list_head *nodelist; + struct cluster_node *node; + uint32_t site; + + ENTER(); + + /* Count current member nodes per site */ + qb_list_for_each(nodelist, &cluster_members_list) { + node = qb_list_entry(nodelist, struct cluster_node, list); + + if (node->state == NODESTATE_MEMBER && + node->site_id != VOTEQUORUM_SITE_UNASSIGNED && + node->site_id < VOTEQUORUM_MAX_SITES) { + site_current_nodes[node->site_id]++; + } + } + + /* Check if any site is complete */ + for (site = 0; site < num_sites; site++) { + if (site_expected_nodes[site] > 0 && + site_current_nodes[site] == site_expected_nodes[site]) { + log_printf(LOGSYS_LEVEL_DEBUG, + "Site %u is complete (%u/%u nodes)", + site, site_current_nodes[site], + site_expected_nodes[site]); + LEAVE(); + return 1; + } + } + + LEAVE(); + return 0; +} + static int calculate_quorum(int allow_decrease, unsigned int max_expected, unsigned int *ret_total_votes) { struct qb_list_head *nodelist; struct cluster_node *node; unsigned int total_votes = 0; unsigned int highest_expected = 0; unsigned int newquorum, q1, q2; unsigned int total_nodes = 0; ENTER(); if ((allow_downscale) && (allow_decrease) && (max_expected)) { max_expected = max(ev_barrier, max_expected); } qb_list_for_each(nodelist, &cluster_members_list) { node = qb_list_entry(nodelist, struct cluster_node, list); log_printf(LOGSYS_LEVEL_DEBUG, "node " CS_PRI_NODE_ID " state=%d, votes=%u, expected=%u", node->node_id, node->state, node->votes, node->expected_votes); if (node->state == NODESTATE_MEMBER) { highest_expected = max(highest_expected, node->expected_votes); total_votes += node->votes; total_nodes++; } } if (us->flags & NODE_FLAGS_QDEVICE_CAST_VOTE) { log_printf(LOGSYS_LEVEL_DEBUG, "node 0 state=1, votes=%u", qdevice->votes); total_votes += qdevice->votes; total_nodes++; } if (max_expected > 0) { highest_expected = max_expected; } /* * This quorum calculation is taken from the OpenVMS Cluster Systems * manual, but, then, you guessed that didn't you */ q1 = (highest_expected + 2) / 2; q2 = (total_votes + 2) / 2; newquorum = max(q1, q2); /* * Normally quorum never decreases but the system administrator can * force it down by setting expected votes to a maximum value */ if (!allow_decrease) { newquorum = max(quorum, newquorum); } /* * The special two_node mode allows each of the two nodes to retain * quorum if the other fails. Only one of the two should live past * fencing (as both nodes try to fence each other in split-brain.) * Also: if there are more than two nodes, force us inquorate to avoid * any damage or confusion. */ if (two_node && total_nodes <= 2) { newquorum = 1; } + /* + * Site-aware quorum: if we are currently quorate and have all nodes + * from at least one complete site, retain quorum even if we drop below + * 50% total votes. This extends two_node logic to multiple sites. + * Only applies when already quorate to prevent split-brain - an inquorate + * partition must achieve standard majority to regain quorum. + */ + if (site_aware && cluster_is_quorate && partition_has_complete_site()) { + newquorum = 1; + } + if (ret_total_votes) { *ret_total_votes = total_votes; } LEAVE(); return newquorum; } static void update_node_expected_votes(int new_expected_votes) { struct qb_list_head *nodelist; struct cluster_node *node; if (new_expected_votes) { qb_list_for_each(nodelist, &cluster_members_list) { node = qb_list_entry(nodelist, struct cluster_node, list); if (node->state == NODESTATE_MEMBER) { node->expected_votes = new_expected_votes; } } } } static void are_we_quorate(unsigned int total_votes) { int quorate; int quorum_change = 0; ENTER(); /* * wait for all nodes to show up before granting quorum */ if ((wait_for_all) && (wait_for_all_status)) { if (total_votes != us->expected_votes) { log_printf(LOGSYS_LEVEL_NOTICE, "Waiting for all cluster members. " "Current votes: %d expected_votes: %d", total_votes, us->expected_votes); assert(!cluster_is_quorate); return; } update_wait_for_all_status(0); } if (quorum > total_votes) { quorate = 0; } else { quorate = 1; get_lowest_node_id(); get_highest_node_id(); } if ((auto_tie_breaker != ATB_NONE) && /* Must be a half (or half-1) split */ (total_votes == (us->expected_votes / 2)) && /* If the 'other' partition in a split might have quorum then we can't run ATB */ (previous_quorum_members_entries - quorum_members_entries < quorum) && (check_auto_tie_breaker() == 1)) { quorate = 1; } if ((qdevice_master_wins) && (!quorate) && (check_qdevice_master() == 1)) { log_printf(LOGSYS_LEVEL_DEBUG, "node is quorate as part of master_wins partition"); quorate = 1; } if (cluster_is_quorate && !quorate) { quorum_change = 1; log_printf(LOGSYS_LEVEL_DEBUG, "quorum lost, blocking activity"); } if (!cluster_is_quorate && quorate) { quorum_change = 1; log_printf(LOGSYS_LEVEL_DEBUG, "quorum regained, resuming activity"); } cluster_is_quorate = quorate; if (cluster_is_quorate) { us->flags |= NODE_FLAGS_QUORATE; } else { us->flags &= ~NODE_FLAGS_QUORATE; } if (wait_for_all) { if (quorate) { update_wait_for_all_status(0); } else { update_wait_for_all_status(1); } } if ((quorum_change) && (sync_in_progress == 0)) { quorum_callback(quorum_members, quorum_members_entries, cluster_is_quorate, &quorum_ringid); votequorum_exec_send_quorum_notification(NULL, 0L); } LEAVE(); } static void get_total_votes(unsigned int *totalvotes, unsigned int *current_members) { unsigned int total_votes = 0; unsigned int cluster_members = 0; struct qb_list_head *nodelist; struct cluster_node *node; ENTER(); qb_list_for_each(nodelist, &cluster_members_list) { node = qb_list_entry(nodelist, struct cluster_node, list); if (node->state == NODESTATE_MEMBER) { cluster_members++; total_votes += node->votes; } } if (qdevice->votes) { total_votes += qdevice->votes; cluster_members++; } *totalvotes = total_votes; *current_members = cluster_members; LEAVE(); } /* * Recalculate cluster quorum, set quorate and notify changes */ static void recalculate_quorum(int allow_decrease, int by_current_nodes) { unsigned int total_votes = 0; unsigned int cluster_members = 0; ENTER(); get_total_votes(&total_votes, &cluster_members); if (!by_current_nodes) { cluster_members = 0; } /* * Keep expected_votes at the highest number of votes in the cluster */ log_printf(LOGSYS_LEVEL_DEBUG, "total_votes=%d, expected_votes=%d", total_votes, us->expected_votes); if (total_votes > us->expected_votes) { us->expected_votes = total_votes; votequorum_exec_send_expectedvotes_notification(); } if ((ev_tracking) && (us->expected_votes > ev_tracking_barrier)) { update_ev_tracking_barrier(us->expected_votes); } quorum = calculate_quorum(allow_decrease, cluster_members, &total_votes); update_node_expected_votes(cluster_members); are_we_quorate(total_votes); LEAVE(); } /* * configuration bits and pieces */ +static uint32_t get_node_site_id(uint32_t nodeid) +{ + icmap_iter_t iter; + const char *iter_key; + char tmp_key[ICMAP_KEYNAME_MAXLEN]; + uint32_t node_pos, last_node_pos=-1; + uint32_t config_nodeid; + uint32_t site_id = VOTEQUORUM_SITE_UNASSIGNED; + int res = 0; + + ENTER(); + + iter = icmap_iter_init("nodelist.node."); + + while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { + + res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, tmp_key); + if (res != 2) { + continue; + } + + if (last_node_pos == node_pos) { + continue; + } + last_node_pos = node_pos; + + /* Get this node's ID */ + snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.nodeid", node_pos); + if (icmap_get_uint32(tmp_key, &config_nodeid) != CS_OK) { + continue; + } + + /* Found our node, get its site */ + if (config_nodeid == nodeid) { + snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.site", node_pos); + if (icmap_get_uint32(tmp_key, &site_id) == CS_OK) { + log_printf(LOGSYS_LEVEL_DEBUG, + "Node " CS_PRI_NODE_ID " assigned to site %u", + nodeid, site_id); + } else { + log_printf(LOGSYS_LEVEL_DEBUG, + "Node " CS_PRI_NODE_ID " has no site assignment", + nodeid); + } + break; + } + } + + icmap_iter_finalize(iter); + + LEAVE(); + return site_id; +} + static int votequorum_read_nodelist_configuration(uint32_t *votes, uint32_t *nodes, uint32_t *expected_votes) { icmap_iter_t iter; const char *iter_key; char tmp_key[ICMAP_KEYNAME_MAXLEN]; uint32_t our_pos, node_pos, last_node_pos=-1; uint32_t nodecount = 0; uint32_t nodelist_expected_votes = 0; uint32_t node_votes = 0; + uint32_t node_site_id; int res = 0; ENTER(); + /* Initialize site tracking */ + memset(site_expected_nodes, 0, sizeof(site_expected_nodes)); + num_sites = 0; + if (icmap_get_uint32("nodelist.local_node_pos", &our_pos) != CS_OK) { log_printf(LOGSYS_LEVEL_DEBUG, "No nodelist defined or our node is not in the nodelist"); return 0; } iter = icmap_iter_init("nodelist.node."); while ((iter_key = icmap_iter_next(iter, NULL, NULL)) != NULL) { res = sscanf(iter_key, "nodelist.node.%u.%s", &node_pos, tmp_key); if (res != 2) { continue; } /* * If current node_pos is the same as the last_node_pos then skip it * so we only do the code below once per node. * (icmap keys are always in order) */ if (last_node_pos == node_pos) { continue; } last_node_pos = node_pos; nodecount++; snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.quorum_votes", node_pos); if (icmap_get_uint32(tmp_key, &node_votes) != CS_OK) { node_votes = 1; } nodelist_expected_votes = nodelist_expected_votes + node_votes; if (node_pos == our_pos) { *votes = node_votes; } + + /* Track site membership for site-aware quorum */ + snprintf(tmp_key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%u.site", node_pos); + if (icmap_get_uint32(tmp_key, &node_site_id) == CS_OK) { + if (node_site_id < VOTEQUORUM_MAX_SITES) { + site_expected_nodes[node_site_id]++; + if (node_site_id + 1 > num_sites) { + num_sites = node_site_id + 1; + } + } else { + log_printf(LOGSYS_LEVEL_WARNING, + "Node %u site ID %u exceeds maximum %u, ignoring", + node_pos, node_site_id, VOTEQUORUM_MAX_SITES - 1); + } + } } *expected_votes = nodelist_expected_votes; *nodes = nodecount; icmap_iter_finalize(iter); LEAVE(); return 1; } static int votequorum_qdevice_is_configured(uint32_t *qdevice_votes) { char *qdevice_model = NULL; int ret = 0; ENTER(); if (icmap_get_string("quorum.device.model", &qdevice_model) == CS_OK) { if (strlen(qdevice_model)) { if (icmap_get_uint32("quorum.device.votes", qdevice_votes) != CS_OK) { *qdevice_votes = -1; } if (icmap_get_uint32("quorum.device.timeout", &qdevice_timeout) != CS_OK) { qdevice_timeout = VOTEQUORUM_QDEVICE_DEFAULT_TIMEOUT; } if (icmap_get_uint32("quorum.device.sync_timeout", &qdevice_sync_timeout) != CS_OK) { qdevice_sync_timeout = VOTEQUORUM_QDEVICE_DEFAULT_SYNC_TIMEOUT; } update_qdevice_can_operate(1); ret = 1; } free(qdevice_model); } LEAVE(); return ret; } #define VOTEQUORUM_READCONFIG_STARTUP 0 #define VOTEQUORUM_READCONFIG_RUNTIME 1 static char *votequorum_readconfig(int runtime) { uint32_t node_votes = 0, qdevice_votes = 0; uint32_t node_expected_votes = 0, expected_votes = 0; uint32_t node_count = 0; uint8_t atb = 0; int have_nodelist, have_qdevice; char *atb_string = NULL; char *error = NULL; ENTER(); log_printf(LOGSYS_LEVEL_DEBUG, "Reading configuration (runtime: %d)", runtime); /* * Set the few things we re-read at runtime back to their defaults */ if (runtime) { two_node = 0; expected_votes = 0; /* auto_tie_breaker cannot be changed by config reload, but * we automatically disable it on odd-sized clusters without * wait_for_all. * We may need to re-enable it when membership changes to ensure * that auto_tie_breaker is consistent across all nodes */ auto_tie_breaker = initial_auto_tie_breaker; icmap_set_uint32("runtime.votequorum.atb_type", auto_tie_breaker); } /* * gather basic data here */ (void)icmap_get_uint32("quorum.expected_votes", &expected_votes); have_nodelist = votequorum_read_nodelist_configuration(&node_votes, &node_count, &node_expected_votes); have_qdevice = votequorum_qdevice_is_configured(&qdevice_votes); (void)icmap_get_uint8("quorum.two_node", &two_node); + (void)icmap_get_uint8("quorum.site_aware", &site_aware); /* * do config verification and enablement */ if ((!have_nodelist) && (!expected_votes)) { if (!runtime) { error = (char *)"configuration error: nodelist or quorum.expected_votes must be configured!"; } else { log_printf(LOGSYS_LEVEL_CRIT, "configuration error: nodelist or quorum.expected_votes must be configured!"); log_printf(LOGSYS_LEVEL_CRIT, "will continue with current runtime data"); } goto out; } /* * two_node and qdevice are not compatible in the same config. * try to make an educated guess of what to do */ if ((two_node) && (have_qdevice)) { if (!runtime) { error = (char *)"configuration error: two_node and quorum device cannot be configured at the same time!"; goto out; } else { log_printf(LOGSYS_LEVEL_CRIT, "configuration error: two_node and quorum device cannot be configured at the same time!"); if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) { log_printf(LOGSYS_LEVEL_CRIT, "quorum device is registered, disabling two_node"); two_node = 0; } else { log_printf(LOGSYS_LEVEL_CRIT, "quorum device is not registered, allowing two_node"); update_qdevice_can_operate(0); } } } /* * Enable special features */ if (!runtime) { (void)icmap_get_uint8("quorum.allow_downscale", &allow_downscale); if (icmap_get_uint8("quorum.wait_for_all", &wait_for_all) != CS_OK) { wait_for_all_autoset = 1; } (void)icmap_get_uint8("quorum.last_man_standing", &last_man_standing); (void)icmap_get_uint32("quorum.last_man_standing_window", &last_man_standing_window); (void)icmap_get_uint8("quorum.expected_votes_tracking", &ev_tracking); (void)icmap_get_uint8("quorum.auto_tie_breaker", &atb); (void)icmap_get_string("quorum.auto_tie_breaker_node", &atb_string); /* auto_tie_breaker defaults to LOWEST */ if (atb) { auto_tie_breaker = ATB_LOWEST; icmap_set_uint32("runtime.votequorum.atb_type", auto_tie_breaker); } else { auto_tie_breaker = ATB_NONE; if (atb_string) { log_printf(LOGSYS_LEVEL_WARNING, "auto_tie_breaker_node: is meaningless if auto_tie_breaker is set to 0"); } } if (atb && atb_string) { parse_atb_string(atb_string); } free(atb_string); initial_auto_tie_breaker = auto_tie_breaker; /* allow_downscale requires ev_tracking */ if (allow_downscale) { ev_tracking = 1; } if (ev_tracking) { if (load_ev_tracking_barrier() < 0) { LEAVE(); return ((char *)"Unable to load ev_tracking file!"); } update_ev_tracking_barrier(ev_tracking_barrier); } } /* * Changing of wait_for_all during runtime is not supported, but changing of two_node is * and two_node may set wfa if not configured explicitly. It is safe to unset it * (or set it back) when two_node changes. */ if (wait_for_all_autoset) { wait_for_all = two_node; } /* two_node and auto_tie_breaker are not compatible as two_node uses * a fence race to decide quorum whereas ATB decides based on node id */ if (two_node && auto_tie_breaker != ATB_NONE) { log_printf(LOGSYS_LEVEL_CRIT, "two_node and auto_tie_breaker are both specified but are not compatible."); log_printf(LOGSYS_LEVEL_CRIT, "two_node has been disabled, please fix your corosync.conf"); two_node = 0; } + /* + * site_aware and two_node are not compatible + */ + if (site_aware && two_node) { + if (!runtime) { + error = (char *)"configuration error: site_aware and two_node cannot be configured at the same time!"; + goto out; + } else { + log_printf(LOGSYS_LEVEL_CRIT, "configuration error: site_aware and two_node cannot be configured at the same time!"); + log_printf(LOGSYS_LEVEL_CRIT, "disabling site_aware"); + site_aware = 0; + } + } + + /* + * site_aware and qdevice are not compatible + */ + if (site_aware && have_qdevice) { + if (!runtime) { + error = (char *)"configuration error: site_aware and quorum device cannot be configured at the same time!"; + goto out; + } else { + log_printf(LOGSYS_LEVEL_CRIT, "configuration error: site_aware and quorum device cannot be configured at the same time!"); + if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) { + log_printf(LOGSYS_LEVEL_CRIT, "quorum device is registered, disabling site_aware"); + site_aware = 0; + } else { + log_printf(LOGSYS_LEVEL_CRIT, "quorum device is not registered, allowing site_aware"); + update_qdevice_can_operate(0); + } + } + } + + /* + * site_aware and auto_tie_breaker are not compatible - similar to two_node + */ + if (site_aware && auto_tie_breaker != ATB_NONE) { + log_printf(LOGSYS_LEVEL_CRIT, "site_aware and auto_tie_breaker are both specified but are not compatible."); + log_printf(LOGSYS_LEVEL_CRIT, "site_aware has been disabled, please fix your corosync.conf"); + site_aware = 0; + } + /* If ATB is set and the cluster has an odd number of nodes then wait_for_all needs * to be set so that an isolated half+1 without the tie breaker node * does not have quorum on reboot. */ if ((auto_tie_breaker != ATB_NONE) && (node_expected_votes % 2) && (!wait_for_all)) { if (last_man_standing) { /* if LMS is set too, it's a fatal configuration error. We can't dictate to the user what * they might want so we'll just quit. */ log_printf(LOGSYS_LEVEL_CRIT, "auto_tie_breaker is set, the cluster has an odd number of nodes\n"); log_printf(LOGSYS_LEVEL_CRIT, "and last_man_standing is also set. With this situation a better\n"); log_printf(LOGSYS_LEVEL_CRIT, "solution would be to disable LMS, leave ATB enabled, and also\n"); log_printf(LOGSYS_LEVEL_CRIT, "enable wait_for_all (mandatory for ATB in odd-numbered clusters).\n"); log_printf(LOGSYS_LEVEL_CRIT, "Due to this ambiguity, corosync will fail to start. Please fix your corosync.conf\n"); error = (char *)"configuration error: auto_tie_breaker & last_man_standing not available in odd sized cluster"; goto out; } else { log_printf(LOGSYS_LEVEL_CRIT, "auto_tie_breaker is set and the cluster has an odd number of nodes.\n"); log_printf(LOGSYS_LEVEL_CRIT, "wait_for_all needs to be set for this configuration but it is missing\n"); log_printf(LOGSYS_LEVEL_CRIT, "Therefore auto_tie_breaker has been disabled. Please fix your corosync.conf\n"); auto_tie_breaker = ATB_NONE; icmap_set_uint32("runtime.votequorum.atb_type", auto_tie_breaker); } } /* * quorum device is not compatible with last_man_standing and auto_tie_breaker * neither lms or atb can be set at runtime, so there is no need to check for * runtime incompatibilities, but qdevice can be configured _after_ LMS and ATB have * been enabled at startup. */ if ((have_qdevice) && (last_man_standing)) { if (!runtime) { error = (char *)"configuration error: quorum.device is not compatible with last_man_standing"; goto out; } else { log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device is not compatible with last_man_standing"); log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations"); update_qdevice_can_operate(0); } } if ((have_qdevice) && (auto_tie_breaker != ATB_NONE)) { if (!runtime) { error = (char *)"configuration error: quorum.device is not compatible with auto_tie_breaker"; goto out; } else { log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device is not compatible with auto_tie_breaker"); log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations"); update_qdevice_can_operate(0); } } if ((have_qdevice) && (allow_downscale)) { if (!runtime) { error = (char *)"configuration error: quorum.device is not compatible with allow_downscale"; goto out; } else { log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device is not compatible with allow_downscale"); log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations"); update_qdevice_can_operate(0); } } /* * if user specifies quorum.expected_votes + quorum.device but NOT the device.votes * we don't know what the quorum device should vote. */ if ((expected_votes) && (have_qdevice) && (qdevice_votes == -1)) { if (!runtime) { error = (char *)"configuration error: quorum.device.votes must be specified when quorum.expected_votes is set"; goto out; } else { log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device.votes must be specified when quorum.expected_votes is set"); log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations"); update_qdevice_can_operate(0); } } /* * if user specifies a node list with uneven votes and no device.votes * we cannot autocalculate the votes */ if ((have_qdevice) && (qdevice_votes == -1) && (have_nodelist) && (node_count != node_expected_votes)) { if (!runtime) { error = (char *)"configuration error: quorum.device.votes must be specified when not all nodes votes 1"; goto out; } else { log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device.votes must be specified when not all nodes votes 1"); log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations"); update_qdevice_can_operate(0); } } /* * validate quorum device votes vs expected_votes */ if ((qdevice_votes > 0) && (expected_votes)) { int delta = expected_votes - qdevice_votes; if (delta < 2) { if (!runtime) { error = (char *)"configuration error: quorum.device.votes is too high or expected_votes is too low"; goto out; } else { log_printf(LOGSYS_LEVEL_CRIT, "configuration error: quorum.device.votes is too high or expected_votes is too low"); log_printf(LOGSYS_LEVEL_CRIT, "disabling quorum device operations"); update_qdevice_can_operate(0); } } } /* * automatically calculate device votes and adjust expected_votes from nodelist */ if ((have_qdevice) && (qdevice_votes == -1) && (!expected_votes) && (have_nodelist) && (node_count == node_expected_votes)) { qdevice_votes = node_expected_votes - 1; node_expected_votes = node_expected_votes + qdevice_votes; } /* * set this node votes and expected_votes */ log_printf(LOGSYS_LEVEL_DEBUG, "ev_tracking=%d, ev_tracking_barrier = %d: expected_votes = %d\n", ev_tracking, ev_tracking_barrier, expected_votes); if (ev_tracking) { expected_votes = ev_tracking_barrier; } if (have_nodelist) { us->votes = node_votes; us->expected_votes = node_expected_votes; } else { us->votes = 1; (void)icmap_get_uint32("quorum.votes", &us->votes); } if (expected_votes) { us->expected_votes = expected_votes; } /* * set qdevice votes */ if (!have_qdevice) { qdevice->votes = 0; } if (qdevice_votes != -1) { qdevice->votes = qdevice_votes; } update_ev_barrier(us->expected_votes); update_two_node(); + update_site_aware(); if (wait_for_all) { if (!runtime) { update_wait_for_all_status(1); } } else if (wait_for_all_autoset && wait_for_all_status) { /* * Reset wait for all status for consistency when wfa is auto-unset by 2node. * wait_for_all_status would be ignored by are_we_quorate anyway. */ update_wait_for_all_status(0); } out: LEAVE(); return error; } static void votequorum_refresh_config( int32_t event, const char *key_name, struct icmap_notify_value new_val, struct icmap_notify_value old_val, void *user_data) { int old_votes, old_expected_votes; uint8_t reloading; uint8_t cancel_wfa; int32_t reload_status; ENTER(); /* * If a full reload is in progress then don't do anything until it's done and * can reconfigure it all atomically */ if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading) == CS_OK && reloading) { return; } /* If a full reload failed, then don't reconfigure */ if ( (strcmp(key_name, "config.totemconfig_reload_in_progress") == 0) && (icmap_get_int32("config.reload_status", &reload_status) == CS_OK) && (reload_status != CS_OK) ) { return; } (void)icmap_get_uint8("quorum.cancel_wait_for_all", &cancel_wfa); if (strcmp(key_name, "quorum.cancel_wait_for_all") == 0 && cancel_wfa >= 1) { icmap_set_uint8("quorum.cancel_wait_for_all", 0); if (votequorum_exec_send_reconfigure(VOTEQUORUM_RECONFIG_PARAM_CANCEL_WFA, us->node_id, 0)) { log_printf(LOGSYS_LEVEL_ERROR, "Failed to send Cancel WFA message to other nodes"); } return; } old_votes = us->votes; old_expected_votes = us->expected_votes; /* * Reload the configuration */ votequorum_readconfig(VOTEQUORUM_READCONFIG_RUNTIME); /* * activate new config */ votequorum_exec_send_nodeinfo(us->node_id); votequorum_exec_send_nodeinfo(VOTEQUORUM_QDEVICE_NODEID); if (us->votes != old_votes) { if (votequorum_exec_send_reconfigure(VOTEQUORUM_RECONFIG_PARAM_NODE_VOTES, us->node_id, us->votes)) { log_printf(LOGSYS_LEVEL_ERROR, "Failed to send new votes message to other nodes"); } } if (us->expected_votes != old_expected_votes) { if (votequorum_exec_send_reconfigure(VOTEQUORUM_RECONFIG_PARAM_EXPECTED_VOTES, us->node_id, us->expected_votes)) { log_printf(LOGSYS_LEVEL_ERROR, "Failed to send expected votes message to other nodes"); } } LEAVE(); } static void votequorum_exec_add_config_notification(void) { icmap_track_t icmap_track_nodelist = NULL; icmap_track_t icmap_track_quorum = NULL; icmap_track_t icmap_track_reload = NULL; ENTER(); icmap_track_add("nodelist.", ICMAP_TRACK_ADD | ICMAP_TRACK_DELETE | ICMAP_TRACK_MODIFY | ICMAP_TRACK_PREFIX, votequorum_refresh_config, NULL, &icmap_track_nodelist); icmap_track_add("quorum.", ICMAP_TRACK_ADD | ICMAP_TRACK_DELETE | ICMAP_TRACK_MODIFY | ICMAP_TRACK_PREFIX, votequorum_refresh_config, NULL, &icmap_track_quorum); icmap_track_add("config.totemconfig_reload_in_progress", ICMAP_TRACK_ADD | ICMAP_TRACK_MODIFY, votequorum_refresh_config, NULL, &icmap_track_reload); LEAVE(); } /* * votequorum_exec core */ static int votequorum_exec_send_reconfigure(uint8_t param, unsigned int nodeid, uint32_t value) { struct req_exec_quorum_reconfigure req_exec_quorum_reconfigure; struct iovec iov[1]; int ret; ENTER(); req_exec_quorum_reconfigure.nodeid = nodeid; req_exec_quorum_reconfigure.value = value; req_exec_quorum_reconfigure.param = param; req_exec_quorum_reconfigure._pad0 = 0; req_exec_quorum_reconfigure._pad1 = 0; req_exec_quorum_reconfigure._pad2 = 0; req_exec_quorum_reconfigure.header.id = SERVICE_ID_MAKE(VOTEQUORUM_SERVICE, MESSAGE_REQ_EXEC_VOTEQUORUM_RECONFIGURE); req_exec_quorum_reconfigure.header.size = sizeof(req_exec_quorum_reconfigure); iov[0].iov_base = (void *)&req_exec_quorum_reconfigure; iov[0].iov_len = sizeof(req_exec_quorum_reconfigure); ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED); LEAVE(); return ret; } static int votequorum_exec_send_nodeinfo(uint32_t nodeid) { struct req_exec_quorum_nodeinfo req_exec_quorum_nodeinfo; struct iovec iov[1]; struct cluster_node *node; int ret; ENTER(); node = find_node_by_nodeid(nodeid); if (!node) { return -1; } memset(&req_exec_quorum_nodeinfo, 0, sizeof(req_exec_quorum_nodeinfo)); req_exec_quorum_nodeinfo.nodeid = nodeid; req_exec_quorum_nodeinfo.votes = node->votes; req_exec_quorum_nodeinfo.expected_votes = node->expected_votes; req_exec_quorum_nodeinfo.flags = node->flags; if (nodeid != VOTEQUORUM_QDEVICE_NODEID) { decode_flags(node->flags); } req_exec_quorum_nodeinfo.header.id = SERVICE_ID_MAKE(VOTEQUORUM_SERVICE, MESSAGE_REQ_EXEC_VOTEQUORUM_NODEINFO); req_exec_quorum_nodeinfo.header.size = sizeof(req_exec_quorum_nodeinfo); iov[0].iov_base = (void *)&req_exec_quorum_nodeinfo; iov[0].iov_len = sizeof(req_exec_quorum_nodeinfo); ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED); LEAVE(); return ret; } static int votequorum_exec_send_qdevice_reconfigure(const char *oldname, const char *newname) { struct req_exec_quorum_qdevice_reconfigure req_exec_quorum_qdevice_reconfigure; struct iovec iov[1]; int ret; ENTER(); req_exec_quorum_qdevice_reconfigure.header.id = SERVICE_ID_MAKE(VOTEQUORUM_SERVICE, MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_RECONFIGURE); req_exec_quorum_qdevice_reconfigure.header.size = sizeof(req_exec_quorum_qdevice_reconfigure); assert(strlen(oldname) < sizeof(req_exec_quorum_qdevice_reconfigure.oldname)); strcpy(req_exec_quorum_qdevice_reconfigure.oldname, oldname); assert(strlen(newname) < sizeof(req_exec_quorum_qdevice_reconfigure.newname)); strcpy(req_exec_quorum_qdevice_reconfigure.newname, newname); iov[0].iov_base = (void *)&req_exec_quorum_qdevice_reconfigure; iov[0].iov_len = sizeof(req_exec_quorum_qdevice_reconfigure); ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED); LEAVE(); return ret; } static int votequorum_exec_send_qdevice_reg(uint32_t operation, const char *qdevice_name_req) { struct req_exec_quorum_qdevice_reg req_exec_quorum_qdevice_reg; struct iovec iov[1]; int ret; ENTER(); req_exec_quorum_qdevice_reg.header.id = SERVICE_ID_MAKE(VOTEQUORUM_SERVICE, MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_REG); req_exec_quorum_qdevice_reg.header.size = sizeof(req_exec_quorum_qdevice_reg); req_exec_quorum_qdevice_reg.operation = operation; assert(strlen(qdevice_name_req) < sizeof(req_exec_quorum_qdevice_reg.qdevice_name)); strcpy(req_exec_quorum_qdevice_reg.qdevice_name, qdevice_name_req); iov[0].iov_base = (void *)&req_exec_quorum_qdevice_reg; iov[0].iov_len = sizeof(req_exec_quorum_qdevice_reg); ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED); LEAVE(); return ret; } static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context) { struct res_lib_votequorum_quorum_notification *res_lib_votequorum_notification; struct qb_list_head *tmp; struct cluster_node *node; int i = 0; int cluster_members = 0; int size; char buf[sizeof(struct res_lib_votequorum_quorum_notification) + sizeof(struct votequorum_node) * (PROCESSOR_COUNT_MAX + 2)]; ENTER(); log_printf(LOGSYS_LEVEL_DEBUG, "Sending quorum callback, quorate = %d", cluster_is_quorate); qb_list_for_each(tmp, &cluster_members_list) { node = qb_list_entry(tmp, struct cluster_node, list); cluster_members++; } if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) { cluster_members++; } size = sizeof(struct res_lib_votequorum_quorum_notification) + sizeof(struct votequorum_node) * cluster_members; res_lib_votequorum_notification = (struct res_lib_votequorum_quorum_notification *)&buf; res_lib_votequorum_notification->quorate = cluster_is_quorate; res_lib_votequorum_notification->context = context; res_lib_votequorum_notification->node_list_entries = cluster_members; res_lib_votequorum_notification->header.id = MESSAGE_RES_VOTEQUORUM_QUORUM_NOTIFICATION; res_lib_votequorum_notification->header.size = size; res_lib_votequorum_notification->header.error = CS_OK; /* Send all known nodes and their states */ qb_list_for_each(tmp, &cluster_members_list) { node = qb_list_entry(tmp, struct cluster_node, list); res_lib_votequorum_notification->node_list[i].nodeid = node->node_id; res_lib_votequorum_notification->node_list[i++].state = node->state; } if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) { res_lib_votequorum_notification->node_list[i].nodeid = VOTEQUORUM_QDEVICE_NODEID; res_lib_votequorum_notification->node_list[i++].state = qdevice->state; } /* Send it to all interested parties */ if (conn) { int ret = corosync_api->ipc_dispatch_send(conn, &buf, size); LEAVE(); return ret; } else { struct quorum_pd *qpd; qb_list_for_each(tmp, &trackers_list) { qpd = qb_list_entry(tmp, struct quorum_pd, list); res_lib_votequorum_notification->context = qpd->tracking_context; corosync_api->ipc_dispatch_send(qpd->conn, &buf, size); } } LEAVE(); return 0; } static int votequorum_exec_send_nodelist_notification(void *conn, uint64_t context) { struct res_lib_votequorum_nodelist_notification *res_lib_votequorum_notification; int i = 0; int size; struct qb_list_head *tmp; char buf[sizeof(struct res_lib_votequorum_nodelist_notification) + sizeof(uint32_t) * quorum_members_entries]; ENTER(); log_printf(LOGSYS_LEVEL_DEBUG, "Sending nodelist callback. ring_id = " CS_PRI_RING_ID, quorum_ringid.nodeid, quorum_ringid.seq); size = sizeof(struct res_lib_votequorum_nodelist_notification) + sizeof(uint32_t) * quorum_members_entries; res_lib_votequorum_notification = (struct res_lib_votequorum_nodelist_notification *)&buf; res_lib_votequorum_notification->node_list_entries = quorum_members_entries; res_lib_votequorum_notification->ring_id.nodeid = quorum_ringid.nodeid; res_lib_votequorum_notification->ring_id.seq = quorum_ringid.seq; res_lib_votequorum_notification->context = context; for (i=0; inode_list[i] = quorum_members[i]; } res_lib_votequorum_notification->header.id = MESSAGE_RES_VOTEQUORUM_NODELIST_NOTIFICATION; res_lib_votequorum_notification->header.size = size; res_lib_votequorum_notification->header.error = CS_OK; /* Send it to all interested parties */ if (conn) { int ret = corosync_api->ipc_dispatch_send(conn, &buf, size); LEAVE(); return ret; } else { struct quorum_pd *qpd; qb_list_for_each(tmp, &trackers_list) { qpd = qb_list_entry(tmp, struct quorum_pd, list); res_lib_votequorum_notification->context = qpd->tracking_context; corosync_api->ipc_dispatch_send(qpd->conn, &buf, size); } } LEAVE(); return 0; } static void votequorum_exec_send_expectedvotes_notification(void) { struct res_lib_votequorum_expectedvotes_notification res_lib_votequorum_expectedvotes_notification; struct quorum_pd *qpd; struct qb_list_head *tmp; ENTER(); log_printf(LOGSYS_LEVEL_DEBUG, "Sending expected votes callback"); res_lib_votequorum_expectedvotes_notification.header.id = MESSAGE_RES_VOTEQUORUM_EXPECTEDVOTES_NOTIFICATION; res_lib_votequorum_expectedvotes_notification.header.size = sizeof(res_lib_votequorum_expectedvotes_notification); res_lib_votequorum_expectedvotes_notification.header.error = CS_OK; res_lib_votequorum_expectedvotes_notification.expected_votes = us->expected_votes; qb_list_for_each(tmp, &trackers_list) { qpd = qb_list_entry(tmp, struct quorum_pd, list); res_lib_votequorum_expectedvotes_notification.context = qpd->tracking_context; corosync_api->ipc_dispatch_send(qpd->conn, &res_lib_votequorum_expectedvotes_notification, sizeof(struct res_lib_votequorum_expectedvotes_notification)); } LEAVE(); } static void exec_votequorum_qdevice_reconfigure_endian_convert (void *message) { ENTER(); LEAVE(); } static void message_handler_req_exec_votequorum_qdevice_reconfigure ( const void *message, unsigned int nodeid) { const struct req_exec_quorum_qdevice_reconfigure *req_exec_quorum_qdevice_reconfigure = message; ENTER(); log_printf(LOGSYS_LEVEL_DEBUG, "Received qdevice name change req from node " CS_PRI_NODE_ID " [from: %s to: %s]", nodeid, req_exec_quorum_qdevice_reconfigure->oldname, req_exec_quorum_qdevice_reconfigure->newname); if (!strcmp(req_exec_quorum_qdevice_reconfigure->oldname, qdevice_name)) { log_printf(LOGSYS_LEVEL_DEBUG, "Allowing qdevice rename"); memset(qdevice_name, 0, VOTEQUORUM_QDEVICE_MAX_NAME_LEN); strcpy(qdevice_name, req_exec_quorum_qdevice_reconfigure->newname); /* * TODO: notify qdevices about name change? * this is not relevant for now and can wait later on since * qdevices are local only and libvotequorum is not final */ } LEAVE(); } static void exec_votequorum_qdevice_reg_endian_convert (void *message) { struct req_exec_quorum_qdevice_reg *req_exec_quorum_qdevice_reg = message; ENTER(); req_exec_quorum_qdevice_reg->operation = swab32(req_exec_quorum_qdevice_reg->operation); LEAVE(); } static void message_handler_req_exec_votequorum_qdevice_reg ( const void *message, unsigned int nodeid) { const struct req_exec_quorum_qdevice_reg *req_exec_quorum_qdevice_reg = message; struct res_lib_votequorum_status res_lib_votequorum_status; int wipe_qdevice_name = 1; struct cluster_node *node = NULL; struct qb_list_head *tmp; cs_error_t error = CS_OK; ENTER(); log_printf(LOGSYS_LEVEL_DEBUG, "Received qdevice op %u req from node " CS_PRI_NODE_ID " [%s]", req_exec_quorum_qdevice_reg->operation, nodeid, req_exec_quorum_qdevice_reg->qdevice_name); switch(req_exec_quorum_qdevice_reg->operation) { case VOTEQUORUM_QDEVICE_OPERATION_REGISTER: if (nodeid != us->node_id) { if (!strlen(qdevice_name)) { log_printf(LOGSYS_LEVEL_DEBUG, "Remote qdevice name recorded"); strcpy(qdevice_name, req_exec_quorum_qdevice_reg->qdevice_name); } LEAVE(); return; } /* * protect against the case where we broadcast qdevice registration * to new memebers, we receive the message back, but there is no registration * connection in progress */ if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) { LEAVE(); return; } /* * this should NEVER happen */ if (!qdevice_reg_conn) { log_printf(LOGSYS_LEVEL_WARNING, "Unable to determine origin of the qdevice register call!"); LEAVE(); return; } /* * registering our own device in this case */ if (!strlen(qdevice_name)) { strcpy(qdevice_name, req_exec_quorum_qdevice_reg->qdevice_name); } /* * check if it is our device or something else */ if ((!strncmp(req_exec_quorum_qdevice_reg->qdevice_name, qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN))) { us->flags |= NODE_FLAGS_QDEVICE_REGISTERED; votequorum_exec_send_nodeinfo(VOTEQUORUM_QDEVICE_NODEID); votequorum_exec_send_nodeinfo(us->node_id); } else { log_printf(LOGSYS_LEVEL_WARNING, "A new qdevice with different name (new: %s old: %s) is trying to register!", req_exec_quorum_qdevice_reg->qdevice_name, qdevice_name); error = CS_ERR_EXIST; } res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; corosync_api->ipc_response_send(qdevice_reg_conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); qdevice_reg_conn = NULL; break; case VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER: qb_list_for_each(tmp, &cluster_members_list) { node = qb_list_entry(tmp, struct cluster_node, list); if ((node->state == NODESTATE_MEMBER) && (node->flags & NODE_FLAGS_QDEVICE_REGISTERED)) { wipe_qdevice_name = 0; } } if (wipe_qdevice_name) { memset(qdevice_name, 0, VOTEQUORUM_QDEVICE_MAX_NAME_LEN); } break; } LEAVE(); } static void exec_votequorum_nodeinfo_endian_convert (void *message) { struct req_exec_quorum_nodeinfo *nodeinfo = message; ENTER(); nodeinfo->nodeid = swab32(nodeinfo->nodeid); nodeinfo->votes = swab32(nodeinfo->votes); nodeinfo->expected_votes = swab32(nodeinfo->expected_votes); nodeinfo->flags = swab32(nodeinfo->flags); LEAVE(); } static void message_handler_req_exec_votequorum_nodeinfo ( const void *message, unsigned int sender_nodeid) { const struct req_exec_quorum_nodeinfo *req_exec_quorum_nodeinfo = message; struct cluster_node *node = NULL; int old_votes; int old_expected; uint32_t old_flags; nodestate_t old_state; int new_node = 0; int allow_downgrade = 0; int by_node = 0; unsigned int nodeid = req_exec_quorum_nodeinfo->nodeid; ENTER(); log_printf(LOGSYS_LEVEL_DEBUG, "got nodeinfo message from cluster node " CS_PRI_NODE_ID, sender_nodeid); log_printf(LOGSYS_LEVEL_DEBUG, "nodeinfo message[" CS_PRI_NODE_ID "]: votes: %d, expected: %d flags: %d", nodeid, req_exec_quorum_nodeinfo->votes, req_exec_quorum_nodeinfo->expected_votes, req_exec_quorum_nodeinfo->flags); if (nodeid != VOTEQUORUM_QDEVICE_NODEID) { decode_flags(req_exec_quorum_nodeinfo->flags); } node = find_node_by_nodeid(nodeid); if (!node) { node = allocate_node(nodeid); new_node = 1; } if (!node) { corosync_api->error_memory_failure(); LEAVE(); return; } if (new_node) { old_votes = 0; old_expected = 0; old_state = NODESTATE_DEAD; old_flags = 0; + /* Set site ID for new node */ + node->site_id = get_node_site_id(nodeid); } else { old_votes = node->votes; old_expected = node->expected_votes; old_state = node->state; old_flags = node->flags; } if (nodeid == VOTEQUORUM_QDEVICE_NODEID) { struct cluster_node *sender_node = find_node_by_nodeid(sender_nodeid); assert(sender_node != NULL); if ((!cluster_is_quorate) && (sender_node->flags & NODE_FLAGS_QUORATE)) { node->votes = req_exec_quorum_nodeinfo->votes; } else { node->votes = max(node->votes, req_exec_quorum_nodeinfo->votes); } goto recalculate; } /* Update node state */ node->flags = req_exec_quorum_nodeinfo->flags; node->votes = req_exec_quorum_nodeinfo->votes; node->state = NODESTATE_MEMBER; if (node->flags & NODE_FLAGS_LEAVING) { node->state = NODESTATE_LEAVING; allow_downgrade = 1; by_node = 1; } if ((!cluster_is_quorate) && (node->flags & NODE_FLAGS_QUORATE)) { allow_downgrade = 1; us->expected_votes = req_exec_quorum_nodeinfo->expected_votes; } if (node->flags & NODE_FLAGS_QUORATE || (ev_tracking)) { node->expected_votes = req_exec_quorum_nodeinfo->expected_votes; } else { node->expected_votes = us->expected_votes; } if ((last_man_standing) && (node->votes > 1)) { log_printf(LOGSYS_LEVEL_WARNING, "Last Man Standing feature is supported only when all" "cluster nodes votes are set to 1. Disabling LMS."); last_man_standing = 0; if (last_man_standing_timer_set) { corosync_api->timer_delete(last_man_standing_timer); last_man_standing_timer_set = 0; } } recalculate: if ((new_node) || (nodeid == us->node_id) || (node->flags & NODE_FLAGS_FIRST) || (old_votes != node->votes) || (old_expected != node->expected_votes) || (old_flags != node->flags) || (old_state != node->state)) { recalculate_quorum(allow_downgrade, by_node); } if ((wait_for_all) && (!(node->flags & NODE_FLAGS_WFASTATUS)) && (node->flags & NODE_FLAGS_QUORATE)) { update_wait_for_all_status(0); } LEAVE(); } static void exec_votequorum_reconfigure_endian_convert (void *message) { struct req_exec_quorum_reconfigure *reconfigure = message; ENTER(); reconfigure->nodeid = swab32(reconfigure->nodeid); reconfigure->value = swab32(reconfigure->value); LEAVE(); } static void message_handler_req_exec_votequorum_reconfigure ( const void *message, unsigned int nodeid) { const struct req_exec_quorum_reconfigure *req_exec_quorum_reconfigure = message; struct cluster_node *node; ENTER(); log_printf(LOGSYS_LEVEL_DEBUG, "got reconfigure message from cluster node " CS_PRI_NODE_ID " for " CS_PRI_NODE_ID, nodeid, req_exec_quorum_reconfigure->nodeid); switch(req_exec_quorum_reconfigure->param) { case VOTEQUORUM_RECONFIG_PARAM_EXPECTED_VOTES: update_node_expected_votes(req_exec_quorum_reconfigure->value); votequorum_exec_send_expectedvotes_notification(); update_ev_barrier(req_exec_quorum_reconfigure->value); if (ev_tracking) { us->expected_votes = max(us->expected_votes, ev_tracking_barrier); } recalculate_quorum(1, 0); /* Allow decrease */ break; case VOTEQUORUM_RECONFIG_PARAM_NODE_VOTES: node = find_node_by_nodeid(req_exec_quorum_reconfigure->nodeid); if (!node) { LEAVE(); return; } node->votes = req_exec_quorum_reconfigure->value; recalculate_quorum(1, 0); /* Allow decrease */ break; case VOTEQUORUM_RECONFIG_PARAM_CANCEL_WFA: update_wait_for_all_status(0); log_printf(LOGSYS_LEVEL_INFO, "wait_for_all_status reset by user on node " CS_PRI_NODE_ID ".", req_exec_quorum_reconfigure->nodeid); recalculate_quorum(0, 0); break; } LEAVE(); } static int votequorum_exec_exit_fn (void) { int ret = 0; ENTER(); /* * tell the other nodes we are leaving */ if (allow_downscale) { us->flags |= NODE_FLAGS_LEAVING; ret = votequorum_exec_send_nodeinfo(us->node_id); } if ((ev_tracking) && (ev_tracking_fd != -1)) { close(ev_tracking_fd); } LEAVE(); return ret; } static void votequorum_set_icmap_ro_keys(void) { icmap_set_ro_access("quorum.allow_downscale", CS_FALSE, CS_TRUE); icmap_set_ro_access("quorum.wait_for_all", CS_FALSE, CS_TRUE); icmap_set_ro_access("quorum.last_man_standing", CS_FALSE, CS_TRUE); icmap_set_ro_access("quorum.last_man_standing_window", CS_FALSE, CS_TRUE); icmap_set_ro_access("quorum.expected_votes_tracking", CS_FALSE, CS_TRUE); icmap_set_ro_access("quorum.auto_tie_breaker", CS_FALSE, CS_TRUE); icmap_set_ro_access("quorum.auto_tie_breaker_node", CS_FALSE, CS_TRUE); } static char *votequorum_exec_init_fn (struct corosync_api_v1 *api) { char *error = NULL; ENTER(); /* * make sure we start clean */ qb_list_init(&cluster_members_list); qb_list_init(&trackers_list); qdevice = NULL; us = NULL; memset(cluster_nodes, 0, sizeof(cluster_nodes)); /* * Allocate a cluster_node for qdevice */ qdevice = allocate_node(VOTEQUORUM_QDEVICE_NODEID); if (!qdevice) { LEAVE(); return ((char *)"Could not allocate node."); } qdevice->votes = 0; memset(qdevice_name, 0, VOTEQUORUM_QDEVICE_MAX_NAME_LEN); /* * Allocate a cluster_node for us */ us = allocate_node(corosync_api->totem_nodeid_get()); if (!us) { LEAVE(); return ((char *)"Could not allocate node."); } icmap_set_uint32("runtime.votequorum.this_node_id", us->node_id); us->state = NODESTATE_MEMBER; us->votes = 1; us->flags |= NODE_FLAGS_FIRST; + us->site_id = VOTEQUORUM_SITE_UNASSIGNED; error = votequorum_readconfig(VOTEQUORUM_READCONFIG_STARTUP); if (error) { return error; } + + /* Set our site ID from configuration */ + us->site_id = get_node_site_id(us->node_id); + recalculate_quorum(0, 0); /* * Set RO keys in icmap */ votequorum_set_icmap_ro_keys(); /* * Listen for changes */ votequorum_exec_add_config_notification(); /* * Start us off with one node */ votequorum_exec_send_nodeinfo(us->node_id); LEAVE(); return (NULL); } /* * votequorum service core */ static void votequorum_last_man_standing_timer_fn(void *arg) { ENTER(); last_man_standing_timer_set = 0; if (cluster_is_quorate) { recalculate_quorum(1,1); } LEAVE(); } static void votequorum_sync_init ( const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id) { int i, j; int found; int left_nodes; struct cluster_node *node; ENTER(); sync_in_progress = 1; sync_nodeinfo_sent = 0; sync_wait_for_poll_or_timeout = 0; if (member_list_entries > 1) { us->flags &= ~NODE_FLAGS_FIRST; } /* * we don't need to track which nodes have left directly, * since that info is in the node db, but we need to know * if somebody has left for last_man_standing */ left_nodes = 0; for (i = 0; i < quorum_members_entries; i++) { found = 0; for (j = 0; j < member_list_entries; j++) { if (quorum_members[i] == member_list[j]) { found = 1; break; } } if (found == 0) { left_nodes = 1; node = find_node_by_nodeid(quorum_members[i]); if (node) { node->state = NODESTATE_DEAD; } } } if (last_man_standing) { if (((member_list_entries >= quorum) && (left_nodes)) || ((member_list_entries <= quorum) && (auto_tie_breaker != ATB_NONE) && (check_low_node_id_partition() == 1))) { if (last_man_standing_timer_set) { corosync_api->timer_delete(last_man_standing_timer); last_man_standing_timer_set = 0; } corosync_api->timer_add_duration((unsigned long long)last_man_standing_window*1000000, NULL, votequorum_last_man_standing_timer_fn, &last_man_standing_timer); last_man_standing_timer_set = 1; } } memcpy(previous_quorum_members, quorum_members, sizeof(unsigned int) * quorum_members_entries); previous_quorum_members_entries = quorum_members_entries; memcpy(quorum_members, member_list, sizeof(unsigned int) * member_list_entries); quorum_members_entries = member_list_entries; memcpy(&quorum_ringid, ring_id, sizeof(*ring_id)); if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED && us->flags & NODE_FLAGS_QDEVICE_ALIVE) { /* * Reset poll timer. Sync waiting is interrupted on valid qdevice poll or after timeout */ if (qdevice_timer_set) { corosync_api->timer_delete(qdevice_timer); } corosync_api->timer_add_duration((unsigned long long)qdevice_sync_timeout*1000000, qdevice, qdevice_timer_fn, &qdevice_timer); qdevice_timer_set = 1; sync_wait_for_poll_or_timeout = 1; log_printf(LOGSYS_LEVEL_INFO, "waiting for quorum device %s poll (but maximum for %u ms)", qdevice_name, qdevice_sync_timeout); } LEAVE(); } static int votequorum_sync_process (void) { if (!sync_nodeinfo_sent) { votequorum_exec_send_nodeinfo(us->node_id); votequorum_exec_send_nodeinfo(VOTEQUORUM_QDEVICE_NODEID); if (strlen(qdevice_name)) { votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_REGISTER, qdevice_name); } votequorum_exec_send_nodelist_notification(NULL, 0LL); sync_nodeinfo_sent = 1; } if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED && sync_wait_for_poll_or_timeout) { /* * Waiting for qdevice to poll with new ringid or timeout */ return (-1); } return 0; } static void votequorum_sync_activate (void) { recalculate_quorum(0, 0); quorum_callback(quorum_members, quorum_members_entries, cluster_is_quorate, &quorum_ringid); votequorum_exec_send_quorum_notification(NULL, 0L); sync_in_progress = 0; } static void votequorum_sync_abort (void) { } char *votequorum_init(struct corosync_api_v1 *api, quorum_set_quorate_fn_t q_set_quorate_fn) { char *error; ENTER(); if (q_set_quorate_fn == NULL) { return ((char *)"Quorate function not set"); } corosync_api = api; quorum_callback = q_set_quorate_fn; error = corosync_service_link_and_init(corosync_api, &votequorum_service[0]); if (error) { return (error); } LEAVE(); return (NULL); } /* * Library Handler init/fini */ static int quorum_lib_init_fn (void *conn) { struct quorum_pd *pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn); ENTER(); qb_list_init (&pd->list); pd->conn = conn; LEAVE(); return (0); } static int quorum_lib_exit_fn (void *conn) { struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn); ENTER(); if (quorum_pd->tracking_enabled) { qb_list_del (&quorum_pd->list); qb_list_init (&quorum_pd->list); } LEAVE(); return (0); } /* * library internal functions */ static void qdevice_timer_fn(void *arg) { ENTER(); if ((!(us->flags & NODE_FLAGS_QDEVICE_ALIVE)) || (!qdevice_timer_set)) { LEAVE(); return; } us->flags &= ~NODE_FLAGS_QDEVICE_ALIVE; us->flags &= ~NODE_FLAGS_QDEVICE_CAST_VOTE; log_printf(LOGSYS_LEVEL_INFO, "lost contact with quorum device %s", qdevice_name); votequorum_exec_send_nodeinfo(us->node_id); qdevice_timer_set = 0; sync_wait_for_poll_or_timeout = 0; LEAVE(); } /* * Library Handler Functions */ static void message_handler_req_lib_votequorum_getinfo (void *conn, const void *message) { const struct req_lib_votequorum_getinfo *req_lib_votequorum_getinfo = message; struct res_lib_votequorum_getinfo res_lib_votequorum_getinfo; struct cluster_node *node; unsigned int highest_expected = 0; unsigned int total_votes = 0; cs_error_t error = CS_OK; uint32_t nodeid = req_lib_votequorum_getinfo->nodeid; ENTER(); log_printf(LOGSYS_LEVEL_DEBUG, "got getinfo request on %p for node " CS_PRI_NODE_ID, conn, req_lib_votequorum_getinfo->nodeid); if (nodeid == VOTEQUORUM_QDEVICE_NODEID) { nodeid = us->node_id; } node = find_node_by_nodeid(nodeid); if (node) { struct cluster_node *iternode; struct qb_list_head *nodelist; qb_list_for_each(nodelist, &cluster_members_list) { iternode = qb_list_entry(nodelist, struct cluster_node, list); if (iternode->state == NODESTATE_MEMBER) { highest_expected = max(highest_expected, iternode->expected_votes); total_votes += iternode->votes; } } if (node->flags & NODE_FLAGS_QDEVICE_CAST_VOTE) { total_votes += qdevice->votes; } switch(node->state) { case NODESTATE_MEMBER: res_lib_votequorum_getinfo.state = VOTEQUORUM_NODESTATE_MEMBER; break; case NODESTATE_DEAD: res_lib_votequorum_getinfo.state = VOTEQUORUM_NODESTATE_DEAD; break; case NODESTATE_LEAVING: res_lib_votequorum_getinfo.state = VOTEQUORUM_NODESTATE_LEAVING; break; default: res_lib_votequorum_getinfo.state = node->state; break; } res_lib_votequorum_getinfo.state = node->state; res_lib_votequorum_getinfo.votes = node->votes; res_lib_votequorum_getinfo.expected_votes = node->expected_votes; res_lib_votequorum_getinfo.highest_expected = highest_expected; res_lib_votequorum_getinfo.quorum = quorum; res_lib_votequorum_getinfo.total_votes = total_votes; res_lib_votequorum_getinfo.flags = 0; res_lib_votequorum_getinfo.nodeid = node->node_id; if (two_node) { res_lib_votequorum_getinfo.flags |= VOTEQUORUM_INFO_TWONODE; } if (cluster_is_quorate) { res_lib_votequorum_getinfo.flags |= VOTEQUORUM_INFO_QUORATE; } if (wait_for_all) { res_lib_votequorum_getinfo.flags |= VOTEQUORUM_INFO_WAIT_FOR_ALL; } if (last_man_standing) { res_lib_votequorum_getinfo.flags |= VOTEQUORUM_INFO_LAST_MAN_STANDING; } if (auto_tie_breaker != ATB_NONE) { res_lib_votequorum_getinfo.flags |= VOTEQUORUM_INFO_AUTO_TIE_BREAKER; } if (allow_downscale) { res_lib_votequorum_getinfo.flags |= VOTEQUORUM_INFO_ALLOW_DOWNSCALE; } memset(res_lib_votequorum_getinfo.qdevice_name, 0, VOTEQUORUM_QDEVICE_MAX_NAME_LEN); strcpy(res_lib_votequorum_getinfo.qdevice_name, qdevice_name); res_lib_votequorum_getinfo.qdevice_votes = qdevice->votes; if (node->flags & NODE_FLAGS_QDEVICE_REGISTERED) { res_lib_votequorum_getinfo.flags |= VOTEQUORUM_INFO_QDEVICE_REGISTERED; } if (node->flags & NODE_FLAGS_QDEVICE_ALIVE) { res_lib_votequorum_getinfo.flags |= VOTEQUORUM_INFO_QDEVICE_ALIVE; } if (node->flags & NODE_FLAGS_QDEVICE_CAST_VOTE) { res_lib_votequorum_getinfo.flags |= VOTEQUORUM_INFO_QDEVICE_CAST_VOTE; } if (node->flags & NODE_FLAGS_QDEVICE_MASTER_WINS) { res_lib_votequorum_getinfo.flags |= VOTEQUORUM_INFO_QDEVICE_MASTER_WINS; } } else { error = CS_ERR_NOT_EXIST; } res_lib_votequorum_getinfo.header.size = sizeof(res_lib_votequorum_getinfo); res_lib_votequorum_getinfo.header.id = MESSAGE_RES_VOTEQUORUM_GETINFO; res_lib_votequorum_getinfo.header.error = error; corosync_api->ipc_response_send(conn, &res_lib_votequorum_getinfo, sizeof(res_lib_votequorum_getinfo)); log_printf(LOGSYS_LEVEL_DEBUG, "getinfo response error: %d", error); LEAVE(); } static void message_handler_req_lib_votequorum_setexpected (void *conn, const void *message) { const struct req_lib_votequorum_setexpected *req_lib_votequorum_setexpected = message; struct res_lib_votequorum_status res_lib_votequorum_status; cs_error_t error = CS_OK; unsigned int newquorum; unsigned int total_votes; uint8_t allow_downscale_status = 0; ENTER(); allow_downscale_status = allow_downscale; allow_downscale = 0; /* * Validate new expected votes */ newquorum = calculate_quorum(1, req_lib_votequorum_setexpected->expected_votes, &total_votes); allow_downscale = allow_downscale_status; /* * Setting expected_votes < total_votes doesn't make sense. * For quorate cluster prevent cluster to become unquorate. */ if (req_lib_votequorum_setexpected->expected_votes < total_votes || (cluster_is_quorate && (newquorum > total_votes))) { error = CS_ERR_INVALID_PARAM; goto error_exit; } update_node_expected_votes(req_lib_votequorum_setexpected->expected_votes); if (votequorum_exec_send_reconfigure(VOTEQUORUM_RECONFIG_PARAM_EXPECTED_VOTES, us->node_id, req_lib_votequorum_setexpected->expected_votes)) { error = CS_ERR_NO_RESOURCES; } error_exit: res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } static void message_handler_req_lib_votequorum_setvotes (void *conn, const void *message) { const struct req_lib_votequorum_setvotes *req_lib_votequorum_setvotes = message; struct res_lib_votequorum_status res_lib_votequorum_status; struct cluster_node *node; unsigned int newquorum; unsigned int total_votes; unsigned int saved_votes; cs_error_t error = CS_OK; unsigned int nodeid; ENTER(); nodeid = req_lib_votequorum_setvotes->nodeid; node = find_node_by_nodeid(nodeid); if (!node) { error = CS_ERR_NAME_NOT_FOUND; goto error_exit; } /* * Check votes is valid */ saved_votes = node->votes; node->votes = req_lib_votequorum_setvotes->votes; newquorum = calculate_quorum(1, 0, &total_votes); if (newquorum < total_votes / 2 || newquorum > total_votes) { node->votes = saved_votes; error = CS_ERR_INVALID_PARAM; goto error_exit; } if (votequorum_exec_send_reconfigure(VOTEQUORUM_RECONFIG_PARAM_NODE_VOTES, nodeid, req_lib_votequorum_setvotes->votes)) { error = CS_ERR_NO_RESOURCES; } error_exit: res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } static void message_handler_req_lib_votequorum_trackstart (void *conn, const void *message) { const struct req_lib_votequorum_trackstart *req_lib_votequorum_trackstart = message; struct res_lib_votequorum_status res_lib_votequorum_status; struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn); cs_error_t error = CS_OK; ENTER(); /* * If an immediate listing of the current cluster membership * is requested, generate membership list */ if (req_lib_votequorum_trackstart->track_flags & CS_TRACK_CURRENT || req_lib_votequorum_trackstart->track_flags & CS_TRACK_CHANGES) { log_printf(LOGSYS_LEVEL_DEBUG, "sending initial status to %p", conn); votequorum_exec_send_nodelist_notification(conn, req_lib_votequorum_trackstart->context); votequorum_exec_send_quorum_notification(conn, req_lib_votequorum_trackstart->context); } if (quorum_pd->tracking_enabled) { error = CS_ERR_EXIST; goto response_send; } /* * Record requests for tracking */ if (req_lib_votequorum_trackstart->track_flags & CS_TRACK_CHANGES || req_lib_votequorum_trackstart->track_flags & CS_TRACK_CHANGES_ONLY) { quorum_pd->track_flags = req_lib_votequorum_trackstart->track_flags; quorum_pd->tracking_enabled = 1; quorum_pd->tracking_context = req_lib_votequorum_trackstart->context; qb_list_add (&quorum_pd->list, &trackers_list); } response_send: res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } static void message_handler_req_lib_votequorum_trackstop (void *conn, const void *message) { struct res_lib_votequorum_status res_lib_votequorum_status; struct quorum_pd *quorum_pd = (struct quorum_pd *)corosync_api->ipc_private_data_get (conn); int error = CS_OK; ENTER(); if (quorum_pd->tracking_enabled) { error = CS_OK; quorum_pd->tracking_enabled = 0; qb_list_del (&quorum_pd->list); qb_list_init (&quorum_pd->list); } else { error = CS_ERR_NOT_EXIST; } res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } static void message_handler_req_lib_votequorum_qdevice_register (void *conn, const void *message) { const struct req_lib_votequorum_qdevice_register *req_lib_votequorum_qdevice_register = message; struct res_lib_votequorum_status res_lib_votequorum_status; cs_error_t error = CS_OK; ENTER(); if (!qdevice_can_operate) { log_printf(LOGSYS_LEVEL_INFO, "Registration of quorum device is disabled by incorrect corosync.conf. See logs for more information"); error = CS_ERR_ACCESS; goto out; } if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) { if ((!strncmp(req_lib_votequorum_qdevice_register->name, qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN))) { goto out; } else { log_printf(LOGSYS_LEVEL_WARNING, "A new qdevice with different name (new: %s old: %s) is trying to re-register!", req_lib_votequorum_qdevice_register->name, qdevice_name); error = CS_ERR_EXIST; goto out; } } else { if (qdevice_reg_conn != NULL) { log_printf(LOGSYS_LEVEL_WARNING, "Registration request already in progress"); error = CS_ERR_TRY_AGAIN; goto out; } qdevice_reg_conn = conn; if (votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_REGISTER, req_lib_votequorum_qdevice_register->name) != 0) { log_printf(LOGSYS_LEVEL_WARNING, "Unable to send qdevice registration request to cluster"); error = CS_ERR_TRY_AGAIN; qdevice_reg_conn = NULL; } else { LEAVE(); return; } } out: res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } static void message_handler_req_lib_votequorum_qdevice_unregister (void *conn, const void *message) { const struct req_lib_votequorum_qdevice_unregister *req_lib_votequorum_qdevice_unregister = message; struct res_lib_votequorum_status res_lib_votequorum_status; cs_error_t error = CS_OK; ENTER(); if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) { if (strncmp(req_lib_votequorum_qdevice_unregister->name, qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN)) { error = CS_ERR_INVALID_PARAM; goto out; } if (qdevice_timer_set) { corosync_api->timer_delete(qdevice_timer); qdevice_timer_set = 0; sync_wait_for_poll_or_timeout = 0; } us->flags &= ~NODE_FLAGS_QDEVICE_REGISTERED; us->flags &= ~NODE_FLAGS_QDEVICE_ALIVE; us->flags &= ~NODE_FLAGS_QDEVICE_CAST_VOTE; us->flags &= ~NODE_FLAGS_QDEVICE_MASTER_WINS; votequorum_exec_send_nodeinfo(us->node_id); votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER, req_lib_votequorum_qdevice_unregister->name); } else { error = CS_ERR_NOT_EXIST; } out: res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } static void message_handler_req_lib_votequorum_qdevice_update (void *conn, const void *message) { const struct req_lib_votequorum_qdevice_update *req_lib_votequorum_qdevice_update = message; struct res_lib_votequorum_status res_lib_votequorum_status; cs_error_t error = CS_OK; ENTER(); if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) { if (strncmp(req_lib_votequorum_qdevice_update->oldname, qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN)) { error = CS_ERR_INVALID_PARAM; goto out; } votequorum_exec_send_qdevice_reconfigure(req_lib_votequorum_qdevice_update->oldname, req_lib_votequorum_qdevice_update->newname); } else { error = CS_ERR_NOT_EXIST; } out: res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } static void message_handler_req_lib_votequorum_qdevice_poll (void *conn, const void *message) { const struct req_lib_votequorum_qdevice_poll *req_lib_votequorum_qdevice_poll = message; struct res_lib_votequorum_status res_lib_votequorum_status; cs_error_t error = CS_OK; uint32_t oldflags; ENTER(); if (!qdevice_can_operate) { error = CS_ERR_ACCESS; goto out; } if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) { if (!(req_lib_votequorum_qdevice_poll->ring_id.nodeid == quorum_ringid.nodeid && req_lib_votequorum_qdevice_poll->ring_id.seq == quorum_ringid.seq)) { log_printf(LOGSYS_LEVEL_DEBUG, "Received poll ring id (" CS_PRI_RING_ID ") != last sync " "ring id (" CS_PRI_RING_ID "). Ignoring poll call.", req_lib_votequorum_qdevice_poll->ring_id.nodeid, req_lib_votequorum_qdevice_poll->ring_id.seq, quorum_ringid.nodeid, quorum_ringid.seq); error = CS_ERR_MESSAGE_ERROR; goto out; } if (strncmp(req_lib_votequorum_qdevice_poll->name, qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN)) { error = CS_ERR_INVALID_PARAM; goto out; } if (qdevice_timer_set) { corosync_api->timer_delete(qdevice_timer); qdevice_timer_set = 0; } oldflags = us->flags; us->flags |= NODE_FLAGS_QDEVICE_ALIVE; if (req_lib_votequorum_qdevice_poll->cast_vote) { us->flags |= NODE_FLAGS_QDEVICE_CAST_VOTE; } else { us->flags &= ~NODE_FLAGS_QDEVICE_CAST_VOTE; } if (us->flags != oldflags) { votequorum_exec_send_nodeinfo(us->node_id); } corosync_api->timer_add_duration((unsigned long long)qdevice_timeout*1000000, qdevice, qdevice_timer_fn, &qdevice_timer); qdevice_timer_set = 1; sync_wait_for_poll_or_timeout = 0; } else { error = CS_ERR_NOT_EXIST; } out: res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } static void message_handler_req_lib_votequorum_qdevice_master_wins (void *conn, const void *message) { const struct req_lib_votequorum_qdevice_master_wins *req_lib_votequorum_qdevice_master_wins = message; struct res_lib_votequorum_status res_lib_votequorum_status; cs_error_t error = CS_OK; uint32_t oldflags = us->flags; ENTER(); if (!qdevice_can_operate) { error = CS_ERR_ACCESS; goto out; } if (us->flags & NODE_FLAGS_QDEVICE_REGISTERED) { if (strncmp(req_lib_votequorum_qdevice_master_wins->name, qdevice_name, VOTEQUORUM_QDEVICE_MAX_NAME_LEN)) { error = CS_ERR_INVALID_PARAM; goto out; } if (req_lib_votequorum_qdevice_master_wins->allow) { us->flags |= NODE_FLAGS_QDEVICE_MASTER_WINS; } else { us->flags &= ~NODE_FLAGS_QDEVICE_MASTER_WINS; } if (us->flags != oldflags) { votequorum_exec_send_nodeinfo(us->node_id); } update_qdevice_master_wins(req_lib_votequorum_qdevice_master_wins->allow); } else { error = CS_ERR_NOT_EXIST; } out: res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status); res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS; res_lib_votequorum_status.header.error = error; corosync_api->ipc_response_send(conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status)); LEAVE(); } diff --git a/tools/corosync-quorumtool.c b/tools/corosync-quorumtool.c index 199bd209..c6fe0a50 100644 --- a/tools/corosync-quorumtool.c +++ b/tools/corosync-quorumtool.c @@ -1,1023 +1,1078 @@ /* * Copyright (c) 2009-2020 Red Hat, Inc. * * All rights reserved. * * Authors: Christine Caulfield * Fabio M. Di Nitto (fdinitto@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the Red Hat Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #include "util.h" typedef enum { NODEID_FORMAT_DECIMAL, NODEID_FORMAT_HEX } nodeid_format_t; typedef enum { ADDRESS_FORMAT_NAME, ADDRESS_FORMAT_IP } name_format_t; typedef enum { CMD_SHOWNODES, CMD_SHOWSTATUS, CMD_SETVOTES, CMD_SETEXPECTED, CMD_MONITOR, CMD_UNREGISTER_QDEVICE } command_t; typedef enum { SORT_ADDR, SORT_NODEID, SORT_NODENAME } sorttype_t; #define EXIT_NOT_QUORATE 2 /* * global vars */ /* * cmap bits */ static cmap_handle_t cmap_handle; /* * quorum bits */ static void quorum_notification_fn( quorum_handle_t handle, uint32_t quorate, uint64_t ring_id, uint32_t view_list_entries, uint32_t *view_list); static quorum_handle_t q_handle; static uint32_t q_type; static quorum_callbacks_t q_callbacks = { .quorum_notify_fn = quorum_notification_fn }; /* * quorum call back vars */ /* Containing struct to keep votequorum & normal quorum bits together */ typedef struct { struct votequorum_info *vq_info; /* Might be NULL if votequorum not present */ char *name; /* Might be IP address or NULL */ int node_id; /* Always present */ } view_list_entry_t; static view_list_entry_t *g_view_list; static uint32_t g_quorate; static uint64_t g_ring_id; static uint32_t g_ring_id_rep_node; static uint32_t g_view_list_entries; static uint32_t g_called; static uint32_t g_vq_called; static uint32_t g_show_all_addrs = 0; /* * votequorum bits */ static void votequorum_notification_fn( votequorum_handle_t handle, uint64_t context, votequorum_ring_id_t ring_id, uint32_t node_list_entries, uint32_t node_list[]); static votequorum_handle_t v_handle; static votequorum_callbacks_t v_callbacks = { .votequorum_quorum_notify_fn = NULL, .votequorum_expectedvotes_notify_fn = NULL, .votequorum_nodelist_notify_fn = votequorum_notification_fn, }; static uint32_t our_nodeid = 0; /* * cfg bits */ static corosync_cfg_handle_t c_handle; static corosync_cfg_callbacks_t c_callbacks = { .corosync_cfg_shutdown_callback = NULL }; /* * global */ static int machine_parsable = 0; static void show_usage(const char *name) { printf("usage: \n"); printf("%s \n", name); printf("\n"); printf(" options:\n"); printf("\n"); printf(" -s show quorum status\n"); printf(" -m constantly monitor quorum status\n"); printf(" -l list nodes\n"); printf(" -a show all names or addresses for each node\n"); printf(" -p when used with -s or -l, generates machine parsable output\n"); printf(" -v change the number of votes for a node (*)\n"); printf(" -n optional nodeid of node for -v\n"); printf(" -e change expected votes for the cluster (*)\n"); printf(" -H show nodeids in hexadecimal rather than decimal\n"); printf(" -i show node IP addresses instead of the resolved name\n"); printf(" -o order by [a] IP address (default), [n] name, [i] nodeid\n"); printf(" -f forcefully unregister a quorum device *DANGEROUS* (*)\n"); printf(" -h show this help text\n"); printf(" -V show version and exit\n"); printf("\n"); printf(" (*) Starred items only work if votequorum is the quorum provider for corosync\n"); printf("\n"); } static int get_quorum_type(char *quorum_type, size_t quorum_type_len) { int err; char *str = NULL; if ((!quorum_type) || (quorum_type_len <= 0)) { return -1; } if (q_type == QUORUM_FREE) { return -1; } if ((err = cmap_get_string(cmap_handle, "quorum.provider", &str)) != CS_OK) { goto out; } if (!str) { return -1; } strncpy(quorum_type, str, quorum_type_len - 1); free(str); return 0; out: return err; } /* * Returns 1 if 'votequorum' is active. The called then knows that * votequorum calls should work and can provide extra information */ static int using_votequorum(void) { char quorumtype[256]; int using_voteq; memset(quorumtype, 0, sizeof(quorumtype)); if (get_quorum_type(quorumtype, sizeof(quorumtype))) { return -1; } if (strcmp(quorumtype, "corosync_votequorum") == 0) { using_voteq = 1; } else { using_voteq = 0; } return using_voteq; } static int set_votes(uint32_t nodeid, int votes) { int err; if ((err=votequorum_setvotes(v_handle, nodeid, votes)) != CS_OK) { fprintf(stderr, "Unable to set votes %d for nodeid: " CS_PRI_NODE_ID ": %s\n", votes, nodeid, cs_strerror(err)); } return (err == CS_OK ? EXIT_SUCCESS : EXIT_FAILURE); } static int set_expected(int expected_votes) { int err; if ((err=votequorum_setexpected(v_handle, expected_votes)) != CS_OK) { fprintf(stderr, "Unable to set expected votes: %s\n", cs_strerror(err)); } return (err == CS_OK ? EXIT_SUCCESS : EXIT_FAILURE); } /* * node name by nodelist */ static const char *node_name_by_nodelist(uint32_t nodeid) { cmap_iter_handle_t iter; char key_name[CMAP_KEYNAME_MAXLEN + 1]; char tmp_key[CMAP_KEYNAME_MAXLEN + 1]; static char ret_buf[_POSIX_HOST_NAME_MAX]; char *str = NULL; uint32_t node_pos, cur_nodeid; int res = 0; if (cmap_iter_init(cmap_handle, "nodelist.node.", &iter) != CS_OK) { return ""; } memset(ret_buf, 0, sizeof(ret_buf)); while ((cmap_iter_next(cmap_handle, iter, key_name, NULL, NULL)) == CS_OK) { res = sscanf(key_name, "nodelist.node.%u.%s", &node_pos, tmp_key); if (res != 2) { continue; } if (strcmp(tmp_key, "nodeid") != 0) { continue; } snprintf(tmp_key, CMAP_KEYNAME_MAXLEN, "nodelist.node.%u.nodeid", node_pos); if (cmap_get_uint32(cmap_handle, tmp_key, &cur_nodeid) != CS_OK) { continue; } if (cur_nodeid != nodeid) { continue; } snprintf(tmp_key, CMAP_KEYNAME_MAXLEN, "nodelist.node.%u.name", node_pos); if (cmap_get_string(cmap_handle, tmp_key, &str) != CS_OK) { continue; } if (!str) { continue; } strncpy(ret_buf, str, sizeof(ret_buf) - 1); free(str); break; } cmap_iter_finalize(cmap_handle, iter); return ret_buf; } /* * This resolves the first address assigned to a node * and returns the name or IP address. Use cfgtool if you need more information. */ static const char *node_name(uint32_t nodeid, name_format_t name_format) { int err; int numaddrs; corosync_cfg_node_address_t addrs[INTERFACE_MAX]; static char buf[(INET6_ADDRSTRLEN + 1) * KNET_MAX_LINK]; const char *nodelist_name = NULL; socklen_t addrlen; struct sockaddr_storage *ss; int start_addr = 0; int i; int bufptr = 0; buf[0] = '\0'; /* If a name is required, always look for the nodelist node0_addr name first */ if (name_format == ADDRESS_FORMAT_NAME) { nodelist_name = node_name_by_nodelist(nodeid); } if ((nodelist_name) && (strlen(nodelist_name) > 0)) { start_addr = 1; assert(strlen(nodelist_name) < sizeof(buf)); strcpy(buf, nodelist_name); bufptr = strlen(buf); } err = corosync_cfg_get_node_addrs(c_handle, nodeid, INTERFACE_MAX, &numaddrs, addrs); if (err != CS_OK) { fprintf(stderr, "Unable to get node address for nodeid " CS_PRI_NODE_ID ": %s\n", nodeid, cs_strerror(err)); return ""; } /* Don't show all addressess */ if (!g_show_all_addrs) { numaddrs = 1; } for (i=start_addr; iss_family) { continue; } if (ss->ss_family == AF_INET6) { addrlen = sizeof(struct sockaddr_in6); } else { addrlen = sizeof(struct sockaddr_in); } if (i) { buf[bufptr++] = ','; buf[bufptr++] = ' '; } if (!getnameinfo( (struct sockaddr *)addrs[i].address, addrlen, buf+bufptr, sizeof(buf)-bufptr, NULL, 0, (name_format == ADDRESS_FORMAT_IP)?NI_NUMERICHOST:0)) { bufptr += strlen(buf+bufptr); } } return buf; } static void votequorum_notification_fn( votequorum_handle_t handle, uint64_t context, votequorum_ring_id_t ring_id, uint32_t node_list_entries, uint32_t node_list[]) { g_ring_id_rep_node = ring_id.nodeid; g_vq_called = 1; } static void quorum_notification_fn( quorum_handle_t handle, uint32_t quorate, uint64_t ring_id, uint32_t view_list_entries, uint32_t *view_list) { int i; g_called = 1; g_quorate = quorate; g_ring_id = ring_id; g_view_list_entries = view_list_entries; if (g_view_list) { free(g_view_list); } g_view_list = malloc(sizeof(view_list_entry_t) * view_list_entries); if (g_view_list) { for (i=0; i< view_list_entries; i++) { g_view_list[i].node_id = view_list[i]; g_view_list[i].name = NULL; g_view_list[i].vq_info = NULL; } } } static void print_string_padded(const char *buf) { int len, delta; len = strlen(buf); delta = 10 - len; while (delta > 0) { printf(" "); delta--; } printf("%s ", buf); } static void print_uint32_padded(uint32_t value) { char buf[12]; snprintf(buf, sizeof(buf) - 1, "%u", value); print_string_padded(buf); } /* for qsort */ static int compare_nodeids(const void *one, const void *two) { const view_list_entry_t *info1 = one; const view_list_entry_t *info2 = two; if (info1->node_id == info2->node_id) { return 0; } if (info1->node_id > info2->node_id) { return 1; } return -1; } static int compare_nodenames(const void *one, const void *two) { const view_list_entry_t *info1 = one; const view_list_entry_t *info2 = two; return strcmp(info1->name, info2->name); } static void display_nodes_data(nodeid_format_t nodeid_format, name_format_t name_format, sorttype_t sort_type) { int i, display_qdevice = 0; unsigned int our_flags = 0; struct votequorum_info info[g_view_list_entries]; /* * cache node info because we need to parse them twice */ if (v_handle) { for (i=0; i < g_view_list_entries; i++) { if (votequorum_getinfo(v_handle, g_view_list[i].node_id, &info[i]) != CS_OK) { printf("Unable to get node " CS_PRI_NODE_ID " info\n", g_view_list[i].node_id); } g_view_list[i].vq_info = &info[i]; if (info[i].flags & VOTEQUORUM_INFO_QDEVICE_REGISTERED) { display_qdevice = 1; } } } /* * Get node names */ for (i=0; i < g_view_list_entries; i++) { g_view_list[i].name = strdup(node_name(g_view_list[i].node_id, name_format)); } printf("\nMembership information\n"); printf("----------------------\n"); print_string_padded("Nodeid"); if (v_handle) { print_string_padded("Votes"); if ((display_qdevice) || (machine_parsable)) { print_string_padded("Qdevice"); } } - printf("Name\n"); + printf("Name"); + /* Check if site_aware is enabled to display site column */ + uint8_t site_aware = 0; + if (cmap_get_uint8(cmap_handle, "runtime.votequorum.site_aware", &site_aware) == CS_OK && site_aware) { + printf("%*s", 20, "Site"); + } + printf("\n"); /* corosync sends them already sorted by address */ if (sort_type == SORT_NODEID) { qsort(g_view_list, g_view_list_entries, sizeof(view_list_entry_t), compare_nodeids); } if (sort_type == SORT_NODENAME) { qsort(g_view_list, g_view_list_entries, sizeof(view_list_entry_t), compare_nodenames); } for (i=0; i < g_view_list_entries; i++) { if (nodeid_format == NODEID_FORMAT_DECIMAL) { print_uint32_padded(g_view_list[i].node_id); } else { printf("0x%08x ", g_view_list[i].node_id); } if (v_handle) { int votes = -1; votes = info[i].node_votes; print_uint32_padded(votes); if ((display_qdevice) || (machine_parsable)) { if (info[i].flags & VOTEQUORUM_INFO_QDEVICE_REGISTERED) { char buf[10]; snprintf(buf, sizeof(buf), "%s,%s,%s", info[i].flags & VOTEQUORUM_INFO_QDEVICE_ALIVE?"A":"NA", info[i].flags & VOTEQUORUM_INFO_QDEVICE_CAST_VOTE?"V":"NV", info[i].flags & VOTEQUORUM_INFO_QDEVICE_MASTER_WINS?"MW":"NMW"); print_string_padded(buf); } else { print_string_padded("NR"); } } } printf("%s", g_view_list[i].name); if (g_view_list[i].node_id == our_nodeid) { printf(" (local)"); if (v_handle) { our_flags = info[i].flags; } } + /* Display site ID if site_aware is enabled */ + if (site_aware) { + char key[ICMAP_KEYNAME_MAXLEN]; + uint32_t node_site_id; + int site_col_offset; + + /* Calculate padding needed */ + site_col_offset = 50 - strlen(g_view_list[i].name); + if (g_view_list[i].node_id == our_nodeid) { + site_col_offset -= 8; /* " (local)" */ + } + if (site_col_offset > 0) { + printf("%*s", site_col_offset, ""); + } + + snprintf(key, ICMAP_KEYNAME_MAXLEN, "nodelist.node.%d.site", i); + if (cmap_get_uint32(cmap_handle, key, &node_site_id) == CS_OK) { + printf("%u", node_site_id); + } else { + printf("N/A"); + } + } printf("\n"); } if (g_view_list_entries) { for (i=0; i < g_view_list_entries; i++) { free(g_view_list[i].name); } free(g_view_list); g_view_list = NULL; } if (display_qdevice) { if (nodeid_format == NODEID_FORMAT_DECIMAL) { print_uint32_padded(VOTEQUORUM_QDEVICE_NODEID); } else { printf("0x%08x ", VOTEQUORUM_QDEVICE_NODEID); } /* If the quorum device is inactive on this node then show votes as 0 so that the display is not confusing */ if (our_flags & VOTEQUORUM_INFO_QDEVICE_CAST_VOTE) { print_uint32_padded(info[0].qdevice_votes); } else { print_uint32_padded(0); } printf(" %s", info[0].qdevice_name); if (our_flags & VOTEQUORUM_INFO_QDEVICE_CAST_VOTE) { printf("\n"); } else { printf(" (votes %d)\n", info[0].qdevice_votes); } } } static int display_quorum_data(int is_quorate, nodeid_format_t nodeid_format, name_format_t name_format, sorttype_t sort_type, int loop) { struct votequorum_info info; int err; char quorumtype[256]; time_t t; memset(quorumtype, 0, sizeof(quorumtype)); printf("Quorum information\n"); printf("------------------\n"); time(&t); printf("Date: %s", ctime((const time_t *)&t)); if (get_quorum_type(quorumtype, sizeof(quorumtype))) { strncpy(quorumtype, "Not configured", sizeof(quorumtype) - 1); } printf("Quorum provider: %s\n", quorumtype); printf("Nodes: %d\n", g_view_list_entries); if (nodeid_format == NODEID_FORMAT_DECIMAL) { printf("Node ID: " CS_PRI_NODE_ID "\n", our_nodeid); } else { printf("Node ID: 0x%08x\n", our_nodeid); } if (v_handle) { printf("Ring ID: " CS_PRI_RING_ID "\n", g_ring_id_rep_node, g_ring_id); } else { printf("Ring ID: " CS_PRI_RING_ID_SEQ "\n", g_ring_id); } printf("Quorate: %s\n", is_quorate?"Yes":"No"); if (!v_handle) { return CS_OK; } err=votequorum_getinfo(v_handle, our_nodeid, &info); if ((err == CS_OK) || (err == CS_ERR_NOT_EXIST)) { printf("\nVotequorum information\n"); printf("----------------------\n"); printf("Expected votes: %d\n", info.node_expected_votes); printf("Highest expected: %d\n", info.highest_expected); printf("Total votes: %d\n", info.total_votes); printf("Quorum: %d %s\n", info.quorum, info.flags & VOTEQUORUM_INFO_QUORATE?" ":"Activity blocked"); printf("Flags: "); if (info.flags & VOTEQUORUM_INFO_TWONODE) printf("2Node "); if (info.flags & VOTEQUORUM_INFO_QUORATE) printf("Quorate "); if (info.flags & VOTEQUORUM_INFO_WAIT_FOR_ALL) printf("WaitForAll "); if (info.flags & VOTEQUORUM_INFO_LAST_MAN_STANDING) printf("LastManStanding "); if (info.flags & VOTEQUORUM_INFO_AUTO_TIE_BREAKER) printf("AutoTieBreaker "); if (info.flags & VOTEQUORUM_INFO_ALLOW_DOWNSCALE) printf("AllowDownscale "); if (info.flags & VOTEQUORUM_INFO_QDEVICE_REGISTERED) printf("Qdevice "); printf("\n"); + + /* Display site-aware quorum information if enabled */ + uint8_t site_aware_status = 0; + uint32_t num_sites = 0; + if (cmap_get_uint8(cmap_handle, "runtime.votequorum.site_aware", &site_aware_status) == CS_OK && + site_aware_status) { + printf("\nSite-aware quorum information\n"); + printf("-----------------------------\n"); + printf("Site-aware: Yes\n"); + + if (cmap_get_uint32(cmap_handle, "runtime.votequorum.num_sites", &num_sites) == CS_OK) { + uint32_t site; + printf("Number of sites: %u\n", num_sites); + + for (site = 0; site < num_sites; site++) { + char key[ICMAP_KEYNAME_MAXLEN]; + uint32_t expected_nodes = 0; + + snprintf(key, ICMAP_KEYNAME_MAXLEN, + "runtime.votequorum.site.%u.expected_nodes", site); + if (cmap_get_uint32(cmap_handle, key, &expected_nodes) == CS_OK) { + printf("Site %u nodes: %u\n", site, expected_nodes); + } + } + } + } } else { fprintf(stderr, "Unable to get node info: %s\n", cs_strerror(err)); } display_nodes_data(nodeid_format, name_format, sort_type); return err; } /* * return EXIT_SUCCESS if quorate * EXIT_NOT_QUORATE if not quorate * EXIT_FAILURE on error */ static int show_status(nodeid_format_t nodeid_format, name_format_t name_format, sorttype_t sort_type) { int is_quorate; int err; err=quorum_getquorate(q_handle, &is_quorate); if (err != CS_OK) { fprintf(stderr, "Unable to get cluster quorate status: %s\n", cs_strerror(err)); goto quorum_err; } err=quorum_trackstart(q_handle, CS_TRACK_CURRENT); if (err != CS_OK) { fprintf(stderr, "Unable to start quorum status tracking: %s\n", cs_strerror(err)); goto quorum_err; } g_called = 0; while (g_called == 0 && err == CS_OK) { err = quorum_dispatch(q_handle, CS_DISPATCH_ONE); if (err != CS_OK) { fprintf(stderr, "Unable to dispatch quorum status: %s\n", cs_strerror(err)); } } if (quorum_trackstop(q_handle) != CS_OK) { fprintf(stderr, "Unable to stop quorum status tracking: %s\n", cs_strerror(err)); } if (using_votequorum()) { if ( (err=votequorum_trackstart(v_handle, 0LL, CS_TRACK_CURRENT)) != CS_OK) { fprintf(stderr, "Unable to start votequorum status tracking: %s\n", cs_strerror(err)); goto quorum_err; } g_vq_called = 0; while (g_vq_called == 0 && err == CS_OK) { err = votequorum_dispatch(v_handle, CS_DISPATCH_ONE); if (err != CS_OK) { fprintf(stderr, "Unable to dispatch votequorum status: %s\n", cs_strerror(err)); } } } quorum_err: if (err != CS_OK) { return EXIT_FAILURE; } err = display_quorum_data(is_quorate, nodeid_format, name_format, sort_type, 0); if (err != CS_OK) { return EXIT_FAILURE; } return (is_quorate ? EXIT_SUCCESS : EXIT_NOT_QUORATE); } static int monitor_status(nodeid_format_t nodeid_format, name_format_t name_format, sorttype_t sort_type) { int err; int loop = 0; if (q_type == QUORUM_FREE) { printf("\nQuorum is not configured - cannot monitor\n"); return show_status(nodeid_format, name_format, sort_type); } err=quorum_trackstart(q_handle, CS_TRACK_CHANGES); if (err != CS_OK) { fprintf(stderr, "Unable to start quorum status tracking: %s\n", cs_strerror(err)); goto quorum_err; } if (using_votequorum()) { if ( (err=votequorum_trackstart(v_handle, 0LL, CS_TRACK_CHANGES)) != CS_OK) { fprintf(stderr, "Unable to start votequorum status tracking: %s\n", cs_strerror(err)); goto quorum_err; } } while (1) { err = quorum_dispatch(q_handle, CS_DISPATCH_ONE); if (err != CS_OK) { fprintf(stderr, "Unable to dispatch quorum status: %s\n", cs_strerror(err)); goto quorum_err; } if (using_votequorum()) { g_vq_called = 0; while (!g_vq_called) { err = votequorum_dispatch(v_handle, CS_DISPATCH_ONE); if (err != CS_OK) { fprintf(stderr, "Unable to dispatch votequorum status: %s\n", cs_strerror(err)); goto quorum_err; } } } err = display_quorum_data(g_quorate, nodeid_format, name_format, sort_type, loop); printf("\n"); loop = 1; if (err != CS_OK) { fprintf(stderr, "Unable to display quorum data: %s\n", cs_strerror(err)); goto quorum_err; } } quorum_err: return EXIT_FAILURE; } static int show_nodes(nodeid_format_t nodeid_format, name_format_t name_format, sorttype_t sort_type) { int err; int result = EXIT_FAILURE; err = quorum_trackstart(q_handle, CS_TRACK_CURRENT); if (err != CS_OK) { fprintf(stderr, "Unable to start quorum status tracking: %s\n", cs_strerror(err)); goto err_exit; } g_called = 0; while (g_called == 0) { err = quorum_dispatch(q_handle, CS_DISPATCH_ONE); if (err != CS_OK) { fprintf(stderr, "Unable to dispatch quorum status: %s\n", cs_strerror(err)); goto err_exit; } } display_nodes_data(nodeid_format, name_format, sort_type); result = EXIT_SUCCESS; err_exit: return result; } static int unregister_qdevice(void) { int err; struct votequorum_info info; int result; result = EXIT_FAILURE; err = votequorum_getinfo(v_handle, our_nodeid, &info); if (err != CS_OK) { fprintf(stderr, "Unable to get quorum device info: %s\n", cs_strerror(err)); goto err_exit; } if (!(info.flags & VOTEQUORUM_INFO_QDEVICE_REGISTERED)) { result = EXIT_SUCCESS; goto err_exit; } err = votequorum_qdevice_unregister(v_handle, info.qdevice_name); if (err != CS_OK) { fprintf(stderr, "Unable to unregister quorum device: %s\n", cs_strerror(err)); goto err_exit; } result = EXIT_SUCCESS; err_exit: return result; } /* * return -1 on error * 0 if OK */ static int init_all(void) { cmap_handle = 0; q_handle = 0; v_handle = 0; c_handle = 0; if (cmap_initialize(&cmap_handle) != CS_OK) { fprintf(stderr, "Cannot initialize CMAP service\n"); cmap_handle = 0; goto out; } if (quorum_initialize(&q_handle, &q_callbacks, &q_type) != CS_OK) { fprintf(stderr, "Cannot initialize QUORUM service\n"); q_handle = 0; goto out; } if (corosync_cfg_initialize(&c_handle, &c_callbacks) != CS_OK) { fprintf(stderr, "Cannot initialise CFG service\n"); c_handle = 0; goto out; } if (using_votequorum() <= 0) { return 0; } if (votequorum_initialize(&v_handle, &v_callbacks) != CS_OK) { fprintf(stderr, "Cannot initialise VOTEQUORUM service\n"); v_handle = 0; goto out; } if (cmap_get_uint32(cmap_handle, "runtime.votequorum.this_node_id", &our_nodeid) != CS_OK) { fprintf(stderr, "Unable to retrieve this_node_id\n"); goto out; } return 0; out: return -1; } static void close_all(void) { if (cmap_handle) { cmap_finalize(cmap_handle); } if (q_handle) { quorum_finalize(q_handle); } if (c_handle) { corosync_cfg_finalize(c_handle); } if (v_handle) { votequorum_finalize(v_handle); } } int main (int argc, char *argv[]) { const char *options = "VHaslpmfe:v:hin:o:"; int opt; int votes = 0; int ret = 0; uint32_t nodeid = 0; uint32_t nodeid_set = 0; nodeid_format_t nodeid_format = NODEID_FORMAT_DECIMAL; name_format_t address_format = ADDRESS_FORMAT_NAME; command_t command_opt = CMD_SHOWSTATUS; sorttype_t sort_opt = SORT_ADDR; long long int l; while ( (opt = getopt(argc, argv, options)) != -1 ) { switch (opt) { case 'f': command_opt = CMD_UNREGISTER_QDEVICE; break; case 's': command_opt = CMD_SHOWSTATUS; break; case 'a': g_show_all_addrs = 1; break; case 'm': command_opt = CMD_MONITOR; break; case 'i': address_format = ADDRESS_FORMAT_IP; break; case 'H': nodeid_format = NODEID_FORMAT_HEX; break; case 'l': command_opt = CMD_SHOWNODES; break; case 'p': machine_parsable = 1; break; case 'e': if (util_strtonum(optarg, 1, INT_MAX, &l) == -1) { fprintf(stderr, "New expected votes value was not valid, try a positive number\n"); exit(EXIT_FAILURE); } votes = l; command_opt = CMD_SETEXPECTED; break; case 'n': if (util_strtonum(optarg, 1, UINT_MAX, &l) == -1) { fprintf(stderr, "The nodeid was not valid, try a positive number\n"); exit(EXIT_FAILURE); } nodeid = l; nodeid_set = 1; break; case 'v': if (util_strtonum(optarg, 0, INT_MAX, &l) == -1) { fprintf(stderr, "New votes value was not valid, try a positive number or zero\n"); exit(EXIT_FAILURE); } votes = l; command_opt = CMD_SETVOTES; break; case 'o': if (strcmp(optarg, "a") == 0) { sort_opt = SORT_ADDR; } else if (strcmp(optarg, "i") == 0) { sort_opt = SORT_NODEID; } else if (strcmp(optarg, "n") == 0) { sort_opt = SORT_NODENAME; } else { fprintf(stderr, "Invalid ordering option. valid orders are a(address), i(node ID) or n(name)\n"); exit(EXIT_FAILURE); } break; case 'V': printf("corosync-quorumtool version: %s\n", VERSION); exit(EXIT_SUCCESS); break; case 'h': show_usage(argv[0]); exit(EXIT_SUCCESS); break; case ':': case '?': default: show_usage(argv[0]); exit(EXIT_FAILURE); break; } } if (init_all()) { close_all(); exit(EXIT_FAILURE); } switch (command_opt) { case CMD_SHOWNODES: ret = show_nodes(nodeid_format, address_format, sort_opt); break; case CMD_SHOWSTATUS: ret = show_status(nodeid_format, address_format, sort_opt); break; case CMD_SETVOTES: if (using_votequorum() > 0) { if (!nodeid_set) { nodeid = our_nodeid; } ret = set_votes(nodeid, votes); } else { fprintf(stderr, "You cannot change node votes, corosync is not using votequorum\n"); ret = EXIT_FAILURE; } break; case CMD_SETEXPECTED: if (using_votequorum() > 0) { ret = set_expected(votes); } else { fprintf(stderr, "You cannot change expected votes, corosync is not using votequorum\n"); ret = EXIT_FAILURE; } break; case CMD_MONITOR: ret = monitor_status(nodeid_format, address_format, sort_opt); break; case CMD_UNREGISTER_QDEVICE: if (using_votequorum() > 0) { ret = unregister_qdevice(); } else { fprintf(stderr, "You cannot unregister quorum device, corosync is not using votequorum\n"); ret = EXIT_FAILURE; } break; } close_all(); return (ret); }