diff --git a/qdevices/qdevice-cmap.c b/qdevices/qdevice-cmap.c index bdd7c77a..ebf9353c 100644 --- a/qdevices/qdevice-cmap.c +++ b/qdevices/qdevice-cmap.c @@ -1,458 +1,465 @@ /* * Copyright (c) 2015-2016 Red Hat, Inc. * * All rights reserved. * * Author: Jan Friesse (jfriesse@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the Red Hat, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include "qdevice-config.h" #include "qdevice-cmap.h" #include "qdevice-log.h" #include "qdevice-log-debug.h" #include "qdevice-model.h" #include "utils.h" static uint32_t qdevice_cmap_autogenerate_node_id(const char *addr, int clear_node_high_byte) { struct addrinfo *ainfo; struct addrinfo ahints; int ret, i; memset(&ahints, 0, sizeof(ahints)); ahints.ai_socktype = SOCK_DGRAM; ahints.ai_protocol = IPPROTO_UDP; /* * Hardcoded AF_INET because autogenerated nodeid is valid only for ipv4 */ ahints.ai_family = AF_INET; ret = getaddrinfo(addr, NULL, &ahints, &ainfo); if (ret != 0) return (0); if (ainfo->ai_family != AF_INET) { freeaddrinfo(ainfo); return (0); } memcpy(&i, &((struct sockaddr_in *)ainfo->ai_addr)->sin_addr, sizeof(struct in_addr)); freeaddrinfo(ainfo); ret = htonl(i); if (clear_node_high_byte) { ret &= 0x7FFFFFFF; } return (ret); } int qdevice_cmap_get_nodelist(cmap_handle_t cmap_handle, struct node_list *list) { cs_error_t cs_err; cmap_iter_handle_t iter_handle; char key_name[CMAP_KEYNAME_MAXLEN + 1]; char tmp_key[CMAP_KEYNAME_MAXLEN + 1]; int res; int ret_value; unsigned int node_pos; uint32_t node_id; uint32_t data_center_id; char *tmp_str; char *addr0_str; int clear_node_high_byte; ret_value = 0; node_list_init(list); cs_err = cmap_iter_init(cmap_handle, "nodelist.node.", &iter_handle); if (cs_err != CS_OK) { return (-1); } while ((cs_err = cmap_iter_next(cmap_handle, iter_handle, key_name, NULL, NULL)) == CS_OK) { res = sscanf(key_name, "nodelist.node.%u.%s", &node_pos, tmp_key); if (res != 2) { continue; } if (strcmp(tmp_key, "ring0_addr") != 0) { continue; } snprintf(tmp_key, CMAP_KEYNAME_MAXLEN, "nodelist.node.%u.nodeid", node_pos); cs_err = cmap_get_uint32(cmap_handle, tmp_key, &node_id); if (cs_err == CS_ERR_NOT_EXIST) { /* * Nodeid doesn't exists -> autogenerate node id */ clear_node_high_byte = 0; if (cmap_get_string(cmap_handle, "totem.clear_node_high_bit", &tmp_str) == CS_OK) { if (strcmp (tmp_str, "yes") == 0) { clear_node_high_byte = 1; } free(tmp_str); } if (cmap_get_string(cmap_handle, key_name, &addr0_str) != CS_OK) { return (-1); } node_id = qdevice_cmap_autogenerate_node_id(addr0_str, clear_node_high_byte); free(addr0_str); } else if (cs_err != CS_OK) { ret_value = -1; goto iter_finalize; } snprintf(tmp_key, CMAP_KEYNAME_MAXLEN, "nodelist.node.%u.datacenterid", node_pos); if (cmap_get_uint32(cmap_handle, tmp_key, &data_center_id) != CS_OK) { data_center_id = 0; } if (node_list_add(list, node_id, data_center_id, TLV_NODE_STATE_NOT_SET) == NULL) { ret_value = -1; goto iter_finalize; } } iter_finalize: cmap_iter_finalize(cmap_handle, iter_handle); if (ret_value != 0) { node_list_free(list); } return (ret_value); } int qdevice_cmap_get_config_version(cmap_handle_t cmap_handle, uint64_t *config_version) { int res; if (cmap_get_uint64(cmap_handle, "totem.config_version", config_version) == CS_OK) { res = 0; } else { *config_version = 0; res = -1; } return (res); } int qdevice_cmap_store_config_node_list(struct qdevice_instance *instance) { int res; node_list_free(&instance->config_node_list); if (qdevice_cmap_get_nodelist(instance->cmap_handle, &instance->config_node_list) != 0) { qdevice_log(LOG_ERR, "Can't get configuration node list."); return (-1); } res = qdevice_cmap_get_config_version(instance->cmap_handle, &instance->config_node_list_version); instance->config_node_list_version_set = (res == 0); return (0); } void qdevice_cmap_init(struct qdevice_instance *instance) { cs_error_t res; int no_retries; no_retries = 0; while ((res = cmap_initialize(&instance->cmap_handle)) == CS_ERR_TRY_AGAIN && no_retries++ < instance->advanced_settings->max_cs_try_again) { (void)poll(NULL, 0, 1000); } if (res != CS_OK) { errx(1, "Failed to initialize the cmap API. Error %s", cs_strerror(res)); } if ((res = cmap_context_set(instance->cmap_handle, (void *)instance)) != CS_OK) { errx(1, "Can't set cmap context. Error %s", cs_strerror(res)); } cmap_fd_get(instance->cmap_handle, &instance->cmap_poll_fd); } static void qdevice_cmap_node_list_event(struct qdevice_instance *instance) { struct node_list nlist; int config_version_set; uint64_t config_version; qdevice_log(LOG_DEBUG, "Node list configuration possibly changed"); if (qdevice_cmap_get_nodelist(instance->cmap_handle, &nlist) != 0) { qdevice_log(LOG_ERR, "Can't get configuration node list."); if (qdevice_model_get_config_node_list_failed(instance) != 0) { qdevice_log(LOG_DEBUG, "qdevice_model_get_config_node_list_failed returned error -> exit"); exit(2); } return ; } config_version_set = (qdevice_cmap_get_config_version(instance->cmap_handle, &config_version) == 0); if (node_list_eq(&instance->config_node_list, &nlist)) { return ; } qdevice_log(LOG_DEBUG, "Node list changed"); if (config_version_set) { qdevice_log(LOG_DEBUG, " config_version = "UTILS_PRI_CONFIG_VERSION, config_version); } qdevice_log_debug_dump_node_list(&nlist); if (qdevice_model_config_node_list_changed(instance, &nlist, config_version_set, config_version) != 0) { qdevice_log(LOG_DEBUG, "qdevice_model_config_node_list_changed returned error -> exit"); exit(2); } node_list_free(&instance->config_node_list); if (node_list_clone(&instance->config_node_list, &nlist) != 0) { qdevice_log(LOG_ERR, "Can't allocate instance->config_node_list clone"); node_list_free(&nlist); if (qdevice_model_get_config_node_list_failed(instance) != 0) { qdevice_log(LOG_DEBUG, "qdevice_model_get_config_node_list_failed returned error -> exit"); exit(2); } return ; } instance->config_node_list_version_set = config_version_set; if (config_version_set) { instance->config_node_list_version = config_version; } } static void qdevice_cmap_logging_event(struct qdevice_instance *instance) { qdevice_log(LOG_DEBUG, "Logging configuration possibly changed"); qdevice_log_configure(instance); } static void qdevice_cmap_reload_cb(cmap_handle_t cmap_handle, cmap_track_handle_t cmap_track_handle, int32_t event, const char *key_name, struct cmap_notify_value new_value, struct cmap_notify_value old_value, void *user_data) { cs_error_t cs_res; uint8_t reload; struct qdevice_instance *instance; int node_list_event; int logging_event; const char *node_list_prefix_str; const char *logging_prefix_str; node_list_event = 0; logging_event = 0; node_list_prefix_str = "nodelist."; logging_prefix_str = "logging."; if (cmap_context_get(cmap_handle, (const void **)&instance) != CS_OK) { qdevice_log(LOG_ERR, "Fatal error. Can't get cmap context"); exit(1); } /* * Wait for full reload */ if (strcmp(key_name, "config.totemconfig_reload_in_progress") == 0 && new_value.type == CMAP_VALUETYPE_UINT8 && new_value.len == sizeof(reload)) { reload = 1; if (memcmp(new_value.data, &reload, sizeof(reload)) == 0) { /* * Ignore nodelist changes */ instance->cmap_reload_in_progress = 1; return ; } else { instance->cmap_reload_in_progress = 0; node_list_event = 1; logging_event = 1; } } if (instance->cmap_reload_in_progress) { return ; } if (((cs_res = cmap_get_uint8(cmap_handle, "config.totemconfig_reload_in_progress", &reload)) == CS_OK) && reload == 1) { return ; } if (strncmp(key_name, node_list_prefix_str, strlen(node_list_prefix_str)) == 0) { node_list_event = 1; } if (strncmp(key_name, logging_prefix_str, strlen(logging_prefix_str)) == 0) { logging_event = 1; } if (logging_event) { qdevice_cmap_logging_event(instance); } if (node_list_event) { qdevice_cmap_node_list_event(instance); } } int qdevice_cmap_add_track(struct qdevice_instance *instance) { cs_error_t res; res = cmap_track_add(instance->cmap_handle, "config.totemconfig_reload_in_progress", CMAP_TRACK_ADD | CMAP_TRACK_MODIFY, qdevice_cmap_reload_cb, NULL, &instance->cmap_reload_track_handle); if (res != CS_OK) { qdevice_log(LOG_ERR, "Can't initialize cmap totemconfig_reload_in_progress tracking"); return (-1); } res = cmap_track_add(instance->cmap_handle, "nodelist.", CMAP_TRACK_ADD | CMAP_TRACK_DELETE | CMAP_TRACK_MODIFY | CMAP_TRACK_PREFIX, qdevice_cmap_reload_cb, NULL, &instance->cmap_nodelist_track_handle); if (res != CS_OK) { qdevice_log(LOG_ERR, "Can't initialize cmap nodelist tracking"); return (-1); } res = cmap_track_add(instance->cmap_handle, "logging.", CMAP_TRACK_ADD | CMAP_TRACK_DELETE | CMAP_TRACK_MODIFY | CMAP_TRACK_PREFIX, qdevice_cmap_reload_cb, NULL, &instance->cmap_logging_track_handle); if (res != CS_OK) { qdevice_log(LOG_ERR, "Can't initialize logging tracking"); return (-1); } return (0); } int qdevice_cmap_del_track(struct qdevice_instance *instance) { cs_error_t res; res = cmap_track_delete(instance->cmap_handle, instance->cmap_reload_track_handle); if (res != CS_OK) { qdevice_log(LOG_WARNING, "Can't delete cmap totemconfig_reload_in_progress tracking"); } res = cmap_track_delete(instance->cmap_handle, instance->cmap_nodelist_track_handle); if (res != CS_OK) { qdevice_log(LOG_WARNING, "Can't delete cmap nodelist tracking"); } res = cmap_track_delete(instance->cmap_handle, instance->cmap_logging_track_handle); if (res != CS_OK) { qdevice_log(LOG_WARNING, "Can't delete cmap logging tracking"); } return (0); } void qdevice_cmap_destroy(struct qdevice_instance *instance) { cs_error_t res; res = cmap_finalize(instance->cmap_handle); if (res != CS_OK) { qdevice_log(LOG_WARNING, "Can't finalize cmap. Error %s", cs_strerror(res)); } } int qdevice_cmap_dispatch(struct qdevice_instance *instance) { cs_error_t res; + /* + * dispatch can block if corosync is during sync phase + */ + if (instance->sync_in_progress) { + return (0); + } + res = cmap_dispatch(instance->cmap_handle, CS_DISPATCH_ALL); if (res != CS_OK && res != CS_ERR_TRY_AGAIN) { qdevice_log(LOG_ERR, "Can't dispatch cmap messages"); return (-1); } return (0); } diff --git a/qdevices/qdevice-instance.h b/qdevices/qdevice-instance.h index bf7b951d..b93c7f54 100644 --- a/qdevices/qdevice-instance.h +++ b/qdevices/qdevice-instance.h @@ -1,119 +1,121 @@ /* * Copyright (c) 2015-2016 Red Hat, Inc. * * All rights reserved. * * Author: Jan Friesse (jfriesse@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the Red Hat, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _QDEVICE_INSTANCE_H_ #define _QDEVICE_INSTANCE_H_ #include #include #include #include #include #include "qdevice-advanced-settings.h" #include "qdevice-model-type.h" #include "node-list.h" #include "unix-socket-ipc.h" #ifdef __cplusplus extern "C" { #endif struct qdevice_instance { cmap_handle_t cmap_handle; int cmap_poll_fd; int cmap_reload_in_progress; cmap_track_handle_t cmap_reload_track_handle; cmap_track_handle_t cmap_nodelist_track_handle; cmap_track_handle_t cmap_logging_track_handle; votequorum_handle_t votequorum_handle; int votequorum_poll_fd; struct unix_socket_ipc local_ipc; enum qdevice_model_type model_type; uint32_t node_id; uint32_t heartbeat_interval; /* Heartbeat interval during normal operation */ uint32_t sync_heartbeat_interval; /* Heartbeat interval during corosync sync */ struct node_list config_node_list; int config_node_list_version_set; uint64_t config_node_list_version; /* * Copy of votequorum_quorum_notify_fn callback paramters. * Set after model callback is called. */ uint32_t vq_quorum_quorate; uint32_t vq_quorum_node_list_entries; votequorum_node_t *vq_quorum_node_list; /* * Copy of votequorum_nodelist_notify_fn callback paramters. * Set after model callback is called. */ votequorum_ring_id_t vq_node_list_ring_id; uint32_t vq_node_list_entries; uint32_t *vq_node_list; /* * Copy of votequorum_expectedvotes_notify_fn callback parameters. * Set after model callback is called. */ uint32_t vq_expected_votes; time_t vq_last_poll; int vq_last_poll_cast_vote; void *model_data; const struct qdevice_advanced_settings *advanced_settings; + + int sync_in_progress; }; extern int qdevice_instance_init(struct qdevice_instance *instance, const struct qdevice_advanced_settings *advanced_settings); extern int qdevice_instance_destroy(struct qdevice_instance *instance); extern int qdevice_instance_configure_from_cmap(struct qdevice_instance *instance); #ifdef __cplusplus } #endif #endif /* _QDEVICE_INSTANCE_H_ */ diff --git a/qdevices/qdevice-net-poll.c b/qdevices/qdevice-net-poll.c index 63c3f22f..72b40489 100644 --- a/qdevices/qdevice-net-poll.c +++ b/qdevices/qdevice-net-poll.c @@ -1,435 +1,437 @@ /* * Copyright (c) 2015-2016 Red Hat, Inc. * * All rights reserved. * * Author: Jan Friesse (jfriesse@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the Red Hat, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include "qdevice-cmap.h" #include "qdevice-net-poll.h" #include "qdevice-log.h" #include "qdevice-net-send.h" #include "qdevice-net-socket.h" #include "qdevice-votequorum.h" #include "qdevice-ipc.h" #include "qdevice-net-poll-array-user-data.h" /* * Needed for creating nspr handle from unix fd */ #include static void qdevice_net_poll_read_socket(struct qdevice_net_instance *instance) { if (qdevice_net_socket_read(instance) == -1) { instance->schedule_disconnect = 1; } } static void qdevice_net_poll_read_votequorum(struct qdevice_net_instance *instance) { if (qdevice_votequorum_dispatch(instance->qdevice_instance_ptr) == -1) { instance->schedule_disconnect = 1; instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED; } } static void qdevice_net_poll_read_cmap(struct qdevice_net_instance *instance) { if (qdevice_cmap_dispatch(instance->qdevice_instance_ptr) == -1) { instance->schedule_disconnect = 1; instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED; } } static void qdevice_net_poll_write_socket(struct qdevice_net_instance *instance, const PRPollDesc *pfd) { int res; if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) { res = nss_sock_non_blocking_client_succeeded(pfd); if (res == -1) { /* * Connect failed -> try next */ res = nss_sock_non_blocking_client_try_next(&instance->non_blocking_client); if (res == -1) { qdevice_log_nss(LOG_ERR, "Can't connect to qnetd host."); nss_sock_non_blocking_client_destroy(&instance->non_blocking_client); } } else if (res == 0) { /* * Poll again */ } else if (res == 1) { /* * Connect success */ instance->socket = instance->non_blocking_client.socket; nss_sock_non_blocking_client_destroy(&instance->non_blocking_client); instance->non_blocking_client.socket = NULL; instance->state = QDEVICE_NET_INSTANCE_STATE_SENDING_PREINIT_REPLY; qdevice_log(LOG_DEBUG, "Sending preinit msg to qnetd"); if (qdevice_net_send_preinit(instance) != 0) { instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_CANT_ALLOCATE_MSG_BUFFER; instance->schedule_disconnect = 1; } } else { qdevice_log(LOG_CRIT, "Unhandled nss_sock_non_blocking_client_succeeded"); exit(1); } } else { if (qdevice_net_socket_write(instance) == -1) { instance->schedule_disconnect = 1; } } } static void qdevice_net_poll_err_socket(struct qdevice_net_instance *instance, const PRPollDesc *pfd) { if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) { /* * Workaround for RHEL<7. Pollout is never set for nonblocking connect (doesn't work * only with poll, select works as expected!???). * So test if client is still valid and if pollout was not already called (ensured * by default because of order in PR_Poll). * If both applies it's possible to emulate pollout set by calling poll_write. */ if (!instance->non_blocking_client.destroyed) { qdevice_net_poll_write_socket(instance, pfd); } } else { qdevice_log(LOG_ERR, "POLL_ERR (%u) on main socket", pfd->out_flags); instance->schedule_disconnect = 1; instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_SERVER_CLOSED_CONNECTION; } } static void qdevice_net_poll_read_ipc_socket(struct qdevice_net_instance *instance) { struct unix_socket_client *client; PRFileDesc *prfd; struct qdevice_ipc_user_data *user_data; if (qdevice_ipc_accept(instance->qdevice_instance_ptr, &client) != 0) { return ; } prfd = PR_CreateSocketPollFd(client->socket); if (prfd == NULL) { qdevice_log_nss(LOG_CRIT, "Can't create NSPR poll fd for IPC client. " "Disconnecting client"); qdevice_ipc_client_disconnect(instance->qdevice_instance_ptr, client); return ; } user_data = (struct qdevice_ipc_user_data *)client->user_data; user_data->model_data = (void *)prfd; } static PRPollDesc * qdevice_net_pr_poll_array_create(struct qdevice_net_instance *instance) { struct pr_poll_array *poll_array; PRPollDesc *poll_desc; struct qdevice_net_poll_array_user_data *user_data; struct unix_socket_client *ipc_client; const struct unix_socket_client_list *ipc_client_list; struct qdevice_ipc_user_data *qdevice_ipc_user_data; poll_array = &instance->poll_array; ipc_client_list = &instance->qdevice_instance_ptr->local_ipc.clients; - pr_poll_array_clean(poll_array); + if (qdevice_ipc_is_closed(instance->qdevice_instance_ptr)) { + qdevice_log(LOG_DEBUG, "Local socket is closed"); + instance->schedule_disconnect = 1; + instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_LOCAL_SOCKET_CLOSED; - if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) { return (NULL); } - poll_desc->fd = instance->votequorum_poll_fd; - poll_desc->in_flags = PR_POLL_READ; - user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM; + + pr_poll_array_clean(poll_array); if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) { return (NULL); } - poll_desc->fd = instance->cmap_poll_fd; + poll_desc->fd = instance->votequorum_poll_fd; poll_desc->in_flags = PR_POLL_READ; - user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP; - - if (qdevice_ipc_is_closed(instance->qdevice_instance_ptr)) { - qdevice_log(LOG_DEBUG, "Local socket is closed"); - instance->schedule_disconnect = 1; - instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_LOCAL_SOCKET_CLOSED; + user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM; - return (NULL); + if (!instance->qdevice_instance_ptr->sync_in_progress) { + if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) { + return (NULL); + } + poll_desc->fd = instance->cmap_poll_fd; + poll_desc->in_flags = PR_POLL_READ; + user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP; } if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) { return (NULL); } poll_desc->fd = instance->ipc_socket_poll_fd; poll_desc->in_flags = PR_POLL_READ; user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET; if (instance->state != QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT || !instance->non_blocking_client.destroyed) { if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) { return (NULL); } user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET; if (instance->state == QDEVICE_NET_INSTANCE_STATE_WAITING_CONNECT) { poll_desc->fd = instance->non_blocking_client.socket; poll_desc->in_flags = PR_POLL_WRITE | PR_POLL_EXCEPT; } else { poll_desc->fd = instance->socket; poll_desc->in_flags = PR_POLL_READ; if (!send_buffer_list_empty(&instance->send_buffer_list)) { poll_desc->in_flags |= PR_POLL_WRITE; } } } TAILQ_FOREACH(ipc_client, ipc_client_list, entries) { if (!ipc_client->reading_line && !ipc_client->writing_buffer) { continue; } if (pr_poll_array_add(poll_array, &poll_desc, (void **)&user_data) < 0) { return (NULL); } qdevice_ipc_user_data = (struct qdevice_ipc_user_data *)ipc_client->user_data; poll_desc->fd = (PRFileDesc *)qdevice_ipc_user_data->model_data; if (ipc_client->reading_line) { poll_desc->in_flags |= PR_POLL_READ; } if (ipc_client->writing_buffer) { poll_desc->in_flags |= PR_POLL_WRITE; } user_data->type = QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT; user_data->ipc_client = ipc_client; } pr_poll_array_gc(poll_array); return (poll_array->array); } int qdevice_net_poll(struct qdevice_net_instance *instance) { PRPollDesc *pfds; PRFileDesc *prfd; PRInt32 poll_res; ssize_t i; struct qdevice_net_poll_array_user_data *user_data; struct unix_socket_client *ipc_client; struct qdevice_ipc_user_data *qdevice_ipc_user_data; int case_processed; pfds = qdevice_net_pr_poll_array_create(instance); if (pfds == NULL) { return (-1); } instance->schedule_disconnect = 0; if ((poll_res = PR_Poll(pfds, pr_poll_array_size(&instance->poll_array), timer_list_time_to_expire(&instance->main_timer_list))) > 0) { for (i = 0; i < pr_poll_array_size(&instance->poll_array); i++) { user_data = pr_poll_array_get_user_data(&instance->poll_array, i); ipc_client = user_data->ipc_client; if (pfds[i].out_flags & PR_POLL_READ) { case_processed = 0; switch (user_data->type) { case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET: case_processed = 1; qdevice_net_poll_read_socket(instance); break; case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM: case_processed = 1; qdevice_net_poll_read_votequorum(instance); break; case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP: case_processed = 1; qdevice_net_poll_read_cmap(instance); break; case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET: case_processed = 1; qdevice_net_poll_read_ipc_socket(instance); break; case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT: case_processed = 1; qdevice_ipc_io_read(instance->qdevice_instance_ptr, ipc_client); break; /* * Default is not defined intentionally. Compiler shows warning when * new poll_array_user_data_type is added */ } if (!case_processed) { qdevice_log(LOG_CRIT, "Unhandled read on poll descriptor %u", i); exit(1); } } if (!instance->schedule_disconnect && pfds[i].out_flags & PR_POLL_WRITE) { case_processed = 0; switch (user_data->type) { case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET: case_processed = 1; qdevice_net_poll_write_socket(instance, &pfds[i]); break; case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT: case_processed = 1; qdevice_ipc_io_write(instance->qdevice_instance_ptr, ipc_client); break; case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM: case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP: case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET: /* * Write on votequorum, cmap and ipc socket shouldn't happen. */ break; /* * Default is not defined intentionally. Compiler shows warning when * new poll_array_user_data_type is added */ } if (!case_processed) { qdevice_log(LOG_CRIT, "Unhandled write on poll descriptor %u", i); exit(1); } } if (!instance->schedule_disconnect && (pfds[i].out_flags & (PR_POLL_ERR|PR_POLL_NVAL|PR_POLL_HUP|PR_POLL_EXCEPT)) && !(pfds[i].out_flags & (PR_POLL_READ|PR_POLL_WRITE))) { case_processed = 0; switch (user_data->type) { case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_SOCKET: case_processed = 1; qdevice_net_poll_err_socket(instance, &pfds[i]); break; case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_SOCKET: case_processed = 1; if (pfds[i].out_flags != PR_POLL_NVAL) { qdevice_log(LOG_CRIT, "POLLERR (%u) on local socket", pfds[i].out_flags); exit(1); } else { qdevice_log(LOG_DEBUG, "Local socket is closed"); instance->schedule_disconnect = 1; instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_LOCAL_SOCKET_CLOSED; } break; case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT: case_processed = 1; qdevice_log(LOG_DEBUG, "POLL_ERR (%u) on ipc client socket. " "Disconnecting.", pfds[i].out_flags); ipc_client->schedule_disconnect = 1; break; case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_VOTEQUORUM: case QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_CMAP: case_processed = 1; qdevice_log(LOG_DEBUG, "POLL_ERR (%u) on corosync socket. " "Disconnecting.", pfds[i].out_flags); instance->schedule_disconnect = 1; instance->disconnect_reason = QDEVICE_NET_DISCONNECT_REASON_COROSYNC_CONNECTION_CLOSED; break; /* * Default is not defined intentionally. Compiler shows warning when * new poll_array_user_data_type is added */ } if (!case_processed) { qdevice_log(LOG_CRIT, "Unhandled error on poll descriptor %u", i); exit(1); } } if (user_data->type == QDEVICE_NET_POLL_ARRAY_USER_DATA_TYPE_IPC_CLIENT && ipc_client->schedule_disconnect) { qdevice_ipc_user_data = (struct qdevice_ipc_user_data *)ipc_client->user_data; prfd = (PRFileDesc *)qdevice_ipc_user_data->model_data; if (PR_DestroySocketPollFd(prfd) != PR_SUCCESS) { qdevice_log_nss(LOG_WARNING, "Unable to destroy client IPC poll socket fd"); } qdevice_ipc_client_disconnect(instance->qdevice_instance_ptr, ipc_client); } } } if (!instance->schedule_disconnect) { timer_list_expire(&instance->main_timer_list); } if (instance->schedule_disconnect) { /* * Schedule disconnect can be set by this function, by some timer_list callback * or cmap/votequorum callbacks */ return (-1); } return (0); } diff --git a/qdevices/qdevice-net-send.c b/qdevices/qdevice-net-send.c index ea3d0855..f7e68cdc 100644 --- a/qdevices/qdevice-net-send.c +++ b/qdevices/qdevice-net-send.c @@ -1,314 +1,313 @@ /* * Copyright (c) 2015-2016 Red Hat, Inc. * * All rights reserved. * * Author: Jan Friesse (jfriesse@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the Red Hat, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include "qdevice-log.h" #include "qdevice-log-debug.h" #include "qdevice-net-send.h" -#include "qdevice-cmap.h" #include "qdevice-net-votequorum.h" #include "msg.h" #include "utils.h" int qdevice_net_send_echo_request(struct qdevice_net_instance *instance) { struct send_buffer_list_entry *send_buffer; send_buffer = send_buffer_list_get_new(&instance->send_buffer_list); if (send_buffer == NULL) { qdevice_log(LOG_CRIT, "Can't allocate send list buffer for reply msg."); return (-1); } instance->echo_request_expected_msg_seq_num++; if (msg_create_echo_request(&send_buffer->buffer, 1, instance->echo_request_expected_msg_seq_num) == -1) { qdevice_log(LOG_ERR, "Can't allocate send buffer for echo request msg"); return (-1); } send_buffer_list_put(&instance->send_buffer_list, send_buffer); return (0); } int qdevice_net_send_preinit(struct qdevice_net_instance *instance) { struct send_buffer_list_entry *send_buffer; send_buffer = send_buffer_list_get_new(&instance->send_buffer_list); if (send_buffer == NULL) { qdevice_log(LOG_ERR, "Can't allocate send list buffer for preinit msg"); return (-1); } if (msg_create_preinit(&send_buffer->buffer, instance->cluster_name, 1, instance->last_msg_seq_num) == 0) { qdevice_log(LOG_ERR, "Can't allocate buffer"); send_buffer_list_discard_new(&instance->send_buffer_list, send_buffer); return (-1); } send_buffer_list_put(&instance->send_buffer_list, send_buffer); instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_PREINIT_REPLY; return (0); } int qdevice_net_send_init(struct qdevice_net_instance *instance) { enum msg_type *supported_msgs; size_t no_supported_msgs; enum tlv_opt_type *supported_opts; size_t no_supported_opts; struct send_buffer_list_entry *send_buffer; tlv_get_supported_options(&supported_opts, &no_supported_opts); msg_get_supported_messages(&supported_msgs, &no_supported_msgs); instance->last_msg_seq_num++; send_buffer = send_buffer_list_get_new(&instance->send_buffer_list); if (send_buffer == NULL) { qdevice_log(LOG_ERR, "Can't allocate send list buffer for init msg"); return (-1); } if (msg_create_init(&send_buffer->buffer, 1, instance->last_msg_seq_num, instance->decision_algorithm, supported_msgs, no_supported_msgs, supported_opts, no_supported_opts, instance->qdevice_instance_ptr->node_id, instance->heartbeat_interval, &instance->tie_breaker) == 0) { qdevice_log(LOG_ERR, "Can't allocate send buffer for init msg"); send_buffer_list_discard_new(&instance->send_buffer_list, send_buffer); return (-1); } send_buffer_list_put(&instance->send_buffer_list, send_buffer); instance->state = QDEVICE_NET_INSTANCE_STATE_WAITING_INIT_REPLY; return (0); } int qdevice_net_send_ask_for_vote(struct qdevice_net_instance *instance) { struct send_buffer_list_entry *send_buffer; send_buffer = send_buffer_list_get_new(&instance->send_buffer_list); if (send_buffer == NULL) { qdevice_log(LOG_ERR, "Can't allocate send list buffer for ask for vote msg"); return (-1); } instance->last_msg_seq_num++; qdevice_log(LOG_DEBUG, "Sending ask for vote seq = "UTILS_PRI_MSG_SEQ, instance->last_msg_seq_num); if (msg_create_ask_for_vote(&send_buffer->buffer, instance->last_msg_seq_num) == 0) { qdevice_log(LOG_ERR, "Can't allocate send buffer for ask for vote msg"); send_buffer_list_discard_new(&instance->send_buffer_list, send_buffer); return (-1); } send_buffer_list_put(&instance->send_buffer_list, send_buffer); return (0); } int qdevice_net_send_config_node_list(struct qdevice_net_instance *instance, const struct node_list *nlist, int config_version_set, uint64_t config_version, int initial) { struct send_buffer_list_entry *send_buffer; send_buffer = send_buffer_list_get_new(&instance->send_buffer_list); if (send_buffer == NULL) { qdevice_log(LOG_ERR, "Can't allocate send list buffer for config " "node list msg"); return (-1); } instance->last_msg_seq_num++; qdevice_log(LOG_DEBUG, "Sending config node list seq = "UTILS_PRI_MSG_SEQ, instance->last_msg_seq_num); qdevice_log_debug_dump_node_list(nlist); if (msg_create_node_list(&send_buffer->buffer, instance->last_msg_seq_num, (initial ? TLV_NODE_LIST_TYPE_INITIAL_CONFIG : TLV_NODE_LIST_TYPE_CHANGED_CONFIG), 0, NULL, config_version_set, config_version, 0, TLV_QUORATE_INQUORATE, nlist) == 0) { qdevice_log(LOG_ERR, "Can't allocate send buffer for config list msg"); send_buffer_list_discard_new(&instance->send_buffer_list, send_buffer); return (-1); } send_buffer_list_put(&instance->send_buffer_list, send_buffer); return (0); } int qdevice_net_send_membership_node_list(struct qdevice_net_instance *instance, const struct tlv_ring_id *ring_id, uint32_t node_list_entries, uint32_t node_list[]) { struct node_list nlist; struct send_buffer_list_entry *send_buffer; uint32_t i; node_list_init(&nlist); for (i = 0; i < node_list_entries; i++) { if (node_list_add(&nlist, node_list[i], 0, TLV_NODE_STATE_NOT_SET) == NULL) { qdevice_log(LOG_ERR, "Can't allocate membership node list."); node_list_free(&nlist); return (-1); } } send_buffer = send_buffer_list_get_new(&instance->send_buffer_list); if (send_buffer == NULL) { qdevice_log(LOG_ERR, "Can't allocate send list buffer for membership " "node list msg"); node_list_free(&nlist); return (-1); } instance->last_msg_seq_num++; qdevice_log(LOG_DEBUG, "Sending membership node list seq = "UTILS_PRI_MSG_SEQ", " "ringid = ("UTILS_PRI_RING_ID").", instance->last_msg_seq_num, ring_id->node_id, ring_id->seq); qdevice_log_debug_dump_node_list(&nlist); if (msg_create_node_list(&send_buffer->buffer, instance->last_msg_seq_num, TLV_NODE_LIST_TYPE_MEMBERSHIP, 1, ring_id, 0, 0, 0, 0, &nlist) == 0) { qdevice_log(LOG_ERR, "Can't allocate send buffer for membership list msg"); node_list_free(&nlist); send_buffer_list_discard_new(&instance->send_buffer_list, send_buffer); return (-1); } memcpy(&instance->last_sent_ring_id, ring_id, sizeof(instance->last_sent_ring_id)); node_list_free(&nlist); send_buffer_list_put(&instance->send_buffer_list, send_buffer); return (0); } int qdevice_net_send_quorum_node_list(struct qdevice_net_instance *instance, enum tlv_quorate quorate, uint32_t node_list_entries, votequorum_node_t node_list[]) { struct node_list nlist; struct send_buffer_list_entry *send_buffer; uint32_t i; node_list_init(&nlist); for (i = 0; i < node_list_entries; i++) { if (node_list[i].nodeid == 0) { continue; } if (node_list_add(&nlist, node_list[i].nodeid, 0, qdevice_net_votequorum_node_state_to_tlv(node_list[i].state)) == NULL) { qdevice_log(LOG_ERR, "Can't allocate quorum node list."); node_list_free(&nlist); return (-1); } } send_buffer = send_buffer_list_get_new(&instance->send_buffer_list); if (send_buffer == NULL) { qdevice_log(LOG_ERR, "Can't allocate send list buffer for quorum " "node list msg"); node_list_free(&nlist); return (-1); } instance->last_msg_seq_num++; qdevice_log(LOG_DEBUG, "Sending quorum node list seq = "UTILS_PRI_MSG_SEQ", quorate = %u", instance->last_msg_seq_num, quorate); qdevice_log_debug_dump_node_list(&nlist); if (msg_create_node_list(&send_buffer->buffer, instance->last_msg_seq_num, TLV_NODE_LIST_TYPE_QUORUM, 0, NULL, 0, 0, 1, quorate, &nlist) == 0) { qdevice_log(LOG_ERR, "Can't allocate send buffer for quorum list msg"); node_list_free(&nlist); send_buffer_list_discard_new(&instance->send_buffer_list, send_buffer); return (-1); } node_list_free(&nlist); send_buffer_list_put(&instance->send_buffer_list, send_buffer); return (0); } diff --git a/qdevices/qdevice-votequorum.c b/qdevices/qdevice-votequorum.c index a25934b9..946854c4 100644 --- a/qdevices/qdevice-votequorum.c +++ b/qdevices/qdevice-votequorum.c @@ -1,275 +1,279 @@ /* * Copyright (c) 2015-2016 Red Hat, Inc. * * All rights reserved. * * Author: Jan Friesse (jfriesse@redhat.com) * * This software licensed under BSD license, the text of which follows: * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * - Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * - Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * - Neither the name of the Red Hat, Inc. nor the names of its * contributors may be used to endorse or promote products derived from this * software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #include "qdevice-config.h" #include "qdevice-log.h" #include "qdevice-votequorum.h" #include "qdevice-model.h" #include "utils.h" static void qdevice_votequorum_quorum_notify_callback(votequorum_handle_t votequorum_handle, uint64_t context, uint32_t quorate, uint32_t node_list_entries, votequorum_node_t node_list[]) { struct qdevice_instance *instance; uint32_t u32; if (votequorum_context_get(votequorum_handle, (void **)&instance) != CS_OK) { qdevice_log(LOG_CRIT, "Fatal error. Can't get votequorum context"); exit(1); } + instance->sync_in_progress = 0; + qdevice_log(LOG_DEBUG, "Votequorum quorum notify callback:"); qdevice_log(LOG_DEBUG, " Quorate = %u", quorate); qdevice_log(LOG_DEBUG, " Node list (size = %"PRIu32"):", node_list_entries); for (u32 = 0; u32 < node_list_entries; u32++) { qdevice_log(LOG_DEBUG, " %"PRIu32" nodeid = "UTILS_PRI_NODE_ID", state = %"PRIu32, u32, node_list[u32].nodeid, node_list[u32].state); } if (qdevice_model_votequorum_quorum_notify(instance, quorate, node_list_entries, node_list) != 0) { qdevice_log(LOG_DEBUG, "qdevice_model_votequorum_quorum_notify returned error -> exit"); exit(2); } instance->vq_quorum_quorate = quorate; instance->vq_quorum_node_list_entries = node_list_entries; free(instance->vq_quorum_node_list); instance->vq_quorum_node_list = malloc(sizeof(*node_list) * node_list_entries); if (instance->vq_quorum_node_list == NULL) { qdevice_log(LOG_CRIT, "Can't alloc votequorum node list memory"); exit(1); } memcpy(instance->vq_quorum_node_list, node_list, sizeof(*node_list) * node_list_entries); } static void qdevice_votequorum_node_list_notify_callback(votequorum_handle_t votequorum_handle, uint64_t context, votequorum_ring_id_t votequorum_ring_id, uint32_t node_list_entries, uint32_t node_list[]) { struct qdevice_instance *instance; uint32_t u32; if (votequorum_context_get(votequorum_handle, (void **)&instance) != CS_OK) { qdevice_log(LOG_CRIT, "Fatal error. Can't get votequorum context"); exit(1); } + instance->sync_in_progress = 1; + qdevice_log(LOG_DEBUG, "Votequorum nodelist notify callback:"); qdevice_log(LOG_DEBUG, " Ring_id = ("UTILS_PRI_RING_ID")", votequorum_ring_id.nodeid, votequorum_ring_id.seq); qdevice_log(LOG_DEBUG, " Node list (size = %"PRIu32"):", node_list_entries); for (u32 = 0; u32 < node_list_entries; u32++) { qdevice_log(LOG_DEBUG, " %"PRIu32" nodeid = "UTILS_PRI_NODE_ID, u32, node_list[u32]); } if (qdevice_model_votequorum_node_list_notify(instance, votequorum_ring_id, node_list_entries, node_list) != 0) { qdevice_log(LOG_DEBUG, "qdevice_votequorum_node_list_notify_callback returned error -> exit"); exit(2); } memcpy(&instance->vq_node_list_ring_id, &votequorum_ring_id, sizeof(votequorum_ring_id)); instance->vq_node_list_entries = node_list_entries; free(instance->vq_node_list); instance->vq_node_list = malloc(sizeof(*node_list) * node_list_entries); if (instance->vq_node_list == NULL) { qdevice_log(LOG_CRIT, "Can't alloc votequorum node list memory"); exit(1); } memcpy(instance->vq_node_list, node_list, sizeof(*node_list) * node_list_entries); } static void qdevice_votequorum_expected_votes_notify_callback(votequorum_handle_t votequorum_handle, uint64_t context, uint32_t expected_votes) { struct qdevice_instance *instance; if (votequorum_context_get(votequorum_handle, (void **)&instance) != CS_OK) { qdevice_log(LOG_CRIT, "Fatal error. Can't get votequorum context"); exit(1); } qdevice_log(LOG_DEBUG, "Votequorum expected_votes notify callback:"); qdevice_log(LOG_DEBUG, " Expected_votes: "UTILS_PRI_EXPECTED_VOTES, expected_votes); if (qdevice_model_votequorum_expected_votes_notify(instance, expected_votes) != 0) { qdevice_log(LOG_DEBUG, "qdevice_votequorum_expected_votes_notify_callback returned error -> exit"); exit(2); } instance->vq_expected_votes = expected_votes; } void qdevice_votequorum_init(struct qdevice_instance *instance) { votequorum_callbacks_t votequorum_callbacks; votequorum_handle_t votequorum_handle; cs_error_t res; int no_retries; struct votequorum_info vq_info; memset(&votequorum_callbacks, 0, sizeof(votequorum_callbacks)); votequorum_callbacks.votequorum_quorum_notify_fn = qdevice_votequorum_quorum_notify_callback; votequorum_callbacks.votequorum_nodelist_notify_fn = qdevice_votequorum_node_list_notify_callback; votequorum_callbacks.votequorum_expectedvotes_notify_fn = qdevice_votequorum_expected_votes_notify_callback; no_retries = 0; while ((res = votequorum_initialize(&votequorum_handle, &votequorum_callbacks)) == CS_ERR_TRY_AGAIN && no_retries++ < instance->advanced_settings->max_cs_try_again) { (void)poll(NULL, 0, 1000); } if (res != CS_OK) { qdevice_log(LOG_CRIT, "Failed to initialize the votequorum API. Error %s", cs_strerror(res)); exit(1); } if ((res = votequorum_qdevice_register(votequorum_handle, instance->advanced_settings->votequorum_device_name)) != CS_OK) { qdevice_log(LOG_CRIT, "Can't register votequorum device. Error %s", cs_strerror(res)); exit(1); } if ((res = votequorum_context_set(votequorum_handle, (void *)instance)) != CS_OK) { qdevice_log(LOG_CRIT, "Can't set votequorum context. Error %s", cs_strerror(res)); exit(1); } if ((res = votequorum_getinfo(votequorum_handle, VOTEQUORUM_QDEVICE_NODEID, &vq_info)) != CS_OK) { qdevice_log(LOG_CRIT, "Can't get votequorum information. Error %s", cs_strerror(res)); exit(1); } instance->vq_expected_votes = vq_info.node_expected_votes; instance->votequorum_handle = votequorum_handle; votequorum_fd_get(votequorum_handle, &instance->votequorum_poll_fd); if ((res = votequorum_trackstart(instance->votequorum_handle, 0, CS_TRACK_CHANGES)) != CS_OK) { qdevice_log(LOG_CRIT, "Can't start tracking votequorum changes. Error %s", cs_strerror(res)); exit(1); } } void qdevice_votequorum_destroy(struct qdevice_instance *instance) { cs_error_t res; free(instance->vq_quorum_node_list); instance->vq_quorum_node_list = NULL; free(instance->vq_node_list); instance->vq_node_list = NULL; res = votequorum_trackstop(instance->votequorum_handle); if (res != CS_OK) { qdevice_log(LOG_WARNING, "Can't start tracking votequorum changes. Error %s", cs_strerror(res)); } res = votequorum_qdevice_unregister(instance->votequorum_handle, instance->advanced_settings->votequorum_device_name); if (res != CS_OK) { qdevice_log(LOG_WARNING, "Unable to unregister votequorum device. Error %s", cs_strerror(res)); } res = votequorum_finalize(instance->votequorum_handle); if (res != CS_OK) { qdevice_log(LOG_WARNING, "Unable to finalize votequorum. Error %s", cs_strerror(res)); } } int qdevice_votequorum_dispatch(struct qdevice_instance *instance) { cs_error_t res; res = votequorum_dispatch(instance->votequorum_handle, CS_DISPATCH_ALL); if (res != CS_OK && res != CS_ERR_TRY_AGAIN) { qdevice_log(LOG_ERR, "Can't dispatch votequorum messages"); return (-1); } return (0); } int qdevice_votequorum_poll(struct qdevice_instance *instance, int cast_vote) { cs_error_t res; instance->vq_last_poll = time(NULL); instance->vq_last_poll_cast_vote = cast_vote; res = votequorum_qdevice_poll(instance->votequorum_handle, instance->advanced_settings->votequorum_device_name, cast_vote, instance->vq_node_list_ring_id); if (res != CS_OK && res != CS_ERR_TRY_AGAIN) { if (res == CS_ERR_MESSAGE_ERROR) { qdevice_log(LOG_INFO, "qdevice_votequorum_poll called with old ring id"); } else { qdevice_log(LOG_CRIT, "Can't call votequorum_qdevice_poll. Error %s", cs_strerror(res)); return (-1); } } return (0); }