diff --git a/autogen.sh b/autogen.sh index c4a7c1a6dc..b7b3f839de 100755 --- a/autogen.sh +++ b/autogen.sh @@ -1,193 +1,193 @@ #!/bin/sh # # License: GNU General Public License (GPL) # Copyright 2001 horms # (heavily mangled by alanr) # # bootstrap: set up the project and get it ready to make # # Basically, we run autoconf, automake and libtool in the # right way to get things set up for this environment. # # We also look and see if those tools are installed, and # tell you where to get them if they're not. # # Our goal is to not require dragging along anything # more than we need. If this doesn't work on your system, # (i.e., your /bin/sh is broken) send us a patch. # # This code loosely based on the corresponding named script in # enlightenment, and also on the sort-of-standard autoconf # bootstrap script. # Run this to generate all the initial makefiles, etc. testProgram() { cmd=$1 if [ -z "$cmd" ]; then return 1; fi arch=`uname -s` # Make sure the which is in an if-block... on some platforms it throws exceptions # # The ERR trap is not executed if the failed command is part # of an until or while loop, part of an if statement, part of a && # or || list. if which $cmd /dev/null 2>&1 then : else return 1 fi # The GNU standard is --version if $cmd --version /dev/null 2>&1 then return 0 fi # Maybe it suppports -V instead if $cmd -V /dev/null 2>&1 then return 0 fi # Nope, the program seems broken return 1 } arch=`uname -s` # Disable the errors on FreeBSD until a fix can be found. if [ ! "$arch" = "FreeBSD" ]; then set -e # # All errors are fatal from here on out... # The shell will complain and exit on any "uncaught" error code. # # # And the trap will ensure sure some kind of error message comes out. # trap 'echo ""; echo "$0 exiting due to error (sorry!)." >&2' 0 fi gnu="ftp://ftp.gnu.org/pub/gnu" for command in autoconf autoconf213 autoconf253 autoconf259 do if testProgram $command == 1 then autoconf=$command autoheader=`echo "$autoconf" | sed -e 's/autoconf/autoheader/'` autom4te=`echo "$autoconf" | sed -e 's/autoconf/autmo4te/'` autoreconf=`echo "$autoconf" | sed -e 's/autoconf/autoreconf/'` autoscan=`echo "$autoconf" | sed -e 's/autoconf/autoscan/'` autoupdate=`echo "$autoconf" | sed -e 's/autoconf/autoupdate/'` ifnames=`echo "$autoconf" | sed -e 's/autoconf/ifnames/'` fi done for command in automake automake14 automake-1.4 automake15 automake-1.5 automake17 automake-1.7 automake19 automake-1.9 do if testProgram $command then : OK $pkg is installed automake=$command aclocal=`echo "$automake" | sed -e 's/automake/aclocal/'` fi done for command in libtool libtool14 libtool15 glibtool do URL=$gnu/$pkg/ if testProgram $command then libtool=$command libtoolize=`echo "$libtool" | sed -e 's/libtool/libtoolize/'` fi done if [ -z $autoconf ]; then echo You must have automake installed to compile the linux-ha package. echo Download the appropriate package for your system, echo or get the source tarball at: $gnu/autoconf/ exit 1 elif [ -z $automake ]; then echo You must have automake installed to compile the linux-ha package. echo Download the appropriate package for your system, echo or get the source tarball at: $gnu/automake/ exit 1 elif [ -z $libtool ]; then echo You must have automake installed to compile the linux-ha package. echo Download the appropriate package for your system, echo or get the source tarball at: $gnu/libtool/ exit 1 fi oneline() { read x; echo "$x" } LT_version=`$libtool --version | oneline | sed -e 's%^[^0-9]*%%' -e s'% .*%%'` LT_majvers=`echo "$LT_version" | sed -e 's%\..*%%'` LT_minvers=`echo "$LT_version" | sed -e 's%^[^.]*\.%%' ` LT_minnum=`echo "$LT_minvers" | sed -e 's%[^0-9].*%%'` if [ $LT_majvers -lt 1 ] || [ $LT_majvers = 1 -a $LT_minnum -lt 4 ] then echo "Minimum version of libtool is 1.4. You have $LT_version installed." exit 1 fi # Create local copies so that the incremental updates will work. rm -f ./autoconf ./automake ./autoheader ./libtool ln -s `which $libtool` ./libtool ln -s `which $autoconf` ./autoconf ln -s `which $automake` ./automake ln -s `which $autoheader` ./autoheader -echo "$autoconf:\t" | tr '\n' ' ' +printf "$autoconf:\t" $autoconf --version | head -n 1 -echo "$automake:\t" | tr '\n' ' ' +printf "$automake:\t" $automake --version | head -n 1 rm -rf libltdl libltdl.tar echo $libtoolize --ltdl --force --copy # Unset GREP_OPTIONS as any coloring can mess up the AC_CONFIG_AUX_DIR matching patterns GREP_OPTIONS= $libtoolize --ltdl --force --copy # Emulate the old --ltdl-tar option... # If the libltdl directory is required we will unpack it later tar -cf libltdl.tar libltdl rm -rf libltdl echo $aclocal $ACLOCAL_FLAGS $aclocal $ACLOCAL_FLAGS echo $autoheader $autoheader echo $automake --add-missing --include-deps --copy $automake --add-missing --include-deps --copy echo $autoconf $autoconf test -f libtool.m4 || touch libtool.m4 test -f ltdl.m4 || touch ltdl.m4 echo Now run ./configure trap '' 0 diff --git a/cib/main.c b/cib/main.c index 43e7e61780..8cda87bea8 100644 --- a/cib/main.c +++ b/cib/main.c @@ -1,641 +1,636 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if HAVE_LIBXML2 # include #endif #ifdef HAVE_GETOPT_H # include #endif #if HAVE_BZLIB_H # include #endif extern int init_remote_listener(int port); extern gboolean stand_alone; gboolean cib_shutdown_flag = FALSE; enum cib_errors cib_status = cib_ok; #if SUPPORT_HEARTBEAT oc_ev_t *cib_ev_token; ll_cluster_t *hb_conn = NULL; extern void oc_ev_special(const oc_ev_t *, oc_ev_class_t , int ); gboolean cib_register_ha(ll_cluster_t *hb_cluster, const char *client_name); #endif GMainLoop* mainloop = NULL; const char* cib_root = WORKING_DIR; char *cib_our_uname = NULL; gboolean preserve_status = FALSE; gboolean cib_writes_enabled = TRUE; int remote_fd = 0; void usage(const char* cmd, int exit_status); int cib_init(void); gboolean cib_shutdown(int nsig, gpointer unused); void cib_ha_connection_destroy(gpointer user_data); gboolean startCib(const char *filename); extern int write_cib_contents(gpointer p); GTRIGSource *cib_writer = NULL; GHashTable *client_list = NULL; char *channel1 = NULL; char *channel2 = NULL; char *channel3 = NULL; char *channel4 = NULL; char *channel5 = NULL; #define OPTARGS "aswr:V?" void cib_cleanup(void); static void cib_diskwrite_complete(gpointer userdata, int status, int signo, int exitcode) { if(exitcode != LSB_EXIT_OK || signo != 0 || status != 0) { crm_err("Disk write failed: status=%d, signo=%d, exitcode=%d", status, signo, exitcode); if(cib_writes_enabled) { crm_err("Disabling disk writes after write failure"); cib_writes_enabled = FALSE; } } else { crm_debug_2("Disk write passed"); } } int main(int argc, char ** argv) { int flag; int rc = 0; int argerr = 0; #ifdef HAVE_GETOPT_H int option_index = 0; static struct option long_options[] = { {"per-action-cib", 0, 0, 'a'}, {"stand-alone", 0, 0, 's'}, {"disk-writes", 0, 0, 'w'}, {"cib-root", 1, 0, 'r'}, {"verbose", 0, 0, 'V'}, {"help", 0, 0, '?'}, {0, 0, 0, 0} }; #endif struct passwd *pwentry = NULL; crm_log_init(CRM_SYSTEM_CIB, LOG_INFO, TRUE, TRUE, 0, NULL); G_main_add_SignalHandler( G_PRIORITY_HIGH, SIGTERM, cib_shutdown, NULL, NULL); cib_writer = G_main_add_tempproc_trigger( G_PRIORITY_LOW, write_cib_contents, "write_cib_contents", NULL, NULL, NULL, cib_diskwrite_complete); /* EnableProcLogging(); */ set_sigchld_proctrack(G_PRIORITY_HIGH,DEFAULT_MAXDISPATCHTIME); crm_peer_init(); client_list = g_hash_table_new(g_str_hash, g_str_equal); while (1) { #ifdef HAVE_GETOPT_H flag = getopt_long(argc, argv, OPTARGS, long_options, &option_index); #else flag = getopt(argc, argv, OPTARGS); #endif if (flag == -1) break; switch(flag) { case 'V': alter_debug(DEBUG_INC); break; case 's': stand_alone = TRUE; preserve_status = TRUE; cib_writes_enabled = FALSE; pwentry = getpwnam(HA_CCMUSER); CRM_CHECK(pwentry != NULL, crm_perror(LOG_ERR,"Invalid uid (%s) specified", HA_CCMUSER); return 100); rc = setgid(pwentry->pw_gid); if(rc < 0) { crm_perror(LOG_ERR,"Could not set group to %d", pwentry->pw_gid); return 100; } rc = setuid(pwentry->pw_uid); if(rc < 0) { crm_perror(LOG_ERR,"Could not set user to %d", pwentry->pw_uid); return 100; } cl_log_enable_stderr(1); break; case '?': /* Help message */ usage(crm_system_name, LSB_EXIT_OK); break; case 'w': cib_writes_enabled = TRUE; break; case 'r': cib_root = optarg; break; default: ++argerr; break; } } if (optind > argc) { ++argerr; } if (argerr) { usage(crm_system_name,LSB_EXIT_GENERIC); } /* read local config file */ rc = cib_init(); CRM_CHECK(g_hash_table_size(client_list) == 0, crm_warn("Not all clients gone at exit")); cib_cleanup(); #if SUPPORT_HEARTBEAT if(hb_conn) { hb_conn->llc_ops->delete(hb_conn); } #endif crm_info("Done"); return rc; } void cib_cleanup(void) { crm_peer_destroy(); g_hash_table_destroy(client_list); crm_free(cib_our_uname); #if HAVE_LIBXML2 xmlCleanupParser(); #endif crm_free(channel1); crm_free(channel2); crm_free(channel3); crm_free(channel4); crm_free(channel5); } unsigned long cib_num_ops = 0; const char *cib_stat_interval = "10min"; unsigned long cib_num_local = 0, cib_num_updates = 0, cib_num_fail = 0; unsigned long cib_bad_connects = 0, cib_num_timeouts = 0; longclock_t cib_call_time = 0; gboolean cib_stats(gpointer data); gboolean cib_stats(gpointer data) { int local_log_level = LOG_DEBUG; static unsigned long last_stat = 0; unsigned int cib_calls_ms = 0; static unsigned long cib_stat_interval_ms = 0; if(cib_stat_interval_ms == 0) { cib_stat_interval_ms = crm_get_msec(cib_stat_interval); } cib_calls_ms = longclockto_ms(cib_call_time); if((cib_num_ops - last_stat) > 0) { unsigned long calls_diff = cib_num_ops - last_stat; double stat_1 = (1000*cib_calls_ms)/calls_diff; local_log_level = LOG_INFO; do_crm_log(local_log_level, "Processed %lu operations" " (%.2fus average, %lu%% utilization) in the last %s", calls_diff, stat_1, (100*cib_calls_ms)/cib_stat_interval_ms, cib_stat_interval); } do_crm_log_unlikely(local_log_level+1, "\tDetail: %lu operations (%ums total)" " (%lu local, %lu updates, %lu failures," " %lu timeouts, %lu bad connects)", cib_num_ops, cib_calls_ms, cib_num_local, cib_num_updates, cib_num_fail, cib_bad_connects, cib_num_timeouts); last_stat = cib_num_ops; cib_call_time = 0; return TRUE; } #if SUPPORT_HEARTBEAT gboolean ccm_connect(void); static void ccm_connection_destroy(gpointer user_data) { crm_err("CCM connection failed... blocking while we reconnect"); CRM_ASSERT(ccm_connect()); return; } gboolean ccm_connect(void) { gboolean did_fail = TRUE; int num_ccm_fails = 0; int max_ccm_fails = 30; int ret; int cib_ev_fd; while(did_fail) { did_fail = FALSE; crm_info("Registering with CCM..."); ret = oc_ev_register(&cib_ev_token); if (ret != 0) { did_fail = TRUE; } if(did_fail == FALSE) { crm_debug_3("Setting up CCM callbacks"); ret = oc_ev_set_callback( cib_ev_token, OC_EV_MEMB_CLASS, cib_ccm_msg_callback, NULL); if (ret != 0) { crm_warn("CCM callback not set"); did_fail = TRUE; } } if(did_fail == FALSE) { oc_ev_special(cib_ev_token, OC_EV_MEMB_CLASS, 0); crm_debug_3("Activating CCM token"); ret = oc_ev_activate(cib_ev_token, &cib_ev_fd); if (ret != 0){ crm_warn("CCM Activation failed"); did_fail = TRUE; } } if(did_fail) { num_ccm_fails++; oc_ev_unregister(cib_ev_token); if(num_ccm_fails < max_ccm_fails){ crm_warn("CCM Connection failed %d times (%d max)", num_ccm_fails, max_ccm_fails); sleep(3); } else { crm_err("CCM Activation failed %d (max) times", num_ccm_fails); return FALSE; } } } crm_debug("CCM Activation passed... all set to go!"); G_main_add_fd(G_PRIORITY_HIGH, cib_ev_fd, FALSE, cib_ccm_dispatch, cib_ev_token, ccm_connection_destroy); return TRUE; } #endif #if SUPPORT_AIS static gboolean cib_ais_dispatch(AIS_Message *wrapper, char *data, int sender) { xmlNode *xml = NULL; - switch(wrapper->header.id) { - case crm_class_members: - case crm_class_notify: - break; - default: + if(wrapper->header.id == crm_class_cluster) { xml = string2xml(data); if(xml == NULL) { goto bail; } crm_xml_add(xml, F_ORIG, wrapper->sender.uname); crm_xml_add_int(xml, F_SEQ, wrapper->id); cib_peer_callback(xml, NULL); - break; } free_xml(xml); return TRUE; bail: crm_err("Invalid XML: '%.120s'", data); return TRUE; } static void cib_ais_destroy(gpointer user_data) { crm_err("AIS connection terminated"); ais_fd_sync = -1; exit(1); } #endif int cib_init(void) { gboolean was_error = FALSE; if(startCib("cib.xml") == FALSE){ crm_crit("Cannot start CIB... terminating"); exit(1); } if(stand_alone == FALSE) { void *dispatch = cib_ha_peer_callback; void *destroy = cib_ha_connection_destroy; if(is_openais_cluster()) { #if SUPPORT_AIS destroy = cib_ais_destroy; dispatch = cib_ais_dispatch; #endif } if(crm_cluster_connect(&cib_our_uname, NULL, dispatch, destroy, #if SUPPORT_HEARTBEAT &hb_conn #else NULL #endif ) == FALSE){ crm_crit("Cannot sign in to the cluster... terminating"); exit(100); } #if 0 if(is_openais_cluster()) { crm_info("Requesting the list of configured nodes"); send_ais_text( crm_class_members, __FUNCTION__, TRUE, NULL, crm_msg_ais); } #endif #if SUPPORT_HEARTBEAT if(is_heartbeat_cluster()) { if(was_error == FALSE) { if (HA_OK != hb_conn->llc_ops->set_cstatus_callback( hb_conn, cib_client_status_callback, hb_conn)) { crm_err("Cannot set cstatus callback: %s", hb_conn->llc_ops->errmsg(hb_conn)); was_error = TRUE; } } if(was_error == FALSE) { was_error = (ccm_connect() == FALSE); } if(was_error == FALSE) { /* Async get client status information in the cluster */ crm_info("Requesting the list of configured nodes"); hb_conn->llc_ops->client_status( hb_conn, NULL, CRM_SYSTEM_CIB, -1); } } #endif } else { cib_our_uname = crm_strdup("localhost"); } channel1 = crm_strdup(cib_channel_callback); was_error = init_server_ipc_comms( channel1, cib_client_connect, default_ipc_connection_destroy); channel2 = crm_strdup(cib_channel_ro); was_error = was_error || init_server_ipc_comms( channel2, cib_client_connect, default_ipc_connection_destroy); channel3 = crm_strdup(cib_channel_rw); was_error = was_error || init_server_ipc_comms( channel3, cib_client_connect, default_ipc_connection_destroy); if(stand_alone) { if(was_error) { crm_err("Couldnt start"); return 1; } cib_is_master = TRUE; /* Create the mainloop and run it... */ mainloop = g_main_new(FALSE); crm_info("Starting %s mainloop", crm_system_name); g_main_run(mainloop); return_to_orig_privs(); return 0; } if(was_error == FALSE) { /* Create the mainloop and run it... */ mainloop = g_main_new(FALSE); crm_info("Starting %s mainloop", crm_system_name); Gmain_timeout_add( crm_get_msec(cib_stat_interval), cib_stats, NULL); g_main_run(mainloop); return_to_orig_privs(); } else { crm_err("Couldnt start all communication channels, exiting."); } return 0; } void usage(const char* cmd, int exit_status) { FILE* stream; stream = exit_status ? stderr : stdout; fprintf(stream, "usage: %s [-%s]\n", cmd, OPTARGS); fprintf(stream, "\t--%s (-%c)\t\tTurn on debug info." " Additional instances increase verbosity\n", "verbose", 'V'); fprintf(stream, "\t--%s (-%c)\t\tThis help message\n", "help", '?'); fprintf(stream, "\t--%s (-%c)\tAdvanced use only\n", "per-action-cib", 'a'); fprintf(stream, "\t--%s (-%c)\tAdvanced use only\n", "stand-alone", 's'); fprintf(stream, "\t--%s (-%c)\tAdvanced use only\n", "disk-writes", 'w'); fprintf(stream, "\t--%s (-%c)\t\tAdvanced use only\n", "cib-root", 'r'); fflush(stream); exit(exit_status); } void cib_ha_connection_destroy(gpointer user_data) { if(cib_shutdown_flag) { crm_info("Heartbeat disconnection complete... exiting"); } else { crm_err("Heartbeat connection lost! Exiting."); } uninitializeCib(); crm_info("Exiting..."); if (mainloop != NULL && g_main_is_running(mainloop)) { g_main_quit(mainloop); } else { exit(LSB_EXIT_OK); } } static void disconnect_cib_client(gpointer key, gpointer value, gpointer user_data) { cib_client_t *a_client = value; crm_debug_2("Processing client %s/%s... send=%d, recv=%d", crm_str(a_client->name), crm_str(a_client->channel_name), (int)a_client->channel->send_queue->current_qlen, (int)a_client->channel->recv_queue->current_qlen); if(a_client->channel->ch_status == IPC_CONNECT) { a_client->channel->ops->resume_io(a_client->channel); if(a_client->channel->send_queue->current_qlen != 0 || a_client->channel->recv_queue->current_qlen != 0) { crm_info("Flushed messages to/from %s/%s... send=%d, recv=%d", crm_str(a_client->name), crm_str(a_client->channel_name), (int)a_client->channel->send_queue->current_qlen, (int)a_client->channel->recv_queue->current_qlen); } } if(a_client->channel->ch_status == IPC_CONNECT) { crm_warn("Disconnecting %s/%s...", crm_str(a_client->name), crm_str(a_client->channel_name)); a_client->channel->ops->disconnect(a_client->channel); } } extern gboolean cib_process_disconnect( IPC_Channel *channel, cib_client_t *cib_client); gboolean cib_shutdown(int nsig, gpointer unused) { if(cib_shutdown_flag == FALSE) { cib_shutdown_flag = TRUE; crm_debug("Disconnecting %d clients", g_hash_table_size(client_list)); g_hash_table_foreach(client_list, disconnect_cib_client, NULL); crm_info("Disconnected %d clients", g_hash_table_size(client_list)); cib_process_disconnect(NULL, NULL); } else { crm_info("Waiting for %d clients to disconnect...", g_hash_table_size(client_list)); } return TRUE; } gboolean startCib(const char *filename) { gboolean active = FALSE; xmlNode *cib = readCibXmlFile(cib_root, filename, !preserve_status); CRM_ASSERT(cib != NULL); if(activateCibXml(cib, TRUE, "start") == 0) { int port = 0; const char *port_s = crm_element_value(cib, "remote_access_port"); active = TRUE; if(port_s) { port = crm_parse_int(port_s, NULL); remote_fd = init_remote_listener(port); } crm_info("CIB Initialization completed successfully"); } return active; } diff --git a/crmd/ccm.c b/crmd/ccm.c index 7459ebe437..bc79f076cd 100644 --- a/crmd/ccm.c +++ b/crmd/ccm.c @@ -1,476 +1,476 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ /* put these first so that uuid_t is defined without conflicts */ #include #if SUPPORT_HEARTBEAT #include #include #endif #include #include #include #include #include #include #include #include #include #include #include #include gboolean membership_flux_hack = FALSE; void post_cache_update(int instance); #if SUPPORT_HEARTBEAT oc_ev_t *fsa_ev_token; void oc_ev_special(const oc_ev_t *, oc_ev_class_t , int ); void crmd_ccm_msg_callback( oc_ed_t event, void *cookie, size_t size, const void *data); #endif void ghash_update_cib_node(gpointer key, gpointer value, gpointer user_data); void check_dead_member(const char *uname, GHashTable *members); void reap_dead_ccm_nodes(gpointer key, gpointer value, gpointer user_data); #define CCM_EVENT_DETAIL 0 #define CCM_EVENT_DETAIL_PARTIAL 0 int num_ccm_register_fails = 0; int max_ccm_register_fails = 30; int last_peer_update = 0; extern GHashTable *voted; struct update_data_s { const char *state; const char *caller; xmlNode *updates; gboolean overwrite_join; }; void reap_dead_ccm_nodes(gpointer key, gpointer value, gpointer user_data) { crm_node_t *node = value; if(crm_is_member_active(node) == FALSE) { check_dead_member(node->uname, NULL); } } extern gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source); void check_dead_member(const char *uname, GHashTable *members) { CRM_CHECK(uname != NULL, return); if(members != NULL && g_hash_table_lookup(members, uname) != NULL) { crm_err("%s didnt really leave the membership!", uname); return; } erase_node_from_join(uname); if(voted != NULL) { g_hash_table_remove(voted, uname); } if(safe_str_eq(fsa_our_uname, uname)) { crm_err("We're not part of the cluster anymore"); } else if(AM_I_DC == FALSE && safe_str_eq(uname, fsa_our_dc)) { crm_warn("Our DC node (%s) left the cluster", uname); register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); } else if(fsa_state == S_INTEGRATION || fsa_state == S_FINALIZE_JOIN) { check_join_state(fsa_state, __FUNCTION__); } } /* A_CCM_CONNECT */ void do_ccm_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { #if SUPPORT_HEARTBEAT if(is_heartbeat_cluster()) { if(action & A_CCM_DISCONNECT){ set_bit_inplace(fsa_input_register, R_CCM_DISCONNECTED); oc_ev_unregister(fsa_ev_token); } if(action & A_CCM_CONNECT) { int ret; int fsa_ev_fd; gboolean did_fail = FALSE; crm_peer_init(); crm_debug_3("Registering with CCM"); clear_bit_inplace(fsa_input_register, R_CCM_DISCONNECTED); ret = oc_ev_register(&fsa_ev_token); if (ret != 0) { crm_warn("CCM registration failed"); did_fail = TRUE; } if(did_fail == FALSE) { crm_debug_3("Setting up CCM callbacks"); ret = oc_ev_set_callback(fsa_ev_token, OC_EV_MEMB_CLASS, crmd_ccm_msg_callback, NULL); if (ret != 0) { crm_warn("CCM callback not set"); did_fail = TRUE; } } if(did_fail == FALSE) { oc_ev_special(fsa_ev_token, OC_EV_MEMB_CLASS, 0/*don't care*/); crm_debug_3("Activating CCM token"); ret = oc_ev_activate(fsa_ev_token, &fsa_ev_fd); if (ret != 0){ crm_warn("CCM Activation failed"); did_fail = TRUE; } } if(did_fail) { num_ccm_register_fails++; oc_ev_unregister(fsa_ev_token); if(num_ccm_register_fails < max_ccm_register_fails) { crm_warn("CCM Connection failed" " %d times (%d max)", num_ccm_register_fails, max_ccm_register_fails); crm_timer_start(wait_timer); crmd_fsa_stall(NULL); return; } else { crm_err("CCM Activation failed %d (max) times", num_ccm_register_fails); register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL); return; } } crm_info("CCM connection established..." " waiting for first callback"); G_main_add_fd(G_PRIORITY_HIGH, fsa_ev_fd, FALSE, ccm_dispatch, fsa_ev_token, default_ipc_connection_destroy); } } #endif if(action & ~(A_CCM_CONNECT|A_CCM_DISCONNECT)) { crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__); } } #if SUPPORT_HEARTBEAT void ccm_event_detail(const oc_ev_membership_t *oc, oc_ed_t event) { int lpc; gboolean member = FALSE; member = FALSE; crm_debug_2("-----------------------"); crm_info("%s: trans=%d, nodes=%d, new=%d, lost=%d n_idx=%d, " "new_idx=%d, old_idx=%d", ccm_event_name(event), oc->m_instance, oc->m_n_member, oc->m_n_in, oc->m_n_out, oc->m_memb_idx, oc->m_in_idx, oc->m_out_idx); #if !CCM_EVENT_DETAIL_PARTIAL for(lpc=0; lpc < oc->m_n_member; lpc++) { crm_info("\tCURRENT: %s [nodeid=%d, born=%d]", oc->m_array[oc->m_memb_idx+lpc].node_uname, oc->m_array[oc->m_memb_idx+lpc].node_id, oc->m_array[oc->m_memb_idx+lpc].node_born_on); if(safe_str_eq(fsa_our_uname, oc->m_array[oc->m_memb_idx+lpc].node_uname)) { member = TRUE; } } if (member == FALSE) { crm_warn("MY NODE IS NOT IN CCM THE MEMBERSHIP LIST"); } #endif for(lpc=0; lpc<(int)oc->m_n_in; lpc++) { crm_info("\tNEW: %s [nodeid=%d, born=%d]", oc->m_array[oc->m_in_idx+lpc].node_uname, oc->m_array[oc->m_in_idx+lpc].node_id, oc->m_array[oc->m_in_idx+lpc].node_born_on); } for(lpc=0; lpc<(int)oc->m_n_out; lpc++) { crm_info("\tLOST: %s [nodeid=%d, born=%d]", oc->m_array[oc->m_out_idx+lpc].node_uname, oc->m_array[oc->m_out_idx+lpc].node_id, oc->m_array[oc->m_out_idx+lpc].node_born_on); } crm_debug_2("-----------------------"); } #endif gboolean ever_had_quorum = FALSE; void post_cache_update(int instance) { xmlNode *no_op = NULL; crm_peer_seq = instance; - ever_had_quorum |= crm_have_quorum; crm_debug("Updated cache after membership event %d.", instance); g_hash_table_foreach(crm_peer_cache, reap_dead_ccm_nodes, NULL); set_bit_inplace(fsa_input_register, R_CCM_DATA); if(AM_I_DC) { populate_cib_nodes(FALSE); do_update_cib_nodes(FALSE, __FUNCTION__); } /* Membership changed, remind everyone we're here. * This will aid detection of duplicate DCs */ no_op = create_request( CRM_OP_NOOP, NULL, NULL, CRM_SYSTEM_CRMD, AM_I_DC?CRM_SYSTEM_DC:CRM_SYSTEM_CRMD, NULL); send_cluster_message(NULL, crm_msg_crmd, no_op, FALSE); free_xml(no_op); } /* A_CCM_UPDATE_CACHE */ /* * Take the opportunity to update the node status in the CIB as well */ #if SUPPORT_HEARTBEAT void do_ccm_update_cache( enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, oc_ed_t event, const oc_ev_membership_t *oc, xmlNode *xml) { unsigned long long instance = 0; unsigned int lpc = 0; if(is_heartbeat_cluster()) { CRM_ASSERT(oc != NULL); instance = oc->m_instance; } CRM_ASSERT(crm_peer_seq <= instance); switch(cur_state) { case S_STOPPING: case S_TERMINATE: case S_HALT: crm_debug("Ignoring %s CCM event %llu, we're in state %s", ccm_event_name(event), instance, fsa_state2string(cur_state)); return; case S_ELECTION: register_fsa_action(A_ELECTION_CHECK); break; default: break; } if(is_heartbeat_cluster()) { ccm_event_detail(oc, event); /*--*-- Recently Dead Member Nodes --*--*/ for(lpc=0; lpc < oc->m_n_out; lpc++) { crm_update_ccm_node(oc, lpc+oc->m_out_idx, CRM_NODE_LOST, instance); } /*--*-- All Member Nodes --*--*/ for(lpc=0; lpc < oc->m_n_member; lpc++) { crm_update_ccm_node(oc, lpc+oc->m_memb_idx, CRM_NODE_ACTIVE, instance); } } if(event == OC_EV_MS_EVICTED) { crm_update_peer( 0, 0, 0, -1, 0, fsa_our_uuid, fsa_our_uname, NULL, CRM_NODE_EVICTED); /* todo: drop back to S_PENDING instead */ /* get out... NOW! * * go via the error recovery process so that HA will * restart us if required */ register_fsa_error_adv(cause, I_ERROR, NULL, NULL, __FUNCTION__); } post_cache_update(instance); return; } #endif static void ccm_node_update_complete(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { fsa_data_t *msg_data = NULL; last_peer_update = 0; if(rc == cib_ok) { crm_debug_2("Node update %d complete", call_id); } else { crm_err("Node update %d failed", call_id); crm_log_xml(LOG_DEBUG, "failed", msg); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } void ghash_update_cib_node(gpointer key, gpointer value, gpointer user_data) { xmlNode *tmp1 = NULL; const char *join = NULL; crm_node_t *node = value; struct update_data_s* data = (struct update_data_s*)user_data; data->state = XML_BOOLEAN_NO; if(safe_str_eq(node->state, CRM_NODE_ACTIVE)) { data->state = XML_BOOLEAN_YES; } crm_debug("Updating %s: %s (overwrite=%s) hash_size=%d", node->uname, data->state, data->overwrite_join?"true":"false", g_hash_table_size(confirmed_nodes)); if(data->overwrite_join) { if((node->processes & crm_proc_crmd) == FALSE) { join = CRMD_JOINSTATE_DOWN; } else { const char *peer_member = g_hash_table_lookup( confirmed_nodes, node->uname); if(peer_member != NULL) { join = CRMD_JOINSTATE_MEMBER; } else { join = CRMD_JOINSTATE_PENDING; } } } tmp1 = create_node_state( node->uname, (node->processes&crm_proc_ais)?ACTIVESTATUS:DEADSTATUS, data->state, (node->processes&crm_proc_crmd)?ONLINESTATUS:OFFLINESTATUS, join, NULL, FALSE, data->caller); add_node_copy(data->updates, tmp1); free_xml(tmp1); } void do_update_cib_nodes(gboolean overwrite, const char *caller) { int call_id = 0; int call_options = cib_scope_local|cib_quorum_override; struct update_data_s update_data; xmlNode *fragment = NULL; if(crm_peer_cache == NULL) { /* We got a replace notification before being connected to * the CCM. * So there is no need to update the local CIB with our values * - since we have none. */ return; } else if(AM_I_DC == FALSE) { return; } fragment = create_xml_node(NULL, XML_CIB_TAG_STATUS); update_data.caller = caller; update_data.updates = fragment; update_data.overwrite_join = overwrite; g_hash_table_foreach(crm_peer_cache, ghash_update_cib_node, &update_data); fsa_cib_update(XML_CIB_TAG_STATUS, fragment, call_options, call_id); add_cib_op_callback(fsa_cib_conn, call_id, FALSE, NULL, ccm_node_update_complete); last_peer_update = call_id; free_xml(fragment); } static void cib_quorum_update_complete( xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { fsa_data_t *msg_data = NULL; if(rc == cib_ok) { crm_debug_2("Quorum update %d complete", call_id); } else { crm_err("Quorum update %d failed", call_id); crm_log_xml(LOG_DEBUG, "failed", msg); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } void crm_update_quorum(gboolean quorum, gboolean force_update) { + ever_had_quorum |= quorum; if(AM_I_DC && (force_update || fsa_has_quorum != quorum)) { int call_id = 0; xmlNode *update = NULL; int call_options = cib_scope_local|cib_quorum_override; update = create_xml_node(NULL, XML_TAG_CIB); crm_xml_add_int(update, XML_ATTR_HAVE_QUORUM, quorum); set_uuid(update, XML_ATTR_DC_UUID, fsa_our_uname); fsa_cib_update(XML_TAG_CIB, update, call_options, call_id); crm_info("Updating quorum status to %s (call=%d)", quorum?"true":"false", call_id); add_cib_op_callback(fsa_cib_conn, call_id, FALSE, NULL, cib_quorum_update_complete); free_xml(update); } fsa_has_quorum = quorum; } diff --git a/crmd/control.c b/crmd/control.c index 05d370494c..1b390dc826 100644 --- a/crmd/control.c +++ b/crmd/control.c @@ -1,960 +1,969 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include char *ipc_server = NULL; extern void post_cache_update(int seq); extern void crmd_ha_connection_destroy(gpointer user_data); gboolean crm_shutdown(int nsig, gpointer unused); gboolean crm_read_options(gpointer user_data); gboolean fsa_has_quorum = FALSE; GHashTable *ipc_clients = NULL; GTRIGSource *fsa_source = NULL; GTRIGSource *config_read = NULL; /* A_HA_CONNECT */ #if SUPPORT_AIS extern void crmd_ha_msg_filter(xmlNode * msg); static gboolean crm_ais_dispatch(AIS_Message *wrapper, char *data, int sender) { int seq = 0; xmlNode *xml = NULL; const char *seq_s = NULL; - if(wrapper->header.id == crm_class_notify) { - return TRUE; - } - - xml = string2xml(data); - if(xml == NULL) { - crm_err("Message received: %d:'%.120s'", wrapper->id, data); - return TRUE; - } - crm_xml_add(xml, F_ORIG, wrapper->sender.uname); crm_xml_add_int(xml, F_SEQ, wrapper->id); switch(wrapper->header.id) { case crm_class_members: + xml = string2xml(data); seq_s = crm_element_value(xml, "seq"); seq = crm_int_helper(seq_s, NULL); set_bit_inplace(fsa_input_register, R_PEER_DATA); - post_cache_update(seq); + + /* fall through */ + case crm_class_quorum: crm_update_quorum(crm_have_quorum, FALSE); + if(AM_I_DC) { + struct crm_ais_quorum_resp_s *msg = (struct crm_ais_quorum_resp_s *)wrapper; + char *votes = crm_itoa(msg->expected_votes); + crm_info("Updating %s=%d", XML_ATTR_EXPECTED_VOTES, msg->expected_votes); + update_attr(fsa_cib_conn, cib_inhibit_notify, XML_CIB_TAG_CRMCONFIG, + NULL, NULL, NULL, XML_ATTR_EXPECTED_VOTES, votes, FALSE); + crm_free(votes); + } break; - default: + case crm_class_cluster: + xml = string2xml(data); crmd_ha_msg_filter(xml); break; + case crm_class_notify: + case crm_class_nodeid: + case crm_class_rmpeer: + break; } free_xml(xml); return TRUE; } static void crm_ais_destroy(gpointer user_data) { crm_err("AIS connection terminated"); ais_fd_sync = -1; exit(1); } #endif void do_ha_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { gboolean registered = FALSE; if(action & A_HA_DISCONNECT) { if(is_openais_cluster()) { crm_peer_destroy(); crm_info("Disconnected from OpenAIS"); #if SUPPORT_HEARTBEAT } else if(fsa_cluster_conn != NULL) { set_bit_inplace(fsa_input_register, R_HA_DISCONNECTED); fsa_cluster_conn->llc_ops->signoff(fsa_cluster_conn, FALSE); crm_info("Disconnected from Heartbeat"); #endif } } if(action & A_HA_CONNECT) { void *dispatch = NULL; void *destroy = NULL; if(is_openais_cluster()) { #if SUPPORT_AIS destroy = crm_ais_destroy; dispatch = crm_ais_dispatch; #endif } else if(is_heartbeat_cluster()) { #if SUPPORT_HEARTBEAT dispatch = crmd_ha_msg_callback; destroy = crmd_ha_connection_destroy; #endif } crm_set_status_callback(&ais_status_callback); registered = crm_cluster_connect( &fsa_our_uname, &fsa_our_uuid, dispatch, destroy, #if SUPPORT_HEARTBEAT &fsa_cluster_conn #else NULL #endif ); #if SUPPORT_HEARTBEAT if(is_heartbeat_cluster()) { crm_debug_3("Be informed of Node Status changes"); if (registered && fsa_cluster_conn->llc_ops->set_nstatus_callback( fsa_cluster_conn, crmd_ha_status_callback, fsa_cluster_conn) != HA_OK){ crm_err("Cannot set nstatus callback: %s", fsa_cluster_conn->llc_ops->errmsg(fsa_cluster_conn)); registered = FALSE; } crm_debug_3("Be informed of CRM Client Status changes"); if (registered && fsa_cluster_conn->llc_ops->set_cstatus_callback( fsa_cluster_conn, crmd_client_status_callback, fsa_cluster_conn) != HA_OK) { crm_err("Cannot set cstatus callback: %s", fsa_cluster_conn->llc_ops->errmsg(fsa_cluster_conn)); registered = FALSE; } if(registered) { crm_debug_3("Requesting an initial dump of CRMD client_status"); fsa_cluster_conn->llc_ops->client_status( fsa_cluster_conn, NULL, CRM_SYSTEM_CRMD, -1); } } #endif if(registered == FALSE) { set_bit_inplace(fsa_input_register, R_HA_DISCONNECTED); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); return; } clear_bit_inplace(fsa_input_register, R_HA_DISCONNECTED); crm_info("Connected to Heartbeat"); } if(action & ~(A_HA_CONNECT|A_HA_DISCONNECT)) { crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__); } } /* A_SHUTDOWN */ void do_shutdown(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { /* just in case */ set_bit_inplace(fsa_input_register, R_SHUTDOWN); if(is_heartbeat_cluster()) { if(is_set(fsa_input_register, pe_subsystem->flag_connected)) { crm_info("Terminating the %s", pe_subsystem->name); if(stop_subsystem(pe_subsystem, TRUE) == FALSE) { /* its gone... */ crm_err("Faking %s exit", pe_subsystem->name); clear_bit_inplace(fsa_input_register, pe_subsystem->flag_connected); } else { crm_info("Waiting for subsystems to exit"); crmd_fsa_stall(NULL); } } crm_info("All subsystems stopped, continuing"); } } /* A_SHUTDOWN_REQ */ void do_shutdown_req(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { xmlNode *msg = NULL; crm_info("Sending shutdown request to DC: %s", crm_str(fsa_our_dc)); msg = create_request( CRM_OP_SHUTDOWN_REQ, NULL, NULL, CRM_SYSTEM_DC, CRM_SYSTEM_CRMD, NULL); /* set_bit_inplace(fsa_input_register, R_STAYDOWN); */ if(send_request(msg, NULL) == FALSE) { if(AM_I_DC) { crm_info("Processing shutdown locally"); } else { register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } free_xml(msg); } extern char *max_generation_from; extern xmlNode *max_generation_xml; extern GHashTable *resources; extern GHashTable *voted; extern GHashTable *reload_hash; void log_connected_client(gpointer key, gpointer value, gpointer user_data); void log_connected_client(gpointer key, gpointer value, gpointer user_data) { crmd_client_t *client = value; crm_err("%s is still connected at exit", client->table_key); } static void free_mem(fsa_data_t *msg_data) { #if SUPPORT_HEARTBEAT if(fsa_cluster_conn) { fsa_cluster_conn->llc_ops->delete(fsa_cluster_conn); fsa_cluster_conn = NULL; } #endif slist_destroy(fsa_data_t, fsa_data, fsa_message_queue, crm_info("Dropping %s: [ state=%s cause=%s origin=%s ]", fsa_input2string(fsa_data->fsa_input), fsa_state2string(fsa_state), fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin); delete_fsa_input(fsa_data); ); delete_fsa_input(msg_data); if(ipc_clients) { crm_debug("Number of connected clients: %d", g_hash_table_size(ipc_clients)); /* g_hash_table_foreach(ipc_clients, log_connected_client, NULL); */ g_hash_table_destroy(ipc_clients); } empty_uuid_cache(); crm_peer_destroy(); clear_bit_inplace(fsa_input_register, R_CCM_DATA); if(te_subsystem->client && te_subsystem->client->client_source) { crm_debug("Full destroy: TE"); G_main_del_IPC_Channel(te_subsystem->client->client_source); } else { crm_debug("Partial destroy: TE"); crmd_ipc_connection_destroy(te_subsystem->client); } crm_free(te_subsystem); if(pe_subsystem->client && pe_subsystem->client->client_source) { crm_debug("Full destroy: PE"); G_main_del_IPC_Channel(pe_subsystem->client->client_source); } else { crm_debug("Partial destroy: PE"); crmd_ipc_connection_destroy(pe_subsystem->client); } crm_free(pe_subsystem); crm_free(cib_subsystem); if(integrated_nodes) { g_hash_table_destroy(integrated_nodes); } if(finalized_nodes) { g_hash_table_destroy(finalized_nodes); } if(confirmed_nodes) { g_hash_table_destroy(confirmed_nodes); } if(reload_hash) { g_hash_table_destroy(reload_hash); } if(resources) { g_hash_table_destroy(resources); } if(voted) { g_hash_table_destroy(voted); } cib_delete(fsa_cib_conn); fsa_cib_conn = NULL; if(fsa_lrm_conn) { fsa_lrm_conn->lrm_ops->delete(fsa_lrm_conn); } crm_free(integration_timer); crm_free(finalization_timer); crm_free(election_trigger); crm_free(election_timeout); crm_free(shutdown_escalation_timer); crm_free(wait_timer); crm_free(recheck_timer); crm_free(fsa_our_dc_version); crm_free(fsa_our_uname); crm_free(fsa_our_uuid); crm_free(fsa_our_dc); crm_free(ipc_server); crm_free(max_generation_from); free_xml(max_generation_xml); } /* A_EXIT_0, A_EXIT_1 */ void do_exit(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { int exit_code = 0; int log_level = LOG_INFO; const char *exit_type = "gracefully"; if(action & A_EXIT_1) { exit_code = 1; log_level = LOG_ERR; exit_type = "forcefully"; } verify_stopped(cur_state, LOG_ERR); do_crm_log(log_level, "Performing %s - %s exiting the CRMd", fsa_action2string(action), exit_type); if(is_set(fsa_input_register, R_IN_RECOVERY)) { crm_err("Could not recover from internal error"); exit_code = 2; } if(is_set(fsa_input_register, R_STAYDOWN)) { crm_warn("Inhibiting respawn by Heartbeat"); exit_code = 100; } free_mem(msg_data); crm_info("[%s] stopped (%d)", crm_system_name, exit_code); cl_flush_logs(); exit(exit_code); } /* A_STARTUP */ void do_startup(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { int was_error = 0; int interval = 1; /* seconds between DC heartbeats */ crm_debug("Registering Signal Handlers"); G_main_add_SignalHandler( G_PRIORITY_HIGH, SIGTERM, crm_shutdown, NULL, NULL); fsa_source = G_main_add_TriggerHandler( G_PRIORITY_HIGH, crm_fsa_trigger, NULL, NULL); config_read = G_main_add_TriggerHandler( G_PRIORITY_HIGH, crm_read_options, NULL, NULL); ipc_clients = g_hash_table_new(g_str_hash, g_str_equal); crm_debug("Creating CIB and LRM objects"); fsa_cib_conn = cib_new(); fsa_lrm_conn = ll_lrm_new(XML_CIB_TAG_LRM); /* set up the timers */ crm_malloc0(integration_timer, sizeof(fsa_timer_t)); crm_malloc0(finalization_timer, sizeof(fsa_timer_t)); crm_malloc0(election_trigger, sizeof(fsa_timer_t)); crm_malloc0(election_timeout, sizeof(fsa_timer_t)); crm_malloc0(shutdown_escalation_timer, sizeof(fsa_timer_t)); crm_malloc0(wait_timer, sizeof(fsa_timer_t)); crm_malloc0(recheck_timer, sizeof(fsa_timer_t)); interval = interval * 1000; if(election_trigger != NULL) { election_trigger->source_id = 0; election_trigger->period_ms = -1; election_trigger->fsa_input = I_DC_TIMEOUT; election_trigger->callback = crm_timer_popped; election_trigger->repeat = FALSE; } else { was_error = TRUE; } if(election_timeout != NULL) { election_timeout->source_id = 0; election_timeout->period_ms = -1; election_timeout->fsa_input = I_ELECTION_DC; election_timeout->callback = crm_timer_popped; election_timeout->repeat = FALSE; } else { was_error = TRUE; } if(integration_timer != NULL) { integration_timer->source_id = 0; integration_timer->period_ms = -1; integration_timer->fsa_input = I_INTEGRATED; integration_timer->callback = crm_timer_popped; integration_timer->repeat = FALSE; } else { was_error = TRUE; } if(finalization_timer != NULL) { finalization_timer->source_id = 0; finalization_timer->period_ms = -1; finalization_timer->fsa_input = I_FINALIZED; finalization_timer->callback = crm_timer_popped; finalization_timer->repeat = FALSE; /* for possible enabling... a bug in the join protocol left * a slave in S_PENDING while we think its in S_NOT_DC * * raising I_FINALIZED put us into a transition loop which is * never resolved. * in this loop we continually send probes which the node * NACK's because its in S_PENDING * * if we have nodes where heartbeat is active but the * CRM is not... then this will be handled in the * integration phase */ finalization_timer->fsa_input = I_ELECTION; } else { was_error = TRUE; } if(shutdown_escalation_timer != NULL) { shutdown_escalation_timer->source_id = 0; shutdown_escalation_timer->period_ms = -1; shutdown_escalation_timer->fsa_input = I_STOP; shutdown_escalation_timer->callback = crm_timer_popped; shutdown_escalation_timer->repeat = FALSE; } else { was_error = TRUE; } if(wait_timer != NULL) { wait_timer->source_id = 0; wait_timer->period_ms = 2000; wait_timer->fsa_input = I_NULL; wait_timer->callback = crm_timer_popped; wait_timer->repeat = FALSE; } else { was_error = TRUE; } if(recheck_timer != NULL) { recheck_timer->source_id = 0; recheck_timer->period_ms = -1; recheck_timer->fsa_input = I_PE_CALC; recheck_timer->callback = crm_timer_popped; recheck_timer->repeat = FALSE; } else { was_error = TRUE; } /* set up the sub systems */ crm_malloc0(cib_subsystem, sizeof(struct crm_subsystem_s)); crm_malloc0(te_subsystem, sizeof(struct crm_subsystem_s)); crm_malloc0(pe_subsystem, sizeof(struct crm_subsystem_s)); if(cib_subsystem != NULL) { cib_subsystem->pid = -1; cib_subsystem->path = BIN_DIR; cib_subsystem->name = CRM_SYSTEM_CIB; cib_subsystem->command = BIN_DIR"/"CRM_SYSTEM_CIB; cib_subsystem->args = "-VVc"; cib_subsystem->flag_connected = R_CIB_CONNECTED; cib_subsystem->flag_required = R_CIB_REQUIRED; } else { was_error = TRUE; } if(te_subsystem != NULL) { te_subsystem->pid = -1; te_subsystem->path = BIN_DIR; te_subsystem->name = CRM_SYSTEM_TENGINE; te_subsystem->command = BIN_DIR"/"CRM_SYSTEM_TENGINE; te_subsystem->args = NULL; te_subsystem->flag_connected = R_TE_CONNECTED; te_subsystem->flag_required = R_TE_REQUIRED; } else { was_error = TRUE; } if(pe_subsystem != NULL) { pe_subsystem->pid = -1; pe_subsystem->path = BIN_DIR; pe_subsystem->name = CRM_SYSTEM_PENGINE; pe_subsystem->command = BIN_DIR"/"CRM_SYSTEM_PENGINE; pe_subsystem->args = NULL; pe_subsystem->flag_connected = R_PE_CONNECTED; pe_subsystem->flag_required = R_PE_REQUIRED; } else { was_error = TRUE; } if(was_error) { register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } welcomed_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); integrated_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); finalized_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); confirmed_nodes = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); set_sigchld_proctrack(G_PRIORITY_HIGH,DEFAULT_MAXDISPATCHTIME); } /* A_STOP */ void do_stop(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL); } /* A_STARTED */ void do_started(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { if(cur_state != S_STARTING) { crm_err("Start cancelled... %s", fsa_state2string(cur_state)); return; } else if(is_set(fsa_input_register, R_CCM_DATA) == FALSE) { crm_info("Delaying start, CCM (%.16llx) not connected", R_CCM_DATA); crmd_fsa_stall(NULL); return; } else if(is_set(fsa_input_register, R_LRM_CONNECTED) == FALSE) { crm_info("Delaying start, LRM (%.16llx) not connected", R_LRM_CONNECTED); crmd_fsa_stall(NULL); return; } else if(is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) { crm_info("Delaying start, CIB (%.16llx) not connected", R_CIB_CONNECTED); crmd_fsa_stall(NULL); return; } else if(is_set(fsa_input_register, R_READ_CONFIG) == FALSE) { crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG); crmd_fsa_stall(NULL); return; } else if(is_set(fsa_input_register, R_PEER_DATA) == FALSE) { HA_Message *msg = NULL; /* try reading from HA */ crm_info("Delaying start, Peer data (%.16llx) not recieved", R_PEER_DATA); crm_debug_3("Looking for a HA message"); #if SUPPORT_HEARTBEAT if(is_heartbeat_cluster()) { msg = fsa_cluster_conn->llc_ops->readmsg(fsa_cluster_conn, 0); } #endif if(msg != NULL) { crm_debug_3("There was a HA message"); crm_msg_del(msg); } /* this should no longer be required */ /* crm_timer_start(wait_timer); */ crmd_fsa_stall(NULL); return; } crm_debug("Init server comms"); if(ipc_server == NULL) { ipc_server = crm_strdup(CRM_SYSTEM_CRMD); } if(init_server_ipc_comms(ipc_server, crmd_client_connect, default_ipc_connection_destroy)) { crm_err("Couldn't start IPC server"); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } crm_info("The local CRM is operational"); clear_bit_inplace(fsa_input_register, R_STARTING); register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL); } /* A_RECOVER */ void do_recover(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { set_bit_inplace(fsa_input_register, R_IN_RECOVERY); crm_err("Action %s (%.16llx) not supported", fsa_action2string(action), action); register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL); } pe_cluster_option crmd_opts[] = { /* name, old-name, validate, default, description */ { XML_CONFIG_ATTR_DC_DEADTIME, "dc_deadtime", "time", NULL, "10s", &check_time, "How long to wait for a response from other nodes during startup.", "The \"correct\" value will depend on the speed and load of your network." }, { XML_CONFIG_ATTR_RECHECK, "cluster_recheck_interval", "time", "Zero disables polling. Positive values are an interval in seconds (unless other SI units are specified. eg. 5min)", "15min", &check_timer, "Polling interval for time based changes to options, resource parameters and constraints.", "The Cluster is primarily event driven, however the configuration can have elements that change based on time. To ensure these changes take effect, we can optionally poll the cluster's status for changes." }, { XML_CONFIG_ATTR_ELECTION_FAIL, "election_timeout", "time", NULL, "2min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." }, { XML_CONFIG_ATTR_FORCE_QUIT, "shutdown_escalation", "time", NULL, "20min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." }, { "crmd-integration-timeout", NULL, "time", NULL, "3min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." }, { "crmd-finalization-timeout", NULL, "time", NULL, "30min", &check_timer, "*** Advanced Use Only ***.", "If you need to adjust this value, it probably indicates the presence of a bug." }, }; void crmd_metadata(void) { config_metadata("CRM Daemon", "1.0", "CRM Daemon Options", "This is a fake resource that details the options that can be configured for the CRM Daemon.", crmd_opts, DIMOF(crmd_opts)); } static void verify_crmd_options(GHashTable *options) { verify_all_options(options, crmd_opts, DIMOF(crmd_opts)); } static const char * crmd_pref(GHashTable *options, const char *name) { return get_cluster_pref(options, crmd_opts, DIMOF(crmd_opts), name); } static void config_query_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { const char *value = NULL; GHashTable *config_hash = NULL; ha_time_t *now = new_ha_date(TRUE); if(rc != cib_ok) { fsa_data_t *msg_data = NULL; crm_err("Local CIB query resulted in an error: %s", cib_error2string(rc)); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); if(rc == cib_bad_permissions || rc == cib_dtd_validation || rc == cib_bad_digest || rc == cib_bad_config) { crm_err("The cluster is mis-configured - shutting down and staying down"); set_bit_inplace(fsa_input_register, R_STAYDOWN); } goto bail; } crm_info("Call %d : Parsing CIB options", call_id); config_hash = g_hash_table_new_full( g_str_hash,g_str_equal, g_hash_destroy_str,g_hash_destroy_str); unpack_instance_attributes( output, XML_CIB_TAG_PROPSET, NULL, config_hash, CIB_OPTIONS_FIRST, FALSE, now); verify_crmd_options(config_hash); value = crmd_pref(config_hash, XML_CONFIG_ATTR_DC_DEADTIME); election_trigger->period_ms = crm_get_msec(value); value = crmd_pref(config_hash, XML_CONFIG_ATTR_FORCE_QUIT); shutdown_escalation_timer->period_ms = crm_get_msec(value); value = crmd_pref(config_hash, XML_CONFIG_ATTR_ELECTION_FAIL); election_timeout->period_ms = crm_get_msec(value); value = crmd_pref(config_hash, XML_CONFIG_ATTR_RECHECK); recheck_timer->period_ms = crm_get_msec(value); crm_info("Checking for expired actions every %dms", recheck_timer->period_ms); value = crmd_pref(config_hash, "crmd-integration-timeout"); integration_timer->period_ms = crm_get_msec(value); value = crmd_pref(config_hash, "crmd-finalization-timeout"); finalization_timer->period_ms = crm_get_msec(value); + value = crmd_pref(config_hash, XML_ATTR_EXPECTED_VOTES); + send_ais_text(crm_class_quorum, value, TRUE, NULL, crm_msg_ais); + set_bit_inplace(fsa_input_register, R_READ_CONFIG); crm_debug_3("Triggering FSA: %s", __FUNCTION__); G_main_set_trigger(fsa_source); g_hash_table_destroy(config_hash); bail: free_ha_date(now); } gboolean crm_read_options(gpointer user_data) { int call_id = fsa_cib_conn->cmds->query( fsa_cib_conn, XML_CIB_TAG_CRMCONFIG, NULL, cib_scope_local); add_cib_op_callback(fsa_cib_conn, call_id, FALSE, NULL, config_query_callback); crm_debug_2("Querying the CIB... call %d", call_id); return TRUE; } /* A_READCONFIG */ void do_read_config(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { G_main_set_trigger(config_read); } gboolean crm_shutdown(int nsig, gpointer unused) { if (crmd_mainloop != NULL && g_main_is_running(crmd_mainloop)) { if(is_set(fsa_input_register, R_SHUTDOWN)) { crm_err("Escalating the shutdown"); register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL); } else { crm_info("Requesting shutdown"); set_bit_inplace(fsa_input_register, R_SHUTDOWN); register_fsa_input(C_SHUTDOWN,I_SHUTDOWN,NULL); if(shutdown_escalation_timer->period_ms < 1) { GHashTable *config_hash = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); const char *value = crmd_pref( config_hash, XML_CONFIG_ATTR_FORCE_QUIT); int msec = crm_get_msec(value); crm_info("Using default shutdown escalation: %dms", msec); shutdown_escalation_timer->period_ms = msec; g_hash_table_destroy(config_hash); } /* cant rely on this... */ crm_timer_start(shutdown_escalation_timer); } } else { crm_info("exit from shutdown"); exit(LSB_EXIT_OK); } return TRUE; } static void default_cib_update_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { if(rc != cib_ok) { fsa_data_t *msg_data = NULL; crm_err("CIB Update failed: %s", cib_error2string(rc)); crm_log_xml_warn(output, "update:failed"); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } #if SUPPORT_HEARTBEAT static void populate_cib_nodes_ha(gboolean with_client_status) { int call_id = 0; const char *ha_node = NULL; xmlNode *cib_node_list = NULL; if(fsa_cluster_conn == NULL) { crm_debug("Not connected"); return; } /* Async get client status information in the cluster */ crm_info("Requesting the list of configured nodes"); fsa_cluster_conn->llc_ops->init_nodewalk(fsa_cluster_conn); cib_node_list = create_xml_node(NULL, XML_CIB_TAG_NODES); do { const char *ha_node_type = NULL; const char *ha_node_uuid = NULL; xmlNode *cib_new_node = NULL; ha_node = fsa_cluster_conn->llc_ops->nextnode(fsa_cluster_conn); if(ha_node == NULL) { continue; } ha_node_type = fsa_cluster_conn->llc_ops->node_type( fsa_cluster_conn, ha_node); if(safe_str_neq(NORMALNODE, ha_node_type)) { crm_debug("Node %s: skipping '%s'", ha_node, ha_node_type); continue; } ha_node_uuid = get_uuid(ha_node); if(ha_node_uuid == NULL) { crm_warn("Node %s: no uuid found", ha_node); continue; } crm_debug("Node: %s (uuid: %s)", ha_node, ha_node_uuid); cib_new_node = create_xml_node(cib_node_list, XML_CIB_TAG_NODE); crm_xml_add(cib_new_node, XML_ATTR_ID, ha_node_uuid); crm_xml_add(cib_new_node, XML_ATTR_UNAME, ha_node); crm_xml_add(cib_new_node, XML_ATTR_TYPE, ha_node_type); } while(ha_node != NULL); fsa_cluster_conn->llc_ops->end_nodewalk(fsa_cluster_conn); /* Now update the CIB with the list of nodes */ fsa_cib_update( XML_CIB_TAG_NODES, cib_node_list, cib_scope_local|cib_quorum_override, call_id); add_cib_op_callback(fsa_cib_conn, call_id, FALSE, NULL, default_cib_update_callback); free_xml(cib_node_list); crm_debug_2("Complete"); } #endif static void create_cib_node_definition( gpointer key, gpointer value, gpointer user_data) { crm_node_t *node = value; xmlNode *cib_nodes = user_data; xmlNode *cib_new_node = NULL; cib_new_node = create_xml_node(cib_nodes, XML_CIB_TAG_NODE); crm_xml_add(cib_new_node, XML_ATTR_ID, node->uuid); crm_xml_add(cib_new_node, XML_ATTR_UNAME, node->uname); crm_xml_add(cib_new_node, XML_ATTR_TYPE, NORMALNODE); } void populate_cib_nodes(gboolean with_client_status) { int call_id = 0; xmlNode *cib_node_list = NULL; #if SUPPORT_HEARTBEAT if(is_heartbeat_cluster()) { populate_cib_nodes_ha(with_client_status); return; } #endif cib_node_list = create_xml_node(NULL, XML_CIB_TAG_NODES); g_hash_table_foreach( crm_peer_cache, create_cib_node_definition, cib_node_list); fsa_cib_update( XML_CIB_TAG_NODES, cib_node_list, cib_scope_local|cib_quorum_override, call_id); add_cib_op_callback(fsa_cib_conn, call_id, FALSE, NULL, default_cib_update_callback); free_xml(cib_node_list); crm_debug_2("Complete"); } diff --git a/crmd/election.c b/crmd/election.c index e83124b2ed..619bf2c20e 100644 --- a/crmd/election.c +++ b/crmd/election.c @@ -1,477 +1,478 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include GHashTable *voted = NULL; uint highest_born_on = -1; static int current_election_id = 1; const char *get_hg_version(void); const char *get_hg_version(void) { /* limit this #define's use to a single file to avoid rebuilding more than necessary */ return BUILD_VERSION; } /* A_ELECTION_VOTE */ void do_election_vote(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { gboolean not_voting = FALSE; xmlNode *vote = NULL; /* don't vote if we're in one of these states or wanting to shut down */ switch(cur_state) { case S_STARTING: case S_RECOVERY: case S_STOPPING: case S_TERMINATE: crm_warn("Not voting in election, we're in state %s", fsa_state2string(cur_state)); not_voting = TRUE; break; default: break; } if(not_voting == FALSE) { if(is_set(fsa_input_register, R_STARTING)) { not_voting = TRUE; } } if(not_voting) { if(AM_I_DC) { register_fsa_input(C_FSA_INTERNAL, I_RELEASE_DC, NULL); } else { register_fsa_input(C_FSA_INTERNAL, I_PENDING, NULL); } return; } vote = create_request( CRM_OP_VOTE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL); current_election_id++; crm_xml_add(vote, F_CRM_ELECTION_OWNER, fsa_our_uuid); crm_xml_add_int(vote, F_CRM_ELECTION_ID, current_election_id); send_request(vote, NULL); crm_debug("Destroying voted hash"); if(voted) { g_hash_table_destroy(voted); } free_xml(vote); voted = NULL; if(cur_state == S_ELECTION || cur_state == S_RELEASE_DC) { crm_timer_start(election_timeout); } else if(cur_state != S_INTEGRATION) { crm_err("Broken? Voting in state %s", fsa_state2string(cur_state)); } return; } char *dc_hb_msg = NULL; int beat_num = 0; gboolean do_dc_heartbeat(gpointer data) { return TRUE; } struct election_data_s { const char *winning_uname; unsigned int winning_bornon; }; static void log_member_uname(gpointer key, gpointer value, gpointer user_data) { if(crm_is_member_active(value)) { crm_err("%s: %s", (char*)user_data, (char*)key); } } static void log_node(gpointer key, gpointer value, gpointer user_data) { crm_err("%s: %s", (char*)user_data, (char*)key); } void do_election_check(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { int voted_size = 0; int num_members = crm_active_members(); if(voted) { voted_size = g_hash_table_size(voted); } /* in the case of #voted > #members, it is better to * wait for the timeout and give the cluster time to * stabilize */ if(fsa_state != S_ELECTION) { crm_debug("Ignore election check: we not in an election"); } else if(voted_size >= num_members) { /* we won and everyone has voted */ crm_timer_stop(election_timeout); register_fsa_input(C_FSA_INTERNAL, I_ELECTION_DC, NULL); if(voted_size > num_members) { char *data = NULL; data = crm_strdup("member"); g_hash_table_foreach(crm_peer_cache, log_member_uname, data); crm_free(data); data = crm_strdup("voted"); g_hash_table_foreach(voted, log_node, data); crm_free(data); } crm_debug("Destroying voted hash"); g_hash_table_destroy(voted); voted = NULL; } else { crm_info("Still waiting on %d non-votes (%d total)", num_members - voted_size, num_members); } return; } /* A_ELECTION_COUNT */ void do_election_count_vote(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { int election_id = -1; gboolean we_loose = FALSE; const char *op = NULL; const char *vote_from = NULL; const char *your_version = NULL; const char *election_owner = NULL; const char *reason = "unknown"; crm_node_t *our_node = NULL, *your_node = NULL; ha_msg_input_t *vote = fsa_typed_data(fsa_dt_ha_msg); static int win_dampen = 1; /* in seconds */ static int loss_dampen = 2; /* in seconds */ static time_t last_election_win = 0; static time_t last_election_loss = 0; /* if the membership copy is NULL we REALLY shouldnt be voting * the question is how we managed to get here. */ CRM_CHECK(msg_data != NULL, return); CRM_CHECK(crm_peer_cache != NULL, return); CRM_CHECK(vote != NULL, crm_err("Bogus data from %s", msg_data->origin); return); CRM_CHECK(vote->msg != NULL, crm_err("Bogus data from %s", msg_data->origin); return); vote_from = crm_element_value(vote->msg, F_CRM_HOST_FROM); CRM_CHECK(vote_from != NULL, vote_from = fsa_our_uname); your_node = crm_get_peer(0, vote_from); if(your_node == NULL || crm_is_member_active(your_node) == FALSE) { crm_debug("Election ignore: The other side doesn't exist in CCM: %s", vote_from); return; } if(voted == NULL) { crm_debug("Created voted hash"); voted = g_hash_table_new_full( g_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); } op = crm_element_value(vote->msg, F_CRM_TASK); election_owner = crm_element_value(vote->msg, F_CRM_ELECTION_OWNER); crm_element_value_int(vote->msg, F_CRM_ELECTION_ID, &election_id); our_node = crm_get_peer(0, fsa_our_uname); CRM_ASSERT(our_node != NULL && crm_is_member_active(our_node)); crm_debug("Election %d, owner: %s", election_id, election_owner); /* update the list of nodes that have voted */ if(crm_str_eq(fsa_our_uuid, election_owner, TRUE) || crm_str_eq(fsa_our_uname, election_owner, TRUE)) { if(election_id == current_election_id) { char *uname_copy = NULL; char *op_copy = crm_strdup(op); uname_copy = crm_strdup(your_node->uname); g_hash_table_replace(voted, uname_copy, op_copy); crm_info("Updated voted hash for %s to %s", your_node->uname, op); } else { crm_debug("Ignore old '%s' from %s: %d vs. %d", op, your_node->uname, election_id, current_election_id); return; } } else { CRM_CHECK(safe_str_neq(op, CRM_OP_NOVOTE), return); } if(vote_from == NULL || crm_str_eq(vote_from, fsa_our_uname, TRUE)) { /* don't count our own vote */ crm_info("Election ignore: our %s (%s)", op,crm_str(vote_from)); return; } else if(crm_str_eq(op, CRM_OP_NOVOTE, TRUE)) { crm_info("Election ignore: no-vote from %s", vote_from); return; } crm_info("Election check: %s from %s", op, vote_from); your_version = crm_element_value(vote->msg, F_CRM_VERSION); if(cur_state == S_STARTING) { reason = "still starting"; we_loose = TRUE; } else if(our_node == NULL || safe_str_neq(our_node->state, CRM_NODE_MEMBER)) { reason = "we don't exist in CCM"; we_loose = TRUE; } else if(compare_version(your_version, CRM_FEATURE_SET) < 0) { reason = "version"; we_loose = TRUE; } else if(compare_version(your_version, CRM_FEATURE_SET) > 0) { reason = "version"; } else if(your_node->born < our_node->born) { reason = "born_on"; we_loose = TRUE; } else if(your_node->born > our_node->born) { reason = "born_on"; } else if(fsa_our_uname == NULL || strcasecmp(fsa_our_uname, vote_from) > 0) { reason = "uname"; we_loose = TRUE; } else { CRM_CHECK(strcmp(fsa_our_uname, vote_from) != 0, ;); crm_debug("Them: %s (born="U64T") Us: %s (born="U64T")", vote_from, your_node->born, fsa_our_uname, our_node->born); /* cant happen... * } else if(strcasecmp(fsa_our_uname, vote_from) == 0) { * * default... * } else { // strcasecmp(fsa_our_uname, vote_from) < 0 * we win */ } if(we_loose) { xmlNode *novote = create_request( CRM_OP_NOVOTE, NULL, vote_from, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL); update_dc(NULL, FALSE); crm_timer_stop(election_timeout); crm_debug("Election %d lost to %s: %s", election_id, vote_from, reason); if(fsa_input_register & R_THE_DC) { crm_debug_3("Give up the DC to %s", vote_from); register_fsa_input(C_FSA_INTERNAL, I_RELEASE_DC, NULL); } else if(cur_state != S_STARTING) { crm_debug_3("We werent the DC anyway"); register_fsa_input(C_FSA_INTERNAL, I_PENDING, NULL); } crm_xml_add(novote, F_CRM_ELECTION_OWNER, election_owner); crm_xml_add_int(novote, F_CRM_ELECTION_ID, election_id); CRM_CHECK(send_request(novote, NULL),;); free_xml(novote); fsa_cib_conn->cmds->set_slave(fsa_cib_conn, cib_scope_local); last_election_loss = time(NULL); last_election_win = 0; } else { if(last_election_loss) { time_t tm_now = time(NULL); if(tm_now - last_election_loss < (time_t)loss_dampen) { crm_info("Election %d ignore: We already lost an election less than %ds ago", election_id, loss_dampen); return; } last_election_loss = 0; } if(last_election_win) { time_t tm_now = time(NULL); if(tm_now - last_election_win < (time_t)win_dampen) { crm_info("Election %d ignore: We already won an election less than %ds ago", election_id, win_dampen); return; } } #if 0 /* Enabling this code can lead to multiple DCs during SimulStart. * Specifically when a node comes up after our last 'win' vote. * * Fixing and enabling this functionality might become important when * we start running realy big clusters, but for now leave it disabled. */ last_election_win = time(NULL); #endif register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); crm_info("Election %d won over %s: %s", election_id, vote_from, reason); g_hash_table_destroy(voted); voted = NULL; } } /* A_ELECT_TIMER_START, A_ELECTION_TIMEOUT */ /* we won */ void do_election_timer_ctrl(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { } static void feature_update_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { if(rc != cib_ok) { fsa_data_t *msg_data = NULL; register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } /* A_DC_TAKEOVER */ void do_dc_takeover(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { int rc = cib_ok; xmlNode *cib = NULL; crm_info("Taking over DC status for this partition"); set_bit_inplace(fsa_input_register, R_THE_DC); + send_ais_text(crm_class_quorum, NULL, TRUE, NULL, crm_msg_ais); if(voted != NULL) { crm_debug_2("Destroying voted hash"); g_hash_table_destroy(voted); voted = NULL; } set_bit_inplace(fsa_input_register, R_JOIN_OK); set_bit_inplace(fsa_input_register, R_INVOKE_PE); fsa_cib_conn->cmds->set_slave_all(fsa_cib_conn, cib_none); fsa_cib_conn->cmds->set_master(fsa_cib_conn, cib_none); cib = create_xml_node(NULL, XML_TAG_CIB); crm_xml_add(cib, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET); fsa_cib_update(XML_TAG_CIB, cib, cib_quorum_override, rc); add_cib_op_callback(fsa_cib_conn, rc, FALSE, NULL, feature_update_callback); update_attr(fsa_cib_conn, cib_none, XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, "dc-version", VERSION"-"BUILD_VERSION, FALSE); G_main_set_trigger(config_read); free_xml(cib); } /* A_DC_RELEASE */ void do_dc_release(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t *msg_data) { if(action & A_DC_RELEASE) { crm_debug("Releasing the role of DC"); clear_bit_inplace(fsa_input_register, R_THE_DC); } else if (action & A_DC_RELEASED) { crm_info("DC role released"); #if 0 if( are there errors ) { /* we cant stay up if not healthy */ /* or perhaps I_ERROR and go to S_RECOVER? */ result = I_SHUTDOWN; } #endif register_fsa_input(C_FSA_INTERNAL, I_RELEASE_SUCCESS, NULL); } else { crm_err("Unknown action %s", fsa_action2string(action)); } crm_debug_2("Am I still the DC? %s", AM_I_DC?XML_BOOLEAN_YES:XML_BOOLEAN_NO); } diff --git a/crmd/te_callbacks.c b/crmd/te_callbacks.c index 04d64b3815..8ca6f67990 100644 --- a/crmd/te_callbacks.c +++ b/crmd/te_callbacks.c @@ -1,511 +1,511 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include void te_update_confirm(const char *event, xmlNode *msg); extern char *te_uuid; gboolean shuttingdown = FALSE; crm_graph_t *transition_graph; GTRIGSource *transition_trigger = NULL; /* #define rsc_op_template "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_CIB_TAG_STATE"[@uname='%s']"//"XML_LRM_TAG_RSC_OP"[@id='%s]" */ #define rsc_op_template "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_LRM_TAG_RSC_OP"[@id='%s']" static const char *get_node_id(xmlNode *rsc_op) { xmlNode *node = rsc_op; while(node != NULL && safe_str_neq(XML_CIB_TAG_STATE, TYPE(node))) { node = node->parent; } CRM_CHECK(node != NULL, return NULL); return ID(node); } static void process_resource_updates(xmlXPathObject *xpathObj) { /* */ int lpc = 0, max = xpathObj->nodesetval->nodeNr; for(lpc = 0; lpc < max; lpc++) { xmlNode *rsc_op = getXpathResult(xpathObj, lpc); const char *node = get_node_id(rsc_op); process_graph_event(rsc_op, node); } } void te_update_diff(const char *event, xmlNode *msg) { int rc = -1; const char *op = NULL; xmlNode *diff = NULL; xmlNode *cib_top = NULL; xmlXPathObject *xpathObj = NULL; int diff_add_updates = 0; int diff_add_epoch = 0; int diff_add_admin_epoch = 0; int diff_del_updates = 0; int diff_del_epoch = 0; int diff_del_admin_epoch = 0; CRM_CHECK(msg != NULL, return); crm_element_value_int(msg, F_CIB_RC, &rc); if(transition_graph == NULL) { crm_debug_3("No graph"); return; } else if(rc < cib_ok) { crm_debug_3("Filter rc=%d (%s)", rc, cib_error2string(rc)); return; } else if(transition_graph->complete == TRUE && fsa_state != S_IDLE && fsa_state != S_TRANSITION_ENGINE && fsa_state != S_POLICY_ENGINE) { crm_debug_2("Filter state=%s, complete=%d", fsa_state2string(fsa_state), transition_graph->complete); return; } op = crm_element_value(msg, F_CIB_OPERATION); diff = get_message_xml(msg, F_CIB_UPDATE_RESULT); cib_diff_version_details( diff, &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates, &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates); crm_debug("Processing diff (%s): %d.%d.%d -> %d.%d.%d (%s)", op, diff_del_admin_epoch,diff_del_epoch,diff_del_updates, diff_add_admin_epoch,diff_add_epoch,diff_add_updates, fsa_state2string(fsa_state)); log_cib_diff(LOG_DEBUG_2, diff, op); /* Process crm_config updates */ cib_top = get_xpath_object("//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_CIB_TAG_CRMCONFIG, diff, LOG_DEBUG); if(cib_top != NULL) { G_main_set_trigger(config_read); } - + /* Process anything that was added */ cib_top = get_xpath_object("//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB, diff, LOG_ERR); if(need_abort(cib_top)) { goto bail; /* configuration changed */ } /* Process anything that was removed */ cib_top = get_xpath_object("//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_REMOVED"//"XML_TAG_CIB, diff, LOG_ERR); if(need_abort(cib_top)) { goto bail; /* configuration changed */ } /* Transient Attributes - Added/Updated */ xpathObj = xpath_search(diff,"//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_TAG_TRANSIENT_NODEATTRS); if(xpathObj && xpathObj->nodesetval->nodeNr > 0) { xmlNode *aborted = getXpathResult(xpathObj, 0); abort_transition(INFINITY, tg_restart, "Transient attribute: update", aborted); goto bail; } else if(xpathObj) { xmlXPathFreeObject(xpathObj); } /* Transient Attributes - Removed */ xpathObj = xpath_search(diff,"//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_REMOVED"//"XML_TAG_TRANSIENT_NODEATTRS); if(xpathObj && xpathObj->nodesetval->nodeNr > 0) { xmlNode *aborted = getXpathResult(xpathObj, 0); abort_transition(INFINITY, tg_restart, "Transient attribute: removal", aborted); goto bail; } else if(xpathObj) { xmlXPathFreeObject(xpathObj); } /* Check for node state updates... possibly from a shutdown we requested */ xpathObj = xpath_search(diff, "//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_CIB_TAG_STATE); if(xpathObj && xpathObj->nodesetval->nodeNr > 0) { int lpc = 0, max = xpathObj->nodesetval->nodeNr; for(lpc = 0; lpc < max; lpc++) { xmlNode *node = getXpathResult(xpathObj, lpc); const char *event_node = crm_element_value(node, XML_ATTR_ID); const char *ccm_state = crm_element_value(node, XML_CIB_ATTR_INCCM); const char *ha_state = crm_element_value(node, XML_CIB_ATTR_HASTATE); const char *shutdown_s = crm_element_value(node, XML_CIB_ATTR_SHUTDOWN); const char *crmd_state = crm_element_value(node, XML_CIB_ATTR_CRMDSTATE); if(safe_str_eq(ccm_state, XML_BOOLEAN_FALSE) || safe_str_eq(ha_state, DEADSTATUS) || safe_str_eq(crmd_state, CRMD_JOINSTATE_DOWN)) { crm_action_t *shutdown = match_down_event(0, event_node, NULL); if(shutdown != NULL) { const char *task = crm_element_value(shutdown->xml, XML_LRM_ATTR_TASK); if(safe_str_neq(task, CRM_OP_FENCE)) { /* Wait for stonithd to tell us it is complete via tengine_stonith_callback() */ update_graph(transition_graph, shutdown); trigger_graph(); } } else { crm_info("Stonith/shutdown of %s not matched", event_node); abort_transition(INFINITY, tg_restart, "Node failure", node); } fail_incompletable_actions(transition_graph, event_node); } if(shutdown_s) { int shutdown = crm_parse_int(shutdown_s, NULL); if(shutdown > 0) { crm_info("Aborting on "XML_CIB_ATTR_SHUTDOWN" attribute for %s", event_node); abort_transition(INFINITY, tg_restart, "Shutdown request", node); } } } } if(xpathObj) { xmlXPathFreeObject(xpathObj); } /* * Check for and fast-track the processing of LRM refreshes * In large clusters this can result in _huge_ speedups * * Unfortunately we can only do so when there are no pending actions * Otherwise we could miss updates we're waiting for and stall * */ xpathObj = NULL; if(transition_graph->pending == 0) { xpathObj = xpath_search(diff, "//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_LRM_TAG_RESOURCE); } if(xpathObj && xpathObj->nodesetval->nodeNr > 0) { int updates = xpathObj->nodesetval->nodeNr; xmlXPathFreeObject(xpathObj); xpathObj = NULL; if(updates > 1) { /* Updates by, or in response to, TE actions will never contain updates * for more than one resource at a time */ crm_info("Detected LRM refresh - %d resources updated: Skipping all resource events", updates); abort_transition(INFINITY, tg_restart, "LRM Refresh", diff); goto bail; } } /* Process operation updates */ xpathObj = xpath_search(diff, "//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_LRM_TAG_RSC_OP); if(xpathObj && xpathObj->nodesetval->nodeNr > 0) { process_resource_updates(xpathObj); xmlXPathFreeObject(xpathObj); } /* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */ xpathObj = xpath_search(diff, "//"XML_TAG_DIFF_REMOVED"//"XML_LRM_TAG_RSC_OP); if(xpathObj) { int lpc = 0, max = xpathObj->nodesetval->nodeNr; for(lpc = 0; lpc < max; lpc++) { int max = 0; const char *op_id = NULL; char *rsc_op_xpath = NULL; xmlXPathObject *op_match = NULL; xmlNode *match = getXpathResult(xpathObj, lpc); CRM_CHECK(match != NULL, continue); op_id = ID(match); max = strlen(rsc_op_template) + strlen(op_id) + 1; crm_malloc0(rsc_op_xpath, max); snprintf(rsc_op_xpath, max, rsc_op_template, op_id); op_match = xpath_search(diff, rsc_op_xpath); if(op_match == NULL || op_match->nodesetval->nodeNr == 0) { /* Prevent false positives by matching cancelations too */ const char *node = get_node_id(match); crm_action_t *cancelled = get_cancel_action(op_id, node); if(cancelled == NULL) { crm_info("No match for deleted action %s (%s on %s)", rsc_op_xpath, op_id, node); abort_transition(INFINITY, tg_restart, "Resource op removal", match); goto bail; } else { crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d", op_id, node, cancelled->id); } } if(op_match) { xmlXPathFreeObject(op_match); } crm_free(rsc_op_xpath); } } bail: if(xpathObj) { xmlXPathFreeObject(xpathObj); } } gboolean process_te_message(xmlNode *msg, xmlNode *xml_data) { const char *from = crm_element_value(msg, F_ORIG); const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO); const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM); const char *ref = crm_element_value(msg, XML_ATTR_REFERENCE); const char *op = crm_element_value(msg, F_CRM_TASK); const char *type = crm_element_value(msg, F_CRM_MSG_TYPE); crm_debug_2("Processing %s (%s) message", op, ref); crm_log_xml(LOG_DEBUG_3, "ipc", msg); if(op == NULL){ /* error */ } else if(sys_to == NULL || strcasecmp(sys_to, CRM_SYSTEM_TENGINE) != 0) { crm_debug_2("Bad sys-to %s", crm_str(sys_to)); return FALSE; } else if(safe_str_eq(op, CRM_OP_INVOKE_LRM) && safe_str_eq(sys_from, CRM_SYSTEM_LRMD) /* && safe_str_eq(type, XML_ATTR_RESPONSE) */ ){ xmlXPathObject *xpathObj = NULL; crm_log_xml(LOG_DEBUG_2, "Processing (N)ACK", msg); crm_info("Processing (N)ACK %s from %s", crm_element_value(msg, XML_ATTR_REFERENCE), from); xpathObj = xpath_search(xml_data, "//"XML_LRM_TAG_RSC_OP); if(xpathObj) { process_resource_updates(xpathObj); xmlXPathFreeObject(xpathObj); xpathObj = NULL; } else { crm_log_xml(LOG_ERR, "Invalid (N)ACK", msg); return FALSE; } } else { crm_err("Unknown command: %s::%s from %s", type, op, sys_from); } crm_debug_3("finished processing message"); return TRUE; } void tengine_stonith_callback(stonith_ops_t * op) { const char *allow_fail = NULL; int target_rc = -1; int stonith_id = -1; int transition_id = -1; char *uuid = NULL; crm_action_t *stonith_action = NULL; if(op == NULL) { crm_err("Called with a NULL op!"); return; } crm_info("call=%d, optype=%d, node_name=%s, result=%d, node_list=%s, action=%s", op->call_id, op->optype, op->node_name, op->op_result, (char *)op->node_list, op->private_data); /* this will mark the event complete if a match is found */ CRM_CHECK(op->private_data != NULL, return); /* filter out old STONITH actions */ CRM_CHECK(decode_transition_key( op->private_data, &uuid, &transition_id, &stonith_id, &target_rc), crm_err("Invalid event detected"); goto bail; ); if(transition_graph->complete || stonith_id < 0 || safe_str_neq(uuid, te_uuid) || transition_graph->id != transition_id) { crm_info("Ignoring STONITH action initiated outside" " of the current transition"); } stonith_action = get_action(stonith_id, TRUE); if(stonith_action == NULL) { crm_err("Stonith action not matched"); goto bail; } switch(op->op_result) { case STONITH_SUCCEEDED: send_stonith_update(op); break; case STONITH_CANNOT: case STONITH_TIMEOUT: case STONITH_GENERIC: stonith_action->failed = TRUE; allow_fail = crm_meta_value(stonith_action->params, XML_ATTR_TE_ALLOWFAIL); if(FALSE == crm_is_true(allow_fail)) { crm_err("Stonith of %s failed (%d)..." " aborting transition.", op->node_name, op->op_result); abort_transition(INFINITY, tg_restart, "Stonith failed", NULL); } break; default: crm_err("Unsupported action result: %d", op->op_result); abort_transition(INFINITY, tg_restart, "Unsupport Stonith result", NULL); } update_graph(transition_graph, stonith_action); trigger_graph(); bail: crm_free(uuid); return; } void cib_fencing_updated(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { if(rc < cib_ok) { crm_err("CIB update failed: %s", cib_error2string(rc)); crm_log_xml_warn(msg, "Failed update"); } crm_free(user_data); } void cib_action_updated(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { if(rc < cib_ok) { crm_err("Update %d FAILED: %s", call_id, cib_error2string(rc)); } } void cib_failcount_updated(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { if(rc < cib_ok) { crm_err("Update %d FAILED: %s", call_id, cib_error2string(rc)); } } gboolean action_timer_callback(gpointer data) { crm_action_timer_t *timer = NULL; CRM_CHECK(data != NULL, return FALSE); timer = (crm_action_timer_t*)data; stop_te_timer(timer); crm_warn("Timer popped (timeout=%d, abort_level=%d, complete=%s)", timer->timeout, transition_graph->abort_priority, transition_graph->complete?"true":"false"); CRM_CHECK(timer->action != NULL, return FALSE); if(transition_graph->complete) { crm_warn("Ignoring timeout while not in transition"); } else if(timer->reason == timeout_action_warn) { print_action( LOG_WARNING,"Action missed its timeout: ", timer->action); } else if(fsa_state != S_TRANSITION_ENGINE && fsa_state != S_POLICY_ENGINE) { crm_err("Discarding action timeout in state: %s", fsa_state2string(fsa_state)); } else if(transition_graph->complete) { crm_err("Ignoring action timeout while not in transition"); } else { /* fail the action */ gboolean send_update = TRUE; const char *task = crm_element_value(timer->action->xml, XML_LRM_ATTR_TASK); print_action(LOG_ERR, "Aborting transition, action lost: ", timer->action); timer->action->failed = TRUE; timer->action->confirmed = TRUE; abort_transition(INFINITY, tg_restart, "Action lost", NULL); update_graph(transition_graph, timer->action); trigger_graph(); if(timer->action->type != action_type_rsc) { send_update = FALSE; } else if(safe_str_eq(task, "cancel")) { /* we dont need to update the CIB with these */ send_update = FALSE; } else if(safe_str_eq(task, "stop")) { /* *never* update the CIB with these */ send_update = FALSE; } if(send_update) { /* cib_action_update(timer->action, LRM_OP_PENDING, EXECRA_STATUS_UNKNOWN); */ cib_action_update(timer->action, LRM_OP_TIMEOUT, EXECRA_UNKNOWN_ERROR); } } return FALSE; } diff --git a/include/crm/ais_common.h b/include/crm/ais_common.h index 91b7ea74c1..7284708c85 100644 --- a/include/crm/ais_common.h +++ b/include/crm/ais_common.h @@ -1,475 +1,484 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef CRM_AIS_COMMON__H #define CRM_AIS_COMMON__H #include #include #include #if SUPPORT_AIS # ifdef AIS_WHITETANK /* cheap hacks for building against the stable series of openais */ # include enum service_types { EVS_SERVICE = 0, CLM_SERVICE = 1, AMF_SERVICE = 2, CKPT_SERVICE = 3, EVT_SERVICE = 4, LCK_SERVICE = 5, MSG_SERVICE = 6, CFG_SERVICE = 7, CPG_SERVICE = 8 }; typedef struct { int size; __attribute__((aligned(8))) int id __attribute__((aligned(8))); SaAisErrorT error __attribute__((aligned(8))); } mar_res_header_t __attribute__((aligned(8))); typedef struct { int size __attribute__((aligned(8))); int id __attribute__((aligned(8))); } mar_req_header_t __attribute__((aligned(8))); # ifdef TRADITIONAL_AIS_IPC extern SaAisErrorT saSendReceiveReply ( int s, void *requestMessage, int requestLen, void *responseMessage, int responseLen); extern SaAisErrorT saRecvRetry (int s, void *msg, size_t len); extern SaAisErrorT saServiceConnect (int *responseOut, int *callbackOut, enum service_types service); #else extern SaAisErrorT openais_service_connect ( enum service_types service, void **ipc_context); extern SaAisErrorT openais_service_disconnect ( void *ipc_context); extern int openais_dispatch_recv ( void *ipc_context, void *buf, int timeout); extern SaAisErrorT openais_msg_send_reply_receive ( void *ipc_context, struct iovec *iov, int iov_len, void *res_msg, int res_len); extern int openais_fd_get(void *ipc_context); #endif static inline const char *ais_error2text(int error) { const char *text = "unknown"; switch(error) { case SA_AIS_OK: text = "None"; break; case SA_AIS_ERR_LIBRARY: text = "Library error"; break; case SA_AIS_ERR_VERSION: text = "Version error"; break; case SA_AIS_ERR_INIT: text = "Initialization error"; break; case SA_AIS_ERR_TIMEOUT: text = "Timeout"; break; case SA_AIS_ERR_TRY_AGAIN: text = "Try again"; break; case SA_AIS_ERR_INVALID_PARAM: text = "Invalid parameter"; break; case SA_AIS_ERR_NO_MEMORY: text = "No memory"; break; case SA_AIS_ERR_BAD_HANDLE: text = "Bad handle"; break; case SA_AIS_ERR_BUSY: text = "Busy"; break; case SA_AIS_ERR_ACCESS: text = "Access error"; break; case SA_AIS_ERR_NOT_EXIST: text = "Doesn't exist"; break; case SA_AIS_ERR_NAME_TOO_LONG: text = "Name too long"; break; case SA_AIS_ERR_EXIST: text = "Exists"; break; case SA_AIS_ERR_NO_SPACE: text = "No space"; break; case SA_AIS_ERR_INTERRUPT: text = "Interrupt"; break; case SA_AIS_ERR_NAME_NOT_FOUND: text = "Name not found"; break; case SA_AIS_ERR_NO_RESOURCES: text = "No resources"; break; case SA_AIS_ERR_NOT_SUPPORTED: text = "Not supported"; break; case SA_AIS_ERR_BAD_OPERATION: text = "Bad operation"; break; case SA_AIS_ERR_FAILED_OPERATION: text = "Failed operation"; break; case SA_AIS_ERR_MESSAGE_ERROR: text = "Message error"; break; case SA_AIS_ERR_QUEUE_FULL: text = "Queue full"; break; case SA_AIS_ERR_QUEUE_NOT_AVAILABLE: text = "Queue not available"; break; case SA_AIS_ERR_BAD_FLAGS: text = "Bad flags"; break; case SA_AIS_ERR_TOO_BIG: text = "To big"; break; case SA_AIS_ERR_NO_SECTIONS: text = "No sections"; break; } return text; } # endif # ifdef AIS_COROSYNC # include # include #define SA_AIS_OK CS_OK #define SA_AIS_ERR_TRY_AGAIN CS_ERR_TRY_AGAIN #define SA_AIS_OK CS_OK static inline const char *ais_error2text(int error) { const char *text = "unknown"; switch(error) { case CS_OK: text = "None"; break; case CS_ERR_LIBRARY: text = "Library error"; break; case CS_ERR_VERSION: text = "Version error"; break; case CS_ERR_INIT: text = "Initialization error"; break; case CS_ERR_TIMEOUT: text = "Timeout"; break; case CS_ERR_TRY_AGAIN: text = "Try again"; break; case CS_ERR_INVALID_PARAM: text = "Invalid parameter"; break; case CS_ERR_NO_MEMORY: text = "No memory"; break; case CS_ERR_BAD_HANDLE: text = "Bad handle"; break; case CS_ERR_BUSY: text = "Busy"; break; case CS_ERR_ACCESS: text = "Access error"; break; case CS_ERR_NOT_EXIST: text = "Doesn't exist"; break; case CS_ERR_NAME_TOO_LONG: text = "Name too long"; break; case CS_ERR_EXIST: text = "Exists"; break; case CS_ERR_NO_SPACE: text = "No space"; break; case CS_ERR_INTERRUPT: text = "Interrupt"; break; case CS_ERR_NAME_NOT_FOUND: text = "Name not found"; break; case CS_ERR_NO_RESOURCES: text = "No resources"; break; case CS_ERR_NOT_SUPPORTED: text = "Not supported"; break; case CS_ERR_BAD_OPERATION: text = "Bad operation"; break; case CS_ERR_FAILED_OPERATION: text = "Failed operation"; break; case CS_ERR_MESSAGE_ERROR: text = "Message error"; break; case CS_ERR_QUEUE_FULL: text = "Queue full"; break; case CS_ERR_QUEUE_NOT_AVAILABLE: text = "Queue not available"; break; case CS_ERR_BAD_FLAGS: text = "Bad flags"; break; case CS_ERR_TOO_BIG: text = "To big"; break; case CS_ERR_NO_SECTIONS: text = "No sections"; break; } return text; } # endif #else typedef struct { int size __attribute__((aligned(8))); int id __attribute__((aligned(8))); } mar_req_header_t __attribute__((aligned(8))); typedef struct { int size; __attribute__((aligned(8))) int id __attribute__((aligned(8))); int error __attribute__((aligned(8))); } mar_res_header_t __attribute__((aligned(8))); #endif #define CRM_SERVICE 9 #define CRM_MESSAGE_IPC_ACK 0 #define MAX_NAME 256 #define AIS_IPC_NAME "ais-crm-ipc" #define CRM_NODE_LOST "lost" #define CRM_NODE_MEMBER "member" #define CRM_NODE_ACTIVE CRM_NODE_MEMBER #define CRM_NODE_INACTIVE CRM_NODE_LOST #define CRM_NODE_EVICTED "evicted" typedef struct crm_ais_host_s AIS_Host; typedef struct crm_ais_msg_s AIS_Message; enum crm_ais_msg_class { crm_class_cluster = 0, crm_class_members = 1, crm_class_notify = 2, crm_class_nodeid = 3, crm_class_rmpeer = 4, + crm_class_quorum = 5, }; /* order here matters - its used to index into the crm_children array */ enum crm_ais_msg_types { crm_msg_none = 0, crm_msg_ais = 1, crm_msg_lrmd = 2, crm_msg_cib = 3, crm_msg_crmd = 4, crm_msg_attrd = 5, crm_msg_stonithd = 6, crm_msg_te = 7, crm_msg_pe = 8, }; enum crm_proc_flag { crm_proc_none = 0x00000001, crm_proc_ais = 0x00000002, crm_proc_lrmd = 0x00000010, crm_proc_cib = 0x00000100, crm_proc_crmd = 0x00000200, crm_proc_attrd = 0x00001000, crm_proc_stonithd = 0x00002000, crm_proc_pe = 0x00010000, crm_proc_te = 0x00020000, crm_proc_mgmtd = 0x00040000, }; typedef struct crm_peer_node_s { uint32_t id; uint64_t born; uint64_t last_seen; int32_t votes; uint32_t processes; char *uname; char *state; char *uuid; char *addr; char *version; } crm_node_t; struct crm_ais_host_s { uint32_t id; uint32_t pid; gboolean local; enum crm_ais_msg_types type; uint32_t size; char uname[256]; } __attribute__((packed)); struct crm_ais_msg_s { mar_res_header_t header __attribute__((aligned(8))); uint32_t id; gboolean is_compressed; AIS_Host host; AIS_Host sender; uint32_t size; uint32_t compressed_size; /* 584 bytes */ char data[0]; } __attribute__((packed)); struct crm_ais_nodeid_resp_s { mar_res_header_t header __attribute__((aligned(8))); uint32_t id; uint32_t counter; char uname[256]; } __attribute__((packed)); +struct crm_ais_quorum_resp_s +{ + mar_res_header_t header __attribute__((aligned(8))); + uint64_t id; + uint32_t votes; + uint32_t expected_votes; + uint32_t quorate; +} __attribute__((packed)); static inline const char *msg_type2text(enum crm_ais_msg_types type) { const char *text = "unknown"; switch(type) { case crm_msg_none: text = "unknown"; break; case crm_msg_ais: text = "ais"; break; case crm_msg_cib: text = "cib"; break; case crm_msg_crmd: text = "crmd"; break; case crm_msg_pe: text = "pengine"; break; case crm_msg_te: text = "tengine"; break; case crm_msg_lrmd: text = "lrmd"; break; case crm_msg_attrd: text = "attrd"; break; case crm_msg_stonithd: text = "stonithd"; break; } return text; } static inline const char *peer2text(enum crm_proc_flag proc) { const char *text = "unknown"; switch(proc) { case crm_proc_none: text = "unknown"; break; case crm_proc_ais: text = "ais"; break; case crm_proc_cib: text = "cib"; break; case crm_proc_crmd: text = "crmd"; break; case crm_proc_pe: text = "pengine"; break; case crm_proc_te: text = "tengine"; break; case crm_proc_lrmd: text = "lrmd"; break; case crm_proc_attrd: text = "attrd"; break; case crm_proc_stonithd: text = "stonithd"; break; case crm_proc_mgmtd: text = "mgmtd"; break; } return text; } static inline const char *ais_dest(struct crm_ais_host_s *host) { if(host->local) { return "local"; } else if(host->size > 0) { return host->uname; } else { return ""; } } #define ais_data_len(msg) (msg->is_compressed?msg->compressed_size:msg->size) #endif diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h index 2be33a0f17..075fcf7f53 100644 --- a/include/crm/msg_xml.h +++ b/include/crm/msg_xml.h @@ -1,269 +1,270 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef XML_TAGS__H #define XML_TAGS__H #define CIB_OPTIONS_FIRST "cib-bootstrap-options" #define F_CRM_TASK "crm_task" #define F_CRM_HOST_TO "crm_host_to" #define F_CRM_MSG_TYPE F_SUBTYPE #define F_CRM_SYS_TO "crm_sys_to" #define F_CRM_SYS_FROM "crm_sys_from" #define F_CRM_HOST_FROM F_ORIG #define F_CRM_REFERENCE XML_ATTR_REFERENCE #define F_CRM_VERSION XML_ATTR_VERSION #define F_CRM_ORIGIN "origin" #define F_CRM_JOIN_ID "join_id" #define F_CRM_ELECTION_ID "election-id" #define F_CRM_ELECTION_OWNER "election-owner" #define F_CRM_TGRAPH "crm-tgraph" #define F_CRM_TGRAPH_INPUT "crm-tgraph-in" /*---- Common tags/attrs */ #define XML_DIFF_MARKER "__crm_diff_marker__" #define XML_ATTR_TAGNAME F_XML_TAGNAME #define XML_TAG_CIB "cib" #define XML_TAG_FAILED "failed" #define XML_ATTR_CRM_VERSION "crm_feature_set" #define XML_ATTR_DIGEST "digest" #define XML_ATTR_VALIDATION "validate-with" #define XML_ATTR_QUORUM_PANIC "no-quorum-panic" #define XML_ATTR_HAVE_QUORUM "have-quorum" +#define XML_ATTR_EXPECTED_VOTES "expected-quorum-votes" #define XML_ATTR_GENERATION "epoch" #define XML_ATTR_GENERATION_ADMIN "admin_epoch" #define XML_ATTR_NUMUPDATES "num_updates" #define XML_ATTR_TIMEOUT "timeout" #define XML_ATTR_ORIGIN "crm-debug-origin" #define XML_ATTR_TSTAMP "crm-timestamp" #define XML_CIB_ATTR_WRITTEN "cib-last-written" #define XML_ATTR_VERSION "version" #define XML_ATTR_DESC "description" #define XML_ATTR_ID "id" #define XML_ATTR_IDREF "id-ref" #define XML_ATTR_ID_LONG "long-id" #define XML_ATTR_TYPE "type" #define XML_ATTR_FILTER_TYPE "type-filter" #define XML_ATTR_FILTER_ID "id-filter" #define XML_ATTR_FILTER_PRIORITY "priority-filter" #define XML_ATTR_VERBOSE "verbose" #define XML_ATTR_OP "op" #define XML_ATTR_DC "is_dc" #define XML_ATTR_DC_UUID "dc-uuid" #define XML_BOOLEAN_TRUE "true" #define XML_BOOLEAN_FALSE "false" #define XML_BOOLEAN_YES XML_BOOLEAN_TRUE #define XML_BOOLEAN_NO XML_BOOLEAN_FALSE #define XML_TAG_OPTIONS "options" /*---- top level tags/attrs */ #define XML_MSG_TAG "crm_message" #define XML_MSG_TAG_DATA "msg_data" #define XML_ATTR_REQUEST "request" #define XML_ATTR_RESPONSE "response" #define XML_ATTR_UNAME "uname" #define XML_ATTR_UUID "id" #define XML_ATTR_REFERENCE "reference" #define XML_FAIL_TAG_RESOURCE "failed_resource" #define XML_FAILRES_ATTR_RESID "resource_id" #define XML_FAILRES_ATTR_REASON "reason" #define XML_FAILRES_ATTR_RESSTATUS "resource_status" #define XML_CRM_TAG_PING "ping_response" #define XML_PING_ATTR_STATUS "result" #define XML_PING_ATTR_SYSFROM "crm_subsystem" #define XML_TAG_FRAGMENT "cib_fragment" #define XML_ATTR_RESULT "result" #define XML_ATTR_SECTION "section" #define XML_FAIL_TAG_CIB "failed_update" #define XML_FAILCIB_ATTR_ID "id" #define XML_FAILCIB_ATTR_OBJTYPE "object_type" #define XML_FAILCIB_ATTR_OP "operation" #define XML_FAILCIB_ATTR_REASON "reason" /*---- CIB specific tags/attrs */ #define XML_CIB_TAG_SECTION_ALL "all" #define XML_CIB_TAG_CONFIGURATION "configuration" #define XML_CIB_TAG_STATUS "status" #define XML_CIB_TAG_RESOURCES "resources" #define XML_CIB_TAG_NODES "nodes" #define XML_CIB_TAG_CONSTRAINTS "constraints" #define XML_CIB_TAG_CRMCONFIG "crm_config" #define XML_CIB_TAG_OPCONFIG "op_defaults" #define XML_CIB_TAG_RSCCONFIG "rsc_defaults" #define XML_CIB_TAG_STATE "node_state" #define XML_CIB_TAG_NODE "node" #define XML_CIB_TAG_CONSTRAINT "constraint" #define XML_CIB_TAG_NVPAIR "nvpair" #define XML_CIB_TAG_PROPSET "cluster_property_set" #define XML_TAG_ATTR_SETS "instance_attributes" #define XML_TAG_META_SETS "meta_attributes" #define XML_TAG_ATTRS "attributes" #define XML_TAG_PARAMS "parameters" #define XML_TAG_PARAM "param" #define XML_TAG_RESOURCE_REF "resource_ref" #define XML_CIB_TAG_RESOURCE "primitive" #define XML_CIB_TAG_GROUP "group" #define XML_CIB_TAG_INCARNATION "clone" #define XML_CIB_TAG_MASTER "master" #define XML_RSC_ATTR_RESTART "restart-type" #define XML_RSC_ATTR_ORDERED "ordered" #define XML_RSC_ATTR_INTERLEAVE "interleave" #define XML_RSC_ATTR_INCARNATION "clone" #define XML_RSC_ATTR_INCARNATION_MAX "clone-max" #define XML_RSC_ATTR_INCARNATION_NODEMAX "clone-node-max" #define XML_RSC_ATTR_MASTER_MAX "master-max" #define XML_RSC_ATTR_MASTER_NODEMAX "master-node-max" #define XML_RSC_ATTR_STATE "clone-state" #define XML_RSC_ATTR_MANAGED "is-managed" #define XML_RSC_ATTR_TARGET_ROLE "target-role" #define XML_RSC_ATTR_UNIQUE "globally-unique" #define XML_RSC_ATTR_NOTIFY "notify" #define XML_RSC_ATTR_STICKINESS "resource-stickiness" #define XML_RSC_ATTR_FAIL_STICKINESS "migration-threshold" #define XML_RSC_ATTR_FAIL_TIMEOUT "failure-timeout" #define XML_RSC_ATTR_MULTIPLE "multiple-active" #define XML_RSC_ATTR_PRIORITY "priority" #define XML_OP_ATTR_ON_FAIL "on-fail" #define XML_OP_ATTR_START_DELAY "start-delay" #define XML_OP_ATTR_ALLOW_MIGRATE "allow-migrate" #define XML_OP_ATTR_ORIGIN "interval-origin" #define XML_OP_ATTR_PENDING "record-pending" #define XML_CIB_TAG_LRM "lrm" #define XML_LRM_TAG_RESOURCES "lrm_resources" #define XML_LRM_TAG_RESOURCE "lrm_resource" #define XML_LRM_TAG_AGENTS "lrm_agents" #define XML_LRM_TAG_AGENT "lrm_agent" #define XML_LRM_TAG_RSC_OP "lrm_rsc_op" #define XML_AGENT_ATTR_CLASS "class" #define XML_AGENT_ATTR_PROVIDER "provider" #define XML_LRM_TAG_ATTRIBUTES "attributes" #define XML_CIB_ATTR_REPLACE "replace" #define XML_CIB_ATTR_SOURCE "source" #define XML_CIB_ATTR_HEALTH "health" #define XML_CIB_ATTR_WEIGHT "weight" #define XML_CIB_ATTR_PRIORITY "priority" #define XML_CIB_ATTR_CLEAR "clear_on" #define XML_CIB_ATTR_SOURCE "source" #define XML_CIB_ATTR_JOINSTATE "join" #define XML_CIB_ATTR_EXPSTATE "expected" #define XML_CIB_ATTR_INCCM "in_ccm" #define XML_CIB_ATTR_CRMDSTATE "crmd" #define XML_CIB_ATTR_HASTATE "ha" #define XML_CIB_ATTR_SHUTDOWN "shutdown" #define XML_CIB_ATTR_STONITH "stonith" #define XML_LRM_ATTR_INTERVAL "interval" #define XML_LRM_ATTR_TASK "operation" #define XML_LRM_ATTR_TASK_KEY "operation_key" #define XML_LRM_ATTR_TARGET "on_node" #define XML_LRM_ATTR_TARGET_UUID "on_node_uuid" #define XML_LRM_ATTR_RSCID "rsc-id" #define XML_LRM_ATTR_OPSTATUS "op-status" #define XML_LRM_ATTR_RC "rc-code" #define XML_LRM_ATTR_CALLID "call-id" #define XML_LRM_ATTR_OP_DIGEST "op-digest" #define XML_LRM_ATTR_OP_RESTART "op-force-restart" #define XML_LRM_ATTR_RESTART_DIGEST "op-restart-digest" #define XML_TAG_GRAPH "transition_graph" #define XML_GRAPH_TAG_RSC_OP "rsc_op" #define XML_GRAPH_TAG_PSEUDO_EVENT "pseudo_event" #define XML_GRAPH_TAG_CRM_EVENT "crm_event" #define XML_TAG_RULE "rule" #define XML_RULE_ATTR_SCORE "score" #define XML_RULE_ATTR_SCORE_ATTRIBUTE "score-attribute" #define XML_RULE_ATTR_SCORE_MANGLED "score-attribute-mangled" #define XML_RULE_ATTR_ROLE "role" #define XML_RULE_ATTR_RESULT "result" #define XML_RULE_ATTR_BOOLEAN_OP "boolean-op" #define XML_TAG_EXPRESSION "expression" #define XML_EXPR_ATTR_ATTRIBUTE "attribute" #define XML_EXPR_ATTR_OPERATION "operation" #define XML_EXPR_ATTR_VALUE "value" #define XML_EXPR_ATTR_TYPE "type" #define XML_CONS_TAG_RSC_DEPEND "rsc_colocation" #define XML_CONS_TAG_RSC_ORDER "rsc_order" #define XML_CONS_TAG_RSC_LOCATION "rsc_location" #define XML_CONS_ATTR_SYMMETRICAL "symmetrical" #define XML_COLOC_ATTR_SOURCE "rsc" #define XML_COLOC_ATTR_SOURCE_ROLE "rsc-role" #define XML_COLOC_ATTR_TARGET "with-rsc" #define XML_COLOC_ATTR_TARGET_ROLE "with-rsc-role" #define XML_COLOC_ATTR_NODE_ATTR "node-attribute" #define XML_ORDER_ATTR_FIRST "first" #define XML_ORDER_ATTR_THEN "then" #define XML_ORDER_ATTR_FIRST_ACTION "first-action" #define XML_ORDER_ATTR_THEN_ACTION "then-action" #define XML_NVPAIR_ATTR_NAME "name" #define XML_NVPAIR_ATTR_VALUE "value" #define XML_NODE_ATTR_STATE "state" #define XML_CONFIG_ATTR_DC_DEADTIME "dc-deadtime" #define XML_CONFIG_ATTR_ELECTION_FAIL "election-timeout" #define XML_CONFIG_ATTR_FORCE_QUIT "shutdown-escalation" #define XML_CONFIG_ATTR_RECHECK "cluster-recheck-interval" #define XML_CIB_TAG_GENERATION_TUPPLE "generation_tuple" #define XML_ATTR_TRANSITION_MAGIC "transition-magic" #define XML_ATTR_TRANSITION_KEY "transition-key" #define XML_ATTR_TE_NOWAIT "op_no_wait" #define XML_ATTR_TE_TARGET_RC "op_target_rc" #define XML_ATTR_TE_ALLOWFAIL "op_allow_fail" #define XML_ATTR_LRM_PROBE "lrm-is-probe" #define XML_TAG_TRANSIENT_NODEATTRS "transient_attributes" #define XML_TAG_DIFF_ADDED "diff-added" #define XML_TAG_DIFF_REMOVED "diff-removed" #include #define ID(x) crm_element_value(x, XML_ATTR_ID) #define INSTANCE(x) crm_element_value(x, XML_CIB_ATTR_INSTANCE) #define TSTAMP(x) crm_element_value(x, XML_ATTR_TSTAMP) #define TYPE(x) crm_element_name(x) #endif diff --git a/lib/ais/plugin.c b/lib/ais/plugin.c index a0b06e767a..435726cf2c 100644 --- a/lib/ais/plugin.c +++ b/lib/ais/plugin.c @@ -1,1335 +1,1455 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "plugin.h" #include "utils.h" #ifdef AIS_COROSYNC # include #endif #include #include #include #include #include #include plugin_init_type *crm_api = NULL; +uint32_t plugin_has_votes = 0; +uint32_t plugin_expected_votes = 1024; + int use_mgmtd = 0; int plugin_log_level = LOG_DEBUG; char *local_uname = NULL; int local_uname_len = 0; uint32_t local_nodeid = 0; char *ipc_channel_name = NULL; uint64_t membership_seq = 0; pthread_t crm_wait_thread; gboolean wait_active = TRUE; gboolean have_reliable_membership_id = FALSE; GHashTable *membership_list = NULL; GHashTable *membership_notify_list = NULL; #define MAX_RESPAWN 100 #define crm_flag_none 0x00000000 #define crm_flag_members 0x00000001 struct crm_identify_msg_s { mar_req_header_t header __attribute__((aligned(8))); uint32_t id; uint32_t pid; int32_t votes; uint32_t processes; char uname[256]; char version[256]; uint64_t born_on; } __attribute__((packed)); static crm_child_t crm_children[] = { { 0, crm_proc_none, crm_flag_none, 0, 0, FALSE, "none", NULL, NULL, NULL, NULL }, { 0, crm_proc_ais, crm_flag_none, 0, 0, FALSE, "ais", NULL, NULL, NULL, NULL }, { 0, crm_proc_lrmd, crm_flag_none, 3, 0, TRUE, "lrmd", NULL, HA_LIBHBDIR"/lrmd", NULL, NULL }, { 0, crm_proc_cib, crm_flag_members, 2, 0, TRUE, "cib", HA_CCMUSER, HA_LIBHBDIR"/cib", NULL, NULL }, { 0, crm_proc_crmd, crm_flag_members, 6, 0, TRUE, "crmd", HA_CCMUSER, HA_LIBHBDIR"/crmd", NULL, NULL }, { 0, crm_proc_attrd, crm_flag_none, 4, 0, TRUE, "attrd", HA_CCMUSER, HA_LIBHBDIR"/attrd", NULL, NULL }, { 0, crm_proc_stonithd, crm_flag_none, 1, 0, TRUE, "stonithd", NULL, HA_LIBHBDIR"/stonithd", NULL, NULL }, { 0, crm_proc_pe, crm_flag_none, 5, 0, TRUE, "pengine", HA_CCMUSER, HA_LIBHBDIR"/pengine", NULL, NULL }, { 0, crm_proc_mgmtd, crm_flag_none, 7, 0, TRUE, "mgmtd", NULL, HA_LIBHBDIR"/mgmtd", NULL, NULL }, }; void send_cluster_id(void); int send_cluster_msg_raw(AIS_Message *ais_msg); char *ais_generate_membership_data(void); extern totempg_groups_handle openais_group_handle; void global_confchg_fn ( enum totem_configuration_type configuration_type, unsigned int *member_list, int member_list_entries, unsigned int *left_list, int left_list_entries, unsigned int *joined_list, int joined_list_entries, struct memb_ring_id *ring_id); #ifdef AIS_WHITETANK int crm_exec_init_fn (struct objdb_iface_ver0 *objdb); int crm_exec_exit_fn (struct objdb_iface_ver0 *objdb); int crm_config_init_fn(struct objdb_iface_ver0 *objdb); #endif #ifdef AIS_COROSYNC int crm_exec_init_fn (struct corosync_api_v1 *corosync_api); int crm_exec_exit_fn (void); int crm_config_init_fn(struct corosync_api_v1 *corosync_api); #endif int ais_ipc_client_connect_callback (void *conn); int ais_ipc_client_exit_callback (void *conn); void ais_cluster_message_swab(void *msg); void ais_cluster_message_callback(void *message, unsigned int nodeid); void ais_ipc_message_callback(void *conn, void *msg); void ais_our_nodeid(void *conn, void *msg); void ais_quorum_query(void *conn, void *msg); void ais_node_list_query(void *conn, void *msg); void ais_manage_notification(void *conn, void *msg); void ais_plugin_remove_member(void *conn, void *msg); +void ais_plugin_quorum(void *conn, void *msg); void ais_cluster_id_swab(void *msg); void ais_cluster_id_callback(void *message, unsigned int nodeid); static plugin_lib_handler crm_lib_service[] = { { /* 0 */ .lib_handler_fn = ais_ipc_message_callback, .response_size = sizeof (mar_res_header_t), .response_id = CRM_MESSAGE_IPC_ACK, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED }, { /* 1 */ .lib_handler_fn = ais_node_list_query, .response_size = sizeof (mar_res_header_t), .response_id = CRM_MESSAGE_IPC_ACK, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED }, { /* 2 */ .lib_handler_fn = ais_manage_notification, .response_size = sizeof (mar_res_header_t), .response_id = CRM_MESSAGE_IPC_ACK, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED }, { /* 3 */ .lib_handler_fn = ais_our_nodeid, .response_size = sizeof (struct crm_ais_nodeid_resp_s), .response_id = crm_class_nodeid, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED }, { /* 4 */ .lib_handler_fn = ais_plugin_remove_member, .response_size = sizeof (mar_res_header_t), .response_id = CRM_MESSAGE_IPC_ACK, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED }, + { /* 5 */ + .lib_handler_fn = ais_plugin_quorum, + .response_size = sizeof (struct crm_ais_quorum_resp_s), + .response_id = CRM_MESSAGE_IPC_ACK, + .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED + }, }; static plugin_exec_handler crm_exec_service[] = { { /* 0 */ .exec_handler_fn = ais_cluster_message_callback, .exec_endian_convert_fn = ais_cluster_message_swab }, { /* 1 */ .exec_handler_fn = ais_cluster_id_callback, .exec_endian_convert_fn = ais_cluster_id_swab } }; static void crm_exec_dump_fn(void) { ais_err("Called after SIG_USR2"); } /* * Exports the interface for the service */ plugin_service_handler crm_service_handler = { .name = (unsigned char *)"Pacemaker Cluster Manager", .id = CRM_SERVICE, .private_data_size = 0, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED, .lib_init_fn = ais_ipc_client_connect_callback, .lib_exit_fn = ais_ipc_client_exit_callback, .exec_init_fn = crm_exec_init_fn, .exec_exit_fn = crm_exec_exit_fn, .config_init_fn = crm_config_init_fn, #ifdef AIS_WHITETANK .lib_service = crm_lib_service, .lib_service_count = sizeof (crm_lib_service) / sizeof (plugin_lib_handler), .exec_service = crm_exec_service, .exec_service_count = sizeof (crm_exec_service) / sizeof (plugin_exec_handler), #endif #ifdef AIS_COROSYNC .lib_engine = crm_lib_service, .lib_engine_count = sizeof (crm_lib_service) / sizeof (plugin_lib_handler), .exec_engine = crm_exec_service, .exec_engine_count = sizeof (crm_exec_service) / sizeof (plugin_exec_handler), #endif .confchg_fn = global_confchg_fn, .exec_dump_fn = crm_exec_dump_fn, /* void (*sync_init) (void); */ /* int (*sync_process) (void); */ /* void (*sync_activate) (void); */ /* void (*sync_abort) (void); */ }; /* * Dynamic Loader definition */ plugin_service_handler *crm_get_handler_ver0 (void); #ifdef AIS_WHITETANK struct openais_service_handler_iface_ver0 crm_service_handler_iface = { .openais_get_service_handler_ver0 = crm_get_handler_ver0 }; #endif #ifdef AIS_COROSYNC struct corosync_service_engine_iface_ver0 crm_service_handler_iface = { .corosync_get_service_engine_ver0 = crm_get_handler_ver0 }; #endif static struct lcr_iface openais_crm_ver0[1] = { { .name = "pacemaker", .version = 0, .versions_replace = 0, .versions_replace_count = 0, .dependencies = 0, .dependency_count = 0, .constructor = NULL, .destructor = NULL, .interfaces = NULL } }; static struct lcr_comp crm_comp_ver0 = { .iface_count = 1, .ifaces = openais_crm_ver0 }; plugin_service_handler *crm_get_handler_ver0 (void) { return (&crm_service_handler); } __attribute__ ((constructor)) static void register_this_component (void) { lcr_interfaces_set (&openais_crm_ver0[0], &crm_service_handler_iface); lcr_component_register (&crm_comp_ver0); } +static int plugin_has_quorum(void) +{ + if(plugin_expected_votes < (2 * plugin_has_votes) + 1) { + return 1; + } + return 0; +} + #ifdef AIS_COROSYNC #include #endif /* Create our own local copy of the config so we can navigate it */ static void process_ais_conf(void) { char *value = NULL; unsigned int top_handle = 0; unsigned int local_handle = 0; ais_info("Reading configure"); top_handle = config_find_init(crm_api, "logging"); local_handle = config_find_next(crm_api, "logging", top_handle); get_config_opt(crm_api, local_handle, "debug", &value, "on"); if(ais_get_boolean(value)) { plugin_log_level = LOG_DEBUG; setenv("HA_debug", "1", 1); } else { plugin_log_level = LOG_INFO; setenv("HA_debug", "0", 1); } get_config_opt(crm_api, local_handle, "to_syslog", &value, "on"); if(ais_get_boolean(value)) { get_config_opt(crm_api, local_handle, "syslog_facility", &value, "daemon"); setenv("HA_logfacility", value, 1); } else { setenv("HA_logfacility", "none", 1); } get_config_opt(crm_api, local_handle, "to_file", &value, "off"); if(ais_get_boolean(value)) { get_config_opt(crm_api, local_handle, "logfile", &value, NULL); if(value == NULL) { ais_err("Logging to a file requested but no log file specified"); } else { setenv("HA_logfile", value, 1); } } config_find_done(crm_api, local_handle); top_handle = config_find_init(crm_api, "service"); local_handle = config_find_next(crm_api, "service", top_handle); while(local_handle) { value = NULL; crm_api->object_key_get(local_handle, "name", strlen("name"), (void**)&value, NULL); if(ais_str_eq("pacemaker", value)) { break; } local_handle = config_find_next(crm_api, "service", top_handle); } - get_config_opt(crm_api, local_handle, "expected_nodes", &value, "1"); - setenv("HA_expected_nodes", value, 1); - - get_config_opt(crm_api, local_handle, "expected_votes", &value, "2"); - setenv("HA_expected_votes", value, 1); - - get_config_opt(crm_api, local_handle, "quorum_votes", &value, "1"); - setenv("HA_votes", value, 1); - get_config_opt(crm_api, local_handle, "use_logd", &value, "no"); setenv("HA_use_logd", value, 1); get_config_opt(crm_api, local_handle, "use_mgmtd", &value, "no"); if(ais_get_boolean(value) == FALSE) { int lpc = 0; for (; lpc < SIZEOF(crm_children); lpc++) { if(crm_proc_mgmtd & crm_children[lpc].flag) { /* Disable mgmtd startup */ crm_children[lpc].start_seq = 0; break; } } } config_find_done(crm_api, local_handle); } static void crm_plugin_init(void) { int rc = 0; struct utsname us; #ifdef AIS_WHITETANK log_init ("crm"); #endif process_ais_conf(); membership_list = g_hash_table_new_full( g_direct_hash, g_direct_equal, NULL, destroy_ais_node); membership_notify_list = g_hash_table_new(g_direct_hash, g_direct_equal); setenv("HA_COMPRESSION", "bz2", 1); setenv("HA_cluster_type", "openais", 1); if(system("echo 1 > /proc/sys/kernel/core_uses_pid") != 0) { ais_perror("Could not enable /proc/sys/kernel/core_uses_pid"); } ais_info("CRM: Initialized"); log_printf(LOG_INFO, "Logging: Initialized %s\n", __PRETTY_FUNCTION__); rc = uname(&us); AIS_ASSERT(rc == 0); local_uname = ais_strdup(us.nodename); local_uname_len = strlen(local_uname); #if AIS_WHITETANK local_nodeid = totempg_my_nodeid_get(); #endif #if AIS_COROSYNC local_nodeid = crm_api->totem_nodeid_get(); #endif ais_info("Service: %d", CRM_SERVICE); ais_info("Local node id: %u", local_nodeid); ais_info("Local hostname: %s", local_uname); update_member(local_nodeid, 0, 0, 1, 0, local_uname, CRM_NODE_MEMBER, NULL); } int crm_config_init_fn(plugin_init_type *unused) { return 0; } static void *crm_wait_dispatch (void *arg) { struct timespec waitsleep = { .tv_sec = 0, .tv_nsec = 100000 /* 100 msec */ }; while(wait_active) { int lpc = 0; for (; lpc < SIZEOF(crm_children); lpc++) { if(crm_children[lpc].pid > 0) { int status; pid_t pid = wait4( crm_children[lpc].pid, &status, WNOHANG, NULL); if(pid == 0) { continue; } else if(pid < 0) { ais_perror("crm_wait_dispatch: Call to wait4(%s) failed", crm_children[lpc].name); continue; } /* cleanup */ crm_children[lpc].pid = 0; crm_children[lpc].conn = NULL; crm_children[lpc].async_conn = NULL; if(WIFSIGNALED(status)) { int sig = WTERMSIG(status); ais_err("Child process %s terminated with signal %d" " (pid=%d, core=%s)", crm_children[lpc].name, sig, pid, WCOREDUMP(status)?"true":"false"); } else if (WIFEXITED(status)) { int rc = WEXITSTATUS(status); do_ais_log(rc==0?LOG_NOTICE:LOG_ERR, "Child process %s exited (pid=%d, rc=%d)", crm_children[lpc].name, pid, rc); if(rc == 100) { ais_notice("Child process %s no longer wishes" " to be respawned", crm_children[lpc].name); crm_children[lpc].respawn = FALSE; } } crm_children[lpc].respawn_count += 1; if(crm_children[lpc].respawn_count > MAX_RESPAWN) { ais_err("Child respawn count exceeded by %s", crm_children[lpc].name); crm_children[lpc].respawn = FALSE; } if(crm_children[lpc].respawn) { ais_notice("Respawning failed child process: %s", crm_children[lpc].name); spawn_child(&(crm_children[lpc])); } else { send_cluster_id(); } } } sched_yield (); nanosleep (&waitsleep, 0); } return 0; } #include #include int crm_exec_init_fn(plugin_init_type *init_with) { int lpc = 0; int start_seq = 1; static gboolean need_init = TRUE; static int max = SIZEOF(crm_children); crm_api = init_with; if(need_init) { struct passwd *pwentry = NULL; need_init = FALSE; crm_plugin_init(); pthread_create (&crm_wait_thread, NULL, crm_wait_dispatch, NULL); pwentry = getpwnam(HA_CCMUSER); AIS_CHECK(pwentry != NULL, ais_err("Cluster user %s does not exist", HA_CCMUSER); return TRUE); mkdir(HA_VARRUNDIR, 750); mkdir(HA_VARRUNDIR"/crm", 750); mkdir(HA_VARRUNHBDIR"/rsctmp", 755); /* Used by RAs - Leave owned by root */ chown(HA_VARRUNDIR"/crm", pwentry->pw_uid, pwentry->pw_gid); chown(HA_VARRUNDIR, pwentry->pw_uid, pwentry->pw_gid); for (start_seq = 1; start_seq < max; start_seq++) { /* dont start anything with start_seq < 1 */ for (lpc = 0; lpc < max; lpc++) { if(start_seq == crm_children[lpc].start_seq) { spawn_child(&(crm_children[lpc])); } } } } ais_info("CRM: Initialized"); return 0; } /* static void ais_print_node(const char *prefix, struct totem_ip_address *host) { int len = 0; char *buffer = NULL; ais_malloc0(buffer, INET6_ADDRSTRLEN+1); inet_ntop(host->family, host->addr, buffer, INET6_ADDRSTRLEN); len = strlen(buffer); ais_info("%s: %.*s", prefix, len, buffer); ais_free(buffer); } */ #if 0 /* copied here for reference from exec/totempg.c */ char *totempg_ifaces_print (unsigned int nodeid) { static char iface_string[256 * INTERFACE_MAX]; char one_iface[64]; struct totem_ip_address interfaces[INTERFACE_MAX]; char **status; unsigned int iface_count; unsigned int i; int res; iface_string[0] = '\0'; res = totempg_ifaces_get (nodeid, interfaces, &status, &iface_count); if (res == -1) { return ("no interface found for nodeid"); } for (i = 0; i < iface_count; i++) { sprintf (one_iface, "r(%d) ip(%s), ", i, totemip_print (&interfaces[i])); strcat (iface_string, one_iface); } return (iface_string); } #endif static void ais_mark_unseen_peer_dead( gpointer key, gpointer value, gpointer user_data) { int *changed = user_data; crm_node_t *node = value; if(node->last_seen != membership_seq && ais_str_eq(CRM_NODE_LOST, node->state) == FALSE) { ais_info("Node %s was not seen in the previous transition", node->uname); *changed += update_member(node->id, 0, membership_seq, node->votes, node->processes, node->uname, CRM_NODE_LOST, NULL); } } void global_confchg_fn ( enum totem_configuration_type configuration_type, unsigned int *member_list, int member_list_entries, unsigned int *left_list, int left_list_entries, unsigned int *joined_list, int joined_list_entries, struct memb_ring_id *ring_id) { int lpc = 0; int changed = 0; int do_update = 0; AIS_ASSERT(ring_id != NULL); switch(configuration_type) { case TOTEM_CONFIGURATION_REGULAR: do_update = 1; break; case TOTEM_CONFIGURATION_TRANSITIONAL: break; } membership_seq = ring_id->seq; ais_notice("%s membership event on ring %lld: memb=%d, new=%d, lost=%d", do_update?"Stable":"Transitional", ring_id->seq, member_list_entries, joined_list_entries, left_list_entries); if(do_update == 0) { for(lpc = 0; lpc < joined_list_entries; lpc++) { const char *prefix = "new: "; uint32_t nodeid = joined_list[lpc]; ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid); } for(lpc = 0; lpc < member_list_entries; lpc++) { const char *prefix = "memb:"; uint32_t nodeid = member_list[lpc]; ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid); } for(lpc = 0; lpc < left_list_entries; lpc++) { const char *prefix = "lost:"; uint32_t nodeid = left_list[lpc]; ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid); } return; } for(lpc = 0; lpc < joined_list_entries; lpc++) { const char *prefix = "NEW: "; uint32_t nodeid = joined_list[lpc]; crm_node_t *node = NULL; changed += update_member( nodeid, 0, membership_seq, -1, 0, NULL, CRM_NODE_MEMBER, NULL); ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid); node = g_hash_table_lookup(membership_list, GUINT_TO_POINTER(nodeid)); if(node->addr == NULL) { const char *addr = totempg_ifaces_print(nodeid); node->addr = ais_strdup(addr); ais_debug("Node %u has address %s", nodeid, node->addr); } } + plugin_has_votes = 0; for(lpc = 0; lpc < member_list_entries; lpc++) { const char *prefix = "MEMB:"; uint32_t nodeid = member_list[lpc]; + plugin_has_votes++; changed += update_member( nodeid, 0, membership_seq, -1, 0, NULL, CRM_NODE_MEMBER, NULL); ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid); } - + for(lpc = 0; lpc < left_list_entries; lpc++) { const char *prefix = "LOST:"; uint32_t nodeid = left_list[lpc]; changed += update_member( nodeid, 0, membership_seq, -1, 0, NULL, CRM_NODE_LOST, NULL); ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid); } if(changed && joined_list_entries == 0 && left_list_entries == 0) { ais_err("Something strange happened: %d", changed); changed = 0; } ais_debug_2("Reaping unseen nodes..."); g_hash_table_foreach(membership_list, ais_mark_unseen_peer_dead, &changed); + if(plugin_has_votes > plugin_expected_votes) { + plugin_expected_votes = plugin_has_votes; + changed = 1; + } + if(member_list_entries > 1) { /* Used to set born-on in send_cluster_id()) * We need to wait until we have at least one peer since first * membership id is based on the one before we stopped and isn't reliable */ have_reliable_membership_id = TRUE; } if(changed) { ais_debug("%d nodes changed", changed); send_member_notification(); } send_cluster_id(); } int ais_ipc_client_exit_callback (void *conn) { int lpc = 0; const char *client = NULL; void *async_conn = openais_conn_partner_get(conn); for (; lpc < SIZEOF(crm_children); lpc++) { if(crm_children[lpc].conn == conn) { if(wait_active == FALSE) { /* Make sure the shutdown loop exits */ crm_children[lpc].pid = 0; } crm_children[lpc].conn = NULL; crm_children[lpc].async_conn = NULL; client = crm_children[lpc].name; break; } } g_hash_table_remove(membership_notify_list, async_conn); do_ais_log(client?LOG_INFO:(LOG_DEBUG+1), "Client %s (conn=%p, async-conn=%p) left", client?client:"unknown-transient", conn, async_conn); return (0); } int ais_ipc_client_connect_callback (void *conn) { /* OpenAIS hasn't finished setting up the connection at this point * Sending messages now messes up the protocol! */ return (0); } /* * Executive message handlers */ void ais_cluster_message_swab(void *msg) { AIS_Message *ais_msg = msg; ais_debug_3("Performing endian conversion..."); ais_msg->id = swab32 (ais_msg->id); ais_msg->size = swab32 (ais_msg->size); ais_msg->is_compressed = swab32 (ais_msg->is_compressed); ais_msg->compressed_size = swab32 (ais_msg->compressed_size); ais_msg->host.id = swab32 (ais_msg->host.id); ais_msg->host.pid = swab32 (ais_msg->host.pid); ais_msg->host.type = swab32 (ais_msg->host.type); ais_msg->host.size = swab32 (ais_msg->host.size); ais_msg->host.local = swab32 (ais_msg->host.local); ais_msg->sender.id = swab32 (ais_msg->sender.id); ais_msg->sender.pid = swab32 (ais_msg->sender.pid); ais_msg->sender.type = swab32 (ais_msg->sender.type); ais_msg->sender.size = swab32 (ais_msg->sender.size); ais_msg->sender.local = swab32 (ais_msg->sender.local); } void ais_cluster_message_callback ( void *message, unsigned int nodeid) { AIS_Message *ais_msg = message; ais_debug_2("Message from node %u (%s)", nodeid, nodeid==local_nodeid?"local":"remote"); /* Shouldn't be required... update_member( ais_msg->sender.id, membership_seq, -1, 0, ais_msg->sender.uname, NULL); */ if(ais_msg->host.size == 0 || ais_str_eq(ais_msg->host.uname, local_uname)) { route_ais_message(ais_msg, FALSE); } else { ais_debug_3("Discarding Msg[%d] (dest=%s:%s, from=%s:%s)", ais_msg->id, ais_dest(&(ais_msg->host)), msg_type2text(ais_msg->host.type), ais_dest(&(ais_msg->sender)), msg_type2text(ais_msg->sender.type)); } } void ais_cluster_id_swab(void *msg) { struct crm_identify_msg_s *ais_msg = msg; ais_debug_3("Performing endian conversion..."); ais_msg->id = swab32 (ais_msg->id); ais_msg->pid = swab32 (ais_msg->pid); ais_msg->votes = swab32 (ais_msg->votes); ais_msg->processes = swab32 (ais_msg->processes); } void ais_cluster_id_callback (void *message, unsigned int nodeid) { int changed = 0; struct crm_identify_msg_s *msg = message; if(nodeid != msg->id) { ais_err("Invalid message: Node %u claimed to be node %d", nodeid, msg->id); return; } ais_debug("Node update: %s (%s)", msg->uname, msg->version); changed = update_member( nodeid, msg->born_on, membership_seq, msg->votes, msg->processes, msg->uname, NULL, msg->version); if(changed) { send_member_notification(); } } struct res_overlay { mar_res_header_t header __attribute((aligned(8))); char buf[4096]; }; struct res_overlay *res_overlay = NULL; static void send_ipc_ack(void *conn, int class) { if(res_overlay == NULL) { ais_malloc0(res_overlay, sizeof(struct res_overlay)); } res_overlay->header.size = crm_lib_service[class].response_size; res_overlay->header.id = crm_lib_service[class].response_id; res_overlay->header.error = SA_AIS_OK; #ifdef AIS_WHITETANK openais_response_send (conn, res_overlay, res_overlay->header.size); #endif #ifdef AIS_COROSYNC crm_api->ipc_conn_send_response (conn, res_overlay, res_overlay->header.size); #endif } /* local callbacks */ void ais_ipc_message_callback(void *conn, void *msg) { gboolean transient = TRUE; AIS_Message *ais_msg = msg; int type = ais_msg->sender.type; void *async_conn = openais_conn_partner_get(conn); ais_debug_2("Message from client %p", conn); send_ipc_ack(conn, 0); ais_debug_3("type: %d local: %d conn: %p host type: %d ais: %d sender pid: %d child pid: %d size: %d", type, ais_msg->host.local, crm_children[type].conn, ais_msg->host.type, crm_msg_ais, ais_msg->sender.pid, crm_children[type].pid, ((int)SIZEOF(crm_children))); if(type > crm_msg_none && type < SIZEOF(crm_children)) { /* known child process */ transient = FALSE; } /* If this check fails, the order of crm_children probably * doesn't match that of the crm_ais_msg_types enum */ AIS_CHECK(transient || ais_msg->sender.pid == crm_children[type].pid, ais_err("Sender: %d, child[%d]: %d", ais_msg->sender.pid, type, crm_children[type].pid); return); if(transient == FALSE && type > crm_msg_none && ais_msg->host.local && crm_children[type].conn == NULL && ais_msg->host.type == crm_msg_ais) { ais_info("Recorded connection %p for %s/%d", conn, crm_children[type].name, crm_children[type].pid); crm_children[type].conn = conn; crm_children[type].async_conn = async_conn; /* Make sure they have the latest membership */ if(crm_children[type].flags & crm_flag_members) { char *update = ais_generate_membership_data(); g_hash_table_replace(membership_notify_list, async_conn, async_conn); ais_info("Sending membership update "U64T" to %s", membership_seq, crm_children[type].name); send_client_msg(async_conn, crm_class_members, crm_msg_none,update); } } ais_msg->sender.id = local_nodeid; ais_msg->sender.size = local_uname_len; memset(ais_msg->sender.uname, 0, MAX_NAME); memcpy(ais_msg->sender.uname, local_uname, ais_msg->sender.size); route_ais_message(msg, TRUE); } int crm_exec_exit_fn ( #ifdef AIS_WHITETANK struct objdb_iface_ver0 *objdb #endif #ifdef AIS_COROSYNC void #endif ) { int lpc = 0; int start_seq = 1; static int max = SIZEOF(crm_children); struct timespec waitsleep = { .tv_sec = 1, .tv_nsec = 0 }; ais_notice("Begining shutdown"); in_shutdown = TRUE; wait_active = FALSE; /* stop the wait loop */ for (start_seq = max; start_seq > 0; start_seq--) { /* dont stop anything with start_seq < 1 */ for (lpc = max - 1; lpc >= 0; lpc--) { int orig_pid = 0, iter = 0; if(start_seq != crm_children[lpc].start_seq) { continue; } orig_pid = crm_children[lpc].pid; crm_children[lpc].respawn = FALSE; stop_child(&(crm_children[lpc]), SIGTERM); while(crm_children[lpc].command && crm_children[lpc].pid) { int status; pid_t pid = 0; pid = wait4( crm_children[lpc].pid, &status, WNOHANG, NULL); if(pid == 0) { if((++iter % 30) == 0) { ais_notice("Still waiting for %s (pid=%d) to terminate...", crm_children[lpc].name, orig_pid); } sched_yield (); nanosleep (&waitsleep, 0); continue; } else if(pid < 0) { ais_perror("crm_exec_exit_fn: Call to wait4(%s) failed", crm_children[lpc].name); } /* cleanup */ crm_children[lpc].pid = 0; crm_children[lpc].conn = NULL; crm_children[lpc].async_conn = NULL; break; } ais_notice("%s (pid=%d) confirmed dead", crm_children[lpc].name, orig_pid); } } send_cluster_id(); ais_notice("Shutdown complete"); #ifndef AIS_WHITETANK logsys_flush (); #endif return 0; } struct member_loop_data { char *string; }; void member_loop_fn(gpointer key, gpointer value, gpointer user_data) { crm_node_t *node = value; struct member_loop_data *data = user_data; ais_debug_2("Dumping node %u", node->id); data->string = append_member(data->string, node); } char *ais_generate_membership_data(void) { int size = 0; struct member_loop_data data; size = 14 + 32; /* + int */ ais_malloc0(data.string, size); - sprintf(data.string, "", membership_seq); + sprintf(data.string, "", membership_seq, plugin_has_quorum()); g_hash_table_foreach(membership_list, member_loop_fn, &data); size = strlen(data.string); data.string = realloc(data.string, size + 9) ;/* 9 = + nul */ sprintf(data.string + size, ""); return data.string; } void ais_node_list_query(void *conn, void *msg) { char *data = ais_generate_membership_data(); void *async_conn = openais_conn_partner_get(conn); /* send the ACK before we send any other messages */ send_ipc_ack(conn, 1); if(async_conn) { send_client_msg(async_conn, crm_class_members, crm_msg_none, data); } ais_free(data); } void ais_plugin_remove_member(void *conn, void *msg) { AIS_Message *ais_msg = msg; char *data = get_ais_data(ais_msg); if(data != NULL) { char *bcast = ais_concat("remove-peer", data, ':'); send_cluster_msg(crm_msg_ais, NULL, bcast); ais_info("Sent: %s", bcast); ais_free(bcast); } send_ipc_ack(conn, 2); ais_free(data); } +static void send_quorum_details(void *conn, int sync) +{ + struct crm_ais_quorum_resp_s resp; + + resp.header.size = crm_lib_service[crm_class_quorum].response_size; + resp.header.id = crm_lib_service[crm_class_quorum].response_id; + resp.header.error = SA_AIS_OK; + + resp.id = membership_seq; + resp.votes = plugin_has_votes; + resp.expected_votes = plugin_expected_votes; + resp.quorate = plugin_has_quorum(); + + if(sync) { +#ifdef AIS_WHITETANK + openais_response_send (conn, &resp, resp.header.size); +#endif +#ifdef AIS_COROSYNC + crm_api->ipc_conn_send_response (conn, &resp, resp.header.size); +#endif + } else { +#ifdef AIS_WHITETANK + openais_dispatch_send (conn, &resp, resp.header.size); +#endif +#ifdef AIS_COROSYNC + crm_api->ipc_dispatch_send (conn, &resp, resp.header.size); +#endif + } +} + + +void ais_plugin_quorum(void *conn, void *msg) +{ + AIS_Message *ais_msg = msg; + char *data = get_ais_data(ais_msg); + + if(data != NULL) { + int value = ais_get_int(data, NULL); + if(value > 0) { + ais_info("Expected quorum votes %d -> %d", plugin_expected_votes, value); + plugin_expected_votes = value; + } + } + send_quorum_details(conn, TRUE); + ais_free(data); +} + void ais_manage_notification(void *conn, void *msg) { int enable = 0; AIS_Message *ais_msg = msg; char *data = get_ais_data(ais_msg); void *async_conn = openais_conn_partner_get(conn); if(ais_str_eq("true", data)) { enable = 1; } ais_info("%s node notifications for child %d (%p)", enable?"Enabling":"Disabling", ais_msg->sender.pid, async_conn); if(enable) { g_hash_table_replace(membership_notify_list, async_conn, async_conn); } else { g_hash_table_remove(membership_notify_list, async_conn); } send_ipc_ack(conn, 2); ais_free(data); } void ais_our_nodeid(void *conn, void *msg) { static int counter = 0; struct crm_ais_nodeid_resp_s resp; ais_debug_2("Sending local nodeid: %d to %p[%d]", local_nodeid, conn, counter); resp.header.size = crm_lib_service[crm_class_nodeid].response_size; resp.header.id = crm_lib_service[crm_class_nodeid].response_id; resp.header.error = SA_AIS_OK; resp.id = local_nodeid; resp.counter = counter++; memset(resp.uname, 0, 256); memcpy(resp.uname, local_uname, local_uname_len); #ifdef AIS_WHITETANK openais_response_send (conn, &resp, resp.header.size); #endif #ifdef AIS_COROSYNC crm_api->ipc_conn_send_response (conn, &resp, resp.header.size); #endif } static gboolean ghash_send_update(gpointer key, gpointer value, gpointer data) { + send_quorum_details(value, FALSE); if(send_client_msg(value, crm_class_members, crm_msg_none, data) != 0) { /* remove it */ return TRUE; } return FALSE; } void send_member_notification(void) { char *update = ais_generate_membership_data(); ais_info("Sending membership update "U64T" to %d children", membership_seq, g_hash_table_size(membership_notify_list)); g_hash_table_foreach_remove(membership_notify_list, ghash_send_update, update); ais_free(update); } static gboolean check_message_sanity(AIS_Message *msg, char *data) { gboolean sane = TRUE; gboolean repaired = FALSE; int dest = msg->host.type; int tmp_size = msg->header.size - sizeof(AIS_Message); if(sane && msg->header.size == 0) { ais_warn("Message with no size"); sane = FALSE; } if(sane && msg->header.error != SA_AIS_OK) { ais_warn("Message header contains an error: %d", msg->header.error); sane = FALSE; } if(sane && tmp_size < 0) { /* not an AIS message */ return TRUE; } if(sane && ais_data_len(msg) != tmp_size) { int cur_size = ais_data_len(msg); repaired = TRUE; if(msg->is_compressed) { msg->compressed_size = tmp_size; } else { msg->size = tmp_size; } ais_warn("Repaired message payload size %d -> %d", cur_size, tmp_size); } if(sane && ais_data_len(msg) == 0) { ais_warn("Message with no payload"); sane = FALSE; } if(sane && data && msg->is_compressed == FALSE) { int str_size = strlen(data) + 1; if(ais_data_len(msg) != str_size) { int lpc = 0; ais_warn("Message payload is corrupted: expected %d bytes, got %d", ais_data_len(msg), str_size); sane = FALSE; for(lpc = (str_size - 10); lpc < msg->size; lpc++) { if(lpc < 0) { lpc = 0; } ais_debug_2("bad_data[%d]: %d / '%c'", lpc, data[lpc], data[lpc]); } } } if(sane == FALSE) { ais_err("Invalid message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)", msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed, ais_data_len(msg), msg->header.size); } else if(repaired) { ais_err("Repaired message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)", msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed, ais_data_len(msg), msg->header.size); } else { ais_debug_3("Verified message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)", msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed, ais_data_len(msg), msg->header.size); } return sane; } gboolean route_ais_message(AIS_Message *msg, gboolean local_origin) { int rc = 0; int dest = msg->host.type; const char *reason = "unknown"; ais_debug_3("Msg[%d] (dest=%s:%s, from=%s:%s.%d, remote=%s, size=%d)", msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, local_origin?"false":"true", ais_data_len(msg)); if(local_origin == FALSE) { if(msg->host.size == 0 || ais_str_eq(local_uname, msg->host.uname)) { msg->host.local = TRUE; } } if(check_message_sanity(msg, msg->data) == FALSE) { /* Dont send this message to anyone */ return FALSE; } if(msg->host.local) { void *conn = NULL; const char *lookup = NULL; if(dest == crm_msg_ais) { process_ais_message(msg); return TRUE; } else if(dest == crm_msg_lrmd) { /* lrmd messages are routed via the crm */ dest = crm_msg_crmd; } else if(dest == crm_msg_te) { /* te messages are routed via the crm */ dest = crm_msg_crmd; } AIS_CHECK(dest > 0 && dest < SIZEOF(crm_children), ais_err("Invalid destination: %d", dest); log_ais_message(LOG_ERR, msg); return FALSE; ); lookup = msg_type2text(dest); conn = crm_children[dest].async_conn; /* the cluster fails in weird and wonderfully obscure ways when this is not true */ AIS_ASSERT(ais_str_eq(lookup, crm_children[dest].name)); rc = send_client_ipc(conn, msg); } else if(local_origin) { /* forward to other hosts */ ais_debug_3("Forwarding to cluster"); reason = "cluster delivery failed"; rc = send_cluster_msg_raw(msg); } else { ais_debug_3("Ignoring..."); } if(rc != 0) { ais_warn("Sending message to %s.%s failed: %s (rc=%d)", ais_dest(&(msg->host)), msg_type2text(dest), reason, rc); log_ais_message(LOG_DEBUG, msg); return FALSE; } return TRUE; } int send_cluster_msg_raw(AIS_Message *ais_msg) { int rc = 0; struct iovec iovec; static uint32_t msg_id = 0; AIS_Message *bz2_msg = NULL; AIS_ASSERT(local_nodeid != 0); if(ais_msg->header.size != (sizeof(AIS_Message) + ais_data_len(ais_msg))) { ais_err("Repairing size mismatch: %u + %d = %d", (unsigned int)sizeof(AIS_Message), ais_data_len(ais_msg), ais_msg->header.size); ais_msg->header.size = sizeof(AIS_Message) + ais_data_len(ais_msg); } if(ais_msg->id == 0) { msg_id++; AIS_CHECK(msg_id != 0 /* detect wrap-around */, msg_id++; ais_err("Message ID wrapped around")); ais_msg->id = msg_id; } ais_msg->header.error = SA_AIS_OK; ais_msg->header.id = SERVICE_ID_MAKE(CRM_SERVICE, 0); ais_msg->sender.id = local_nodeid; ais_msg->sender.size = local_uname_len; memset(ais_msg->sender.uname, 0, MAX_NAME); memcpy(ais_msg->sender.uname, local_uname, ais_msg->sender.size); iovec.iov_base = (char *)ais_msg; iovec.iov_len = ais_msg->header.size; ais_debug_3("Sending message (size=%u)", (unsigned int)iovec.iov_len); rc = totempg_groups_mcast_joined ( openais_group_handle, &iovec, 1, TOTEMPG_SAFE); if(rc == 0 && ais_msg->is_compressed == FALSE) { ais_debug_2("Message sent: %.80s", ais_msg->data); } AIS_CHECK(rc == 0, ais_err("Message not sent (%d): %.120s", rc, ais_msg->data)); ais_free(bz2_msg); return rc; } #define min(x,y) (x)<(y)?(x):(y) void send_cluster_id(void) { int rc = 0; int lpc = 0; int len = 0; struct iovec iovec; struct crm_identify_msg_s *msg = NULL; static uint64_t local_born_on = 0; AIS_ASSERT(local_nodeid != 0); if(local_born_on == 0 && have_reliable_membership_id) { local_born_on = membership_seq; } ais_malloc0(msg, sizeof(struct crm_identify_msg_s)); msg->header.size = sizeof(struct crm_identify_msg_s); msg->id = local_nodeid; /* msg->header.error = SA_AIS_OK; */ msg->header.id = SERVICE_ID_MAKE(CRM_SERVICE, 1); len = min(local_uname_len, MAX_NAME-1); memset(msg->uname, 0, MAX_NAME); memcpy(msg->uname, local_uname, len); len = min(strlen(VERSION), MAX_NAME-1); memset(msg->version, 0, MAX_NAME); memcpy(msg->version, VERSION, len); msg->votes = 1; msg->pid = getpid(); msg->processes = crm_proc_ais; msg->born_on = local_born_on; for (lpc = 0; lpc < SIZEOF(crm_children); lpc++) { if(crm_children[lpc].pid != 0) { msg->processes |= crm_children[lpc].flag; } } ais_debug("Local update: id=%u, born="U64T", seq="U64T"", local_nodeid, local_born_on, membership_seq); update_member( local_nodeid, local_born_on, membership_seq, msg->votes, msg->processes, NULL, NULL, VERSION); iovec.iov_base = (char *)msg; iovec.iov_len = msg->header.size; rc = totempg_groups_mcast_joined ( openais_group_handle, &iovec, 1, TOTEMPG_SAFE); AIS_CHECK(rc == 0, ais_err("Message not sent (%d)", rc)); ais_free(msg); } + +static gboolean +ghash_send_removal(gpointer key, gpointer value, gpointer data) +{ + send_quorum_details(value, FALSE); + if(send_client_msg(value, crm_class_rmpeer, crm_msg_none, data) != 0) { + /* remove it */ + return TRUE; + } + return FALSE; +} + +static void ais_remove_peer(char *node_id) +{ + uint32_t id = ais_get_int(node_id, NULL); + crm_node_t *node = g_hash_table_lookup(membership_list, GUINT_TO_POINTER(id)); + if(node == NULL) { + ais_info("Peer %u is unknown", id); + + } else if(ais_str_eq(CRM_NODE_MEMBER, node->state)) { + ais_warn("Peer %u/%s is still active", id, node->uname); + + } else if(g_hash_table_remove(membership_list, GUINT_TO_POINTER(id))) { + plugin_expected_votes--; + ais_notice("Removed dead peer %u from the membership list", id); + ais_info("Sending removal of %u to %d children", + id, g_hash_table_size(membership_notify_list)); + + g_hash_table_foreach_remove(membership_notify_list, ghash_send_removal, node_id); + + } else { + ais_warn("Peer %u/%s was not removed", id, node->uname); + } + +} + +gboolean process_ais_message(AIS_Message *msg) +{ + int len = ais_data_len(msg); + char *data = get_ais_data(msg); + do_ais_log(LOG_DEBUG, + "Msg[%d] (dest=%s:%s, from=%s:%s.%d, remote=%s, size=%d): %.90s", + msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type), + ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), + msg->sender.pid, + msg->sender.uname==local_uname?"false":"true", + ais_data_len(msg), data); + + if(data && len > 12 && strncmp("remove-peer:", data, 12) == 0) { + char *node = data+12; + ais_remove_peer(node); + } + + ais_free(data); + return TRUE; +} diff --git a/lib/ais/utils.c b/lib/ais/utils.c index 7b977cbd90..5cf98a0336 100644 --- a/lib/ais/utils.c +++ b/lib/ais/utils.c @@ -1,740 +1,686 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "./utils.h" int in_shutdown = FALSE; extern GHashTable *membership_notify_list; extern int send_cluster_msg_raw(AIS_Message *ais_msg); void log_ais_message(int level, AIS_Message *msg) { char *data = get_ais_data(msg); log_printf(level, "Msg[%d] (dest=%s:%s, from=%s:%s.%d, remote=%s, size=%d): %.90s", msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, msg->sender.uname==local_uname?"false":"true", ais_data_len(msg), data); /* do_ais_log(level, */ /* "Msg[%d] (dest=%s:%s, from=%s:%s.%d, remote=%s, size=%d): %.90s", */ /* msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type), */ /* ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), */ /* msg->sender.pid, */ /* msg->sender.uname==local_uname?"false":"true", */ /* ais_data_len(msg), data); */ ais_free(data); } /* static gboolean ghash_find_by_uname(gpointer key, gpointer value, gpointer user_data) { crm_node_t *node = value; int id = GPOINTER_TO_INT(user_data); if (node->id == id) { return TRUE; } return FALSE; } */ -static gboolean -ghash_send_removal(gpointer key, gpointer value, gpointer data) -{ - if(send_client_msg(value, crm_class_rmpeer, crm_msg_none, data) != 0) { - /* remove it */ - return TRUE; - } - return FALSE; -} - -static void ais_remove_peer(char *node_id) -{ - uint32_t id = ais_get_int(node_id, NULL); - crm_node_t *node = g_hash_table_lookup(membership_list, GUINT_TO_POINTER(id)); - if(node == NULL) { - ais_info("Peer %u is unknown", id); - - } else if(ais_str_eq(CRM_NODE_MEMBER, node->state)) { - ais_warn("Peer %u/%s is still active", id, node->uname); - - } else if(g_hash_table_remove(membership_list, GUINT_TO_POINTER(id))) { - ais_notice("Removed dead peer %u from the membership list", id); - ais_info("Sending removal of %u to %d children", - id, g_hash_table_size(membership_notify_list)); - - g_hash_table_foreach_remove(membership_notify_list, ghash_send_removal, node_id); - - } else { - ais_warn("Peer %u/%s was not removed", id, node->uname); - } - -} - -gboolean process_ais_message(AIS_Message *msg) -{ - int len = ais_data_len(msg); - char *data = get_ais_data(msg); - do_ais_log(LOG_DEBUG, - "Msg[%d] (dest=%s:%s, from=%s:%s.%d, remote=%s, size=%d): %.90s", - msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type), - ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), - msg->sender.pid, - msg->sender.uname==local_uname?"false":"true", - ais_data_len(msg), data); - - if(data && len > 12 && strncmp("remove-peer:", data, 12) == 0) { - char *node = data+12; - ais_remove_peer(node); - } - - ais_free(data); - return TRUE; -} - static int ais_string_to_boolean(const char * s) { int rc = 0; if(s == NULL) { return rc; } if(strcasecmp(s, "true") == 0 || strcasecmp(s, "on") == 0 || strcasecmp(s, "yes") == 0 || strcasecmp(s, "y") == 0 || strcasecmp(s, "1") == 0) { rc = 1; } return rc; } gboolean spawn_child(crm_child_t *child) { int rc = 0; int lpc = 0; int uid = 0; int gid = 0; struct rlimit oflimits; struct passwd *pwentry = NULL; gboolean use_valgrind = FALSE; const char *devnull = "/dev/null"; const char *env_valgrind = getenv("HA_VALGRIND_ENABLED"); if(child->command == NULL) { ais_info("Nothing to do for child \"%s\"", child->name); return TRUE; } if(child->uid) { pwentry = getpwnam(child->uid); if(pwentry == NULL) { ais_err("Invalid uid (%s) specified for %s", child->uid, child->name); return TRUE; } uid = pwentry->pw_uid; gid = pwentry->pw_gid; } if(env_valgrind == NULL) { use_valgrind = FALSE; } else if(ais_string_to_boolean(env_valgrind)) { use_valgrind = TRUE; } else if(strstr(env_valgrind, child->name)) { use_valgrind = TRUE; } if(use_valgrind && strlen(VALGRIND_BIN) == 0) { ais_warn("Cannot enable valgrind for %s:" " The location of the valgrind binary is unknown", child->name); use_valgrind = FALSE; } child->pid = fork(); AIS_ASSERT(child->pid != -1); if(child->pid > 0) { /* parent */ ais_info("Forked child %d for process %s%s", child->pid, child->name, use_valgrind?" (valgrind enabled)":""); return TRUE; } /* else child */ /* The child should only log fatal errors, there is a possibility of a * deadlock if the parent was logging just before the fork() executed. * So any logging must relate to conditions worse than a deadlock */ if(0 && gid) { rc = setgid(gid); if(rc < 0) { ais_perror("Could not set group to %d", gid); } } if(uid) { rc = setuid(uid); if(rc < 0) { ais_perror("Could not set user to %d (%s)", uid, child->uid); } } /* A precautionary measure */ getrlimit(RLIMIT_NOFILE, &oflimits); for (; lpc < oflimits.rlim_cur; lpc++) { close(lpc); } (void)open(devnull, O_RDONLY); /* Stdin: fd 0 */ (void)open(devnull, O_WRONLY); /* Stdout: fd 1 */ (void)open(devnull, O_WRONLY); /* Stderr: fd 2 */ if(use_valgrind) { char *opts[] = { ais_strdup(VALGRIND_BIN), ais_strdup(child->command), NULL }; (void)execvp(VALGRIND_BIN, opts); } else { char *opts[] = { ais_strdup(child->command), NULL }; (void)execvp(child->command, opts); } ais_perror("FATAL: Cannot exec %s", child->command); exit(100); return TRUE; /* never reached */ } gboolean stop_child(crm_child_t *child, int signal) { if(signal == 0) { signal = SIGTERM; } if(child->command == NULL) { ais_info("Nothing to do for child \"%s\"", child->name); return TRUE; } ais_debug("Stopping CRM child \"%s\"", child->name); if (child->pid <= 0) { ais_debug_2("Client %s not running", child->name); return TRUE; } errno = 0; if(kill(child->pid, signal) == 0) { ais_notice("Sent -%d to %s: [%d]", signal, child->name, child->pid); } else { ais_perror("Sent -%d to %s: [%d]", signal, child->name, child->pid); } return TRUE; } void destroy_ais_node(gpointer data) { crm_node_t *node = data; ais_info("Destroying entry for node %u", node->id); ais_free(node->addr); ais_free(node->uname); ais_free(node->state); ais_free(node); } int update_member(unsigned int id, uint64_t born, uint64_t seq, int32_t votes, uint32_t procs, const char *uname, const char *state, const char *version) { int changed = 0; crm_node_t *node = NULL; node = g_hash_table_lookup(membership_list, GUINT_TO_POINTER(id)); if(node == NULL) { ais_malloc0(node, sizeof(crm_node_t)); ais_info("Creating entry for node %u born on "U64T"", id, seq); node->id = id; node->addr = NULL; node->state = ais_strdup("unknown"); g_hash_table_insert(membership_list, GUINT_TO_POINTER(id), node); node = g_hash_table_lookup(membership_list, GUINT_TO_POINTER(id)); } if(seq != 0) { node->last_seen = seq; } if(born != 0) { changed = TRUE; node->born = born; ais_info("%p Node %u (%s) born on: "U64T, node, id, uname, born); } if(version != NULL) { ais_free(node->version); node->version = ais_strdup(version); } if(uname != NULL) { if(node->uname == NULL || ais_str_eq(node->uname, uname) == FALSE) { ais_info("%p Node %u now known as %s (was: %s)", node, id, uname, node->uname); ais_free(node->uname); node->uname = ais_strdup(uname); changed = TRUE; } } if(procs != 0 && procs != node->processes) { ais_info("Node %s now has process list: %.32x (%u)", node->uname, procs, procs); node->processes = procs; changed = TRUE; } if(votes >= 0 && votes != node->votes) { ais_info("Node %s now has %d quorum votes (was %d)", node->uname, votes, node->votes); node->votes = votes; changed = TRUE; } if(state != NULL) { if(node->state == NULL || ais_str_eq(node->state, state) == FALSE) { ais_free(node->state); node->state = ais_strdup(state); ais_info("Node %u/%s is now: %s", id, node->uname?node->uname:"unknown", state); changed = TRUE; } } AIS_ASSERT(node != NULL); return changed; } void delete_member(uint32_t id, const char *uname) { if(uname == NULL) { g_hash_table_remove(membership_list, GUINT_TO_POINTER(id)); return; } ais_err("Deleting by uname is not yet supported"); } const char *member_uname(uint32_t id) { crm_node_t *node = g_hash_table_lookup( membership_list, GUINT_TO_POINTER(id)); if(node == NULL) { return ".unknown."; } if(node->uname == NULL) { return ".pending."; } return node->uname; } char *append_member(char *data, crm_node_t *node) { int size = 1; /* nul */ int offset = 0; static int fixed_len = 4 + 8 + 7 + 6 + 6 + 7 + 11; if(data) { size = strlen(data); } offset = size; size += fixed_len; size += 32; /* node->id */ size += 100; /* node->seq, node->born */ size += strlen(node->state); if(node->uname) { size += (7 + strlen(node->uname)); } if(node->addr) { size += (6 + strlen(node->addr)); } if(node->version) { size += (9 + strlen(node->version)); } data = realloc(data, size); offset += snprintf(data + offset, size - offset, "id); if(node->uname) { offset += snprintf(data + offset, size - offset, "uname=\"%s\" ", node->uname); } offset += snprintf(data + offset, size - offset, "state=\"%s\" ", node->state); offset += snprintf(data + offset, size - offset, "born=\""U64T"\" ", node->born); offset += snprintf(data + offset, size - offset, "seen=\""U64T"\" ", node->last_seen); offset += snprintf(data + offset, size - offset, "votes=\"%d\" ", node->votes); offset += snprintf(data + offset, size - offset, "processes=\"%u\" ", node->processes); if(node->addr) { offset += snprintf(data + offset, size - offset, "addr=\"%s\" ", node->addr); } if(node->version) { offset += snprintf(data + offset, size - offset, "version=\"%s\" ", node->version); } offset += snprintf(data + offset, size - offset, "/>"); return data; } void swap_sender(AIS_Message *msg) { int tmp = 0; char tmp_s[256]; tmp = msg->host.type; msg->host.type = msg->sender.type; msg->sender.type = tmp; tmp = msg->host.type; msg->host.size = msg->sender.type; msg->sender.type = tmp; memcpy(tmp_s, msg->host.uname, 256); memcpy(msg->host.uname, msg->sender.uname, 256); memcpy(msg->sender.uname, tmp_s, 256); } char *get_ais_data(AIS_Message *msg) { int rc = BZ_OK; char *uncompressed = NULL; unsigned int new_size = msg->size + 1; if(msg->is_compressed == FALSE) { uncompressed = strdup(msg->data); } else { ais_malloc0(uncompressed, new_size); rc = BZ2_bzBuffToBuffDecompress( uncompressed, &new_size, msg->data, msg->compressed_size, 1, 0); if(rc != BZ_OK) { ais_info("rc=%d, new=%u expected=%u", rc, new_size, msg->size); } AIS_ASSERT(rc == BZ_OK); AIS_ASSERT(new_size == msg->size); } return uncompressed; } int send_cluster_msg( enum crm_ais_msg_types type, const char *host, const char *data) { int rc = 0; int data_len = 0; AIS_Message *ais_msg = NULL; int total_size = sizeof(AIS_Message); AIS_ASSERT(local_nodeid != 0); if(data != NULL) { data_len = 1 + strlen(data); total_size += data_len; } ais_malloc0(ais_msg, total_size); ais_msg->header.size = total_size; ais_msg->header.error = SA_AIS_OK; ais_msg->header.id = 0; ais_msg->size = data_len; memcpy(ais_msg->data, data, data_len); ais_msg->sender.type = crm_msg_ais; ais_msg->host.type = type; ais_msg->host.id = 0; if(host) { ais_msg->host.size = strlen(host); memset(ais_msg->host.uname, 0, MAX_NAME); memcpy(ais_msg->host.uname, host, ais_msg->host.size); /* ais_msg->host.id = nodeid_lookup(host); */ } else { ais_msg->host.type = type; ais_msg->host.size = 0; memset(ais_msg->host.uname, 0, MAX_NAME); } rc = send_cluster_msg_raw(ais_msg); ais_free(ais_msg); return rc; } extern struct corosync_api_v1 *crm_api; int send_client_ipc(void *conn, AIS_Message *ais_msg) { int rc = -1; if (conn == NULL) { rc = -2; } else if (!libais_connection_active(conn)) { ais_warn("Connection no longer active"); rc = -3; /* } else if ((queue->size - 1) == queue->used) { */ /* ais_err("Connection is throttled: %d", queue->size); */ } else { #ifdef AIS_WHITETANK rc = openais_dispatch_send (conn, ais_msg, ais_msg->header.size); #endif #ifdef AIS_COROSYNC rc = crm_api->ipc_dispatch_send (conn, ais_msg, ais_msg->header.size); #endif } return rc; } int send_client_msg( void *conn, enum crm_ais_msg_class class, enum crm_ais_msg_types type, const char *data) { int rc = 0; int data_len = 0; int total_size = sizeof(AIS_Message); AIS_Message *ais_msg = NULL; static int msg_id = 0; AIS_ASSERT(local_nodeid != 0); msg_id++; AIS_ASSERT(msg_id != 0 /* wrap-around */); if(data != NULL) { data_len = 1 + strlen(data); } total_size += data_len; ais_malloc0(ais_msg, total_size); ais_msg->id = msg_id; ais_msg->header.id = class; ais_msg->header.size = total_size; ais_msg->header.error = SA_AIS_OK; ais_msg->size = data_len; memcpy(ais_msg->data, data, data_len); ais_msg->host.size = 0; ais_msg->host.type = type; memset(ais_msg->host.uname, 0, MAX_NAME); ais_msg->host.id = 0; ais_msg->sender.type = crm_msg_ais; ais_msg->sender.size = local_uname_len; memset(ais_msg->sender.uname, 0, MAX_NAME); memcpy(ais_msg->sender.uname, local_uname, ais_msg->sender.size); ais_msg->sender.id = local_nodeid; rc = send_client_ipc(conn, ais_msg); if(rc != 0) { ais_warn("Sending message to %s failed: %d", msg_type2text(type), rc); log_ais_message(LOG_DEBUG, ais_msg); return FALSE; } ais_free(ais_msg); return rc; } char * ais_concat(const char *prefix, const char *suffix, char join) { int len = 0; char *new_str = NULL; AIS_ASSERT(prefix != NULL); AIS_ASSERT(suffix != NULL); len = strlen(prefix) + strlen(suffix) + 2; ais_malloc0(new_str, (len)); sprintf(new_str, "%s%c%s", prefix, join, suffix); new_str[len-1] = 0; return new_str; } unsigned int config_find_init(plugin_init_type *config, char *name) { unsigned int local_handle = 0; #ifdef AIS_COROSYNC config->object_find_create(OBJECT_PARENT_HANDLE, name, strlen(name), &local_handle); #endif #ifdef AIS_WHITETANK config->object_find_reset (OBJECT_PARENT_HANDLE); #endif return local_handle; } unsigned int config_find_next(plugin_init_type *config, char *name, unsigned int top_handle) { int rc = 0; unsigned int local_handle = 0; #ifdef AIS_COROSYNC rc = config->object_find_next (top_handle, &local_handle); #endif #ifdef AIS_WHITETANK rc = config->object_find(OBJECT_PARENT_HANDLE, name, strlen (name), &local_handle); #endif if(rc < 0) { ais_info("No additional configuration supplied for: %s", name); local_handle = 0; } else { ais_info("Processing additional %s options...", name); } return local_handle; } void config_find_done(plugin_init_type *config, unsigned int local_handle) { #ifdef AIS_COROSYNC config->object_find_destroy (local_handle); #endif } int get_config_opt( plugin_init_type *config, unsigned int object_service_handle, char *key, char **value, const char *fallback) { char *env_key = NULL; *value = NULL; if(object_service_handle > 0) { config->object_key_get( object_service_handle, key, strlen(key), (void**)value, NULL); } if (*value) { ais_info("Found '%s' for option: %s", *value, key); return 0; } env_key = ais_concat("HA", key, '_'); *value = getenv(env_key); ais_free(env_key); if (*value) { ais_info("Found '%s' in ENV for option: %s", *value, key); return 0; } if(fallback) { ais_info("Defaulting to '%s' for option: %s", fallback, key); *value = ais_strdup(fallback); } else { ais_info("No default for option: %s", key); } return -1; } int ais_get_boolean(const char * value) { if(value == NULL) { return 0; } else if (strcasecmp(value, "true") == 0 || strcasecmp(value, "on") == 0 || strcasecmp(value, "yes") == 0 || strcasecmp(value, "y") == 0 || strcasecmp(value, "1") == 0){ return 1; } return 0; } long long ais_get_int(const char *text, char **end_text) { long long result = -1; char *local_end_text = NULL; errno = 0; if(text != NULL) { #ifdef ANSI_ONLY if(end_text != NULL) { result = strtol(text, end_text, 10); } else { result = strtol(text, &local_end_text, 10); } #else if(end_text != NULL) { result = strtoll(text, end_text, 10); } else { result = strtoll(text, &local_end_text, 10); } #endif if(errno == EINVAL) { ais_err("Conversion of %s failed", text); result = -1; } else if(errno == ERANGE) { ais_err("Conversion of %s was clipped: %lld", text, result); } else if(errno != 0) { ais_perror("Conversion of %s failed:", text); } if(local_end_text != NULL && local_end_text[0] != '\0') { ais_err("Characters left over after parsing '%s': '%s'", text, local_end_text); } } return result; } diff --git a/lib/common/ais.c b/lib/common/ais.c index 68ea66dd46..292d31c6b2 100644 --- a/lib/common/ais.c +++ b/lib/common/ais.c @@ -1,749 +1,770 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include "stack.h" #include #include enum crm_ais_msg_types text2msg_type(const char *text) { int type = crm_msg_none; CRM_CHECK(text != NULL, return type); if(safe_str_eq(text, "ais")) { type = crm_msg_ais; } else if(safe_str_eq(text, "crm_plugin")) { type = crm_msg_ais; } else if(safe_str_eq(text, CRM_SYSTEM_CIB)) { type = crm_msg_cib; } else if(safe_str_eq(text, CRM_SYSTEM_CRMD)) { type = crm_msg_crmd; } else if(safe_str_eq(text, CRM_SYSTEM_DC)) { type = crm_msg_crmd; } else if(safe_str_eq(text, CRM_SYSTEM_TENGINE)) { type = crm_msg_te; } else if(safe_str_eq(text, CRM_SYSTEM_PENGINE)) { type = crm_msg_pe; } else if(safe_str_eq(text, CRM_SYSTEM_LRMD)) { type = crm_msg_lrmd; } else if(safe_str_eq(text, CRM_SYSTEM_STONITHD)) { type = crm_msg_stonithd; } else if(safe_str_eq(text, "attrd")) { type = crm_msg_attrd; } else { crm_debug_2("Unknown message type: %s", text); } return type; } char *get_ais_data(AIS_Message *msg) { int rc = BZ_OK; char *uncompressed = NULL; unsigned int new_size = msg->size + 1; if(msg->is_compressed == FALSE) { crm_debug_2("Returning uncompressed message data"); uncompressed = strdup(msg->data); } else { crm_debug_2("Decompressing message data"); crm_malloc0(uncompressed, new_size); rc = BZ2_bzBuffToBuffDecompress( uncompressed, &new_size, msg->data, msg->compressed_size, 1, 0); CRM_ASSERT(rc = BZ_OK); CRM_ASSERT(new_size == msg->size); } return uncompressed; } #if SUPPORT_AIS int ais_fd_sync = -1; int ais_fd_async = -1; /* never send messages via this channel */ void *ais_ipc_ctx = NULL; GFDSource *ais_source = NULL; GFDSource *ais_source_sync = NULL; gboolean get_ais_nodeid(uint32_t *id, char **uname) { struct iovec iov; int retries = 0; int rc = SA_AIS_OK; mar_res_header_t header; struct crm_ais_nodeid_resp_s answer; header.error = SA_AIS_OK; header.id = crm_class_nodeid; header.size = sizeof(mar_res_header_t); CRM_CHECK(id != NULL, return FALSE); CRM_CHECK(uname != NULL, return FALSE); iov.iov_base = &header; iov.iov_len = header.size; retry: errno = 0; #ifdef TRADITIONAL_AIS_IPC rc = saSendReceiveReply(ais_fd_sync, &header, header.size, &answer, sizeof (struct crm_ais_nodeid_resp_s)); #else rc = openais_msg_send_reply_receive( ais_ipc_ctx, &iov, 1, &answer, sizeof (answer)); #endif if(rc == SA_AIS_OK) { CRM_CHECK(answer.header.size == sizeof (struct crm_ais_nodeid_resp_s), crm_err("Odd message: id=%d, size=%d, error=%d", answer.header.id, answer.header.size, answer.header.error)); CRM_CHECK(answer.header.id == crm_class_nodeid, crm_err("Bad response id: %d", answer.header.id)); } if(rc == SA_AIS_ERR_TRY_AGAIN && retries < 20) { retries++; crm_info("Peer overloaded: Re-sending message (Attempt %d of 20)", retries); mssleep(retries * 100); /* Proportional back off */ goto retry; } if(rc != SA_AIS_OK) { crm_err("Sending nodeid request: FAILED (rc=%d): %s", rc, ais_error2text(rc)); return FALSE; } else if(answer.header.error != SA_AIS_OK) { crm_err("Bad response from peer: (rc=%d): %s", rc, ais_error2text(rc)); return FALSE; } crm_info("Server details: id=%u uname=%s", answer.id, answer.uname); *id = answer.id; *uname = crm_strdup(answer.uname); return TRUE; } gboolean send_ais_text(int class, const char *data, gboolean local, const char *node, enum crm_ais_msg_types dest) { static int msg_id = 0; static int local_pid = 0; int retries = 0; int rc = SA_AIS_OK; int buf_len = sizeof(mar_res_header_t); char *buf = NULL; struct iovec iov; mar_res_header_t *header; AIS_Message *ais_msg = NULL; enum crm_ais_msg_types sender = text2msg_type(crm_system_name); - /* There are only 5 handlers registered to crm_lib_service in plugin.c */ - CRM_CHECK(class < 5, crm_err("Invalid message class: %d", class); return FALSE); - CRM_CHECK(data != NULL, return FALSE); - + /* There are only 6 handlers registered to crm_lib_service in plugin.c */ + CRM_CHECK(class < 6, crm_err("Invalid message class: %d", class); return FALSE); + if(local_pid == 0) { local_pid = getpid(); } crm_malloc0(ais_msg, sizeof(AIS_Message)); ais_msg->id = msg_id++; ais_msg->header.id = class; ais_msg->header.error = SA_AIS_OK; ais_msg->host.type = dest; ais_msg->host.local = local; if(node) { ais_msg->host.size = strlen(node); memset(ais_msg->host.uname, 0, MAX_NAME); memcpy(ais_msg->host.uname, node, ais_msg->host.size); ais_msg->host.id = 0; } else { ais_msg->host.size = 0; memset(ais_msg->host.uname, 0, MAX_NAME); ais_msg->host.id = 0; } ais_msg->sender.type = sender; ais_msg->sender.pid = local_pid; ais_msg->sender.size = 0; memset(ais_msg->sender.uname, 0, MAX_NAME); ais_msg->sender.id = 0; - - ais_msg->size = 1 + strlen(data); - if(ais_msg->size < CRM_BZ2_THRESHOLD) { + if(data) { + ais_msg->size = 1 + strlen(data); + } + + if(data == NULL) { + /* nothing to do */ + } else if(ais_msg->size < CRM_BZ2_THRESHOLD) { failback: crm_realloc(ais_msg, sizeof(AIS_Message) + ais_msg->size); memcpy(ais_msg->data, data, ais_msg->size); } else { char *compressed = NULL; char *uncompressed = crm_strdup(data); unsigned int len = (ais_msg->size * 1.1) + 600; /* recomended size */ crm_debug_5("Compressing message payload"); crm_malloc(compressed, len); rc = BZ2_bzBuffToBuffCompress( compressed, &len, uncompressed, ais_msg->size, CRM_BZ2_BLOCKS, 0, CRM_BZ2_WORK); crm_free(uncompressed); if(rc != BZ_OK) { crm_err("Compression failed: %d", rc); crm_free(compressed); goto failback; } crm_realloc(ais_msg, sizeof(AIS_Message) + len + 1); memcpy(ais_msg->data, compressed, len); ais_msg->data[len] = 0; crm_free(compressed); ais_msg->is_compressed = TRUE; ais_msg->compressed_size = len; crm_debug_2("Compression details: %d -> %d", ais_msg->size, ais_data_len(ais_msg)); } ais_msg->header.size = sizeof(AIS_Message) + ais_data_len(ais_msg); crm_debug_3("Sending%s message %d to %s.%s (data=%d, total=%d)", ais_msg->is_compressed?" compressed":"", ais_msg->id, ais_dest(&(ais_msg->host)), msg_type2text(dest), ais_data_len(ais_msg), ais_msg->header.size); iov.iov_base = ais_msg; iov.iov_len = ais_msg->header.size; retry: errno = 0; crm_realloc(buf, buf_len); #ifdef TRADITIONAL_AIS_IPC rc = saSendReceiveReply(ais_fd_sync, ais_msg, ais_msg->header.size, buf, buf_len); #else rc = openais_msg_send_reply_receive(ais_ipc_ctx, &iov, 1, buf, buf_len); #endif header = (mar_res_header_t *)buf; if(rc == SA_AIS_ERR_TRY_AGAIN && retries < 20) { retries++; crm_info("Peer overloaded: Re-sending message (Attempt %d of 20)", retries); mssleep(retries * 100); /* Proportional back off */ goto retry; } else if(rc == SA_AIS_OK) { CRM_CHECK_AND_STORE(header->size == sizeof (mar_res_header_t), crm_err("Odd message: id=%d, size=%d, class=%d, error=%d", header->id, header->size, class, header->error)); if(buf_len < header->size) { crm_err("Increasing buffer length to %d and retrying", header->size); buf_len = header->size + 1; goto retry; } else if(header->id == crm_class_nodeid && header->size == sizeof (struct crm_ais_nodeid_resp_s)){ struct crm_ais_nodeid_resp_s *answer = (struct crm_ais_nodeid_resp_s *)header; crm_err("Server details: id=%u uname=%s counter=%u", answer->id, answer->uname, answer->counter); } else { CRM_CHECK_AND_STORE(header->id == CRM_MESSAGE_IPC_ACK, crm_err("Bad response id (%d) for request (%d)", header->id, ais_msg->header.id)); CRM_CHECK(header->error == SA_AIS_OK, rc = header->error); } } if(rc != SA_AIS_OK) { crm_perror(LOG_ERR,"Sending message %d: FAILED (rc=%d): %s", ais_msg->id, rc, ais_error2text(rc)); ais_fd_async = -1; } else { crm_debug_4("Message %d: sent", ais_msg->id); } crm_free(buf); crm_free(ais_msg); return (rc == SA_AIS_OK); } gboolean send_ais_message(xmlNode *msg, gboolean local, const char *node, enum crm_ais_msg_types dest) { gboolean rc = TRUE; char *data = NULL; if(ais_fd_async < 0 || ais_source == NULL) { crm_err("Not connected to AIS"); return FALSE; } data = dump_xml_unformatted(msg); rc = send_ais_text(0, data, local, node, dest); crm_free(data); return rc; } void terminate_ais_connection(void) { #ifndef TRADITIONAL_AIS_IPC if(ais_ipc_ctx) { openais_service_disconnect(ais_ipc_ctx); } #else if(ais_fd_sync > 0) { close(ais_fd_sync); } if(ais_fd_async > 0) { close(ais_fd_async); } #endif crm_notice("Disconnected from AIS"); /* G_main_del_fd(ais_source); */ /* G_main_del_fd(ais_source_sync); */ } int ais_membership_timer = 0; gboolean ais_membership_force = FALSE; static gboolean ais_membership_dampen(gpointer data) { crm_debug_2("Requesting cluster membership after stabilization delay"); send_ais_text(crm_class_members, __FUNCTION__, TRUE, NULL, crm_msg_ais); ais_membership_force = TRUE; ais_membership_timer = 0; return FALSE; /* never repeat automatically */ } gboolean ais_dispatch(int sender, gpointer user_data) { char *data = NULL; char *uncompressed = NULL; int rc = SA_AIS_OK; AIS_Message *msg = NULL; gboolean (*dispatch)(AIS_Message*,char*,int) = user_data; #ifdef TRADITIONAL_AIS_IPC mar_res_header_t *header = NULL; static int header_len = sizeof(mar_res_header_t); crm_malloc0(header, header_len); errno = 0; rc = saRecvRetry(sender, header, header_len); if (rc != SA_AIS_OK) { crm_perror(LOG_ERR, "Receiving message header failed: (%d/%d) %s", rc, errno, ais_error2text(rc)); goto bail; } else if(header->size == header_len) { crm_err("Empty message: id=%d, size=%d, error=%d, header_len=%d", header->id, header->size, header->error, header_len); goto done; } else if(header->size == 0 || header->size < header_len) { crm_err("Mangled header: size=%d, header=%d, error=%d", header->size, header_len, header->error); goto done; } else if(header->error != SA_AIS_OK) { crm_err("Header contined error: %d", header->error); } crm_debug_2("Looking for %d (%d - %d) more bytes", header->size - header_len, header->size, header_len); crm_realloc(header, header->size); /* Use a char* so we can store the remainder into an offset */ data = (char*)header; errno = 0; rc = saRecvRetry(sender, data+header_len, header->size - header_len); #else crm_malloc0(data, 1000000); rc = openais_dispatch_recv (ais_ipc_ctx, data, 0); #endif - msg = (AIS_Message*)data; if (rc != SA_AIS_OK) { crm_perror(LOG_ERR,"Receiving message body failed: (%d) %s", rc, ais_error2text(rc)); goto bail; } + msg = (AIS_Message*)data; + + if(msg->id == crm_class_quorum) { + struct crm_ais_quorum_resp_s *resp = (struct crm_ais_quorum_resp_s *)data; + crm_debug("known: %u, available: %u: %s", + resp->expected_votes, resp->votes, resp->quorate?"true":"false"); + + if(resp->quorate != crm_have_quorum) { + crm_notice("Membership "U64T": quorum %s", + resp->id, resp->quorate?"attained":"lost"); + + } else { + crm_debug_2("Membership "U64T": quorum %s", + resp->id, resp->quorate?"retained":"lost"); + } + + crm_have_quorum = resp->quorate; + goto dispatch; + } + crm_debug_3("Got new%s message (size=%d, %d, %d)", msg->is_compressed?" compressed":"", ais_data_len(msg), msg->size, msg->compressed_size); data = msg->data; if(msg->is_compressed && msg->size > 0) { int rc = BZ_OK; unsigned int new_size = msg->size + 1; if(check_message_sanity(msg, NULL) == FALSE) { goto badmsg; } crm_debug_5("Decompressing message data"); crm_malloc0(uncompressed, new_size); rc = BZ2_bzBuffToBuffDecompress( uncompressed, &new_size, data, msg->compressed_size, 1, 0); if(rc != BZ_OK) { crm_err("Decompression failed: %d", rc); goto badmsg; } CRM_ASSERT(rc == BZ_OK); CRM_ASSERT(new_size == msg->size); data = uncompressed; } else if(check_message_sanity(msg, data) == FALSE) { goto badmsg; } else if(safe_str_eq("identify", data)) { int pid = getpid(); char *pid_s = crm_itoa(pid); send_ais_text(0, pid_s, TRUE, NULL, crm_msg_ais); crm_free(pid_s); goto done; } if(msg->header.id == crm_class_rmpeer) { uint32_t id = crm_int_helper(data, NULL); crm_info("Removing peer %s/%u", data, id); reap_crm_member(id); - crm_calculate_quorum(); goto done; } if(msg->header.id == crm_class_members) { xmlNode *xml = string2xml(data); if(xml != NULL) { gboolean do_ask = FALSE; gboolean do_process = TRUE; int new_size = 0; unsigned long long seq = 0; int current_size = crm_active_members(); const char *reason = "unknown"; const char *value = crm_element_value(xml, "id"); seq = crm_int_helper(value, NULL); crm_debug_2("Received membership %llu", seq); xml_child_iter(xml, node, const char *state = crm_element_value(node, "state"); if(safe_str_eq(state, CRM_NODE_MEMBER)) { new_size++; } ); if(ais_membership_force) { /* always process */ crm_debug_2("Processing delayed membership change"); #if 0 } else if(current_size == 0 && new_size == 1) { do_ask = TRUE; do_process = FALSE; reason = "We've come up alone"; #endif } else if(new_size < (current_size/2)) { do_process = FALSE; reason = "We've lost more than half our peers"; if(ais_membership_timer == 0) { reason = "We've lost more than half our peers"; crm_log_xml_debug(xml, __PRETTY_FUNCTION__); do_ask = TRUE; } } if(do_process) { static long long last = 0; /* if there is a timer running - let it run * there is no harm in getting an extra membership message */ crm_peer_seq = seq; /* Skip resends */ if(last < seq) { crm_info("Processing membership %llu", seq); } /* crm_log_xml_debug(xml, __PRETTY_FUNCTION__); */ if(ais_membership_force) { ais_membership_force = FALSE; } xml_child_iter(xml, node, crm_update_ais_node(node, seq)); - crm_calculate_quorum(); last = seq; } else if(do_ask) { dispatch = NULL; crm_warn("Pausing to allow membership stability (size %d -> %d): %s", current_size, new_size, reason); ais_membership_timer = Gmain_timeout_add(4*1000, ais_membership_dampen, NULL); /* process node additions */ xml_child_iter(xml, node, const char *state = crm_element_value(node, "state"); if(crm_str_eq(state, CRM_NODE_MEMBER, FALSE)) { crm_update_ais_node(node, seq); } ); } else { dispatch = NULL; crm_warn("Membership is still unstable (size %d -> %d): %s", current_size, new_size, reason); } } else { crm_warn("Invalid peer update: %s", data); } free_xml(xml); } else { const char *uuid = msg->sender.uname; crm_update_peer(msg->sender.id, 0,0,0,0, uuid, msg->sender.uname, NULL, NULL); } + dispatch: if(dispatch != NULL) { dispatch(msg, data, sender); } done: crm_free(uncompressed); crm_free(msg); return TRUE; badmsg: crm_err("Invalid message (id=%d, dest=%s:%s, from=%s:%s.%d):" " min=%d, total=%d, size=%d, bz2_size=%d", msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, (int)sizeof(AIS_Message), msg->header.size, msg->size, msg->compressed_size); goto done; bail: crm_err("AIS connection failed"); return FALSE; } static void ais_destroy(gpointer user_data) { crm_err("AIS connection terminated"); ais_fd_sync = -1; exit(1); } gboolean init_ais_connection( gboolean (*dispatch)(AIS_Message*,char*,int), void (*destroy)(gpointer), char **our_uuid, char **our_uname, int *nodeid) { int pid = 0; int retries = 0; int rc = SA_AIS_OK; char *pid_s = NULL; struct utsname name; uint32_t local_nodeid = 0; char *local_uname = NULL; retry: crm_info("Creating connection to our AIS plugin"); #ifdef TRADITIONAL_AIS_IPC rc = saServiceConnect (&ais_fd_sync, &ais_fd_async, CRM_SERVICE); #else rc = openais_service_connect(CRM_SERVICE, &ais_ipc_ctx); if(ais_ipc_ctx) { ais_fd_async = openais_fd_get(ais_ipc_ctx); } else if(rc == SA_AIS_OK) { crm_err("No context created, but connection reported 'ok'"); rc = SA_AIS_ERR_LIBRARY; } #endif if (rc != SA_AIS_OK) { crm_info("Connection to our AIS plugin (%d) failed: %s (%d)", CRM_SERVICE, ais_error2text(rc), rc); } switch(rc) { case SA_AIS_OK: break; case SA_AIS_ERR_TRY_AGAIN: if(retries < 30) { sleep(1); retries++; goto retry; } crm_err("Retry count exceeded"); return FALSE; default: return FALSE; } if(destroy == NULL) { crm_debug("Using the default destroy handler"); destroy = ais_destroy; } crm_info("AIS connection established"); pid = getpid(); pid_s = crm_itoa(pid); send_ais_text(0, pid_s, TRUE, NULL, crm_msg_ais); crm_free(pid_s); crm_peer_init(); get_ais_nodeid(&local_nodeid, &local_uname); if(uname(&name) < 0) { crm_perror(LOG_ERR,"uname(2) call failed"); exit(100); } if(safe_str_neq(name.nodename, local_uname)) { crm_crit("Node name mismatch! OpenAIS supplied %s, our lookup returned %s", local_uname, name.nodename); crm_notice("Node name mismatches usually occur when assigned automatically by DHCP servers"); crm_notice("If this node was part of the cluster with a different name," " you will need to remove the old entry with crm_node --remove"); } if(our_uuid != NULL) { *our_uuid = crm_strdup(local_uname); } if(our_uname != NULL) { *our_uname = local_uname; } if(nodeid != NULL) { *nodeid = local_nodeid; } if(local_nodeid != 0) { /* Ensure the local node always exists */ crm_update_peer(local_nodeid, 0, 0, 0, 0, local_uname, local_uname, NULL, NULL); } if(dispatch) { ais_source = G_main_add_fd( G_PRIORITY_HIGH, ais_fd_async, FALSE, ais_dispatch, dispatch, destroy); } return TRUE; } gboolean check_message_sanity(AIS_Message *msg, char *data) { gboolean sane = TRUE; gboolean repaired = FALSE; int dest = msg->host.type; int tmp_size = msg->header.size - sizeof(AIS_Message); if(sane && msg->header.size == 0) { crm_warn("Message with no size"); sane = FALSE; } if(sane && msg->header.error != SA_AIS_OK) { crm_warn("Message header contains an error: %d", msg->header.error); sane = FALSE; } if(sane && ais_data_len(msg) != tmp_size) { int cur_size = ais_data_len(msg); repaired = TRUE; if(msg->is_compressed) { msg->compressed_size = tmp_size; } else { msg->size = tmp_size; } crm_warn("Repaired message payload size %d -> %d", cur_size, tmp_size); } if(sane && ais_data_len(msg) == 0) { crm_warn("Message with no payload"); sane = FALSE; } if(sane && data && msg->is_compressed == FALSE) { int str_size = strlen(data) + 1; if(ais_data_len(msg) != str_size) { int lpc = 0; crm_warn("Message payload is corrupted: expected %d bytes, got %d", ais_data_len(msg), str_size); sane = FALSE; for(lpc = (str_size - 10); lpc < msg->size; lpc++) { if(lpc < 0) { lpc = 0; } crm_debug("bad_data[%d]: %d / '%c'", lpc, data[lpc], data[lpc]); } } } if(sane == FALSE) { crm_err("Invalid message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)", msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed, ais_data_len(msg), msg->header.size); } else if(repaired) { crm_err("Repaired message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)", msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed, ais_data_len(msg), msg->header.size); } else { crm_debug_3("Verfied message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)", msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed, ais_data_len(msg), msg->header.size); } return sane; } #endif diff --git a/lib/common/membership.c b/lib/common/membership.c index b62d3fb591..fc0da71e70 100644 --- a/lib/common/membership.c +++ b/lib/common/membership.c @@ -1,575 +1,426 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #ifndef _GNU_SOURCE # define _GNU_SOURCE #endif #include #include #include #include #include #include #include #include #include -struct quorum_count_s -{ - guint votes_max; - guint votes_active; - guint votes_total; - guint nodes_max; - guint nodes_total; -}; - GHashTable *crm_peer_id_cache = NULL; GHashTable *crm_peer_cache = NULL; unsigned long long crm_peer_seq = 0; -unsigned long long crm_max_peers = 0; -struct quorum_count_s quorum_stats; gboolean crm_have_quorum = FALSE; gboolean crm_is_member_active(const crm_node_t *node) { if(node && safe_str_eq(node->state, CRM_NODE_MEMBER)) { return TRUE; } return FALSE; } static gboolean crm_reap_dead_member( gpointer key, gpointer value, gpointer user_data) { crm_node_t *node = value; crm_node_t *search = user_data; if(search != NULL && node->id != search->id) { return FALSE; } else if(crm_is_member_active(value) == FALSE) { - quorum_stats.nodes_total -= 1; - quorum_stats.votes_total -= node->votes; - crm_notice("Removing %s/%u from the membership list (votes=%u, nodes=%u)", - node->uname, node->id, quorum_stats.votes_total, quorum_stats.nodes_total); + crm_notice("Removing %s/%u from the membership list", node->uname, node->id); return TRUE; } return FALSE; } guint reap_crm_member(uint32_t id) { int matches = 0; crm_node_t *node = g_hash_table_lookup(crm_peer_id_cache, GUINT_TO_POINTER(id)); if(node == NULL) { crm_info("Peer %u is unknown", id); } else if(crm_is_member_active(node)) { crm_warn("Peer %u/%s is still active", id, node->uname); } else { if(g_hash_table_remove(crm_peer_id_cache, GUINT_TO_POINTER(id))) { crm_notice("Removed dead peer %u from the uuid cache", id); } else { crm_warn("Peer %u/%s was not removed", id, node->uname); } matches = g_hash_table_foreach_remove( crm_peer_cache, crm_reap_dead_member, node); crm_notice("Removed %d dead peers with id=%u from the membership list", matches, id); } return matches; } static void crm_count_member( gpointer key, gpointer value, gpointer user_data) { guint *count = user_data; if(crm_is_member_active(value)) { *count = *count + 1; } } guint crm_active_members(void) { guint count = 0; g_hash_table_foreach(crm_peer_cache, crm_count_member, &count); return count; } struct peer_count_s { uint32_t peer; guint count; }; static void crm_count_peer( gpointer key, gpointer value, gpointer user_data) { crm_node_t *node = value; struct peer_count_s *search = user_data; if(crm_is_member_active(node) && (node->processes & search->peer)) { search->count = search->count + 1; } } guint crm_active_peers(uint32_t peer) { struct peer_count_s search; search.count = 0; search.peer = peer; g_hash_table_foreach(crm_peer_cache, crm_count_peer, &search); return search.count; } void destroy_crm_node(gpointer data) { crm_node_t *node = data; crm_debug_2("Destroying entry for node %u", node->id); crm_free(node->addr); crm_free(node->uname); crm_free(node->state); crm_free(node->uuid); crm_free(node); } void crm_peer_init(void) { - const char *value = NULL; static gboolean initialized = FALSE; if(initialized) { return; } initialized = TRUE; - quorum_stats.votes_max = 2; - quorum_stats.votes_active = 0; - quorum_stats.votes_total = 0; - quorum_stats.nodes_max = 1; - quorum_stats.nodes_total = 0; - - value = getenv("HA_expected_votes"); - if(value) { - crm_notice("%s expected quorum votes", value); - quorum_stats.votes_max = crm_int_helper(value, NULL); - } - value = getenv("HA_expected_nodes"); - if(value) { - crm_notice("%s expected nodes", value); - quorum_stats.nodes_max = crm_int_helper(value, NULL); - } - - if(quorum_stats.votes_max < 1) { - quorum_stats.votes_max = 1; - } - - if(quorum_stats.nodes_max < 1) { - quorum_stats.nodes_max = 1; - } - crm_peer_destroy(); if(crm_peer_cache == NULL) { crm_peer_cache = g_hash_table_new_full( g_str_hash, g_str_equal, NULL, destroy_crm_node); } if(crm_peer_id_cache == NULL) { crm_peer_id_cache = g_hash_table_new_full( g_direct_hash, g_direct_equal, NULL, NULL); } } void crm_peer_destroy(void) { if(crm_peer_cache != NULL) { g_hash_table_destroy(crm_peer_cache); crm_peer_cache = NULL; } if(crm_peer_id_cache != NULL) { g_hash_table_destroy(crm_peer_id_cache); crm_peer_id_cache = NULL; } } void (*crm_status_callback)(enum crm_status_type, crm_node_t*, const void*) = NULL; void crm_set_status_callback( void (*dispatch)(enum crm_status_type,crm_node_t*, const void*)) { crm_status_callback = dispatch; } static crm_node_t *crm_new_peer(unsigned int id, const char *uname) { crm_node_t *node = NULL; CRM_CHECK(uname != NULL || id > 0, return NULL); crm_debug("Creating entry for node %s/%u", uname, id); crm_malloc0(node, sizeof(crm_node_t)); node->state = crm_strdup("unknown"); if(id > 0) { node->id = id; crm_info("Node %s now has id: %u", crm_str(uname), id); g_hash_table_replace(crm_peer_id_cache, GUINT_TO_POINTER(node->id), node); } if(uname) { node->uname = crm_strdup(uname); CRM_ASSERT(node->uname != NULL); crm_info("Node %u is now known as %s", id, node->uname); g_hash_table_replace(crm_peer_cache, node->uname, node); if(is_openais_cluster()) { node->uuid = crm_strdup(node->uname); } if(crm_status_callback) { crm_status_callback(crm_status_uname, node, NULL); } } return node; } crm_node_t *crm_get_peer(unsigned int id, const char *uname) { crm_node_t *node = NULL; if(uname != NULL) { node = g_hash_table_lookup(crm_peer_cache, uname); } if(node == NULL && id > 0) { node = g_hash_table_lookup(crm_peer_id_cache, GUINT_TO_POINTER(id)); if(node && node->uname && uname) { crm_crit("Node %s and %s share the same cluster node id '%u'!", node->uname, uname, id); /* NOTE: Calling crm_new_peer() means the entry in * crm_peer_id_cache will point to the new entity */ /* TODO: Replace the old uname instead? */ node = crm_new_peer(id, uname); CRM_ASSERT(node->uname != NULL); } } if(node && uname && node->uname == NULL) { node->uname = crm_strdup(uname); crm_info("Node %u is now known as %s", id, uname); g_hash_table_insert(crm_peer_cache, node->uname, node); if(crm_status_callback) { crm_status_callback(crm_status_uname, node, NULL); } } if(node && id > 0 && id != node->id) { g_hash_table_remove(crm_peer_id_cache, GUINT_TO_POINTER(node->id)); g_hash_table_insert(crm_peer_id_cache, GUINT_TO_POINTER(id), node); node->id = id; crm_info("Node %s now has id: %u", crm_str(uname), id); } return node; } crm_node_t *crm_update_peer( unsigned int id, uint64_t born, uint64_t seen, int32_t votes, uint32_t children, const char *uuid, const char *uname, const char *addr, const char *state) { gboolean state_changed = FALSE; gboolean addr_changed = FALSE; gboolean procs_changed = FALSE; gboolean votes_changed = FALSE; crm_node_t *node = NULL; CRM_CHECK(uname != NULL || id > 0, return NULL); CRM_ASSERT(crm_peer_cache != NULL); CRM_ASSERT(crm_peer_id_cache != NULL); node = crm_get_peer(id, uname); if(node == NULL) { node = crm_new_peer(id, uname); /* do it now so we don't get '(new)' everywhere */ node->votes = votes; node->processes = children; if(addr) { node->addr = crm_strdup(addr); } } if(votes > 0 && node->votes != votes) { votes_changed = TRUE; node->votes = votes; } if(node->uuid == NULL) { if(uuid != NULL) { node->uuid = crm_strdup(uuid); } else if(node->uname != NULL && is_openais_cluster()) { node->uuid = crm_strdup(node->uname); } } if(children > 0 && children != node->processes) { uint32_t last = node->processes; node->processes = children; procs_changed = TRUE; if(crm_status_callback) { crm_status_callback(crm_status_processes, node, &last); } } if(born != 0) { node->born = born; } if(state != NULL && safe_str_neq(node->state, state)) { char *last = node->state; node->state = crm_strdup(state); state_changed = TRUE; if(crm_status_callback) { crm_status_callback(crm_status_nstate, node, last); } crm_free(last); } if(seen != 0 && crm_is_member_active(node)) { node->last_seen = seen; } if(addr != NULL) { if(node->addr == NULL || crm_str_eq(node->addr, addr, FALSE) == FALSE) { addr_changed = TRUE; crm_free(node->addr); node->addr = crm_strdup(addr); } } if(state_changed || addr_changed || votes_changed || procs_changed) { crm_info("Node %s: id=%u state=%s%s addr=%s%s votes=%d%s born="U64T" seen="U64T" proc=%.32x%s", node->uname, node->id, node->state, state_changed?" (new)":"", node->addr, addr_changed?" (new)":"", node->votes, votes_changed?" (new)":"", node->born, node->last_seen, node->processes, procs_changed?" (new)":"" ); } return node; } crm_node_t *crm_update_ais_node(xmlNode *member, long long seq) { const char *id_s = crm_element_value(member, "id"); const char *addr = crm_element_value(member, "addr"); const char *uname = crm_element_value(member, "uname"); const char *state = crm_element_value(member, "state"); const char *born_s = crm_element_value(member, "born"); const char *seen_s = crm_element_value(member, "seen"); const char *votes_s = crm_element_value(member, "votes"); const char *procs_s = crm_element_value(member, "processes"); int votes = crm_int_helper(votes_s, NULL); unsigned int id = crm_int_helper(id_s, NULL); unsigned int procs = crm_int_helper(procs_s, NULL); /* TODO: These values will contain garbage if version < 0.7.1 */ uint64_t born = crm_int_helper(born_s, NULL); uint64_t seen = crm_int_helper(seen_s, NULL); return crm_update_peer(id, born, seen, votes, procs, uname, uname, addr, state); } #if SUPPORT_HEARTBEAT crm_node_t *crm_update_ccm_node( const oc_ev_membership_t *oc, int offset, const char *state, uint64_t seq) { crm_node_t *node = NULL; const char *uuid = NULL; CRM_CHECK(oc->m_array[offset].node_uname != NULL, return NULL); uuid = get_uuid(oc->m_array[offset].node_uname); node = crm_update_peer(oc->m_array[offset].node_id, oc->m_array[offset].node_born_on, seq, -1, 0, uuid, oc->m_array[offset].node_uname, NULL, state); if(safe_str_eq(CRM_NODE_ACTIVE, state)) { /* Heartbeat doesn't send status notifications for nodes that were already part of the cluster */ crm_update_peer_proc( oc->m_array[offset].node_uname, crm_proc_ais, ONLINESTATUS); /* Nor does it send status notifications for processes that were already active */ crm_update_peer_proc( oc->m_array[offset].node_uname, crm_proc_crmd, ONLINESTATUS); } return node; } #endif void crm_update_peer_proc(const char *uname, uint32_t flag, const char *status) { crm_node_t *node = NULL; gboolean changed = FALSE; CRM_ASSERT(crm_peer_cache != NULL); CRM_CHECK(uname != NULL, return); node = g_hash_table_lookup(crm_peer_cache, uname); CRM_CHECK(node != NULL, crm_err("Could not set %s.%s to %s", uname, peer2text(flag), status); return); if(safe_str_eq(status, ONLINESTATUS)) { if((node->processes & flag) == 0) { set_bit_inplace(node->processes, flag); changed = TRUE; } } else if(node->processes & flag) { clear_bit_inplace(node->processes, flag); changed = TRUE; } if(changed) { crm_info("%s.%s is now %s", uname, peer2text(flag), status); } } - -static void crm_count_quorum( - gpointer key, gpointer value, gpointer user_data) -{ - crm_node_t *node = value; - quorum_stats.nodes_total += 1; - quorum_stats.votes_total += node->votes; - if(crm_is_member_active(node)) { - quorum_stats.votes_active = quorum_stats.votes_active + node->votes; - } -} - -gboolean crm_calculate_quorum(void) -{ - unsigned int limit = 0; - gboolean quorate = TRUE; - quorum_stats.votes_total = 0; - quorum_stats.nodes_total = 0; - quorum_stats.votes_active = 0; - - g_hash_table_foreach(crm_peer_cache, crm_count_quorum, NULL); - - if(quorum_stats.votes_total > quorum_stats.votes_max) { - crm_info("Known quorum votes: %u -> %u", - quorum_stats.votes_max, quorum_stats.votes_total); - quorum_stats.votes_max = quorum_stats.votes_total; - } - - if(quorum_stats.nodes_total > quorum_stats.nodes_max) { - crm_debug("Known quorum nodes: %u -> %u", - quorum_stats.nodes_max, quorum_stats.nodes_total); - quorum_stats.nodes_max = quorum_stats.nodes_total; - } - - limit = (quorum_stats.votes_max + 2) / 2; - if(quorum_stats.votes_active < limit) { - quorate = FALSE; - } - - crm_debug("known: %u, available: %u, limit: %u, active: %u: %s", - quorum_stats.votes_max, quorum_stats.votes_total, - limit, quorum_stats.votes_active, quorate?"true":"false"); - - if(quorate != crm_have_quorum) { - crm_notice("Membership %llu: quorum %s", - crm_peer_seq, quorate?"attained":"lost"); - - } else { - crm_debug_2("Membership %llu: quorum %s", - crm_peer_seq, quorate?"retained":"lost"); - } - - crm_have_quorum = quorate; - return quorate; -} - -/* Code appropriated (with permission) from cman/daemon/commands.c under GPLv2 */ - -#if 0 -static int calculate_quorum(int allow_decrease, int max_expected, unsigned int *ret_total_votes) -{ - struct list *nodelist; - struct cluster_node *node; - unsigned int total_votes = 0; - unsigned int highest_expected = 0; - unsigned int newquorum, q1, q2; - unsigned int total_nodes = 0; - - list_iterate(nodelist, &cluster_members_list) { - node = list_item(nodelist, struct cluster_node); - - if (node->state == NODESTATE_MEMBER) { - highest_expected = - max(highest_expected, node->expected_votes); - total_votes += node->votes; - total_nodes++; - } - } - if (quorum_device && quorum_device->state == NODESTATE_MEMBER) - total_votes += quorum_device->votes; - - if (max_expected > 0) - highest_expected = max_expected; - - /* This quorum calculation is taken from the OpenVMS Cluster Systems - * manual, but, then, you guessed that didn't you */ - q1 = (highest_expected + 2) / 2; - q2 = (total_votes + 2) / 2; - newquorum = max(q1, q2); - - /* Normally quorum never decreases but the system administrator can - * force it down by setting expected votes to a maximum value */ - if (!allow_decrease) - newquorum = max(quorum, newquorum); - - /* The special two_node mode allows each of the two nodes to retain - * quorum if the other fails. Only one of the two should live past - * fencing (as both nodes try to fence each other in split-brain.) - * Also: if there are more than two nodes, force us inquorate to avoid - * any damage or confusion. - */ - if (two_node && total_nodes <= 2) - newquorum = 1; - - if (ret_total_votes) - *ret_total_votes = total_votes; - return newquorum; -} -#endif diff --git a/tools/attrd.c b/tools/attrd.c index 33e265510a..62f0596578 100644 --- a/tools/attrd.c +++ b/tools/attrd.c @@ -1,784 +1,779 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define OPTARGS "hV" #if SUPPORT_HEARTBEAT ll_cluster_t *attrd_cluster_conn; #endif GMainLoop* mainloop = NULL; char *attrd_uname = NULL; char *attrd_uuid = NULL; gboolean need_shutdown = FALSE; GHashTable *attr_hash = NULL; cib_t *cib_conn = NULL; typedef struct attr_hash_entry_s { char *uuid; char *id; char *set; char *section; char *value; char *stored_value; int timeout; char *dampen; guint timer_id; } attr_hash_entry_t; static void free_hash_entry(gpointer data) { attr_hash_entry_t *entry = data; if (entry == NULL) { return; } crm_free(entry->id); crm_free(entry->set); crm_free(entry->dampen); crm_free(entry->section); crm_free(entry->uuid); crm_free(entry->value); crm_free(entry->stored_value); crm_free(entry); } void attrd_local_callback(xmlNode * msg); gboolean attrd_timer_callback(void *user_data); gboolean attrd_trigger_update(attr_hash_entry_t *hash_entry); void attrd_perform_update(attr_hash_entry_t *hash_entry); static gboolean attrd_shutdown(int nsig, gpointer unused) { need_shutdown = TRUE; crm_info("Exiting"); if (mainloop != NULL && g_main_is_running(mainloop)) { g_main_quit(mainloop); } else { exit(0); } return FALSE; } static void usage(const char* cmd, int exit_status) { FILE* stream; stream = exit_status ? stderr : stdout; fprintf(stream, "usage: %s [-srkh] [-c configure file]\n", cmd); /* fprintf(stream, "\t-d\tsets debug level\n"); */ /* fprintf(stream, "\t-s\tgets daemon status\n"); */ /* fprintf(stream, "\t-r\trestarts daemon\n"); */ /* fprintf(stream, "\t-k\tstops daemon\n"); */ /* fprintf(stream, "\t-h\thelp message\n"); */ fflush(stream); exit(exit_status); } typedef struct attrd_client_s { char *id; char *name; IPC_Channel *channel; GCHSource *source; } attrd_client_t; static void stop_attrd_timer(attr_hash_entry_t *hash_entry) { if(hash_entry != NULL && hash_entry->timer_id != 0) { crm_debug_2("Stopping %s timer", hash_entry->id); Gmain_timeout_remove(hash_entry->timer_id); hash_entry->timer_id = 0; } } static gboolean attrd_ipc_callback(IPC_Channel *client, gpointer user_data) { int lpc = 0; xmlNode *msg = NULL; attrd_client_t *curr_client = (attrd_client_t*)user_data; gboolean stay_connected = TRUE; crm_debug_2("Invoked: %s", curr_client->id); while(IPC_ISRCONN(client)) { if(client->ops->is_message_pending(client) == 0) { break; } msg = xmlfromIPC(client, MAX_IPC_DELAY); if (msg == NULL) { break; } lpc++; crm_debug_2("Processing msg from %s", curr_client->id); crm_log_xml(LOG_DEBUG_3, __PRETTY_FUNCTION__, msg); attrd_local_callback(msg); free_xml(msg); msg = NULL; if(client->ch_status != IPC_CONNECT) { break; } } crm_debug_2("Processed %d messages", lpc); if (client->ch_status != IPC_CONNECT) { stay_connected = FALSE; } return stay_connected; } static void attrd_connection_destroy(gpointer user_data) { attrd_client_t *client = user_data; /* cib_process_disconnect */ if(client == NULL) { return; } if(client->source != NULL) { crm_debug_4("Deleting %s (%p) from mainloop", client->name, client->source); G_main_del_IPC_Channel(client->source); client->source = NULL; } crm_debug_3("Destroying %s (%p)", client->name, client); crm_free(client->name); crm_free(client->id); crm_free(client); crm_debug_4("Freed the cib client"); return; } static gboolean attrd_connect(IPC_Channel *channel, gpointer user_data) { attrd_client_t *new_client = NULL; crm_debug_3("Connecting channel"); if(channel == NULL) { crm_err("Channel was NULL"); return FALSE; } else if(channel->ch_status != IPC_CONNECT) { crm_err("Channel was disconnected"); return FALSE; } else if(need_shutdown) { crm_info("Ignoring connection request during shutdown"); return FALSE; } crm_malloc0(new_client, sizeof(attrd_client_t)); new_client->channel = channel; crm_debug_3("Created channel %p for channel %s", new_client, new_client->id); /* channel->ops->set_recv_qlen(channel, 100); */ /* channel->ops->set_send_qlen(channel, 400); */ new_client->source = G_main_add_IPC_Channel( G_PRIORITY_DEFAULT, channel, FALSE, attrd_ipc_callback, new_client, attrd_connection_destroy); crm_debug_3("Client %s connected", new_client->id); return TRUE; } static void log_hash_entry(int level, attr_hash_entry_t *entry, const char *text) { do_crm_log(level, "%s", text); do_crm_log(level, "Set: %s", entry->section); do_crm_log(level, "Name: %s", entry->id); do_crm_log(level, "Value: %s", entry->value); do_crm_log(level, "Timeout: %s", entry->dampen); } static attr_hash_entry_t * find_hash_entry(xmlNode * msg) { const char *value = NULL; const char *attr = crm_element_value(msg, F_ATTRD_ATTRIBUTE); attr_hash_entry_t *hash_entry = NULL; if(attr == NULL) { crm_info("Ignoring message with no attribute name"); return NULL; } hash_entry = g_hash_table_lookup(attr_hash, attr); if(hash_entry == NULL) { /* create one and add it */ crm_info("Creating hash entry for %s", attr); crm_malloc0(hash_entry, sizeof(attr_hash_entry_t)); hash_entry->id = crm_strdup(attr); g_hash_table_insert(attr_hash, hash_entry->id, hash_entry); hash_entry = g_hash_table_lookup(attr_hash, attr); CRM_CHECK(hash_entry != NULL, return NULL); } value = crm_element_value(msg, F_ATTRD_SET); if(value != NULL) { crm_free(hash_entry->set); hash_entry->set = crm_strdup(value); crm_debug("\t%s->set: %s", attr, value); } value = crm_element_value(msg, F_ATTRD_SECTION); if(value == NULL) { value = XML_CIB_TAG_STATUS; } crm_free(hash_entry->section); hash_entry->section = crm_strdup(value); crm_debug_2("\t%s->section: %s", attr, value); value = crm_element_value(msg, F_ATTRD_DAMPEN); if(value != NULL) { crm_free(hash_entry->dampen); hash_entry->dampen = crm_strdup(value); hash_entry->timeout = crm_get_msec(value); crm_debug_2("\t%s->timeout: %s", attr, value); } log_hash_entry(LOG_DEBUG_2, hash_entry, "Found (and updated) entry:"); return hash_entry; } #if SUPPORT_HEARTBEAT static void attrd_ha_connection_destroy(gpointer user_data) { crm_debug_3("Invoked"); if(need_shutdown) { /* we signed out, so this is expected */ crm_info("Heartbeat disconnection complete"); return; } crm_crit("Lost connection to heartbeat service!"); if (mainloop != NULL && g_main_is_running(mainloop)) { g_main_quit(mainloop); return; } exit(LSB_EXIT_OK); } static void attrd_ha_callback(HA_Message *msg, void* private_data) { attr_hash_entry_t *hash_entry = NULL; xmlNode *xml = convert_ha_message(NULL, msg, __FUNCTION__); const char *from = crm_element_value(xml, F_ORIG); const char *op = crm_element_value(xml, F_ATTRD_TASK); const char *host = crm_element_value(xml, F_ATTRD_HOST); const char *ignore = crm_element_value(xml, F_ATTRD_IGNORE_LOCALLY); if(host != NULL && safe_str_eq(host, attrd_uname)) { crm_info("Update relayed from %s", from); attrd_local_callback(xml); } else if(ignore == NULL || safe_str_neq(from, attrd_uname)) { crm_info("%s message from %s", op, from); hash_entry = find_hash_entry(xml); stop_attrd_timer(hash_entry); attrd_perform_update(hash_entry); } free_xml(xml); } #endif #if SUPPORT_AIS static gboolean attrd_ais_dispatch(AIS_Message *wrapper, char *data, int sender) { xmlNode *xml = NULL; - switch(wrapper->header.id) { - case crm_class_notify: - case crm_class_members: - break; - default: + if(wrapper->header.id == crm_class_cluster) { xml = string2xml(data); if(xml == NULL) { crm_err("Bad message received: %d:'%.120s'", wrapper->id, data); } - break; } if(xml != NULL) { attr_hash_entry_t *hash_entry = NULL; const char *op = crm_element_value(xml, F_ATTRD_TASK); const char *host = crm_element_value(xml, F_ATTRD_HOST); const char *ignore = crm_element_value(xml, F_ATTRD_IGNORE_LOCALLY); crm_xml_add_int(xml, F_SEQ, wrapper->id); crm_xml_add(xml, F_ORIG, wrapper->sender.uname); if(host != NULL && safe_str_eq(host, attrd_uname)) { crm_info("Update relayed from %s", wrapper->sender.uname); attrd_local_callback(xml); } else if(ignore == NULL || safe_str_neq(wrapper->sender.uname, attrd_uname)) { crm_info("%s message from %s", op, wrapper->sender.uname); hash_entry = find_hash_entry(xml); stop_attrd_timer(hash_entry); attrd_perform_update(hash_entry); } free_xml(xml); } return TRUE; } static void attrd_ais_destroy(gpointer unused) { ais_fd_sync = -1; if(need_shutdown) { /* we signed out, so this is expected */ crm_info("OpenAIS disconnection complete"); return; } crm_crit("Lost connection to OpenAIS service!"); if (mainloop != NULL && g_main_is_running(mainloop)) { g_main_quit(mainloop); return; } exit(LSB_EXIT_GENERIC); } #endif static gboolean register_with_ha(void) { void *destroy = NULL; void *dispatch = NULL; if(is_openais_cluster()) { #if SUPPORT_AIS destroy = attrd_ais_destroy; dispatch = attrd_ais_dispatch; #endif } if(is_heartbeat_cluster()) { #if SUPPORT_HEARTBEAT dispatch = attrd_ha_callback; destroy = attrd_ha_connection_destroy; #endif } return crm_cluster_connect(&attrd_uname, &attrd_uuid, dispatch, destroy, #if SUPPORT_HEARTBEAT &attrd_cluster_conn #else NULL #endif ); } static void attrd_cib_connection_destroy(gpointer user_data) { if(need_shutdown) { crm_info("Connection to the CIB terminated..."); } else { /* eventually this will trigger a reconnect, not a shutdown */ crm_err("Connection to the CIB terminated..."); exit(1); } return; } static void update_for_hash_entry(gpointer key, gpointer value, gpointer user_data) { attrd_timer_callback(value); } static void do_cib_replaced(const char *event, xmlNode *msg) { crm_info("Sending full refresh"); g_hash_table_foreach(attr_hash, update_for_hash_entry, NULL); } int main(int argc, char ** argv) { int flag; int argerr = 0; gboolean was_err = FALSE; char *channel_name = crm_strdup(T_ATTRD); crm_log_init(T_ATTRD, LOG_INFO, TRUE, FALSE, 0, NULL); G_main_add_SignalHandler( G_PRIORITY_HIGH, SIGTERM, attrd_shutdown, NULL, NULL); while ((flag = getopt(argc, argv, OPTARGS)) != EOF) { switch(flag) { case 'V': cl_log_enable_stderr(1); alter_debug(DEBUG_INC); break; case 'h': /* Help message */ usage(T_ATTRD, LSB_EXIT_OK); break; default: ++argerr; break; } } if (optind > argc) { ++argerr; } if (argerr) { usage(T_ATTRD, LSB_EXIT_GENERIC); } attr_hash = g_hash_table_new_full( g_str_hash, g_str_equal, NULL, free_hash_entry); crm_info("Starting up...."); if(register_with_ha() == FALSE) { crm_err("HA Signon failed"); was_err = TRUE; } if(was_err == FALSE) { int rc = init_server_ipc_comms( channel_name, attrd_connect, default_ipc_connection_destroy); if(rc != 0) { crm_err("Could not start IPC server"); was_err = TRUE; } } if(was_err == FALSE) { int lpc = 0; int max_retry = 20; enum cib_errors rc = cib_not_connected; cib_conn = cib_new(); for(lpc = 0; lpc < max_retry && rc != cib_ok; lpc++) { crm_debug("CIB signon attempt %d", lpc); rc = cib_conn->cmds->signon( cib_conn, T_ATTRD, cib_command); sleep(5); } if(rc != cib_ok) { crm_err("Signon to CIB failed: %s", cib_error2string(rc)); was_err = TRUE; } } if(was_err == FALSE) { enum cib_errors rc = cib_conn->cmds->set_connection_dnotify( cib_conn, attrd_cib_connection_destroy); if(rc != cib_ok) { crm_err("Could not set dnotify callback"); was_err = TRUE; } } if(was_err == FALSE) { if(cib_ok != cib_conn->cmds->add_notify_callback( cib_conn, T_CIB_REPLACE_NOTIFY, do_cib_replaced)) { crm_err("Could not set CIB notification callback"); was_err = TRUE; } } if(was_err) { crm_err("Aborting startup"); return 100; } crm_info("Sending full refresh"); g_hash_table_foreach(attr_hash, update_for_hash_entry, NULL); crm_info("Starting mainloop..."); mainloop = g_main_new(FALSE); g_main_run(mainloop); crm_info("Exiting..."); #if SUPPORT_HEARTBEAT if(is_heartbeat_cluster()) { attrd_cluster_conn->llc_ops->signoff(attrd_cluster_conn, TRUE); attrd_cluster_conn->llc_ops->delete(attrd_cluster_conn); } #endif cib_conn->cmds->signoff(cib_conn); cib_delete(cib_conn); g_hash_table_destroy(attr_hash); crm_free(channel_name); crm_free(attrd_uuid); empty_uuid_cache(); return 0; } struct attrd_callback_s { char *attr; char *value; }; static void attrd_cib_callback(xmlNode *msg, int call_id, int rc, xmlNode *output, void *user_data) { int err_level = LOG_ERR; attr_hash_entry_t *hash_entry = NULL; struct attrd_callback_s *data = user_data; if(data->value == NULL && rc == cib_NOTEXISTS) { rc = cib_ok; } switch(rc) { case cib_ok: crm_debug("Update %d for %s=%s passed", call_id, data->attr, data->value); hash_entry = g_hash_table_lookup(attr_hash, data->attr); if(hash_entry) { crm_free(hash_entry->stored_value); hash_entry->stored_value = NULL; if(data->value != NULL) { hash_entry->stored_value = crm_strdup(data->value); } } break; case cib_remote_timeout: /* When an attr changes while there is a DC election */ case cib_NOTEXISTS: /* When an attr changes while the CIB is syncing a * newer config from a node that just came up */ err_level = LOG_WARNING; default: do_crm_log(err_level, "Update %d for %s=%s failed: %s", call_id, data->attr, data->value, cib_error2string(rc)); } crm_free(data->value); crm_free(data->attr); crm_free(data); } void attrd_perform_update(attr_hash_entry_t *hash_entry) { int rc = cib_ok; struct attrd_callback_s *data = NULL; if(hash_entry == NULL) { return; } else if(cib_conn == NULL) { crm_info("Delaying operation %s=%s: cib not connected", hash_entry->id, crm_str(hash_entry->value)); return; } else if(hash_entry->value == NULL) { /* delete the attr */ rc = delete_attr(cib_conn, cib_none, hash_entry->section, attrd_uuid, hash_entry->set, hash_entry->uuid, hash_entry->id, NULL, FALSE); crm_info("Sent delete %d: node=%s, attr=%s, id=%s, set=%s, section=%s", rc, attrd_uuid, hash_entry->id, hash_entry->uuid?hash_entry->uuid:"", hash_entry->set, hash_entry->section); } else { /* send update */ rc = update_attr(cib_conn, cib_none, hash_entry->section, attrd_uuid, hash_entry->set, hash_entry->uuid, hash_entry->id, hash_entry->value, FALSE); crm_info("Sent update %d: %s=%s", rc, hash_entry->id, hash_entry->value); } crm_malloc0(data, sizeof(struct attrd_callback_s)); data->attr = crm_strdup(hash_entry->id); if(hash_entry->value != NULL) { data->value = crm_strdup(hash_entry->value); } add_cib_op_callback(cib_conn, rc, FALSE, data, attrd_cib_callback); return; } void attrd_local_callback(xmlNode * msg) { attr_hash_entry_t *hash_entry = NULL; const char *from = crm_element_value(msg, F_ORIG); const char *op = crm_element_value(msg, F_ATTRD_TASK); const char *attr = crm_element_value(msg, F_ATTRD_ATTRIBUTE); const char *value = crm_element_value(msg, F_ATTRD_VALUE); const char *host = crm_element_value(msg, F_ATTRD_HOST); if(safe_str_eq(op, "refresh")) { crm_info("Sending full refresh (origin=%s)", from); g_hash_table_foreach(attr_hash, update_for_hash_entry, NULL); return; } if(host != NULL && safe_str_neq(host, attrd_uname)) { send_cluster_message(host, crm_msg_attrd, msg, FALSE); return; } crm_debug("%s message from %s: %s=%s", op, from, attr, crm_str(value)); hash_entry = find_hash_entry(msg); if(hash_entry == NULL) { return; } if(hash_entry->uuid == NULL) { const char *key = crm_element_value(msg, F_ATTRD_KEY); if(key) { hash_entry->uuid = crm_strdup(key); } } crm_debug("Supplied: %s, Current: %s, Stored: %s", value, hash_entry->value, hash_entry->stored_value); if(safe_str_eq(value, hash_entry->value)) { crm_debug_2("Ignoring non-change"); return; } crm_free(hash_entry->value); hash_entry->value = NULL; if(value != NULL) { hash_entry->value = crm_strdup(value); crm_debug("New value of %s is %s", attr, value); } stop_attrd_timer(hash_entry); if(hash_entry->timeout > 0) { hash_entry->timer_id = Gmain_timeout_add( hash_entry->timeout, attrd_timer_callback, hash_entry); } else { attrd_trigger_update(hash_entry); } return; } gboolean attrd_timer_callback(void *user_data) { stop_attrd_timer(user_data); attrd_trigger_update(user_data); return TRUE; } gboolean attrd_trigger_update(attr_hash_entry_t *hash_entry) { xmlNode *msg = NULL; /* send HA message to everyone */ crm_info("Sending flush op to all hosts for: %s", hash_entry->id); log_hash_entry(LOG_DEBUG_2, hash_entry, "Sending flush op to all hosts for:"); msg = create_xml_node(NULL, __FUNCTION__); crm_xml_add(msg, F_TYPE, T_ATTRD); crm_xml_add(msg, F_ORIG, attrd_uname); crm_xml_add(msg, F_ATTRD_TASK, "flush"); crm_xml_add(msg, F_ATTRD_ATTRIBUTE, hash_entry->id); crm_xml_add(msg, F_ATTRD_SET, hash_entry->set); crm_xml_add(msg, F_ATTRD_SECTION, hash_entry->section); crm_xml_add(msg, F_ATTRD_DAMPEN, hash_entry->dampen); crm_xml_add(msg, F_ATTRD_VALUE, hash_entry->value); if(hash_entry->timeout <= 0) { crm_xml_add(msg, F_ATTRD_IGNORE_LOCALLY, hash_entry->value); attrd_perform_update(hash_entry); } send_cluster_message(NULL, crm_msg_attrd, msg, FALSE); free_xml(msg); return TRUE; } diff --git a/tools/ccm_epoche.c b/tools/ccm_epoche.c index 9f7baec570..533aed22f6 100644 --- a/tools/ccm_epoche.c +++ b/tools/ccm_epoche.c @@ -1,371 +1,382 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #ifdef HAVE_GETOPT_H # include #endif #include /* for basename() */ #include #include #include #include #include #include int command = 0; #define OPTARGS "hVqepHR:" int ccm_fd = 0; int try_hb = 1; int try_ais = 1; const char *uname = NULL; void usage(const char* cmd, int exit_status); void ais_membership_destroy(gpointer user_data); gboolean ais_membership_dispatch(AIS_Message *wrapper, char *data, int sender); #include <../lib/common/stack.h> #if SUPPORT_HEARTBEAT # include # include oc_ev_t *ccm_token = NULL; void oc_ev_special(const oc_ev_t *, oc_ev_class_t , int ); void ccm_age_callback( oc_ed_t event, void *cookie, size_t size, const void *data); gboolean ccm_age_connect(int *ccm_fd); #endif int main(int argc, char ** argv) { int flag = 0; int argerr = 0; gboolean force_flag = FALSE; gboolean dangerous_cmd = FALSE; #ifdef HAVE_GETOPT_H int option_index = 0; static struct option long_options[] = { /* Top-level Options */ {"remove", 1, 0, 'R'}, {"partition", 0, 0, 'p'}, {"epoch", 0, 0, 'e'}, {"quorum", 0, 0, 'q'}, {"force", 0, 0, 'f'}, {"verbose", 0, 0, 'V'}, {"help", 0, 0, '?'}, {0, 0, 0, 0} }; #endif crm_peer_init(); crm_log_init(basename(argv[0]), LOG_WARNING, FALSE, FALSE, 0, NULL); while (flag >= 0) { #ifdef HAVE_GETOPT_H flag = getopt_long(argc, argv, OPTARGS, long_options, &option_index); #else flag = getopt(argc, argv, OPTARGS); #endif switch(flag) { case -1: break; case 'V': cl_log_enable_stderr(TRUE); alter_debug(DEBUG_INC); break; case 'h': /* Help message */ usage(crm_system_name, LSB_EXIT_OK); break; case 'H': try_ais = 0; break; case 'A': try_hb = 0; break; case 'f': force_flag = TRUE; break; case 'R': dangerous_cmd = TRUE; command = flag; uname = optarg; break; case 'p': case 'e': case 'q': command = flag; break; default: ++argerr; break; } } if (optind > argc) { ++argerr; } if (argerr) { usage(crm_system_name,LSB_EXIT_GENERIC); } if(dangerous_cmd && force_flag == FALSE) { fprintf(stderr, "The supplied command is considered dangerous." " To prevent accidental destruction of the cluster," " the --force flag is required in order to proceed.\n"); fflush(stderr); exit(LSB_EXIT_GENERIC); } #if SUPPORT_AIS if(try_ais && init_ais_connection( ais_membership_dispatch, ais_membership_destroy, NULL, NULL, NULL)) { GMainLoop* amainloop = NULL; - if(command == 'r') { - send_ais_text(crm_class_rmpeer, uname, TRUE, NULL, crm_msg_ais); - return 0; + switch(command) { + case 'R': + send_ais_text(crm_class_rmpeer, uname, TRUE, NULL, crm_msg_ais); + return 0; - } else { - crm_info("Requesting the list of configured nodes"); - send_ais_text(crm_class_members, __FUNCTION__, TRUE, NULL, crm_msg_ais); + case 'e': + /* Age makes no sense (yet) in an AIS cluster */ + fprintf(stdout, "1\n"); + return 0; + + case 'q': + send_ais_text(crm_class_quorum, NULL, TRUE, NULL, crm_msg_ais); + break; + + case 'p': + crm_info("Requesting the list of configured nodes"); + send_ais_text(crm_class_members, __FUNCTION__, TRUE, NULL, crm_msg_ais); + break; + + default: + fprintf(stderr, "Unknown option '%c'\n", command); + usage(crm_system_name, LSB_EXIT_GENERIC); } amainloop = g_main_new(FALSE); g_main_run(amainloop); } #endif #if SUPPORT_HEARTBEAT if(try_hb && ccm_age_connect(&ccm_fd)) { int rc = 0; fd_set rset; oc_ev_t *ccm_token = NULL; while (1) { FD_ZERO(&rset); FD_SET(ccm_fd, &rset); rc = select(ccm_fd + 1, &rset, NULL,NULL,NULL); if(rc < 0) { crm_perror(LOG_ERR, "select failed"); if(errno == EINTR) { crm_debug("Retry..."); continue; } } else if(oc_ev_handle_event(ccm_token) != 0){ crm_err("oc_ev_handle_event failed"); } return(1); } } #endif return(1); } void usage(const char* cmd, int exit_status) { FILE* stream; stream = exit_status ? stderr : stdout; fprintf(stream, "usage: %s [-V] [-p|-e|-q]\n", cmd); fprintf(stream, "\t--%s (-%c)\tprint the members of this partition\n", "partition", 'p'); fprintf(stream, "\t--%s (-%c)\tprint the epoch this node joined the partition\n", "epoch", 'e'); fprintf(stream, "\t--%s (-%c)\tprint a 1 if our partition has quorum\n", "quorum", 'q'); fflush(stream); exit(exit_status); } #if SUPPORT_HEARTBEAT gboolean ccm_age_connect(int *ccm_fd) { gboolean did_fail = FALSE; int ret = 0; crm_debug("Registering with CCM"); ret = oc_ev_register(&ccm_token); if (ret != 0) { crm_warn("CCM registration failed"); did_fail = TRUE; } if(did_fail == FALSE) { crm_debug("Setting up CCM callbacks"); ret = oc_ev_set_callback(ccm_token, OC_EV_MEMB_CLASS, ccm_age_callback, NULL); if (ret != 0) { crm_warn("CCM callback not set"); did_fail = TRUE; } } if(did_fail == FALSE) { oc_ev_special(ccm_token, OC_EV_MEMB_CLASS, 0/*don't care*/); crm_debug("Activating CCM token"); ret = oc_ev_activate(ccm_token, ccm_fd); if (ret != 0){ crm_warn("CCM Activation failed"); did_fail = TRUE; } } return !did_fail; } void ccm_age_callback(oc_ed_t event, void *cookie, size_t size, const void *data) { int lpc; int node_list_size; const oc_ev_membership_t *oc = (const oc_ev_membership_t *)data; node_list_size = oc->m_n_member; if(command == 'q') { crm_debug("Processing \"%s\" event.", event==OC_EV_MS_NEW_MEMBERSHIP?"NEW MEMBERSHIP": event==OC_EV_MS_NOT_PRIMARY?"NOT PRIMARY": event==OC_EV_MS_PRIMARY_RESTORED?"PRIMARY RESTORED": event==OC_EV_MS_EVICTED?"EVICTED": "NO QUORUM MEMBERSHIP"); if(ccm_have_quorum(event)) { fprintf(stdout, "1\n"); } else { fprintf(stdout, "0\n"); } } else if(command == 'e') { crm_debug("Searching %d members for our birth", oc->m_n_member); } for(lpc=0; lpcm_array[oc->m_memb_idx+lpc].node_uname); } else if(command == 'e') { if(oc_ev_is_my_nodeid(ccm_token, &(oc->m_array[lpc]))){ crm_debug("MATCH: nodeid=%d, uname=%s, born=%d", oc->m_array[oc->m_memb_idx+lpc].node_id, oc->m_array[oc->m_memb_idx+lpc].node_uname, oc->m_array[oc->m_memb_idx+lpc].node_born_on); fprintf(stdout, "%d\n", oc->m_array[oc->m_memb_idx+lpc].node_born_on); } } } oc_ev_callback_done(cookie); if(command == 'p') { fprintf(stdout, "\n"); } fflush(stdout); exit(0); } #endif #if SUPPORT_AIS void ais_membership_destroy(gpointer user_data) { crm_err("AIS connection terminated"); ais_fd_sync = -1; exit(1); } #endif static gint member_sort(gconstpointer a, gconstpointer b) { const crm_node_t *node_a = a; const crm_node_t *node_b = b; return strcmp(node_a->uname, node_b->uname); } static void crm_add_member( gpointer key, gpointer value, gpointer user_data) { GList **list = user_data; crm_node_t *node = value; if(node->uname != NULL) { *list = g_list_insert_sorted(*list, node, member_sort); } } gboolean ais_membership_dispatch(AIS_Message *wrapper, char *data, int sender) { - crm_info("Message received"); switch(wrapper->header.id) { case crm_class_members: case crm_class_notify: + case crm_class_quorum: break; default: return TRUE; break; } if(command == 'q') { if(crm_have_quorum) { fprintf(stdout, "1\n"); } else { fprintf(stdout, "0\n"); } - } else if(command == 'e') { - /* Age makes no sense (yet) in an AIS cluster */ - fprintf(stdout, "1\n"); - } else if(command == 'p') { GList *nodes = NULL; g_hash_table_foreach(crm_peer_cache, crm_add_member, &nodes); slist_iter(node, crm_node_t, nodes, lpc, if(node->uname && crm_is_member_active(node)) { fprintf(stdout, "%s ", node->uname); } ); fprintf(stdout, "\n"); } exit(0); return TRUE; }