diff --git a/daemons/execd/pacemaker_remote.service.in b/daemons/execd/pacemaker_remote.service.in index 3189a59865..cad70d23bd 100644 --- a/daemons/execd/pacemaker_remote.service.in +++ b/daemons/execd/pacemaker_remote.service.in @@ -1,49 +1,52 @@ [Unit] Description=Pacemaker Remote executor daemon Documentation=man:pacemaker-remoted Documentation=https://clusterlabs.org/pacemaker/doc/en-US/Pacemaker/2.0/html-single/Pacemaker_Remote/index.html # See main pacemaker unit file for descriptions of why these are needed After=network.target After=time-sync.target After=dbus.service Wants=dbus.service After=resource-agents-deps.target Wants=resource-agents-deps.target After=syslog.service After=rsyslog.service [Install] Alias=pacemaker-remote.service WantedBy=multi-user.target [Service] Type=simple KillMode=process NotifyAccess=none EnvironmentFile=-@CONFIGDIR@/pacemaker EnvironmentFile=-@CONFIGDIR@/sbd +# Not actually success, but fatal failure -- this ensures no respawn +SuccessExitStatus=100 + ExecStart=@sbindir@/pacemaker-remoted # Systemd v227 and above can limit the number of processes spawned by a # service. That is a bad idea for an HA cluster resource manager, so disable it # by default. The administrator can create a local override if they really want # a limit. If your systemd version does not support TasksMax, and you want to # get rid of the resulting log warnings, comment out this option. TasksMax=infinity # If connected to the cluster and when the service functions properly, it will # wait to exit until the cluster notifies it all resources on the remote node # have been stopped. The default of 30min should cover most typical cluster # configurations, but it may need an increase to adapt to local conditions # (e.g. a large, clustered database could conceivably take longer to stop). TimeoutStopSec=30min TimeoutStartSec=30s # Restart options include: no, on-success, on-failure, on-abort or always Restart=on-failure # crm_perror() writes directly to stderr, so ignore it here # to avoid double-logging with the wrong format StandardError=null diff --git a/daemons/execd/remoted_tls.c b/daemons/execd/remoted_tls.c index 8681419421..9af88b369e 100644 --- a/daemons/execd/remoted_tls.c +++ b/daemons/execd/remoted_tls.c @@ -1,408 +1,418 @@ /* * Copyright 2012-2019 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pacemaker-execd.h" #ifdef HAVE_GNUTLS_GNUTLS_H # include // Hidden in liblrmd extern int lrmd_tls_set_key(gnutls_datum_t *key); # define LRMD_REMOTE_AUTH_TIMEOUT 10000 gnutls_psk_server_credentials_t psk_cred_s; gnutls_dh_params_t dh_params; static int ssock = -1; extern int lrmd_call_id; static void debug_log(int level, const char *str) { fputs(str, stderr); } /*! * \internal * \brief Read (more) TLS handshake data from client */ static int remoted__read_handshake_data(crm_client_t *client) { int rc = pcmk__read_handshake_data(client); if (rc == 0) { /* No more data is available at the moment. Just return for now; * we'll get invoked again once the client sends more. */ return 0; } else if (rc < 0) { crm_err("TLS handshake with remote client failed: %s " CRM_XS " rc=%d", gnutls_strerror(rc), rc); return -1; } if (client->remote->auth_timeout) { g_source_remove(client->remote->auth_timeout); } client->remote->auth_timeout = 0; client->remote->tls_handshake_complete = TRUE; crm_notice("Remote client connection accepted"); // Alert other clients of the new connection notify_of_new_client(client); return 0; } static int lrmd_remote_client_msg(gpointer data) { int id = 0; int rc = 0; int disconnected = 0; xmlNode *request = NULL; crm_client_t *client = data; if (client->remote->tls_handshake_complete == FALSE) { return remoted__read_handshake_data(client); } rc = crm_remote_ready(client->remote, 0); if (rc == 0) { /* no msg to read */ return 0; } else if (rc < 0) { crm_info("Remote client disconnected while polling it"); return -1; } crm_remote_recv(client->remote, -1, &disconnected); request = crm_remote_parse_buffer(client->remote); while (request) { crm_element_value_int(request, F_LRMD_REMOTE_MSG_ID, &id); crm_trace("Processing remote client request %d", id); if (!client->name) { const char *value = crm_element_value(request, F_LRMD_CLIENTNAME); if (value) { client->name = strdup(value); } } lrmd_call_id++; if (lrmd_call_id < 1) { lrmd_call_id = 1; } crm_xml_add(request, F_LRMD_CLIENTID, client->id); crm_xml_add(request, F_LRMD_CLIENTNAME, client->name); crm_xml_add_int(request, F_LRMD_CALLID, lrmd_call_id); process_lrmd_message(client, id, request); free_xml(request); /* process all the messages in the current buffer */ request = crm_remote_parse_buffer(client->remote); } if (disconnected) { crm_info("Remote client disconnected while reading from it"); return -1; } return 0; } static void lrmd_remote_client_destroy(gpointer user_data) { crm_client_t *client = user_data; if (client == NULL) { return; } crm_notice("Cleaning up after remote client %s disconnected " CRM_XS " id=%s", (client->name? client->name : ""), client->id); ipc_proxy_remove_provider(client); /* if this is the last remote connection, stop recurring * operations */ if (crm_hash_table_size(client_connections) == 1) { client_disconnect_cleanup(NULL); } if (client->remote->tls_session) { void *sock_ptr; int csock; sock_ptr = gnutls_transport_get_ptr(*client->remote->tls_session); csock = GPOINTER_TO_INT(sock_ptr); gnutls_bye(*client->remote->tls_session, GNUTLS_SHUT_RDWR); gnutls_deinit(*client->remote->tls_session); gnutls_free(client->remote->tls_session); close(csock); } lrmd_client_destroy(client); return; } static gboolean lrmd_auth_timeout_cb(gpointer data) { crm_client_t *client = data; client->remote->auth_timeout = 0; if (client->remote->tls_handshake_complete == TRUE) { return FALSE; } mainloop_del_fd(client->remote->source); client->remote->source = NULL; crm_err("Remote client authentication timed out"); return FALSE; } static int lrmd_remote_listen(gpointer data) { int csock = 0; gnutls_session_t *session = NULL; crm_client_t *new_client = NULL; static struct mainloop_fd_callbacks lrmd_remote_fd_cb = { .dispatch = lrmd_remote_client_msg, .destroy = lrmd_remote_client_destroy, }; csock = crm_remote_accept(ssock); if (csock < 0) { return TRUE; } session = pcmk__new_tls_session(csock, GNUTLS_SERVER, GNUTLS_CRD_PSK, psk_cred_s); if (session == NULL) { close(csock); return TRUE; } new_client = crm_client_alloc(NULL); new_client->remote = calloc(1, sizeof(crm_remote_t)); new_client->kind = CRM_CLIENT_TLS; new_client->remote->tls_session = session; // Require the client to authenticate within this time new_client->remote->auth_timeout = g_timeout_add(LRMD_REMOTE_AUTH_TIMEOUT, lrmd_auth_timeout_cb, new_client); crm_info("Remote client pending authentication " CRM_XS " %p id: %s", new_client, new_client->id); new_client->remote->source = mainloop_add_fd("pacemaker-remote-client", G_PRIORITY_DEFAULT, csock, new_client, &lrmd_remote_fd_cb); return TRUE; } static void lrmd_remote_connection_destroy(gpointer user_data) { crm_notice("TLS server session ended"); return; } static int lrmd_tls_server_key_cb(gnutls_session_t session, const char *username, gnutls_datum_t * key) { return lrmd_tls_set_key(key); } static int bind_and_listen(struct addrinfo *addr) { int optval; int fd; int rc; char buffer[INET6_ADDRSTRLEN] = { 0, }; crm_sockaddr2str(addr->ai_addr, buffer); crm_trace("Attempting to bind to address %s", buffer); fd = socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol); if (fd < 0) { crm_perror(LOG_ERR, "Listener socket creation failed"); return -1; } /* reuse address */ optval = 1; rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); if (rc < 0) { crm_perror(LOG_ERR, "Local address reuse not allowed on %s", buffer); close(fd); return -1; } if (addr->ai_family == AF_INET6) { optval = 0; rc = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &optval, sizeof(optval)); if (rc < 0) { crm_perror(LOG_INFO, "Couldn't disable IPV6-only on %s", buffer); close(fd); return -1; } } if (bind(fd, addr->ai_addr, addr->ai_addrlen) != 0) { crm_perror(LOG_ERR, "Cannot bind to %s", buffer); close(fd); return -1; } if (listen(fd, 10) == -1) { crm_perror(LOG_ERR, "Cannot listen on %s", buffer); close(fd); return -1; } return fd; } int lrmd_init_remote_tls_server() { int rc; int filter; int port = crm_default_remote_port(); struct addrinfo hints, *res = NULL, *iter; char port_str[6]; // at most "65535" gnutls_datum_t psk_key = { NULL, 0 }; + const char *bind_name = getenv("PCMK_remote_address"); static struct mainloop_fd_callbacks remote_listen_fd_callbacks = { .dispatch = lrmd_remote_listen, .destroy = lrmd_remote_connection_destroy, }; - crm_debug("Starting TLS listener on port %d", port); + crm_debug("Starting TLS listener on %s port %d", + (bind_name? bind_name : "all addresses on"), port); crm_gnutls_global_init(); gnutls_global_set_log_function(debug_log); if (pcmk__init_tls_dh(&dh_params) != GNUTLS_E_SUCCESS) { return -1; } gnutls_psk_allocate_server_credentials(&psk_cred_s); gnutls_psk_set_server_credentials_function(psk_cred_s, lrmd_tls_server_key_cb); gnutls_psk_set_server_dh_params(psk_cred_s, dh_params); /* The key callback won't get called until the first client connection * attempt. Do it once here, so we can warn the user at start-up if we can't * read the key. We don't error out, though, because it's fine if the key is * going to be added later. */ rc = lrmd_tls_set_key(&psk_key); if (rc != 0) { crm_warn("A cluster connection will not be possible until the key is available"); } gnutls_free(psk_key.data); memset(&hints, 0, sizeof(struct addrinfo)); - /* Bind to the wildcard address (INADDR_ANY or IN6ADDR_ANY_INIT). - * @TODO allow user to specify a specific address - */ hints.ai_flags = AI_PASSIVE; hints.ai_family = AF_UNSPEC; /* Return IPv6 or IPv4 */ hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = IPPROTO_TCP; snprintf(port_str, sizeof(port_str), "%d", port); - rc = getaddrinfo(NULL, port_str, &hints, &res); + rc = getaddrinfo(bind_name, port_str, &hints, &res); if (rc) { - crm_err("Unable to get IP address info for local node: %s", - gai_strerror(rc)); + crm_err("Unable to get IP address(es) for %s: %s", + (bind_name? bind_name : "local node"), gai_strerror(rc)); return -1; } + /* Currently we listen on only one address from the resulting list (the + * first IPv6 address we can bind to if possible, otherwise the first IPv4 + * address we can bind to). When bind_name is NULL, this should be the + * respective wildcard address. + * + * @TODO If there is demand for specifying more than one address, allow + * bind_name to be a space-separated list, call getaddrinfo() for each, + * and create a socket for each result (set IPV6_V6ONLY on IPv6 sockets + * since IPv4 listeners will have their own sockets). + */ + rc = -1; iter = res; filter = AF_INET6; - /* Try IPv6 addresses first, then IPv4 */ while (iter) { if (iter->ai_family == filter) { ssock = bind_and_listen(iter); } if (ssock != -1) { break; } iter = iter->ai_next; if (iter == NULL && filter == AF_INET6) { iter = res; filter = AF_INET; } } if (ssock < 0) { goto init_remote_cleanup; } mainloop_add_fd("pacemaker-remote-server", G_PRIORITY_DEFAULT, ssock, NULL, &remote_listen_fd_callbacks); rc = ssock; init_remote_cleanup: if (rc < 0) { close(ssock); ssock = 0; } else { - crm_debug("Started TLS listener on port %d", port); + crm_debug("Started TLS listener on %s port %d", + (bind_name? bind_name : "all addresses on"), port); } freeaddrinfo(res); return rc; } void lrmd_tls_server_destroy(void) { if (psk_cred_s) { gnutls_psk_free_server_credentials(psk_cred_s); psk_cred_s = 0; } if (ssock > 0) { close(ssock); ssock = 0; } } #endif diff --git a/daemons/pacemakerd/pacemaker.sysconfig b/daemons/pacemakerd/pacemaker.sysconfig index 27002a4b0d..c7745d85dd 100644 --- a/daemons/pacemakerd/pacemaker.sysconfig +++ b/daemons/pacemakerd/pacemaker.sysconfig @@ -1,160 +1,168 @@ #==#==# Variables that control logging # Enable debug logging globally (yes|no) or by subsystem. Multiple subsystems # may be comma-separated, for example: PCMK_debug=pacemakerd,pacemaker-execd # Subsystems are: # pacemakerd # pacemaker-attrd # pacemaker-based # pacemaker-controld # pacemaker-execd # pacemaker-fenced # pacemaker-schedulerd # PCMK_debug=no # Send detailed log messages to the specified file. Compared to messages logged # via syslog, messages in this file may have extended information, and will # include messages of "info" severity (and, if debug and/or trace logging # has been enabled, those as well). This log is of more use to developers and # advanced system administrators, and when reporting problems. # PCMK_logfile=/var/log/pacemaker/pacemaker.log # Enable logging via syslog, using the specified syslog facility. Messages sent # here are of value to all Pacemaker users. This can be disabled using "none", # but that is not recommended. The default is "daemon". # PCMK_logfacility=none|daemon|user|local0|local1|local2|local3|local4|local5|local6|local7 # Unless syslog logging is disabled using PCMK_logfacility=none, messages of # the specified severity and higher will be sent to syslog. The default value # of "notice" is appropriate for most installations; "info" is highly verbose # and "debug" is almost certain to send you blind (which is why there is a # separate detail log specified by PCMK_logfile). # PCMK_logpriority=emerg|alert|crit|error|warning|notice|info|debug # Log all messages from a comma-separated list of functions. # PCMK_trace_functions=function1,function2,function3 # Log all messages from a comma-separated list of files (no path). # Wildcards are supported, e.g. PCMK_trace_files=prefix*.c # PCMK_trace_files=file.c,other.h # Log all messages matching comma-separated list of formats. # PCMK_trace_formats="Sent delete %d" # Log all messages from a comma-separated list of tags. # PCMK_trace_tags=tag1,tag2 # Dump the blackbox whenever the message at function and line is emitted, # e.g. PCMK_trace_blackbox=te_graph_trigger:223,unpack_clone:81 # PCMK_trace_blackbox=fn:line,fn2:line2,... # Enable blackbox logging globally or per-subsystem. The blackbox contains a # rolling buffer of all logs (including info, debug, and trace) and is written # after a crash or assertion failure, and/or when SIGTRAP is received. The # blackbox recorder can also be enabled for Pacemaker daemons at runtime by # sending SIGUSR1 (or SIGTRAP), and disabled by sending SIGUSR2. Specify value # as for PCMK_debug above. # PCMK_blackbox=no #==#==# Advanced use only # By default, nodes will join the cluster in an online state when they first # start, unless they were previously put into standby mode. If this variable is # set to "standby" or "online", it will force this node to join in the # specified state when starting. # (only supported for cluster nodes, not Pacemaker Remote nodes) # PCMK_node_start_state=default # Specify an alternate location for RNG schemas and XSL transforms. # (This is of use only to developers.) # PCMK_schema_directory=/some/path # Pacemaker consists of a master process with multiple subsidiary daemons. If # one of the daemons crashes, the master process will normally attempt to # restart it. If this is set to "true", the master process will instead panic # the host (see PCMK_panic_action). The default is unset. # PCMK_fail_fast=no # Pacemaker will panic its host under certain conditions. If this is set to # "crash", Pacemaker will trigger a kernel crash (which is useful if you want a # kernel dump to investigate). For any other value, Pacemaker will trigger a # host reboot. The default is unset. # PCMK_panic_action=crash #==#==# Pacemaker Remote # Use the contents of this file as the authorization key to use with Pacemaker # Remote connections. This file must be readable by Pacemaker daemons (that is, # it must allow read permissions to either the hacluster user or the haclient # group), and its contents must be identical on all nodes. The default is # "/etc/pacemaker/authkey". # PCMK_authkey_location=/etc/pacemaker/authkey +# If the Pacemaker Remote service is run on the local node, it will listen +# for connections on this address. The value may be a resolvable hostname or an +# IPv4 or IPv6 numeric address. When resolving names or using the default +# wildcard address (i.e. listen on all available addresses), IPv6 will be +# preferred if available. When listening on an IPv6 address, IPv4 clients will +# be supported (via IPv4-mapped IPv6 addresses). +# PCMK_remote_address="192.0.2.1" + # Use this TCP port number when connecting to a Pacemaker Remote node. This # value must be the same on all nodes. The default is "3121". # PCMK_remote_port=3121 # Use these GnuTLS cipher priorities for TLS connections. See: # # https://gnutls.org/manual/html_node/Priority-Strings.html # # Pacemaker will append ":+ANON-DH" for remote CIB access (when enabled) and # ":+DHE-PSK:+PSK" for Pacemaker Remote connections, as they are required for # the respective functionality. # PCMK_tls_priorities="NORMAL" # Set bounds on the bit length of the prime number generated for Diffie-Hellman # parameters needed by TLS connections. The default is not to set any bounds. # # If these values are specified, the server (Pacemaker Remote daemon, or CIB # manager configured to accept remote clients) will use these values to provide # a floor and/or ceiling for the value recommended by the GnuTLS library. The # library will only accept a limited number of specific values, which vary by # library version, so setting these is recommended only when required for # compatibility with specific client versions. # # If PCMK_dh_min_bits is specified, the client (connecting cluster node or # remote CIB command) will require that the server use a prime of at least this # size. This is only recommended when the value must be lowered in order for # the client's GnuTLS library to accept a connection to an older server. # The client side does not use PCMK_dh_max_bits. # # PCMK_dh_min_bits=1024 # PCMK_dh_max_bits=2048 #==#==# IPC # Force use of a particular class of IPC connection. # PCMK_ipc_type=shared-mem|socket|posix|sysv # Specify an IPC buffer size in bytes. This is useful when connecting to really # big clusters that exceed the default 128KB buffer. # PCMK_ipc_buffer=131072 #==#==# Profiling and memory leak testing (mainly useful to developers) # Affect the behavior of glib's memory allocator. Setting to "always-malloc" # when running under valgrind will help valgrind track malloc/free better; # setting to "debug-blocks" when not running under valgrind will perform # (somewhat expensive) memory checks. # G_SLICE=always-malloc # Uncommenting this will make malloc() initialize newly allocated memory # and free() wipe it (to help catch uninitialized-memory/use-after-free). # MALLOC_PERTURB_=221 # Uncommenting this will make malloc() and friends print to stderr and abort # for some (inexpensive) memory checks. # MALLOC_CHECK_=3 # Set as for PCMK_debug above to run some or all daemons under valgrind. # PCMK_valgrind_enabled=no # Set as for PCMK_debug above to run some or all daemons under valgrind with # the callgrind tool enabled. # PCMK_callgrind_enabled=no # Set the options to pass to valgrind, when valgrind is enabled. See # valgrind(1) man page for details. "--vgdb=no" is specified because # pacemaker-execd can lower privileges when executing commands, which would # otherwise leave a bunch of unremovable files in /tmp. VALGRIND_OPTS="--leak-check=full --trace-children=no --vgdb=no --num-callers=25 --log-file=/var/lib/pacemaker/valgrind-%p --suppressions=/usr/share/pacemaker/tests/valgrind-pcmk.suppressions --gen-suppressions=all" diff --git a/extra/logrotate/pacemaker.in b/extra/logrotate/pacemaker.in index fb361c6f3d..ec6fc38e52 100644 --- a/extra/logrotate/pacemaker.in +++ b/extra/logrotate/pacemaker.in @@ -1,12 +1,12 @@ -@CRM_LOG_DIR@/pacemaker.log @CRM_BUNDLE_DIR@/*/pacemaker.log { +@CRM_LOG_DIR@/pacemaker.log @CRM_BUNDLE_DIR@/*/pacemaker.log @CRM_BUNDLE_DIR@/*/pcmk-init.log { compress dateext weekly su hacluster haclient rotate 99 maxage 365 maxsize 100M notifempty missingok copytruncate } diff --git a/lib/common/output.c b/lib/common/output.c index bd427fda3b..870c5ba06c 100644 --- a/lib/common/output.c +++ b/lib/common/output.c @@ -1,150 +1,150 @@ /* * Copyright 2019 the Pacemaker project contributors * * The version control history for this file may have further details. * * This source code is licensed under the GNU Lesser General Public License * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. */ #include #include #include #include #include static GHashTable *formatters = NULL; void pcmk__output_free(pcmk__output_t *out) { out->free_priv(out); g_hash_table_destroy(out->messages); free(out->request); free(out); } int pcmk__output_new(pcmk__output_t **out, const char *fmt_name, const char *filename, char **argv) { pcmk__output_factory_t create = NULL;; if (formatters == NULL) { return EINVAL; } /* If no name was given, just try "text". It's up to each tool to register * what it supports so this also may not be valid. */ if (fmt_name == NULL) { create = g_hash_table_lookup(formatters, "text"); } else { create = g_hash_table_lookup(formatters, fmt_name); } if (create == NULL) { return pcmk_err_unknown_format; } *out = create(argv); if (*out == NULL) { return ENOMEM; } if (filename == NULL || safe_str_eq(filename, "-")) { (*out)->dest = stdout; } else { (*out)->dest = fopen(filename, "w"); if ((*out)->dest == NULL) { return errno; } } (*out)->messages = g_hash_table_new_full(crm_str_hash, g_str_equal, free, NULL); if ((*out)->init(*out) == false) { pcmk__output_free(*out); return ENOMEM; } return 0; } int pcmk__register_format(GOptionContext *context, const char *name, pcmk__output_factory_t create, GOptionEntry *options) { if (create == NULL) { return -EINVAL; } if (formatters == NULL) { formatters = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL); } if (options != NULL && context != NULL) { char *group_name = crm_strdup_printf("output-%s", name); - char *group_desc = crm_strdup_printf("%s Output Options:", name); + char *group_desc = crm_strdup_printf("Output Options (%s):", name); char *group_help = crm_strdup_printf("Show %s output help", name); GOptionGroup *group = g_option_group_new(group_name, group_desc, group_help, NULL, NULL); g_option_group_add_entries(group, options); g_option_context_add_group(context, group); free(group_name); free(group_desc); free(group_help); } g_hash_table_insert(formatters, strdup(name), create); return 0; } void pcmk__register_formats(GOptionContext *context, pcmk__supported_format_t *formats) { pcmk__supported_format_t *entry = NULL; if (formats == NULL) { return; } for (entry = formats; entry->name != NULL; entry++) { pcmk__register_format(context, entry->name, entry->create, entry->options); } } int pcmk__call_message(pcmk__output_t *out, const char *message_id, ...) { va_list args; int rc = 0; pcmk__message_fn_t fn; fn = g_hash_table_lookup(out->messages, message_id); if (fn == NULL) { return -EINVAL; } va_start(args, message_id); rc = fn(out, args); va_end(args); return rc; } void pcmk__register_message(pcmk__output_t *out, const char *message_id, pcmk__message_fn_t fn) { g_hash_table_replace(out->messages, strdup(message_id), fn); } void pcmk__register_messages(pcmk__output_t *out, pcmk__message_entry_t *table) { pcmk__message_entry_t *entry; for (entry = table; entry->message_id != NULL; entry++) { if (safe_str_eq(out->fmt_name, entry->fmt_name)) { pcmk__register_message(out, entry->message_id, entry->fn); } } } diff --git a/tools/report.common.in b/tools/report.common.in index 73ec0dce40..4b39e2a27c 100644 --- a/tools/report.common.in +++ b/tools/report.common.in @@ -1,885 +1,885 @@ # # Originally based on hb_report # Copyright 2007 Dejan Muhamedagic # Later changes copyright 2010-2018 the Pacemaker project contributors # # The version control history for this file may have further details. # # This source code is licensed under the GNU General Public License version 2 # or later (GPLv2+) WITHOUT ANY WARRANTY. # host=`uname -n` shorthost=`echo $host | sed s:\\\\..*::` if [ -z $verbose ]; then verbose=0 fi # Target Files EVENTS_F=events.txt ANALYSIS_F=analysis.txt HALOG_F=cluster-log.txt BT_F=backtraces.txt SYSINFO_F=sysinfo.txt SYSSTATS_F=sysstats.txt DLM_DUMP_F=dlm_dump.txt CRM_MON_F=crm_mon.txt MEMBERSHIP_F=members.txt CRM_VERIFY_F=crm_verify.txt PERMISSIONS_F=permissions.txt CIB_F=cib.xml CIB_TXT_F=cib.txt DRBD_INFO_F=drbd_info.txt EVENT_PATTERNS=" state do_state_transition membership pcmk_peer_update.*(lost|memb): quorum (crmd|pacemaker-controld).*crm_update_quorum pause Process.pause.detected resources (lrmd|pacemaker-execd).*rsc:(start|stop) stonith te_fence_node|fenced.*(requests|(Succeeded|Failed).to.|result=) start_stop shutdown.decision|Corosync.Cluster.Engine|corosync.*Initializing.transport|Executive.Service.RELEASE|crm_shutdown:.Requesting.shutdown|pcmk_shutdown:.Shutdown.complete " # superset of all packages of interest on all distros # (the package manager will be used to validate the installation # of any of these packages that are installed) PACKAGES="pacemaker pacemaker-libs pacemaker-cluster-libs libpacemaker3 pacemaker-remote pacemaker-pygui pacemaker-pymgmt pymgmt-client corosync corosynclib libcorosync4 resource-agents cluster-glue-libs cluster-glue libglue2 ldirectord ocfs2-tools ocfs2-tools-o2cb ocfs2console ocfs2-kmp-default ocfs2-kmp-pae ocfs2-kmp-xen ocfs2-kmp-debug ocfs2-kmp-trace drbd drbd-kmp-xen drbd-kmp-pae drbd-kmp-default drbd-kmp-debug drbd-kmp-trace drbd-pacemaker drbd-utils drbd-bash-completion drbd-xen lvm2 lvm2-clvm cmirrord libdlm libdlm2 libdlm3 hawk ruby lighttpd kernel-default kernel-pae kernel-xen glibc " # Potential locations of system log files SYSLOGS=" /var/log/* /var/logs/* /var/syslog/* /var/adm/* /var/log/ha/* /var/log/cluster/* " # Whether pacemaker-remoted was found (0 = yes, 1 = no, -1 = haven't looked yet) REMOTED_STATUS=-1 # # keep the user posted # record() { if [ x != x"$REPORT_HOME" -a -d "${REPORT_HOME}/$shorthost" ]; then rec="${REPORT_HOME}/$shorthost/report.out" elif [ x != x"${l_base}" -a -d "${l_base}" ]; then rec="${l_base}/report.summary" else rec="/dev/null" fi printf "%-10s $*\n" "$shorthost:" 2>&1 >> "${rec}" } log() { printf "%-10s $*\n" "$shorthost:" 1>&2 record "$*" } debug() { if [ $verbose -gt 0 ]; then log "Debug: $*" else record "Debug: $*" fi } info() { log "$*" } warning() { log "WARN: $*" } fatal() { log "ERROR: $*" exit 1 } require_tar() { which tar >/dev/null 2>&1 if [ $? -ne 0 ]; then fatal "Required program 'tar' not found, please install and re-run" fi } # check if process of given substring in its name does exist; # only look for processes originated by user 0 (by UID), "@CRM_DAEMON_USER@" # or effective user running this script, and/or group 0 (by GID), # "@CRM_DAEMON_GROUP@" or one of the groups the effective user belongs to # (there's no business in probing any other processes) is_running() { ps -G "0 $(getent group '@CRM_DAEMON_GROUP@' 2>/dev/null | cut -d: -f3) $(id -G)" \ -u "0 @CRM_DAEMON_USER@ $(id -u)" -f \ | grep -Eqs $(echo "$1" | sed -e 's/^\(.\)/[\1]/') } has_remoted() { if [ $REMOTED_STATUS -eq -1 ]; then REMOTED_STATUS=1 if which pacemaker-remoted >/dev/null 2>&1; then REMOTED_STATUS=0 # Check for pre-2.0.0 daemon name in case we have mixed-version cluster elif which pacemaker_remoted >/dev/null 2>&1; then REMOTED_STATUS=0 elif [ -x "@sbindir@/pacemaker-remoted" ]; then REMOTED_STATUS=0 elif [ -x "@sbindir@/pacemaker_remoted" ]; then REMOTED_STATUS=0 else # @TODO: the binary might be elsewhere, # but a global search is too expensive for d in /{usr,opt}/{local/,}{s,}bin; do if [ -x "${d}/pacemaker-remoted" ]; then REMOTED_STATUS=0 elif [ -x "${d}/pacemaker_remoted" ]; then REMOTED_STATUS=0 fi done fi fi return $REMOTED_STATUS } # found_dir found_dir() { echo "$2" info "Pacemaker $1 found in: $2" } detect_daemon_dir() { info "Searching for where Pacemaker daemons live... this may take a while" for d in \ {/usr,/usr/local,/opt/local,@exec_prefix@}/{libexec,lib64,lib}/pacemaker do # pacemaker and pacemaker-cts packages can install to daemon directory, # so check for a file from each if [ -e $d/pacemaker-schedulerd ] || [ -e $d/cts-exec-helper ]; then found_dir "daemons" "$d" return fi done # Pacemaker Remote nodes don't need to install daemons if has_remoted; then info "Pacemaker daemons not found (this appears to be a Pacemaker Remote node)" return fi for f in $(find / -maxdepth $maxdepth -type f -name pacemaker-schedulerd -o -name cts-exec-helper); do d=$(dirname "$f") found_dir "daemons" "$d" return done fatal "Pacemaker daemons not found (nonstandard installation?)" } detect_cib_dir() { d="${local_state_dir}/lib/pacemaker/cib" if [ -f "$d/cib.xml" ]; then found_dir "config files" "$d" return fi # Pacemaker Remote nodes don't need a CIB if has_remoted; then info "Pacemaker config not found (this appears to be a Pacemaker Remote node)" return fi info "Searching for where Pacemaker keeps config information... this may take a while" # TODO: What about false positives where someone copied the CIB? for f in $(find / -maxdepth $maxdepth -type f -name cib.xml); do d=$(dirname $f) found_dir "config files" "$d" return done warning "Pacemaker config not found (nonstandard installation?)" } detect_state_dir() { if [ -n "$CRM_CONFIG_DIR" ]; then # Assume new layout # $local_state_dir/lib/pacemaker/(cib,pengine,blackbox,cores) dirname "$CRM_CONFIG_DIR" # Pacemaker Remote nodes might not have a CRM_CONFIG_DIR elif [ -d "$local_state_dir/lib/pacemaker" ]; then echo $local_state_dir/lib/pacemaker fi } detect_pe_dir() { config_root="$1" d="$config_root/pengine" if [ -d "$d" ]; then found_dir "scheduler inputs" "$d" return fi if has_remoted; then info "Pacemaker scheduler inputs not found (this appears to be a Pacemaker Remote node)" return fi info "Searching for where Pacemaker keeps scheduler inputs... this may take a while" for d in $(find / -maxdepth $maxdepth -type d -name pengine); do found_dir "scheduler inputs" "$d" return done fatal "Pacemaker scheduler inputs not found (nonstandard installation?)" } detect_host() { local_state_dir=@localstatedir@ if [ -d $local_state_dir/run ]; then CRM_STATE_DIR=$local_state_dir/run/crm else info "Searching for where Pacemaker keeps runtime data... this may take a while" for d in `find / -maxdepth $maxdepth -type d -name run`; do local_state_dir=`dirname $d` CRM_STATE_DIR=$d/crm break done info "Found: $CRM_STATE_DIR" fi debug "Machine runtime directory: $local_state_dir" debug "Pacemaker runtime data located in: $CRM_STATE_DIR" CRM_DAEMON_DIR=$(detect_daemon_dir) CRM_CONFIG_DIR=$(detect_cib_dir) config_root=$(detect_state_dir) # Older versions had none BLACKBOX_DIR=$config_root/blackbox debug "Pacemaker blackboxes (if any) located in: $BLACKBOX_DIR" PE_STATE_DIR=$(detect_pe_dir "$config_root") CRM_CORE_DIRS="" for d in $config_root/cores $local_state_dir/lib/corosync; do if [ -d $d ]; then CRM_CORE_DIRS="$CRM_CORE_DIRS $d" fi done debug "Core files located under: $CRM_CORE_DIRS" } time2str() { perl -e "use POSIX; print strftime('%x %X',localtime($1));" } get_time() { perl -e "\$time=\"$*\";" -e ' $unix_tm = 0; eval "use Date::Parse"; if (index($time, ":") < 0) { } elsif (!$@) { $unix_tm = str2time($time); } else { eval "use Date::Manip"; if (!$@) { $unix_tm = UnixDate(ParseDateString($time), "%s"); } } if ($unix_tm != "") { print int($unix_tm); } else { print ""; } ' } get_time_syslog() { awk '{print $1,$2,$3}' } get_time_legacy() { awk '{print $2}' | sed 's/_/ /' } get_time_iso8601() { awk '{print $1}' } get_time_format_for_string() { l="$*" t=$(get_time `echo $l | get_time_syslog`) if [ "x$t" != x ]; then echo syslog return fi t=$(get_time `echo $l | get_time_iso8601`) if [ "x$t" != x ]; then echo iso8601 return fi t=$(get_time `echo $l | get_time_legacy`) if [ "x$t" != x ]; then echo legacy return fi } get_time_format() { t=0 l="" func="" trycnt=10 while [ $trycnt -gt 0 ] && read l; do func=$(get_time_format_for_string $l) if [ "x$func" != x ]; then break fi trycnt=$(($trycnt-1)) done #debug "Logfile uses the $func time format" echo $func } get_time_from_line() { GTFL_FORMAT="$1" shift if [ "$GTFL_FORMAT" = "" ]; then GTFL_FORMAT=$(get_time_format_for_string "$@") fi case $GTFL_FORMAT in syslog|legacy|iso8601) get_time $(echo "$@" | get_time_${GTFL_FORMAT}) ;; *) warning "Unknown time format in: $@" ;; esac } get_first_time() { l="" format=$1 while read l; do ts=$(get_time_from_line "$format" "$l") if [ "x$ts" != x ]; then echo "$ts" return fi done } get_last_time() { l="" best=`date +%s` # Now format=$1 while read l; do ts=$(get_time_from_line "$format" "$l") if [ "x$ts" != x ]; then best=$ts fi done echo $best } linetime() { get_time_from_line "" $(tail -n +$2 $1 | grep -a ":[0-5][0-9]:" | head -n 1) } # # findmsg # # Print the names of up to system logs that contain , # ordered by most recently modified. # findmsg() { max=$1 pattern="$2" found=0 # List all potential system logs ordered by most recently modified. candidates=$(ls -1td $SYSLOGS 2>/dev/null) if [ -z "$candidates" ]; then debug "No system logs found to search for pattern \'$pattern\'" return fi # Portable way to handle files with spaces in their names. SAVE_IFS=$IFS IFS=" " # Check each log file for matches. logfiles="" for f in $candidates; do local cat="" # We only care about readable files with something in them. if [ ! -f "$f" ] || [ ! -r "$f" ] || [ ! -s "$f" ] ; then continue fi cat=$(find_decompressor "$f") # We want to avoid grepping through potentially huge binary logs such # as lastlog. However, control characters sometimes find their way into # text logs, so we use a heuristic of more than 256 nonprintable # characters in the file's first kilobyte. if [ $($cat "$f" 2>/dev/null | head -c 1024 | tr -d '[:print:][:space:]' | wc -c) -gt 256 ] then continue fi # Our patterns are ASCII, so we can use LC_ALL="C" to speed up grep $cat "$f" 2>/dev/null | LC_ALL="C" grep -q -e "$pattern" if [ $? -eq 0 ]; then # Add this file to the list of hits # (using newline as separator to handle spaces in names). if [ -z "$logfiles" ]; then logfiles="$f" else logfiles="$logfiles $f" fi # If we have enough hits, print them and return. found=$(($found+1)) if [ $found -ge $max ]; then break fi fi done 2>/dev/null IFS=$SAVE_IFS if [ -z "$logfiles" ]; then debug "Pattern \'$pattern\' not found in any system logs" else debug "Pattern \'$pattern\' found in: [ $logfiles ]" echo "$logfiles" fi } node_events() { if [ -e $1 ]; then Epatt=`echo "$EVENT_PATTERNS" | while read title p; do [ -n "$p" ] && echo -n "|$p"; done | sed 's/.//' ` grep -E "$Epatt" $1 fi } pickfirst() { for x; do which $x >/dev/null 2>&1 && { echo $x return 0 } done return 1 } shrink() { olddir=$PWD dir=`dirname $1` base=`basename $1` target=$1.tar tar_options="cf" variant=`pickfirst bzip2 gzip xz false` case $variant in bz*) tar_options="jcf" target="$target.bz2" ;; gz*) tar_options="zcf" target="$target.gz" ;; xz*) tar_options="Jcf" target="$target.xz" ;; *) warning "Could not find a compression program, the resulting tarball may be huge" ;; esac if [ -e $target ]; then fatal "Destination $target already exists, specify an alternate name with --dest" fi cd $dir >/dev/null 2>&1 tar $tar_options $target $base >/dev/null 2>&1 if [ $? -ne 0 ]; then fatal "Could not archive $base, please investigate and collect manually" fi cd $olddir >/dev/null 2>&1 echo $target } findln_by_time() { local logf=$1 local tm=$2 local first=1 # Some logs can be massive (over 1,500,000,000 lines have been seen in the wild) # Even just 'wc -l' on these files can take 10+ minutes local fileSize=`ls -lh "$logf" | awk '{ print $5 }' | grep -ie G` if [ x$fileSize != x ]; then warning "$logf is ${fileSize} in size and could take many hours to process. Skipping." return fi local last=`wc -l < $logf` while [ $first -le $last ]; do mid=$((($last+$first)/2)) trycnt=10 while [ $trycnt -gt 0 ]; do tmid=`linetime $logf $mid` [ "$tmid" ] && break warning "cannot extract time: $logf:$mid; will try the next one" trycnt=$(($trycnt-1)) # shift the whole first-last segment first=$(($first-1)) last=$(($last-1)) mid=$((($last+$first)/2)) done if [ -z "$tmid" ]; then warning "giving up on log..." return fi if [ $tmid -gt $tm ]; then last=$(($mid-1)) elif [ $tmid -lt $tm ]; then first=$(($mid+1)) else break fi done echo $mid } dumplog() { local logf=$1 local from_line=$2 local to_line=$3 [ "$from_line" ] || return tail -n +$from_line $logf | if [ "$to_line" ]; then head -$(($to_line-$from_line+1)) else cat fi } # # find log/set of logs which are interesting for us # # # find log slices # find_decompressor() { case $1 in *bz2) echo "bzip2 -dc" ;; *gz) echo "gzip -dc" ;; *xz) echo "xz -dc" ;; *) echo "cat" ;; esac } # # check if the log contains a piece of our segment # is_our_log() { local logf=$1 local from_time=$2 local to_time=$3 local cat=`find_decompressor $logf` local format=`$cat $logf | get_time_format` local first_time=`$cat $logf | head -10 | get_first_time $format` - local last_time=`$cat $logf | tail -1 | get_last_time $format` + local last_time=`$cat $logf | tail -10 | get_last_time $format` if [ x = "x$first_time" -o x = "x$last_time" ]; then warning "Skipping bad logfile '$1': Could not determine log dates" return 0 # skip (empty log?) fi if [ $from_time -gt $last_time ]; then # we shouldn't get here anyway if the logs are in order return 2 # we're past good logs; exit fi if [ $from_time -ge $first_time ]; then return 3 # this is the last good log fi # have to go further back if [ x = "x$to_time" -o $to_time -ge $first_time ]; then return 1 # include this log else return 0 # don't include this log fi } # # go through archived logs (timewise backwards) and see if there # are lines belonging to us # (we rely on untouched log files, i.e. that modify time # hasn't been changed) # arch_logs() { local logf=$1 local from_time=$2 local to_time=$3 # look for files such as: ha-log-20090308 or # ha-log-20090308.gz (.bz2) or ha-log.0, etc ls -t $logf $logf*[0-9z] 2>/dev/null | while read next_log; do is_our_log $next_log $from_time $to_time case $? in 0) ;; # noop, continue 1) echo $next_log # include log and continue debug "Found log $next_log" ;; 2) break;; # don't go through older logs! 3) echo $next_log # include log and continue debug "Found log $next_log" break ;; # don't go through older logs! esac done } # # print part of the log # drop_tmp_file() { [ -z "$tmp" ] || rm -f "$tmp" } print_logseg() { local logf=$1 local from_time=$2 local to_time=$3 # uncompress to a temp file (if necessary) local cat=`find_decompressor $logf` if [ "$cat" != "cat" ]; then tmp=`mktemp` $cat $logf > $tmp trap drop_tmp_file 0 sourcef=$tmp else sourcef=$logf tmp="" fi if [ "$from_time" = 0 ]; then FROM_LINE=1 else FROM_LINE=`findln_by_time $sourcef $from_time` fi if [ -z "$FROM_LINE" ]; then warning "couldn't find line for time $from_time; corrupt log file?" return fi TO_LINE="" if [ "$to_time" != 0 ]; then TO_LINE=`findln_by_time $sourcef $to_time` if [ -z "$TO_LINE" ]; then warning "couldn't find line for time $to_time; corrupt log file?" return fi if [ $FROM_LINE -lt $TO_LINE ]; then dumplog $sourcef $FROM_LINE $TO_LINE log "Including segment [$FROM_LINE-$TO_LINE] from $logf" else debug "Empty segment [$FROM_LINE-$TO_LINE] from $logf" fi else dumplog $sourcef $FROM_LINE $TO_LINE log "Including all logs after line $FROM_LINE from $logf" fi drop_tmp_file trap "" 0 } # # find log/set of logs which are interesting for us # dumplogset() { local logf=$1 local from_time=$2 local to_time=$3 local logf_set=`arch_logs $logf $from_time $to_time` if [ x = "x$logf_set" ]; then return fi local num_logs=`echo "$logf_set" | wc -l` local oldest=`echo $logf_set | awk '{print $NF}'` local newest=`echo $logf_set | awk '{print $1}'` local mid_logfiles=`echo $logf_set | awk '{for(i=NF-1; i>1; i--) print $i}'` # the first logfile: from $from_time to $to_time (or end) # logfiles in the middle: all # the last logfile: from beginning to $to_time (or end) case $num_logs in 1) print_logseg $newest $from_time $to_time;; *) print_logseg $oldest $from_time 0 for f in $mid_logfiles; do `find_decompressor $f` $f debug "including complete $f logfile" done print_logseg $newest 0 $to_time ;; esac } # cut out a stanza getstanza() { awk -v name="$1" ' !in_stanza && NF==2 && /^[a-z][a-z]*[[:space:]]*{/ { # stanza start if ($1 == name) in_stanza = 1 } in_stanza { print } in_stanza && NF==1 && $1 == "}" { exit } ' } # supply stanza in $1 and variable name in $2 # (stanza is optional) getcfvar() { cf_type=$1; shift; cf_var=$1; shift; cf_file=$* [ -f "$cf_file" ] || return case $cf_type in corosync) sed 's/#.*//' < $cf_file | if [ $# -eq 2 ]; then getstanza "$cf_var" shift 1 else cat fi | awk -v varname="$cf_var" ' NF==2 && match($1,varname":$")==1 { print $2; exit; } ' ;; esac } pickfirst() { for x; do which $x >/dev/null 2>&1 && { echo $x return 0 } done return 1 } # # figure out the cluster type, depending on the process list # and existence of configuration files # get_cluster_type() { if is_running corosync; then tool=`pickfirst corosync-objctl corosync-cmapctl` case $tool in *objctl) quorum=`$tool -a | grep quorum.provider | sed 's/.*=\s*//'`;; *cmapctl) quorum=`$tool | grep quorum.provider | sed 's/.*=\s*//'`;; esac stack="corosync" # Now we're guessing... # TODO: Technically these could be anywhere :-/ elif [ -f /etc/corosync/corosync.conf ]; then stack="corosync" else # We still don't know. This might be a Pacemaker Remote node, # or the configuration might be in a nonstandard location. stack="any" fi debug "Detected the '$stack' cluster stack" echo $stack } find_cluster_cf() { case $1 in corosync) best_size=0 best_file="" # TODO: Technically these could be anywhere :-/ for cf in /etc/corosync/corosync.conf; do if [ -f $cf ]; then size=`wc -l $cf | awk '{print $1}'` if [ $size -gt $best_size ]; then best_size=$size best_file=$cf fi fi done if [ -z "$best_file" ]; then debug "Looking for corosync configuration file. This may take a while..." for f in `find / -maxdepth $maxdepth -type f -name corosync.conf`; do best_file=$f break done fi debug "Located corosync config file: $best_file" echo "$best_file" ;; any) # Cluster type is undetermined. Don't complain, because this # might be a Pacemaker Remote node. ;; *) warning "Unknown cluster type: $1" ;; esac } # # check for the major prereq for a) parameter parsing and b) # parsing logs # t=`get_time "12:00"` if [ "$t" = "" ]; then fatal "please install the perl Date::Parse module (perl-DateTime-Format-DateParse on Fedora/Red Hat)" fi # vim: set expandtab tabstop=8 softtabstop=4 shiftwidth=4 textwidth=80: