diff --git a/mcp/Makefile.am b/mcp/Makefile.am new file mode 100644 index 0000000000..badb656527 --- /dev/null +++ b/mcp/Makefile.am @@ -0,0 +1,48 @@ +# +# Copyright (C) 2004-2009 Andrew Beekhof +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +MAINTAINERCLEANFILES = Makefile.in + +INCLUDES = -I$(top_builddir)/include -I$(top_srcdir)/include \ + -I$(top_builddir)/libltdl -I$(top_srcdir)/libltdl + +sbin_PROGRAMS = pacemaker + +if BUILD_HELP +man8_MANS = $(sbin_PROGRAMS:%=%.8) +endif + +## SOURCES + +noinst_HEADERS = + +pacemaker_SOURCES = pacemaker.c corosync.c +pacemaker_LDADD = $(CLUSTERLIBS) $(top_builddir)/lib/common/libcrmcommon.la -lcfg -lconfdb + +%.8: % + echo Creating $@ + chmod a+x $(top_builddir)/mcp/$< + help2man --output $@ --no-info --section 8 --name "Part of the Pacemaker cluster resource manager" $(top_builddir)/mcp/$< + +clean-generic: + rm -f *.log *.debug *.xml *~ + +install-exec-local: + +uninstall-local: + +.PHONY: install-exec-hook diff --git a/mcp/corosync.c b/mcp/corosync.c new file mode 100644 index 0000000000..b8ae057866 --- /dev/null +++ b/mcp/corosync.c @@ -0,0 +1,565 @@ +/* + * Copyright (C) 2010 Andrew Beekhof + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +#include +#include + +#include +#include +#include + +static struct cpg_name cpg_group = { + .length = 0, + .value[0] = 0, +}; + +static cpg_handle_t cpg_handle; +static corosync_cfg_handle_t cfg_handle; +static corosync_cfg_state_notification_t cfg_buffer; + +static inline const char *ais_error2text(int error) +{ + const char *text = "unknown"; + switch(error) { + case CS_OK: + text = "None"; + break; + case CS_ERR_LIBRARY: + text = "Library error"; + break; + case CS_ERR_VERSION: + text = "Version error"; + break; + case CS_ERR_INIT: + text = "Initialization error"; + break; + case CS_ERR_TIMEOUT: + text = "Timeout"; + break; + case CS_ERR_TRY_AGAIN: + text = "Try again"; + break; + case CS_ERR_INVALID_PARAM: + text = "Invalid parameter"; + break; + case CS_ERR_NO_MEMORY: + text = "No memory"; + break; + case CS_ERR_BAD_HANDLE: + text = "Bad handle"; + break; + case CS_ERR_BUSY: + text = "Busy"; + break; + case CS_ERR_ACCESS: + text = "Access error"; + break; + case CS_ERR_NOT_EXIST: + text = "Doesn't exist"; + break; + case CS_ERR_NAME_TOO_LONG: + text = "Name too long"; + break; + case CS_ERR_EXIST: + text = "Exists"; + break; + case CS_ERR_NO_SPACE: + text = "No space"; + break; + case CS_ERR_INTERRUPT: + text = "Interrupt"; + break; + case CS_ERR_NAME_NOT_FOUND: + text = "Name not found"; + break; + case CS_ERR_NO_RESOURCES: + text = "No resources"; + break; + case CS_ERR_NOT_SUPPORTED: + text = "Not supported"; + break; + case CS_ERR_BAD_OPERATION: + text = "Bad operation"; + break; + case CS_ERR_FAILED_OPERATION: + text = "Failed operation"; + break; + case CS_ERR_MESSAGE_ERROR: + text = "Message error"; + break; + case CS_ERR_QUEUE_FULL: + text = "Queue full"; + break; + case CS_ERR_QUEUE_NOT_AVAILABLE: + text = "Queue not available"; + break; + case CS_ERR_BAD_FLAGS: + text = "Bad flags"; + break; + case CS_ERR_TOO_BIG: + text = "To big"; + break; + case CS_ERR_NO_SECTIONS: + text = "No sections"; + break; + } + return text; +} + +/* =::=::=::= CFG - Shutdown stuff =::=::=::= */ + +static void cfg_shutdown_callback(corosync_cfg_handle_t h, + corosync_cfg_shutdown_flags_t flags) +{ + if (flags & COROSYNC_CFG_SHUTDOWN_FLAG_REQUEST) { + /* Never allow corosync to shut down while we're running */ + corosync_cfg_replyto_shutdown(h, COROSYNC_CFG_SHUTDOWN_FLAG_NO); + } +} + +static corosync_cfg_callbacks_t cfg_callbacks = +{ + .corosync_cfg_shutdown_callback = cfg_shutdown_callback, + .corosync_cfg_state_track_callback = NULL, +}; + +static gboolean pcmk_cfg_dispatch(int sender, gpointer user_data) +{ + corosync_cfg_handle_t *handle = (corosync_cfg_handle_t*)user_data; + cs_error_t rc = corosync_cfg_dispatch(*handle, CS_DISPATCH_ALL); + if (rc != CS_OK) { + return FALSE; + } + return TRUE; +} + +static void +cfg_connection_destroy(gpointer user_data) +{ + crm_err("Connection destroyed"); + cfg_handle = 0; + return; +} + +gboolean cluster_disconnect_cfg(void) +{ + if(cfg_handle) { + corosync_cfg_finalize(cfg_handle); + cfg_handle = 0; + } + return TRUE; +} + +gboolean cluster_connect_cfg(uint32_t *nodeid) +{ + cs_error_t rc; + int fd; + + rc = corosync_cfg_initialize(&cfg_handle, &cfg_callbacks); + if (rc != CS_OK) { + crm_err("corosync cfg init error %d", rc); + return FALSE; + } + + rc = corosync_cfg_fd_get(cfg_handle, &fd); + if (rc != CS_OK) { + crm_err("corosync cfg fd_get error %d", rc); + goto bail; + } + + rc = corosync_cfg_local_get(cfg_handle, nodeid); + if (rc != CS_OK) { + crm_err("corosync cfg local_get error %d", rc); + goto bail; + } + + crm_debug("Our nodeid: %d", *nodeid); + + rc = corosync_cfg_state_track (cfg_handle, 0, &cfg_buffer); + if (rc != CS_OK) { + crm_err("corosync cfg stack_track error %d", rc); + goto bail; + } + + crm_debug("Adding fd=%d to mainloop", fd); + G_main_add_fd( + G_PRIORITY_HIGH, fd, FALSE, pcmk_cfg_dispatch, &cfg_handle, cfg_connection_destroy); + + return TRUE; + + bail: + corosync_cfg_finalize(cfg_handle); + return FALSE; +} + +/* =::=::=::= CPG - Closed Process Group Messaging =::=::=::= */ + +static gboolean pcmk_cpg_dispatch(int sender, gpointer user_data) +{ + cpg_handle_t *handle = (cpg_handle_t*)user_data; + cs_error_t rc = cpg_dispatch(*handle, CS_DISPATCH_ALL); + if (rc != CS_OK) { + return FALSE; + } + return TRUE; +} + +static void +cpg_connection_destroy(gpointer user_data) +{ + crm_err("Connection destroyed"); + cpg_handle = 0; + return; +} + +static void pcmk_cpg_deliver ( + cpg_handle_t handle, + const struct cpg_name *groupName, + uint32_t nodeid, + uint32_t pid, + void *msg, + size_t msg_len) +{ + if(nodeid != local_nodeid) { + uint32_t procs = 0; + xmlNode *xml = string2xml(msg); + const char *uname = crm_element_value(xml, "uname"); + + crm_element_value_int(xml, "proclist", (int*)&procs); + /* crm_debug("Got proclist %.32x from %s", procs, uname); */ + if(update_node_processes(nodeid, uname, procs)) { + update_process_clients(); + } + } +} + +static void pcmk_cpg_membership( + cpg_handle_t handle, + const struct cpg_name *groupName, + const struct cpg_address *member_list, size_t member_list_entries, + const struct cpg_address *left_list, size_t left_list_entries, + const struct cpg_address *joined_list, size_t joined_list_entries) +{ + /* Don't care about CPG membership */ +} + +cpg_callbacks_t cpg_callbacks = { + .cpg_deliver_fn = pcmk_cpg_deliver, + .cpg_confchg_fn = pcmk_cpg_membership, +}; + +gboolean cluster_disconnect_cpg(void) +{ + if(cpg_handle) { + cpg_finalize(cpg_handle); + cpg_handle = 0; + } + return TRUE; +} + +gboolean cluster_connect_cpg(void) +{ + cs_error_t rc; + unsigned int nodeid; + int fd; + int retries = 0; + + strcpy(cpg_group.value, "pcmk"); + cpg_group.length = strlen(cpg_group.value)+1; + + rc = cpg_initialize(&cpg_handle, &cpg_callbacks); + if (rc != CS_OK) { + crm_err("corosync cpg init error %d", rc); + return FALSE; + } + + rc = cpg_fd_get(cpg_handle, &fd); + if (rc != CS_OK) { + crm_err("corosync cpg fd_get error %d", rc); + goto bail; + } + + rc = cpg_local_get(cpg_handle, &nodeid); + if (rc != CS_OK) { + crm_err("corosync cpg local_get error %d", rc); + goto bail; + } + + crm_debug("Our nodeid: %d", nodeid); + + do { + rc = cpg_join(cpg_handle, &cpg_group); + if(rc == CS_ERR_TRY_AGAIN) { + retries++; + sleep(retries); + } + + } while(rc == CS_ERR_TRY_AGAIN && retries < 30); + + if (rc != CS_OK) { + crm_err("Could not join the CPG group '%s': %d", crm_system_name, rc); + goto bail; + } + + crm_debug("Adding fd=%d to mainloop", fd); + G_main_add_fd( + G_PRIORITY_HIGH, fd, FALSE, pcmk_cpg_dispatch, &cpg_handle, cpg_connection_destroy); + + return TRUE; + + bail: + cpg_finalize(cpg_handle); + return FALSE; +} + +gboolean send_cpg_message(struct iovec *iov) +{ + int rc = CS_OK; + int retries = 0; + + errno = 0; + do { + rc = cpg_mcast_joined(cpg_handle, CPG_TYPE_AGREED, iov, 1); + if(rc == CS_ERR_TRY_AGAIN) { + retries++; + sleep(retries); + } + + } while(rc == CS_ERR_TRY_AGAIN && retries < 30); + + if(rc != CS_OK) { + crm_perror(LOG_ERR,"Sending message via cpg FAILED: (rc=%d) %s", + rc, ais_error2text(rc)); + } + + return (rc == CS_OK); +} + + +/* =::=::=::= Configuration =::=::=::= */ + +static int get_config_opt( + confdb_handle_t config, + hdb_handle_t object_handle, + const char *key, char **value, const char *fallback) +{ + size_t len = 0; + char *env_key = NULL; + const char *env_value = NULL; + char buffer[256]; + + if(*value) { + crm_free(*value); + *value = NULL; + } + + if(object_handle > 0) { + if(CS_OK == confdb_key_get(config, object_handle, key, strlen(key), &buffer, &len)) { + *value = crm_strdup(buffer); + } + } + + if (*value) { + crm_info("Found '%s' for option: %s", *value, key); + return 0; + } + + env_key = crm_concat("HA", key, '_'); + env_value = getenv(env_key); + crm_free(env_key); + + if (*value) { + crm_info("Found '%s' in ENV for option: %s", *value, key); + *value = crm_strdup(env_value); + return 0; + } + + if(fallback) { + crm_info("Defaulting to '%s' for option: %s", fallback, key); + *value = crm_strdup(fallback); + + } else { + crm_info("No default for option: %s", key); + } + + return -1; +} + +static confdb_handle_t config_find_init(confdb_handle_t config) +{ + cs_error_t rc = CS_OK; + confdb_handle_t local_handle = OBJECT_PARENT_HANDLE; + + rc = confdb_object_find_start(config, local_handle); + if(rc == CS_OK) { + return local_handle; + } else { + crm_err("Couldn't create search context: %d", rc); + } + return 0; +} + +static hdb_handle_t config_find_next( + confdb_handle_t config, const char *name, confdb_handle_t top_handle) +{ + cs_error_t rc = CS_OK; + hdb_handle_t local_handle = 0; + + if(top_handle == 0) { + crm_err("Couldn't search for %s: no valid context", name); + return 0; + } + + crm_info("Searching for %s in "HDB_X_FORMAT, name, top_handle); + rc = confdb_object_find(config, top_handle, name, strlen(name), &local_handle); + if(rc != CS_OK) { + crm_info("No additional configuration supplied for: %s", name); + local_handle = 0; + } else { + crm_info("Processing additional %s options...", name); + } + return local_handle; +} + +gboolean read_config(void) +{ + confdb_handle_t config; + + int rc; + char *value = NULL; + gboolean have_log = FALSE; + gboolean have_quorum = FALSE; + confdb_handle_t top_handle = 0; + hdb_handle_t local_handle = 0; + static confdb_callbacks_t callbacks = {}; + + + rc = confdb_initialize (&config, &callbacks); + if (rc != CS_OK) { + printf ("Could not initialize Cluster Configuration Database API instance error %d\n", rc); + return FALSE; + } + + crm_info("Reading configure"); + + /* =::=::= Defaults =::=::= */ + setenv("HA_COMPRESSION", "bz2", 1); + setenv("HA_cluster_type", "openais", 1); + setenv("HA_debug", "0", 1); + setenv("HA_logfacility", "daemon", 1); + setenv("HA_LOGFACILITY", "daemon", 1); + setenv("HA_use_logd", "off", 1); + setenv("HA_quorum_type", "pcmk", 1); + + /* =::=::= Should we be here =::=::= */ + + top_handle = config_find_init(config); + local_handle = config_find_next(config, "service", top_handle); + while(local_handle) { + get_config_opt(config, local_handle, "name", &value, NULL); + if(safe_str_eq("pacemaker", value)) { + crm_err("Pacemaker is already loaded as a plugin"); + crm_free(value); + exit(100); + } + local_handle = config_find_next(config, "service", top_handle); + } + + /* =::=::= Logging =::=::= */ + + top_handle = config_find_init(config); + local_handle = config_find_next(config, "logging", top_handle); + + get_config_opt(config, local_handle, "debug", &value, "on"); + if(crm_is_true(value) && crm_log_level < LOG_DEBUG) { + crm_log_level = LOG_DEBUG; + } + + if(crm_log_level >= LOG_DEBUG) { + char *level = crm_itoa(crm_log_level - LOG_INFO); + setenv("HA_debug", level, 1); + crm_free(level); + } + + get_config_opt(config, local_handle, "to_logfile", &value, "off"); + if(crm_is_true(value)) { + get_config_opt(config, local_handle, "logfile", &value, NULL); + + if(value == NULL) { + crm_err("Logging to a file requested but no log file specified"); + + } else { + uid_t pcmk_uid = geteuid(); + uid_t pcmk_gid = getegid(); + + FILE *logfile = fopen(value, "a"); + if(logfile) { + int logfd = fileno(logfile); + + setenv("HA_debugfile", value, 1); + + /* Ensure the file has the correct permissions */ + fchown(logfd, pcmk_uid, pcmk_gid); + fchmod(logfd, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP); + + fprintf(logfile, "Set r/w permissions for uid=%d, gid=%d on %s\n", + pcmk_uid, pcmk_gid, value); + fflush(logfile); + fsync(logfd); + fclose(logfile); + have_log = TRUE; + + } else { + crm_err("Couldn't create logfile: %s", value); + } + } + } + + get_config_opt(config, local_handle, "to_syslog", &value, "on"); + if(have_log && crm_is_true(value) == FALSE) { + crm_info("User configured file based logging and explicitly disabled syslog."); + value = NULL; + + } else { + if(crm_is_true(value) == FALSE) { + crm_err("Please enable some sort of logging, either 'to_file: on' or 'to_syslog: on'."); + crm_err("If you use file logging, be sure to also define a value for 'logfile'"); + } + get_config_opt(config, local_handle, "syslog_facility", &value, "daemon"); + } + + setenv("HA_logfacility", value?value:"none", 1); + setenv("HA_LOGFACILITY", value?value:"none", 1); + + /* =::=::= Quorum =::=::= */ + + top_handle = config_find_init(config); + local_handle = config_find_next(config, "quorum", top_handle); + get_config_opt(config, local_handle, "provider", &value, NULL); + if(value) { + have_quorum = TRUE; + if(safe_str_eq("quorum_cman", value)) { + setenv("HA_quorum_type", "cman", 1); + } else { + setenv("HA_quorum_type", "corosync", 1); + } + } + + confdb_finalize (config); + crm_free(value); + return TRUE; +} diff --git a/mcp/pacemaker.c b/mcp/pacemaker.c new file mode 100644 index 0000000000..8eef566f53 --- /dev/null +++ b/mcp/pacemaker.c @@ -0,0 +1,749 @@ +/* + * Copyright (C) 2010 Andrew Beekhof + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include +#include + +#include +#include + +#include + + +GMainLoop *mainloop = NULL; +GHashTable *client_list = NULL; +GHashTable *peers = NULL; + +char ipc_name[] = "pcmk"; +uint32_t local_nodeid = 0; +crm_trigger_t *shutdown_trigger = NULL; +const char *pid_file = "/var/run/pacemaker.pid"; + +enum crm_proc_flag { + crm_proc_none = 0x00000001, + crm_proc_ais = 0x00000002, + crm_proc_lrmd = 0x00000010, + crm_proc_cib = 0x00000100, + crm_proc_crmd = 0x00000200, + crm_proc_attrd = 0x00001000, + crm_proc_stonithd = 0x00002000, + crm_proc_pe = 0x00010000, + crm_proc_te = 0x00020000, + crm_proc_mgmtd = 0x00040000, + crm_proc_stonith_ng = 0x00100000, +}; + +/* order here matters - its used to index into the crm_children array */ +enum crm_ais_msg_types { + crm_msg_none = 0, + crm_msg_ais = 1, + crm_msg_lrmd = 2, + crm_msg_cib = 3, + crm_msg_crmd = 4, + crm_msg_attrd = 5, + crm_msg_stonithd = 6, + crm_msg_te = 7, + crm_msg_pe = 8, + crm_msg_stonith_ng = 9, +}; + +typedef struct pcmk_child_s { + int pid; + long flag; + long flags; + int start_seq; + int respawn_count; + gboolean respawn; + const char *name; + const char *uid; + const char *command; + +} pcmk_child_t; + +static pcmk_child_t pcmk_children[] = { + { 0, crm_proc_none, crm_flag_none, 0, 0, FALSE, "none", NULL, NULL }, + { 0, crm_proc_ais, crm_flag_none, 0, 0, FALSE, "ais", NULL, NULL }, + { 0, crm_proc_lrmd, crm_flag_none, 3, 0, TRUE, "lrmd", NULL, HB_DAEMON_DIR"/lrmd" }, + { 0, crm_proc_cib, crm_flag_members, 2, 0, TRUE, "cib", CRM_DAEMON_USER, CRM_DAEMON_DIR"/cib" }, + { 0, crm_proc_crmd, crm_flag_members, 6, 0, TRUE, "crmd", CRM_DAEMON_USER, CRM_DAEMON_DIR"/crmd" }, + { 0, crm_proc_attrd, crm_flag_none, 4, 0, TRUE, "attrd", CRM_DAEMON_USER, CRM_DAEMON_DIR"/attrd" }, + { 0, crm_proc_stonithd, crm_flag_none, 0, 0, TRUE, "stonithd", NULL, "/bin/false" }, + { 0, crm_proc_pe, crm_flag_none, 5, 0, TRUE, "pengine", CRM_DAEMON_USER, CRM_DAEMON_DIR"/pengine" }, + { 0, crm_proc_mgmtd, crm_flag_none, 7, 0, TRUE, "mgmtd", NULL, HB_DAEMON_DIR"/mgmtd" }, + { 0, crm_proc_stonith_ng, crm_flag_none, 1, 0, TRUE, "stonith-ng", NULL, CRM_DAEMON_DIR"/stonithd" }, +}; + +static gboolean start_child(pcmk_child_t *child); + +static uint32_t get_process_list(void) +{ + int lpc = 0; + uint32_t procs = crm_proc_ais; + for (lpc = 0; lpc < SIZEOF(pcmk_children); lpc++) { + if(pcmk_children[lpc].pid != 0) { + procs |= pcmk_children[lpc].flag; + } + } + return procs; +} + +static int pcmk_user_lookup(const char *name, uid_t *uid, gid_t *gid) +{ + int rc = -1; + char *buffer = NULL; + struct passwd pwd; + struct passwd *pwentry = NULL; + + crm_malloc0(buffer, PW_BUFFER_LEN); + getpwnam_r(name, &pwd, buffer, PW_BUFFER_LEN, &pwentry); + if(pwentry) { + rc = 0; + if(uid) { *uid = pwentry->pw_uid; } + if(gid) { *gid = pwentry->pw_gid; } + crm_debug("Cluster user %s has uid=%d gid=%d", + name, pwentry->pw_uid, pwentry->pw_gid); + + } else { + crm_err("Cluster user %s does not exist", name); + } + + crm_free(buffer); + return rc; +} + +static void +pcmk_child_exit( + ProcTrack* p, int status, int signo, int exitcode, int waslogged) +{ + pcmk_child_t *child = p->privatedata; + p->privatedata = NULL; + + crm_notice("Process %s [%d] exited (signal=%d, exitcode=%d)", + child->name, child->pid, signo, exitcode); + + child->pid = 0; + if(exitcode == 100) { + crm_notice("Child process %s no longer wishes" + " to be respawned", child->name); + child->respawn = FALSE; + } + + child->respawn_count += 1; + if(child->respawn_count > MAX_RESPAWN) { + crm_err("Child respawn count exceeded by %s", child->name); + child->respawn = FALSE; + } + + if(shutdown_trigger) { + mainloop_set_trigger(shutdown_trigger); + update_node_processes(local_nodeid, NULL, get_process_list()); + + } else if(child->respawn) { + crm_notice("Respawning failed child process: %s", child->name); + start_child(child); + } +} + +static void +pcmkManagedChildRegistered(ProcTrack* p) +{ + pcmk_child_t *child = p->privatedata; + child->pid = p->pid; +} + +static const char * +pcmkManagedChildName(ProcTrack* p) +{ + pcmk_child_t *child = p->privatedata; + return child->name; +} + +static ProcTrack_ops pcmk_managed_child_ops = { + pcmk_child_exit, + pcmkManagedChildRegistered, + pcmkManagedChildName +}; + +static gboolean +stop_child(pcmk_child_t *child, int signal) +{ + if(signal == 0) { + signal = SIGTERM; + } + + if(child->command == NULL) { + crm_debug("Nothing to do for child \"%s\"", child->name); + return TRUE; + } + + crm_info("Stopping CRM child \"%s\"", child->name); + + if (child->pid <= 0) { + crm_debug_2("Client %s not running", child->name); + return TRUE; + } + + errno = 0; + if(kill(child->pid, signal) == 0) { + crm_notice("Sent -%d to %s: [%d]", signal, child->name, child->pid); + + } else { + crm_perror(LOG_ERR, "Sent -%d to %s: [%d]", signal, child->name, child->pid); + } + + return TRUE; +} + +static char *opts_default[] = { NULL, NULL }; +static char *opts_vgrind[] = { NULL, NULL, NULL }; + +static gboolean +start_child(pcmk_child_t *child) +{ + int lpc = 0; + uid_t uid = 0; + struct rlimit oflimits; + gboolean use_valgrind = FALSE; + const char *devnull = "/dev/null"; + const char *env_valgrind = getenv("HA_VALGRIND_ENABLED"); + + if(child->command == NULL) { + crm_info("Nothing to do for child \"%s\"", child->name); + return TRUE; + } + + if(env_valgrind == NULL) { + use_valgrind = FALSE; + + } else if(crm_is_true(env_valgrind)) { + use_valgrind = TRUE; + + } else if(strstr(env_valgrind, child->name)) { + use_valgrind = TRUE; + } + + if(use_valgrind && strlen(VALGRIND_BIN) == 0) { + crm_warn("Cannot enable valgrind for %s:" + " The location of the valgrind binary is unknown", child->name); + use_valgrind = FALSE; + } + + child->pid = fork(); + CRM_ASSERT(child->pid != -1); + + if(child->pid > 0) { + /* parent */ + NewTrackedProc(child->pid, 0, PT_LOGNORMAL, child, &pcmk_managed_child_ops); + crm_info("Forked child %d for process %s%s", child->pid, child->name, + use_valgrind?" (valgrind enabled: "VALGRIND_BIN")":""); + update_node_processes(local_nodeid, NULL, get_process_list()); + return TRUE; + + } else { + /* Start a new session */ + (void)setsid(); + + /* Setup the two alternate arg arrarys */ + opts_vgrind[0] = crm_strdup(VALGRIND_BIN); + opts_vgrind[1] = crm_strdup(child->command); + opts_default[0] = opts_vgrind[1]; + +#if 0 + /* Dont set the group for now - it prevents connection to the cluster */ + if(gid && setgid(gid) < 0) { + crm_perror("Could not set group to %d", gid); + } +#endif + + if(child->uid) { + if(pcmk_user_lookup(child->uid, &uid, NULL) < 0) { + crm_err("Invalid uid (%s) specified for %s", + child->uid, child->name); + return TRUE; + } + } + + if(uid && setuid(uid) < 0) { + crm_perror(LOG_ERR, "Could not set user to %d (%s)", uid, child->uid); + } + + /* Close all open file descriptors */ + getrlimit(RLIMIT_NOFILE, &oflimits); + for (lpc = 0; lpc < oflimits.rlim_cur; lpc++) { + close(lpc); + } + + (void)open(devnull, O_RDONLY); /* Stdin: fd 0 */ + (void)open(devnull, O_WRONLY); /* Stdout: fd 1 */ + (void)open(devnull, O_WRONLY); /* Stderr: fd 2 */ + + setenv("HA_COMPRESSION", "bz2", 1); + setenv("HA_cluster_type", "openais", 1); +/* + setenv("HA_debug", pcmk_env.debug, 1); + setenv("HA_logfacility", pcmk_env.syslog, 1); + setenv("HA_LOGFACILITY", pcmk_env.syslog, 1); + setenv("HA_use_logd", pcmk_env.use_logd, 1); + setenv("HA_quorum_type", pcmk_env.quorum, 1); + if(pcmk_env.logfile) { + setenv("HA_debugfile", pcmk_env.logfile, 1); + } +*/ + + if(use_valgrind) { + (void)execvp(VALGRIND_BIN, opts_vgrind); + } else { + (void)execvp(child->command, opts_default); + } + crm_perror(LOG_ERR, "FATAL: Cannot exec %s", child->command); + exit(100); + } + return TRUE; /* never reached */ +} + +static gboolean +escalate_shutdown(gpointer data) +{ + + pcmk_child_t *child = data; + if(child->pid) { + crm_err("Child %s not terminating in a timely manner, forcing", child->name); + stop_child(child, SIGKILL); + } + return FALSE; +} + +static gboolean +pcmk_shutdown_worker(gpointer user_data) +{ + static int phase = 0; + static time_t next_log = 0; + static int max = SIZEOF(pcmk_children); + + int lpc = 0; + + if(phase == 0) { + crm_notice("Shuting down Pacemaker"); + phase = max; + } + + for (; phase > 0; phase--) { + /* dont stop anything with start_seq < 1 */ + + for (lpc = max - 1; lpc >= 0; lpc--) { + pcmk_child_t *child = &(pcmk_children[lpc]); + if(phase != child->start_seq) { + continue; + } + + if(child->pid) { + time_t now = time(NULL); + + if(child->respawn) { + next_log = now + 30; + child->respawn = FALSE; + stop_child(child, SIGTERM); + if(phase < pcmk_children[crm_msg_crmd].start_seq) { + g_timeout_add(180000/* 3m */, escalate_shutdown, child); + } + + } else if(now >= next_log) { + next_log = now + 30; + crm_notice("Still waiting for %s (pid=%d, seq=%d) to terminate...", + child->name, child->pid, child->start_seq); + } + return TRUE; + } + + /* cleanup */ + crm_notice("%s confirmed stopped", child->name); + child->pid = 0; + } + } + + /* send_cluster_id(); */ + crm_notice("Shutdown complete"); + g_main_loop_quit(mainloop); + return TRUE; +} + +static void +pcmk_shutdown(int nsig) +{ + shutdown_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, pcmk_shutdown_worker, NULL); + mainloop_set_trigger(shutdown_trigger); +} + + +static void build_path(const char *path_c, mode_t mode) +{ + int offset = 1, len = 0; + char *path = crm_strdup(path_c); + + CRM_CHECK(path != NULL, return); + for(len = strlen(path); offset < len; offset++) { + if(path[offset] == '/') { + path[offset] = 0; + if(mkdir(path, mode) < 0 && errno != EEXIST) { + crm_perror(LOG_ERR, "Could not create directory '%s'", path); + break; + } + path[offset] = '/'; + } + } + if(mkdir(path, mode) < 0 && errno != EEXIST) { + crm_perror(LOG_ERR, "Could not create directory '%s'", path); + } + crm_free(path); +} + +static void +pcmk_server_destroy(gpointer user_data) +{ + crm_info("Server destroyed"); + return; +} + +static void +pcmk_client_destroy(gpointer user_data) +{ + crm_debug("Client %p disconnected", user_data); + g_hash_table_remove(client_list, user_data); + return; +} + +static gboolean +pcmk_client_msg(IPC_Channel *client, gpointer user_data) +{ + xmlNode *msg = NULL; + gboolean stay_connected = TRUE; + + while(IPC_ISRCONN(client)) { + if(client->ops->is_message_pending(client) == 0) { + break; + } + + msg = xmlfromIPC(client, MAX_IPC_DELAY); + free_xml(msg); + + if(client->ch_status != IPC_CONNECT) { + break; + } + } + + if (client->ch_status != IPC_CONNECT) { + stay_connected = FALSE; + } + return stay_connected; +} + +static gboolean +pcmk_client_connect(IPC_Channel *ch, gpointer user_data) +{ + if (ch == NULL) { + crm_err("Channel was invalid"); + + } else if (ch->ch_status == IPC_DISCONNECT) { + crm_err("Channel was disconnected"); + + } else { + ch->ops->set_recv_qlen(ch, 1024); + ch->ops->set_send_qlen(ch, 1024); + + g_hash_table_insert(client_list, ch, user_data); + update_process_clients(); + + G_main_add_IPC_Channel( + G_PRIORITY_LOW, ch, FALSE, pcmk_client_msg, ch, pcmk_client_destroy); + } + return TRUE; +} + +static gboolean +ghash_send_proc_details(gpointer key, gpointer value, gpointer data) +{ + if(send_ipc_message(key, data) == FALSE) { + /* remove it */ + return TRUE; + } + return FALSE; +} + +static void peer_loop_fn(gpointer key, gpointer value, gpointer user_data) +{ + pcmk_peer_t *node = value; + xmlNode *update = user_data; + + xmlNode *xml = create_xml_node(update, "node"); + crm_xml_add_int(xml, "id", node->id); + crm_xml_add(xml, "uname", node->uname); + crm_xml_add_int(xml, "processes", node->processes); +} + +void update_process_clients(void) +{ + xmlNode *update = create_xml_node(NULL, "nodes"); + + crm_debug("Sending process list to %d children", + g_hash_table_size(client_list)); + + g_hash_table_foreach(peers, peer_loop_fn, update); + g_hash_table_foreach_remove(client_list, ghash_send_proc_details, update); + + crm_log_xml_debug(update, "update"); + free_xml(update); +} + +void update_process_peers(pcmk_peer_t *node) +{ + char buffer[1024]; + struct iovec iov; + int rc = 0; + + memset(buffer, SIZEOF(buffer), 0); + + if(node->uname) { + rc = snprintf(buffer, SIZEOF(buffer) - 1, "", node->uname, get_process_list()); + } else { + rc = snprintf(buffer, SIZEOF(buffer) - 1, "", get_process_list()); + } + + iov.iov_base = buffer; + iov.iov_len = rc + 1; + + send_cpg_message(&iov); +} + +gboolean update_node_processes(uint32_t id, const char *uname, uint32_t procs) +{ + gboolean changed = FALSE; + pcmk_peer_t *node = g_hash_table_lookup(peers, GUINT_TO_POINTER(id)); + if(node == NULL) { + changed = TRUE; + + crm_malloc0(node, sizeof(pcmk_peer_t)); + node->id = id; + + g_hash_table_insert(peers, GUINT_TO_POINTER(id), node); + node = g_hash_table_lookup(peers, GUINT_TO_POINTER(id)); + CRM_ASSERT(node != NULL); + } + + if(uname != NULL) { + if(node->uname == NULL || safe_str_eq(node->uname, uname) == FALSE) { + crm_info("%p Node %u now known as %s (was: %s)", + node, id, uname, node->uname); + crm_free(node->uname); + node->uname = crm_strdup(uname); + changed = TRUE; + } + } + + if(procs != 0 && procs != node->processes) { + crm_info("Node %s now has process list: %.32x (was %.32x)", + node->uname, procs, node->processes); + node->processes = procs; + changed = TRUE; + } + + if(changed && id == local_nodeid) { + update_process_clients(); + update_process_peers(node); + } + return changed; +} + +static struct crm_option long_options[] = { + /* Top-level Options */ + {"help", 0, 0, '?', "\tThis text"}, + {"version", 0, 0, '$', "\tVersion information" }, + {"verbose", 0, 0, 'V', "\tIncrease debug output"}, + + {"-spacer-", 1, 0, '-', "\nAdditional Options:"}, + {"foreground", 0, 0, 'f', "\tRun in the foreground instead of as a daemon"}, + {"pid-file", 1, 0, 'p', "\t(Advanced) Daemon pid file location"}, + + {NULL, 0, 0, 0} +}; + +int +main(int argc, char **argv) +{ + int rc; + int flag; + int argerr = 0; + + int option_index = 0; + gboolean daemonize = TRUE; + + int start_seq = 1, lpc = 0; + static int max = SIZEOF(pcmk_children); + + uid_t pcmk_uid = 0; + gid_t pcmk_gid = 0; + struct rlimit cores; + struct utsname name; + + crm_log_init(NULL, LOG_INFO, FALSE, FALSE, argc, argv, TRUE); + crm_set_options("V?$fp:", "mode [options]", long_options, + "Start/Stop Pacemaker\n"); + +#ifndef ON_DARWIN + /* prevent zombies */ + signal(SIGCLD, SIG_IGN); +#endif + + while (1) { + flag = crm_get_option(argc, argv, &option_index); + if (flag == -1) + break; + + switch(flag) { + case 'V': + cl_log_enable_stderr(TRUE); + alter_debug(DEBUG_INC); + break; + case 'f': + daemonize = FALSE; + break; + case 'p': + pid_file = optarg; + break; + case '$': + case '?': + crm_help(flag, LSB_EXIT_OK); + break; + default: + printf("Argument code 0%o (%c) is not (?yet?) supported\n", flag, flag); + ++argerr; + break; + } + } + + if (optind < argc) { + printf("non-option ARGV-elements: "); + while (optind < argc) + printf("%s ", argv[optind++]); + printf("\n"); + } + if (argerr) { + crm_help('?', LSB_EXIT_GENERIC); + } + + if(daemonize) { + cl_log_enable_stderr(FALSE); + crm_make_daemon(crm_system_name, TRUE, pid_file); + } + + crm_info("Starting %s", crm_system_name); + mainloop = g_main_new(FALSE); + + rc = getrlimit(RLIMIT_CORE, &cores); + if(rc < 0) { + crm_perror(LOG_ERR, "Cannot determine current maximum core size."); + } + + if(cores.rlim_max <= 0) { + cores.rlim_max = RLIM_INFINITY; + + rc = setrlimit(RLIMIT_CORE, &cores); + if(rc < 0) { + crm_perror(LOG_ERR, + "Core file generation will remain disabled." + " Core files are an important diagnositic tool," + " please consider enabling them by default."); + } + + } else { + crm_info("Maximum core file size is: %lu", cores.rlim_max); +#if 0 + /* system() is not thread-safe, can't call from here + * Actually, its a pretty hacky way to try and achieve this anyway + */ + if(system("echo 1 > /proc/sys/kernel/core_uses_pid") != 0) { + crm_perror(LOG_ERR, "Could not enable /proc/sys/kernel/core_uses_pid"); + } +#endif + } + + if(pcmk_user_lookup(CRM_DAEMON_USER, &pcmk_uid, &pcmk_gid) < 0) { + crm_err("Cluster user %s does not exist, aborting Pacemaker startup", CRM_DAEMON_USER); + return TRUE; + } + + mkdir(CRM_STATE_DIR, 0750); + chown(CRM_STATE_DIR, pcmk_uid, pcmk_gid); + + /* Used by stonithd */ + build_path(HA_STATE_DIR"/heartbeat", 0755); + + /* Used by RAs - Leave owned by root */ + build_path(CRM_RSCTMP_DIR, 0755); + + if(uname(&name) < 0) { + crm_perror(LOG_ERR,"uname(2) call failed"); + exit(100); + } + + if(read_config() == FALSE) { + return 1; + } + + client_list = g_hash_table_new(g_direct_hash, g_direct_equal); + peers = g_hash_table_new(g_direct_hash, g_direct_equal); + + if(init_server_ipc_comms(ipc_name, pcmk_client_connect, pcmk_server_destroy)) { + crm_err("Couldn't start IPC server"); + return 1; + } + + if(cluster_connect_cfg(&local_nodeid) == FALSE) { + return 1; + } + + if(cluster_connect_cpg() == FALSE) { + return 1; + } + + update_node_processes(local_nodeid, name.nodename, get_process_list()); + + mainloop_add_signal(SIGTERM, pcmk_shutdown); + mainloop_add_signal(SIGINT, pcmk_shutdown); + set_sigchld_proctrack(G_PRIORITY_HIGH, DEFAULT_MAXDISPATCHTIME); + + for (start_seq = 1; start_seq < max; start_seq++) { + /* dont start anything with start_seq < 1 */ + for (lpc = 0; lpc < max; lpc++) { + if(start_seq == pcmk_children[lpc].start_seq) { + start_child(&(pcmk_children[lpc])); + } + } + } + + crm_info("Starting mainloop"); + + g_main_run(mainloop); + g_main_destroy(mainloop); + + cluster_disconnect_cpg(); + cluster_disconnect_cfg(); + + crm_info("Exiting %s", crm_system_name); + + return 0; +} diff --git a/mcp/pacemaker.h b/mcp/pacemaker.h new file mode 100644 index 0000000000..fac57bf2c0 --- /dev/null +++ b/mcp/pacemaker.h @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2010 Andrew Beekhof + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This software is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include + +#include +#include +#include + +#include +#include + +#define SIZEOF(a) (sizeof(a) / sizeof(a[0])) +#define crm_flag_none 0x00000000 +#define crm_flag_members 0x00000001 +#define MAX_RESPAWN 100 +#define PW_BUFFER_LEN 500 + +extern uint32_t local_nodeid; + +typedef struct pcmk_peer_s +{ + uint32_t id; + uint32_t processes; + char *uname; +} pcmk_peer_t; + +extern gboolean read_config(void); + +extern gboolean cluster_connect_cfg(uint32_t *nodeid); +extern gboolean cluster_disconnect_cfg(void); + +extern gboolean cluster_connect_cpg(void); +extern gboolean cluster_disconnect_cpg(void); +extern gboolean send_cpg_message(struct iovec *iov); + +extern void update_process_clients(void); +extern void update_process_peers(pcmk_peer_t *node); +extern gboolean update_node_processes(uint32_t node, const char *uname, uint32_t procs);