diff --git a/lib/ais/plugin.c b/lib/ais/plugin.c index 1b38882387..36cfbc2a66 100644 --- a/lib/ais/plugin.c +++ b/lib/ais/plugin.c @@ -1,1742 +1,1747 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef AIS_COROSYNC # include # include # include #endif #include #include #include "plugin.h" #include "utils.h" #include #include #include #include #include #include #include #include #include struct corosync_api_v1 *pcmk_api = NULL; uint32_t plugin_has_votes = 0; uint32_t plugin_expected_votes = 2; int use_mgmtd = 0; int plugin_log_level = LOG_DEBUG; char *local_uname = NULL; int local_uname_len = 0; char *local_cname = NULL; int local_cname_len = 0; uint32_t local_nodeid = 0; char *ipc_channel_name = NULL; static uint64_t local_born_on = 0; uint64_t membership_seq = 0; pthread_t pcmk_wait_thread; gboolean wait_active = TRUE; gboolean have_reliable_membership_id = FALSE; GHashTable *ipc_client_list = NULL; GHashTable *membership_list = NULL; GHashTable *membership_notify_list = NULL; #define MAX_RESPAWN 100 #define LOOPBACK_ID 16777343 #define crm_flag_none 0x00000000 #define crm_flag_members 0x00000001 struct crm_identify_msg_s { coroipc_request_header_t header __attribute__((aligned(8))); uint32_t id; uint32_t pid; int32_t votes; uint32_t processes; char uname[256]; char version[256]; uint64_t born_on; } __attribute__((packed)); static crm_child_t pcmk_children[] = { { 0, crm_proc_none, crm_flag_none, 0, 0, FALSE, "none", NULL, NULL, NULL, NULL }, { 0, crm_proc_ais, crm_flag_none, 0, 0, FALSE, "ais", NULL, NULL, NULL, NULL }, { 0, crm_proc_lrmd, crm_flag_none, 3, 0, TRUE, "lrmd", NULL, HB_DAEMON_DIR"/lrmd", NULL, NULL }, { 0, crm_proc_cib, crm_flag_members, 2, 0, TRUE, "cib", CRM_DAEMON_USER, CRM_DAEMON_DIR"/cib", NULL, NULL }, { 0, crm_proc_crmd, crm_flag_members, 6, 0, TRUE, "crmd", CRM_DAEMON_USER, CRM_DAEMON_DIR"/crmd", NULL, NULL }, { 0, crm_proc_attrd, crm_flag_none, 4, 0, TRUE, "attrd", CRM_DAEMON_USER, CRM_DAEMON_DIR"/attrd", NULL, NULL }, { 0, crm_proc_stonithd, crm_flag_none, 0, 0, TRUE, "stonithd", NULL, "/bin/false", NULL, NULL }, { 0, crm_proc_pe, crm_flag_none, 5, 0, TRUE, "pengine", CRM_DAEMON_USER, CRM_DAEMON_DIR"/pengine", NULL, NULL }, { 0, crm_proc_mgmtd, crm_flag_none, 7, 0, TRUE, "mgmtd", NULL, HB_DAEMON_DIR"/mgmtd", NULL, NULL }, { 0, crm_proc_stonith_ng, crm_flag_none, 1, 0, TRUE, "stonith-ng", NULL, CRM_DAEMON_DIR"/stonithd", NULL, NULL }, }; void send_cluster_id(void); int send_cluster_msg_raw(const AIS_Message *ais_msg); char *pcmk_generate_membership_data(void); gboolean check_message_sanity(const AIS_Message *msg, const char *data); #ifdef AIS_COROSYNC typedef const void ais_void_ptr; int pcmk_shutdown(void); void pcmk_peer_update(enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id); #else typedef void ais_void_ptr; extern totempg_groups_handle openais_group_handle; int pcmk_shutdown(struct objdb_iface_ver0 *objdb); void pcmk_peer_update(enum totem_configuration_type configuration_type, unsigned int *member_list, int member_list_entries, unsigned int *left_list, int left_list_entries, unsigned int *joined_list, int joined_list_entries, struct memb_ring_id *ring_id); #endif int pcmk_startup (struct corosync_api_v1 *corosync_api); int pcmk_config_init(struct corosync_api_v1 *corosync_api); int pcmk_ipc_exit (void *conn); int pcmk_ipc_connect (void *conn); void pcmk_ipc(void *conn, ais_void_ptr *msg); void pcmk_exec_dump(void); void pcmk_cluster_swab(void *msg); void pcmk_cluster_callback(ais_void_ptr *message, unsigned int nodeid); void pcmk_nodeid(void *conn, ais_void_ptr *msg); void pcmk_nodes(void *conn, ais_void_ptr *msg); void pcmk_notify(void *conn, ais_void_ptr *msg); void pcmk_remove_member(void *conn, ais_void_ptr *msg); void pcmk_quorum(void *conn, ais_void_ptr *msg); void pcmk_cluster_id_swab(void *msg); void pcmk_cluster_id_callback(ais_void_ptr *message, unsigned int nodeid); void ais_remove_peer(char *node_id); static uint32_t get_process_list(void) { int lpc = 0; uint32_t procs = crm_proc_ais; for (lpc = 0; lpc < SIZEOF(pcmk_children); lpc++) { if(pcmk_children[lpc].pid != 0) { procs |= pcmk_children[lpc].flag; } } return procs; } static struct corosync_lib_handler pcmk_lib_service[] = { { /* 0 */ .lib_handler_fn = pcmk_ipc, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED, #ifdef AIS_WHITETANK .response_size = sizeof (coroipc_response_header_t), .response_id = CRM_MESSAGE_IPC_ACK, #endif }, { /* 1 */ .lib_handler_fn = pcmk_nodes, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED, #ifdef AIS_WHITETANK .response_size = sizeof (coroipc_response_header_t), .response_id = CRM_MESSAGE_IPC_ACK, #endif }, { /* 2 */ .lib_handler_fn = pcmk_notify, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED, #ifdef AIS_WHITETANK .response_size = sizeof (coroipc_response_header_t), .response_id = CRM_MESSAGE_IPC_ACK, #endif }, { /* 3 */ .lib_handler_fn = pcmk_nodeid, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED, #ifdef AIS_WHITETANK .response_size = sizeof (struct crm_ais_nodeid_resp_s), .response_id = crm_class_nodeid, #endif }, { /* 4 */ .lib_handler_fn = pcmk_remove_member, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED, #ifdef AIS_WHITETANK .response_size = sizeof (coroipc_response_header_t), .response_id = CRM_MESSAGE_IPC_ACK, #endif }, { /* 5 */ .lib_handler_fn = pcmk_quorum, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED, #ifdef AIS_WHITETANK .response_size = sizeof (coroipc_response_header_t), .response_id = CRM_MESSAGE_IPC_ACK, #endif }, }; static struct corosync_exec_handler pcmk_exec_service[] = { { /* 0 */ .exec_handler_fn = pcmk_cluster_callback, .exec_endian_convert_fn = pcmk_cluster_swab }, { /* 1 */ .exec_handler_fn = pcmk_cluster_id_callback, .exec_endian_convert_fn = pcmk_cluster_id_swab } }; /* * Exports the interface for the service */ struct corosync_service_engine pcmk_service_handler = { .name = (unsigned char *)"Pacemaker Cluster Manager "PACKAGE_VERSION, .id = PCMK_SERVICE_ID, .private_data_size = 0, .flow_control = COROSYNC_LIB_FLOW_CONTROL_NOT_REQUIRED, .lib_init_fn = pcmk_ipc_connect, .lib_exit_fn = pcmk_ipc_exit, .exec_init_fn = pcmk_startup, .exec_exit_fn = pcmk_shutdown, .config_init_fn = pcmk_config_init, #ifdef AIS_COROSYNC .priority = 50, .lib_engine = pcmk_lib_service, .lib_engine_count = sizeof (pcmk_lib_service) / sizeof (struct corosync_lib_handler), .exec_engine = pcmk_exec_service, .exec_engine_count = sizeof (pcmk_exec_service) / sizeof (struct corosync_exec_handler), #else .lib_service = pcmk_lib_service, .lib_service_count = sizeof (pcmk_lib_service) / sizeof (struct corosync_lib_handler), .exec_service = pcmk_exec_service, .exec_service_count = sizeof (pcmk_exec_service) / sizeof (struct corosync_exec_handler), #endif .confchg_fn = pcmk_peer_update, .exec_dump_fn = pcmk_exec_dump, /* void (*sync_init) (void); */ /* int (*sync_process) (void); */ /* void (*sync_activate) (void); */ /* void (*sync_abort) (void); */ }; /* * Dynamic Loader definition */ struct corosync_service_engine *pcmk_get_handler_ver0 (void); #ifdef AIS_COROSYNC struct corosync_service_engine_iface_ver0 pcmk_service_handler_iface = { .corosync_get_service_engine_ver0 = pcmk_get_handler_ver0 }; #else struct openais_service_handler_iface_ver0 pcmk_service_handler_iface = { .openais_get_service_handler_ver0 = pcmk_get_handler_ver0 }; #endif static struct lcr_iface openais_pcmk_ver0[1] = { { .name = "pacemaker", .version = 0, .versions_replace = 0, .versions_replace_count = 0, .dependencies = 0, .dependency_count = 0, .constructor = NULL, .destructor = NULL, .interfaces = NULL } }; static struct lcr_comp pcmk_comp_ver0 = { .iface_count = 1, .ifaces = openais_pcmk_ver0 }; struct corosync_service_engine *pcmk_get_handler_ver0 (void) { return (&pcmk_service_handler); } __attribute__ ((constructor)) static void register_this_component (void) { lcr_interfaces_set (&openais_pcmk_ver0[0], &pcmk_service_handler_iface); lcr_component_register (&pcmk_comp_ver0); } static int plugin_has_quorum(void) { if((plugin_expected_votes >> 1) < plugin_has_votes) { return 1; } return 0; } static void update_expected_votes(int value) { if(value < plugin_has_votes) { /* Never drop below the number of connected nodes */ ais_info("Cannot update expected quorum votes %d -> %d:" " value cannot be less that the current number of votes", plugin_expected_votes, value); } else if(plugin_expected_votes != value) { ais_info("Expected quorum votes %d -> %d", plugin_expected_votes, value); plugin_expected_votes = value; } } /* Create our own local copy of the config so we can navigate it */ static void process_ais_conf(void) { char *value = NULL; gboolean any_log = FALSE; hdb_handle_t top_handle = 0; hdb_handle_t local_handle = 0; ais_info("Reading configure"); top_handle = config_find_init(pcmk_api, "logging"); local_handle = config_find_next(pcmk_api, "logging", top_handle); get_config_opt(pcmk_api, local_handle, "debug", &value, "on"); if(ais_get_boolean(value)) { plugin_log_level = LOG_DEBUG; pcmk_env.debug = "1"; } else { plugin_log_level = LOG_INFO; pcmk_env.debug = "0"; } get_config_opt(pcmk_api, local_handle, "to_logfile", &value, "off"); if(ais_get_boolean(value)) { get_config_opt(pcmk_api, local_handle, "logfile", &value, NULL); if(value == NULL) { ais_err("Logging to a file requested but no log file specified"); } else { uid_t pcmk_uid = geteuid(); uid_t pcmk_gid = getegid(); FILE *logfile = fopen(value, "a"); - int logfd = fileno(logfile); + if(logfile) { + int logfd = fileno(logfile); - pcmk_env.logfile = value; + pcmk_env.logfile = value; - /* Ensure the file has the correct permissions */ - fchown(logfd, pcmk_uid, pcmk_gid); - fchmod(logfd, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP); - - fprintf(logfile, "Set r/w permissions for uid=%d, gid=%d on %s\n", - pcmk_uid, pcmk_gid, value); - fflush(logfile); - fsync(fileno(logfile)); - fclose(logfile); - any_log = TRUE; + /* Ensure the file has the correct permissions */ + fchown(logfd, pcmk_uid, pcmk_gid); + fchmod(logfd, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP); + + fprintf(logfile, "Set r/w permissions for uid=%d, gid=%d on %s\n", + pcmk_uid, pcmk_gid, value); + fflush(logfile); + fsync(fileno(logfile)); + fclose(logfile); + any_log = TRUE; + + } else { + ais_err("Couldn't create logfile: %s", value); + } } } get_config_opt(pcmk_api, local_handle, "to_syslog", &value, "on"); if(any_log && ais_get_boolean(value) == FALSE) { ais_info("User configured file based logging and explicitly disabled syslog."); value = "none"; } else { if(ais_get_boolean(value) == FALSE) { ais_err("Please enable some sort of logging, either 'to_file: on' or 'to_syslog: on'."); ais_err("If you use file logging, be sure to also define a value for 'logfile'"); } get_config_opt(pcmk_api, local_handle, "syslog_facility", &value, "daemon"); } pcmk_env.syslog = value; config_find_done(pcmk_api, local_handle); top_handle = config_find_init(pcmk_api, "service"); local_handle = config_find_next(pcmk_api, "service", top_handle); while(local_handle) { value = NULL; pcmk_api->object_key_get(local_handle, "name", strlen("name"), (void**)&value, NULL); if(ais_str_eq("pacemaker", value)) { break; } local_handle = config_find_next(pcmk_api, "service", top_handle); } get_config_opt(pcmk_api, local_handle, "clustername", &local_cname, "pcmk"); local_cname_len = strlen(local_cname); get_config_opt(pcmk_api, local_handle, "use_logd", &value, "no"); pcmk_env.use_logd = value; get_config_opt(pcmk_api, local_handle, "use_mgmtd", &value, "no"); if(ais_get_boolean(value) == FALSE) { int lpc = 0; for (; lpc < SIZEOF(pcmk_children); lpc++) { if(crm_proc_mgmtd & pcmk_children[lpc].flag) { /* Disable mgmtd startup */ pcmk_children[lpc].start_seq = 0; break; } } } config_find_done(pcmk_api, local_handle); } int pcmk_config_init(struct corosync_api_v1 *unused) { return 0; } static void *pcmk_wait_dispatch (void *arg) { struct timespec waitsleep = { .tv_sec = 1, .tv_nsec = 0 }; while(wait_active) { int lpc = 0; for (; lpc < SIZEOF(pcmk_children); lpc++) { if(pcmk_children[lpc].pid > 0) { int status; pid_t pid = wait4( pcmk_children[lpc].pid, &status, WNOHANG, NULL); if(pid == 0) { continue; } else if(pid < 0) { ais_perror("Call to wait4(%s) failed", pcmk_children[lpc].name); continue; } /* cleanup */ pcmk_children[lpc].pid = 0; pcmk_children[lpc].conn = NULL; pcmk_children[lpc].async_conn = NULL; if(WIFSIGNALED(status)) { int sig = WTERMSIG(status); ais_err("Child process %s terminated with signal %d" " (pid=%d, core=%s)", pcmk_children[lpc].name, sig, pid, WCOREDUMP(status)?"true":"false"); } else if (WIFEXITED(status)) { int rc = WEXITSTATUS(status); do_ais_log(rc==0?LOG_NOTICE:LOG_ERR, "Child process %s exited (pid=%d, rc=%d)", pcmk_children[lpc].name, pid, rc); if(rc == 100) { ais_notice("Child process %s no longer wishes" " to be respawned", pcmk_children[lpc].name); pcmk_children[lpc].respawn = FALSE; } } pcmk_children[lpc].respawn_count += 1; if(pcmk_children[lpc].respawn_count > MAX_RESPAWN) { ais_err("Child respawn count exceeded by %s", pcmk_children[lpc].name); pcmk_children[lpc].respawn = FALSE; } if(pcmk_children[lpc].respawn) { ais_notice("Respawning failed child process: %s", pcmk_children[lpc].name); spawn_child(&(pcmk_children[lpc])); } send_cluster_id(); } } sched_yield (); nanosleep (&waitsleep, 0); } return 0; } static uint32_t pcmk_update_nodeid(void) { int last = local_nodeid; #if AIS_COROSYNC local_nodeid = pcmk_api->totem_nodeid_get(); #else local_nodeid = totempg_my_nodeid_get(); #endif if(last != local_nodeid) { if(last == 0) { ais_info("Local node id: %u", local_nodeid); } else { char *last_s = NULL; ais_malloc0(last_s, 32); ais_warn("Detected local node id change: %u -> %u", last, local_nodeid); snprintf(last_s, 31, "%u", last); ais_remove_peer(last_s); ais_free(last_s); } update_member(local_nodeid, 0, 0, 1, 0, local_uname, CRM_NODE_MEMBER, NULL); } return local_nodeid; } static void build_path(const char *path_c, mode_t mode) { int offset = 1, len = 0; char *path = ais_strdup(path_c); AIS_CHECK(path != NULL, return); for(len = strlen(path); offset < len; offset++) { if(path[offset] == '/') { path[offset] = 0; if(mkdir(path, mode) < 0 && errno != EEXIST) { ais_perror("Could not create directory '%s'", path); break; } path[offset] = '/'; } } if(mkdir(path, mode) < 0 && errno != EEXIST) { ais_perror("Could not create directory '%s'", path); } ais_free(path); } int pcmk_startup(struct corosync_api_v1 *init_with) { int rc = 0; int lpc = 0; int start_seq = 1; struct utsname us; struct rlimit cores; static int max = SIZEOF(pcmk_children); uid_t pcmk_uid = 0; gid_t pcmk_gid = 0; uid_t root_uid = -1; uid_t cs_uid = geteuid(); pcmk_user_lookup("root", &root_uid, NULL); pcmk_api = init_with; #ifdef AIS_WHITETANK log_init ("crm"); #endif pcmk_env.debug = "0"; pcmk_env.logfile = NULL; pcmk_env.use_logd = "false"; pcmk_env.syslog = "daemon"; if(cs_uid != root_uid) { ais_err("Corosync must be configured to start as 'root'," " otherwise Pacemaker cannot manage services." " Expected %d got %d", root_uid, cs_uid); return -1; } process_ais_conf(); membership_list = g_hash_table_new_full( g_direct_hash, g_direct_equal, NULL, destroy_ais_node); membership_notify_list = g_hash_table_new(g_direct_hash, g_direct_equal); ipc_client_list = g_hash_table_new(g_direct_hash, g_direct_equal); ais_info("CRM: Initialized"); log_printf(LOG_INFO, "Logging: Initialized %s\n", __PRETTY_FUNCTION__); rc = getrlimit(RLIMIT_CORE, &cores); if(rc < 0) { ais_perror("Cannot determine current maximum core size."); } if(cores.rlim_max <= 0) { cores.rlim_max = RLIM_INFINITY; rc = setrlimit(RLIMIT_CORE, &cores); if(rc < 0) { ais_perror("Core file generation will remain disabled." " Core files are an important diagnositic tool," " please consider enabling them by default."); } } else { ais_info("Maximum core file size is: %lu", cores.rlim_max); #if 0 /* system() is not thread-safe, can't call from here * Actually, its a pretty hacky way to try and achieve this anyway */ if(system("echo 1 > /proc/sys/kernel/core_uses_pid") != 0) { ais_perror("Could not enable /proc/sys/kernel/core_uses_pid"); } #endif } if(pcmk_user_lookup(CRM_DAEMON_USER, &pcmk_uid, &pcmk_gid) < 0) { ais_err("Cluster user %s does not exist, aborting Pacemaker startup", CRM_DAEMON_USER); return TRUE; } mkdir(CRM_STATE_DIR, 0750); chown(CRM_STATE_DIR, pcmk_uid, pcmk_gid); /* Used by stonithd */ build_path(HA_STATE_DIR"/heartbeat", 0755); /* Used by RAs - Leave owned by root */ build_path(CRM_RSCTMP_DIR, 0755); rc = uname(&us); AIS_ASSERT(rc == 0); local_uname = ais_strdup(us.nodename); local_uname_len = strlen(local_uname); ais_info("Service: %d", PCMK_SERVICE_ID); ais_info("Local hostname: %s", local_uname); pcmk_update_nodeid(); pthread_create (&pcmk_wait_thread, NULL, pcmk_wait_dispatch, NULL); for (start_seq = 1; start_seq < max; start_seq++) { /* dont start anything with start_seq < 1 */ for (lpc = 0; lpc < max; lpc++) { if(start_seq == pcmk_children[lpc].start_seq) { spawn_child(&(pcmk_children[lpc])); } } } send_cluster_id(); return 0; } /* static void ais_print_node(const char *prefix, struct totem_ip_address *host) { int len = 0; char *buffer = NULL; ais_malloc0(buffer, INET6_ADDRSTRLEN+1); inet_ntop(host->family, host->addr, buffer, INET6_ADDRSTRLEN); len = strlen(buffer); ais_info("%s: %.*s", prefix, len, buffer); ais_free(buffer); } */ #if 0 /* copied here for reference from exec/totempg.c */ char *totempg_ifaces_print (unsigned int nodeid) { static char iface_string[256 * INTERFACE_MAX]; char one_iface[64]; struct totem_ip_address interfaces[INTERFACE_MAX]; char **status; unsigned int iface_count; unsigned int i; int res; iface_string[0] = '\0'; res = totempg_ifaces_get (nodeid, interfaces, &status, &iface_count); if (res == -1) { return ("no interface found for nodeid"); } for (i = 0; i < iface_count; i++) { sprintf (one_iface, "r(%d) ip(%s), ", i, totemip_print (&interfaces[i])); strcat (iface_string, one_iface); } return (iface_string); } #endif static void ais_mark_unseen_peer_dead( gpointer key, gpointer value, gpointer user_data) { int *changed = user_data; crm_node_t *node = value; if(node->last_seen != membership_seq && ais_str_eq(CRM_NODE_LOST, node->state) == FALSE) { ais_info("Node %s was not seen in the previous transition", node->uname); *changed += update_member(node->id, 0, membership_seq, node->votes, node->processes, node->uname, CRM_NODE_LOST, NULL); } } void pcmk_peer_update ( enum totem_configuration_type configuration_type, #ifdef AIS_COROSYNC const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id #else unsigned int *member_list, int member_list_entries, unsigned int *left_list, int left_list_entries, unsigned int *joined_list, int joined_list_entries, struct memb_ring_id *ring_id #endif ) { int lpc = 0; int changed = 0; int do_update = 0; AIS_ASSERT(ring_id != NULL); switch(configuration_type) { case TOTEM_CONFIGURATION_REGULAR: do_update = 1; break; case TOTEM_CONFIGURATION_TRANSITIONAL: break; } membership_seq = ring_id->seq; ais_notice("%s membership event on ring %lld: memb=%ld, new=%ld, lost=%ld", do_update?"Stable":"Transitional", ring_id->seq, (long)member_list_entries, (long)joined_list_entries, (long)left_list_entries); if(do_update == 0) { for(lpc = 0; lpc < joined_list_entries; lpc++) { const char *prefix = "new: "; uint32_t nodeid = joined_list[lpc]; ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid); } for(lpc = 0; lpc < member_list_entries; lpc++) { const char *prefix = "memb:"; uint32_t nodeid = member_list[lpc]; ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid); } for(lpc = 0; lpc < left_list_entries; lpc++) { const char *prefix = "lost:"; uint32_t nodeid = left_list[lpc]; ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid); } return; } for(lpc = 0; lpc < joined_list_entries; lpc++) { const char *prefix = "NEW: "; uint32_t nodeid = joined_list[lpc]; crm_node_t *node = NULL; changed += update_member( nodeid, 0, membership_seq, -1, 0, NULL, CRM_NODE_MEMBER, NULL); ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid); node = g_hash_table_lookup(membership_list, GUINT_TO_POINTER(nodeid)); if(node->addr == NULL) { const char *addr = totempg_ifaces_print(nodeid); node->addr = ais_strdup(addr); ais_debug("Node %u has address %s", nodeid, node->addr); } } for(lpc = 0; lpc < member_list_entries; lpc++) { const char *prefix = "MEMB:"; uint32_t nodeid = member_list[lpc]; changed += update_member( nodeid, 0, membership_seq, -1, 0, NULL, CRM_NODE_MEMBER, NULL); ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid); } for(lpc = 0; lpc < left_list_entries; lpc++) { const char *prefix = "LOST:"; uint32_t nodeid = left_list[lpc]; changed += update_member( nodeid, 0, membership_seq, -1, 0, NULL, CRM_NODE_LOST, NULL); ais_info("%s %s %u", prefix, member_uname(nodeid), nodeid); } if(changed && joined_list_entries == 0 && left_list_entries == 0) { ais_err("Something strange happened: %d", changed); changed = 0; } ais_debug_2("Reaping unseen nodes..."); g_hash_table_foreach(membership_list, ais_mark_unseen_peer_dead, &changed); if(member_list_entries > 1) { /* Used to set born-on in send_cluster_id()) * We need to wait until we have at least one peer since first * membership id is based on the one before we stopped and isn't reliable */ have_reliable_membership_id = TRUE; } if(changed) { ais_debug("%d nodes changed", changed); pcmk_update_nodeid(); send_member_notification(); } send_cluster_id(); } int pcmk_ipc_exit (void *conn) { int lpc = 0; const char *client = NULL; void *async_conn = conn; for (; lpc < SIZEOF(pcmk_children); lpc++) { if(pcmk_children[lpc].conn == conn) { if(wait_active == FALSE) { /* Make sure the shutdown loop exits */ pcmk_children[lpc].pid = 0; } pcmk_children[lpc].conn = NULL; pcmk_children[lpc].async_conn = NULL; client = pcmk_children[lpc].name; break; } } g_hash_table_remove(membership_notify_list, async_conn); g_hash_table_remove(ipc_client_list, async_conn); do_ais_log(client?LOG_INFO:(LOG_DEBUG+1), "Client %s (conn=%p, async-conn=%p) left", client?client:"unknown-transient", conn, async_conn); return (0); } int pcmk_ipc_connect (void *conn) { /* OpenAIS hasn't finished setting up the connection at this point * Sending messages now messes up the protocol! */ return (0); } /* * Executive message handlers */ void pcmk_cluster_swab(void *msg) { AIS_Message *ais_msg = msg; ais_debug_3("Performing endian conversion..."); ais_msg->id = swab32 (ais_msg->id); ais_msg->size = swab32 (ais_msg->size); ais_msg->is_compressed = swab32 (ais_msg->is_compressed); ais_msg->compressed_size = swab32 (ais_msg->compressed_size); ais_msg->host.id = swab32 (ais_msg->host.id); ais_msg->host.pid = swab32 (ais_msg->host.pid); ais_msg->host.type = swab32 (ais_msg->host.type); ais_msg->host.size = swab32 (ais_msg->host.size); ais_msg->host.local = swab32 (ais_msg->host.local); ais_msg->sender.id = swab32 (ais_msg->sender.id); ais_msg->sender.pid = swab32 (ais_msg->sender.pid); ais_msg->sender.type = swab32 (ais_msg->sender.type); ais_msg->sender.size = swab32 (ais_msg->sender.size); ais_msg->sender.local = swab32 (ais_msg->sender.local); ais_msg->header.size = swab32 (ais_msg->header.size); ais_msg->header.id = swab32 (ais_msg->header.id); ais_msg->header.error = swab32 (ais_msg->header.error); } void pcmk_cluster_callback ( ais_void_ptr *message, unsigned int nodeid) { const AIS_Message *ais_msg = message; ais_debug_2("Message from node %u (%s)", nodeid, nodeid==local_nodeid?"local":"remote"); /* Shouldn't be required... update_member( ais_msg->sender.id, membership_seq, -1, 0, ais_msg->sender.uname, NULL); */ if(ais_msg->host.size == 0 || ais_str_eq(ais_msg->host.uname, local_uname)) { route_ais_message(ais_msg, FALSE); } else { ais_debug_3("Discarding Msg[%d] (dest=%s:%s, from=%s:%s)", ais_msg->id, ais_dest(&(ais_msg->host)), msg_type2text(ais_msg->host.type), ais_dest(&(ais_msg->sender)), msg_type2text(ais_msg->sender.type)); } } void pcmk_cluster_id_swab(void *msg) { struct crm_identify_msg_s *ais_msg = msg; ais_debug_3("Performing endian conversion..."); ais_msg->id = swab32 (ais_msg->id); ais_msg->pid = swab32 (ais_msg->pid); ais_msg->votes = swab32 (ais_msg->votes); ais_msg->processes = swab32 (ais_msg->processes); ais_msg->born_on = swab64 (ais_msg->born_on); ais_msg->header.size = swab32 (ais_msg->header.size); ais_msg->header.id = swab32 (ais_msg->header.id); } void pcmk_cluster_id_callback (ais_void_ptr *message, unsigned int nodeid) { int changed = 0; const struct crm_identify_msg_s *msg = message; if(nodeid != msg->id) { ais_err("Invalid message: Node %u claimed to be node %d", nodeid, msg->id); return; } ais_debug("Node update: %s (%s)", msg->uname, msg->version); changed = update_member( nodeid, msg->born_on, membership_seq, msg->votes, msg->processes, msg->uname, NULL, msg->version); if(changed) { send_member_notification(); } } struct res_overlay { coroipc_response_header_t header __attribute((aligned(8))); char buf[4096]; }; struct res_overlay *res_overlay = NULL; static void send_ipc_ack(void *conn) { if(res_overlay == NULL) { ais_malloc0(res_overlay, sizeof(struct res_overlay)); } res_overlay->header.id = CRM_MESSAGE_IPC_ACK; res_overlay->header.size = sizeof (coroipc_response_header_t); res_overlay->header.error = CS_OK; #ifdef AIS_COROSYNC pcmk_api->ipc_response_send (conn, res_overlay, res_overlay->header.size); #else openais_response_send (conn, res_overlay, res_overlay->header.size); #endif } /* local callbacks */ void pcmk_ipc(void *conn, ais_void_ptr *msg) { AIS_Message *mutable; int type = 0, size = 0; gboolean transient = TRUE; const AIS_Message *ais_msg = (const AIS_Message*)msg; void *async_conn = conn; ais_debug_2("Message from client %p", conn); if(check_message_sanity(msg, ((const AIS_Message*)msg)->data) == FALSE) { /* The message is corrupted - ignore */ send_ipc_ack(conn); msg = NULL; return; } /* Make a copy of the message here and ACK it * The message is only valid until a response is sent * but the response must also be sent _before_ we send anything else */ mutable = ais_msg_copy(ais_msg); AIS_ASSERT(check_message_sanity(mutable, mutable->data)); size = mutable->header.size; /* ais_malloc0(ais_msg, size); */ /* memcpy(ais_msg, msg, size); */ type = mutable->sender.type; ais_debug_3("type: %d local: %d conn: %p host type: %d ais: %d sender pid: %d child pid: %d size: %d", type, mutable->host.local, pcmk_children[type].conn, mutable->host.type, crm_msg_ais, mutable->sender.pid, pcmk_children[type].pid, ((int)SIZEOF(pcmk_children))); if(type > crm_msg_none && type < SIZEOF(pcmk_children)) { /* known child process */ transient = FALSE; } /* If this check fails, the order of pcmk_children probably * doesn't match that of the crm_ais_msg_types enum */ AIS_CHECK(transient || mutable->sender.pid == pcmk_children[type].pid, ais_err("Sender: %d, child[%d]: %d", mutable->sender.pid, type, pcmk_children[type].pid); return); if(transient == FALSE && type > crm_msg_none && mutable->host.local && pcmk_children[type].conn == NULL && mutable->host.type == crm_msg_ais) { AIS_CHECK(mutable->sender.type != mutable->sender.pid, ais_err("Pid=%d, type=%d", mutable->sender.pid, mutable->sender.type)); ais_info("Recorded connection %p for %s/%d", conn, pcmk_children[type].name, pcmk_children[type].pid); pcmk_children[type].conn = conn; pcmk_children[type].async_conn = async_conn; /* Make sure they have the latest membership */ if(pcmk_children[type].flags & crm_flag_members) { char *update = pcmk_generate_membership_data(); g_hash_table_replace(membership_notify_list, async_conn, async_conn); ais_info("Sending membership update "U64T" to %s", membership_seq, pcmk_children[type].name); send_client_msg(async_conn, crm_class_members, crm_msg_none,update); } } else if(transient) { AIS_CHECK(mutable->sender.type == mutable->sender.pid, ais_err("Pid=%d, type=%d", mutable->sender.pid, mutable->sender.type)); g_hash_table_replace(ipc_client_list, async_conn, GUINT_TO_POINTER(mutable->sender.pid)); } mutable->sender.id = local_nodeid; mutable->sender.size = local_uname_len; memset(mutable->sender.uname, 0, MAX_NAME); memcpy(mutable->sender.uname, local_uname, mutable->sender.size); route_ais_message(mutable, TRUE); send_ipc_ack(conn); msg = NULL; ais_free(mutable); } int pcmk_shutdown ( #ifdef AIS_COROSYNC void #else struct objdb_iface_ver0 *objdb #endif ) { int lpc = 0; static int phase = 0; static int max_wait = 0; static time_t next_log = 0; static int max = SIZEOF(pcmk_children); if(phase == 0) { ais_notice("Shuting down Pacemaker"); phase = max; } wait_active = FALSE; /* stop the wait loop */ for (; phase > 0; phase--) { /* dont stop anything with start_seq < 1 */ for (lpc = max - 1; lpc >= 0; lpc--) { if(phase != pcmk_children[lpc].start_seq) { continue; } #ifdef AIS_WHITETANK retry: #endif if(pcmk_children[lpc].pid) { pid_t pid = 0; int status = 0; time_t now = time(NULL); if(pcmk_children[lpc].respawn) { max_wait = 5; /* 5 * 30s = 2.5 minutes... plenty once the crmd is gone */ next_log = now + 30; pcmk_children[lpc].respawn = FALSE; stop_child(&(pcmk_children[lpc]), SIGTERM); } pid = wait4(pcmk_children[lpc].pid, &status, WNOHANG, NULL); if(pid < 0) { ais_perror("Call to wait4(%s/%d) failed - treating it as stopped", pcmk_children[lpc].name, pcmk_children[lpc].pid); } else if(pid == 0) { if(now >= next_log) { max_wait--; next_log = now + 30; ais_notice("Still waiting for %s (pid=%d, seq=%d) to terminate...", pcmk_children[lpc].name, pcmk_children[lpc].pid, pcmk_children[lpc].start_seq); if(max_wait <= 0 && phase < pcmk_children[crm_msg_crmd].start_seq) { ais_err("Child %s taking too long to terminate, sending SIGKILL", pcmk_children[lpc].name); stop_child(&(pcmk_children[lpc]), SIGKILL); } } #ifdef AIS_WHITETANK { struct timespec waitsleep = { .tv_sec = 1, .tv_nsec = 0 }; sched_yield (); nanosleep (&waitsleep, 0); goto retry; } #else /* Return control to corosync */ return -1; #endif } } /* cleanup */ ais_notice("%s confirmed stopped", pcmk_children[lpc].name); pcmk_children[lpc].async_conn = NULL; pcmk_children[lpc].conn = NULL; pcmk_children[lpc].pid = 0; } } send_cluster_id(); ais_notice("Shutdown complete"); /* TODO: Add back the logsys flush call once its written */ #ifdef AIS_WHITETANK /* Bug bnc#482847, bnc#482905 * * All cluster services are now down, we could allow OpenAIS to continue * unloading plugins, but its kinda new at that and there are a bunch of * race conditions that get exercised. * * Take the easy way out for now (on whitetank) and eventually fix for * CoroSync which is where everyone wants to be eventually anyway */ ais_notice("Forcing clean exit of OpenAIS"); exit(0); #endif return 0; } struct member_loop_data { char *string; }; void member_vote_count_fn(gpointer key, gpointer value, gpointer user_data) { crm_node_t *node = value; if (ais_str_eq(CRM_NODE_MEMBER, node->state)) { plugin_has_votes += node->votes; } } void member_loop_fn(gpointer key, gpointer value, gpointer user_data) { crm_node_t *node = value; struct member_loop_data *data = user_data; ais_debug_2("Dumping node %u", node->id); data->string = append_member(data->string, node); } char *pcmk_generate_membership_data(void) { int size = 0; struct member_loop_data data; size = 256; ais_malloc0(data.string, size); /* Ensure the list of active processes is up-to-date */ update_member(local_nodeid, 0, 0, -1, get_process_list(), local_uname, NULL, NULL); plugin_has_votes = 0; g_hash_table_foreach(membership_list, member_vote_count_fn, NULL); if(plugin_has_votes > plugin_expected_votes) { update_expected_votes(plugin_has_votes); } snprintf(data.string, size, "", membership_seq, plugin_has_quorum()?"true":"false", plugin_expected_votes, plugin_has_votes); g_hash_table_foreach(membership_list, member_loop_fn, &data); size = strlen(data.string); data.string = realloc(data.string, size + 9) ;/* 9 = + nul */ sprintf(data.string + size, ""); return data.string; } void pcmk_nodes(void *conn, ais_void_ptr *msg) { char *data = pcmk_generate_membership_data(); void *async_conn = conn; /* send the ACK before we send any other messages * - but after we no longer need to access the message */ send_ipc_ack(conn); msg = NULL; if(async_conn) { send_client_msg(async_conn, crm_class_members, crm_msg_none, data); } ais_free(data); } void pcmk_remove_member(void *conn, ais_void_ptr *msg) { const AIS_Message *ais_msg = msg; char *data = get_ais_data(ais_msg); send_ipc_ack(conn); msg = NULL; if(data != NULL) { char *bcast = ais_concat("remove-peer", data, ':'); send_cluster_msg(crm_msg_ais, NULL, bcast); ais_info("Sent: %s", bcast); ais_free(bcast); } ais_free(data); } static void send_quorum_details(void *conn) { int size = 256; char *data = NULL; ais_malloc0(data, size); snprintf(data, size, "", membership_seq, plugin_has_quorum()?"true":"false", plugin_expected_votes, plugin_has_votes); send_client_msg(conn, crm_class_quorum, crm_msg_none, data); ais_free(data); } void pcmk_quorum(void *conn, ais_void_ptr *msg) { char *dummy = NULL; const AIS_Message *ais_msg = msg; char *data = get_ais_data(ais_msg); send_ipc_ack(conn); msg = NULL; /* Make sure the current number of votes is accurate */ dummy = pcmk_generate_membership_data(); ais_free(dummy); /* Calls without data just want the current quorum details */ if(data != NULL && strlen(data) > 0) { int value = ais_get_int(data, NULL); update_expected_votes(value); } send_quorum_details(conn); ais_free(data); } void pcmk_notify(void *conn, ais_void_ptr *msg) { const AIS_Message *ais_msg = msg; char *data = get_ais_data(ais_msg); void *async_conn = conn; int enable = 0; int sender = ais_msg->sender.pid; send_ipc_ack(conn); msg = NULL; if(ais_str_eq("true", data)) { enable = 1; } ais_info("%s node notifications for child %d (%p)", enable?"Enabling":"Disabling", sender, async_conn); if(enable) { g_hash_table_replace(membership_notify_list, async_conn, async_conn); } else { g_hash_table_remove(membership_notify_list, async_conn); } ais_free(data); } void pcmk_nodeid(void *conn, ais_void_ptr *msg) { static int counter = 0; struct crm_ais_nodeid_resp_s resp; ais_debug_2("Sending local nodeid: %d to %p[%d]", local_nodeid, conn, counter); resp.header.id = crm_class_nodeid; resp.header.size = sizeof (struct crm_ais_nodeid_resp_s); resp.header.error = CS_OK; resp.id = local_nodeid; resp.counter = counter++; memset(resp.uname, 0, MAX_NAME); memcpy(resp.uname, local_uname, local_uname_len); memset(resp.cname, 0, MAX_NAME); memcpy(resp.cname, local_cname, local_cname_len); #ifdef AIS_COROSYNC pcmk_api->ipc_response_send (conn, &resp, resp.header.size); #else openais_response_send (conn, &resp, resp.header.size); #endif } static gboolean ghash_send_update(gpointer key, gpointer value, gpointer data) { if(send_client_msg(value, crm_class_members, crm_msg_none, data) != 0) { /* remove it */ return TRUE; } return FALSE; } void send_member_notification(void) { char *update = pcmk_generate_membership_data(); ais_info("Sending membership update "U64T" to %d children", membership_seq, g_hash_table_size(membership_notify_list)); g_hash_table_foreach_remove(membership_notify_list, ghash_send_update, update); ais_free(update); } gboolean check_message_sanity(const AIS_Message *msg, const char *data) { gboolean sane = TRUE; gboolean repaired = FALSE; int dest = msg->host.type; int tmp_size = msg->header.size - sizeof(AIS_Message); if(sane && msg->header.size == 0) { ais_err("Message with no size"); sane = FALSE; } if(sane && msg->header.error != CS_OK) { ais_err("Message header contains an error: %d", msg->header.error); sane = FALSE; } AIS_CHECK(msg->header.size > sizeof(AIS_Message), ais_err("Message %d size too small: %d < %zu", msg->header.id, msg->header.size, sizeof(AIS_Message)); return FALSE); if(sane && ais_data_len(msg) != tmp_size) { ais_warn("Message payload size is incorrect: expected %d, got %d", ais_data_len(msg), tmp_size); sane = TRUE; } if(sane && ais_data_len(msg) == 0) { ais_err("Message with no payload"); sane = FALSE; } if(sane && data && msg->is_compressed == FALSE) { int str_size = strlen(data) + 1; if(ais_data_len(msg) != str_size) { int lpc = 0; ais_err("Message payload is corrupted: expected %d bytes, got %d", ais_data_len(msg), str_size); sane = FALSE; for(lpc = (str_size - 10); lpc < msg->size; lpc++) { if(lpc < 0) { lpc = 0; } ais_debug_2("bad_data[%d]: %d / '%c'", lpc, data[lpc], data[lpc]); } } } if(sane == FALSE) { AIS_CHECK(sane, ais_err("Invalid message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)", msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed, ais_data_len(msg), msg->header.size)); } else if(repaired) { ais_err("Repaired message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)", msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed, ais_data_len(msg), msg->header.size); } else { ais_debug_3("Verified message %d: (dest=%s:%s, from=%s:%s.%d, compressed=%d, size=%d, total=%d)", msg->id, ais_dest(&(msg->host)), msg_type2text(dest), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, msg->is_compressed, ais_data_len(msg), msg->header.size); } return sane; } static int delivered_transient = 0; static void deliver_transient_msg(gpointer key, gpointer value, gpointer user_data) { int pid = GPOINTER_TO_INT(value); AIS_Message *mutable = user_data; if(pid == mutable->host.type) { int rc = send_client_ipc(key, mutable); delivered_transient++; ais_info("Sent message to %s.%d (rc=%d)", ais_dest(&(mutable->host)), pid, rc); if(rc != 0) { ais_warn("Sending message to %s.%d failed (rc=%d)", ais_dest(&(mutable->host)), pid, rc); log_ais_message(LOG_DEBUG, mutable); } } } gboolean route_ais_message(const AIS_Message *msg, gboolean local_origin) { int rc = 0; int dest = msg->host.type; const char *reason = "unknown"; AIS_Message *mutable = ais_msg_copy(msg); static int service_id = SERVICE_ID_MAKE(PCMK_SERVICE_ID, 0); ais_debug_3("Msg[%d] (dest=%s:%s, from=%s:%s.%d, remote=%s, size=%d)", mutable->id, ais_dest(&(mutable->host)), msg_type2text(dest), ais_dest(&(mutable->sender)), msg_type2text(mutable->sender.type), mutable->sender.pid, local_origin?"false":"true", ais_data_len((mutable))); if(local_origin == FALSE) { if(mutable->host.size == 0 || ais_str_eq(local_uname, mutable->host.uname)) { mutable->host.local = TRUE; } } if(check_message_sanity(mutable, mutable->data) == FALSE) { /* Dont send this message to anyone */ rc = 1; goto bail; } if(mutable->host.local) { void *conn = NULL; const char *lookup = NULL; if(dest == crm_msg_ais) { process_ais_message(mutable); goto bail; } else if(dest == crm_msg_lrmd) { /* lrmd messages are routed via the crm */ dest = crm_msg_crmd; } else if(dest == crm_msg_te) { /* te messages are routed via the crm */ dest = crm_msg_crmd; } else if(dest >= SIZEOF(pcmk_children)) { /* Transient client */ delivered_transient = 0; g_hash_table_foreach(ipc_client_list, deliver_transient_msg, mutable); if(delivered_transient) { ais_debug_2("Sent message to %d transient clients: %d", delivered_transient, dest); goto bail; } else { /* try the crmd */ ais_debug_2("Sending message to transient client %d via crmd", dest); dest = crm_msg_crmd; } } else if(dest == 0) { ais_err("Invalid destination: %d", dest); log_ais_message(LOG_ERR, mutable); log_printf(LOG_ERR, "%s", get_ais_data(mutable)); rc = 1; goto bail; } lookup = msg_type2text(dest); conn = pcmk_children[dest].async_conn; /* the cluster fails in weird and wonderfully obscure ways when this is not true */ AIS_ASSERT(ais_str_eq(lookup, pcmk_children[dest].name)); if(mutable->header.id == service_id) { mutable->header.id = 0; /* reset this back to zero for IPC messages */ } else if(mutable->header.id != 0) { ais_err("reset header id back to zero from %d", mutable->header.id); mutable->header.id = 0; /* reset this back to zero for IPC messages */ } reason = "ipc delivery failed"; rc = send_client_ipc(conn, mutable); } else if(local_origin) { /* forward to other hosts */ ais_debug_3("Forwarding to cluster"); reason = "cluster delivery failed"; rc = send_cluster_msg_raw(mutable); } if(rc != 0) { ais_warn("Sending message to %s.%s failed: %s (rc=%d)", ais_dest(&(mutable->host)), msg_type2text(dest), reason, rc); log_ais_message(LOG_DEBUG, mutable); } bail: ais_free(mutable); return rc==0?TRUE:FALSE; } int send_cluster_msg_raw(const AIS_Message *ais_msg) { int rc = 0; struct iovec iovec; static uint32_t msg_id = 0; AIS_Message *mutable = ais_msg_copy(ais_msg); AIS_ASSERT(local_nodeid != 0); AIS_ASSERT(ais_msg->header.size == (sizeof(AIS_Message) + ais_data_len(ais_msg))); if(mutable->id == 0) { msg_id++; AIS_CHECK(msg_id != 0 /* detect wrap-around */, msg_id++; ais_err("Message ID wrapped around")); mutable->id = msg_id; } mutable->header.error = CS_OK; mutable->header.id = SERVICE_ID_MAKE(PCMK_SERVICE_ID, 0); mutable->sender.id = local_nodeid; mutable->sender.size = local_uname_len; memset(mutable->sender.uname, 0, MAX_NAME); memcpy(mutable->sender.uname, local_uname, mutable->sender.size); iovec.iov_base = (char *)mutable; iovec.iov_len = mutable->header.size; ais_debug_3("Sending message (size=%u)", (unsigned int)iovec.iov_len); #if AIS_COROSYNC rc = pcmk_api->totem_mcast(&iovec, 1, TOTEMPG_SAFE); #else rc = totempg_groups_mcast_joined(openais_group_handle, &iovec, 1, TOTEMPG_SAFE); #endif if(rc == 0 && mutable->is_compressed == FALSE) { ais_debug_2("Message sent: %.80s", mutable->data); } AIS_CHECK(rc == 0, ais_err("Message not sent (%d): %.120s", rc, mutable->data)); ais_free(mutable); return rc; } #define min(x,y) (x)<(y)?(x):(y) void send_cluster_id(void) { int rc = 0; int lpc = 0; int len = 0; time_t now = time(NULL); struct iovec iovec; struct crm_identify_msg_s *msg = NULL; static time_t started = 0; static uint64_t first_seq = 0; AIS_ASSERT(local_nodeid != 0); if(started == 0) { started = now; first_seq = membership_seq; } if(local_born_on == 0) { if(started + 15 < now) { ais_debug("Born-on set to: "U64T" (age)", first_seq); local_born_on = first_seq; } else if(have_reliable_membership_id) { ais_debug("Born-on set to: "U64T" (peer)", membership_seq); local_born_on = membership_seq; } else { ais_debug("Leaving born-on unset: "U64T, membership_seq); } } ais_malloc0(msg, sizeof(struct crm_identify_msg_s)); msg->header.size = sizeof(struct crm_identify_msg_s); msg->id = local_nodeid; /* msg->header.error = CS_OK; */ msg->header.id = SERVICE_ID_MAKE(PCMK_SERVICE_ID, 1); len = min(local_uname_len, MAX_NAME-1); memset(msg->uname, 0, MAX_NAME); memcpy(msg->uname, local_uname, len); len = min(strlen(VERSION), MAX_NAME-1); memset(msg->version, 0, MAX_NAME); memcpy(msg->version, VERSION, len); msg->votes = 1; msg->pid = getpid(); msg->processes = get_process_list(); msg->born_on = local_born_on; ais_debug("Local update: id=%u, born="U64T", seq="U64T"", local_nodeid, local_born_on, membership_seq); update_member( local_nodeid, local_born_on, membership_seq, msg->votes, msg->processes, NULL, NULL, VERSION); iovec.iov_base = (char *)msg; iovec.iov_len = msg->header.size; #if AIS_COROSYNC rc = pcmk_api->totem_mcast(&iovec, 1, TOTEMPG_SAFE); #else rc = totempg_groups_mcast_joined(openais_group_handle, &iovec, 1, TOTEMPG_SAFE); #endif AIS_CHECK(rc == 0, ais_err("Message not sent (%d)", rc)); ais_free(msg); } static gboolean ghash_send_removal(gpointer key, gpointer value, gpointer data) { send_quorum_details(value); if(send_client_msg(value, crm_class_rmpeer, crm_msg_none, data) != 0) { /* remove it */ return TRUE; } return FALSE; } void ais_remove_peer(char *node_id) { uint32_t id = ais_get_int(node_id, NULL); crm_node_t *node = g_hash_table_lookup(membership_list, GUINT_TO_POINTER(id)); if(node == NULL) { ais_info("Peer %u is unknown", id); } else if(ais_str_eq(CRM_NODE_MEMBER, node->state)) { ais_warn("Peer %u/%s is still active", id, node->uname); } else if(g_hash_table_remove(membership_list, GUINT_TO_POINTER(id))) { plugin_expected_votes--; ais_notice("Removed dead peer %u from the membership list", id); ais_info("Sending removal of %u to %d children", id, g_hash_table_size(membership_notify_list)); g_hash_table_foreach_remove(membership_notify_list, ghash_send_removal, node_id); } else { ais_warn("Peer %u/%s was not removed", id, node->uname); } } gboolean process_ais_message(const AIS_Message *msg) { int len = ais_data_len(msg); char *data = get_ais_data(msg); do_ais_log(LOG_DEBUG, "Msg[%d] (dest=%s:%s, from=%s:%s.%d, remote=%s, size=%d): %.90s", msg->id, ais_dest(&(msg->host)), msg_type2text(msg->host.type), ais_dest(&(msg->sender)), msg_type2text(msg->sender.type), msg->sender.pid, msg->sender.uname==local_uname?"false":"true", ais_data_len(msg), data); if(data && len > 12 && strncmp("remove-peer:", data, 12) == 0) { char *node = data+12; ais_remove_peer(node); } ais_free(data); return TRUE; } static void member_dump_fn(gpointer key, gpointer value, gpointer user_data) { crm_node_t *node = value; ais_info(" node id:%u, uname=%s state=%s processes=%.16x born="U64T" seen="U64T" addr=%s version=%s", node->id, node->uname?node->uname:"-unknown-", node->state, node->processes, node->born, node->last_seen, node->addr?node->addr:"-unknown-", node->version?node->version:"-unknown-"); } void pcmk_exec_dump(void) { /* Called after SIG_USR2 */ process_ais_conf(); ais_info("Local id: %u, uname: %s, born: "U64T, local_nodeid, local_uname, local_born_on); ais_info("Membership id: "U64T", quorate: %s, expected: %u, actual: %u", membership_seq, plugin_has_quorum()?"true":"false", plugin_expected_votes, plugin_has_votes); g_hash_table_foreach(membership_list, member_dump_fn, NULL); } diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c index 8b53007a0c..1d26f61bc5 100644 --- a/lib/fencing/st_client.c +++ b/lib/fencing/st_client.c @@ -1,1591 +1,1593 @@ /* * Copyright (c) 2004 Andrew Beekhof * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define FE_AGENT_FORK -2 #define FE_AGENT_ERROR -3 typedef struct stonith_private_s { char *token; IPC_Channel *command_channel; IPC_Channel *callback_channel; GCHSource *callback_source; GHashTable *stonith_op_callback_table; void (*op_callback)( stonith_t *st, const xmlNode *msg, int call, int rc, xmlNode *output, void *userdata); } stonith_private_t; typedef struct stonith_notify_client_s { const char *event; const char *obj_id; /* implement one day */ const char *obj_type; /* implement one day */ void (*notify)(stonith_t *st, const char *event, xmlNode *msg); } stonith_notify_client_t; typedef struct stonith_callback_client_s { void (*callback)( stonith_t *st, const xmlNode *msg, int call, int rc, xmlNode *output, void *userdata); const char *id; void *user_data; gboolean only_success; struct timer_rec_s *timer; } stonith_callback_client_t; struct notify_blob_s { stonith_t *stonith; xmlNode *xml; }; struct timer_rec_s { int call_id; int timeout; guint ref; stonith_t *stonith; }; typedef enum stonith_errors (*stonith_op_t)( const char *, int, const char *, xmlNode *, xmlNode*, xmlNode*, xmlNode**, xmlNode**); static const char META_TEMPLATE[] = "\n" "\n" "\n" " 1.0\n" " \n" "%s\n" " \n" " %s\n" "%s\n" " \n" " \n" " \n" " \n" " \n" " \n" " \n" " \n" " 2.0\n" " \n" "\n"; gboolean stonith_dispatch(IPC_Channel *channel, gpointer user_data); void stonith_perform_callback(stonith_t *stonith, xmlNode *msg, int call_id, int rc); xmlNode *stonith_create_op( int call_id, const char *token, const char *op, xmlNode *data, int call_options); int stonith_send_command( stonith_t *stonith, const char *op, xmlNode *data, xmlNode **output_data, int call_options, int timeout); static void stonith_connection_destroy(gpointer user_data); static void stonith_send_notification(gpointer data, gpointer user_data); static void stonith_connection_destroy(gpointer user_data) { stonith_t *stonith = user_data; stonith_private_t *native = NULL; struct notify_blob_s blob; blob.stonith = stonith; blob.xml = create_xml_node(NULL, "notify");; native = stonith->private; native->callback_source = NULL; stonith->state = stonith_disconnected; crm_xml_add(blob.xml, F_TYPE, T_STONITH_NOTIFY); crm_xml_add(blob.xml, F_SUBTYPE, T_STONITH_NOTIFY_DISCONNECT); g_list_foreach(stonith->notify_list, stonith_send_notification, &blob); free_xml(blob.xml); } static int stonith_api_register_device( stonith_t *stonith, int call_options, const char *id, const char *namespace, const char *agent, GHashTable *params) { int rc = 0; xmlNode *data = create_xml_node(NULL, F_STONITH_DEVICE); xmlNode *args = create_xml_node(data, XML_TAG_ATTRS); crm_xml_add(data, XML_ATTR_ID, id); crm_xml_add(data, "origin", __FUNCTION__); crm_xml_add(data, "agent", agent); crm_xml_add(data, "namespace", namespace); g_hash_table_foreach(params, hash2field, args); rc = stonith_send_command(stonith, STONITH_OP_DEVICE_ADD, data, NULL, call_options, 0); free_xml(data); return rc; } static int stonith_api_remove_device( stonith_t *stonith, int call_options, const char *name) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, F_STONITH_DEVICE); crm_xml_add(data, "origin", __FUNCTION__); crm_xml_add(data, XML_ATTR_ID, name); rc = stonith_send_command(stonith, STONITH_OP_DEVICE_DEL, data, NULL, call_options, 0); free_xml(data); return rc; } static void append_arg( gpointer key, gpointer value, gpointer user_data) { int len = 3; /* =, \n, \0 */ int last = 0; char **args = user_data; CRM_CHECK(key != NULL, return); CRM_CHECK(value != NULL, return); if(strstr(key, "pcmk_")) { return; } else if(strstr(key, CRM_META)) { return; } else if(safe_str_eq(key, "crm_feature_set")) { return; } len += strlen(key); len += strlen(value); if(*args != NULL) { last = strlen(*args); } crm_realloc(*args, last+len); crm_debug_2("Appending: %s=%s", (char *)key, (char *)value); sprintf((*args)+last, "%s=%s\n", (char *)key, (char *)value); } static void append_const_arg(const char *key, const char *value, char **arg_list) { char *glib_sucks_key = crm_strdup(key); char *glib_sucks_value = crm_strdup(value); append_arg(glib_sucks_key, glib_sucks_value, arg_list); crm_free(glib_sucks_value); crm_free(glib_sucks_key); } static void append_host_specific_args(const char *victim, const char *map, GHashTable *params, char **arg_list) { char *name = NULL; int last = 0, lpc = 0, max = 0; if(map == NULL) { /* The best default there is for now... */ crm_debug("Using default arg map: port=uname"); append_const_arg("port", victim, arg_list); return; } max = strlen(map); crm_debug("Processing arg map: %s", map); for(; lpc < max + 1; lpc++) { if(isalpha(map[lpc])) { /* keep going */ } else if(map[lpc] == '=' || map[lpc] == ':') { crm_malloc0(name, 1 + lpc - last); strncpy(name, map + last, lpc - last); crm_debug("Got name: %s", name); last = lpc + 1; } else if(map[lpc] == 0 || map[lpc] == ',' || isspace(map[lpc])) { char *param = NULL; const char *value = NULL; crm_malloc0(param, 1 + lpc - last); strncpy(param, map + last, lpc - last); last = lpc + 1; crm_debug("Got key: %s", param); if(name == NULL) { crm_err("Misparsed '%s', found '%s' without a name", map, param); crm_free(param); continue; } if(safe_str_eq(param, "uname")) { value = victim; } else { char *key = crm_meta_name(param); value = g_hash_table_lookup(params, key); crm_free(key); } if(value) { crm_debug("Setting '%s'='%s' (%s) for %s", name, value, param, victim); append_const_arg(name, value, arg_list); } else { crm_err("No node attribute '%s' for '%s'", name, victim); } crm_free(name); name=NULL; crm_free(param); if(map[lpc] == 0) { break; } } else if(isspace(map[lpc])) { last = lpc; } } } static char *make_args(GHashTable *dev_hash, GHashTable *node_hash, const char *action, const char *victim) { char *arg_list = NULL; const char *map = NULL; CRM_CHECK(action != NULL, return NULL); if(dev_hash) { map = g_hash_table_lookup(dev_hash, STONITH_ATTR_ARGMAP); g_hash_table_foreach(dev_hash, append_arg, &arg_list); } append_const_arg(STONITH_ATTR_ACTION_OP, action, &arg_list); if(victim) { append_const_arg("nodename", victim, &arg_list); append_host_specific_args(victim, map, node_hash, &arg_list); } crm_debug_3("Calculated: %s", arg_list); return arg_list; } /* Borrowed from libfence and extended */ int run_stonith_agent( const char *agent, GHashTable *dev_hash, GHashTable *node_hash, const char *action, const char *victim, int *agent_result, char **output, async_command_t *track) { char *args = make_args(dev_hash, node_hash, action, victim); int pid, status, len, rc = -1; int p_read_fd, p_write_fd; /* parent read/write file descriptors */ int c_read_fd, c_write_fd; /* child read/write file descriptors */ int fd1[2]; int fd2[2]; c_read_fd = c_write_fd = p_read_fd = p_write_fd = -1; if (args == NULL || agent == NULL) goto fail; len = strlen(args); if (pipe(fd1)) goto fail; p_read_fd = fd1[0]; c_write_fd = fd1[1]; if (pipe(fd2)) goto fail; c_read_fd = fd2[0]; p_write_fd = fd2[1]; crm_debug("forking"); pid = fork(); if (pid < 0) { *agent_result = FE_AGENT_FORK; goto fail; } if (pid) { /* parent */ int ret; fcntl(p_read_fd, F_SETFL, fcntl(p_read_fd, F_GETFL, 0) | O_NONBLOCK); do { crm_debug("sending args"); ret = write(p_write_fd, args, len); } while (ret < 0 && errno == EINTR); if (ret != len) { if(ret >= 0) { rc = st_err_generic; } goto fail; } close(p_write_fd); if(track) { NewTrackedProc(pid, 0, PT_LOGNORMAL, track, track->pt_ops); #if 0 ProcTrackKillInfo *info = NULL; crm_malloc0(info, sizeof(ProcTrackKillInfo) * 3); killseq[0].mstimeout = timeout; /* after timeout send TERM */ killseq[0].signalno = SIGTERM; killseq[1].mstimeout = 5000; /* after 5 secs remove it */ killseq[1].signalno = SIGKILL; killseq[2].mstimeout = 5000; /* if it's still there after 5, complain */ killseq[2].signalno = 0; SetTrackedProcTimeouts(pid,killseq); #endif track->stdout = p_read_fd; crm_free(args); close(c_write_fd); close(c_read_fd); return pid; } else { waitpid(pid, &status, 0); if(output != NULL) { len = 0; do { char buf[500]; ret = read(p_read_fd, buf, 500); if(ret > 0) { buf[ret] = 0; crm_realloc(*output, len + ret + 1); sprintf((*output)+len, "%s", buf); crm_debug("%d: %s", ret, (*output)+len); len += ret; } } while (ret == 500 || (ret < 0 && errno == EINTR)); } *agent_result = FE_AGENT_ERROR; if (WIFEXITED(status)) { crm_debug("result = %d", WEXITSTATUS(status)); *agent_result = -WEXITSTATUS(status); rc = 0; } if(node_hash) { g_hash_table_destroy(node_hash); } } } else { /* child */ close(1); if (dup(c_write_fd) < 0) goto fail; close(2); if (dup(c_write_fd) < 0) goto fail; close(0); if (dup(c_read_fd) < 0) goto fail; /* keep c_write_fd open so parent can report all errors. */ close(c_read_fd); close(p_read_fd); close(p_write_fd); execlp(agent, agent, NULL); exit(EXIT_FAILURE); } fail: crm_free(args); close(p_read_fd); close(p_write_fd); close(c_read_fd); close(c_write_fd); return rc; } static int stonith_api_device_metadata( stonith_t *stonith, int call_options, const char *agent, const char *namespace, char **output, int timeout) { int rc = 0; int bufferlen = 0; char *buffer = NULL; char *xml_meta_longdesc = NULL; char *xml_meta_shortdesc = NULL; char *meta_param = NULL; char *meta_longdesc = NULL; char *meta_shortdesc = NULL; const char *provider = get_stonith_provider(agent, namespace); Stonith *stonith_obj = NULL; static const char *no_parameter_info = ""; crm_info("looking up %s/%s metadata", agent, provider); /* By having this in a library, we can access it from stonith_admin * when neither lrmd or stonith-ng are running * Important for the crm shell's validations... */ if(safe_str_eq(provider, "redhat")) { int exec_rc = run_stonith_agent( agent, NULL, NULL, "metadata", NULL, &rc, &buffer, NULL); if(exec_rc < 0 || rc != 0 || buffer == NULL) { /* failed */ crm_debug("Query failed: %d %d: %s", exec_rc, rc, crm_str(buffer)); /* provide a fake metadata entry */ meta_longdesc = crm_strdup(no_parameter_info); meta_shortdesc = crm_strdup(no_parameter_info); meta_param = crm_strdup( " \n" " \n" " \n" " \n" " Fencing action (null, off, on, [reboot], status, hostlist, devstatus)\n" " \n" " "); goto build; } } else { stonith_obj = stonith_new(agent); meta_longdesc = crm_strdup(stonith_get_info(stonith_obj, ST_DEVICEDESCR)); if (meta_longdesc == NULL) { crm_warn("no long description in %s's metadata.", agent); meta_longdesc = crm_strdup(no_parameter_info); } meta_shortdesc = crm_strdup(stonith_get_info(stonith_obj, ST_DEVICENAME)); crm_info("short description: %s", meta_shortdesc); if (meta_shortdesc == NULL) { crm_warn("no short description in %s's metadata.", agent); meta_shortdesc = crm_strdup(no_parameter_info); } meta_param = crm_strdup(stonith_get_info(stonith_obj, ST_CONF_XML)); if (meta_param == NULL) { crm_warn("no list of parameters in %s's metadata.", agent); meta_param = crm_strdup(no_parameter_info); } build: xml_meta_longdesc = (char *)xmlEncodeEntitiesReentrant(NULL, (const unsigned char *)meta_longdesc); xml_meta_shortdesc = (char *)xmlEncodeEntitiesReentrant(NULL, (const unsigned char *)meta_shortdesc); bufferlen = strlen(META_TEMPLATE) + strlen(agent) + strlen(xml_meta_longdesc) + strlen(xml_meta_shortdesc) + strlen(meta_param) + 1; crm_malloc0(buffer, bufferlen); snprintf(buffer, bufferlen-1, META_TEMPLATE, agent, xml_meta_longdesc, xml_meta_shortdesc, meta_param); xmlFree(xml_meta_longdesc); xmlFree(xml_meta_shortdesc); if(stonith_obj) { stonith_delete(stonith_obj); } crm_free(meta_shortdesc); crm_free(meta_longdesc); crm_free(meta_param); } if(output) { *output = buffer; } else { crm_free(buffer); } return rc; } static int stonith_api_query( stonith_t *stonith, int call_options, const char *target, GListPtr *devices, int timeout) { int rc = 0, lpc = 0, max = 0; xmlNode *data = NULL; xmlNode *output = NULL; xmlXPathObjectPtr xpathObj = NULL; CRM_CHECK(devices != NULL, return st_err_missing); data = create_xml_node(NULL, F_STONITH_DEVICE); crm_xml_add(data, "origin", __FUNCTION__); crm_xml_add(data, F_STONITH_TARGET, target); rc = stonith_send_command(stonith, STONITH_OP_QUERY, data, &output, call_options, timeout); if(rc < 0) { return rc; } xpathObj = xpath_search(output, "//@agent"); - max = xpathObj->nodesetval->nodeNr; + if(xpathObj) { + max = xpathObj->nodesetval->nodeNr; - for(lpc = 0; lpc < max; lpc++) { - xmlNode *match = getXpathResult(xpathObj, lpc); - CRM_CHECK(match != NULL, continue); - - crm_info("%s[%d] = %s", "//@agent", lpc, xmlGetNodePath(match)); - *devices = g_list_append(*devices, crm_element_value_copy(match, XML_ATTR_ID)); + for(lpc = 0; lpc < max; lpc++) { + xmlNode *match = getXpathResult(xpathObj, lpc); + CRM_CHECK(match != NULL, continue); + + crm_info("%s[%d] = %s", "//@agent", lpc, xmlGetNodePath(match)); + *devices = g_list_append(*devices, crm_element_value_copy(match, XML_ATTR_ID)); + } } - + free_xml(output); free_xml(data); return max; } static int stonith_api_call( stonith_t *stonith, int call_options, const char *id, const char *action, const char *victim, int timeout) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, F_STONITH_DEVICE); crm_xml_add(data, "origin", __FUNCTION__); crm_xml_add(data, F_STONITH_DEVICE, id); crm_xml_add(data, F_STONITH_ACTION, action); crm_xml_add(data, F_STONITH_TARGET, victim); crm_xml_add_int(data, "timeout", timeout); rc = stonith_send_command(stonith, STONITH_OP_EXEC, data, NULL, call_options, timeout); free_xml(data); return rc; } static int stonith_api_fence( stonith_t *stonith, int call_options, const char *node, GHashTable *parameters, const char *action, int timeout) { int rc = 0; xmlNode *data = NULL; xmlNode *params = NULL; data = create_xml_node(NULL, __FUNCTION__); crm_xml_add(data, F_STONITH_TARGET, node); crm_xml_add(data, F_STONITH_ACTION, action); crm_xml_add_int(data, "timeout", timeout); params = create_xml_node(data, XML_TAG_ATTRS); g_hash_table_foreach(parameters, hash2nvpair, params); rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, timeout); free_xml(data); return rc; } const char * stonith_error2string(enum stonith_errors return_code) { const char *error_msg = NULL; switch(return_code) { case stonith_ok: error_msg = "OK"; break; case st_err_not_supported: error_msg = "Not supported"; break; case st_err_authentication: error_msg = "Not authenticated"; break; case st_err_generic: error_msg = "Generic error"; break; case st_err_internal: error_msg = "Internal error"; break; case st_err_unknown_device: error_msg = "Unknown device"; break; case st_err_unknown_operation: error_msg = "Unknown operation"; break; case st_err_unknown_port: error_msg = "Unknown victim"; break; case st_err_none_available: error_msg = "No available fencing devices"; break; case st_err_connection: error_msg = "Not connected"; break; case st_err_missing: error_msg = "Missing input"; break; case st_err_exists: error_msg = "Device exists"; break; case st_err_timeout: error_msg = "Operation timed out"; break; case st_err_signal: error_msg = "Killed by signal"; break; case st_err_ipc: error_msg = "IPC connection failed"; break; case st_err_peer: error_msg = "Error from peer"; break; } if(error_msg == NULL) { crm_err("Unknown Stonith error code: %d", return_code); error_msg = ""; } return error_msg; } gboolean is_redhat_agent(const char *agent) { int rc = 0; struct stat prop; char buffer[FILENAME_MAX+1]; snprintf(buffer,FILENAME_MAX,"%s/%s", RH_STONITH_DIR, agent); rc = stat(buffer, &prop); if (rc >= 0 && S_ISREG(prop.st_mode)) { return TRUE; } return FALSE; } const char *get_stonith_provider(const char *agent, const char *provider) { /* This function sucks */ if(is_redhat_agent(agent)) { return "redhat"; } else { Stonith *stonith_obj = stonith_new(agent); if(stonith_obj) { stonith_delete(stonith_obj); return "heartbeat"; } } crm_err("No such device: %s", agent); return NULL; } static gint stonithlib_GCompareFunc(gconstpointer a, gconstpointer b) { int rc = 0; const stonith_notify_client_t *a_client = a; const stonith_notify_client_t *b_client = b; CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0); rc = strcmp(a_client->event, b_client->event); if(rc == 0) { if(a_client->notify == NULL || b_client->notify == NULL) { return 0; } else if(a_client->notify == b_client->notify) { return 0; } else if(((long)a_client->notify) < ((long)b_client->notify)) { crm_err("callbacks for %s are not equal: %p vs. %p", a_client->event, a_client->notify, b_client->notify); return -1; } crm_err("callbacks for %s are not equal: %p vs. %p", a_client->event, a_client->notify, b_client->notify); return 1; } return rc; } static int get_stonith_token(IPC_Channel *ch, char **token) { int rc = stonith_ok; xmlNode *reg_msg = NULL; const char *msg_type = NULL; const char *tmp_ticket = NULL; CRM_CHECK(ch != NULL, return st_err_missing); CRM_CHECK(token != NULL, return st_err_missing); crm_debug_4("Waiting for msg on command channel"); reg_msg = xmlfromIPC(ch, MAX_IPC_DELAY); if(ch->ops->get_chan_status(ch) != IPC_CONNECT) { crm_err("No reply message - disconnected"); free_xml(reg_msg); return st_err_connection; } else if(reg_msg == NULL) { crm_err("No reply message - empty"); return st_err_ipc; } msg_type = crm_element_value(reg_msg, F_STONITH_OPERATION); tmp_ticket = crm_element_value(reg_msg, F_STONITH_CLIENTID); if(safe_str_neq(msg_type, CRM_OP_REGISTER) ) { crm_err("Invalid registration message: %s", msg_type); rc = st_err_internal; } else if(tmp_ticket == NULL) { crm_err("No registration token provided"); crm_log_xml_warn(reg_msg, "Bad reply") rc = st_err_internal; } else { crm_debug("Obtained registration token: %s", tmp_ticket); *token = crm_strdup(tmp_ticket); } free_xml(reg_msg); return rc; } xmlNode *stonith_create_op( int call_id, const char *token, const char *op, xmlNode *data, int call_options) { int rc = HA_OK; xmlNode *op_msg = create_xml_node(NULL, "stonith_command"); CRM_CHECK(op_msg != NULL, return NULL); CRM_CHECK(token != NULL, return NULL); crm_xml_add(op_msg, F_XML_TAGNAME, "stonith_command"); crm_xml_add(op_msg, F_TYPE, T_STONITH_NG); crm_xml_add(op_msg, F_STONITH_CALLBACK_TOKEN, token); crm_xml_add(op_msg, F_STONITH_OPERATION, op); crm_xml_add_int(op_msg, F_STONITH_CALLID, call_id); crm_debug_4("Sending call options: %.8lx, %d", (long)call_options, call_options); crm_xml_add_int(op_msg, F_STONITH_CALLOPTS, call_options); if(data != NULL) { add_message_xml(op_msg, F_STONITH_CALLDATA, data); } if (rc != HA_OK) { crm_err("Failed to create STONITH operation message"); crm_log_xml(LOG_ERR, "BadOp", op_msg); free_xml(op_msg); return NULL; } return op_msg; } static void stonith_destroy_op_callback(gpointer data) { stonith_callback_client_t *blob = data; if(blob->timer && blob->timer->ref > 0) { g_source_remove(blob->timer->ref); } crm_free(blob->timer); crm_free(blob); } static int stonith_api_signoff(stonith_t* stonith) { stonith_private_t *native = stonith->private; crm_debug("Signing out of the STONITH Service"); /* close channels */ if (native->command_channel != NULL) { native->command_channel->ops->destroy( native->command_channel); native->command_channel = NULL; } if (native->callback_source != NULL) { G_main_del_IPC_Channel(native->callback_source); native->callback_source = NULL; } if (native->callback_channel != NULL) { #ifdef BUG native->callback_channel->ops->destroy( native->callback_channel); #endif native->callback_channel = NULL; } stonith->state = stonith_disconnected; return stonith_ok; } static int stonith_api_signon( stonith_t* stonith, const char *name, int *async_fd, int *sync_fd) { int rc = stonith_ok; xmlNode *hello = NULL; char *uuid_ticket = NULL; stonith_private_t *native = stonith->private; crm_debug_4("Connecting command channel"); stonith->state = stonith_connected_command; native->command_channel = init_client_ipc_comms_nodispatch(stonith_channel); if(native->command_channel == NULL) { crm_debug("Connection to command channel failed"); rc = st_err_connection; } else if(native->command_channel->ch_status != IPC_CONNECT) { crm_err("Connection may have succeeded," " but authentication to command channel failed"); rc = st_err_authentication; } if(rc == stonith_ok) { rc = get_stonith_token(native->command_channel, &uuid_ticket); if(rc == stonith_ok) { native->token = uuid_ticket; uuid_ticket = NULL; } else { stonith->state = stonith_disconnected; native->command_channel->ops->disconnect(native->command_channel); return rc; } } native->callback_channel = init_client_ipc_comms_nodispatch( stonith_channel_callback); if(native->callback_channel == NULL) { crm_debug("Connection to callback channel failed"); rc = st_err_connection; } else if(native->callback_channel->ch_status != IPC_CONNECT) { crm_err("Connection may have succeeded," " but authentication to command channel failed"); rc = st_err_authentication; } if(rc == stonith_ok) { native->callback_channel->send_queue->max_qlen = 500; rc = get_stonith_token(native->callback_channel, &uuid_ticket); if(rc == stonith_ok) { crm_free(native->token); native->token = uuid_ticket; } } if(rc == stonith_ok) { CRM_CHECK(native->token != NULL, ;); hello = stonith_create_op(0, native->token, CRM_OP_REGISTER, NULL, 0); crm_xml_add(hello, F_STONITH_CLIENTNAME, name); if(send_ipc_message(native->command_channel, hello) == FALSE) { rc = st_err_internal; } free_xml(hello); } if(rc == stonith_ok) { gboolean do_mainloop = TRUE; if(async_fd != NULL) { do_mainloop = FALSE; *async_fd = native->callback_channel->ops->get_recv_select_fd(native->callback_channel); } if(sync_fd != NULL) { do_mainloop = FALSE; *sync_fd = native->callback_channel->ops->get_send_select_fd(native->callback_channel); } if(do_mainloop) { crm_debug_4("Connecting callback channel"); native->callback_source = G_main_add_IPC_Channel( G_PRIORITY_HIGH, native->callback_channel, FALSE, stonith_dispatch, stonith, default_ipc_connection_destroy); if(native->callback_source == NULL) { crm_err("Callback source not recorded"); rc = st_err_connection; } else { set_IPC_Channel_dnotify( native->callback_source, stonith_connection_destroy); } } } if(rc == stonith_ok) { #if HAVE_MSGFROMIPC_TIMEOUT stonith->call_timeout = MAX_IPC_DELAY; #endif crm_debug("Connection to STONITH successful"); return stonith_ok; } crm_debug("Connection to STONITH failed: %s", stonith_error2string(rc)); stonith->cmds->disconnect(stonith); return rc; } static int stonith_set_notification(stonith_t* stonith, const char *callback, int enabled) { xmlNode *notify_msg = create_xml_node(NULL, __FUNCTION__); stonith_private_t *native = stonith->private; if(stonith->state != stonith_disconnected) { crm_xml_add(notify_msg, F_STONITH_OPERATION, T_STONITH_NOTIFY); if(enabled) { crm_xml_add(notify_msg, F_STONITH_NOTIFY_ACTIVATE, callback); } else { crm_xml_add(notify_msg, F_STONITH_NOTIFY_DEACTIVATE, callback); } send_ipc_message(native->callback_channel, notify_msg); } free_xml(notify_msg); return stonith_ok; } static int stonith_api_add_notification( stonith_t *stonith, const char *event, void (*callback)(stonith_t *stonith, const char *event, xmlNode *msg)) { GList *list_item = NULL; stonith_notify_client_t *new_client = NULL; crm_debug_2("Adding callback for %s events (%d)", event, g_list_length(stonith->notify_list)); crm_malloc0(new_client, sizeof(stonith_notify_client_t)); new_client->event = event; new_client->notify = callback; list_item = g_list_find_custom( stonith->notify_list, new_client, stonithlib_GCompareFunc); if(list_item != NULL) { crm_warn("Callback already present"); crm_free(new_client); return st_err_exists; } else { stonith->notify_list = g_list_append( stonith->notify_list, new_client); stonith_set_notification(stonith, event, 1); crm_debug_3("Callback added (%d)", g_list_length(stonith->notify_list)); } return stonith_ok; } static int stonith_api_del_notification(stonith_t *stonith, const char *event) { GList *list_item = NULL; stonith_notify_client_t *new_client = NULL; crm_debug("Removing callback for %s events", event); crm_malloc0(new_client, sizeof(stonith_notify_client_t)); new_client->event = event; new_client->notify = NULL; list_item = g_list_find_custom( stonith->notify_list, new_client, stonithlib_GCompareFunc); stonith_set_notification(stonith, event, 0); if(list_item != NULL) { stonith_notify_client_t *list_client = list_item->data; stonith->notify_list = g_list_remove(stonith->notify_list, list_client); crm_free(list_client); crm_debug_3("Removed callback"); } else { crm_debug_3("Callback not present"); } crm_free(new_client); return stonith_ok; } static gboolean stonith_async_timeout_handler(gpointer data) { struct timer_rec_s *timer = data; crm_debug("Async call %d timed out after %dms", timer->call_id, timer->timeout); stonith_perform_callback(timer->stonith, NULL, timer->call_id, st_err_timeout); /* Always return TRUE, never remove the handler * We do that in stonith_del_callback() */ return TRUE; } static int stonith_api_add_callback( stonith_t *stonith, int call_id, int timeout, gboolean only_success, void *user_data, const char *callback_name, void (*callback)( stonith_t *st, const xmlNode *msg, int call, int rc, xmlNode *output, void *userdata)) { stonith_callback_client_t *blob = NULL; stonith_private_t *private = NULL; CRM_CHECK(stonith != NULL, return st_err_missing); CRM_CHECK(stonith->private != NULL, return st_err_missing); private = stonith->private; if(call_id == 0) { private->op_callback = callback; } else if(call_id < 0) { if(only_success == FALSE) { callback(stonith, NULL, call_id, call_id, NULL, user_data); } else { crm_warn("STONITH call failed: %s", stonith_error2string(call_id)); } return FALSE; } crm_malloc0(blob, sizeof(stonith_callback_client_t)); blob->id = callback_name; blob->only_success = only_success; blob->user_data = user_data; blob->callback = callback; if(timeout > 0) { struct timer_rec_s *async_timer = NULL; crm_malloc0(async_timer, sizeof(struct timer_rec_s)); blob->timer = async_timer; async_timer->stonith = stonith; async_timer->call_id = call_id; async_timer->timeout = timeout*1100; async_timer->ref = g_timeout_add( async_timer->timeout, stonith_async_timeout_handler, async_timer); } g_hash_table_insert(private->stonith_op_callback_table, GINT_TO_POINTER(call_id), blob); return TRUE; } static int stonith_api_del_callback(stonith_t *stonith, int call_id, gboolean all_callbacks) { stonith_private_t *private = stonith->private; if(all_callbacks) { private->op_callback = NULL; g_hash_table_destroy(private->stonith_op_callback_table); private->stonith_op_callback_table = g_hash_table_new_full( g_direct_hash, g_direct_equal, NULL, stonith_destroy_op_callback); } else if(call_id == 0) { private->op_callback = NULL; } else { g_hash_table_remove(private->stonith_op_callback_table, GINT_TO_POINTER(call_id)); } return stonith_ok; } static void stonith_dump_pending_op( gpointer key, gpointer value, gpointer user_data) { int call = GPOINTER_TO_INT(key); stonith_callback_client_t *blob = value; crm_debug("Call %d (%s): pending", call, crm_str(blob->id)); } void stonith_dump_pending_callbacks(stonith_t *stonith) { stonith_private_t *private = stonith->private; if(private->stonith_op_callback_table == NULL) { return; } return g_hash_table_foreach( private->stonith_op_callback_table, stonith_dump_pending_op, NULL); } void stonith_perform_callback(stonith_t *stonith, xmlNode *msg, int call_id, int rc) { xmlNode *output = NULL; stonith_private_t *private = NULL; stonith_callback_client_t *blob = NULL; stonith_callback_client_t local_blob; CRM_CHECK(stonith != NULL, return); CRM_CHECK(stonith->private != NULL, return); private = stonith->private; local_blob.id = NULL; local_blob.callback = NULL; local_blob.user_data = NULL; local_blob.only_success = FALSE; if(msg != NULL) { crm_element_value_int(msg, F_STONITH_RC, &rc); crm_element_value_int(msg, F_STONITH_CALLID, &call_id); output = get_message_xml(msg, F_STONITH_CALLDATA); } CRM_CHECK(call_id > 0, crm_warn("Strange or missing call-id")); blob = g_hash_table_lookup( private->stonith_op_callback_table, GINT_TO_POINTER(call_id)); if(blob != NULL) { local_blob = *blob; blob = NULL; stonith_api_del_callback(stonith, call_id, FALSE); } else { crm_debug_2("No callback found for call %d", call_id); local_blob.callback = NULL; } if(stonith == NULL) { crm_debug("No stonith object supplied"); } if(local_blob.callback != NULL && (rc == stonith_ok || local_blob.only_success == FALSE)) { crm_debug_2("Invoking callback %s for call %d", crm_str(local_blob.id), call_id); local_blob.callback(stonith, msg, call_id, rc, output, local_blob.user_data); } else if(private->op_callback == NULL && rc != stonith_ok) { crm_warn("STONITH command failed: %s", stonith_error2string(rc)); crm_log_xml(LOG_DEBUG, "Failed STONITH Update", msg); } if(private->op_callback != NULL) { crm_debug_2("Invoking global callback for call %d", call_id); private->op_callback(stonith, msg, call_id, rc, output, NULL); } crm_debug_4("OP callback activated."); } static void stonith_send_notification(gpointer data, gpointer user_data) { struct notify_blob_s *blob = user_data; stonith_notify_client_t *entry = data; const char *event = NULL; if(blob->xml == NULL) { crm_warn("Skipping callback - NULL message"); return; } event = crm_element_value(blob->xml, F_SUBTYPE); if(entry == NULL) { crm_warn("Skipping callback - NULL callback client"); return; } else if(entry->notify == NULL) { crm_warn("Skipping callback - NULL callback"); return; } else if(safe_str_neq(entry->event, event)) { crm_debug_4("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event); return; } crm_debug_4("Invoking callback for %p/%s event...", entry, event); entry->notify(blob->stonith, event, blob->xml); crm_debug_4("Callback invoked..."); } int stonith_send_command( stonith_t *stonith, const char *op, xmlNode *data, xmlNode **output_data, int call_options, int timeout) { int rc = HA_OK; xmlNode *op_msg = NULL; xmlNode *op_reply = NULL; stonith_private_t *native = stonith->private; if(stonith->state == stonith_disconnected) { return st_err_connection; } if(output_data != NULL) { *output_data = NULL; } if(op == NULL) { crm_err("No operation specified"); return st_err_missing; } stonith->call_id++; /* prevent call_id from being negative (or zero) and conflicting * with the stonith_errors enum * use 2 because we use it as (stonith->call_id - 1) below */ if(stonith->call_id < 1) { stonith->call_id = 1; } CRM_CHECK(native->token != NULL, ;); op_msg = stonith_create_op(stonith->call_id, native->token, op, data, call_options); if(op_msg == NULL) { return st_err_missing; } crm_debug_3("Sending %s message to STONITH service", op); if(send_ipc_message(native->command_channel, op_msg) == FALSE) { crm_err("Sending message to STONITH service FAILED"); free_xml(op_msg); return st_err_ipc; } else { crm_debug_3("Message sent"); } free_xml(op_msg); if((call_options & st_opt_discard_reply)) { crm_debug_3("Discarding reply"); return stonith_ok; } else if(!(call_options & st_opt_sync_call)) { crm_debug_3("Async call, returning"); CRM_CHECK(stonith->call_id != 0, return st_err_ipc); return stonith->call_id; } rc = IPC_OK; crm_debug_3("Waiting for a syncronous reply"); rc = stonith_ok; while(IPC_ISRCONN(native->command_channel)) { int reply_id = -1; int msg_id = stonith->call_id; op_reply = xmlfromIPC(native->command_channel, stonith->call_timeout); if(op_reply == NULL) { rc = st_err_peer; break; } crm_element_value_int(op_reply, F_STONITH_CALLID, &reply_id); if(reply_id <= 0) { rc = st_err_peer; break; } else if(reply_id == msg_id) { crm_debug_3("Syncronous reply received"); crm_log_xml(LOG_MSG, "Reply", op_reply); if(crm_element_value_int(op_reply, F_STONITH_RC, &rc) != 0) { rc = st_err_peer; } if(output_data != NULL && is_not_set(call_options, st_opt_discard_reply)) { *output_data = op_reply; op_reply = NULL; } break; } else if(reply_id < msg_id) { crm_debug("Recieved old reply: %d (wanted %d)", reply_id, msg_id); crm_log_xml(LOG_MSG, "Old reply", op_reply); } else if((reply_id - 10000) > msg_id) { /* wrap-around case */ crm_debug("Recieved old reply: %d (wanted %d)", reply_id, msg_id); crm_log_xml(LOG_MSG, "Old reply", op_reply); } else { crm_err("Received a __future__ reply:" " %d (wanted %d)", reply_id, msg_id); } free_xml(op_reply); op_reply = NULL; } if(IPC_ISRCONN(native->command_channel) == FALSE) { crm_err("STONITH disconnected: %d", native->command_channel->ch_status); stonith->state = stonith_disconnected; } if(op_reply == NULL && stonith->state == stonith_disconnected) { rc = st_err_connection; } else if(rc == stonith_ok && op_reply == NULL) { rc = st_err_peer; } free_xml(op_reply); return rc; } static gboolean stonith_msgready(stonith_t* stonith) { stonith_private_t *private = NULL; if (stonith == NULL) { crm_err("No STONITH!"); return FALSE; } private = stonith->private; if(private->command_channel != NULL) { /* drain the channel */ IPC_Channel *cmd_ch = private->command_channel; xmlNode *cmd_msg = NULL; while(cmd_ch->ch_status != IPC_DISCONNECT && cmd_ch->ops->is_message_pending(cmd_ch)) { /* this will happen when the STONITH exited from beneath us */ cmd_msg = xmlfromIPC(cmd_ch, MAX_IPC_DELAY); free_xml(cmd_msg); } } else { crm_err("No command channel"); } if(private->callback_channel == NULL) { crm_err("No callback channel"); return FALSE; } else if(private->callback_channel->ch_status == IPC_DISCONNECT) { crm_info("Lost connection to the STONITH service [%d].", private->callback_channel->farside_pid); return FALSE; } else if(private->callback_channel->ops->is_message_pending( private->callback_channel)) { crm_debug_4("Message pending on command channel [%d]", private->callback_channel->farside_pid); return TRUE; } crm_debug_3("No message pending"); return FALSE; } static int stonith_rcvmsg(stonith_t* stonith) { const char *type = NULL; stonith_private_t *private = NULL; struct notify_blob_s blob; if (stonith == NULL) { crm_err("No STONITH!"); return FALSE; } blob.stonith = stonith; private = stonith->private; /* if it is not blocking mode and no message in the channel, return */ if (stonith_msgready(stonith) == FALSE) { crm_debug_3("No message ready and non-blocking..."); return 0; } /* IPC_INTR is not a factor here */ blob.xml = xmlfromIPC(private->callback_channel, MAX_IPC_DELAY); if (blob.xml == NULL) { crm_warn("Received a NULL msg from STONITH service."); return 0; } /* do callbacks */ type = crm_element_value(blob.xml, F_TYPE); crm_debug_4("Activating %s callbacks...", type); if(safe_str_eq(type, T_STONITH_NG)) { stonith_perform_callback(stonith, blob.xml, 0, 0); } else if(safe_str_eq(type, T_STONITH_NOTIFY)) { g_list_foreach(stonith->notify_list, stonith_send_notification, &blob); } else { crm_err("Unknown message type: %s", type); crm_log_xml_warn(blob.xml, "BadReply"); } free_xml(blob.xml); return 1; } gboolean stonith_dispatch(IPC_Channel *channel, gpointer user_data) { stonith_t *stonith = user_data; stonith_private_t *private = NULL; gboolean stay_connected = TRUE; CRM_CHECK(stonith != NULL, return FALSE); private = stonith->private; CRM_CHECK(private->callback_channel == channel, return FALSE); while(stonith_msgready(stonith)) { /* invoke the callbacks but dont block */ int rc = stonith_rcvmsg(stonith); if( rc < 0) { crm_err("Message acquisition failed: %d", rc); break; } else if(rc == 0) { break; } } if(private->callback_channel && private->callback_channel->ch_status != IPC_CONNECT) { crm_crit("Lost connection to the STONITH service [%d/callback].", channel->farside_pid); private->callback_source = NULL; stay_connected = FALSE; } if(private->command_channel && private->command_channel->ch_status != IPC_CONNECT) { crm_crit("Lost connection to the STONITH service [%d/command].", channel->farside_pid); private->callback_source = NULL; stay_connected = FALSE; } return stay_connected; } static int stonith_api_free (stonith_t* stonith) { int rc = stonith_ok; if(stonith->state != stonith_disconnected) { rc = stonith->cmds->disconnect(stonith); } if(stonith->state == stonith_disconnected) { stonith_private_t *private = stonith->private; g_hash_table_destroy(private->stonith_op_callback_table); crm_free(private->token); crm_free(stonith->private); crm_free(stonith->cmds); crm_free(stonith); } return rc; } void stonith_api_delete(stonith_t *stonith) { GList *list = stonith->notify_list; while(list != NULL) { stonith_notify_client_t *client = g_list_nth_data(list, 0); list = g_list_remove(list, client); crm_free(client); } stonith->cmds->free(stonith); stonith = NULL; } stonith_t *stonith_api_new(void) { stonith_t* new_stonith = NULL; stonith_private_t* private = NULL; crm_malloc0(new_stonith, sizeof(stonith_t)); crm_malloc0(private, sizeof(stonith_private_t)); new_stonith->private = private; private->stonith_op_callback_table = g_hash_table_new_full( g_direct_hash, g_direct_equal, NULL, stonith_destroy_op_callback); new_stonith->call_id = 1; new_stonith->notify_list = NULL; new_stonith->state = stonith_disconnected; crm_malloc0(new_stonith->cmds, sizeof(stonith_api_operations_t)); new_stonith->cmds->free = stonith_api_free; new_stonith->cmds->connect = stonith_api_signon; new_stonith->cmds->disconnect = stonith_api_signoff; new_stonith->cmds->call = stonith_api_call; new_stonith->cmds->fence = stonith_api_fence; new_stonith->cmds->metadata = stonith_api_device_metadata; new_stonith->cmds->query = stonith_api_query; new_stonith->cmds->remove_device = stonith_api_remove_device; new_stonith->cmds->register_device = stonith_api_register_device; new_stonith->cmds->remove_callback = stonith_api_del_callback; new_stonith->cmds->register_callback = stonith_api_add_callback; new_stonith->cmds->remove_notification = stonith_api_del_notification; new_stonith->cmds->register_notification = stonith_api_add_notification; return new_stonith; } diff --git a/pengine/master.c b/pengine/master.c index 86717b35ae..2ada551c57 100644 --- a/pengine/master.c +++ b/pengine/master.c @@ -1,898 +1,898 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include #include #include #include #include #define VARIANT_CLONE 1 #include extern gint sort_clone_instance(gconstpointer a, gconstpointer b, gpointer data_set); extern int master_score(resource_t *rsc, node_t *node, int not_set_value); static void child_promoting_constraints( clone_variant_data_t *clone_data, enum pe_ordering type, resource_t *rsc, resource_t *child, resource_t *last, pe_working_set_t *data_set) { if(child == NULL) { if(clone_data->ordered && last != NULL) { crm_debug_4("Ordered version (last node)"); /* last child promote before promoted started */ new_rsc_order(last, RSC_PROMOTE, rsc, RSC_PROMOTED, type, data_set); } return; } /* child promote before global promoted */ new_rsc_order(child, RSC_PROMOTE, rsc, RSC_PROMOTED, type, data_set); /* global promote before child promote */ new_rsc_order(rsc, RSC_PROMOTE, child, RSC_PROMOTE, type, data_set); if(clone_data->ordered) { crm_debug_4("Ordered version"); if(last == NULL) { /* global promote before first child promote */ last = rsc; } /* else: child/child relative promote */ order_start_start(last, child, type); new_rsc_order(last, RSC_PROMOTE, child, RSC_PROMOTE, type, data_set); } else { crm_debug_4("Un-ordered version"); } } static void child_demoting_constraints( clone_variant_data_t *clone_data, enum pe_ordering type, resource_t *rsc, resource_t *child, resource_t *last, pe_working_set_t *data_set) { if(child == NULL) { if(clone_data->ordered && last != NULL) { crm_debug_4("Ordered version (last node)"); /* global demote before first child demote */ new_rsc_order(rsc, RSC_DEMOTE, last, RSC_DEMOTE, pe_order_implies_left, data_set); } return; } /* child demote before global demoted */ new_rsc_order(child, RSC_DEMOTE, rsc, RSC_DEMOTED, pe_order_implies_right_printed, data_set); /* global demote before child demote */ new_rsc_order(rsc, RSC_DEMOTE, child, RSC_DEMOTE, pe_order_implies_left_printed, data_set); if(clone_data->ordered && last != NULL) { crm_debug_4("Ordered version"); /* child/child relative demote */ new_rsc_order(child, RSC_DEMOTE, last, RSC_DEMOTE, type, data_set); } else if(clone_data->ordered) { crm_debug_4("Ordered version (1st node)"); /* first child stop before global stopped */ new_rsc_order(child, RSC_DEMOTE, rsc, RSC_DEMOTED, type, data_set); } else { crm_debug_4("Un-ordered version"); } } static void master_update_pseudo_status( resource_t *rsc, gboolean *demoting, gboolean *promoting) { if(rsc->children) { slist_iter(child, resource_t, rsc->children, lpc, master_update_pseudo_status(child, demoting, promoting) ); return; } CRM_ASSERT(demoting != NULL); CRM_ASSERT(promoting != NULL); slist_iter( action, action_t, rsc->actions, lpc, if(*promoting && *demoting) { return; } else if(action->optional) { continue; } else if(safe_str_eq(RSC_DEMOTE, action->task)) { *demoting = TRUE; } else if(safe_str_eq(RSC_PROMOTE, action->task)) { *promoting = TRUE; } ); } #define apply_master_location(list) \ slist_iter( \ cons, rsc_to_node_t, list, lpc2, \ cons_node = NULL; \ if(cons->role_filter == RSC_ROLE_MASTER) { \ crm_debug_2("Applying %s to %s", \ cons->id, child_rsc->id); \ cons_node = pe_find_node_id( \ cons->node_list_rh, chosen->details->id); \ } \ if(cons_node != NULL) { \ int new_priority = merge_weights( \ child_rsc->priority, cons_node->weight); \ crm_debug_2("\t%s: %d->%d (%d)", child_rsc->id, \ child_rsc->priority, new_priority, cons_node->weight); \ child_rsc->priority = new_priority; \ } \ ); static node_t * can_be_master(resource_t *rsc) { node_t *node = NULL; node_t *local_node = NULL; resource_t *parent = uber_parent(rsc); clone_variant_data_t *clone_data = NULL; int level = LOG_DEBUG_2; #if 0 enum rsc_role_e role = RSC_ROLE_UNKNOWN; role = rsc->fns->state(rsc, FALSE); crm_info("%s role: %s", rsc->id, role2text(role)); #endif if(rsc->children) { slist_iter( child, resource_t, rsc->children, lpc, if(can_be_master(child) == NULL) { do_crm_log_unlikely(level, "Child %s of %s can't be promoted", child->id, rsc->id); return NULL; } ); } node = rsc->fns->location(rsc, NULL, FALSE); if(node == NULL) { do_crm_log_unlikely(level, "%s cannot be master: not allocated", rsc->id); return NULL; } else if(is_not_set(rsc->flags, pe_rsc_managed)) { if(rsc->fns->state(rsc, TRUE) == RSC_ROLE_MASTER) { crm_notice("Forcing unmanaged master %s to remain promoted on %s", rsc->id, node->details->uname); } else { return NULL; } } else if(rsc->priority < 0) { do_crm_log_unlikely(level, "%s cannot be master: preference: %d", rsc->id, rsc->priority); return NULL; } else if(can_run_resources(node) == FALSE) { do_crm_log_unlikely(level, "Node cant run any resources: %s", node->details->uname); return NULL; } get_clone_variant_data(clone_data, parent); local_node = pe_find_node_id( parent->allowed_nodes, node->details->id); if(local_node == NULL) { crm_err("%s cannot run on %s: node not allowed", rsc->id, node->details->uname); return NULL; } else if(local_node->count < clone_data->master_node_max || is_not_set(rsc->flags, pe_rsc_managed)) { return local_node; } else { do_crm_log_unlikely(level, "%s cannot be master on %s: node full", rsc->id, node->details->uname); } return NULL; } static gint sort_master_instance(gconstpointer a, gconstpointer b, gpointer data_set) { int rc; enum rsc_role_e role1 = RSC_ROLE_UNKNOWN; enum rsc_role_e role2 = RSC_ROLE_UNKNOWN; const resource_t *resource1 = (const resource_t*)a; const resource_t *resource2 = (const resource_t*)b; CRM_ASSERT(resource1 != NULL); CRM_ASSERT(resource2 != NULL); role1 = resource1->fns->state(resource1, TRUE); role2 = resource2->fns->state(resource2, TRUE); rc = sort_rsc_index(a, b); if( rc != 0 ) { return rc; } if(role1 > role2) { return -1; } else if(role1 < role2) { return 1; } return sort_clone_instance(a, b, data_set); } static void master_promotion_order(resource_t *rsc, pe_working_set_t *data_set) { node_t *node = NULL; node_t *chosen = NULL; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); if(clone_data->merged_master_weights) { return; } clone_data->merged_master_weights = TRUE; crm_debug_2("Merging weights for %s", rsc->id); slist_iter( child, resource_t, rsc->children, lpc, crm_debug_2("%s: %d", child->id, child->sort_index); ); dump_node_scores(LOG_DEBUG_3, rsc, "Before", rsc->allowed_nodes); slist_iter( child, resource_t, rsc->children, lpc, chosen = child->fns->location(child, NULL, FALSE); if(chosen == NULL || child->sort_index < 0) { crm_debug_3("Skipping %s", child->id); continue; } node = (node_t*)pe_find_node_id( rsc->allowed_nodes, chosen->details->id); CRM_ASSERT(node != NULL); /* adds in master preferences and rsc_location.role=Master */ node->weight = merge_weights(child->sort_index, node->weight); ); dump_node_scores(LOG_DEBUG_3, rsc, "Middle", rsc->allowed_nodes); slist_iter( constraint, rsc_colocation_t, rsc->rsc_cons, lpc, /* (re-)adds location preferences of resources that the * master instance should/must be colocated with */ if(constraint->role_lh == RSC_ROLE_MASTER) { crm_debug_2("RHS: %s with %s: %d", constraint->rsc_lh->id, constraint->rsc_rh->id, constraint->score); rsc->allowed_nodes = constraint->rsc_rh->cmds->merge_weights( constraint->rsc_rh, rsc->id, rsc->allowed_nodes, constraint->node_attribute, constraint->score/INFINITY, constraint->score==INFINITY?FALSE:TRUE); } ); slist_iter( constraint, rsc_colocation_t, rsc->rsc_cons_lhs, lpc, /* (re-)adds location preferences of resource that wish to be * colocated with the master instance */ if(constraint->role_rh == RSC_ROLE_MASTER) { crm_debug_2("LHS: %s with %s: %d", constraint->rsc_lh->id, constraint->rsc_rh->id, constraint->score); rsc->allowed_nodes = constraint->rsc_lh->cmds->merge_weights( constraint->rsc_lh, rsc->id, rsc->allowed_nodes, constraint->node_attribute, constraint->score/INFINITY, TRUE); } ); dump_node_scores(LOG_DEBUG_3, rsc, "After", rsc->allowed_nodes); /* write them back and sort */ slist_iter( child, resource_t, rsc->children, lpc, chosen = child->fns->location(child, NULL, FALSE); if(chosen == NULL || child->sort_index < 0) { crm_debug_2("%s: %d", child->id, child->sort_index); continue; } node = (node_t*)pe_find_node_id( rsc->allowed_nodes, chosen->details->id); CRM_ASSERT(node != NULL); child->sort_index = node->weight; crm_debug_2("%s: %d", child->id, child->sort_index); ); rsc->children = g_list_sort_with_data(rsc->children, sort_master_instance, data_set); } int master_score(resource_t *rsc, node_t *node, int not_set_value) { char *attr_name; char *name = rsc->id; const char *attr_value; int score = not_set_value, len = 0; if(rsc->children) { slist_iter( child, resource_t, rsc->children, lpc, int c_score = master_score(child, node, not_set_value); if(score == not_set_value) { score = c_score; } else { score += c_score; } ); return score; } if(rsc->fns->state(rsc, TRUE) < RSC_ROLE_STARTED) { return score; } if(rsc->running_on) { node_t *match = pe_find_node_id(rsc->allowed_nodes, node->details->id); - if(match->weight < 0) { + if(match && match->weight < 0) { crm_debug_2("%s on %s has score: %d - ignoring master pref", rsc->id, match->details->uname, match->weight); return score; } } if(rsc->clone_name) { /* Use the name the lrm knows this resource as, * since that's what crm_master would have used too */ name = rsc->clone_name; } len = 8 + strlen(name); crm_malloc0(attr_name, len); sprintf(attr_name, "master-%s", name); crm_debug_3("looking for %s on %s", attr_name, node->details->uname); attr_value = g_hash_table_lookup( node->details->attrs, attr_name); if(attr_value == NULL) { crm_free(attr_name); len = 8 + strlen(rsc->long_name); crm_malloc0(attr_name, len); sprintf(attr_name, "master-%s", rsc->long_name); crm_debug_3("looking for %s on %s", attr_name, node->details->uname); attr_value = g_hash_table_lookup( node->details->attrs, attr_name); } if(attr_value != NULL) { crm_debug_2("%s[%s] = %s", attr_name, node->details->uname, crm_str(attr_value)); score = char2score(attr_value); } crm_free(attr_name); return score; } #define max(a, b) aapplied_master_prefs) { /* Make sure we only do this once */ return; } clone_data->applied_master_prefs = TRUE; slist_iter( child_rsc, resource_t, rsc->children, lpc, slist_iter( node, node_t, child_rsc->allowed_nodes, lpc, if(can_run_resources(node) == FALSE) { /* This node will never be promoted to master, * so don't apply the master score as that may * lead to clone shuffling */ continue; } score = master_score(child_rsc, node, 0); if(score > 0) { new_score = merge_weights(node->weight, score); if(new_score != node->weight) { crm_debug_2("\t%s: Updating preference for %s (%d->%d)", child_rsc->id, node->details->uname, node->weight, new_score); node->weight = new_score; } } new_score = max(child_rsc->priority, score); if(new_score != child_rsc->priority) { crm_debug_2("\t%s: Updating priority (%d->%d)", child_rsc->id, child_rsc->priority, new_score); child_rsc->priority = new_score; } ); ); } static void set_role_slave(resource_t *rsc, gboolean current) { if(current) { if(rsc->role == RSC_ROLE_STARTED) { rsc->role = RSC_ROLE_SLAVE; } } else { GListPtr allocated = NULL; rsc->fns->location(rsc, &allocated, FALSE); if(allocated) { rsc->next_role = RSC_ROLE_SLAVE; } else { rsc->next_role = RSC_ROLE_STOPPED; } g_list_free(allocated); } slist_iter( child_rsc, resource_t, rsc->children, lpc, set_role_slave(child_rsc, current); ); } static void set_role_master(resource_t *rsc) { if(rsc->next_role == RSC_ROLE_UNKNOWN) { rsc->next_role = RSC_ROLE_MASTER; } slist_iter( child_rsc, resource_t, rsc->children, lpc, set_role_master(child_rsc); ); } node_t * master_color(resource_t *rsc, pe_working_set_t *data_set) { int promoted = 0; node_t *chosen = NULL; node_t *cons_node = NULL; enum rsc_role_e next_role = RSC_ROLE_UNKNOWN; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); apply_master_prefs(rsc); clone_color(rsc, data_set); /* count now tracks the number of masters allocated */ slist_iter(node, node_t, rsc->allowed_nodes, lpc, node->count = 0; ); /* * assign priority */ slist_iter( child_rsc, resource_t, rsc->children, lpc, GListPtr list = NULL; crm_debug_2("Assigning priority for %s: %s", child_rsc->id, role2text(child_rsc->next_role)); if(child_rsc->fns->state(child_rsc, TRUE) == RSC_ROLE_STARTED) { set_role_slave(child_rsc, TRUE); } chosen = child_rsc->fns->location(child_rsc, &list, FALSE); if(g_list_length(list) > 1) { crm_config_err("Cannot promote non-colocated child %s", child_rsc->id); } g_list_free(list); if(chosen == NULL) { continue; } next_role = child_rsc->fns->state(child_rsc, FALSE); switch(next_role) { case RSC_ROLE_STARTED: case RSC_ROLE_UNKNOWN: CRM_CHECK(chosen != NULL, break); /* * Default to -1 if no value is set * * This allows master locations to be specified * based solely on rsc_location constraints, * but prevents anyone from being promoted if * neither a constraint nor a master-score is present */ child_rsc->priority = master_score(child_rsc, chosen, -1); break; case RSC_ROLE_SLAVE: case RSC_ROLE_STOPPED: child_rsc->priority = -INFINITY; break; case RSC_ROLE_MASTER: /* We will arrive here if we're re-creating actions after a stonith * OR target-role is set */ break; default: CRM_CHECK(FALSE/* unhandled */, crm_err("Unknown resource role: %d for %s", next_role, child_rsc->id)); } apply_master_location(child_rsc->rsc_location); apply_master_location(rsc->rsc_location); slist_iter( cons, rsc_colocation_t, child_rsc->rsc_cons, lpc2, child_rsc->cmds->rsc_colocation_lh(child_rsc, cons->rsc_rh, cons); ); child_rsc->sort_index = child_rsc->priority; crm_debug_2("Assigning priority for %s: %d", child_rsc->id, child_rsc->priority); if(next_role == RSC_ROLE_MASTER) { child_rsc->sort_index = INFINITY; } ); master_promotion_order(rsc, data_set); /* mark the first N as masters */ slist_iter( child_rsc, resource_t, rsc->children, lpc, char *score = score2char(child_rsc->sort_index); chosen = child_rsc->fns->location(child_rsc, NULL, FALSE); if(show_scores) { fprintf(stdout, "%s promotion score on %s: %s\n", child_rsc->id, chosen?chosen->details->uname:"none", score); } else { do_crm_log_unlikely(scores_log_level, "%s promotion score on %s: %s", child_rsc->id, chosen?chosen->details->uname:"none", score); } crm_free(score); chosen = NULL; /* nuke 'chosen' so that we don't promote more than the * required number of instances */ if(child_rsc->sort_index < 0) { crm_debug_2("Not supposed to promote child: %s", child_rsc->id); } else if(promoted < clone_data->master_max || is_not_set(rsc->flags, pe_rsc_managed)) { chosen = can_be_master(child_rsc); } crm_debug("%s master score: %d", child_rsc->id, child_rsc->priority); if(chosen == NULL) { set_role_slave(child_rsc, FALSE); continue; } chosen->count++; crm_info("Promoting %s (%s %s)", child_rsc->id, role2text(child_rsc->role), chosen->details->uname); set_role_master(child_rsc); promoted++; ); clone_data->masters_allocated = promoted; crm_info("%s: Promoted %d instances of a possible %d to master", rsc->id, promoted, clone_data->master_max); return NULL; } void master_create_actions(resource_t *rsc, pe_working_set_t *data_set) { action_t *action = NULL; action_t *action_complete = NULL; gboolean any_promoting = FALSE; gboolean any_demoting = FALSE; resource_t *last_promote_rsc = NULL; resource_t *last_demote_rsc = NULL; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); crm_debug("Creating actions for %s", rsc->id); /* create actions as normal */ clone_create_actions(rsc, data_set); slist_iter( child_rsc, resource_t, rsc->children, lpc, gboolean child_promoting = FALSE; gboolean child_demoting = FALSE; crm_debug_2("Creating actions for %s", child_rsc->id); child_rsc->cmds->create_actions(child_rsc, data_set); master_update_pseudo_status( child_rsc, &child_demoting, &child_promoting); any_demoting = any_demoting || child_demoting; any_promoting = any_promoting || child_promoting; crm_debug_2("Created actions for %s: %d %d", child_rsc->id, child_promoting, child_demoting); ); /* promote */ action = promote_action(rsc, NULL, !any_promoting); action_complete = custom_action( rsc, promoted_key(rsc), RSC_PROMOTED, NULL, !any_promoting, TRUE, data_set); action->pseudo = TRUE; action->runnable = FALSE; action_complete->pseudo = TRUE; action_complete->runnable = FALSE; action_complete->priority = INFINITY; if(clone_data->masters_allocated > 0) { action->runnable = TRUE; action_complete->runnable = TRUE; } child_promoting_constraints(clone_data, pe_order_optional, rsc, NULL, last_promote_rsc, data_set); if(clone_data->promote_notify == NULL) { clone_data->promote_notify = create_notification_boundaries( rsc, RSC_PROMOTE, action, action_complete, data_set); } /* demote */ action = demote_action(rsc, NULL, !any_demoting); action_complete = custom_action( rsc, demoted_key(rsc), RSC_DEMOTED, NULL, !any_demoting, TRUE, data_set); action_complete->priority = INFINITY; action->pseudo = TRUE; action->runnable = TRUE; action_complete->pseudo = TRUE; action_complete->runnable = TRUE; child_demoting_constraints(clone_data, pe_order_optional, rsc, NULL, last_demote_rsc, data_set); if(clone_data->demote_notify == NULL) { clone_data->demote_notify = create_notification_boundaries( rsc, RSC_DEMOTE, action, action_complete, data_set); if(clone_data->promote_notify) { /* If we ever wanted groups to have notifications we'd need to move this to native_internal_constraints() one day * Requires exposing *_notify */ order_actions(clone_data->stop_notify->post_done, clone_data->promote_notify->pre, pe_order_optional); order_actions(clone_data->start_notify->post_done, clone_data->promote_notify->pre, pe_order_optional); order_actions(clone_data->demote_notify->post_done, clone_data->promote_notify->pre, pe_order_optional); order_actions(clone_data->demote_notify->post_done, clone_data->start_notify->pre, pe_order_optional); order_actions(clone_data->demote_notify->post_done, clone_data->stop_notify->pre, pe_order_optional); } } /* restore the correct priority */ slist_iter( child_rsc, resource_t, rsc->children, lpc, child_rsc->priority = rsc->priority; ); } void master_internal_constraints(resource_t *rsc, pe_working_set_t *data_set) { resource_t *last_rsc = NULL; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); clone_internal_constraints(rsc, data_set); /* global stopped before start */ new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_START, pe_order_optional, data_set); /* global stopped before promote */ new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_PROMOTE, pe_order_optional, data_set); /* global demoted before start */ new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_START, pe_order_optional, data_set); /* global started before promote */ new_rsc_order(rsc, RSC_STARTED, rsc, RSC_PROMOTE, pe_order_optional, data_set); /* global demoted before stop */ new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_STOP, pe_order_optional, data_set); /* global demote before demoted */ new_rsc_order(rsc, RSC_DEMOTE, rsc, RSC_DEMOTED, pe_order_optional, data_set); /* global demoted before promote */ new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_PROMOTE, pe_order_optional, data_set); slist_iter( child_rsc, resource_t, rsc->children, lpc, /* child demote before promote */ new_rsc_order(child_rsc, RSC_DEMOTE, child_rsc, RSC_PROMOTE, pe_order_optional, data_set); child_promoting_constraints(clone_data, pe_order_optional, rsc, child_rsc, last_rsc, data_set); child_demoting_constraints(clone_data, pe_order_optional, rsc, child_rsc, last_rsc, data_set); last_rsc = child_rsc; ); } static void node_list_update_one(GListPtr list, node_t *other, const char *attr, int score) { const char *value = NULL; if(other == NULL) { return; } else if(attr == NULL) { attr = "#"XML_ATTR_UNAME; } value = g_hash_table_lookup(other->details->attrs, attr); slist_iter(node, node_t, list, lpc, const char *tmp = g_hash_table_lookup(node->details->attrs, attr); if(safe_str_eq(value, tmp)) { crm_debug_2("%s: %d + %d", node->details->uname, node->weight, other->weight); node->weight = merge_weights(node->weight, score); } ); } void master_rsc_colocation_rh( resource_t *rsc_lh, resource_t *rsc_rh, rsc_colocation_t *constraint) { clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc_rh); CRM_CHECK(rsc_rh != NULL, return); if(is_set(rsc_rh->flags, pe_rsc_provisional)) { return; } else if(constraint->role_rh == RSC_ROLE_UNKNOWN) { crm_debug_3("Handling %s as a clone colocation", constraint->id); clone_rsc_colocation_rh(rsc_lh, rsc_rh, constraint); return; } CRM_CHECK(rsc_lh != NULL, return); CRM_CHECK(rsc_lh->variant == pe_native, return); crm_debug_2("Processing constraint %s: %d", constraint->id, constraint->score); if(constraint->role_rh == RSC_ROLE_UNKNOWN) { slist_iter( child_rsc, resource_t, rsc_rh->children, lpc, child_rsc->cmds->rsc_colocation_rh(rsc_lh, child_rsc, constraint); ); } else if(is_set(rsc_lh->flags, pe_rsc_provisional)) { GListPtr lhs = NULL, rhs = NULL; lhs = rsc_lh->allowed_nodes; slist_iter( child_rsc, resource_t, rsc_rh->children, lpc, node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE); enum rsc_role_e next_role = child_rsc->fns->state(child_rsc, FALSE); crm_debug_3("Processing: %s", child_rsc->id); if(chosen != NULL && next_role == constraint->role_rh) { crm_debug_3("Applying: %s %s %s %d", child_rsc->id, role2text(next_role), chosen->details->uname, constraint->score); if(constraint->score < INFINITY) { node_list_update_one(rsc_lh->allowed_nodes, chosen, constraint->node_attribute, constraint->score); } rhs = g_list_append(rhs, chosen); } ); /* Only do this if its not a master-master colocation * Doing this unconditionally would prevent the slaves from being started */ if(constraint->role_lh != RSC_ROLE_MASTER || constraint->role_rh != RSC_ROLE_MASTER) { if(constraint->score > 0) { rsc_lh->allowed_nodes = node_list_exclude(lhs, rhs, TRUE); pe_free_shallow(lhs); } } pe_free_shallow_adv(rhs, FALSE); } else if(constraint->role_lh == RSC_ROLE_MASTER) { resource_t *rh_child = find_compatible_child(rsc_lh, rsc_rh, constraint->role_rh, FALSE); if(rh_child == NULL && constraint->score >= INFINITY) { crm_debug_2("%s can't be promoted %s", rsc_lh->id, constraint->id); rsc_lh->priority = -INFINITY; } else if(rh_child != NULL) { int new_priority = merge_weights(rsc_lh->priority, constraint->score); crm_debug("Applying %s to %s", constraint->id, rsc_lh->id); crm_debug("\t%s: %d->%d", rsc_lh->id, rsc_lh->priority, new_priority); rsc_lh->priority = new_priority; } } return; } void master_append_meta(resource_t *rsc, xmlNode *xml) { char *name = NULL; clone_variant_data_t *clone_data = NULL; get_clone_variant_data(clone_data, rsc); clone_append_meta(rsc, xml); name = crm_meta_name(XML_RSC_ATTR_MASTER_MAX); crm_xml_add_int(xml, name, clone_data->master_max); crm_free(name); name = crm_meta_name(XML_RSC_ATTR_MASTER_NODEMAX); crm_xml_add_int(xml, name, clone_data->master_node_max); crm_free(name); }