diff --git a/attrd/attrd_alerts.c b/attrd/attrd_alerts.c index d187fab7b2..58562e8c14 100644 --- a/attrd/attrd_alerts.c +++ b/attrd/attrd_alerts.c @@ -1,536 +1,380 @@ /* * Copyright (C) 2015 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include -#include #include #include #include #include "attrd_alerts.h" #include -#include #include +#include +#include GHashTable *alert_info_cache = NULL; lrmd_t * attrd_lrmd_connect(int max_retry, void callback(lrmd_event_data_t * op)) { int ret = -ENOTCONN; int fails = 0; if (!the_lrmd) { the_lrmd = lrmd_api_new(); } while(fails < max_retry) { the_lrmd->cmds->set_callback(the_lrmd, callback); ret = the_lrmd->cmds->connect(the_lrmd, T_ATTRD, NULL); if (ret != pcmk_ok) { fails++; crm_trace("lrmd_connect RETRY!(%d)", fails); } else { crm_trace("lrmd_connect OK!"); break; } } if (ret != pcmk_ok) { if (the_lrmd->cmds->is_connected(the_lrmd)) { lrmd_api_delete(the_lrmd); } the_lrmd = NULL; } return the_lrmd; } -static void -attrd_parse_alerts(xmlNode *notifications) -{ - xmlNode *alert; - crm_alert_entry_t entry; - guint max_timeout = 0; - - crm_free_alert_list(); - crm_alert_max_alert_timeout = CRM_ALERT_DEFAULT_TIMEOUT_MS; - - if (crm_alert_kind_default == NULL) { - crm_alert_kind_default = g_strsplit(CRM_ALERT_KIND_DEFAULT, ",", 0); - } - - if (notifications) { - crm_info("We have an alerts section in the cib"); - } else { - crm_info("No optional alerts section in cib"); - return; - } - - for (alert = first_named_child(notifications, XML_CIB_TAG_ALERT); - alert; alert = __xml_next(alert)) { - xmlNode *recipient; - int recipients = 0, envvars = 0; - GHashTable *config_hash = NULL; - - entry = (crm_alert_entry_t) { - .id = (char *) crm_element_value(alert, XML_ATTR_ID), - .path = (char *) crm_element_value(alert, XML_ALERT_ATTR_PATH), - .timeout = CRM_ALERT_DEFAULT_TIMEOUT_MS, - .tstamp_format = (char *) CRM_ALERT_DEFAULT_TSTAMP_FORMAT, - .select_kind_orig = NULL, - .select_kind = NULL, - .select_attribute_name_orig = NULL, - .select_attribute_name = NULL - }; - - crm_get_envvars_from_cib(alert, - &entry, - &envvars); - - config_hash = - get_meta_attrs_from_cib(alert, &entry, &max_timeout); - - crm_debug("Found alert: id=%s, path=%s, timeout=%d, " - "tstamp_format=%s, select_kind=%s, select_attribute_name=%s, %d additional environment variables", - entry.id, entry.path, entry.timeout, - entry.tstamp_format, entry.select_kind_orig, entry.select_attribute_name_orig, envvars); - - for (recipient = first_named_child(alert, - XML_CIB_TAG_ALERT_RECIPIENT); - recipient; recipient = __xml_next(recipient)) { - int envvars_added = 0; - - entry.recipient = (char *) crm_element_value(recipient, - XML_ALERT_ATTR_REC_VALUE); - recipients++; - - crm_get_envvars_from_cib(recipient, - &entry, - &envvars_added); - - { - crm_alert_entry_t recipient_entry = entry; - GHashTable *config_hash = - get_meta_attrs_from_cib(recipient, - &recipient_entry, - &max_timeout); - - crm_add_dup_alert_list_entry(&recipient_entry); - - crm_debug("Alert has recipient: id=%s, value=%s, " - "%d additional environment variables", - crm_element_value(recipient, XML_ATTR_ID), - recipient_entry.recipient, envvars_added); - - g_hash_table_destroy(config_hash); - } - - entry.envvars = - crm_drop_envvars(&entry, envvars_added); - } - - if (recipients == 0) { - crm_add_dup_alert_list_entry(&entry); - } - - crm_drop_envvars(&entry, -1); - g_hash_table_destroy(config_hash); - } - - if (max_timeout > 0) { - crm_alert_max_alert_timeout = max_timeout; - } -} - static void config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) { crm_time_t *now = crm_time_new(NULL); xmlNode *crmalerts = NULL; if (rc != pcmk_ok) { crm_err("Local CIB query resulted in an error: %s", pcmk_strerror(rc)); goto bail; } crmalerts = output; if ((crmalerts) && (crm_element_name(crmalerts)) && (strcmp(crm_element_name(crmalerts), XML_CIB_TAG_ALERTS) != 0)) { crmalerts = first_named_child(crmalerts, XML_CIB_TAG_ALERTS); } if (!crmalerts) { crm_err("Local CIB query for " XML_CIB_TAG_ALERTS " section failed"); goto bail; } - attrd_parse_alerts(crmalerts); + pe_unpack_alerts(crmalerts); bail: crm_time_free(now); } gboolean attrd_read_options(gpointer user_data) { int call_id; if (the_cib) { call_id = the_cib->cmds->query(the_cib, "//" XML_CIB_TAG_ALERTS, NULL, cib_xpath | cib_scope_local); the_cib->cmds->register_callback_full(the_cib, call_id, 120, FALSE, NULL, "config_query_callback", config_query_callback, free); crm_trace("Querying the CIB... call %d", call_id); } else { crm_err("Querying the CIB...CIB connection not active"); } return TRUE; } void attrd_cib_updated_cb(const char *event, xmlNode * msg) { int rc = -1; int format= 1; xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT); xmlNode *change = NULL; xmlXPathObject *xpathObj = NULL; CRM_CHECK(msg != NULL, return); crm_element_value_int(msg, F_CIB_RC, &rc); if (rc < pcmk_ok) { crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc)); return; } crm_element_value_int(patchset, "format", &format); if (format == 1) { if ((xpathObj = xpath_search( msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_ALERTS )) != NULL) { freeXpathObject(xpathObj); mainloop_set_trigger(attrd_config_read); } } else if (format == 2) { for (change = __xml_first_child(patchset); change != NULL; change = __xml_next(change)) { const char *xpath = crm_element_value(change, XML_DIFF_PATH); if (xpath == NULL) { continue; } /* modifying properties */ if (!strstr(xpath, "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_ALERTS)) { xmlNode *section = NULL; const char *name = NULL; /* adding notifications section */ if ((strcmp(xpath, "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION) != 0) || ((section = __xml_first_child(change)) == NULL) || ((name = crm_element_name(section)) == NULL) || (strcmp(name, XML_CIB_TAG_ALERTS) != 0)) { continue; } } mainloop_set_trigger(attrd_config_read); break; } } else { crm_warn("Unknown patch format: %d", format); } } -GHashTable * -get_meta_attrs_from_cib(xmlNode *basenode, crm_alert_entry_t *entry, - guint *max_timeout) -{ - GHashTable *config_hash = - g_hash_table_new_full(crm_str_hash, g_str_equal, - g_hash_destroy_str, g_hash_destroy_str); - crm_time_t *now = crm_time_new(NULL); - const char *value = NULL; - - unpack_instance_attributes(basenode, basenode, XML_TAG_META_SETS, NULL, - config_hash, NULL, FALSE, now); - - value = g_hash_table_lookup(config_hash, XML_ALERT_ATTR_TIMEOUT); - if (value) { - entry->timeout = crm_get_msec(value); - if (entry->timeout <= 0) { - if (entry->timeout == 0) { - crm_trace("Setting timeout to default %dmsec", - CRM_ALERT_DEFAULT_TIMEOUT_MS); - } else { - crm_warn("Invalid timeout value setting to default %dmsec", - CRM_ALERT_DEFAULT_TIMEOUT_MS); - } - entry->timeout = CRM_ALERT_DEFAULT_TIMEOUT_MS; - } else { - crm_trace("Found timeout %dmsec", entry->timeout); - } - if (entry->timeout > *max_timeout) { - *max_timeout = entry->timeout; - } - } - value = g_hash_table_lookup(config_hash, XML_ALERT_ATTR_TSTAMP_FORMAT); - if (value) { - /* hard to do any checks here as merely anything can - * can be a valid time-format-string - */ - entry->tstamp_format = (char *) value; - crm_trace("Found timestamp format string '%s'", value); - } - - value = g_hash_table_lookup(config_hash, XML_ALERT_ATTR_SELECT_KIND); - if (value) { - entry->select_kind_orig = (char*) value; - entry->select_kind = g_strsplit((char*) value, ",", 0); - crm_trace("Found select_kind string '%s'", (char *) value); - } - - value = g_hash_table_lookup(config_hash, XML_ALERT_ATTR_SELECT_ATTRIBUTE_NAME); - if (value) { - entry->select_attribute_name_orig = (char*) value; - entry->select_attribute_name = g_strsplit((char*) value, ",", 0); - crm_trace("Found attribute_name string '%s'", (char *) value); - } - - crm_time_free(now); - return config_hash; /* keep hash as long as strings are needed */ -} - void attrd_alert_fini() { if (alert_info_cache) { g_hash_table_destroy(alert_info_cache); alert_info_cache = NULL; } if (crm_alert_kind_default) { g_strfreev(crm_alert_kind_default); crm_alert_kind_default = NULL; } } static int exec_alerts(lrmd_t *lrmd, const char *kind, const char *attribute_name, lrmd_key_value_t * params, GListPtr alert_list, GHashTable *info_cache) { int call_id = 0; static int operations = 0; GListPtr l; crm_time_hr_t *now = crm_time_hr_new(NULL); params = lrmd_set_alert_key_to_lrmd_params(params, CRM_alert_kind, kind); params = lrmd_set_alert_key_to_lrmd_params(params, CRM_alert_version, VERSION); for (l = g_list_first(alert_list); l; l = g_list_next(l)) { lrmd_rsc_info_t *rsc = NULL; crm_alert_entry_t *entry = (crm_alert_entry_t *)(l->data); char *timestamp = crm_time_format_hr(entry->tstamp_format, now); lrmd_key_value_t * copy_params = NULL; lrmd_key_value_t *head, *p; if (crm_is_target_alert(entry->select_kind == NULL ? crm_alert_kind_default : entry->select_kind, kind) == FALSE) { crm_trace("Cannot sending '%s' alert to '%s' via '%s'(select_kind=%s)", kind, entry->recipient, entry->path, entry->select_kind == NULL ? CRM_ALERT_KIND_DEFAULT : entry->select_kind_orig); free(timestamp); continue; } if (crm_is_target_alert(entry->select_attribute_name, attribute_name) == FALSE) { crm_trace("Cannot sending '%s' alert to '%s' via '%s'(select_attribute_name=%s attribute_name=%s)", kind, entry->recipient, entry->path, entry->select_attribute_name_orig, attribute_name); free(timestamp); continue; } crm_info("Sending '%s' alert to '%s' via '%s'", kind, entry->recipient, entry->path); rsc = g_hash_table_lookup(alert_info_cache, entry->id); if (rsc == NULL) { rsc = lrmd->cmds->get_rsc_info(lrmd, entry->id, 0); if (!rsc) { lrmd->cmds->register_rsc(lrmd, entry->id, PCMK_ALERT_CLASS, "pacemaker", entry->path, lrmd_opt_drop_recurring); rsc = lrmd->cmds->get_rsc_info(lrmd, entry->id, 0); if (!rsc) { crm_err("Could not add alert %s : %s", entry->id, entry->path); return -1; } /* cache the result */ g_hash_table_insert(alert_info_cache, entry->id, rsc); } } /* Because there is a parameter to turn into every transmission, Copy a parameter. */ head = params; while (head) { p = head->next; copy_params = lrmd_key_value_add(copy_params, head->key, head->value); head = p; } operations++; copy_params = lrmd_key_value_add(copy_params, CRM_ALERT_KEY_PATH, entry->path); copy_params = lrmd_set_alert_key_to_lrmd_params(copy_params, CRM_alert_recipient, entry->recipient); copy_params = lrmd_set_alert_key_to_lrmd_params(copy_params, CRM_alert_node_sequence, crm_itoa(operations)); copy_params = lrmd_set_alert_key_to_lrmd_params(copy_params, CRM_alert_timestamp, timestamp); lrmd_set_alert_envvar_to_lrmd_params(copy_params, entry); call_id = lrmd->cmds->exec_alert(lrmd, strdup(entry->id), entry->timeout, lrmd_opt_notify_orig_only, copy_params); if (call_id <= 0) { crm_err("Operation %s on %s failed: %d", "start", rsc->id, call_id); } else { crm_info("Operation %s on %s compete: %d", "start", rsc->id, call_id); } free(timestamp); } if (now) { free(now); } return call_id; } static void free_alert_info(gpointer value) { lrmd_rsc_info_t *rsc_info = value; lrmd_free_rsc_info(rsc_info); } static void attrd_alert_lrm_op_callback(lrmd_event_data_t * op) { CRM_CHECK(op != NULL, return); if (op->type == lrmd_event_disconnect) { crm_info("Lost connection to LRMD service!"); if (the_lrmd->cmds->is_connected(the_lrmd)) { the_lrmd->cmds->disconnect(the_lrmd); lrmd_api_delete(the_lrmd); } the_lrmd = NULL; return; } else if (op->type != lrmd_event_exec_complete) { return; } if (op->params != NULL) { void *value_tmp1, *value_tmp2; value_tmp1 = g_hash_table_lookup(op->params, CRM_ALERT_KEY_PATH); if (value_tmp1 != NULL) { value_tmp2 = g_hash_table_lookup(op->params, CRM_ALERT_NODE_SEQUENCE); if(op->rc == 0) { crm_info("Alert %s (%s) complete", value_tmp2, value_tmp1); } else { crm_warn("Alert %s (%s) failed: %d", value_tmp2, value_tmp1, op->rc); } } } } int attrd_send_alerts(lrmd_t *lrmd, const char *node, uint32_t nodeid, const char *attribute_name, const char *attribute_value, GListPtr alert_list) { int ret = pcmk_ok; lrmd_key_value_t *params = NULL; if (lrmd == NULL) { lrmd = attrd_lrmd_connect(10, attrd_alert_lrm_op_callback); if (lrmd == NULL) { crm_warn("LRMD connection not active"); return ret; } } crm_trace("LRMD connection active"); if (alert_info_cache == NULL) { alert_info_cache = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_alert_info); } params = lrmd_set_alert_key_to_lrmd_params(params, CRM_alert_node, node); params = lrmd_set_alert_key_to_lrmd_params(params, CRM_alert_nodeid, crm_itoa(nodeid)); params = lrmd_set_alert_key_to_lrmd_params(params, CRM_alert_attribute_name, attribute_name); params = lrmd_set_alert_key_to_lrmd_params(params, CRM_alert_attribute_value, attribute_value == NULL ? "null" : attribute_value); ret = exec_alerts(lrmd, "attribute", attribute_name, params, alert_list, alert_info_cache); crm_trace("ret : %d, node : %s, nodeid: %s, name: %s, value : %s", ret, node, crm_itoa(nodeid), attribute_name, attribute_value); if (params) { lrmd_key_value_freeall(params); } return ret; } #if HAVE_ATOMIC_ATTRD void set_alert_attribute_value(GHashTable *t, attribute_value_t *v) { attribute_value_t *a_v = NULL; a_v = calloc(1, sizeof(attribute_value_t)); CRM_ASSERT(a_v != NULL); a_v->nodeid = v->nodeid; a_v->nodename = strdup(v->nodename); if (v->current != NULL) { a_v->current = strdup(v->current); } g_hash_table_replace(t, a_v->nodename, a_v); } void send_alert_attributes_value(attribute_t *a, GHashTable *t) { int call_id = 0; attribute_value_t *at = NULL; GHashTableIter vIter; g_hash_table_iter_init(&vIter, t); while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & at)) { call_id = attrd_send_alerts(the_lrmd, at->nodename, at->nodeid, a->id, at->current, crm_alert_list); crm_trace("call_id : %d, nodename : %s, nodeid: %d, name: %s, value : %s", call_id, at->nodename, at->nodeid, a->id, at->current); } } #endif diff --git a/attrd/attrd_alerts.h b/attrd/attrd_alerts.h index 6949c1f1bf..ecd50e1e0f 100644 --- a/attrd/attrd_alerts.h +++ b/attrd/attrd_alerts.h @@ -1,42 +1,41 @@ /* * Copyright (C) 2015 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef ATTRD_ALERT__H # define ATTRD_ALERT__H # include # include # include extern cib_t *the_cib; extern lrmd_t *the_lrmd; extern crm_trigger_t *attrd_config_read; lrmd_t *attrd_lrmd_connect(int max_retry, void callback(lrmd_event_data_t * op)); gboolean attrd_read_options(gpointer user_data); void attrd_cib_updated_cb(const char *event, xmlNode * msg); -GHashTable *get_meta_attrs_from_cib(xmlNode *basenode, crm_alert_entry_t *entry, guint *max_timeout); void attrd_enable_alerts(const char *script, const char *target); void attrd_alert_fini(void); int attrd_send_alerts(lrmd_t *lrmd, const char *node, uint32_t nodeid, const char *attribute_name, const char *attribute_value, GListPtr alert_list); #if HAVE_ATOMIC_ATTRD void set_alert_attribute_value(GHashTable *t, attribute_value_t *v); void send_alert_attributes_value(attribute_t *a, GHashTable *t); #endif #endif diff --git a/cib/main.c b/cib/main.c index 820692ecb3..b8af86e667 100644 --- a/cib/main.c +++ b/cib/main.c @@ -1,593 +1,585 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "common.h" #if HAVE_LIBXML2 # include #endif #ifdef HAVE_GETOPT_H # include #endif #if HAVE_BZLIB_H # include #endif extern int init_remote_listener(int port, gboolean encrypted); gboolean cib_shutdown_flag = FALSE; int cib_status = pcmk_ok; crm_cluster_t crm_cluster; #if SUPPORT_HEARTBEAT oc_ev_t *cib_ev_token; ll_cluster_t *hb_conn = NULL; extern void oc_ev_special(const oc_ev_t *, oc_ev_class_t, int); gboolean cib_register_ha(ll_cluster_t * hb_cluster, const char *client_name); #else void *hb_conn = NULL; #endif GMainLoop *mainloop = NULL; const char *cib_root = NULL; char *cib_our_uname = NULL; gboolean preserve_status = FALSE; /* volatile because it may be changed in a signal handler */ volatile gboolean cib_writes_enabled = TRUE; int remote_fd = 0; int remote_tls_fd = 0; int cib_init(void); void cib_shutdown(int nsig); gboolean startCib(const char *filename); extern int write_cib_contents(gpointer p); GHashTable *config_hash = NULL; GHashTable *local_notify_queue = NULL; char *channel1 = NULL; char *channel2 = NULL; char *channel3 = NULL; char *channel4 = NULL; char *channel5 = NULL; #define OPTARGS "maswr:V?" void cib_cleanup(void); static void cib_enable_writes(int nsig) { crm_info("(Re)enabling disk writes"); cib_writes_enabled = TRUE; } static void log_cib_client(gpointer key, gpointer value, gpointer user_data) { crm_info("Client %s", crm_client_name(value)); } /* *INDENT-OFF* */ static struct crm_option long_options[] = { /* Top-level Options */ {"help", 0, 0, '?', "\tThis text"}, {"verbose", 0, 0, 'V', "\tIncrease debug output"}, {"per-action-cib", 0, 0, 'a', "\tAdvanced use only"}, {"stand-alone", 0, 0, 's', "\tAdvanced use only"}, {"disk-writes", 0, 0, 'w', "\tAdvanced use only"}, {"cib-root", 1, 0, 'r', "\tAdvanced use only"}, {0, 0, 0, 0} }; /* *INDENT-ON* */ int main(int argc, char **argv) { int flag; int rc = 0; int index = 0; int argerr = 0; struct passwd *pwentry = NULL; crm_log_preinit(NULL, argc, argv); crm_set_options(NULL, "[options]", long_options, "Daemon for storing and replicating the cluster configuration"); crm_peer_init(); mainloop_add_signal(SIGTERM, cib_shutdown); mainloop_add_signal(SIGPIPE, cib_enable_writes); cib_writer = mainloop_add_trigger(G_PRIORITY_LOW, write_cib_contents, NULL); while (1) { flag = crm_get_option(argc, argv, &index); if (flag == -1) break; switch (flag) { case 'V': crm_bump_log_level(argc, argv); break; case 's': stand_alone = TRUE; preserve_status = TRUE; cib_writes_enabled = FALSE; pwentry = getpwnam(CRM_DAEMON_USER); CRM_CHECK(pwentry != NULL, crm_perror(LOG_ERR, "Invalid uid (%s) specified", CRM_DAEMON_USER); return 100); rc = setgid(pwentry->pw_gid); if (rc < 0) { crm_perror(LOG_ERR, "Could not set group to %d", pwentry->pw_gid); return 100; } rc = initgroups(CRM_DAEMON_GROUP, pwentry->pw_gid); if (rc < 0) { crm_perror(LOG_ERR, "Could not setup groups for user %d", pwentry->pw_uid); return 100; } rc = setuid(pwentry->pw_uid); if (rc < 0) { crm_perror(LOG_ERR, "Could not set user to %d", pwentry->pw_uid); return 100; } break; case '?': /* Help message */ crm_help(flag, EX_OK); break; case 'w': cib_writes_enabled = TRUE; break; case 'r': cib_root = optarg; break; case 'm': cib_metadata(); return 0; default: ++argerr; break; } } if (argc - optind == 1 && safe_str_eq("metadata", argv[optind])) { cib_metadata(); return 0; } if (optind > argc) { ++argerr; } if (argerr) { crm_help('?', EX_USAGE); } crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE); - if (cib_root == NULL) { - char *path = crm_strdup_printf("%s/cib.xml", CRM_CONFIG_DIR); - char *legacy = crm_strdup_printf("%s/cib.xml", CRM_LEGACY_CONFIG_DIR); - if (g_file_test(path, G_FILE_TEST_EXISTS)) { - cib_root = CRM_CONFIG_DIR; + if (cib_root == NULL) { + if ((g_file_test(CRM_CONFIG_DIR "/cib.xml", G_FILE_TEST_EXISTS) == FALSE) + && (g_file_test(CRM_LEGACY_CONFIG_DIR "/cib.xml", G_FILE_TEST_EXISTS) == TRUE)) { - } else if (g_file_test(legacy, G_FILE_TEST_EXISTS)) { + crm_notice("Using legacy config location: " CRM_LEGACY_CONFIG_DIR); cib_root = CRM_LEGACY_CONFIG_DIR; - crm_notice("Using legacy config location: %s", cib_root); } else { cib_root = CRM_CONFIG_DIR; - crm_notice("Using new config location: %s", cib_root); } - - free(legacy); - free(path); - } else { crm_notice("Using custom config location: %s", cib_root); } if (crm_is_writable(cib_root, NULL, CRM_DAEMON_USER, CRM_DAEMON_GROUP, FALSE) == FALSE) { crm_err("Bad permissions on %s. Terminating", cib_root); fprintf(stderr, "ERROR: Bad permissions on %s. See logs for details\n", cib_root); fflush(stderr); return 100; } /* read local config file */ rc = cib_init(); CRM_CHECK(crm_hash_table_size(client_connections) == 0, crm_warn("Not all clients gone at exit")); g_hash_table_foreach(client_connections, log_cib_client, NULL); cib_cleanup(); #if SUPPORT_HEARTBEAT if (hb_conn) { hb_conn->llc_ops->delete(hb_conn); } #endif crm_info("Done"); return rc; } void cib_cleanup(void) { crm_peer_destroy(); if (local_notify_queue) { g_hash_table_destroy(local_notify_queue); } crm_client_cleanup(); g_hash_table_destroy(config_hash); free(cib_our_uname); free(channel1); free(channel2); free(channel3); free(channel4); free(channel5); } unsigned long cib_num_ops = 0; const char *cib_stat_interval = "10min"; unsigned long cib_num_local = 0, cib_num_updates = 0, cib_num_fail = 0; unsigned long cib_bad_connects = 0, cib_num_timeouts = 0; #if SUPPORT_HEARTBEAT gboolean ccm_connect(void); static void ccm_connection_destroy(gpointer user_data) { crm_err("CCM connection failed... blocking while we reconnect"); CRM_ASSERT(ccm_connect()); return; } static void *ccm_library = NULL; gboolean ccm_connect(void) { gboolean did_fail = TRUE; int num_ccm_fails = 0; int max_ccm_fails = 30; int ret; int cib_ev_fd; int (*ccm_api_register) (oc_ev_t ** token) = find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_register", 1); int (*ccm_api_set_callback) (const oc_ev_t * token, oc_ev_class_t class, oc_ev_callback_t * fn, oc_ev_callback_t ** prev_fn) = find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_set_callback", 1); void (*ccm_api_special) (const oc_ev_t *, oc_ev_class_t, int) = find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_special", 1); int (*ccm_api_activate) (const oc_ev_t * token, int *fd) = find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_activate", 1); int (*ccm_api_unregister) (oc_ev_t * token) = find_library_function(&ccm_library, CCM_LIBRARY, "oc_ev_unregister", 1); static struct mainloop_fd_callbacks ccm_fd_callbacks = { .dispatch = cib_ccm_dispatch, .destroy = ccm_connection_destroy, }; while (did_fail) { did_fail = FALSE; crm_info("Registering with CCM..."); ret = (*ccm_api_register) (&cib_ev_token); if (ret != 0) { did_fail = TRUE; } if (did_fail == FALSE) { crm_trace("Setting up CCM callbacks"); ret = (*ccm_api_set_callback) (cib_ev_token, OC_EV_MEMB_CLASS, cib_ccm_msg_callback, NULL); if (ret != 0) { crm_warn("CCM callback not set"); did_fail = TRUE; } } if (did_fail == FALSE) { (*ccm_api_special) (cib_ev_token, OC_EV_MEMB_CLASS, 0); crm_trace("Activating CCM token"); ret = (*ccm_api_activate) (cib_ev_token, &cib_ev_fd); if (ret != 0) { crm_warn("CCM Activation failed"); did_fail = TRUE; } } if (did_fail) { num_ccm_fails++; (*ccm_api_unregister) (cib_ev_token); if (num_ccm_fails < max_ccm_fails) { crm_warn("CCM Connection failed %d times (%d max)", num_ccm_fails, max_ccm_fails); sleep(3); } else { crm_err("CCM Activation failed %d (max) times", num_ccm_fails); return FALSE; } } } crm_debug("CCM Activation passed... all set to go!"); mainloop_add_fd("heartbeat-ccm", G_PRIORITY_MEDIUM, cib_ev_fd, cib_ev_token, &ccm_fd_callbacks); return TRUE; } #endif #if SUPPORT_COROSYNC static void cib_cs_dispatch(cpg_handle_t handle, const struct cpg_name *groupName, uint32_t nodeid, uint32_t pid, void *msg, size_t msg_len) { uint32_t kind = 0; xmlNode *xml = NULL; const char *from = NULL; char *data = pcmk_message_common_cs(handle, nodeid, pid, msg, &kind, &from); if(data == NULL) { return; } if (kind == crm_class_cluster) { xml = string2xml(data); if (xml == NULL) { crm_err("Invalid XML: '%.120s'", data); free(data); return; } crm_xml_add(xml, F_ORIG, from); /* crm_xml_add_int(xml, F_SEQ, wrapper->id); */ cib_peer_callback(xml, NULL); } free_xml(xml); free(data); } static void cib_cs_destroy(gpointer user_data) { if (cib_shutdown_flag) { crm_info("Corosync disconnection complete"); } else { crm_err("Corosync connection lost! Exiting."); terminate_cib(__FUNCTION__, -1); } } #endif static void cib_peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data) { switch (type) { case crm_status_processes: #if !SUPPORT_PLUGIN if (cib_legacy_mode() && is_not_set(node->processes, crm_get_cluster_proc())) { uint32_t old = data? *(const uint32_t *)data : 0; if ((node->processes ^ old) & crm_proc_cpg) { crm_info("Attempting to disable legacy mode after %s left the cluster", node->uname); legacy_mode = FALSE; } } #endif break; case crm_status_uname: case crm_status_rstate: case crm_status_nstate: if (cib_shutdown_flag && (crm_active_peers() < 2) && crm_hash_table_size(client_connections) == 0) { crm_info("No more peers"); terminate_cib(__FUNCTION__, 1); } break; } } #if SUPPORT_HEARTBEAT static void cib_ha_connection_destroy(gpointer user_data) { if (cib_shutdown_flag) { crm_info("Heartbeat disconnection complete... exiting"); terminate_cib(__FUNCTION__, 0); } else { crm_err("Heartbeat connection lost! Exiting."); terminate_cib(__FUNCTION__, -1); } } #endif int cib_init(void) { if (is_openais_cluster()) { #if SUPPORT_COROSYNC crm_cluster.destroy = cib_cs_destroy; crm_cluster.cpg.cpg_deliver_fn = cib_cs_dispatch; crm_cluster.cpg.cpg_confchg_fn = pcmk_cpg_membership; #endif } else if (is_heartbeat_cluster()) { #if SUPPORT_HEARTBEAT crm_cluster.hb_dispatch = cib_ha_peer_callback; crm_cluster.destroy = cib_ha_connection_destroy; #endif } config_hash = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); if (startCib("cib.xml") == FALSE) { crm_crit("Cannot start CIB... terminating"); crm_exit(ENODATA); } if (stand_alone == FALSE) { if (is_openais_cluster()) { crm_set_status_callback(&cib_peer_update_callback); } if (crm_cluster_connect(&crm_cluster) == FALSE) { crm_crit("Cannot sign in to the cluster... terminating"); crm_exit(DAEMON_RESPAWN_STOP); } cib_our_uname = crm_cluster.uname; #if SUPPORT_HEARTBEAT if (is_heartbeat_cluster()) { gboolean was_error = FALSE; hb_conn = crm_cluster.hb_conn; if (was_error == FALSE) { if (HA_OK != hb_conn->llc_ops->set_cstatus_callback(hb_conn, cib_client_status_callback, hb_conn)) { crm_err("Cannot set cstatus callback: %s", hb_conn->llc_ops->errmsg(hb_conn)); was_error = TRUE; } } if (was_error == FALSE) { was_error = (ccm_connect() == FALSE); } if (was_error == FALSE) { /* Async get client status information in the cluster */ crm_info("Requesting the list of configured nodes"); hb_conn->llc_ops->client_status(hb_conn, NULL, CRM_SYSTEM_CIB, -1); } } #endif } else { cib_our_uname = strdup("localhost"); } cib_ipc_servers_init(&ipcs_ro, &ipcs_rw, &ipcs_shm, &ipc_ro_callbacks, &ipc_rw_callbacks); if (stand_alone) { cib_is_master = TRUE; } /* Create the mainloop and run it... */ mainloop = g_main_new(FALSE); crm_info("Starting %s mainloop", crm_system_name); g_main_run(mainloop); /* If main loop returned, clean up and exit. We disconnect in case * terminate_cib() was called with fast=1. */ crm_cluster_disconnect(&crm_cluster); cib_ipc_servers_destroy(ipcs_ro, ipcs_rw, ipcs_shm); return crm_exit(pcmk_ok); } gboolean startCib(const char *filename) { gboolean active = FALSE; xmlNode *cib = readCibXmlFile(cib_root, filename, !preserve_status); CRM_ASSERT(cib != NULL); if (activateCibXml(cib, TRUE, "start") == 0) { int port = 0; const char *port_s = NULL; active = TRUE; cib_read_config(config_hash, cib); port_s = crm_element_value(cib, "remote-tls-port"); if (port_s) { port = crm_parse_int(port_s, "0"); remote_tls_fd = init_remote_listener(port, TRUE); } port_s = crm_element_value(cib, "remote-clear-port"); if (port_s) { port = crm_parse_int(port_s, "0"); remote_fd = init_remote_listener(port, FALSE); } crm_info("CIB Initialization completed successfully"); } return active; } diff --git a/crmd/cib.c b/crmd/cib.c index 41f4f3d6e9..b10256a7d1 100644 --- a/crmd/cib.c +++ b/crmd/cib.c @@ -1,272 +1,272 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include /* for access */ #include /* for calls to open */ #include /* for calls to open */ #include /* for calls to open */ #include /* for getpwuid */ #include /* for initgroups */ #include /* for getrlimit */ #include /* for getrlimit */ #include #include #include #include #include #include #include struct crm_subsystem_s *cib_subsystem = NULL; int cib_retries = 0; static void do_cib_updated(const char *event, xmlNode * msg) { int rc = -1; int format= 1; xmlNode *patchset = get_message_xml(msg, F_CIB_UPDATE_RESULT); xmlNode *change = NULL; xmlXPathObject *xpathObj = NULL; CRM_CHECK(msg != NULL, return); crm_element_value_int(msg, F_CIB_RC, &rc); if (rc < pcmk_ok) { crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc)); return; } crm_element_value_int(patchset, "format", &format); if (format == 1) { if ((xpathObj = xpath_search( msg, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_CRMCONFIG " | " \ "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_ALERTS )) != NULL) { freeXpathObject(xpathObj); mainloop_set_trigger(config_read); } } else if (format == 2) { for (change = __xml_first_child(patchset); change != NULL; change = __xml_next(change)) { const char *xpath = crm_element_value(change, XML_DIFF_PATH); if (xpath == NULL) { continue; } /* modifying properties */ if (!strstr(xpath, "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_CRMCONFIG "/") && !strstr(xpath, "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_ALERTS)) { xmlNode *section = NULL; const char *name = NULL; /* adding notifications section */ if ((strcmp(xpath, "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION) != 0) || ((section = __xml_first_child(change)) == NULL) || ((name = crm_element_name(section)) == NULL) || (strcmp(name, XML_CIB_TAG_ALERTS) != 0)) { continue; } } mainloop_set_trigger(config_read); break; } } else { crm_warn("Unknown patch format: %d", format); } } static void revision_check_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) { int cmp = -1; xmlNode *generation = NULL; const char *revision = NULL; if (rc != pcmk_ok) { fsa_data_t *msg_data = NULL; register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); return; } generation = output; CRM_CHECK(safe_str_eq(crm_element_name(generation), XML_TAG_CIB), crm_log_xml_err(output, __FUNCTION__); return); - crm_trace("Checking our feature revision is allowed: %s", CIB_FEATURE_SET); + crm_trace("Checking our feature revision %s is allowed", CRM_FEATURE_SET); revision = crm_element_value(generation, XML_ATTR_CRM_VERSION); cmp = compare_version(revision, CRM_FEATURE_SET); if (cmp > 0) { crm_err("Shutting down because the current configuration is not supported by this version " CRM_XS " build=%s supported=%s current=%s", PACEMAKER_VERSION, CRM_FEATURE_SET, revision); /* go into a stall state */ register_fsa_error_adv(C_FSA_INTERNAL, I_SHUTDOWN, NULL, NULL, __FUNCTION__); return; } } static void do_cib_replaced(const char *event, xmlNode * msg) { crm_debug("Updating the CIB after a replace: DC=%s", AM_I_DC ? "true" : "false"); if (AM_I_DC == FALSE) { return; } else if (fsa_state == S_FINALIZE_JOIN && is_set(fsa_input_register, R_CIB_ASKED)) { /* no need to restart the join - we asked for this replace op */ return; } /* start the join process again so we get everyone's LRM status */ populate_cib_nodes(node_update_quick|node_update_all, __FUNCTION__); register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL); } /* A_CIB_STOP, A_CIB_START, O_CIB_RESTART */ void do_cib_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { CRM_ASSERT(fsa_cib_conn != NULL); if (action & A_CIB_STOP) { if (fsa_cib_conn->state != cib_disconnected && last_resource_update != 0) { crm_info("Waiting for resource update %d to complete", last_resource_update); crmd_fsa_stall(FALSE); return; } crm_info("Disconnecting CIB"); clear_bit(fsa_input_register, R_CIB_CONNECTED); fsa_cib_conn->cmds->del_notify_callback(fsa_cib_conn, T_CIB_DIFF_NOTIFY, do_cib_updated); if (fsa_cib_conn->state != cib_disconnected) { fsa_cib_conn->cmds->set_slave(fsa_cib_conn, cib_scope_local); fsa_cib_conn->cmds->signoff(fsa_cib_conn); } crm_notice("Disconnected from the CIB"); } if (action & A_CIB_START) { int rc = pcmk_ok; if (cur_state == S_STOPPING) { crm_err("Ignoring request to start the CIB after shutdown"); return; } rc = fsa_cib_conn->cmds->signon(fsa_cib_conn, CRM_SYSTEM_CRMD, cib_command_nonblocking); if (rc != pcmk_ok) { /* a short wait that usually avoids stalling the FSA */ sleep(1); rc = fsa_cib_conn->cmds->signon(fsa_cib_conn, CRM_SYSTEM_CRMD, cib_command_nonblocking); } if (rc != pcmk_ok) { crm_info("Could not connect to the CIB service: %s", pcmk_strerror(rc)); } else if (pcmk_ok != fsa_cib_conn->cmds->set_connection_dnotify(fsa_cib_conn, crmd_cib_connection_destroy)) { crm_err("Could not set dnotify callback"); } else if (pcmk_ok != fsa_cib_conn->cmds->add_notify_callback(fsa_cib_conn, T_CIB_REPLACE_NOTIFY, do_cib_replaced)) { crm_err("Could not set CIB notification callback (replace)"); } else if (pcmk_ok != fsa_cib_conn->cmds->add_notify_callback(fsa_cib_conn, T_CIB_DIFF_NOTIFY, do_cib_updated)) { crm_err("Could not set CIB notification callback (update)"); } else { set_bit(fsa_input_register, R_CIB_CONNECTED); } if (is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) { cib_retries++; crm_warn("Couldn't complete CIB registration %d" " times... pause and retry", cib_retries); if (cib_retries < 30) { crm_timer_start(wait_timer); crmd_fsa_stall(FALSE); } else { crm_err("Could not complete CIB" " registration %d times..." " hard error", cib_retries); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } else { int call_id = 0; crm_info("CIB connection established"); call_id = fsa_cib_conn->cmds->query(fsa_cib_conn, NULL, NULL, cib_scope_local); fsa_register_cib_callback(call_id, FALSE, NULL, revision_check_callback); cib_retries = 0; } } } /*! * \internal * \brief Get CIB call options to use local scope if master unavailable * * \return CIB call options */ int crmd_cib_smart_opt() { int call_opt = cib_quorum_override; if (fsa_state == S_ELECTION || fsa_state == S_PENDING) { crm_info("Sending update to local CIB in state: %s", fsa_state2string(fsa_state)); call_opt |= cib_scope_local; } return call_opt; } diff --git a/crmd/control.c b/crmd/control.c index 6b4aa79302..f8d78d4f5b 100644 --- a/crmd/control.c +++ b/crmd/control.c @@ -1,1170 +1,1171 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include +#include #include #include #include #include #include #include #include #include #include #include #include #include #include qb_ipcs_service_t *ipcs = NULL; extern gboolean crm_connect_corosync(crm_cluster_t * cluster); extern void crmd_ha_connection_destroy(gpointer user_data); void crm_shutdown(int nsig); gboolean crm_read_options(gpointer user_data); gboolean fsa_has_quorum = FALSE; crm_trigger_t *fsa_source = NULL; crm_trigger_t *config_read = NULL; bool no_quorum_suicide_escalation = FALSE; static gboolean election_timeout_popped(gpointer data) { /* Not everyone voted */ crm_info("Election failed: Declaring ourselves the winner"); register_fsa_input(C_TIMER_POPPED, I_ELECTION_DC, NULL); return FALSE; } /* A_HA_CONNECT */ void do_ha_control(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { gboolean registered = FALSE; static crm_cluster_t *cluster = NULL; if (cluster == NULL) { cluster = calloc(1, sizeof(crm_cluster_t)); } if (action & A_HA_DISCONNECT) { crm_cluster_disconnect(cluster); crm_info("Disconnected from the cluster"); set_bit(fsa_input_register, R_HA_DISCONNECTED); } if (action & A_HA_CONNECT) { crm_set_status_callback(&peer_update_callback); crm_set_autoreap(FALSE); if (is_openais_cluster()) { #if SUPPORT_COROSYNC registered = crm_connect_corosync(cluster); #endif } else if (is_heartbeat_cluster()) { #if SUPPORT_HEARTBEAT cluster->destroy = crmd_ha_connection_destroy; cluster->hb_dispatch = crmd_ha_msg_callback; registered = crm_cluster_connect(cluster); fsa_cluster_conn = cluster->hb_conn; crm_trace("Be informed of Node Status changes"); if (registered && fsa_cluster_conn->llc_ops->set_nstatus_callback(fsa_cluster_conn, crmd_ha_status_callback, fsa_cluster_conn) != HA_OK) { crm_err("Cannot set nstatus callback: %s", fsa_cluster_conn->llc_ops->errmsg(fsa_cluster_conn)); registered = FALSE; } crm_trace("Be informed of CRM Client Status changes"); if (registered && fsa_cluster_conn->llc_ops->set_cstatus_callback(fsa_cluster_conn, crmd_client_status_callback, fsa_cluster_conn) != HA_OK) { crm_err("Cannot set cstatus callback: %s", fsa_cluster_conn->llc_ops->errmsg(fsa_cluster_conn)); registered = FALSE; } if (registered) { crm_trace("Requesting an initial dump of CRMD client_status"); fsa_cluster_conn->llc_ops->client_status(fsa_cluster_conn, NULL, CRM_SYSTEM_CRMD, -1); } #endif } fsa_election = election_init(NULL, cluster->uname, 60000/*60s*/, election_timeout_popped); fsa_our_uname = cluster->uname; fsa_our_uuid = cluster->uuid; if(cluster->uuid == NULL) { crm_err("Could not obtain local uuid"); registered = FALSE; } if (registered == FALSE) { set_bit(fsa_input_register, R_HA_DISCONNECTED); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); return; } populate_cib_nodes(node_update_none, __FUNCTION__); clear_bit(fsa_input_register, R_HA_DISCONNECTED); crm_info("Connected to the cluster"); } if (action & ~(A_HA_CONNECT | A_HA_DISCONNECT)) { crm_err("Unexpected action %s in %s", fsa_action2string(action), __FUNCTION__); } } static bool need_spawn_pengine_from_crmd(void) { static int result = -1; if (result != -1) return result; if (!is_heartbeat_cluster()) { result = 0; return result; } /* NULL, or "strange" value: rather spawn from here. */ result = TRUE; crm_str_to_boolean(daemon_option("crmd_spawns_pengine"), &result); return result; } /* A_SHUTDOWN */ void do_shutdown(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { /* just in case */ set_bit(fsa_input_register, R_SHUTDOWN); if (need_spawn_pengine_from_crmd()) { if (is_set(fsa_input_register, pe_subsystem->flag_connected)) { crm_info("Terminating the %s", pe_subsystem->name); if (stop_subsystem(pe_subsystem, TRUE) == FALSE) { /* It's gone ... */ crm_err("Faking %s exit", pe_subsystem->name); clear_bit(fsa_input_register, pe_subsystem->flag_connected); } else { crm_info("Waiting for subsystems to exit"); crmd_fsa_stall(FALSE); } } crm_info("All subsystems stopped, continuing"); } if (stonith_api) { /* Prevent it from coming up again */ clear_bit(fsa_input_register, R_ST_REQUIRED); crm_info("Disconnecting STONITH..."); stonith_api->cmds->disconnect(stonith_api); } } /* A_SHUTDOWN_REQ */ void do_shutdown_req(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { xmlNode *msg = NULL; set_bit(fsa_input_register, R_SHUTDOWN); crm_info("Sending shutdown request to all peers (DC is %s)", (fsa_our_dc? fsa_our_dc : "not set")); msg = create_request(CRM_OP_SHUTDOWN_REQ, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL); /* set_bit(fsa_input_register, R_STAYDOWN); */ if (send_cluster_message(NULL, crm_msg_crmd, msg, TRUE) == FALSE) { register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } free_xml(msg); } extern crm_ipc_t *attrd_ipc; extern char *max_generation_from; extern xmlNode *max_generation_xml; extern GHashTable *resource_history; extern GHashTable *voted; extern char *te_client_id; extern regex_t *version_format_regex; void log_connected_client(gpointer key, gpointer value, gpointer user_data); void log_connected_client(gpointer key, gpointer value, gpointer user_data) { crm_client_t *client = value; crm_err("%s is still connected at exit", crm_client_name(client)); } int crmd_fast_exit(int rc) { if (is_set(fsa_input_register, R_STAYDOWN)) { crm_warn("Inhibiting respawn "CRM_XS" remapping exit code %d to %d", rc, DAEMON_RESPAWN_STOP); rc = DAEMON_RESPAWN_STOP; } if (rc == pcmk_ok && is_set(fsa_input_register, R_IN_RECOVERY)) { crm_err("Could not recover from internal error"); rc = pcmk_err_generic; } return crm_exit(rc); } int crmd_exit(int rc) { GListPtr gIter = NULL; GMainLoop *mloop = crmd_mainloop; static bool in_progress = FALSE; if(in_progress && rc == 0) { crm_debug("Exit is already in progress"); return rc; } else if(in_progress) { crm_notice("Error during shutdown process, terminating now with status %d: %s", rc, pcmk_strerror(rc)); crm_write_blackbox(SIGTRAP, NULL); crmd_fast_exit(rc); } in_progress = TRUE; crm_trace("Preparing to exit: %d", rc); /* Suppress secondary errors resulting from us disconnecting everything */ set_bit(fsa_input_register, R_HA_DISCONNECTED); /* Close all IPC servers and clients to ensure any and all shared memory files are cleaned up */ if(ipcs) { crm_trace("Closing IPC server"); mainloop_del_ipc_server(ipcs); ipcs = NULL; } if (attrd_ipc) { crm_trace("Closing attrd connection"); crm_ipc_close(attrd_ipc); crm_ipc_destroy(attrd_ipc); attrd_ipc = NULL; } if (pe_subsystem && pe_subsystem->client && pe_subsystem->client->ipcs) { crm_trace("Disconnecting Policy Engine"); qb_ipcs_disconnect(pe_subsystem->client->ipcs); } if(stonith_api) { crm_trace("Disconnecting fencing API"); clear_bit(fsa_input_register, R_ST_REQUIRED); stonith_api->cmds->free(stonith_api); stonith_api = NULL; } if (rc == pcmk_ok && crmd_mainloop == NULL) { crm_debug("No mainloop detected"); rc = EPROTO; } /* On an error, just get out. * * Otherwise, make the effort to have mainloop exit gracefully so * that it (mostly) cleans up after itself and valgrind has less * to report on - allowing real errors stand out */ if(rc != pcmk_ok) { crm_notice("Forcing immediate exit with status %d: %s", rc, pcmk_strerror(rc)); crm_write_blackbox(SIGTRAP, NULL); return crmd_fast_exit(rc); } /* Clean up as much memory as possible for valgrind */ for (gIter = fsa_message_queue; gIter != NULL; gIter = gIter->next) { fsa_data_t *fsa_data = gIter->data; crm_info("Dropping %s: [ state=%s cause=%s origin=%s ]", fsa_input2string(fsa_data->fsa_input), fsa_state2string(fsa_state), fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin); delete_fsa_input(fsa_data); } clear_bit(fsa_input_register, R_MEMBERSHIP); g_list_free(fsa_message_queue); fsa_message_queue = NULL; free(pe_subsystem); pe_subsystem = NULL; free(te_subsystem); te_subsystem = NULL; free(cib_subsystem); cib_subsystem = NULL; if (version_format_regex) { regfree(version_format_regex); free(version_format_regex); } election_fini(fsa_election); fsa_election = NULL; /* Tear down the CIB connection, but don't free it yet -- it could be used * when we drain the mainloop later. */ cib_free_callbacks(fsa_cib_conn); fsa_cib_conn->cmds->signoff(fsa_cib_conn); verify_stopped(fsa_state, LOG_WARNING); clear_bit(fsa_input_register, R_LRM_CONNECTED); lrm_state_destroy_all(); /* This basically will not work, since mainloop has a reference to it */ mainloop_destroy_trigger(fsa_source); fsa_source = NULL; mainloop_destroy_trigger(config_read); config_read = NULL; mainloop_destroy_trigger(stonith_reconnect); stonith_reconnect = NULL; mainloop_destroy_trigger(transition_trigger); transition_trigger = NULL; crm_client_cleanup(); crm_peer_destroy(); crm_timer_stop(transition_timer); crm_timer_stop(integration_timer); crm_timer_stop(finalization_timer); crm_timer_stop(election_trigger); election_timeout_stop(fsa_election); crm_timer_stop(shutdown_escalation_timer); crm_timer_stop(wait_timer); crm_timer_stop(recheck_timer); free(transition_timer); transition_timer = NULL; free(integration_timer); integration_timer = NULL; free(finalization_timer); finalization_timer = NULL; free(election_trigger); election_trigger = NULL; free(shutdown_escalation_timer); shutdown_escalation_timer = NULL; free(wait_timer); wait_timer = NULL; free(recheck_timer); recheck_timer = NULL; free(fsa_our_dc_version); fsa_our_dc_version = NULL; free(fsa_our_uname); fsa_our_uname = NULL; free(fsa_our_uuid); fsa_our_uuid = NULL; free(fsa_our_dc); fsa_our_dc = NULL; free(fsa_cluster_name); fsa_cluster_name = NULL; free(te_uuid); te_uuid = NULL; free(te_client_id); te_client_id = NULL; free(fsa_pe_ref); fsa_pe_ref = NULL; free(failed_stop_offset); failed_stop_offset = NULL; free(failed_start_offset); failed_start_offset = NULL; free(max_generation_from); max_generation_from = NULL; free_xml(max_generation_xml); max_generation_xml = NULL; mainloop_destroy_signal(SIGPIPE); mainloop_destroy_signal(SIGUSR1); mainloop_destroy_signal(SIGTERM); mainloop_destroy_signal(SIGTRAP); /* leave SIGCHLD engaged as we might still want to drain some service-actions */ if (mloop) { GMainContext *ctx = g_main_loop_get_context(crmd_mainloop); /* Don't re-enter this block */ crmd_mainloop = NULL; crmd_drain_alerts(ctx); /* no signals on final draining anymore */ mainloop_destroy_signal(SIGCHLD); crm_trace("Draining mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx)); { int lpc = 0; while((g_main_context_pending(ctx) && lpc < 10)) { lpc++; crm_trace("Iteration %d", lpc); g_main_context_dispatch(ctx); } } crm_trace("Closing mainloop %d %d", g_main_loop_is_running(mloop), g_main_context_pending(ctx)); g_main_loop_quit(mloop); #if SUPPORT_HEARTBEAT /* Do this only after g_main_loop_quit(). * * This interface was broken (incomplete) since it was introduced. * ->delete() does cleanup and free most of it, but it does not * actually remove and destroy the corresponding GSource, so the next * prepare/check iteratioin would find a corrupt (because partially * freed) GSource, and segfault. * * Apparently one was supposed to store the GSource as returned by * G_main_add_ll_cluster(), and g_source_destroy() that "by hand". * * But no-one ever did this, not even in the old hb code when this was * introduced. * * Note that fsa_cluster_conn was set as an "alias" to cluster->hb_conn * in do_ha_control() right after crm_cluster_connect(), and only * happens to still point at that object, because do_ha_control() does * not reset it to NULL after crm_cluster_disconnect() above does * reset cluster->hb_conn to NULL. * Not sure if that's something to cleanup, too. * * I'll try to fix this up in heartbeat proper, so ->delete * will actually remove, and destroy, and unref, and free this thing. * Doing so after g_main_loop_quit() is valid with both old, * and eventually fixed heartbeat. * * If we introduce the "by hand" destroy/remove/unref, * this may break again once heartbeat is fixed :-( * * -- Lars Ellenberg */ if (fsa_cluster_conn) { crm_trace("Deleting heartbeat api object"); fsa_cluster_conn->llc_ops->delete(fsa_cluster_conn); fsa_cluster_conn = NULL; } #endif /* Won't do anything yet, since we're inside it now */ g_main_loop_unref(mloop); crm_trace("Done %d", rc); } else { mainloop_destroy_signal(SIGCHLD); } cib_delete(fsa_cib_conn); fsa_cib_conn = NULL; throttle_fini(); /* Graceful */ return rc; } /* A_EXIT_0, A_EXIT_1 */ void do_exit(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { int exit_code = pcmk_ok; int log_level = LOG_INFO; const char *exit_type = "gracefully"; if (action & A_EXIT_1) { /* exit_code = pcmk_err_generic; */ log_level = LOG_ERR; exit_type = "forcefully"; exit_code = pcmk_err_generic; } verify_stopped(cur_state, LOG_ERR); do_crm_log(log_level, "Performing %s - %s exiting the CRMd", fsa_action2string(action), exit_type); crm_info("[%s] stopped (%d)", crm_system_name, exit_code); crmd_exit(exit_code); } static void sigpipe_ignore(int nsig) { return; } /* A_STARTUP */ void do_startup(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { int was_error = 0; crm_debug("Registering Signal Handlers"); mainloop_add_signal(SIGTERM, crm_shutdown); mainloop_add_signal(SIGPIPE, sigpipe_ignore); fsa_source = mainloop_add_trigger(G_PRIORITY_HIGH, crm_fsa_trigger, NULL); config_read = mainloop_add_trigger(G_PRIORITY_HIGH, crm_read_options, NULL); transition_trigger = mainloop_add_trigger(G_PRIORITY_LOW, te_graph_trigger, NULL); crm_debug("Creating CIB and LRM objects"); fsa_cib_conn = cib_new(); lrm_state_init_local(); /* set up the timers */ transition_timer = calloc(1, sizeof(fsa_timer_t)); integration_timer = calloc(1, sizeof(fsa_timer_t)); finalization_timer = calloc(1, sizeof(fsa_timer_t)); election_trigger = calloc(1, sizeof(fsa_timer_t)); shutdown_escalation_timer = calloc(1, sizeof(fsa_timer_t)); wait_timer = calloc(1, sizeof(fsa_timer_t)); recheck_timer = calloc(1, sizeof(fsa_timer_t)); if (election_trigger != NULL) { election_trigger->source_id = 0; election_trigger->period_ms = -1; election_trigger->fsa_input = I_DC_TIMEOUT; election_trigger->callback = crm_timer_popped; election_trigger->repeat = FALSE; } else { was_error = TRUE; } if (transition_timer != NULL) { transition_timer->source_id = 0; transition_timer->period_ms = -1; transition_timer->fsa_input = I_PE_CALC; transition_timer->callback = crm_timer_popped; transition_timer->repeat = FALSE; } else { was_error = TRUE; } if (integration_timer != NULL) { integration_timer->source_id = 0; integration_timer->period_ms = -1; integration_timer->fsa_input = I_INTEGRATED; integration_timer->callback = crm_timer_popped; integration_timer->repeat = FALSE; } else { was_error = TRUE; } if (finalization_timer != NULL) { finalization_timer->source_id = 0; finalization_timer->period_ms = -1; finalization_timer->fsa_input = I_FINALIZED; finalization_timer->callback = crm_timer_popped; finalization_timer->repeat = FALSE; /* for possible enabling... a bug in the join protocol left * a slave in S_PENDING while we think it's in S_NOT_DC * * raising I_FINALIZED put us into a transition loop which is * never resolved. * in this loop we continually send probes which the node * NACK's because it's in S_PENDING * * if we have nodes where heartbeat is active but the * CRM is not... then this will be handled in the * integration phase */ finalization_timer->fsa_input = I_ELECTION; } else { was_error = TRUE; } if (shutdown_escalation_timer != NULL) { shutdown_escalation_timer->source_id = 0; shutdown_escalation_timer->period_ms = -1; shutdown_escalation_timer->fsa_input = I_STOP; shutdown_escalation_timer->callback = crm_timer_popped; shutdown_escalation_timer->repeat = FALSE; } else { was_error = TRUE; } if (wait_timer != NULL) { wait_timer->source_id = 0; wait_timer->period_ms = 2000; wait_timer->fsa_input = I_NULL; wait_timer->callback = crm_timer_popped; wait_timer->repeat = FALSE; } else { was_error = TRUE; } if (recheck_timer != NULL) { recheck_timer->source_id = 0; recheck_timer->period_ms = -1; recheck_timer->fsa_input = I_PE_CALC; recheck_timer->callback = crm_timer_popped; recheck_timer->repeat = FALSE; } else { was_error = TRUE; } /* set up the sub systems */ cib_subsystem = calloc(1, sizeof(struct crm_subsystem_s)); te_subsystem = calloc(1, sizeof(struct crm_subsystem_s)); pe_subsystem = calloc(1, sizeof(struct crm_subsystem_s)); if (cib_subsystem != NULL) { cib_subsystem->pid = -1; cib_subsystem->name = CRM_SYSTEM_CIB; cib_subsystem->flag_connected = R_CIB_CONNECTED; cib_subsystem->flag_required = R_CIB_REQUIRED; } else { was_error = TRUE; } if (te_subsystem != NULL) { te_subsystem->pid = -1; te_subsystem->name = CRM_SYSTEM_TENGINE; te_subsystem->flag_connected = R_TE_CONNECTED; te_subsystem->flag_required = R_TE_REQUIRED; } else { was_error = TRUE; } if (pe_subsystem != NULL) { pe_subsystem->pid = -1; pe_subsystem->path = CRM_DAEMON_DIR; pe_subsystem->name = CRM_SYSTEM_PENGINE; pe_subsystem->command = CRM_DAEMON_DIR "/" CRM_SYSTEM_PENGINE; pe_subsystem->args = NULL; pe_subsystem->flag_connected = R_PE_CONNECTED; pe_subsystem->flag_required = R_PE_REQUIRED; } else { was_error = TRUE; } if (was_error == FALSE && need_spawn_pengine_from_crmd()) { if (start_subsystem(pe_subsystem) == FALSE) { was_error = TRUE; } } if (was_error) { register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } } static int32_t crmd_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid) { crm_trace("Connection %p", c); if (crm_client_new(c, uid, gid) == NULL) { return -EIO; } return 0; } static void crmd_ipc_created(qb_ipcs_connection_t * c) { crm_trace("Connection %p", c); } static int32_t crmd_ipc_dispatch(qb_ipcs_connection_t * c, void *data, size_t size) { uint32_t id = 0; uint32_t flags = 0; crm_client_t *client = crm_client_get(c); xmlNode *msg = crm_ipcs_recv(client, data, size, &id, &flags); crm_trace("Invoked: %s", crm_client_name(client)); crm_ipcs_send_ack(client, id, flags, "ack", __FUNCTION__, __LINE__); if (msg == NULL) { return 0; } #if ENABLE_ACL CRM_ASSERT(client->user != NULL); crm_acl_get_set_user(msg, F_CRM_USER, client->user); #endif crm_trace("Processing msg from %s", crm_client_name(client)); crm_log_xml_trace(msg, "CRMd[inbound]"); crm_xml_add(msg, F_CRM_SYS_FROM, client->id); if (crmd_authorize_message(msg, client, NULL)) { route_message(C_IPC_MESSAGE, msg); } trigger_fsa(fsa_source); free_xml(msg); return 0; } static int32_t crmd_ipc_closed(qb_ipcs_connection_t * c) { crm_client_t *client = crm_client_get(c); struct crm_subsystem_s *the_subsystem = NULL; if (client == NULL) { return 0; } crm_trace("Connection %p", c); if (client->userdata == NULL) { crm_trace("Client hadn't registered with us yet"); } else if (strcasecmp(CRM_SYSTEM_PENGINE, client->userdata) == 0) { the_subsystem = pe_subsystem; } else if (strcasecmp(CRM_SYSTEM_TENGINE, client->userdata) == 0) { the_subsystem = te_subsystem; } else if (strcasecmp(CRM_SYSTEM_CIB, client->userdata) == 0) { the_subsystem = cib_subsystem; } if (the_subsystem != NULL) { the_subsystem->source = NULL; the_subsystem->client = NULL; crm_info("Received HUP from %s:[%d]", the_subsystem->name, the_subsystem->pid); } else { /* else that was a transient client */ crm_trace("Received HUP from transient client"); } crm_trace("Disconnecting client %s (%p)", crm_client_name(client), client); free(client->userdata); crm_client_destroy(client); trigger_fsa(fsa_source); return 0; } static void crmd_ipc_destroy(qb_ipcs_connection_t * c) { crm_trace("Connection %p", c); crmd_ipc_closed(c); } /* A_STOP */ void do_stop(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { crm_trace("Closing IPC server"); mainloop_del_ipc_server(ipcs); ipcs = NULL; register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL); } /* A_STARTED */ void do_started(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { static struct qb_ipcs_service_handlers crmd_callbacks = { .connection_accept = crmd_ipc_accept, .connection_created = crmd_ipc_created, .msg_process = crmd_ipc_dispatch, .connection_closed = crmd_ipc_closed, .connection_destroyed = crmd_ipc_destroy }; if (cur_state != S_STARTING) { crm_err("Start cancelled... %s", fsa_state2string(cur_state)); return; } else if (is_set(fsa_input_register, R_MEMBERSHIP) == FALSE) { crm_info("Delaying start, no membership data (%.16llx)", R_MEMBERSHIP); crmd_fsa_stall(TRUE); return; } else if (is_set(fsa_input_register, R_LRM_CONNECTED) == FALSE) { crm_info("Delaying start, LRM not connected (%.16llx)", R_LRM_CONNECTED); crmd_fsa_stall(TRUE); return; } else if (is_set(fsa_input_register, R_CIB_CONNECTED) == FALSE) { crm_info("Delaying start, CIB not connected (%.16llx)", R_CIB_CONNECTED); crmd_fsa_stall(TRUE); return; } else if (is_set(fsa_input_register, R_READ_CONFIG) == FALSE) { crm_info("Delaying start, Config not read (%.16llx)", R_READ_CONFIG); crmd_fsa_stall(TRUE); return; } else if (is_set(fsa_input_register, R_PEER_DATA) == FALSE) { /* try reading from HA */ crm_info("Delaying start, No peer data (%.16llx)", R_PEER_DATA); #if SUPPORT_HEARTBEAT if (is_heartbeat_cluster()) { HA_Message *msg = NULL; crm_trace("Looking for a HA message"); msg = fsa_cluster_conn->llc_ops->readmsg(fsa_cluster_conn, 0); if (msg != NULL) { crm_trace("There was a HA message"); ha_msg_del(msg); } } #endif crmd_fsa_stall(TRUE); return; } crm_debug("Init server comms"); ipcs = crmd_ipc_server_init(&crmd_callbacks); if (ipcs == NULL) { crm_err("Failed to create IPC server: shutting down and inhibiting respawn"); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); } if (stonith_reconnect == NULL) { int dummy; stonith_reconnect = mainloop_add_trigger(G_PRIORITY_LOW, te_connect_stonith, &dummy); } set_bit(fsa_input_register, R_ST_REQUIRED); mainloop_set_trigger(stonith_reconnect); crm_notice("The local CRM is operational"); clear_bit(fsa_input_register, R_STARTING); register_fsa_input(msg_data->fsa_cause, I_PENDING, NULL); } /* A_RECOVER */ void do_recover(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { set_bit(fsa_input_register, R_IN_RECOVERY); crm_warn("Fast-tracking shutdown in response to errors"); register_fsa_input(C_FSA_INTERNAL, I_TERMINATE, NULL); } /* *INDENT-OFF* */ pe_cluster_option crmd_opts[] = { /* name, old-name, validate, values, default, short description, long description */ { "dc-version", NULL, "string", NULL, "none", NULL, "Version of Pacemaker on the cluster's DC.", "Includes the hash which identifies the exact changeset it was built from. Used for diagnostic purposes." }, { "cluster-infrastructure", NULL, "string", NULL, "heartbeat", NULL, "The messaging stack on which Pacemaker is currently running.", "Used for informational and diagnostic purposes." }, { XML_CONFIG_ATTR_DC_DEADTIME, "dc_deadtime", "time", NULL, "20s", &check_time, "How long to wait for a response from other nodes during startup.", "The \"correct\" value will depend on the speed/load of your network and the type of switches used." }, { XML_CONFIG_ATTR_RECHECK, "cluster_recheck_interval", "time", "Zero disables polling. Positive values are an interval in seconds (unless other SI units are specified. eg. 5min)", "15min", &check_timer, "Polling interval for time based changes to options, resource parameters and constraints.", "The Cluster is primarily event driven, however the configuration can have elements that change based on time." " To ensure these changes take effect, we can optionally poll the cluster's status for changes." }, #ifdef RHEL7_COMPAT /* These options were superseded by the alerts feature and now are just an * alternate interface to it. It was never released upstream, but was * released in RHEL 7, so we allow it to be enabled at compile-time by * defining RHEL7_COMPAT. */ { "notification-agent", NULL, "string", NULL, "/dev/null", &check_script, "Deprecated", "Use alert path in alerts section instead" }, { "notification-recipient", NULL, "string", NULL, "", NULL, "Deprecated", "Use recipient value in alerts section instead" }, #endif { "load-threshold", NULL, "percentage", NULL, "80%", &check_utilization, "The maximum amount of system resources that should be used by nodes in the cluster", "The cluster will slow down its recovery process when the amount of system resources used" " (currently CPU) approaches this limit", }, { "node-action-limit", NULL, "integer", NULL, "0", &check_number, "The maximum number of jobs that can be scheduled per node. Defaults to 2x cores"}, { XML_CONFIG_ATTR_ELECTION_FAIL, "election_timeout", "time", NULL, "2min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." }, { XML_CONFIG_ATTR_FORCE_QUIT, "shutdown_escalation", "time", NULL, "20min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." }, { "crmd-integration-timeout", NULL, "time", NULL, "3min", &check_timer, "*** Advanced Use Only ***.", "If need to adjust this value, it probably indicates the presence of a bug." }, { "crmd-finalization-timeout", NULL, "time", NULL, "30min", &check_timer, "*** Advanced Use Only ***.", "If you need to adjust this value, it probably indicates the presence of a bug." }, { "crmd-transition-delay", NULL, "time", NULL, "0s", &check_timer, "*** Advanced Use Only ***\n" "Enabling this option will slow down cluster recovery under all conditions", "Delay cluster recovery for the configured interval to allow for additional/related events to occur.\n" "Useful if your configuration is sensitive to the order in which ping updates arrive." }, { "stonith-watchdog-timeout", NULL, "time", NULL, NULL, &check_sbd_timeout, "How long to wait before we can assume nodes are safely down", NULL }, { "stonith-max-attempts",NULL,"integer",NULL,"10",&check_positive_number, "How many times stonith can fail before it will no longer be attempted on a target" }, { "no-quorum-policy", "no_quorum_policy", "enum", "stop, freeze, ignore, suicide", "stop", &check_quorum, NULL, NULL }, #if SUPPORT_PLUGIN { XML_ATTR_EXPECTED_VOTES, NULL, "integer", NULL, "2", &check_number, "The number of nodes expected to be in the cluster", "Used to calculate quorum in openais based clusters." }, #endif }; /* *INDENT-ON* */ void crmd_metadata(void) { config_metadata("CRM Daemon", "1.0", "CRM Daemon Options", "This is a fake resource that details the options that can be configured for the CRM Daemon.", crmd_opts, DIMOF(crmd_opts)); } static void verify_crmd_options(GHashTable * options) { verify_all_options(options, crmd_opts, DIMOF(crmd_opts)); } static const char * crmd_pref(GHashTable * options, const char *name) { return get_cluster_pref(options, crmd_opts, DIMOF(crmd_opts), name); } static void config_query_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) { -#ifdef RHEL7_COMPAT - const char *script = NULL; -#endif const char *value = NULL; GHashTable *config_hash = NULL; crm_time_t *now = crm_time_new(NULL); xmlNode *crmconfig = NULL; xmlNode *alerts = NULL; if (rc != pcmk_ok) { fsa_data_t *msg_data = NULL; crm_err("Local CIB query resulted in an error: %s", pcmk_strerror(rc)); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); if (rc == -EACCES || rc == -pcmk_err_schema_validation) { crm_err("The cluster is mis-configured - shutting down and staying down"); set_bit(fsa_input_register, R_STAYDOWN); } goto bail; } crmconfig = output; if ((crmconfig) && (crm_element_name(crmconfig)) && (strcmp(crm_element_name(crmconfig), XML_CIB_TAG_CRMCONFIG) != 0)) { crmconfig = first_named_child(crmconfig, XML_CIB_TAG_CRMCONFIG); } if (!crmconfig) { fsa_data_t *msg_data = NULL; crm_err("Local CIB query for " XML_CIB_TAG_CRMCONFIG " section failed"); register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL); goto bail; } crm_debug("Call %d : Parsing CIB options", call_id); config_hash = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); unpack_instance_attributes(crmconfig, crmconfig, XML_CIB_TAG_PROPSET, NULL, config_hash, CIB_OPTIONS_FIRST, FALSE, now); verify_crmd_options(config_hash); #ifdef RHEL7_COMPAT - script = crmd_pref(config_hash, "notification-agent"); - value = crmd_pref(config_hash, "notification-recipient"); - crmd_enable_alerts(script, value); + { + const char *script = crmd_pref(config_hash, "notification-agent"); + const char *recip = crmd_pref(config_hash, "notification-recipient"); + + pe_enable_legacy_alerts(script, recip); + } #endif value = crmd_pref(config_hash, XML_CONFIG_ATTR_DC_DEADTIME); election_trigger->period_ms = crm_get_msec(value); value = crmd_pref(config_hash, "node-action-limit"); /* Also checks migration-limit */ throttle_update_job_max(value); value = crmd_pref(config_hash, "load-threshold"); if(value) { throttle_set_load_target(strtof(value, NULL) / 100.0); } value = crmd_pref(config_hash, "no-quorum-policy"); if (safe_str_eq(value, "suicide") && pcmk_locate_sbd()) { no_quorum_suicide_escalation = TRUE; } value = crmd_pref(config_hash,"stonith-max-attempts"); update_stonith_max_attempts(value); value = crmd_pref(config_hash, XML_CONFIG_ATTR_FORCE_QUIT); shutdown_escalation_timer->period_ms = crm_get_msec(value); /* How long to declare an election over - even if not everyone voted */ crm_debug("Shutdown escalation occurs after: %dms", shutdown_escalation_timer->period_ms); value = crmd_pref(config_hash, XML_CONFIG_ATTR_ELECTION_FAIL); election_timeout_set_period(fsa_election, crm_get_msec(value)); value = crmd_pref(config_hash, XML_CONFIG_ATTR_RECHECK); recheck_timer->period_ms = crm_get_msec(value); crm_debug("Checking for expired actions every %dms", recheck_timer->period_ms); value = crmd_pref(config_hash, "crmd-transition-delay"); transition_timer->period_ms = crm_get_msec(value); value = crmd_pref(config_hash, "crmd-integration-timeout"); integration_timer->period_ms = crm_get_msec(value); value = crmd_pref(config_hash, "crmd-finalization-timeout"); finalization_timer->period_ms = crm_get_msec(value); #if SUPPORT_COROSYNC if (is_classic_ais_cluster()) { value = crmd_pref(config_hash, XML_ATTR_EXPECTED_VOTES); crm_debug("Sending expected-votes=%s to corosync", value); send_cluster_text(crm_class_quorum, value, TRUE, NULL, crm_msg_ais); } #endif free(fsa_cluster_name); fsa_cluster_name = NULL; value = g_hash_table_lookup(config_hash, "cluster-name"); if (value) { fsa_cluster_name = strdup(value); } alerts = first_named_child(output, XML_CIB_TAG_ALERTS); - parse_alerts(alerts); + pe_unpack_alerts(alerts); set_bit(fsa_input_register, R_READ_CONFIG); crm_trace("Triggering FSA: %s", __FUNCTION__); mainloop_set_trigger(fsa_source); g_hash_table_destroy(config_hash); bail: crm_time_free(now); } gboolean crm_read_options(gpointer user_data) { int call_id = fsa_cib_conn->cmds->query(fsa_cib_conn, "//" XML_CIB_TAG_CRMCONFIG " | //" XML_CIB_TAG_ALERTS, NULL, cib_xpath | cib_scope_local); fsa_register_cib_callback(call_id, FALSE, NULL, config_query_callback); crm_trace("Querying the CIB... call %d", call_id); return TRUE; } /* A_READCONFIG */ void do_read_config(long long action, enum crmd_fsa_cause cause, enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data) { throttle_init(); mainloop_set_trigger(config_read); } void crm_shutdown(int nsig) { if (crmd_mainloop != NULL && g_main_is_running(crmd_mainloop)) { if (is_set(fsa_input_register, R_SHUTDOWN)) { crm_err("Escalating the shutdown"); register_fsa_input_before(C_SHUTDOWN, I_ERROR, NULL); } else { set_bit(fsa_input_register, R_SHUTDOWN); register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL); if (shutdown_escalation_timer->period_ms < 1) { const char *value = crmd_pref(NULL, XML_CONFIG_ATTR_FORCE_QUIT); int msec = crm_get_msec(value); crm_debug("Using default shutdown escalation: %dms", msec); shutdown_escalation_timer->period_ms = msec; } /* can't rely on this... */ crm_notice("Shutting down cluster resource manager " CRM_XS " limit=%dms", shutdown_escalation_timer->period_ms); crm_timer_start(shutdown_escalation_timer); } } else { crm_info("exit from shutdown"); crmd_exit(pcmk_ok); } } diff --git a/crmd/crmd_alerts.c b/crmd/crmd_alerts.c index 6dda4f66bf..306d0ea4ab 100644 --- a/crmd/crmd_alerts.c +++ b/crmd/crmd_alerts.c @@ -1,408 +1,219 @@ /* * Copyright (C) 2015 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include -#include #include "crmd_alerts.h" #include "crmd_messages.h" #include #include +#include -static char *notify_script = NULL; -static char *notify_target = NULL; static int alerts_inflight = 0; static gboolean draining_alerts = FALSE; -/* - * synchronize local data with cib - */ - -static GHashTable * -get_meta_attrs_from_cib(xmlNode *basenode, crm_alert_entry_t *entry, - guint *max_timeout) -{ - GHashTable *config_hash = - g_hash_table_new_full(crm_str_hash, g_str_equal, - g_hash_destroy_str, g_hash_destroy_str); - crm_time_t *now = crm_time_new(NULL); - const char *value = NULL; - - unpack_instance_attributes(basenode, basenode, XML_TAG_META_SETS, NULL, - config_hash, NULL, FALSE, now); - - value = g_hash_table_lookup(config_hash, XML_ALERT_ATTR_TIMEOUT); - if (value) { - entry->timeout = crm_get_msec(value); - if (entry->timeout <= 0) { - if (entry->timeout == 0) { - crm_trace("Setting timeout to default %dmsec", - CRM_ALERT_DEFAULT_TIMEOUT_MS); - } else { - crm_warn("Invalid timeout value setting to default %dmsec", - CRM_ALERT_DEFAULT_TIMEOUT_MS); - } - entry->timeout = CRM_ALERT_DEFAULT_TIMEOUT_MS; - } else { - crm_trace("Found timeout %dmsec", entry->timeout); - } - if (entry->timeout > *max_timeout) { - *max_timeout = entry->timeout; - } - } - value = g_hash_table_lookup(config_hash, XML_ALERT_ATTR_TSTAMP_FORMAT); - if (value) { - /* hard to do any checks here as merely anything can - * can be a valid time-format-string - */ - entry->tstamp_format = (char *) value; - crm_trace("Found timestamp format string '%s'", value); - } - - value = g_hash_table_lookup(config_hash, XML_ALERT_ATTR_SELECT_KIND); - if (value) { - entry->select_kind_orig = (char *) value; - entry->select_kind = g_strsplit((char *) value, ",", 0); - crm_trace("Found select_kind string '%s'", (char *) value); - } - - crm_time_free(now); - return config_hash; /* keep hash as long as strings are needed */ -} - -void -parse_alerts(xmlNode *alerts) -{ - xmlNode *alert; - crm_alert_entry_t entry; - guint max_timeout = 0; - - crm_free_alert_list(); - crm_alert_max_alert_timeout = CRM_ALERT_DEFAULT_TIMEOUT_MS; - if (crm_alert_kind_default == NULL) { - crm_alert_kind_default = g_strsplit(CRM_ALERT_KIND_DEFAULT, ",", 0); - } - - if (alerts) { - crm_info("We have an alerts section in the cib"); - - if (notify_script) { - crm_warn("Cib contains configuration for Legacy Notifications " - "which is overruled by alerts section"); - } - } else { - crm_info("No optional alerts section in cib"); - - if (notify_script) { - entry = (crm_alert_entry_t) { - .id = (char *) "legacy_notification", - .path = notify_script, - .timeout = CRM_ALERT_DEFAULT_TIMEOUT_MS, - .recipient = notify_target, - .select_kind_orig = NULL, - .select_kind = NULL, - .select_attribute_name_orig = NULL, - .select_attribute_name = NULL - }; - crm_add_dup_alert_list_entry(&entry); - crm_info("Legacy Notifications enabled"); - } - - return; - } - - for (alert = first_named_child(alerts, XML_CIB_TAG_ALERT); - alert; alert = __xml_next(alert)) { - xmlNode *recipient; - int recipients = 0, envvars = 0; - GHashTable *config_hash = NULL; - - entry = (crm_alert_entry_t) { - .id = (char *) crm_element_value(alert, XML_ATTR_ID), - .path = (char *) crm_element_value(alert, XML_ALERT_ATTR_PATH), - .timeout = CRM_ALERT_DEFAULT_TIMEOUT_MS, - .tstamp_format = (char *) CRM_ALERT_DEFAULT_TSTAMP_FORMAT, - .select_kind_orig = NULL, - .select_kind = NULL, - .select_attribute_name_orig = NULL, - .select_attribute_name = NULL - }; - - crm_get_envvars_from_cib(alert, - &entry, - &envvars); - - config_hash = - get_meta_attrs_from_cib(alert, &entry, &max_timeout); - - crm_debug("Found alert: id=%s, path=%s, timeout=%d, " - "tstamp_format=%s, select_kind=%s, %d additional environment variables", - entry.id, entry.path, entry.timeout, - entry.tstamp_format, entry.select_kind_orig, envvars); - - for (recipient = first_named_child(alert, - XML_CIB_TAG_ALERT_RECIPIENT); - recipient; recipient = __xml_next(recipient)) { - int envvars_added = 0; - - entry.recipient = (char *) crm_element_value(recipient, - XML_ALERT_ATTR_REC_VALUE); - recipients++; - - crm_get_envvars_from_cib(recipient, - &entry, - &envvars_added); - - { - crm_alert_entry_t recipient_entry = entry; - GHashTable *config_hash = - get_meta_attrs_from_cib(recipient, - &recipient_entry, - &max_timeout); - - crm_add_dup_alert_list_entry(&recipient_entry); - - crm_debug("Alert has recipient: id=%s, value=%s, " - "%d additional environment variables", - crm_element_value(recipient, XML_ATTR_ID), - recipient_entry.recipient, envvars_added); - - g_hash_table_destroy(config_hash); - } - - crm_drop_envvars(&entry, envvars_added); - } - - if (recipients == 0) { - crm_add_dup_alert_list_entry(&entry); - } - - crm_drop_envvars(&entry, -1); - g_hash_table_destroy(config_hash); - } - - if (max_timeout > 0) { - crm_alert_max_alert_timeout = max_timeout; - } -} - -/* - * end of synchronization of local data with cib - */ - -void -crmd_enable_alerts(const char *script, const char *target) -{ - free(notify_script); - notify_script = ((script) && - (strcmp(script,"/dev/null")))?strdup(script):NULL; - - free(notify_target); - notify_target = (target != NULL)?strdup(target):NULL; -} - static void crmd_alert_complete(svc_action_t *op) { alerts_inflight--; if(op->rc == 0) { crm_info("Alert %d (%s) complete", op->sequence, op->agent); } else { crm_warn("Alert %d (%s) failed: %d", op->sequence, op->agent, op->rc); } } static void send_alerts(const char *kind) { svc_action_t *alert = NULL; static int operations = 0; GListPtr l; crm_time_hr_t *now = crm_time_hr_new(NULL); crm_set_alert_key(CRM_alert_kind, kind); crm_set_alert_key(CRM_alert_version, VERSION); for (l = g_list_first(crm_alert_list); l; l = g_list_next(l)) { crm_alert_entry_t *entry = (crm_alert_entry_t *)(l->data); char *timestamp = crm_time_format_hr(entry->tstamp_format, now); if (crm_is_target_alert(entry->select_kind == NULL ? crm_alert_kind_default : entry->select_kind, kind) == FALSE) { crm_trace("Cannot sending '%s' alert to '%s' via '%s'(select_kind=%s)", kind, entry->recipient, entry->path, entry->select_kind == NULL ? CRM_ALERT_KIND_DEFAULT : entry->select_kind_orig); free(timestamp); continue; } operations++; if (!draining_alerts) { crm_debug("Sending '%s' alert to '%s' via '%s'", kind, entry->recipient, entry->path); crm_set_alert_key(CRM_alert_recipient, entry->recipient); crm_set_alert_key_int(CRM_alert_node_sequence, operations); crm_set_alert_key(CRM_alert_timestamp, timestamp); alert = services_action_create_generic(entry->path, NULL); alert->timeout = entry->timeout; alert->standard = strdup("event"); alert->id = strdup(entry->id); alert->agent = strdup(entry->path); alert->sequence = operations; crm_set_envvar_list(entry); alerts_inflight++; if(services_action_async(alert, &crmd_alert_complete) == FALSE) { services_action_free(alert); alerts_inflight--; } crm_unset_envvar_list(entry); } else { crm_warn("Ignoring '%s' alert to '%s' via '%s' received " "while shutting down", kind, entry->recipient, entry->path); } free(timestamp); } crm_unset_alert_keys(); if (now) { free(now); } } void crmd_alert_node_event(crm_node_t *node) { if(!crm_alert_list) { return; } crm_set_alert_key(CRM_alert_node, node->uname); crm_set_alert_key_int(CRM_alert_nodeid, node->id); crm_set_alert_key(CRM_alert_desc, node->state); send_alerts("node"); } void crmd_alert_fencing_op(stonith_event_t * e) { char *desc = NULL; if (!crm_alert_list) { return; } desc = crm_strdup_printf( "Operation %s of %s by %s for %s@%s: %s (ref=%s)", e->action, e->target, e->executioner ? e->executioner : "", e->client_origin, e->origin, pcmk_strerror(e->result), e->id); crm_set_alert_key(CRM_alert_node, e->target); crm_set_alert_key(CRM_alert_task, e->operation); crm_set_alert_key(CRM_alert_desc, desc); crm_set_alert_key_int(CRM_alert_rc, e->result); send_alerts("fencing"); free(desc); } void crmd_alert_resource_op(const char *node, lrmd_event_data_t * op) { int target_rc = 0; if(!crm_alert_list) { return; } target_rc = rsc_op_expected_rc(op); if(op->interval == 0 && target_rc == op->rc && safe_str_eq(op->op_type, RSC_STATUS)) { /* Leave it up to the script if they want to alert for * 'failed' probes, only swallow ones for which the result was * unexpected. * * Even if we find a resource running, it was probably because * someone erased the status section. */ return; } crm_set_alert_key(CRM_alert_node, node); crm_set_alert_key(CRM_alert_rsc, op->rsc_id); crm_set_alert_key(CRM_alert_task, op->op_type); crm_set_alert_key_int(CRM_alert_interval, op->interval); crm_set_alert_key_int(CRM_alert_target_rc, target_rc); crm_set_alert_key_int(CRM_alert_status, op->op_status); crm_set_alert_key_int(CRM_alert_rc, op->rc); if(op->op_status == PCMK_LRM_OP_DONE) { crm_set_alert_key(CRM_alert_desc, services_ocf_exitcode_str(op->rc)); } else { crm_set_alert_key(CRM_alert_desc, services_lrm_status_str(op->op_status)); } send_alerts("resource"); } static gboolean alert_drain_timeout_callback(gpointer user_data) { gboolean *timeout_popped = (gboolean *) user_data; *timeout_popped = TRUE; return FALSE; } void crmd_drain_alerts(GMainContext *ctx) { guint timer; gboolean timeout_popped = FALSE; draining_alerts = TRUE; timer = g_timeout_add(crm_alert_max_alert_timeout + 5000, alert_drain_timeout_callback, (gpointer) &timeout_popped); while(alerts_inflight && !timeout_popped) { crm_trace("Draining mainloop while still %d alerts are in flight (timeout=%dms)", alerts_inflight, crm_alert_max_alert_timeout + 5000); g_main_context_iteration(ctx, TRUE); } if (!timeout_popped && (timer > 0)) { g_source_remove(timer); } if (crm_alert_kind_default) { g_strfreev(crm_alert_kind_default); crm_alert_kind_default = NULL; } } diff --git a/crmd/crmd_alerts.h b/crmd/crmd_alerts.h index 40089839f1..d31976ee08 100644 --- a/crmd/crmd_alerts.h +++ b/crmd/crmd_alerts.h @@ -1,32 +1,30 @@ /* * Copyright (C) 2015 Andrew Beekhof * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef CRMD_ALERT__H # define CRMD_ALERT__H # include # include # include -void crmd_enable_alerts(const char *script, const char *target); void crmd_alert_node_event(crm_node_t *node); void crmd_alert_fencing_op(stonith_event_t *e); void crmd_alert_resource_op(const char *node, lrmd_event_data_t *op); void crmd_drain_alerts(GMainContext *ctx); -void parse_alerts(xmlNode *alerts); #endif diff --git a/crmd/lrm_state.c b/crmd/lrm_state.c index eef54c126d..d4fe9a13e8 100644 --- a/crmd/lrm_state.c +++ b/crmd/lrm_state.c @@ -1,783 +1,794 @@ /* * Copyright (C) 2012 David Vossel * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #include #include #include GHashTable *lrm_state_table = NULL; extern GHashTable *proxy_table; int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg); void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg)); static void free_rsc_info(gpointer value) { lrmd_rsc_info_t *rsc_info = value; lrmd_free_rsc_info(rsc_info); } static void free_deletion_op(gpointer value) { struct pending_deletion_op_s *op = value; free(op->rsc); delete_ha_msg_input(op->input); free(op); } static void free_recurring_op(gpointer value) { struct recurring_op_s *op = (struct recurring_op_s *)value; free(op->user_data); free(op->rsc_id); free(op->op_type); free(op->op_key); if (op->params) { g_hash_table_destroy(op->params); } free(op); } static gboolean fail_pending_op(gpointer key, gpointer value, gpointer user_data) { lrmd_event_data_t event = { 0, }; lrm_state_t *lrm_state = user_data; struct recurring_op_s *op = (struct recurring_op_s *)value; crm_trace("Pre-emptively failing %s_%s_%d on %s (call=%s, %s)", op->rsc_id, op->op_type, op->interval, lrm_state->node_name, (char*)key, op->user_data); event.type = lrmd_event_exec_complete; event.rsc_id = op->rsc_id; event.op_type = op->op_type; event.user_data = op->user_data; event.timeout = 0; event.interval = op->interval; event.rc = PCMK_OCF_CONNECTION_DIED; event.op_status = PCMK_LRM_OP_ERROR; event.t_run = op->start_time; event.t_rcchange = op->start_time; event.call_id = op->call_id; event.remote_nodename = lrm_state->node_name; event.params = op->params; process_lrm_event(lrm_state, &event, op); return TRUE; } gboolean lrm_state_is_local(lrm_state_t *lrm_state) { if (lrm_state == NULL || fsa_our_uname == NULL) { return FALSE; } if (strcmp(lrm_state->node_name, fsa_our_uname) != 0) { return FALSE; } return TRUE; } lrm_state_t * lrm_state_create(const char *node_name) { lrm_state_t *state = NULL; if (!node_name) { crm_err("No node name given for lrm state object"); return NULL; } state = calloc(1, sizeof(lrm_state_t)); if (!state) { return NULL; } state->node_name = strdup(node_name); state->rsc_info_cache = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_rsc_info); state->deletion_ops = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, free_deletion_op); state->pending_ops = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, free_recurring_op); state->resource_history = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, history_free); state->metadata_cache = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, crm_destroy_xml); g_hash_table_insert(lrm_state_table, (char *)state->node_name, state); return state; } void lrm_state_destroy(const char *node_name) { g_hash_table_remove(lrm_state_table, node_name); } static gboolean remote_proxy_remove_by_node(gpointer key, gpointer value, gpointer user_data) { remote_proxy_t *proxy = value; const char *node_name = user_data; if (safe_str_eq(node_name, proxy->node_name)) { return TRUE; } return FALSE; } static void internal_lrm_state_destroy(gpointer data) { lrm_state_t *lrm_state = data; if (!lrm_state) { return; } crm_trace("Destroying proxy table %s with %d members", lrm_state->node_name, g_hash_table_size(proxy_table)); g_hash_table_foreach_remove(proxy_table, remote_proxy_remove_by_node, (char *) lrm_state->node_name); remote_ra_cleanup(lrm_state); lrmd_api_delete(lrm_state->conn); if (lrm_state->rsc_info_cache) { crm_trace("Destroying rsc info cache with %d members", g_hash_table_size(lrm_state->rsc_info_cache)); g_hash_table_destroy(lrm_state->rsc_info_cache); } if (lrm_state->resource_history) { crm_trace("Destroying history op cache with %d members", g_hash_table_size(lrm_state->resource_history)); g_hash_table_destroy(lrm_state->resource_history); } if (lrm_state->deletion_ops) { crm_trace("Destroying deletion op cache with %d members", g_hash_table_size(lrm_state->deletion_ops)); g_hash_table_destroy(lrm_state->deletion_ops); } if (lrm_state->pending_ops) { crm_trace("Destroying pending op cache with %d members", g_hash_table_size(lrm_state->pending_ops)); g_hash_table_destroy(lrm_state->pending_ops); } if (lrm_state->metadata_cache) { crm_trace("Destroying metadata cache with %d members", g_hash_table_size(lrm_state->metadata_cache)); g_hash_table_destroy(lrm_state->metadata_cache); } free((char *)lrm_state->node_name); free(lrm_state); } void lrm_state_reset_tables(lrm_state_t * lrm_state, gboolean reset_metadata) { if (lrm_state->resource_history) { crm_trace("Re-setting history op cache with %d members", g_hash_table_size(lrm_state->resource_history)); g_hash_table_remove_all(lrm_state->resource_history); } if (lrm_state->deletion_ops) { crm_trace("Re-setting deletion op cache with %d members", g_hash_table_size(lrm_state->deletion_ops)); g_hash_table_remove_all(lrm_state->deletion_ops); } if (lrm_state->pending_ops) { crm_trace("Re-setting pending op cache with %d members", g_hash_table_size(lrm_state->pending_ops)); g_hash_table_remove_all(lrm_state->pending_ops); } if (lrm_state->rsc_info_cache) { crm_trace("Re-setting rsc info cache with %d members", g_hash_table_size(lrm_state->rsc_info_cache)); g_hash_table_remove_all(lrm_state->rsc_info_cache); } if (reset_metadata && lrm_state->metadata_cache) { crm_trace("Re-setting metadata cache with %d members", g_hash_table_size(lrm_state->metadata_cache)); g_hash_table_remove_all(lrm_state->metadata_cache); } } static gboolean has_cached_metadata_for(lrmd_rsc_info_t *rsc, const char *node_name) { - return lrm_state_get_rsc_metadata(lrm_state_find(node_name), rsc) != NULL; + lrm_state_t *lrm_state; + + CRM_CHECK((rsc != NULL) && (node_name != NULL), return FALSE); + + lrm_state = lrm_state_find(node_name); + if (lrm_state == NULL) { + crm_debug("Metadata check requested for %s but we've never connected to it", + node_name); + return FALSE; + } + + return lrm_state_get_rsc_metadata(lrm_state, rsc) != NULL; } gboolean lrm_state_init_local(void) { if (lrm_state_table) { return TRUE; } lrm_state_table = g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, internal_lrm_state_destroy); if (!lrm_state_table) { return FALSE; } proxy_table = g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, NULL, remote_proxy_free); if (!proxy_table) { g_hash_table_destroy(lrm_state_table); lrm_state_table = NULL; return FALSE; } crm_register_cache_check_fn(&has_cached_metadata_for); return TRUE; } void lrm_state_destroy_all(void) { crm_unregister_cache_check_fn(); if (lrm_state_table) { crm_trace("Destroying state table with %d members", g_hash_table_size(lrm_state_table)); g_hash_table_destroy(lrm_state_table); lrm_state_table = NULL; } if(proxy_table) { crm_trace("Destroying proxy table with %d members", g_hash_table_size(proxy_table)); g_hash_table_destroy(proxy_table); proxy_table = NULL; } } lrm_state_t * lrm_state_find(const char *node_name) { if (!node_name) { return NULL; } return g_hash_table_lookup(lrm_state_table, node_name); } lrm_state_t * lrm_state_find_or_create(const char *node_name) { lrm_state_t *lrm_state; lrm_state = g_hash_table_lookup(lrm_state_table, node_name); if (!lrm_state) { lrm_state = lrm_state_create(node_name); } return lrm_state; } GList * lrm_state_get_list(void) { return g_hash_table_get_values(lrm_state_table); } static remote_proxy_t * find_connected_proxy_by_node(const char * node_name) { GHashTableIter gIter; remote_proxy_t *proxy = NULL; CRM_CHECK(proxy_table != NULL, return NULL); g_hash_table_iter_init(&gIter, proxy_table); while (g_hash_table_iter_next(&gIter, NULL, (gpointer *) &proxy)) { if (proxy->source && safe_str_eq(node_name, proxy->node_name)) { return proxy; } } return NULL; } static void remote_proxy_disconnect_by_node(const char * node_name) { remote_proxy_t *proxy = NULL; CRM_CHECK(proxy_table != NULL, return); while ((proxy = find_connected_proxy_by_node(node_name)) != NULL) { /* mainloop_del_ipc_client() eventually calls remote_proxy_disconnected() * , which removes the entry from proxy_table. * Do not do this in a g_hash_table_iter_next() loop. */ if (proxy->source) { mainloop_del_ipc_client(proxy->source); } } return; } void lrm_state_disconnect_only(lrm_state_t * lrm_state) { int removed = 0; if (!lrm_state->conn) { return; } crm_trace("Disconnecting %s", lrm_state->node_name); remote_proxy_disconnect_by_node(lrm_state->node_name); ((lrmd_t *) lrm_state->conn)->cmds->disconnect(lrm_state->conn); if (is_not_set(fsa_input_register, R_SHUTDOWN)) { removed = g_hash_table_foreach_remove(lrm_state->pending_ops, fail_pending_op, lrm_state); crm_trace("Synthesized %d operation failures for %s", removed, lrm_state->node_name); } } void lrm_state_disconnect(lrm_state_t * lrm_state) { if (!lrm_state->conn) { return; } lrm_state_disconnect_only(lrm_state); lrmd_api_delete(lrm_state->conn); lrm_state->conn = NULL; } int lrm_state_is_connected(lrm_state_t * lrm_state) { if (!lrm_state->conn) { return FALSE; } return ((lrmd_t *) lrm_state->conn)->cmds->is_connected(lrm_state->conn); } int lrm_state_poke_connection(lrm_state_t * lrm_state) { if (!lrm_state->conn) { return -1; } return ((lrmd_t *) lrm_state->conn)->cmds->poke_connection(lrm_state->conn); } int lrm_state_ipc_connect(lrm_state_t * lrm_state) { int ret; if (!lrm_state->conn) { lrm_state->conn = lrmd_api_new(); ((lrmd_t *) lrm_state->conn)->cmds->set_callback(lrm_state->conn, lrm_op_callback); } ret = ((lrmd_t *) lrm_state->conn)->cmds->connect(lrm_state->conn, CRM_SYSTEM_CRMD, NULL); if (ret != pcmk_ok) { lrm_state->num_lrm_register_fails++; } else { lrm_state->num_lrm_register_fails = 0; } return ret; } static remote_proxy_t * crmd_remote_proxy_new(lrmd_t *lrmd, const char *node_name, const char *session_id, const char *channel) { static struct ipc_client_callbacks proxy_callbacks = { .dispatch = remote_proxy_dispatch, .destroy = remote_proxy_disconnected }; remote_proxy_t *proxy = remote_proxy_new(lrmd, &proxy_callbacks, node_name, session_id, channel); return proxy; } gboolean crmd_is_proxy_session(const char *session) { return g_hash_table_lookup(proxy_table, session) ? TRUE : FALSE; } void crmd_proxy_send(const char *session, xmlNode *msg) { remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session); lrm_state_t *lrm_state = NULL; if (!proxy) { return; } crm_log_xml_trace(msg, "to-proxy"); lrm_state = lrm_state_find(proxy->node_name); if (lrm_state) { crm_trace("Sending event to %.8s on %s", proxy->session_id, proxy->node_name); remote_proxy_relay_event(proxy, msg); } } static void crmd_proxy_dispatch(const char *session, xmlNode *msg) { crm_log_xml_trace(msg, "CRMd-PROXY[inbound]"); crm_xml_add(msg, F_CRM_SYS_FROM, session); if (crmd_authorize_message(msg, NULL, session)) { route_message(C_IPC_MESSAGE, msg); } trigger_fsa(fsa_source); } static void remote_config_check(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data) { if (rc != pcmk_ok) { crm_err("Query resulted in an error: %s", pcmk_strerror(rc)); if (rc == -EACCES || rc == -pcmk_err_schema_validation) { crm_err("The cluster is mis-configured - shutting down and staying down"); } } else { lrmd_t * lrmd = (lrmd_t *)user_data; crm_time_t *now = crm_time_new(NULL); GHashTable *config_hash = g_hash_table_new_full( crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); crm_debug("Call %d : Parsing CIB options", call_id); unpack_instance_attributes( output, output, XML_CIB_TAG_PROPSET, NULL, config_hash, CIB_OPTIONS_FIRST, FALSE, now); /* Now send it to the remote peer */ remote_proxy_check(lrmd, config_hash); g_hash_table_destroy(config_hash); crm_time_free(now); } } static void crmd_remote_proxy_cb(lrmd_t *lrmd, void *userdata, xmlNode *msg) { lrm_state_t *lrm_state = userdata; const char *session = crm_element_value(msg, F_LRMD_IPC_SESSION); remote_proxy_t *proxy = g_hash_table_lookup(proxy_table, session); const char *op = crm_element_value(msg, F_LRMD_IPC_OP); if (safe_str_eq(op, LRMD_IPC_OP_NEW)) { const char *channel = crm_element_value(msg, F_LRMD_IPC_IPC_SERVER); proxy = crmd_remote_proxy_new(lrmd, lrm_state->node_name, session, channel); if (proxy != NULL) { /* Look up stonith-watchdog-timeout and send to the remote peer for validation */ int rc = fsa_cib_conn->cmds->query(fsa_cib_conn, XML_CIB_TAG_CRMCONFIG, NULL, cib_scope_local); fsa_cib_conn->cmds->register_callback_full(fsa_cib_conn, rc, 10, FALSE, lrmd, "remote_config_check", remote_config_check, NULL); } } else if (safe_str_eq(op, LRMD_IPC_OP_SHUTDOWN_REQ)) { char *now_s = NULL; time_t now = time(NULL); crm_notice("%s requested shutdown of its remote connection", lrm_state->node_name); if (!remote_ra_is_in_maintenance(lrm_state)) { now_s = crm_itoa(now); update_attrd(lrm_state->node_name, XML_CIB_ATTR_SHUTDOWN, now_s, NULL, TRUE); free(now_s); remote_proxy_ack_shutdown(lrmd); crm_warn("Reconnection attempts to %s may result in failures that must be cleared", lrm_state->node_name); } else { remote_proxy_nack_shutdown(lrmd); crm_notice("Remote resource for %s is not managed so no ordered shutdown happening", lrm_state->node_name); } return; } else if (safe_str_eq(op, LRMD_IPC_OP_REQUEST) && proxy && proxy->is_local) { /* this is for the crmd, which we are, so don't try * and connect/send to ourselves over ipc. instead * do it directly. */ int flags = 0; xmlNode *request = get_message_xml(msg, F_LRMD_IPC_MSG); CRM_CHECK(request != NULL, return); #if ENABLE_ACL CRM_CHECK(lrm_state->node_name, return); crm_xml_add(request, XML_ACL_TAG_ROLE, "pacemaker-remote"); crm_acl_get_set_user(request, F_LRMD_IPC_USER, lrm_state->node_name); #endif crmd_proxy_dispatch(session, request); crm_element_value_int(msg, F_LRMD_IPC_MSG_FLAGS, &flags); if (flags & crm_ipc_client_response) { int msg_id = 0; xmlNode *op_reply = create_xml_node(NULL, "ack"); crm_xml_add(op_reply, "function", __FUNCTION__); crm_xml_add_int(op_reply, "line", __LINE__); crm_element_value_int(msg, F_LRMD_IPC_MSG_ID, &msg_id); remote_proxy_relay_response(proxy, op_reply, msg_id); free_xml(op_reply); } } else { remote_proxy_cb(lrmd, lrm_state->node_name, msg); } } int lrm_state_remote_connect_async(lrm_state_t * lrm_state, const char *server, int port, int timeout_ms) { int ret; if (!lrm_state->conn) { lrm_state->conn = lrmd_remote_api_new(lrm_state->node_name, server, port); if (!lrm_state->conn) { return -1; } ((lrmd_t *) lrm_state->conn)->cmds->set_callback(lrm_state->conn, remote_lrm_op_callback); lrmd_internal_set_proxy_callback(lrm_state->conn, lrm_state, crmd_remote_proxy_cb); } crm_trace("initiating remote connection to %s at %d with timeout %d", server, port, timeout_ms); ret = ((lrmd_t *) lrm_state->conn)->cmds->connect_async(lrm_state->conn, lrm_state->node_name, timeout_ms); if (ret != pcmk_ok) { lrm_state->num_lrm_register_fails++; } else { lrm_state->num_lrm_register_fails = 0; } return ret; } int lrm_state_get_metadata(lrm_state_t * lrm_state, const char *class, const char *provider, const char *agent, char **output, enum lrmd_call_options options) { if (!lrm_state->conn) { return -ENOTCONN; } /* Optimize this... only retrieve metadata from local lrmd connection. Perhaps consider * caching result. */ return ((lrmd_t *) lrm_state->conn)->cmds->get_metadata(lrm_state->conn, class, provider, agent, output, options); } int lrm_state_cancel(lrm_state_t * lrm_state, const char *rsc_id, const char *action, int interval) { if (!lrm_state->conn) { return -ENOTCONN; } /* Figure out a way to make this async? * NOTICE: Currently it's synced and directly acknowledged in do_lrm_invoke(). */ if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) { return remote_ra_cancel(lrm_state, rsc_id, action, interval); } return ((lrmd_t *) lrm_state->conn)->cmds->cancel(lrm_state->conn, rsc_id, action, interval); } lrmd_rsc_info_t * lrm_state_get_rsc_info(lrm_state_t * lrm_state, const char *rsc_id, enum lrmd_call_options options) { lrmd_rsc_info_t *rsc = NULL; if (!lrm_state->conn) { return NULL; } if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) { return remote_ra_get_rsc_info(lrm_state, rsc_id); } rsc = g_hash_table_lookup(lrm_state->rsc_info_cache, rsc_id); if (rsc == NULL) { /* only contact the lrmd if we don't already have a cached rsc info */ rsc = ((lrmd_t *) lrm_state->conn)->cmds->get_rsc_info(lrm_state->conn, rsc_id, options); if (rsc == NULL) { return NULL; } /* cache the result */ g_hash_table_insert(lrm_state->rsc_info_cache, rsc->id, rsc); } return lrmd_copy_rsc_info(rsc); } int lrm_state_exec(lrm_state_t * lrm_state, const char *rsc_id, const char *action, const char *userdata, int interval, /* ms */ int timeout, /* ms */ int start_delay, /* ms */ lrmd_key_value_t * params) { if (!lrm_state->conn) { lrmd_key_value_freeall(params); return -ENOTCONN; } if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) { return remote_ra_exec(lrm_state, rsc_id, action, userdata, interval, timeout, start_delay, params); } return ((lrmd_t *) lrm_state->conn)->cmds->exec(lrm_state->conn, rsc_id, action, userdata, interval, timeout, start_delay, lrmd_opt_notify_changes_only, params); } int lrm_state_register_rsc(lrm_state_t * lrm_state, const char *rsc_id, const char *class, const char *provider, const char *agent, enum lrmd_call_options options) { if (!lrm_state->conn) { return -ENOTCONN; } /* optimize this... this function is a synced round trip from client to daemon. * The crmd/lrm.c code path should be re-factored to allow the register of resources * to be performed async. The lrmd client api needs to make an async version * of register available. */ if (is_remote_lrmd_ra(agent, provider, NULL)) { return lrm_state_find_or_create(rsc_id) ? pcmk_ok : -1; } return ((lrmd_t *) lrm_state->conn)->cmds->register_rsc(lrm_state->conn, rsc_id, class, provider, agent, options); } int lrm_state_unregister_rsc(lrm_state_t * lrm_state, const char *rsc_id, enum lrmd_call_options options) { if (!lrm_state->conn) { return -ENOTCONN; } /* optimize this... this function is a synced round trip from client to daemon. * The crmd/lrm.c code path that uses this function should always treat it as an * async operation. The lrmd client api needs to make an async version unreg available. */ if (is_remote_lrmd_ra(NULL, NULL, rsc_id)) { lrm_state_destroy(rsc_id); return pcmk_ok; } g_hash_table_remove(lrm_state->rsc_info_cache, rsc_id); return ((lrmd_t *) lrm_state->conn)->cmds->unregister_rsc(lrm_state->conn, rsc_id, options); } xmlNode * lrm_state_update_rsc_metadata(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, const char *metadata_str) { char *key = NULL; xmlNode *metadata = NULL; CRM_CHECK(lrm_state && rsc && metadata_str, return NULL); key = crm_generate_ra_key(rsc->class, rsc->provider, rsc->type); if (!key) { return NULL; } metadata = string2xml(metadata_str); if (!metadata) { crm_err("Metadata for %s (%s:%s:%s) is not valid XML", rsc->id, rsc->class, rsc->provider, rsc->type); free(key); return NULL; } g_hash_table_replace(lrm_state->metadata_cache, key, metadata); return metadata; } xmlNode * lrm_state_get_rsc_metadata(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc) { char *key = NULL; xmlNode *metadata = NULL; CRM_CHECK(lrm_state && rsc, return NULL); key = crm_generate_ra_key(rsc->class, rsc->provider, rsc->type); if (!key) { return NULL; } metadata = g_hash_table_lookup(lrm_state->metadata_cache, key); free(key); return metadata; } diff --git a/include/crm/Makefile.am b/include/crm/Makefile.am index bdb627ceb3..951d4833ca 100644 --- a/include/crm/Makefile.am +++ b/include/crm/Makefile.am @@ -1,24 +1,26 @@ # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # MAINTAINERCLEANFILES = Makefile.in headerdir=$(pkgincludedir)/crm -header_HEADERS = crm.h cib.h attrd.h msg_xml.h transition.h stonith-ng.h cluster.h lrmd.h services.h error.h compatibility.h +header_HEADERS = attrd.h cib.h cluster.h compatibility.h crm.h \ + error.h lrmd.h msg_xml.h services.h stonith-ng.h \ + transition.h SUBDIRS = common pengine cib fencing cluster diff --git a/include/crm/common/xml_internal.h b/include/crm/common/xml_internal.h index 970e2d9ff5..36604aefd9 100644 --- a/include/crm/common/xml_internal.h +++ b/include/crm/common/xml_internal.h @@ -1,124 +1,122 @@ /* * Copyright 2017 Jan Pokorny * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef CRM_COMMON_XML_INTERNAL__H # define CRM_COMMON_XML_INTERNAL__H -/** - * \file - * \brief Internal-only wrappers for and extensions to libxml2 (libxslt) - * \ingroup core +/* + * Internal-only wrappers for and extensions to libxml2 (libxslt) */ # include # include # include # include /* transitively imports qblog.h */ /*! * \brief Base for directing lib{xml2,xslt} log into standard libqb backend * * This macro implements the core of what can be needed for directing * libxml2 or libxslt error messaging into standard, preconfigured * libqb-backed log stream. * * It's a bit unfortunate that libxml2 (and more sparsely, also libxslt) * emits a single message by chunks (location is emitted separatedly from * the message itself), so we have to take the effort to combine these * chunks back to single message. Whether to do this or not is driven * with \p dechunk toggle. * * The form of a macro was chosen for implicit deriving of __FILE__, etc. * and also because static dechunking buffer should be differentiated per * library (here we assume different functions referring to this macro * will not ever be using both at once), preferably also per-library * context of use to avoid clashes altogether. * * Note that we cannot use qb_logt, because callsite data have to be known * at the moment of compilation, which it is not always the case -- xml_log * (and unfortunately there's no clear explanation of the fail to compile). * * Also note that there's no explicit guard against said libraries producing * never-newline-terminated chunks (which would just keep consuming memory), * as it's quite improbable. Termination of the program in between the * same-message chunks will raise a flag with valgrind and the likes, though. * * \param[in] priority Syslog priority for the message to be logged * \param[in] dechunk Whether to dechunk new-line terminated message * \param[in] postemit Code to be executed once message is sent out * \param[in] prefix How to prefix the message or NULL for raw passing * \param[in] fmt Format string as with printf-like functions * \param[in] ap Variable argument list to supplement \p fmt format string */ #define CRM_XML_LOG_BASE(priority, dechunk, postemit, prefix, fmt, ap) \ do { \ if (!(dechunk) && (prefix) == NULL) { /* quick pass */ \ qb_log_from_external_source_va(__FUNCTION__, __FILE__, (fmt), \ (priority), __LINE__, 0, (ap)); \ (void) (postemit); \ } else { \ int CXLB_len = 0; \ char *CXLB_buf = NULL; \ static int CXLB_buffer_len = 0; \ static char *CXLB_buffer = NULL; \ \ CXLB_len = vasprintf(&CXLB_buf, (fmt), (ap)); \ \ if (CXLB_len <= 0 || CXLB_buf[CXLB_len - 1] == '\n' || !(dechunk)) { \ if (CXLB_len < 0) { \ CXLB_buf = (char *) "LOG CORRUPTION HAZARD"; /*we don't modify*/\ } else if (CXLB_len > 0 /* && (dechunk) */ \ && CXLB_buf[CXLB_len - 1] == '\n') { \ CXLB_buf[CXLB_len - 1] = '\0'; \ } \ if (CXLB_buffer) { \ qb_log_from_external_source(__FUNCTION__, __FILE__, "%s%s%s", \ (priority), __LINE__, 0, \ (prefix) != NULL ? (prefix) : "", \ CXLB_buffer, CXLB_buf); \ free(CXLB_buffer); \ } else { \ qb_log_from_external_source(__FUNCTION__, __FILE__, "%s%s", \ (priority), __LINE__, 0, \ (prefix) != NULL ? (prefix) : "", \ CXLB_buf); \ } \ if (CXLB_len < 0) { \ CXLB_buf = NULL; /* restore temporary override */ \ } \ CXLB_buffer = NULL; \ CXLB_buffer_len = 0; \ (void) (postemit); \ \ } else if (CXLB_buffer == NULL) { \ CXLB_buffer_len = CXLB_len; \ CXLB_buffer = CXLB_buf; \ CXLB_buf = NULL; \ \ } else { \ CXLB_buffer = realloc(CXLB_buffer, 1 + CXLB_buffer_len + CXLB_len); \ memcpy(CXLB_buffer + CXLB_buffer_len, CXLB_buf, CXLB_len); \ CXLB_buffer_len += CXLB_len; \ CXLB_buffer[CXLB_buffer_len] = '\0'; \ } \ free(CXLB_buf); \ } \ } while (0) #endif diff --git a/include/crm/pengine/Makefile.am b/include/crm/pengine/Makefile.am index fa072f7e5c..f41f88782a 100644 --- a/include/crm/pengine/Makefile.am +++ b/include/crm/pengine/Makefile.am @@ -1,23 +1,23 @@ # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # MAINTAINERCLEANFILES = Makefile.in headerdir=$(pkgincludedir)/crm/pengine -noinst_HEADERS = internal.h +noinst_HEADERS = internal.h rules_internal.h header_HEADERS = common.h complex.h remote.h rules.h status.h diff --git a/include/crm/pengine/rules_internal.h b/include/crm/pengine/rules_internal.h new file mode 100644 index 0000000000..28533e38ef --- /dev/null +++ b/include/crm/pengine/rules_internal.h @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2015-2017 Andrew Beekhof + * + * This source code is licensed under the GNU Lesser General Public License + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. + */ +#ifndef RULES_INTERNAL_H +#define RULES_INTERNAL_H + +#include + +void pe_unpack_alerts(xmlNode *alerts); + +#ifdef RHEL7_COMPAT +void pe_enable_legacy_alerts(const char *script, const char *target); +#endif + +#endif diff --git a/lib/common/Makefile.am b/lib/common/Makefile.am index 4201925031..56aa4c54c8 100644 --- a/lib/common/Makefile.am +++ b/lib/common/Makefile.am @@ -1,49 +1,49 @@ # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # include $(top_srcdir)/Makefile.common AM_CPPFLAGS += -I$(top_builddir)/lib/gnu -I$(top_srcdir)/lib/gnu ## libraries lib_LTLIBRARIES = libcrmcommon.la # Can't use -Wcast-qual here because glib insists on pretending things are const # when they're not and thus we need the crm_element_value_const() hack # s390 needs -fPIC # s390-suse-linux/bin/ld: .libs/ipc.o: relocation R_390_PC32DBL against `__stack_chk_fail@@GLIBC_2.4' can not be used when making a shared object; recompile with -fPIC CFLAGS = $(CFLAGS_COPY:-Wcast-qual=) -fPIC libcrmcommon_la_LDFLAGS = -version-info 10:0:7 libcrmcommon_la_CFLAGS = $(CFLAGS_HARDENED_LIB) libcrmcommon_la_LDFLAGS += $(LDFLAGS_HARDENED_LIB) libcrmcommon_la_LIBADD = @LIBADD_DL@ $(GNUTLSLIBS) -lm -libcrmcommon_la_SOURCES = compat.c digest.c ipc.c io.c procfs.c utils.c xml.c \ - iso8601.c remote.c mainloop.c logging.c watchdog.c \ +libcrmcommon_la_SOURCES = compat.c digest.c ipc.c io.c procfs.c utils.c xml.c \ + iso8601.c remote.c mainloop.c logging.c watchdog.c \ schemas.c strings.c xpath.c attrd_client.c alerts.c if BUILD_CIBSECRETS libcrmcommon_la_SOURCES += cib_secrets.c endif libcrmcommon_la_SOURCES += $(top_builddir)/lib/gnu/md5.c clean-generic: rm -f *.log *.debug *.xml *~ diff --git a/lib/common/schemas.c b/lib/common/schemas.c index 5c867dbcc2..77e87dc51c 100644 --- a/lib/common/schemas.c +++ b/lib/common/schemas.c @@ -1,967 +1,967 @@ /* * Copyright (C) 2004-2016 Andrew Beekhof * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include #if HAVE_LIBXML2 # include #endif #if HAVE_LIBXSLT # include # include # include #endif #include #include #include /* CRM_XML_LOG_BASE */ typedef struct { xmlRelaxNGPtr rng; xmlRelaxNGValidCtxtPtr valid; xmlRelaxNGParserCtxtPtr parser; } relaxng_ctx_cache_t; struct schema_s { int type; float version; char *name; char *location; char *transform; int after_transform; void *cache; }; static struct schema_s *known_schemas = NULL; static int xml_schema_max = 0; static void xml_log(int priority, const char *fmt, ...) G_GNUC_PRINTF(2, 3); static void xml_log(int priority, const char *fmt, ...) { va_list ap; va_start(ap, fmt); /* XXX should not this enable dechunking as well? */ CRM_XML_LOG_BASE(priority, FALSE, 0, NULL, fmt, ap); va_end(ap); } static int xml_latest_schema_index(void) { return xml_schema_max - 4; } static int xml_minimum_schema_index(void) { static int best = 0; if (best == 0) { int lpc = 0; float target = 0.0; best = xml_latest_schema_index(); target = floor(known_schemas[best].version); for (lpc = best; lpc > 0; lpc--) { if (known_schemas[lpc].version < target) { return best; } else { best = lpc; } } best = xml_latest_schema_index(); } return best; } const char * xml_latest_schema(void) { return get_schema_name(xml_latest_schema_index()); } static const char * get_schema_root(void) { static const char *base = NULL; if (base == NULL) { base = getenv("PCMK_schema_directory"); } if (base == NULL || strlen(base) == 0) { base = CRM_DTD_DIRECTORY; } return base; } static char * get_schema_path(const char *name, const char *file) { const char *base = get_schema_root(); if (file) { return crm_strdup_printf("%s/%s", base, file); } return crm_strdup_printf("%s/%s.rng", base, name); } static int schema_filter(const struct dirent *a) { int rc = 0; float version = 0; if (strstr(a->d_name, "pacemaker-") != a->d_name) { /* crm_trace("%s - wrong prefix", a->d_name); */ } else if (!crm_ends_with(a->d_name, ".rng")) { /* crm_trace("%s - wrong suffix", a->d_name); */ } else if (sscanf(a->d_name, "pacemaker-%f.rng", &version) == 0) { /* crm_trace("%s - wrong format", a->d_name); */ } else if (strcmp(a->d_name, "pacemaker-1.1.rng") == 0) { /* "-1.1" was used for what later became "-next" */ /* crm_trace("%s - hack", a->d_name); */ } else { /* crm_debug("%s - candidate", a->d_name); */ rc = 1; } return rc; } static int schema_sort(const struct dirent **a, const struct dirent **b) { int rc = 0; float a_version = 0.0; float b_version = 0.0; sscanf(a[0]->d_name, "pacemaker-%f.rng", &a_version); sscanf(b[0]->d_name, "pacemaker-%f.rng", &b_version); if (a_version > b_version) { rc = 1; } else if(a_version < b_version) { rc = -1; } /* crm_trace("%s (%f) vs. %s (%f) : %d", a[0]->d_name, a_version, b[0]->d_name, b_version, rc); */ return rc; } static void __xml_schema_add(int type, float version, const char *name, const char *location, const char *transform, int after_transform) { int last = xml_schema_max; xml_schema_max++; known_schemas = realloc_safe(known_schemas, xml_schema_max * sizeof(struct schema_s)); CRM_ASSERT(known_schemas != NULL); memset(known_schemas+last, 0, sizeof(struct schema_s)); known_schemas[last].type = type; known_schemas[last].after_transform = after_transform; if (version > 0.0) { known_schemas[last].version = version; known_schemas[last].name = crm_strdup_printf("pacemaker-%.1f", version); known_schemas[last].location = crm_strdup_printf("%s.rng", known_schemas[last].name); } else { char dummy[1024]; CRM_ASSERT(name); CRM_ASSERT(location); sscanf(name, "%[^-]-%f", dummy, &version); known_schemas[last].version = version; known_schemas[last].name = strdup(name); known_schemas[last].location = strdup(location); } if (transform) { known_schemas[last].transform = strdup(transform); } if (after_transform == 0) { after_transform = xml_schema_max; /* upgrade is a one-way */ } known_schemas[last].after_transform = after_transform; if (known_schemas[last].after_transform < 0) { crm_debug("Added supported schema %d: %s (%s)", last, known_schemas[last].name, known_schemas[last].location); } else if (known_schemas[last].transform) { crm_debug("Added supported schema %d: %s (%s upgrades to %d with %s)", last, known_schemas[last].name, known_schemas[last].location, known_schemas[last].after_transform, known_schemas[last].transform); } else { crm_debug("Added supported schema %d: %s (%s upgrades to %d)", last, known_schemas[last].name, known_schemas[last].location, known_schemas[last].after_transform); } } /*! * \internal * \brief Load pacemaker schemas into cache */ void crm_schema_init(void) { int lpc, max; const char *base = get_schema_root(); struct dirent **namelist = NULL; max = scandir(base, &namelist, schema_filter, schema_sort); __xml_schema_add(1, 0.0, "pacemaker-0.6", "crm.dtd", "upgrade06.xsl", 3); __xml_schema_add(1, 0.0, "transitional-0.6", "crm-transitional.dtd", "upgrade06.xsl", 3); __xml_schema_add(2, 0.0, "pacemaker-0.7", "pacemaker-1.0.rng", NULL, 0); if (max < 0) { crm_notice("scandir(%s) failed: %s (%d)", base, strerror(errno), errno); } else { for (lpc = 0; lpc < max; lpc++) { int next = 0; float version = 0.0; char *transform = NULL; sscanf(namelist[lpc]->d_name, "pacemaker-%f.rng", &version); if ((lpc + 1) < max) { float next_version = 0.0; sscanf(namelist[lpc+1]->d_name, "pacemaker-%f.rng", &next_version); if (floor(version) < floor(next_version)) { struct stat s; char *xslt = NULL; transform = crm_strdup_printf("upgrade-%.1f.xsl", version); xslt = get_schema_path(NULL, transform); if (stat(xslt, &s) != 0) { crm_err("Transform %s not found", xslt); free(xslt); __xml_schema_add(2, version, NULL, NULL, NULL, -1); break; } else { free(xslt); } } } else { next = -1; } __xml_schema_add(2, version, NULL, NULL, transform, next); free(namelist[lpc]); free(transform); } } /* 1.1 was the old name for -next */ __xml_schema_add(2, 0.0, "pacemaker-1.1", "pacemaker-next.rng", NULL, 0); __xml_schema_add(2, 0.0, "pacemaker-next", "pacemaker-next.rng", NULL, -1); __xml_schema_add(0, 0.0, "none", "N/A", NULL, -1); free(namelist); } static gboolean validate_with_dtd(xmlDocPtr doc, gboolean to_logs, const char *dtd_file) { gboolean valid = TRUE; xmlDtdPtr dtd = NULL; xmlValidCtxtPtr cvp = NULL; CRM_CHECK(doc != NULL, return FALSE); CRM_CHECK(dtd_file != NULL, return FALSE); dtd = xmlParseDTD(NULL, (const xmlChar *)dtd_file); if (dtd == NULL) { crm_err("Could not locate/parse DTD: %s", dtd_file); return TRUE; } cvp = xmlNewValidCtxt(); if (cvp) { if (to_logs) { cvp->userData = (void *)LOG_ERR; cvp->error = (xmlValidityErrorFunc) xml_log; cvp->warning = (xmlValidityWarningFunc) xml_log; } else { cvp->userData = (void *)stderr; cvp->error = (xmlValidityErrorFunc) fprintf; cvp->warning = (xmlValidityWarningFunc) fprintf; } if (!xmlValidateDtd(cvp, doc, dtd)) { valid = FALSE; } xmlFreeValidCtxt(cvp); } else { crm_err("Internal error: No valid context"); } xmlFreeDtd(dtd); return valid; } #if 0 static void relaxng_invalid_stderr(void *userData, xmlErrorPtr error) { /* Structure xmlError struct _xmlError { int domain : What part of the library raised this er int code : The error code, e.g. an xmlParserError char * message : human-readable informative error messag xmlErrorLevel level : how consequent is the error char * file : the filename int line : the line number if available char * str1 : extra string information char * str2 : extra string information char * str3 : extra string information int int1 : extra number information int int2 : column number of the error or 0 if N/A void * ctxt : the parser context if available void * node : the node in the tree } */ crm_err("Structured error: line=%d, level=%d %s", error->line, error->level, error->message); } #endif static gboolean validate_with_relaxng(xmlDocPtr doc, gboolean to_logs, const char *relaxng_file, relaxng_ctx_cache_t **cached_ctx) { int rc = 0; gboolean valid = TRUE; relaxng_ctx_cache_t *ctx = NULL; CRM_CHECK(doc != NULL, return FALSE); CRM_CHECK(relaxng_file != NULL, return FALSE); if (cached_ctx && *cached_ctx) { ctx = *cached_ctx; } else { crm_info("Creating RNG parser context"); ctx = calloc(1, sizeof(relaxng_ctx_cache_t)); xmlLoadExtDtdDefaultValue = 1; ctx->parser = xmlRelaxNGNewParserCtxt(relaxng_file); CRM_CHECK(ctx->parser != NULL, goto cleanup); if (to_logs) { xmlRelaxNGSetParserErrors(ctx->parser, (xmlRelaxNGValidityErrorFunc) xml_log, (xmlRelaxNGValidityWarningFunc) xml_log, GUINT_TO_POINTER(LOG_ERR)); } else { xmlRelaxNGSetParserErrors(ctx->parser, (xmlRelaxNGValidityErrorFunc) fprintf, (xmlRelaxNGValidityWarningFunc) fprintf, stderr); } ctx->rng = xmlRelaxNGParse(ctx->parser); CRM_CHECK(ctx->rng != NULL, crm_err("Could not find/parse %s", relaxng_file); goto cleanup); ctx->valid = xmlRelaxNGNewValidCtxt(ctx->rng); CRM_CHECK(ctx->valid != NULL, goto cleanup); if (to_logs) { xmlRelaxNGSetValidErrors(ctx->valid, (xmlRelaxNGValidityErrorFunc) xml_log, (xmlRelaxNGValidityWarningFunc) xml_log, GUINT_TO_POINTER(LOG_ERR)); } else { xmlRelaxNGSetValidErrors(ctx->valid, (xmlRelaxNGValidityErrorFunc) fprintf, (xmlRelaxNGValidityWarningFunc) fprintf, stderr); } } /* xmlRelaxNGSetValidStructuredErrors( */ /* valid, relaxng_invalid_stderr, valid); */ xmlLineNumbersDefault(1); rc = xmlRelaxNGValidateDoc(ctx->valid, doc); if (rc > 0) { valid = FALSE; } else if (rc < 0) { crm_err("Internal libxml error during validation"); } cleanup: if (cached_ctx) { *cached_ctx = ctx; } else { if (ctx->parser != NULL) { xmlRelaxNGFreeParserCtxt(ctx->parser); } if (ctx->valid != NULL) { xmlRelaxNGFreeValidCtxt(ctx->valid); } if (ctx->rng != NULL) { xmlRelaxNGFree(ctx->rng); } free(ctx); } return valid; } /*! * \internal * \brief Clean up global memory associated with XML schemas */ void crm_schema_cleanup(void) { int lpc; relaxng_ctx_cache_t *ctx = NULL; for (lpc = 0; lpc < xml_schema_max; lpc++) { switch (known_schemas[lpc].type) { case 0: /* None */ break; case 1: /* DTD - Not cached */ break; case 2: /* RNG - Cached */ ctx = (relaxng_ctx_cache_t *) known_schemas[lpc].cache; if (ctx == NULL) { break; } if (ctx->parser != NULL) { xmlRelaxNGFreeParserCtxt(ctx->parser); } if (ctx->valid != NULL) { xmlRelaxNGFreeValidCtxt(ctx->valid); } if (ctx->rng != NULL) { xmlRelaxNGFree(ctx->rng); } free(ctx); known_schemas[lpc].cache = NULL; break; default: break; } free(known_schemas[lpc].name); free(known_schemas[lpc].location); free(known_schemas[lpc].transform); } free(known_schemas); known_schemas = NULL; } static gboolean validate_with(xmlNode *xml, int method, gboolean to_logs) { xmlDocPtr doc = NULL; gboolean valid = FALSE; int type = 0; char *file = NULL; if (method < 0) { return FALSE; } type = known_schemas[method].type; if(type == 0) { return TRUE; } CRM_CHECK(xml != NULL, return FALSE); doc = getDocPtr(xml); file = get_schema_path(known_schemas[method].name, known_schemas[method].location); crm_trace("Validating with: %s (type=%d)", crm_str(file), type); switch (type) { case 1: valid = validate_with_dtd(doc, to_logs, file); break; case 2: valid = validate_with_relaxng(doc, to_logs, file, (relaxng_ctx_cache_t **) & (known_schemas[method].cache)); break; default: crm_err("Unknown validator type: %d", type); break; } free(file); return valid; } static void dump_file(const char *filename) { FILE *fp = NULL; int ch, line = 0; CRM_CHECK(filename != NULL, return); fp = fopen(filename, "r"); if (fp == NULL) { crm_perror(LOG_ERR, "Could not open %s for reading", filename); return; } fprintf(stderr, "%4d ", ++line); do { ch = getc(fp); if (ch == EOF) { putc('\n', stderr); break; } else if (ch == '\n') { fprintf(stderr, "\n%4d ", ++line); } else { putc(ch, stderr); } } while (1); fclose(fp); } gboolean validate_xml_verbose(xmlNode *xml_blob) { int fd = 0; xmlDoc *doc = NULL; xmlNode *xml = NULL; gboolean rc = FALSE; char *filename = strdup(CRM_STATE_DIR "/cib-invalid.XXXXXX"); CRM_CHECK(filename != NULL, return FALSE); umask(S_IWGRP | S_IWOTH | S_IROTH); fd = mkstemp(filename); write_xml_fd(xml_blob, filename, fd, FALSE); dump_file(filename); doc = xmlParseFile(filename); xml = xmlDocGetRootElement(doc); rc = validate_xml(xml, NULL, FALSE); free_xml(xml); unlink(filename); free(filename); return rc; } gboolean validate_xml(xmlNode *xml_blob, const char *validation, gboolean to_logs) { int version = 0; if (validation == NULL) { validation = crm_element_value(xml_blob, XML_ATTR_VALIDATION); } if (validation == NULL) { int lpc = 0; bool valid = FALSE; - validation = crm_element_value(xml_blob, "ignore-dtd"); + /* @COMPAT pre-1.0 configs */ + validation = crm_element_value(xml_blob, "ignore_dtd"); if (crm_is_true(validation)) { - /* Legacy compatibilty */ crm_xml_add(xml_blob, XML_ATTR_VALIDATION, "none"); return TRUE; } /* Work it out */ for (lpc = 0; lpc < xml_schema_max; lpc++) { if (validate_with(xml_blob, lpc, FALSE)) { valid = TRUE; crm_xml_add(xml_blob, XML_ATTR_VALIDATION, known_schemas[lpc].name); crm_info("XML validated against %s", known_schemas[lpc].name); if(known_schemas[lpc].after_transform == 0) { break; } } } return valid; } version = get_schema_version(validation); if (strcmp(validation, "none") == 0) { return TRUE; } else if (version < xml_schema_max) { return validate_with(xml_blob, version, to_logs); } crm_err("Unknown validator: %s", validation); return FALSE; } #if HAVE_LIBXSLT static void cib_upgrade_err(void *ctx, const char *fmt, ...) G_GNUC_PRINTF(2, 3); static void cib_upgrade_err(void *ctx, const char *fmt, ...) { va_list ap; va_start(ap, fmt); CRM_XML_LOG_BASE(LOG_WARNING, TRUE, 0, "CIB upgrade: ", fmt, ap); va_end(ap); } static xmlNode * apply_transformation(xmlNode *xml, const char *transform, gboolean to_logs) { char *xform = NULL; xmlNode *out = NULL; xmlDocPtr res = NULL; xmlDocPtr doc = NULL; xsltStylesheet *xslt = NULL; CRM_CHECK(xml != NULL, return FALSE); doc = getDocPtr(xml); xform = get_schema_path(NULL, transform); xmlLoadExtDtdDefaultValue = 1; xmlSubstituteEntitiesDefault(1); /* for capturing, e.g., what's emitted via */ if (to_logs) { xsltSetGenericErrorFunc(NULL, cib_upgrade_err); } else { xsltSetGenericErrorFunc((void *) stderr, (xmlGenericErrorFunc) fprintf); } xslt = xsltParseStylesheetFile((const xmlChar *)xform); CRM_CHECK(xslt != NULL, goto cleanup); res = xsltApplyStylesheet(xslt, doc, NULL); CRM_CHECK(res != NULL, goto cleanup); xsltSetGenericErrorFunc(NULL, NULL); /* restore default one */ out = xmlDocGetRootElement(res); cleanup: if (xslt) { xsltFreeStylesheet(xslt); } free(xform); return out; } #endif const char * get_schema_name(int version) { if (version < 0 || version >= xml_schema_max) { return "unknown"; } return known_schemas[version].name; } int get_schema_version(const char *name) { int lpc = 0; if (name == NULL) { name = "none"; } for (; lpc < xml_schema_max; lpc++) { if (safe_str_eq(name, known_schemas[lpc].name)) { return lpc; } } return -1; } /* set which validation to use */ int update_validation(xmlNode **xml_blob, int *best, int max, gboolean transform, gboolean to_logs) { xmlNode *xml = NULL; char *value = NULL; int max_stable_schemas = xml_latest_schema_index(); int lpc = 0, match = -1, rc = pcmk_ok; int next = -1; /* -1 denotes "inactive" value */ CRM_CHECK(best != NULL, return -EINVAL); *best = 0; CRM_CHECK(xml_blob != NULL, return -EINVAL); CRM_CHECK(*xml_blob != NULL, return -EINVAL); xml = *xml_blob; value = crm_element_value_copy(xml, XML_ATTR_VALIDATION); if (value != NULL) { match = get_schema_version(value); lpc = match; if (lpc >= 0 && transform == FALSE) { *best = lpc++; } else if (lpc < 0) { crm_debug("Unknown validation type"); lpc = 0; } } if (match >= max_stable_schemas) { /* nothing to do */ free(value); *best = match; return pcmk_ok; } while (lpc <= max_stable_schemas) { crm_debug("Testing '%s' validation (%d of %d)", known_schemas[lpc].name ? known_schemas[lpc].name : "", lpc, max_stable_schemas); if (validate_with(xml, lpc, to_logs) == FALSE) { if (next != -1) { crm_info("Configuration not valid for schema: %s", known_schemas[lpc].name); next = -1; } else { crm_trace("%s validation failed", known_schemas[lpc].name ? known_schemas[lpc].name : ""); } if (*best) { /* we've satisfied the validation, no need to check further */ break; } rc = -pcmk_err_schema_validation; } else { if (next != -1) { crm_debug("Configuration valid for schema: %s", known_schemas[next].name); next = -1; } rc = pcmk_ok; } if (rc == pcmk_ok) { *best = lpc; } if (rc == pcmk_ok && transform) { xmlNode *upgrade = NULL; next = known_schemas[lpc].after_transform; if (next <= lpc) { /* There is no next version, or next would regress */ crm_trace("Stopping at %s", known_schemas[lpc].name); break; } else if (max > 0 && (lpc == max || next > max)) { crm_trace("Upgrade limit reached at %s (lpc=%d, next=%d, max=%d)", known_schemas[lpc].name, lpc, next, max); break; } else if (known_schemas[lpc].transform == NULL) { crm_debug("%s-style configuration is also valid for %s", known_schemas[lpc].name, known_schemas[next].name); lpc = next; } else { crm_debug("Upgrading %s-style configuration to %s with %s", known_schemas[lpc].name, known_schemas[next].name, known_schemas[lpc].transform ? known_schemas[lpc].transform : "no-op"); #if HAVE_LIBXSLT upgrade = apply_transformation(xml, known_schemas[lpc].transform, to_logs); #endif if (upgrade == NULL) { crm_err("Transformation %s failed", known_schemas[lpc].transform); rc = -pcmk_err_transform_failed; } else if (validate_with(upgrade, next, to_logs)) { crm_info("Transformation %s successful", known_schemas[lpc].transform); lpc = next; *best = next; free_xml(xml); xml = upgrade; rc = pcmk_ok; } else { crm_err("Transformation %s did not produce a valid configuration", known_schemas[lpc].transform); crm_log_xml_info(upgrade, "transform:bad"); free_xml(upgrade); rc = -pcmk_err_schema_validation; } next = -1; } } if (transform == FALSE || rc != pcmk_ok) { /* we need some progress! */ lpc++; } } if (*best > match && *best) { crm_info("%s the configuration from %s to %s", transform?"Transformed":"Upgraded", value ? value : "", known_schemas[*best].name); crm_xml_add(xml, XML_ATTR_VALIDATION, known_schemas[*best].name); } *xml_blob = xml; free(value); return rc; } gboolean cli_config_update(xmlNode **xml, int *best_version, gboolean to_logs) { gboolean rc = TRUE; const char *value = crm_element_value(*xml, XML_ATTR_VALIDATION); char *const orig_value = strdup(value == NULL ? "(none)" : value); int version = get_schema_version(value); int orig_version = version; int min_version = xml_minimum_schema_index(); if (version < min_version) { xmlNode *converted = NULL; converted = copy_xml(*xml); update_validation(&converted, &version, 0, TRUE, to_logs); value = crm_element_value(converted, XML_ATTR_VALIDATION); if (version < min_version) { if (version < orig_version || orig_version == -1) { if (to_logs) { crm_config_err("Your current configuration %s could not" " validate with any schema in range [%s, %s]," " cannot upgrade to %s.", orig_value, get_schema_name(orig_version), xml_latest_schema(), get_schema_name(min_version)); } else { fprintf(stderr, "Your current configuration %s could not" " validate with any schema in range [%s, %s]," " cannot upgrade to %s.\n", orig_value, get_schema_name(orig_version), xml_latest_schema(), get_schema_name(min_version)); } } else if (to_logs) { crm_config_err("Your current configuration could only be upgraded to %s... " "the minimum requirement is %s.", crm_str(value), get_schema_name(min_version)); } else { fprintf(stderr, "Your current configuration could only be upgraded to %s... " "the minimum requirement is %s.\n", crm_str(value), get_schema_name(min_version)); } free_xml(converted); converted = NULL; rc = FALSE; } else { free_xml(*xml); *xml = converted; if (version < xml_latest_schema_index()) { crm_config_warn("Your configuration was internally updated to %s... " "which is acceptable but not the most recent", get_schema_name(version)); } else if (to_logs) { crm_info("Your configuration was internally updated to the latest version (%s)", get_schema_name(version)); } } } else if (version >= get_schema_version("none")) { if (to_logs) { crm_config_warn("Configuration validation is currently disabled." " It is highly encouraged and prevents many common cluster issues."); } else { fprintf(stderr, "Configuration validation is currently disabled." " It is highly encouraged and prevents many common cluster issues.\n"); } } if (best_version) { *best_version = version; } free(orig_value); return rc; } diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c index 34bba88298..3b84621f30 100644 --- a/lib/fencing/st_client.c +++ b/lib/fencing/st_client.c @@ -1,2677 +1,2677 @@ /* * Copyright (c) 2004 Andrew Beekhof * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * */ #include #include #include #include #include #include #include #include #include #include #include #include #include /* Add it for compiling on OSX */ #include #include #include #include #include #ifdef HAVE_STONITH_STONITH_H # include # define LHA_STONITH_LIBRARY "libstonith.so.1" static void *lha_agents_lib = NULL; #endif #include CRM_TRACE_INIT_DATA(stonith); struct stonith_action_s { /*! user defined data */ char *agent; char *action; char *victim; char *args; int timeout; int async; void *userdata; void (*done_cb) (GPid pid, gint status, const char *output, gpointer user_data); /*! internal async track data */ int fd_stdout; int fd_stderr; int last_timeout_signo; /*! internal timing information */ time_t initial_start_time; int tries; int remaining_timeout; guint timer_sigterm; guint timer_sigkill; int max_retries; /* device output data */ GPid pid; int rc; char *output; char *error; }; typedef struct stonith_private_s { char *token; crm_ipc_t *ipc; mainloop_io_t *source; GHashTable *stonith_op_callback_table; GList *notify_list; void (*op_callback) (stonith_t * st, stonith_callback_data_t * data); } stonith_private_t; typedef struct stonith_notify_client_s { const char *event; const char *obj_id; /* implement one day */ const char *obj_type; /* implement one day */ void (*notify) (stonith_t * st, stonith_event_t * e); } stonith_notify_client_t; typedef struct stonith_callback_client_s { void (*callback) (stonith_t * st, stonith_callback_data_t * data); const char *id; void *user_data; gboolean only_success; gboolean allow_timeout_updates; struct timer_rec_s *timer; } stonith_callback_client_t; struct notify_blob_s { stonith_t *stonith; xmlNode *xml; }; struct timer_rec_s { int call_id; int timeout; guint ref; stonith_t *stonith; }; typedef int (*stonith_op_t) (const char *, int, const char *, xmlNode *, xmlNode *, xmlNode *, xmlNode **, xmlNode **); #if HAVE_STONITH_STONITH_H static const char META_TEMPLATE[] = "\n" "\n" "\n" " 1.0\n" " \n" "%s\n" " \n" " %s\n" "%s\n" " \n" " \n" " \n" " \n" " \n" " \n" " \n" " \n" " 2.0\n" " \n" "\n"; #endif bool stonith_dispatch(stonith_t * st); int stonith_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata); void stonith_perform_callback(stonith_t * stonith, xmlNode * msg, int call_id, int rc); xmlNode *stonith_create_op(int call_id, const char *token, const char *op, xmlNode * data, int call_options); int stonith_send_command(stonith_t * stonith, const char *op, xmlNode * data, xmlNode ** output_data, int call_options, int timeout); static void stonith_connection_destroy(gpointer user_data); static void stonith_send_notification(gpointer data, gpointer user_data); static int internal_stonith_action_execute(stonith_action_t * action); static void log_action(stonith_action_t *action, pid_t pid); static void log_action(stonith_action_t *action, pid_t pid) { if (action->output) { /* Logging the whole string confuses syslog when the string is xml */ char *prefix = crm_strdup_printf("%s[%d] stdout:", action->agent, pid); crm_log_output(LOG_TRACE, prefix, action->output); free(prefix); } if (action->error) { /* Logging the whole string confuses syslog when the string is xml */ char *prefix = crm_strdup_printf("%s[%d] stderr:", action->agent, pid); crm_log_output(LOG_WARNING, prefix, action->error); free(prefix); } } static void stonith_connection_destroy(gpointer user_data) { stonith_t *stonith = user_data; stonith_private_t *native = NULL; struct notify_blob_s blob; crm_trace("Sending destroyed notification"); blob.stonith = stonith; blob.xml = create_xml_node(NULL, "notify"); native = stonith->private; native->ipc = NULL; native->source = NULL; stonith->state = stonith_disconnected; crm_xml_add(blob.xml, F_TYPE, T_STONITH_NOTIFY); crm_xml_add(blob.xml, F_SUBTYPE, T_STONITH_NOTIFY_DISCONNECT); g_list_foreach(native->notify_list, stonith_send_notification, &blob); free_xml(blob.xml); } xmlNode * create_device_registration_xml(const char *id, const char *namespace, const char *agent, stonith_key_value_t * params, const char *rsc_provides) { xmlNode *data = create_xml_node(NULL, F_STONITH_DEVICE); xmlNode *args = create_xml_node(data, XML_TAG_ATTRS); #if HAVE_STONITH_STONITH_H namespace = get_stonith_provider(agent, namespace); if (safe_str_eq(namespace, "heartbeat")) { hash2field((gpointer) "plugin", (gpointer) agent, args); agent = "fence_legacy"; } #endif crm_xml_add(data, XML_ATTR_ID, id); crm_xml_add(data, F_STONITH_ORIGIN, __FUNCTION__); crm_xml_add(data, "agent", agent); crm_xml_add(data, "namespace", namespace); if (rsc_provides) { crm_xml_add(data, "rsc_provides", rsc_provides); } for (; params; params = params->next) { hash2field((gpointer) params->key, (gpointer) params->value, args); } return data; } static int stonith_api_register_device(stonith_t * st, int call_options, const char *id, const char *namespace, const char *agent, stonith_key_value_t * params) { int rc = 0; xmlNode *data = NULL; data = create_device_registration_xml(id, namespace, agent, params, NULL); rc = stonith_send_command(st, STONITH_OP_DEVICE_ADD, data, NULL, call_options, 0); free_xml(data); return rc; } static int stonith_api_remove_device(stonith_t * st, int call_options, const char *name) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, F_STONITH_DEVICE); crm_xml_add(data, F_STONITH_ORIGIN, __FUNCTION__); crm_xml_add(data, XML_ATTR_ID, name); rc = stonith_send_command(st, STONITH_OP_DEVICE_DEL, data, NULL, call_options, 0); free_xml(data); return rc; } static int stonith_api_remove_level_full(stonith_t *st, int options, const char *node, const char *pattern, const char *attr, const char *value, int level) { int rc = 0; xmlNode *data = NULL; CRM_CHECK(node || pattern || (attr && value), return -EINVAL); data = create_xml_node(NULL, XML_TAG_FENCING_LEVEL); crm_xml_add(data, F_STONITH_ORIGIN, __FUNCTION__); if (node) { crm_xml_add(data, XML_ATTR_STONITH_TARGET, node); } else if (pattern) { crm_xml_add(data, XML_ATTR_STONITH_TARGET_PATTERN, pattern); } else { crm_xml_add(data, XML_ATTR_STONITH_TARGET_ATTRIBUTE, attr); crm_xml_add(data, XML_ATTR_STONITH_TARGET_VALUE, value); } crm_xml_add_int(data, XML_ATTR_STONITH_INDEX, level); rc = stonith_send_command(st, STONITH_OP_LEVEL_DEL, data, NULL, options, 0); free_xml(data); return rc; } static int stonith_api_remove_level(stonith_t * st, int options, const char *node, int level) { return stonith_api_remove_level_full(st, options, node, NULL, NULL, NULL, level); } /*! * \internal * \brief Create XML for stonithd topology level registration request * * \param[in] node If not NULL, target level by this node name * \param[in] pattern If not NULL, target by node name using this regex * \param[in] attr If not NULL, target by this node attribute * \param[in] value If not NULL, target by this node attribute value * \param[in] level Index number of level to register * \param[in] device_list List of devices in level * * \return Newly allocated XML tree on success, NULL otherwise * * \note The caller should set only one of node, pattern or attr/value. */ xmlNode * create_level_registration_xml(const char *node, const char *pattern, const char *attr, const char *value, int level, stonith_key_value_t *device_list) { int len = 0; char *list = NULL; xmlNode *data; CRM_CHECK(node || pattern || (attr && value), return NULL); data = create_xml_node(NULL, XML_TAG_FENCING_LEVEL); CRM_CHECK(data, return NULL); crm_xml_add(data, F_STONITH_ORIGIN, __FUNCTION__); crm_xml_add_int(data, XML_ATTR_ID, level); crm_xml_add_int(data, XML_ATTR_STONITH_INDEX, level); if (node) { crm_xml_add(data, XML_ATTR_STONITH_TARGET, node); } else if (pattern) { crm_xml_add(data, XML_ATTR_STONITH_TARGET_PATTERN, pattern); } else { crm_xml_add(data, XML_ATTR_STONITH_TARGET_ATTRIBUTE, attr); crm_xml_add(data, XML_ATTR_STONITH_TARGET_VALUE, value); } for (; device_list; device_list = device_list->next) { int adding = strlen(device_list->value); if(list) { adding++; /* +1 space */ } crm_trace("Adding %s (%dc) at offset %d", device_list->value, adding, len); list = realloc_safe(list, len + adding + 1); /* +1 EOS */ if (list == NULL) { crm_perror(LOG_CRIT, "Could not create device list"); free_xml(data); return NULL; } sprintf(list + len, "%s%s", len?",":"", device_list->value); len += adding; } crm_xml_add(data, XML_ATTR_STONITH_DEVICES, list); free(list); return data; } static int stonith_api_register_level_full(stonith_t * st, int options, const char *node, const char *pattern, const char *attr, const char *value, int level, stonith_key_value_t *device_list) { int rc = 0; xmlNode *data = create_level_registration_xml(node, pattern, attr, value, level, device_list); CRM_CHECK(data != NULL, return -EINVAL); rc = stonith_send_command(st, STONITH_OP_LEVEL_ADD, data, NULL, options, 0); free_xml(data); return rc; } static int stonith_api_register_level(stonith_t * st, int options, const char *node, int level, stonith_key_value_t * device_list) { return stonith_api_register_level_full(st, options, node, NULL, NULL, NULL, level, device_list); } static void append_arg(const char *key, const char *value, char **args) { int len = 3; /* =, \n, \0 */ int last = 0; CRM_CHECK(key != NULL, return); CRM_CHECK(value != NULL, return); if (strstr(key, "pcmk_")) { return; } else if (strstr(key, CRM_META)) { return; } else if (safe_str_eq(key, "crm_feature_set")) { return; } len += strlen(key); len += strlen(value); if (*args != NULL) { last = strlen(*args); } *args = realloc_safe(*args, last + len); crm_trace("Appending: %s=%s", key, value); sprintf((*args) + last, "%s=%s\n", key, value); } static void append_config_arg(gpointer key, gpointer value, gpointer user_data) { /* stonithd will filter action out when it registers the device, * but ignore it here just in case any other library callers * fail to do so. */ if (safe_str_neq(key, STONITH_ATTR_ACTION_OP)) { append_arg(key, value, user_data); return; } } static void append_host_specific_args(const char *victim, const char *map, GHashTable * params, char **arg_list) { char *name = NULL; int last = 0, lpc = 0, max = 0; if (map == NULL) { /* The best default there is for now... */ crm_debug("Using default arg map: port=uname"); append_arg("port", victim, arg_list); return; } max = strlen(map); crm_debug("Processing arg map: %s", map); for (; lpc < max + 1; lpc++) { if (isalpha(map[lpc])) { /* keep going */ } else if (map[lpc] == '=' || map[lpc] == ':') { free(name); name = calloc(1, 1 + lpc - last); memcpy(name, map + last, lpc - last); crm_debug("Got name: %s", name); last = lpc + 1; } else if (map[lpc] == 0 || map[lpc] == ',' || isspace(map[lpc])) { char *param = NULL; const char *value = NULL; param = calloc(1, 1 + lpc - last); memcpy(param, map + last, lpc - last); last = lpc + 1; crm_debug("Got key: %s", param); if (name == NULL) { crm_err("Misparsed '%s', found '%s' without a name", map, param); free(param); continue; } if (safe_str_eq(param, "uname")) { value = victim; } else { char *key = crm_meta_name(param); value = g_hash_table_lookup(params, key); free(key); } if (value) { crm_debug("Setting '%s'='%s' (%s) for %s", name, value, param, victim); append_arg(name, value, arg_list); } else { crm_err("No node attribute '%s' for '%s'", name, victim); } free(name); name = NULL; free(param); if (map[lpc] == 0) { break; } } else if (isspace(map[lpc])) { last = lpc; } } free(name); } static char * make_args(const char *agent, const char *action, const char *victim, uint32_t victim_nodeid, GHashTable * device_args, GHashTable * port_map) { char buffer[512]; char *arg_list = NULL; const char *value = NULL; CRM_CHECK(action != NULL, return NULL); buffer[511] = 0; snprintf(buffer, 511, "pcmk_%s_action", action); if (device_args) { value = g_hash_table_lookup(device_args, buffer); } if (value == NULL && device_args) { - /* Legacy support for early 1.1 releases - Remove for 1.4 */ + /* @COMPAT deprecated since 1.1.6 */ snprintf(buffer, 511, "pcmk_%s_cmd", action); value = g_hash_table_lookup(device_args, buffer); } if (value == NULL && device_args && safe_str_eq(action, "off")) { - /* Legacy support for late 1.1 releases - Remove for 1.4 */ + /* @COMPAT deprecated since 1.1.8 */ value = g_hash_table_lookup(device_args, "pcmk_poweroff_action"); } if (value) { crm_info("Substituting action '%s' for requested operation '%s'", value, action); action = value; } append_arg(STONITH_ATTR_ACTION_OP, action, &arg_list); if (victim && device_args) { const char *alias = victim; const char *param = g_hash_table_lookup(device_args, STONITH_ATTR_HOSTARG); if (port_map && g_hash_table_lookup(port_map, victim)) { alias = g_hash_table_lookup(port_map, victim); } /* Always supply the node's name too: * https://fedorahosted.org/cluster/wiki/FenceAgentAPI */ append_arg("nodename", victim, &arg_list); if (victim_nodeid) { char nodeid_str[33] = { 0, }; if (snprintf(nodeid_str, 33, "%u", (unsigned int)victim_nodeid)) { crm_info("For stonith action (%s) for victim %s, adding nodeid (%s) to parameters", action, victim, nodeid_str); append_arg("nodeid", nodeid_str, &arg_list); } } /* Check if we need to supply the victim in any other form */ if(safe_str_eq(agent, "fence_legacy")) { value = agent; } else if (param == NULL) { const char *map = g_hash_table_lookup(device_args, STONITH_ATTR_ARGMAP); if (map == NULL) { param = "port"; value = g_hash_table_lookup(device_args, param); } else { /* Legacy handling */ append_host_specific_args(alias, map, device_args, &arg_list); value = map; /* Nothing more to do */ } } else if (safe_str_eq(param, "none")) { value = param; /* Nothing more to do */ } else { value = g_hash_table_lookup(device_args, param); } /* Don't overwrite explictly set values for $param */ if (value == NULL || safe_str_eq(value, "dynamic")) { crm_debug("Performing %s action for node '%s' as '%s=%s'", action, victim, param, alias); append_arg(param, alias, &arg_list); } } if (device_args) { g_hash_table_foreach(device_args, append_config_arg, &arg_list); } return arg_list; } static gboolean st_child_term(gpointer data) { int rc = 0; stonith_action_t *track = data; crm_info("Child %d timed out, sending SIGTERM", track->pid); track->timer_sigterm = 0; track->last_timeout_signo = SIGTERM; rc = kill(-track->pid, SIGTERM); if (rc < 0) { crm_perror(LOG_ERR, "Couldn't send SIGTERM to %d", track->pid); } return FALSE; } static gboolean st_child_kill(gpointer data) { int rc = 0; stonith_action_t *track = data; crm_info("Child %d timed out, sending SIGKILL", track->pid); track->timer_sigkill = 0; track->last_timeout_signo = SIGKILL; rc = kill(-track->pid, SIGKILL); if (rc < 0) { crm_perror(LOG_ERR, "Couldn't send SIGKILL to %d", track->pid); } return FALSE; } static void stonith_action_clear_tracking_data(stonith_action_t * action) { if (action->timer_sigterm > 0) { g_source_remove(action->timer_sigterm); action->timer_sigterm = 0; } if (action->timer_sigkill > 0) { g_source_remove(action->timer_sigkill); action->timer_sigkill = 0; } if (action->fd_stdout) { close(action->fd_stdout); action->fd_stdout = 0; } if (action->fd_stderr) { close(action->fd_stderr); action->fd_stderr = 0; } free(action->output); action->output = NULL; free(action->error); action->error = NULL; action->rc = 0; action->pid = 0; action->last_timeout_signo = 0; } static void stonith_action_destroy(stonith_action_t * action) { stonith_action_clear_tracking_data(action); free(action->agent); free(action->args); free(action->action); free(action->victim); free(action); } #define FAILURE_MAX_RETRIES 2 stonith_action_t * stonith_action_create(const char *agent, const char *_action, const char *victim, uint32_t victim_nodeid, int timeout, GHashTable * device_args, GHashTable * port_map) { stonith_action_t *action; action = calloc(1, sizeof(stonith_action_t)); crm_debug("Initiating action %s for agent %s (target=%s)", _action, agent, victim); action->args = make_args(agent, _action, victim, victim_nodeid, device_args, port_map); action->agent = strdup(agent); action->action = strdup(_action); if (victim) { action->victim = strdup(victim); } action->timeout = action->remaining_timeout = timeout; action->max_retries = FAILURE_MAX_RETRIES; if (device_args) { char buffer[512]; const char *value = NULL; snprintf(buffer, 511, "pcmk_%s_retries", _action); value = g_hash_table_lookup(device_args, buffer); if (value) { action->max_retries = atoi(value); } } return action; } #define READ_MAX 500 static char * read_output(int fd) { char buffer[READ_MAX]; char *output = NULL; int len = 0; int more = 0; if (!fd) { return NULL; } do { errno = 0; memset(&buffer, 0, READ_MAX); more = read(fd, buffer, READ_MAX - 1); if (more > 0) { buffer[more] = 0; /* Make sure it's nul-terminated for logging * 'more' is always less than our buffer size */ output = realloc_safe(output, len + more + 1); snprintf(output + len, more + 1, "%s", buffer); len += more; } } while (more == (READ_MAX - 1) || (more < 0 && errno == EINTR)); return output; } static gboolean update_remaining_timeout(stonith_action_t * action) { int diff = time(NULL) - action->initial_start_time; if (action->tries >= action->max_retries) { crm_info("Attempted to execute agent %s (%s) the maximum number of times (%d) allowed", action->agent, action->action, action->max_retries); action->remaining_timeout = 0; } else if ((action->rc != -ETIME) && diff < (action->timeout * 0.7)) { /* only set remaining timeout period if there is 30% * or greater of the original timeout period left */ action->remaining_timeout = action->timeout - diff; } else { action->remaining_timeout = 0; } return action->remaining_timeout ? TRUE : FALSE; } static void stonith_action_async_done(mainloop_child_t * p, pid_t pid, int core, int signo, int exitcode) { stonith_action_t *action = mainloop_child_userdata(p); if (action->timer_sigterm > 0) { g_source_remove(action->timer_sigterm); action->timer_sigterm = 0; } if (action->timer_sigkill > 0) { g_source_remove(action->timer_sigkill); action->timer_sigkill = 0; } action->output = read_output(action->fd_stdout); action->error = read_output(action->fd_stderr); if (action->last_timeout_signo) { action->rc = -ETIME; crm_notice("Child process %d performing action '%s' timed out with signal %d", pid, action->action, action->last_timeout_signo); } else if (signo) { action->rc = -ECONNABORTED; crm_notice("Child process %d performing action '%s' timed out with signal %d", pid, action->action, signo); } else { crm_debug("Child process %d performing action '%s' exited with rc %d", pid, action->action, exitcode); if (exitcode > 0) { /* Try to provide a useful error code based on the fence agent's * error output. */ if (action->error == NULL) { exitcode = -ENODATA; } else if (strstr(action->error, "imed out")) { /* Some agents have their own internal timeouts */ exitcode = -ETIMEDOUT; } else if (strstr(action->error, "Unrecognised action")) { exitcode = -EOPNOTSUPP; } else { exitcode = -pcmk_err_generic; } } action->rc = exitcode; } log_action(action, pid); if (action->rc != pcmk_ok && update_remaining_timeout(action)) { int rc = internal_stonith_action_execute(action); if (rc == pcmk_ok) { return; } } if (action->done_cb) { action->done_cb(pid, action->rc, action->output, action->userdata); } stonith_action_destroy(action); } static int internal_stonith_action_execute(stonith_action_t * action) { int pid, status = 0, len, rc = -EPROTO; int ret; int total = 0; int p_read_fd, p_write_fd; /* parent read/write file descriptors */ int c_read_fd, c_write_fd; /* child read/write file descriptors */ int c_stderr_fd, p_stderr_fd; /* parent/child side file descriptors for stderr */ int fd1[2]; int fd2[2]; int fd3[2]; int is_retry = 0; /* clear any previous tracking data */ stonith_action_clear_tracking_data(action); if (!action->tries) { action->initial_start_time = time(NULL); } action->tries++; if (action->tries > 1) { crm_info("Attempt %d to execute %s (%s). remaining timeout is %d", action->tries, action->agent, action->action, action->remaining_timeout); is_retry = 1; } c_read_fd = c_write_fd = p_read_fd = p_write_fd = c_stderr_fd = p_stderr_fd = -1; if (action->args == NULL || action->agent == NULL) goto fail; len = strlen(action->args); if (pipe(fd1)) goto fail; p_read_fd = fd1[0]; c_write_fd = fd1[1]; if (pipe(fd2)) goto fail; c_read_fd = fd2[0]; p_write_fd = fd2[1]; if (pipe(fd3)) goto fail; p_stderr_fd = fd3[0]; c_stderr_fd = fd3[1]; crm_debug("forking"); pid = fork(); if (pid < 0) { rc = -ECHILD; goto fail; } if (!pid) { /* child */ setpgid(0, 0); close(1); /* coverity[leaked_handle] False positive */ if (dup(c_write_fd) < 0) goto fail; close(2); /* coverity[leaked_handle] False positive */ if (dup(c_stderr_fd) < 0) goto fail; close(0); /* coverity[leaked_handle] False positive */ if (dup(c_read_fd) < 0) goto fail; /* keep c_stderr_fd open so parent can report all errors. */ /* keep c_write_fd open so hostlist can be sent to parent. */ close(c_read_fd); close(p_read_fd); close(p_write_fd); close(p_stderr_fd); /* keep retries from executing out of control */ if (is_retry) { sleep(1); } execlp(action->agent, action->agent, NULL); exit(EXIT_FAILURE); } /* parent */ action->pid = pid; ret = fcntl(p_read_fd, F_SETFL, fcntl(p_read_fd, F_GETFL, 0) | O_NONBLOCK); if (ret < 0) { crm_perror(LOG_NOTICE, "Could not change the output of %s to be non-blocking", action->agent); } ret = fcntl(p_stderr_fd, F_SETFL, fcntl(p_stderr_fd, F_GETFL, 0) | O_NONBLOCK); if (ret < 0) { crm_perror(LOG_NOTICE, "Could not change the stderr of %s to be non-blocking", action->agent); } do { crm_debug("sending args"); ret = write(p_write_fd, action->args + total, len - total); if (ret > 0) { total += ret; } } while (errno == EINTR && total < len); if (total != len) { crm_perror(LOG_ERR, "Sent %d not %d bytes", total, len); if (ret >= 0) { rc = -ECOMM; } goto fail; } close(p_write_fd); p_write_fd = -1; /* async */ if (action->async) { action->fd_stdout = p_read_fd; action->fd_stderr = p_stderr_fd; mainloop_child_add(pid, 0/* Move the timeout here? */, action->action, action, stonith_action_async_done); crm_trace("Op: %s on %s, pid: %d, timeout: %ds", action->action, action->agent, pid, action->remaining_timeout); action->last_timeout_signo = 0; if (action->remaining_timeout) { action->timer_sigterm = g_timeout_add(1000 * action->remaining_timeout, st_child_term, action); action->timer_sigkill = g_timeout_add(1000 * (action->remaining_timeout + 5), st_child_kill, action); } else { crm_err("No timeout set for stonith operation %s with device %s", action->action, action->agent); } close(c_write_fd); close(c_read_fd); close(c_stderr_fd); return 0; } else { /* sync */ int timeout = action->remaining_timeout + 1; pid_t p = 0; while (action->remaining_timeout < 0 || timeout > 0) { p = waitpid(pid, &status, WNOHANG); if (p > 0) { break; } sleep(1); timeout--; } if (timeout == 0) { int killrc = kill(-pid, SIGKILL); if (killrc && errno != ESRCH) { crm_err("kill(%d, KILL) failed: %s (%d)", pid, pcmk_strerror(errno), errno); } /* * From sigprocmask(2): * It is not possible to block SIGKILL or SIGSTOP. Attempts to do so are silently ignored. * * This makes it safe to skip WNOHANG here */ p = waitpid(pid, &status, 0); } if (p <= 0) { crm_perror(LOG_ERR, "waitpid(%d)", pid); } else if (p != pid) { crm_err("Waited for %d, got %d", pid, p); } action->output = read_output(p_read_fd); action->error = read_output(p_stderr_fd); action->rc = -ECONNABORTED; log_action(action, pid); rc = action->rc; if (timeout == 0) { action->rc = -ETIME; } else if (WIFEXITED(status)) { crm_debug("result = %d", WEXITSTATUS(status)); action->rc = -WEXITSTATUS(status); rc = 0; } else if (WIFSIGNALED(status)) { crm_err("call %s for %s exited due to signal %d", action->action, action->agent, WTERMSIG(status)); } else { crm_err("call %s for %s returned unexpected status %#x", action->action, action->agent, status); } } fail: if (p_read_fd >= 0) { close(p_read_fd); } if (p_write_fd >= 0) { close(p_write_fd); } if (p_stderr_fd >= 0) { close(p_stderr_fd); } if (c_read_fd >= 0) { close(c_read_fd); } if (c_write_fd >= 0) { close(c_write_fd); } if (c_stderr_fd >= 0) { close(c_stderr_fd); } return rc; } GPid stonith_action_execute_async(stonith_action_t * action, void *userdata, void (*done) (GPid pid, int rc, const char *output, gpointer user_data)) { int rc = 0; if (!action) { return -1; } action->userdata = userdata; action->done_cb = done; action->async = 1; rc = internal_stonith_action_execute(action); return rc < 0 ? rc : action->pid; } int stonith_action_execute(stonith_action_t * action, int *agent_result, char **output) { int rc = 0; if (!action) { return -1; } do { rc = internal_stonith_action_execute(action); if (rc == pcmk_ok) { /* success! */ break; } /* keep retrying while we have time left */ } while (update_remaining_timeout(action)); if (rc) { /* error */ return rc; } if (agent_result) { *agent_result = action->rc; } if (output) { *output = action->output; action->output = NULL; /* handed it off, do not free */ } stonith_action_destroy(action); return rc; } static int stonith_api_device_list(stonith_t * stonith, int call_options, const char *namespace, stonith_key_value_t ** devices, int timeout) { int count = 0; if (devices == NULL) { crm_err("Parameter error: stonith_api_device_list"); return -EFAULT; } /* Include Heartbeat agents */ if (namespace == NULL || safe_str_eq("heartbeat", namespace)) { #if HAVE_STONITH_STONITH_H static gboolean need_init = TRUE; char **entry = NULL; char **type_list = NULL; static char **(*type_list_fn) (void) = NULL; static void (*type_free_fn) (char **) = NULL; if (need_init) { need_init = FALSE; type_list_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_types", FALSE); type_free_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_free_hostlist", FALSE); } if (type_list_fn) { type_list = (*type_list_fn) (); } for (entry = type_list; entry != NULL && *entry; ++entry) { crm_trace("Added: %s", *entry); *devices = stonith_key_value_add(*devices, NULL, *entry); count++; } if (type_list && type_free_fn) { (*type_free_fn) (type_list); } #else if (namespace != NULL) { return -EINVAL; /* Heartbeat agents not supported */ } #endif } /* Include Red Hat agents, basically: ls -1 @sbin_dir@/fence_* */ if (namespace == NULL || safe_str_eq("redhat", namespace)) { struct dirent **namelist; int file_num = scandir(RH_STONITH_DIR, &namelist, 0, alphasort); if (file_num > 0) { struct stat prop; char buffer[FILENAME_MAX + 1]; while (file_num--) { if ('.' == namelist[file_num]->d_name[0]) { free(namelist[file_num]); continue; } else if (0 != strncmp(RH_STONITH_PREFIX, namelist[file_num]->d_name, strlen(RH_STONITH_PREFIX))) { free(namelist[file_num]); continue; } snprintf(buffer, FILENAME_MAX, "%s/%s", RH_STONITH_DIR, namelist[file_num]->d_name); if (stat(buffer, &prop) == 0 && S_ISREG(prop.st_mode)) { *devices = stonith_key_value_add(*devices, NULL, namelist[file_num]->d_name); count++; } free(namelist[file_num]); } free(namelist); } } return count; } #if HAVE_STONITH_STONITH_H static inline char * strdup_null(const char *val) { if (val) { return strdup(val); } return NULL; } static void stonith_plugin(int priority, const char *fmt, ...) __attribute__((__format__ (__printf__, 2, 3))); static void stonith_plugin(int priority, const char *format, ...) { int err = errno; va_list ap; int len = 0; char *string = NULL; va_start(ap, format); len = vasprintf (&string, format, ap); CRM_ASSERT(len > 0); do_crm_log_alias(priority, __FILE__, __func__, __LINE__, "%s", string); free(string); errno = err; } #endif static int stonith_api_device_metadata(stonith_t * stonith, int call_options, const char *agent, const char *namespace, char **output, int timeout) { int rc = 0; char *buffer = NULL; const char *provider = get_stonith_provider(agent, namespace); crm_trace("looking up %s/%s metadata", agent, provider); /* By having this in a library, we can access it from stonith_admin * when neither lrmd or stonith-ng are running * Important for the crm shell's validations... */ if (safe_str_eq(provider, "redhat")) { stonith_action_t *action = stonith_action_create(agent, "metadata", NULL, 0, 5, NULL, NULL); int exec_rc = stonith_action_execute(action, &rc, &buffer); xmlNode *xml = NULL; xmlNode *actions = NULL; xmlXPathObject *xpathObj = NULL; if (exec_rc < 0 || rc != 0 || buffer == NULL) { crm_warn("Could not obtain metadata for %s", agent); crm_debug("Query failed: %d %d: %s", exec_rc, rc, crm_str(buffer)); free(buffer); /* Just in case */ return -EINVAL; } xml = string2xml(buffer); if(xml == NULL) { crm_warn("Metadata for %s is invalid", agent); free(buffer); return -EINVAL; } xpathObj = xpath_search(xml, "//actions"); if (numXpathResults(xpathObj) > 0) { actions = getXpathResult(xpathObj, 0); } freeXpathObject(xpathObj); /* Now fudge the metadata so that the start/stop actions appear */ xpathObj = xpath_search(xml, "//action[@name='stop']"); if (numXpathResults(xpathObj) <= 0) { xmlNode *tmp = NULL; tmp = create_xml_node(actions, "action"); crm_xml_add(tmp, "name", "stop"); crm_xml_add(tmp, "timeout", "20s"); tmp = create_xml_node(actions, "action"); crm_xml_add(tmp, "name", "start"); crm_xml_add(tmp, "timeout", "20s"); } freeXpathObject(xpathObj); /* Now fudge the metadata so that the port isn't required in the configuration */ xpathObj = xpath_search(xml, "//parameter[@name='port']"); if (numXpathResults(xpathObj) > 0) { /* We'll fill this in */ xmlNode *tmp = getXpathResult(xpathObj, 0); crm_xml_add(tmp, "required", "0"); } freeXpathObject(xpathObj); free(buffer); buffer = dump_xml_formatted_with_text(xml); free_xml(xml); if (!buffer) { return -EINVAL; } } else { #if !HAVE_STONITH_STONITH_H return -EINVAL; /* Heartbeat agents not supported */ #else int bufferlen = 0; static const char *no_parameter_info = ""; Stonith *stonith_obj = NULL; static gboolean need_init = TRUE; static Stonith *(*st_new_fn) (const char *) = NULL; static const char *(*st_info_fn) (Stonith *, int) = NULL; static void (*st_del_fn) (Stonith *) = NULL; static void (*st_log_fn) (Stonith *, PILLogFun) = NULL; if (need_init) { need_init = FALSE; st_new_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_new", FALSE); st_del_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_delete", FALSE); st_log_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_set_log", FALSE); st_info_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_get_info", FALSE); } if (lha_agents_lib && st_new_fn && st_del_fn && st_info_fn && st_log_fn) { char *xml_meta_longdesc = NULL; char *xml_meta_shortdesc = NULL; char *meta_param = NULL; char *meta_longdesc = NULL; char *meta_shortdesc = NULL; stonith_obj = (*st_new_fn) (agent); if (stonith_obj) { (*st_log_fn) (stonith_obj, (PILLogFun) & stonith_plugin); meta_longdesc = strdup_null((*st_info_fn) (stonith_obj, ST_DEVICEDESCR)); if (meta_longdesc == NULL) { crm_warn("no long description in %s's metadata.", agent); meta_longdesc = strdup(no_parameter_info); } meta_shortdesc = strdup_null((*st_info_fn) (stonith_obj, ST_DEVICEID)); if (meta_shortdesc == NULL) { crm_warn("no short description in %s's metadata.", agent); meta_shortdesc = strdup(no_parameter_info); } meta_param = strdup_null((*st_info_fn) (stonith_obj, ST_CONF_XML)); if (meta_param == NULL) { crm_warn("no list of parameters in %s's metadata.", agent); meta_param = strdup(no_parameter_info); } (*st_del_fn) (stonith_obj); } else { return -EINVAL; /* Heartbeat agents not supported */ } xml_meta_longdesc = (char *)xmlEncodeEntitiesReentrant(NULL, (const unsigned char *)meta_longdesc); xml_meta_shortdesc = (char *)xmlEncodeEntitiesReentrant(NULL, (const unsigned char *)meta_shortdesc); bufferlen = strlen(META_TEMPLATE) + strlen(agent) + strlen(xml_meta_longdesc) + strlen(xml_meta_shortdesc) + strlen(meta_param) + 1; buffer = calloc(1, bufferlen); snprintf(buffer, bufferlen - 1, META_TEMPLATE, agent, xml_meta_longdesc, xml_meta_shortdesc, meta_param); xmlFree(xml_meta_longdesc); xmlFree(xml_meta_shortdesc); free(meta_shortdesc); free(meta_longdesc); free(meta_param); } #endif } if (output) { *output = buffer; } else { free(buffer); } return rc; } static int stonith_api_query(stonith_t * stonith, int call_options, const char *target, stonith_key_value_t ** devices, int timeout) { int rc = 0, lpc = 0, max = 0; xmlNode *data = NULL; xmlNode *output = NULL; xmlXPathObjectPtr xpathObj = NULL; CRM_CHECK(devices != NULL, return -EINVAL); data = create_xml_node(NULL, F_STONITH_DEVICE); crm_xml_add(data, F_STONITH_ORIGIN, __FUNCTION__); crm_xml_add(data, F_STONITH_TARGET, target); crm_xml_add(data, F_STONITH_ACTION, "off"); rc = stonith_send_command(stonith, STONITH_OP_QUERY, data, &output, call_options, timeout); if (rc < 0) { return rc; } xpathObj = xpath_search(output, "//@agent"); if (xpathObj) { max = numXpathResults(xpathObj); for (lpc = 0; lpc < max; lpc++) { xmlNode *match = getXpathResult(xpathObj, lpc); CRM_LOG_ASSERT(match != NULL); if(match != NULL) { xmlChar *match_path = xmlGetNodePath(match); crm_info("%s[%d] = %s", "//@agent", lpc, match_path); free(match_path); *devices = stonith_key_value_add(*devices, NULL, crm_element_value(match, XML_ATTR_ID)); } } freeXpathObject(xpathObj); } free_xml(output); free_xml(data); return max; } static int stonith_api_call(stonith_t * stonith, int call_options, const char *id, const char *action, const char *victim, int timeout, xmlNode ** output) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, F_STONITH_DEVICE); crm_xml_add(data, F_STONITH_ORIGIN, __FUNCTION__); crm_xml_add(data, F_STONITH_DEVICE, id); crm_xml_add(data, F_STONITH_ACTION, action); crm_xml_add(data, F_STONITH_TARGET, victim); rc = stonith_send_command(stonith, STONITH_OP_EXEC, data, output, call_options, timeout); free_xml(data); return rc; } static int stonith_api_list(stonith_t * stonith, int call_options, const char *id, char **list_info, int timeout) { int rc; xmlNode *output = NULL; rc = stonith_api_call(stonith, call_options, id, "list", NULL, timeout, &output); if (output && list_info) { const char *list_str; list_str = crm_element_value(output, "st_output"); if (list_str) { *list_info = strdup(list_str); } } if (output) { free_xml(output); } return rc; } static int stonith_api_monitor(stonith_t * stonith, int call_options, const char *id, int timeout) { return stonith_api_call(stonith, call_options, id, "monitor", NULL, timeout, NULL); } static int stonith_api_status(stonith_t * stonith, int call_options, const char *id, const char *port, int timeout) { return stonith_api_call(stonith, call_options, id, "status", port, timeout, NULL); } static int stonith_api_fence(stonith_t * stonith, int call_options, const char *node, const char *action, int timeout, int tolerance) { int rc = 0; xmlNode *data = NULL; data = create_xml_node(NULL, __FUNCTION__); crm_xml_add(data, F_STONITH_TARGET, node); crm_xml_add(data, F_STONITH_ACTION, action); crm_xml_add_int(data, F_STONITH_TIMEOUT, timeout); crm_xml_add_int(data, F_STONITH_TOLERANCE, tolerance); rc = stonith_send_command(stonith, STONITH_OP_FENCE, data, NULL, call_options, timeout); free_xml(data); return rc; } static int stonith_api_confirm(stonith_t * stonith, int call_options, const char *target) { return stonith_api_fence(stonith, call_options | st_opt_manual_ack, target, "off", 0, 0); } static int stonith_api_history(stonith_t * stonith, int call_options, const char *node, stonith_history_t ** history, int timeout) { int rc = 0; xmlNode *data = NULL; xmlNode *output = NULL; stonith_history_t *last = NULL; *history = NULL; if (node) { data = create_xml_node(NULL, __FUNCTION__); crm_xml_add(data, F_STONITH_TARGET, node); } rc = stonith_send_command(stonith, STONITH_OP_FENCE_HISTORY, data, &output, call_options | st_opt_sync_call, timeout); free_xml(data); if (rc == 0) { xmlNode *op = NULL; xmlNode *reply = get_xpath_object("//" F_STONITH_HISTORY_LIST, output, LOG_ERR); for (op = __xml_first_child(reply); op != NULL; op = __xml_next(op)) { stonith_history_t *kvp; kvp = calloc(1, sizeof(stonith_history_t)); kvp->target = crm_element_value_copy(op, F_STONITH_TARGET); kvp->action = crm_element_value_copy(op, F_STONITH_ACTION); kvp->origin = crm_element_value_copy(op, F_STONITH_ORIGIN); kvp->delegate = crm_element_value_copy(op, F_STONITH_DELEGATE); kvp->client = crm_element_value_copy(op, F_STONITH_CLIENTNAME); crm_element_value_int(op, F_STONITH_DATE, &kvp->completed); crm_element_value_int(op, F_STONITH_STATE, &kvp->state); if (last) { last->next = kvp; } else { *history = kvp; } last = kvp; } } return rc; } gboolean is_redhat_agent(const char *agent) { int rc = 0; struct stat prop; char buffer[FILENAME_MAX + 1]; snprintf(buffer, FILENAME_MAX, "%s/%s", RH_STONITH_DIR, agent); rc = stat(buffer, &prop); if (rc >= 0 && S_ISREG(prop.st_mode)) { return TRUE; } return FALSE; } const char * get_stonith_provider(const char *agent, const char *provider) { /* This function sucks */ if (is_redhat_agent(agent)) { return "redhat"; #if HAVE_STONITH_STONITH_H } else { Stonith *stonith_obj = NULL; static gboolean need_init = TRUE; static Stonith *(*st_new_fn) (const char *) = NULL; static void (*st_del_fn) (Stonith *) = NULL; if (need_init) { need_init = FALSE; st_new_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_new", FALSE); st_del_fn = find_library_function(&lha_agents_lib, LHA_STONITH_LIBRARY, "stonith_delete", FALSE); } if (lha_agents_lib && st_new_fn && st_del_fn) { stonith_obj = (*st_new_fn) (agent); if (stonith_obj) { (*st_del_fn) (stonith_obj); return "heartbeat"; } } #endif } if (safe_str_eq(provider, "internal")) { return provider; } else { crm_err("No such device: %s", agent); return NULL; } } static gint stonithlib_GCompareFunc(gconstpointer a, gconstpointer b) { int rc = 0; const stonith_notify_client_t *a_client = a; const stonith_notify_client_t *b_client = b; CRM_CHECK(a_client->event != NULL && b_client->event != NULL, return 0); rc = strcmp(a_client->event, b_client->event); if (rc == 0) { if (a_client->notify == NULL || b_client->notify == NULL) { return 0; } else if (a_client->notify == b_client->notify) { return 0; } else if (((long)a_client->notify) < ((long)b_client->notify)) { crm_err("callbacks for %s are not equal: %p vs. %p", a_client->event, a_client->notify, b_client->notify); return -1; } crm_err("callbacks for %s are not equal: %p vs. %p", a_client->event, a_client->notify, b_client->notify); return 1; } return rc; } xmlNode * stonith_create_op(int call_id, const char *token, const char *op, xmlNode * data, int call_options) { xmlNode *op_msg = create_xml_node(NULL, "stonith_command"); CRM_CHECK(op_msg != NULL, return NULL); CRM_CHECK(token != NULL, return NULL); crm_xml_add(op_msg, F_XML_TAGNAME, "stonith_command"); crm_xml_add(op_msg, F_TYPE, T_STONITH_NG); crm_xml_add(op_msg, F_STONITH_CALLBACK_TOKEN, token); crm_xml_add(op_msg, F_STONITH_OPERATION, op); crm_xml_add_int(op_msg, F_STONITH_CALLID, call_id); crm_trace("Sending call options: %.8lx, %d", (long)call_options, call_options); crm_xml_add_int(op_msg, F_STONITH_CALLOPTS, call_options); if (data != NULL) { add_message_xml(op_msg, F_STONITH_CALLDATA, data); } return op_msg; } static void stonith_destroy_op_callback(gpointer data) { stonith_callback_client_t *blob = data; if (blob->timer && blob->timer->ref > 0) { g_source_remove(blob->timer->ref); } free(blob->timer); free(blob); } static int stonith_api_signoff(stonith_t * stonith) { stonith_private_t *native = stonith->private; crm_debug("Signing out of the STONITH Service"); if (native->source != NULL) { /* Attached to mainloop */ mainloop_del_ipc_client(native->source); native->source = NULL; native->ipc = NULL; } else if (native->ipc) { /* Not attached to mainloop */ crm_ipc_t *ipc = native->ipc; native->ipc = NULL; crm_ipc_close(ipc); crm_ipc_destroy(ipc); } free(native->token); native->token = NULL; stonith->state = stonith_disconnected; return pcmk_ok; } static int stonith_api_signon(stonith_t * stonith, const char *name, int *stonith_fd) { int rc = pcmk_ok; stonith_private_t *native = stonith->private; static struct ipc_client_callbacks st_callbacks = { .dispatch = stonith_dispatch_internal, .destroy = stonith_connection_destroy }; crm_trace("Connecting command channel"); stonith->state = stonith_connected_command; if (stonith_fd) { /* No mainloop */ native->ipc = crm_ipc_new("stonith-ng", 0); if (native->ipc && crm_ipc_connect(native->ipc)) { *stonith_fd = crm_ipc_get_fd(native->ipc); } else if (native->ipc) { crm_perror(LOG_ERR, "Connection to STONITH manager failed"); rc = -ENOTCONN; } } else { /* With mainloop */ native->source = mainloop_add_ipc_client("stonith-ng", G_PRIORITY_MEDIUM, 0, stonith, &st_callbacks); native->ipc = mainloop_get_ipc_client(native->source); } if (native->ipc == NULL) { crm_debug("Could not connect to the Stonith API"); rc = -ENOTCONN; } if (rc == pcmk_ok) { xmlNode *reply = NULL; xmlNode *hello = create_xml_node(NULL, "stonith_command"); crm_xml_add(hello, F_TYPE, T_STONITH_NG); crm_xml_add(hello, F_STONITH_OPERATION, CRM_OP_REGISTER); crm_xml_add(hello, F_STONITH_CLIENTNAME, name); rc = crm_ipc_send(native->ipc, hello, crm_ipc_client_response, -1, &reply); if (rc < 0) { crm_perror(LOG_DEBUG, "Couldn't complete registration with the fencing API: %d", rc); rc = -ECOMM; } else if (reply == NULL) { crm_err("Did not receive registration reply"); rc = -EPROTO; } else { const char *msg_type = crm_element_value(reply, F_STONITH_OPERATION); const char *tmp_ticket = crm_element_value(reply, F_STONITH_CLIENTID); if (safe_str_neq(msg_type, CRM_OP_REGISTER)) { crm_err("Invalid registration message: %s", msg_type); crm_log_xml_err(reply, "Bad reply"); rc = -EPROTO; } else if (tmp_ticket == NULL) { crm_err("No registration token provided"); crm_log_xml_err(reply, "Bad reply"); rc = -EPROTO; } else { crm_trace("Obtained registration token: %s", tmp_ticket); native->token = strdup(tmp_ticket); rc = pcmk_ok; } } free_xml(reply); free_xml(hello); } if (rc == pcmk_ok) { #if HAVE_MSGFROMIPC_TIMEOUT stonith->call_timeout = MAX_IPC_DELAY; #endif crm_debug("Connection to STONITH successful"); return pcmk_ok; } crm_debug("Connection to STONITH failed: %s", pcmk_strerror(rc)); stonith->cmds->disconnect(stonith); return rc; } static int stonith_set_notification(stonith_t * stonith, const char *callback, int enabled) { int rc = pcmk_ok; xmlNode *notify_msg = create_xml_node(NULL, __FUNCTION__); stonith_private_t *native = stonith->private; if (stonith->state != stonith_disconnected) { crm_xml_add(notify_msg, F_STONITH_OPERATION, T_STONITH_NOTIFY); if (enabled) { crm_xml_add(notify_msg, F_STONITH_NOTIFY_ACTIVATE, callback); } else { crm_xml_add(notify_msg, F_STONITH_NOTIFY_DEACTIVATE, callback); } rc = crm_ipc_send(native->ipc, notify_msg, crm_ipc_client_response, -1, NULL); if (rc < 0) { crm_perror(LOG_DEBUG, "Couldn't register for fencing notifications: %d", rc); rc = -ECOMM; } else { rc = pcmk_ok; } } free_xml(notify_msg); return rc; } static int stonith_api_add_notification(stonith_t * stonith, const char *event, void (*callback) (stonith_t * stonith, stonith_event_t * e)) { GList *list_item = NULL; stonith_notify_client_t *new_client = NULL; stonith_private_t *private = NULL; private = stonith->private; crm_trace("Adding callback for %s events (%d)", event, g_list_length(private->notify_list)); new_client = calloc(1, sizeof(stonith_notify_client_t)); new_client->event = event; new_client->notify = callback; list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc); if (list_item != NULL) { crm_warn("Callback already present"); free(new_client); return -ENOTUNIQ; } else { private->notify_list = g_list_append(private->notify_list, new_client); stonith_set_notification(stonith, event, 1); crm_trace("Callback added (%d)", g_list_length(private->notify_list)); } return pcmk_ok; } static int stonith_api_del_notification(stonith_t * stonith, const char *event) { GList *list_item = NULL; stonith_notify_client_t *new_client = NULL; stonith_private_t *private = NULL; crm_debug("Removing callback for %s events", event); private = stonith->private; new_client = calloc(1, sizeof(stonith_notify_client_t)); new_client->event = event; new_client->notify = NULL; list_item = g_list_find_custom(private->notify_list, new_client, stonithlib_GCompareFunc); stonith_set_notification(stonith, event, 0); if (list_item != NULL) { stonith_notify_client_t *list_client = list_item->data; private->notify_list = g_list_remove(private->notify_list, list_client); free(list_client); crm_trace("Removed callback"); } else { crm_trace("Callback not present"); } free(new_client); return pcmk_ok; } static gboolean stonith_async_timeout_handler(gpointer data) { struct timer_rec_s *timer = data; crm_err("Async call %d timed out after %dms", timer->call_id, timer->timeout); stonith_perform_callback(timer->stonith, NULL, timer->call_id, -ETIME); /* Always return TRUE, never remove the handler * We do that in stonith_del_callback() */ return TRUE; } static void set_callback_timeout(stonith_callback_client_t * callback, stonith_t * stonith, int call_id, int timeout) { struct timer_rec_s *async_timer = callback->timer; if (timeout <= 0) { return; } if (!async_timer) { async_timer = calloc(1, sizeof(struct timer_rec_s)); callback->timer = async_timer; } async_timer->stonith = stonith; async_timer->call_id = call_id; /* Allow a fair bit of grace to allow the server to tell us of a timeout * This is only a fallback */ async_timer->timeout = (timeout + 60) * 1000; if (async_timer->ref) { g_source_remove(async_timer->ref); } async_timer->ref = g_timeout_add(async_timer->timeout, stonith_async_timeout_handler, async_timer); } static void update_callback_timeout(int call_id, int timeout, stonith_t * st) { stonith_callback_client_t *callback = NULL; stonith_private_t *private = st->private; callback = g_hash_table_lookup(private->stonith_op_callback_table, GINT_TO_POINTER(call_id)); if (!callback || !callback->allow_timeout_updates) { return; } set_callback_timeout(callback, st, call_id, timeout); } static void invoke_callback(stonith_t * st, int call_id, int rc, void *userdata, void (*callback) (stonith_t * st, stonith_callback_data_t * data)) { stonith_callback_data_t data = { 0, }; data.call_id = call_id; data.rc = rc; data.userdata = userdata; callback(st, &data); } static int stonith_api_add_callback(stonith_t * stonith, int call_id, int timeout, int options, void *user_data, const char *callback_name, void (*callback) (stonith_t * st, stonith_callback_data_t * data)) { stonith_callback_client_t *blob = NULL; stonith_private_t *private = NULL; CRM_CHECK(stonith != NULL, return -EINVAL); CRM_CHECK(stonith->private != NULL, return -EINVAL); private = stonith->private; if (call_id == 0) { private->op_callback = callback; } else if (call_id < 0) { if (!(options & st_opt_report_only_success)) { crm_trace("Call failed, calling %s: %s", callback_name, pcmk_strerror(call_id)); invoke_callback(stonith, call_id, call_id, user_data, callback); } else { crm_warn("STONITH call failed: %s", pcmk_strerror(call_id)); } return FALSE; } blob = calloc(1, sizeof(stonith_callback_client_t)); blob->id = callback_name; blob->only_success = (options & st_opt_report_only_success) ? TRUE : FALSE; blob->user_data = user_data; blob->callback = callback; blob->allow_timeout_updates = (options & st_opt_timeout_updates) ? TRUE : FALSE; if (timeout > 0) { set_callback_timeout(blob, stonith, call_id, timeout); } g_hash_table_insert(private->stonith_op_callback_table, GINT_TO_POINTER(call_id), blob); crm_trace("Added callback to %s for call %d", callback_name, call_id); return TRUE; } static int stonith_api_del_callback(stonith_t * stonith, int call_id, bool all_callbacks) { stonith_private_t *private = stonith->private; if (all_callbacks) { private->op_callback = NULL; g_hash_table_destroy(private->stonith_op_callback_table); private->stonith_op_callback_table = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, stonith_destroy_op_callback); } else if (call_id == 0) { private->op_callback = NULL; } else { g_hash_table_remove(private->stonith_op_callback_table, GINT_TO_POINTER(call_id)); } return pcmk_ok; } static void stonith_dump_pending_op(gpointer key, gpointer value, gpointer user_data) { int call = GPOINTER_TO_INT(key); stonith_callback_client_t *blob = value; crm_debug("Call %d (%s): pending", call, crm_str(blob->id)); } void stonith_dump_pending_callbacks(stonith_t * stonith) { stonith_private_t *private = stonith->private; if (private->stonith_op_callback_table == NULL) { return; } return g_hash_table_foreach(private->stonith_op_callback_table, stonith_dump_pending_op, NULL); } void stonith_perform_callback(stonith_t * stonith, xmlNode * msg, int call_id, int rc) { stonith_private_t *private = NULL; stonith_callback_client_t *blob = NULL; stonith_callback_client_t local_blob; CRM_CHECK(stonith != NULL, return); CRM_CHECK(stonith->private != NULL, return); private = stonith->private; local_blob.id = NULL; local_blob.callback = NULL; local_blob.user_data = NULL; local_blob.only_success = FALSE; if (msg != NULL) { crm_element_value_int(msg, F_STONITH_RC, &rc); crm_element_value_int(msg, F_STONITH_CALLID, &call_id); } CRM_CHECK(call_id > 0, crm_log_xml_err(msg, "Bad result")); blob = g_hash_table_lookup(private->stonith_op_callback_table, GINT_TO_POINTER(call_id)); if (blob != NULL) { local_blob = *blob; blob = NULL; stonith_api_del_callback(stonith, call_id, FALSE); } else { crm_trace("No callback found for call %d", call_id); local_blob.callback = NULL; } if (local_blob.callback != NULL && (rc == pcmk_ok || local_blob.only_success == FALSE)) { crm_trace("Invoking callback %s for call %d", crm_str(local_blob.id), call_id); invoke_callback(stonith, call_id, rc, local_blob.user_data, local_blob.callback); } else if (private->op_callback == NULL && rc != pcmk_ok) { crm_warn("STONITH command failed: %s", pcmk_strerror(rc)); crm_log_xml_debug(msg, "Failed STONITH Update"); } if (private->op_callback != NULL) { crm_trace("Invoking global callback for call %d", call_id); invoke_callback(stonith, call_id, rc, NULL, private->op_callback); } crm_trace("OP callback activated."); } /* */ static stonith_event_t * xml_to_event(xmlNode * msg) { stonith_event_t *event = calloc(1, sizeof(stonith_event_t)); const char *ntype = crm_element_value(msg, F_SUBTYPE); char *data_addr = crm_strdup_printf("//%s", ntype); xmlNode *data = get_xpath_object(data_addr, msg, LOG_DEBUG); crm_log_xml_trace(msg, "stonith_notify"); crm_element_value_int(msg, F_STONITH_RC, &(event->result)); if (safe_str_eq(ntype, T_STONITH_NOTIFY_FENCE)) { event->operation = crm_element_value_copy(msg, F_STONITH_OPERATION); if (data) { event->origin = crm_element_value_copy(data, F_STONITH_ORIGIN); event->action = crm_element_value_copy(data, F_STONITH_ACTION); event->target = crm_element_value_copy(data, F_STONITH_TARGET); event->executioner = crm_element_value_copy(data, F_STONITH_DELEGATE); event->id = crm_element_value_copy(data, F_STONITH_REMOTE_OP_ID); event->client_origin = crm_element_value_copy(data, F_STONITH_CLIENTNAME); event->device = crm_element_value_copy(data, F_STONITH_DEVICE); } else { crm_err("No data for %s event", ntype); crm_log_xml_notice(msg, "BadEvent"); } } free(data_addr); return event; } static void event_free(stonith_event_t * event) { free(event->id); free(event->type); free(event->message); free(event->operation); free(event->origin); free(event->action); free(event->target); free(event->executioner); free(event->device); free(event->client_origin); free(event); } static void stonith_send_notification(gpointer data, gpointer user_data) { struct notify_blob_s *blob = user_data; stonith_notify_client_t *entry = data; stonith_event_t *st_event = NULL; const char *event = NULL; if (blob->xml == NULL) { crm_warn("Skipping callback - NULL message"); return; } event = crm_element_value(blob->xml, F_SUBTYPE); if (entry == NULL) { crm_warn("Skipping callback - NULL callback client"); return; } else if (entry->notify == NULL) { crm_warn("Skipping callback - NULL callback"); return; } else if (safe_str_neq(entry->event, event)) { crm_trace("Skipping callback - event mismatch %p/%s vs. %s", entry, entry->event, event); return; } st_event = xml_to_event(blob->xml); crm_trace("Invoking callback for %p/%s event...", entry, event); entry->notify(blob->stonith, st_event); crm_trace("Callback invoked..."); event_free(st_event); } int stonith_send_command(stonith_t * stonith, const char *op, xmlNode * data, xmlNode ** output_data, int call_options, int timeout) { int rc = 0; int reply_id = -1; enum crm_ipc_flags ipc_flags = crm_ipc_flags_none; xmlNode *op_msg = NULL; xmlNode *op_reply = NULL; stonith_private_t *native = stonith->private; if (stonith->state == stonith_disconnected) { return -ENOTCONN; } if (output_data != NULL) { *output_data = NULL; } if (op == NULL) { crm_err("No operation specified"); return -EINVAL; } if (call_options & st_opt_sync_call) { ipc_flags |= crm_ipc_client_response; } stonith->call_id++; /* prevent call_id from being negative (or zero) and conflicting * with the stonith_errors enum * use 2 because we use it as (stonith->call_id - 1) below */ if (stonith->call_id < 1) { stonith->call_id = 1; } CRM_CHECK(native->token != NULL,; ); op_msg = stonith_create_op(stonith->call_id, native->token, op, data, call_options); if (op_msg == NULL) { return -EINVAL; } crm_xml_add_int(op_msg, F_STONITH_TIMEOUT, timeout); crm_trace("Sending %s message to STONITH service, Timeout: %ds", op, timeout); rc = crm_ipc_send(native->ipc, op_msg, ipc_flags, 1000 * (timeout + 60), &op_reply); free_xml(op_msg); if (rc < 0) { crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%ds): %d", op, timeout, rc); rc = -ECOMM; goto done; } crm_log_xml_trace(op_reply, "Reply"); if (!(call_options & st_opt_sync_call)) { crm_trace("Async call %d, returning", stonith->call_id); CRM_CHECK(stonith->call_id != 0, return -EPROTO); free_xml(op_reply); return stonith->call_id; } rc = pcmk_ok; crm_element_value_int(op_reply, F_STONITH_CALLID, &reply_id); if (reply_id == stonith->call_id) { crm_trace("Synchronous reply %d received", reply_id); if (crm_element_value_int(op_reply, F_STONITH_RC, &rc) != 0) { rc = -ENOMSG; } if ((call_options & st_opt_discard_reply) || output_data == NULL) { crm_trace("Discarding reply"); } else { *output_data = op_reply; op_reply = NULL; /* Prevent subsequent free */ } } else if (reply_id <= 0) { crm_err("Received bad reply: No id set"); crm_log_xml_err(op_reply, "Bad reply"); free_xml(op_reply); rc = -ENOMSG; } else { crm_err("Received bad reply: %d (wanted %d)", reply_id, stonith->call_id); crm_log_xml_err(op_reply, "Old reply"); free_xml(op_reply); rc = -ENOMSG; } done: if (crm_ipc_connected(native->ipc) == FALSE) { crm_err("STONITH disconnected"); stonith->state = stonith_disconnected; } free_xml(op_reply); return rc; } /* Not used with mainloop */ bool stonith_dispatch(stonith_t * st) { gboolean stay_connected = TRUE; stonith_private_t *private = NULL; CRM_ASSERT(st != NULL); private = st->private; while (crm_ipc_ready(private->ipc)) { if (crm_ipc_read(private->ipc) > 0) { const char *msg = crm_ipc_buffer(private->ipc); stonith_dispatch_internal(msg, strlen(msg), st); } if (crm_ipc_connected(private->ipc) == FALSE) { crm_err("Connection closed"); stay_connected = FALSE; } } return stay_connected; } int stonith_dispatch_internal(const char *buffer, ssize_t length, gpointer userdata) { const char *type = NULL; struct notify_blob_s blob; stonith_t *st = userdata; stonith_private_t *private = NULL; CRM_ASSERT(st != NULL); private = st->private; blob.stonith = st; blob.xml = string2xml(buffer); if (blob.xml == NULL) { crm_warn("Received a NULL msg from STONITH service: %s.", buffer); return 0; } /* do callbacks */ type = crm_element_value(blob.xml, F_TYPE); crm_trace("Activating %s callbacks...", type); if (safe_str_eq(type, T_STONITH_NG)) { stonith_perform_callback(st, blob.xml, 0, 0); } else if (safe_str_eq(type, T_STONITH_NOTIFY)) { g_list_foreach(private->notify_list, stonith_send_notification, &blob); } else if (safe_str_eq(type, T_STONITH_TIMEOUT_VALUE)) { int call_id = 0; int timeout = 0; crm_element_value_int(blob.xml, F_STONITH_TIMEOUT, &timeout); crm_element_value_int(blob.xml, F_STONITH_CALLID, &call_id); update_callback_timeout(call_id, timeout, st); } else { crm_err("Unknown message type: %s", type); crm_log_xml_warn(blob.xml, "BadReply"); } free_xml(blob.xml); return 1; } static int stonith_api_free(stonith_t * stonith) { int rc = pcmk_ok; crm_trace("Destroying %p", stonith); if (stonith->state != stonith_disconnected) { crm_trace("Disconnecting %p first", stonith); rc = stonith->cmds->disconnect(stonith); } if (stonith->state == stonith_disconnected) { stonith_private_t *private = stonith->private; crm_trace("Removing %d callbacks", g_hash_table_size(private->stonith_op_callback_table)); g_hash_table_destroy(private->stonith_op_callback_table); crm_trace("Destroying %d notification clients", g_list_length(private->notify_list)); g_list_free_full(private->notify_list, free); free(stonith->private); free(stonith->cmds); free(stonith); } else { crm_err("Not free'ing active connection: %s (%d)", pcmk_strerror(rc), rc); } return rc; } void stonith_api_delete(stonith_t * stonith) { crm_trace("Destroying %p", stonith); if(stonith) { stonith->cmds->free(stonith); } } stonith_t * stonith_api_new(void) { stonith_t *new_stonith = NULL; stonith_private_t *private = NULL; new_stonith = calloc(1, sizeof(stonith_t)); private = calloc(1, sizeof(stonith_private_t)); new_stonith->private = private; private->stonith_op_callback_table = g_hash_table_new_full(g_direct_hash, g_direct_equal, NULL, stonith_destroy_op_callback); private->notify_list = NULL; new_stonith->call_id = 1; new_stonith->state = stonith_disconnected; new_stonith->cmds = calloc(1, sizeof(stonith_api_operations_t)); /* *INDENT-OFF* */ new_stonith->cmds->free = stonith_api_free; new_stonith->cmds->connect = stonith_api_signon; new_stonith->cmds->disconnect = stonith_api_signoff; new_stonith->cmds->list = stonith_api_list; new_stonith->cmds->monitor = stonith_api_monitor; new_stonith->cmds->status = stonith_api_status; new_stonith->cmds->fence = stonith_api_fence; new_stonith->cmds->confirm = stonith_api_confirm; new_stonith->cmds->history = stonith_api_history; new_stonith->cmds->list_agents = stonith_api_device_list; new_stonith->cmds->metadata = stonith_api_device_metadata; new_stonith->cmds->query = stonith_api_query; new_stonith->cmds->remove_device = stonith_api_remove_device; new_stonith->cmds->register_device = stonith_api_register_device; new_stonith->cmds->remove_level = stonith_api_remove_level; new_stonith->cmds->remove_level_full = stonith_api_remove_level_full; new_stonith->cmds->register_level = stonith_api_register_level; new_stonith->cmds->register_level_full = stonith_api_register_level_full; new_stonith->cmds->remove_callback = stonith_api_del_callback; new_stonith->cmds->register_callback = stonith_api_add_callback; new_stonith->cmds->remove_notification = stonith_api_del_notification; new_stonith->cmds->register_notification = stonith_api_add_notification; /* *INDENT-ON* */ return new_stonith; } stonith_key_value_t * stonith_key_value_add(stonith_key_value_t * head, const char *key, const char *value) { stonith_key_value_t *p, *end; p = calloc(1, sizeof(stonith_key_value_t)); if (key) { p->key = strdup(key); } if (value) { p->value = strdup(value); } end = head; while (end && end->next) { end = end->next; } if (end) { end->next = p; } else { head = p; } return head; } void stonith_key_value_freeall(stonith_key_value_t * head, int keys, int values) { stonith_key_value_t *p; while (head) { p = head->next; if (keys) { free(head->key); } if (values) { free(head->value); } free(head); head = p; } } #define api_log_open() openlog("stonith-api", LOG_CONS | LOG_NDELAY | LOG_PID, LOG_DAEMON) #define api_log(level, fmt, args...) syslog(level, "%s: "fmt, __FUNCTION__, args) int stonith_api_kick(uint32_t nodeid, const char *uname, int timeout, bool off) { char *name = NULL; const char *action = "reboot"; int rc = -EPROTO; stonith_t *st = NULL; enum stonith_call_options opts = st_opt_sync_call | st_opt_allow_suicide; api_log_open(); st = stonith_api_new(); if (st) { rc = st->cmds->connect(st, "stonith-api", NULL); if(rc != pcmk_ok) { api_log(LOG_ERR, "Connection failed, could not kick (%s) node %u/%s : %s (%d)", action, nodeid, uname, pcmk_strerror(rc), rc); } } if (uname != NULL) { name = strdup(uname); } else if (nodeid > 0) { opts |= st_opt_cs_nodeid; name = crm_itoa(nodeid); } if (off) { action = "off"; } if (rc == pcmk_ok) { rc = st->cmds->fence(st, opts, name, action, timeout, 0); if(rc != pcmk_ok) { api_log(LOG_ERR, "Could not kick (%s) node %u/%s : %s (%d)", action, nodeid, uname, pcmk_strerror(rc), rc); } else { api_log(LOG_NOTICE, "Node %u/%s kicked: %s ", nodeid, uname, action); } } if (st) { st->cmds->disconnect(st); stonith_api_delete(st); } free(name); return rc; } time_t stonith_api_time(uint32_t nodeid, const char *uname, bool in_progress) { int rc = 0; char *name = NULL; time_t when = 0; stonith_t *st = NULL; stonith_history_t *history, *hp = NULL; enum stonith_call_options opts = st_opt_sync_call; st = stonith_api_new(); if (st) { rc = st->cmds->connect(st, "stonith-api", NULL); if(rc != pcmk_ok) { api_log(LOG_NOTICE, "Connection failed: %s (%d)", pcmk_strerror(rc), rc); } } if (uname != NULL) { name = strdup(uname); } else if (nodeid > 0) { opts |= st_opt_cs_nodeid; name = crm_itoa(nodeid); } if (st && rc == pcmk_ok) { int entries = 0; int progress = 0; int completed = 0; rc = st->cmds->history(st, opts, name, &history, 120); for (hp = history; hp; hp = hp->next) { entries++; if (in_progress) { progress++; if (hp->state != st_done && hp->state != st_failed) { when = time(NULL); } } else if (hp->state == st_done) { completed++; if (hp->completed > when) { when = hp->completed; } } } if(rc == pcmk_ok) { api_log(LOG_INFO, "Found %d entries for %u/%s: %d in progress, %d completed", entries, nodeid, uname, progress, completed); } else { api_log(LOG_ERR, "Could not retrieve fence history for %u/%s: %s (%d)", nodeid, uname, pcmk_strerror(rc), rc); } } if (st) { st->cmds->disconnect(st); stonith_api_delete(st); } if(when) { api_log(LOG_INFO, "Node %u/%s last kicked at: %ld", nodeid, uname, (long int)when); } free(name); return when; } #if HAVE_STONITH_STONITH_H # include const char *i_hate_pils(int rc); const char * i_hate_pils(int rc) { return PIL_strerror(rc); } #endif diff --git a/lib/pengine/Makefile.am b/lib/pengine/Makefile.am index ff4d1e3bd5..ad2c7ae60f 100644 --- a/lib/pengine/Makefile.am +++ b/lib/pengine/Makefile.am @@ -1,44 +1,44 @@ # # Copyright (C) 2004 Andrew Beekhof # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # include $(top_srcdir)/Makefile.common ## libraries lib_LTLIBRARIES = libpe_rules.la libpe_status.la ## SOURCES noinst_HEADERS = unpack.h variant.h libpe_rules_la_LDFLAGS = -version-info 4:0:2 libpe_rules_la_CFLAGS = $(CFLAGS_HARDENED_LIB) libpe_rules_la_LDFLAGS += $(LDFLAGS_HARDENED_LIB) libpe_rules_la_LIBADD = $(top_builddir)/lib/common/libcrmcommon.la -libpe_rules_la_SOURCES = rules.c common.c +libpe_rules_la_SOURCES = rules.c rules_alerts.c common.c libpe_status_la_LDFLAGS = -version-info 13:0:3 libpe_status_la_CFLAGS = $(CFLAGS_HARDENED_LIB) libpe_status_la_LDFLAGS += $(LDFLAGS_HARDENED_LIB) libpe_status_la_LIBADD = @CURSESLIBS@ $(top_builddir)/lib/common/libcrmcommon.la libpe_status_la_SOURCES = status.c unpack.c utils.c complex.c native.c container.c \ group.c clone.c rules.c common.c failcounts.c remote.c clean-generic: rm -f *.log *.debug *~ diff --git a/lib/pengine/common.c b/lib/pengine/common.c index cd27561c78..446b7f0dd1 100644 --- a/lib/pengine/common.c +++ b/lib/pengine/common.c @@ -1,433 +1,433 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include gboolean was_processing_error = FALSE; gboolean was_processing_warning = FALSE; static gboolean check_health(const char *value) { if (safe_str_eq(value, "none")) { return TRUE; } else if (safe_str_eq(value, "custom")) { return TRUE; } else if (safe_str_eq(value, "only-green")) { return TRUE; } else if (safe_str_eq(value, "progressive")) { return TRUE; } else if (safe_str_eq(value, "migrate-on-red")) { return TRUE; } return FALSE; } static gboolean check_stonith_action(const char *value) { if (safe_str_eq(value, "reboot")) { return TRUE; } else if (safe_str_eq(value, "poweroff")) { return TRUE; } else if (safe_str_eq(value, "off")) { return TRUE; } return FALSE; } static gboolean check_placement_strategy(const char *value) { if (safe_str_eq(value, "default")) { return TRUE; } else if (safe_str_eq(value, "utilization")) { return TRUE; } else if (safe_str_eq(value, "minimal")) { return TRUE; } else if (safe_str_eq(value, "balanced")) { return TRUE; } return FALSE; } /* *INDENT-OFF* */ pe_cluster_option pe_opts[] = { /* name, old-name, validate, default, description */ { "no-quorum-policy", "no_quorum_policy", "enum", "stop, freeze, ignore, suicide", "stop", &check_quorum, "What to do when the cluster does not have quorum", NULL }, { "symmetric-cluster", "symmetric_cluster", "boolean", NULL, "true", &check_boolean, "All resources can run anywhere by default", NULL }, { "default-resource-stickiness", "default_resource_stickiness", "integer", NULL, "0", &check_number, "", NULL }, { "is-managed-default", "is_managed_default", "boolean", NULL, "true", &check_boolean, "Should the cluster start/stop resources as required", NULL }, { "maintenance-mode", NULL, "boolean", NULL, "false", &check_boolean, "Should the cluster monitor resources and start/stop them as required", NULL }, { "start-failure-is-fatal", NULL, "boolean", NULL, "true", &check_boolean, "Always treat start failures as fatal", - "This was the old default. However when set to FALSE, the cluster will instead use the resource's failcount and value for resource-failure-stickiness" }, + "When set to TRUE, the cluster will immediately ban a resource from a node if it fails to start there. When FALSE, the cluster will instead check the resource's fail count against its migration-threshold." }, { "enable-startup-probes", NULL, "boolean", NULL, "true", &check_boolean, "Should the cluster check for active resources during startup", NULL }, /* Stonith Options */ { "stonith-enabled", "stonith_enabled", "boolean", NULL, "true", &check_boolean, "Failed nodes are STONITH'd", NULL }, { "stonith-action", "stonith_action", "enum", "reboot, poweroff, off", "reboot", &check_stonith_action, "Action to send to STONITH device", NULL }, { "stonith-timeout", NULL, "time", NULL, "60s", &check_timer, "How long to wait for the STONITH action (reboot,on,off) to complete", NULL }, { XML_ATTR_HAVE_WATCHDOG, NULL, "boolean", NULL, "false", &check_boolean, "Enable watchdog integration", "Set automatically by the cluster if SBD is detected. User configured values are ignored." }, { "concurrent-fencing", NULL, "boolean", NULL, "false", &check_boolean, "Allow performing fencing operations in parallel", NULL }, { "startup-fencing", "startup_fencing", "boolean", NULL, "true", &check_boolean, "STONITH unseen nodes", "Advanced Use Only! Not using the default is very unsafe!" }, /* Timeouts etc */ { "cluster-delay", "transition_idle_timeout", "time", NULL, "60s", &check_time, "Round trip delay over the network (excluding action execution)", "The \"correct\" value will depend on the speed and load of your network and cluster nodes." }, { "batch-limit", NULL, "integer", NULL, "0", &check_number, "The number of jobs that the TE is allowed to execute in parallel", "The \"correct\" value will depend on the speed and load of your network and cluster nodes." }, { "migration-limit", NULL, "integer", NULL, "-1", &check_number, "The number of migration jobs that the TE is allowed to execute in parallel on a node"}, { "default-action-timeout", "default_action_timeout", "time", NULL, "20s", &check_time, "How long to wait for actions to complete", NULL }, /* Orphans and stopping */ { "stop-all-resources", NULL, "boolean", NULL, "false", &check_boolean, "Should the cluster stop all active resources (except those needed for fencing)", NULL }, { "stop-orphan-resources", "stop_orphan_resources", "boolean", NULL, "true", &check_boolean, "Should deleted resources be stopped", NULL }, { "stop-orphan-actions", "stop_orphan_actions", "boolean", NULL, "true", &check_boolean, "Should deleted actions be cancelled", NULL }, { "remove-after-stop", "remove_after_stop", "boolean", NULL, "false", &check_boolean, "Remove resources from the LRM after they are stopped", "Always set this to false. Other values are, at best, poorly tested and potentially dangerous." }, /* { "", "", , "0", "", NULL }, */ /* Storing inputs */ { "pe-error-series-max", NULL, "integer", NULL, "-1", &check_number, "The number of PE inputs resulting in ERRORs to save", "Zero to disable, -1 to store unlimited." }, { "pe-warn-series-max", NULL, "integer", NULL, "5000", &check_number, "The number of PE inputs resulting in WARNINGs to save", "Zero to disable, -1 to store unlimited." }, { "pe-input-series-max", NULL, "integer", NULL, "4000", &check_number, "The number of other PE inputs to save", "Zero to disable, -1 to store unlimited." }, /* Node health */ { "node-health-strategy", NULL, "enum", "none, migrate-on-red, only-green, progressive, custom", "none", &check_health, "The strategy combining node attributes to determine overall node health.", "Requires external entities to create node attributes (named with the prefix '#health') with values: 'red', 'yellow' or 'green'."}, { "node-health-base", NULL, "integer", NULL, "0", &check_number, "The base score assigned to a node", "Only used when node-health-strategy is set to progressive." }, { "node-health-green", NULL, "integer", NULL, "0", &check_number, "The score 'green' translates to in rsc_location constraints", "Only used when node-health-strategy is set to custom or progressive." }, { "node-health-yellow", NULL, "integer", NULL, "0", &check_number, "The score 'yellow' translates to in rsc_location constraints", "Only used when node-health-strategy is set to custom or progressive." }, { "node-health-red", NULL, "integer", NULL, "-INFINITY", &check_number, "The score 'red' translates to in rsc_location constraints", "Only used when node-health-strategy is set to custom or progressive." }, /*Placement Strategy*/ { "placement-strategy", NULL, "enum", "default, utilization, minimal, balanced", "default", &check_placement_strategy, "The strategy to determine resource placement", NULL}, }; /* *INDENT-ON* */ void pe_metadata(void) { config_metadata("Policy Engine", "1.0", "Policy Engine Options", "This is a fake resource that details the options that can be configured for the Policy Engine.", pe_opts, DIMOF(pe_opts)); } void verify_pe_options(GHashTable * options) { verify_all_options(options, pe_opts, DIMOF(pe_opts)); } const char * pe_pref(GHashTable * options, const char *name) { return get_cluster_pref(options, pe_opts, DIMOF(pe_opts), name); } const char * fail2text(enum action_fail_response fail) { const char *result = ""; switch (fail) { case action_fail_ignore: result = "ignore"; break; case action_fail_block: result = "block"; break; case action_fail_recover: result = "recover"; break; case action_fail_migrate: result = "migrate"; break; case action_fail_stop: result = "stop"; break; case action_fail_fence: result = "fence"; break; case action_fail_standby: result = "standby"; break; case action_fail_restart_container: result = "restart-container"; break; case action_fail_reset_remote: result = "reset-remote"; break; } return result; } enum action_tasks text2task(const char *task) { if (safe_str_eq(task, CRMD_ACTION_STOP)) { return stop_rsc; } else if (safe_str_eq(task, CRMD_ACTION_STOPPED)) { return stopped_rsc; } else if (safe_str_eq(task, CRMD_ACTION_START)) { return start_rsc; } else if (safe_str_eq(task, CRMD_ACTION_STARTED)) { return started_rsc; } else if (safe_str_eq(task, CRM_OP_SHUTDOWN)) { return shutdown_crm; } else if (safe_str_eq(task, CRM_OP_FENCE)) { return stonith_node; } else if (safe_str_eq(task, CRMD_ACTION_STATUS)) { return monitor_rsc; } else if (safe_str_eq(task, CRMD_ACTION_NOTIFY)) { return action_notify; } else if (safe_str_eq(task, CRMD_ACTION_NOTIFIED)) { return action_notified; } else if (safe_str_eq(task, CRMD_ACTION_PROMOTE)) { return action_promote; } else if (safe_str_eq(task, CRMD_ACTION_DEMOTE)) { return action_demote; } else if (safe_str_eq(task, CRMD_ACTION_PROMOTED)) { return action_promoted; } else if (safe_str_eq(task, CRMD_ACTION_DEMOTED)) { return action_demoted; } #if SUPPORT_TRACING if (safe_str_eq(task, CRMD_ACTION_CANCEL)) { return no_action; } else if (safe_str_eq(task, CRMD_ACTION_DELETE)) { return no_action; } else if (safe_str_eq(task, CRMD_ACTION_STATUS)) { return no_action; } else if (safe_str_eq(task, CRM_OP_PROBED)) { return no_action; } else if (safe_str_eq(task, CRM_OP_LRM_REFRESH)) { return no_action; } else if (safe_str_eq(task, CRMD_ACTION_MIGRATE)) { return no_action; } else if (safe_str_eq(task, CRMD_ACTION_MIGRATED)) { return no_action; } else if (safe_str_eq(task, "fail")) { return no_action; } else if (safe_str_eq(task, "stonith_up")) { return no_action; } else if (safe_str_eq(task, "stonith_complete")) { return no_action; } else if (safe_str_eq(task, "all_stopped")) { return no_action; } crm_trace("Unsupported action: %s", task); #endif return no_action; } const char * task2text(enum action_tasks task) { const char *result = ""; switch (task) { case no_action: result = "no_action"; break; case stop_rsc: result = CRMD_ACTION_STOP; break; case stopped_rsc: result = CRMD_ACTION_STOPPED; break; case start_rsc: result = CRMD_ACTION_START; break; case started_rsc: result = CRMD_ACTION_STARTED; break; case shutdown_crm: result = CRM_OP_SHUTDOWN; break; case stonith_node: result = CRM_OP_FENCE; break; case monitor_rsc: result = CRMD_ACTION_STATUS; break; case action_notify: result = CRMD_ACTION_NOTIFY; break; case action_notified: result = CRMD_ACTION_NOTIFIED; break; case action_promote: result = CRMD_ACTION_PROMOTE; break; case action_promoted: result = CRMD_ACTION_PROMOTED; break; case action_demote: result = CRMD_ACTION_DEMOTE; break; case action_demoted: result = CRMD_ACTION_DEMOTED; break; } return result; } const char * role2text(enum rsc_role_e role) { switch (role) { case RSC_ROLE_UNKNOWN: return RSC_ROLE_UNKNOWN_S; case RSC_ROLE_STOPPED: return RSC_ROLE_STOPPED_S; case RSC_ROLE_STARTED: return RSC_ROLE_STARTED_S; case RSC_ROLE_SLAVE: return RSC_ROLE_SLAVE_S; case RSC_ROLE_MASTER: return RSC_ROLE_MASTER_S; } CRM_CHECK(role >= RSC_ROLE_UNKNOWN, return RSC_ROLE_UNKNOWN_S); CRM_CHECK(role < RSC_ROLE_MAX, return RSC_ROLE_UNKNOWN_S); return RSC_ROLE_UNKNOWN_S; } enum rsc_role_e text2role(const char *role) { CRM_ASSERT(role != NULL); if (safe_str_eq(role, RSC_ROLE_STOPPED_S)) { return RSC_ROLE_STOPPED; } else if (safe_str_eq(role, RSC_ROLE_STARTED_S)) { return RSC_ROLE_STARTED; } else if (safe_str_eq(role, RSC_ROLE_SLAVE_S)) { return RSC_ROLE_SLAVE; } else if (safe_str_eq(role, RSC_ROLE_MASTER_S)) { return RSC_ROLE_MASTER; } else if (safe_str_eq(role, RSC_ROLE_UNKNOWN_S)) { return RSC_ROLE_UNKNOWN; } crm_err("Unknown role: %s", role); return RSC_ROLE_UNKNOWN; } int merge_weights(int w1, int w2) { int result = w1 + w2; if (w1 <= -INFINITY || w2 <= -INFINITY) { if (w1 >= INFINITY || w2 >= INFINITY) { crm_trace("-INFINITY + INFINITY == -INFINITY"); } return -INFINITY; } else if (w1 >= INFINITY || w2 >= INFINITY) { return INFINITY; } /* detect wrap-around */ if (result > 0) { if (w1 <= 0 && w2 < 0) { result = -INFINITY; } } else if (w1 > 0 && w2 > 0) { result = INFINITY; } /* detect +/- INFINITY */ if (result >= INFINITY) { result = INFINITY; } else if (result <= -INFINITY) { result = -INFINITY; } crm_trace("%d + %d = %d", w1, w2, result); return result; } void add_hash_param(GHashTable * hash, const char *name, const char *value) { CRM_CHECK(hash != NULL, return); crm_trace("adding: name=%s value=%s", crm_str(name), crm_str(value)); if (name == NULL || value == NULL) { return; } else if (safe_str_eq(value, "#default")) { return; } else if (g_hash_table_lookup(hash, name) == NULL) { g_hash_table_insert(hash, strdup(name), strdup(value)); } } diff --git a/lib/pengine/rules_alerts.c b/lib/pengine/rules_alerts.c new file mode 100644 index 0000000000..16c9dae082 --- /dev/null +++ b/lib/pengine/rules_alerts.c @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2015-2017 Andrew Beekhof + * + * This source code is licensed under the GNU Lesser General Public License + * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY. + */ + +#include +#include +#include +#include +#include +#include + +#ifdef RHEL7_COMPAT +/* @COMPAT An early implementation of alerts was backported to RHEL 7, + * even though it was never in an upstream release. + */ +static char *notify_script = NULL; +static char *notify_target = NULL; + +void +pe_enable_legacy_alerts(const char *script, const char *target) +{ + free(notify_script); + notify_script = (script && strcmp(script, "/dev/null"))? + strdup(script) : NULL; + + free(notify_target); + notify_target = target? strdup(target): NULL; +} +#endif + +static GHashTable * +get_meta_attrs_from_cib(xmlNode *basenode, crm_alert_entry_t *entry, + guint *max_timeout) +{ + GHashTable *config_hash = g_hash_table_new_full(crm_str_hash, g_str_equal, + g_hash_destroy_str, + g_hash_destroy_str); + crm_time_t *now = crm_time_new(NULL); + const char *value = NULL; + + unpack_instance_attributes(basenode, basenode, XML_TAG_META_SETS, NULL, + config_hash, NULL, FALSE, now); + + value = g_hash_table_lookup(config_hash, XML_ALERT_ATTR_TIMEOUT); + if (value) { + entry->timeout = crm_get_msec(value); + if (entry->timeout <= 0) { + if (entry->timeout == 0) { + crm_trace("Setting timeout to default %dmsec", + CRM_ALERT_DEFAULT_TIMEOUT_MS); + } else { + crm_warn("Invalid timeout value setting to default %dmsec", + CRM_ALERT_DEFAULT_TIMEOUT_MS); + } + entry->timeout = CRM_ALERT_DEFAULT_TIMEOUT_MS; + } else { + crm_trace("Found timeout %dmsec", entry->timeout); + } + if (entry->timeout > *max_timeout) { + *max_timeout = entry->timeout; + } + } + value = g_hash_table_lookup(config_hash, XML_ALERT_ATTR_TSTAMP_FORMAT); + if (value) { + /* hard to do any checks here as merely anything can + * can be a valid time-format-string + */ + entry->tstamp_format = (char *) value; + crm_trace("Found timestamp format string '%s'", value); + } + + value = g_hash_table_lookup(config_hash, XML_ALERT_ATTR_SELECT_KIND); + if (value) { + entry->select_kind_orig = (char *) value; + entry->select_kind = g_strsplit((char *) value, ",", 0); + crm_trace("Found select_kind string '%s'", (char *) value); + } + + value = g_hash_table_lookup(config_hash, + XML_ALERT_ATTR_SELECT_ATTRIBUTE_NAME); + if (value) { + entry->select_attribute_name_orig = (char*) value; + entry->select_attribute_name = g_strsplit((char*) value, ",", 0); + crm_trace("Found attribute_name string '%s'", (char *) value); + } + + crm_time_free(now); + return config_hash; /* keep hash as long as strings are needed */ +} + +void +pe_unpack_alerts(xmlNode *alerts) +{ + xmlNode *alert; + crm_alert_entry_t entry; + guint max_timeout = 0; + + crm_free_alert_list(); + crm_alert_max_alert_timeout = CRM_ALERT_DEFAULT_TIMEOUT_MS; + if (crm_alert_kind_default == NULL) { + crm_alert_kind_default = g_strsplit(CRM_ALERT_KIND_DEFAULT, ",", 0); + } + + if (alerts) { +#ifdef RHEL7_COMPAT + if (notify_script) { + crm_warn("Ignoring deprecated notification configuration because alerts available"); + } +#endif + } else { +#ifdef RHEL7_COMPAT + if (notify_script) { + entry = (crm_alert_entry_t) { + .id = (char *) "legacy_notification", + .path = notify_script, + .timeout = CRM_ALERT_DEFAULT_TIMEOUT_MS, + .recipient = notify_target, + .select_kind_orig = NULL, + .select_kind = NULL, + .select_attribute_name_orig = NULL, + .select_attribute_name = NULL + }; + crm_add_dup_alert_list_entry(&entry); + crm_warn("Deprecated notification syntax in use (alerts syntax is preferable)"); + } +#endif + return; + } + + for (alert = first_named_child(alerts, XML_CIB_TAG_ALERT); + alert; alert = __xml_next(alert)) { + + xmlNode *recipient; + int recipients = 0, envvars = 0; + GHashTable *config_hash = NULL; + + entry = (crm_alert_entry_t) { + .id = (char *) crm_element_value(alert, XML_ATTR_ID), + .path = (char *) crm_element_value(alert, XML_ALERT_ATTR_PATH), + .timeout = CRM_ALERT_DEFAULT_TIMEOUT_MS, + .tstamp_format = (char *) CRM_ALERT_DEFAULT_TSTAMP_FORMAT, + .select_kind_orig = NULL, + .select_kind = NULL, + .select_attribute_name_orig = NULL, + .select_attribute_name = NULL + }; + + crm_get_envvars_from_cib(alert, &entry, &envvars); + config_hash = get_meta_attrs_from_cib(alert, &entry, &max_timeout); + + crm_debug("Found alert %s with path=%s timeout=%d tstamp_format=%s " + "select_kind=%s select_attribute_name=%s " + "%d additional environment variables", + entry.id, entry.path, entry.timeout, entry.tstamp_format, + entry.select_kind_orig, entry.select_attribute_name_orig, + envvars); + + for (recipient = first_named_child(alert, XML_CIB_TAG_ALERT_RECIPIENT); + recipient != NULL; recipient = __xml_next(recipient)) { + + int envvars_added = 0; + + entry.recipient = (char *) crm_element_value(recipient, + XML_ALERT_ATTR_REC_VALUE); + recipients++; + + crm_get_envvars_from_cib(recipient, &entry, &envvars_added); + + { + crm_alert_entry_t recipient_entry = entry; + GHashTable *config_hash = get_meta_attrs_from_cib(recipient, + &recipient_entry, + &max_timeout); + + crm_add_dup_alert_list_entry(&recipient_entry); + crm_debug("Alert has recipient: id=%s, value=%s, " + "%d additional environment variables", + crm_element_value(recipient, XML_ATTR_ID), + recipient_entry.recipient, envvars_added); + g_hash_table_destroy(config_hash); + } + + crm_drop_envvars(&entry, envvars_added); + } + + if (recipients == 0) { + crm_add_dup_alert_list_entry(&entry); + } + + crm_drop_envvars(&entry, -1); + g_hash_table_destroy(config_hash); + } + + if (max_timeout > 0) { + crm_alert_max_alert_timeout = max_timeout; + } +} diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c index 488dc42157..77a456e1e1 100644 --- a/lib/pengine/utils.c +++ b/lib/pengine/utils.c @@ -1,2067 +1,2067 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include #include #include #include pe_working_set_t *pe_dataset = NULL; extern xmlNode *get_object_root(const char *object_type, xmlNode * the_root); void print_str_str(gpointer key, gpointer value, gpointer user_data); gboolean ghash_free_str_str(gpointer key, gpointer value, gpointer user_data); void unpack_operation(action_t * action, xmlNode * xml_obj, resource_t * container, pe_working_set_t * data_set); static xmlNode *find_rsc_op_entry_helper(resource_t * rsc, const char *key, gboolean include_disabled); /*! * \internal * \brief Check whether we can fence a particular node * * \param[in] data_set Working set for cluster * \param[in] node Name of node to check * * \return TRUE if node can be fenced, FALSE otherwise * * \note This function should only be called for cluster nodes and baremetal * remote nodes; guest nodes are fenced by stopping their container * resource, so fence execution requirements do not apply to them. */ bool pe_can_fence(pe_working_set_t * data_set, node_t *node) { if(is_not_set(data_set->flags, pe_flag_stonith_enabled)) { return FALSE; /* Turned off */ } else if (is_not_set(data_set->flags, pe_flag_have_stonith_resource)) { return FALSE; /* No devices */ } else if (is_set(data_set->flags, pe_flag_have_quorum)) { return TRUE; } else if (data_set->no_quorum_policy == no_quorum_ignore) { return TRUE; } else if(node == NULL) { return FALSE; } else if(node->details->online) { crm_notice("We can fence %s without quorum because they're in our membership", node->details->uname); return TRUE; } crm_trace("Cannot fence %s", node->details->uname); return FALSE; } node_t * node_copy(const node_t *this_node) { node_t *new_node = NULL; CRM_CHECK(this_node != NULL, return NULL); new_node = calloc(1, sizeof(node_t)); CRM_ASSERT(new_node != NULL); crm_trace("Copying %p (%s) to %p", this_node, this_node->details->uname, new_node); new_node->rsc_discover_mode = this_node->rsc_discover_mode; new_node->weight = this_node->weight; new_node->fixed = this_node->fixed; new_node->details = this_node->details; return new_node; } /* any node in list1 or list2 and not in the other gets a score of -INFINITY */ void node_list_exclude(GHashTable * hash, GListPtr list, gboolean merge_scores) { GHashTable *result = hash; node_t *other_node = NULL; GListPtr gIter = list; GHashTableIter iter; node_t *node = NULL; g_hash_table_iter_init(&iter, hash); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { other_node = pe_find_node_id(list, node->details->id); if (other_node == NULL) { node->weight = -INFINITY; } else if (merge_scores) { node->weight = merge_weights(node->weight, other_node->weight); } } for (; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; other_node = pe_hash_table_lookup(result, node->details->id); if (other_node == NULL) { node_t *new_node = node_copy(node); new_node->weight = -INFINITY; g_hash_table_insert(result, (gpointer) new_node->details->id, new_node); } } } GHashTable * node_hash_from_list(GListPtr list) { GListPtr gIter = list; GHashTable *result = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, g_hash_destroy_str); for (; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; node_t *n = node_copy(node); g_hash_table_insert(result, (gpointer) n->details->id, n); } return result; } GListPtr node_list_dup(GListPtr list1, gboolean reset, gboolean filter) { GListPtr result = NULL; GListPtr gIter = list1; for (; gIter != NULL; gIter = gIter->next) { node_t *new_node = NULL; node_t *this_node = (node_t *) gIter->data; if (filter && this_node->weight < 0) { continue; } new_node = node_copy(this_node); if (reset) { new_node->weight = 0; } if (new_node != NULL) { result = g_list_prepend(result, new_node); } } return result; } gint sort_node_uname(gconstpointer a, gconstpointer b) { const node_t *node_a = a; const node_t *node_b = b; return strcmp(node_a->details->uname, node_b->details->uname); } void dump_node_scores_worker(int level, const char *file, const char *function, int line, resource_t * rsc, const char *comment, GHashTable * nodes) { GHashTable *hash = nodes; GHashTableIter iter; node_t *node = NULL; if (rsc) { hash = rsc->allowed_nodes; } if (rsc && is_set(rsc->flags, pe_rsc_orphan)) { /* Don't show the allocation scores for orphans */ return; } if (level == 0) { char score[128]; int len = sizeof(score); /* For now we want this in sorted order to keep the regression tests happy */ GListPtr gIter = NULL; GListPtr list = g_hash_table_get_values(hash); list = g_list_sort(list, sort_node_uname); gIter = list; for (; gIter != NULL; gIter = gIter->next) { node_t *node = (node_t *) gIter->data; /* This function is called a whole lot, use stack allocated score */ score2char_stack(node->weight, score, len); if (rsc) { printf("%s: %s allocation score on %s: %s\n", comment, rsc->id, node->details->uname, score); } else { printf("%s: %s = %s\n", comment, node->details->uname, score); } } g_list_free(list); } else if (hash) { char score[128]; int len = sizeof(score); g_hash_table_iter_init(&iter, hash); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { /* This function is called a whole lot, use stack allocated score */ score2char_stack(node->weight, score, len); if (rsc) { do_crm_log_alias(LOG_TRACE, file, function, line, "%s: %s allocation score on %s: %s", comment, rsc->id, node->details->uname, score); } else { do_crm_log_alias(LOG_TRACE, file, function, line + 1, "%s: %s = %s", comment, node->details->uname, score); } } } if (rsc && rsc->children) { GListPtr gIter = NULL; gIter = rsc->children; for (; gIter != NULL; gIter = gIter->next) { resource_t *child = (resource_t *) gIter->data; dump_node_scores_worker(level, file, function, line, child, comment, nodes); } } } static void append_dump_text(gpointer key, gpointer value, gpointer user_data) { char **dump_text = user_data; int len = 0; char *new_text = NULL; len = strlen(*dump_text) + strlen(" ") + strlen(key) + strlen("=") + strlen(value) + 1; new_text = calloc(1, len); sprintf(new_text, "%s %s=%s", *dump_text, (char *)key, (char *)value); free(*dump_text); *dump_text = new_text; } void dump_node_capacity(int level, const char *comment, node_t * node) { int len = 0; char *dump_text = NULL; len = strlen(comment) + strlen(": ") + strlen(node->details->uname) + strlen(" capacity:") + 1; dump_text = calloc(1, len); sprintf(dump_text, "%s: %s capacity:", comment, node->details->uname); g_hash_table_foreach(node->details->utilization, append_dump_text, &dump_text); if (level == 0) { fprintf(stdout, "%s\n", dump_text); } else { crm_trace("%s", dump_text); } free(dump_text); } void dump_rsc_utilization(int level, const char *comment, resource_t * rsc, node_t * node) { int len = 0; char *dump_text = NULL; len = strlen(comment) + strlen(": ") + strlen(rsc->id) + strlen(" utilization on ") + strlen(node->details->uname) + strlen(":") + 1; dump_text = calloc(1, len); sprintf(dump_text, "%s: %s utilization on %s:", comment, rsc->id, node->details->uname); g_hash_table_foreach(rsc->utilization, append_dump_text, &dump_text); if (level == 0) { fprintf(stdout, "%s\n", dump_text); } else { crm_trace("%s", dump_text); } free(dump_text); } gint sort_rsc_index(gconstpointer a, gconstpointer b) { const resource_t *resource1 = (const resource_t *)a; const resource_t *resource2 = (const resource_t *)b; if (a == NULL && b == NULL) { return 0; } if (a == NULL) { return 1; } if (b == NULL) { return -1; } if (resource1->sort_index > resource2->sort_index) { return -1; } if (resource1->sort_index < resource2->sort_index) { return 1; } return 0; } gint sort_rsc_priority(gconstpointer a, gconstpointer b) { const resource_t *resource1 = (const resource_t *)a; const resource_t *resource2 = (const resource_t *)b; if (a == NULL && b == NULL) { return 0; } if (a == NULL) { return 1; } if (b == NULL) { return -1; } if (resource1->priority > resource2->priority) { return -1; } if (resource1->priority < resource2->priority) { return 1; } return 0; } action_t * custom_action(resource_t * rsc, char *key, const char *task, node_t * on_node, gboolean optional, gboolean save_action, pe_working_set_t * data_set) { action_t *action = NULL; GListPtr possible_matches = NULL; CRM_CHECK(key != NULL, return NULL); CRM_CHECK(task != NULL, free(key); return NULL); if (save_action && rsc != NULL) { possible_matches = find_actions(rsc->actions, key, on_node); } else if(save_action) { #if 0 action = g_hash_table_lookup(data_set->singletons, key); #else /* More expensive but takes 'node' into account */ possible_matches = find_actions(data_set->actions, key, on_node); #endif } if(data_set->singletons == NULL) { data_set->singletons = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL); } if (possible_matches != NULL) { if (g_list_length(possible_matches) > 1) { pe_warn("Action %s for %s on %s exists %d times", task, rsc ? rsc->id : "", on_node ? on_node->details->uname : "", g_list_length(possible_matches)); } action = g_list_nth_data(possible_matches, 0); pe_rsc_trace(rsc, "Found existing action (%d) %s for %s on %s", action->id, task, rsc ? rsc->id : "", on_node ? on_node->details->uname : ""); g_list_free(possible_matches); } if (action == NULL) { if (save_action) { pe_rsc_trace(rsc, "Creating%s action %d: %s for %s on %s %d", optional ? "" : " mandatory", data_set->action_id, key, rsc ? rsc->id : "", on_node ? on_node->details->uname : "", optional); } action = calloc(1, sizeof(action_t)); if (save_action) { action->id = data_set->action_id++; } else { action->id = 0; } action->rsc = rsc; CRM_ASSERT(task != NULL); action->task = strdup(task); if (on_node) { action->node = node_copy(on_node); } action->uuid = strdup(key); pe_set_action_bit(action, pe_action_runnable); if (optional) { pe_rsc_trace(rsc, "Set optional on %s", action->uuid); pe_set_action_bit(action, pe_action_optional); } else { pe_clear_action_bit(action, pe_action_optional); pe_rsc_trace(rsc, "Unset optional on %s", action->uuid); } /* Implied by calloc()... action->actions_before = NULL; action->actions_after = NULL; action->pseudo = FALSE; action->dumped = FALSE; action->processed = FALSE; action->seen_count = 0; */ action->extra = g_hash_table_new_full(crm_str_hash, g_str_equal, free, free); action->meta = g_hash_table_new_full(crm_str_hash, g_str_equal, free, free); action->versioned_parameters = create_xml_node(NULL, XML_TAG_OP_VER_ATTRS); action->versioned_meta = create_xml_node(NULL, XML_TAG_OP_VER_META); if (save_action) { data_set->actions = g_list_prepend(data_set->actions, action); if(rsc == NULL) { g_hash_table_insert(data_set->singletons, action->uuid, action); } } if (rsc != NULL) { action->op_entry = find_rsc_op_entry_helper(rsc, key, TRUE); unpack_operation(action, action->op_entry, rsc->container, data_set); if (save_action) { rsc->actions = g_list_prepend(rsc->actions, action); } } if (save_action) { pe_rsc_trace(rsc, "Action %d created", action->id); } } if (optional == FALSE) { pe_rsc_trace(rsc, "Unset optional on %s", action->uuid); pe_clear_action_bit(action, pe_action_optional); } if (rsc != NULL) { enum action_tasks a_task = text2task(action->task); int warn_level = LOG_TRACE; if (save_action) { warn_level = LOG_WARNING; } if (is_set(action->flags, pe_action_have_node_attrs) == FALSE && action->node != NULL && action->op_entry != NULL) { pe_set_action_bit(action, pe_action_have_node_attrs); unpack_instance_attributes(data_set->input, action->op_entry, XML_TAG_ATTR_SETS, action->node->details->attrs, action->extra, NULL, FALSE, data_set->now); } if (is_set(action->flags, pe_action_pseudo)) { /* leave untouched */ } else if (action->node == NULL) { pe_rsc_trace(rsc, "Unset runnable on %s", action->uuid); pe_clear_action_bit(action, pe_action_runnable); } else if (is_not_set(rsc->flags, pe_rsc_managed) && g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL) == NULL) { crm_debug("Action %s (unmanaged)", action->uuid); pe_rsc_trace(rsc, "Set optional on %s", action->uuid); pe_set_action_bit(action, pe_action_optional); /* action->runnable = FALSE; */ } else if (action->node->details->online == FALSE && (!is_container_remote_node(action->node) || action->node->details->remote_requires_reset)) { pe_clear_action_bit(action, pe_action_runnable); do_crm_log(warn_level, "Action %s on %s is unrunnable (offline)", action->uuid, action->node->details->uname); if (is_set(action->rsc->flags, pe_rsc_managed) && save_action && a_task == stop_rsc && action->node->details->unclean == FALSE) { pe_fence_node(data_set, action->node, "resource actions are unrunnable"); } } else if (action->node->details->pending) { pe_clear_action_bit(action, pe_action_runnable); do_crm_log(warn_level, "Action %s on %s is unrunnable (pending)", action->uuid, action->node->details->uname); } else if (action->needs == rsc_req_nothing) { pe_rsc_trace(rsc, "Action %s does not require anything", action->uuid); pe_set_action_bit(action, pe_action_runnable); #if 0 /* * No point checking this * - if we don't have quorum we can't stonith anyway */ } else if (action->needs == rsc_req_stonith) { crm_trace("Action %s requires only stonith", action->uuid); action->runnable = TRUE; #endif } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE && data_set->no_quorum_policy == no_quorum_stop) { pe_clear_action_bit(action, pe_action_runnable); crm_debug("%s\t%s (cancelled : quorum)", action->node->details->uname, action->uuid); } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE && data_set->no_quorum_policy == no_quorum_freeze) { pe_rsc_trace(rsc, "Check resource is already active: %s %s %s %s", rsc->id, action->uuid, role2text(rsc->next_role), role2text(rsc->role)); if (rsc->fns->active(rsc, TRUE) == FALSE || rsc->next_role > rsc->role) { pe_clear_action_bit(action, pe_action_runnable); pe_rsc_debug(rsc, "%s\t%s (cancelled : quorum freeze)", action->node->details->uname, action->uuid); } } else { pe_rsc_trace(rsc, "Action %s is runnable", action->uuid); pe_set_action_bit(action, pe_action_runnable); } if (save_action) { switch (a_task) { case stop_rsc: set_bit(rsc->flags, pe_rsc_stopping); break; case start_rsc: clear_bit(rsc->flags, pe_rsc_starting); if (is_set(action->flags, pe_action_runnable)) { set_bit(rsc->flags, pe_rsc_starting); } break; default: break; } } } free(key); return action; } static const char * unpack_operation_on_fail(action_t * action) { const char *value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ON_FAIL); if (safe_str_eq(action->task, CRMD_ACTION_STOP) && safe_str_eq(value, "standby")) { crm_config_err("on-fail=standby is not allowed for stop actions: %s", action->rsc->id); return NULL; } else if (safe_str_eq(action->task, CRMD_ACTION_DEMOTE) && !value) { /* demote on_fail defaults to master monitor value if present */ xmlNode *operation = NULL; const char *name = NULL; const char *role = NULL; const char *on_fail = NULL; const char *interval = NULL; const char *enabled = NULL; CRM_CHECK(action->rsc != NULL, return NULL); for (operation = __xml_first_child(action->rsc->ops_xml); operation && !value; operation = __xml_next_element(operation)) { if (!crm_str_eq((const char *)operation->name, "op", TRUE)) { continue; } name = crm_element_value(operation, "name"); role = crm_element_value(operation, "role"); on_fail = crm_element_value(operation, XML_OP_ATTR_ON_FAIL); enabled = crm_element_value(operation, "enabled"); interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL); if (!on_fail) { continue; } else if (enabled && !crm_is_true(enabled)) { continue; } else if (safe_str_neq(name, "monitor") || safe_str_neq(role, "Master")) { continue; } else if (crm_get_interval(interval) <= 0) { continue; } value = on_fail; } } return value; } static xmlNode * find_min_interval_mon(resource_t * rsc, gboolean include_disabled) { int number = 0; int min_interval = -1; const char *name = NULL; const char *value = NULL; const char *interval = NULL; xmlNode *op = NULL; xmlNode *operation = NULL; for (operation = __xml_first_child(rsc->ops_xml); operation != NULL; operation = __xml_next_element(operation)) { if (crm_str_eq((const char *)operation->name, "op", TRUE)) { name = crm_element_value(operation, "name"); interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL); value = crm_element_value(operation, "enabled"); if (!include_disabled && value && crm_is_true(value) == FALSE) { continue; } if (safe_str_neq(name, RSC_STATUS)) { continue; } number = crm_get_interval(interval); if (number < 0) { continue; } if (min_interval < 0 || number < min_interval) { min_interval = number; op = operation; } } } return op; } static int unpack_start_delay(const char *value, GHashTable *meta) { int start_delay = 0; if (value != NULL) { start_delay = crm_get_msec(value); if (start_delay < 0) { start_delay = 0; } if (meta) { g_hash_table_replace(meta, strdup(XML_OP_ATTR_START_DELAY), crm_itoa(start_delay)); } } return start_delay; } static int unpack_interval_origin(const char *value, GHashTable *meta, xmlNode *xml_obj, unsigned long long interval, crm_time_t *now) { int start_delay = 0; if (interval > 0 && value) { crm_time_t *origin = crm_time_new(value); if (origin && now) { crm_time_t *delay = NULL; int rc = crm_time_compare(origin, now); long long delay_s = 0; int interval_s = (interval / 1000); crm_trace("Origin: %s, interval: %d", value, interval_s); /* If 'origin' is in the future, find the most recent "multiple" that occurred in the past */ while(rc > 0) { crm_time_add_seconds(origin, -interval_s); rc = crm_time_compare(origin, now); } /* Now find the first "multiple" that occurs after 'now' */ while (rc < 0) { crm_time_add_seconds(origin, interval_s); rc = crm_time_compare(origin, now); } delay = crm_time_calculate_duration(origin, now); crm_time_log(LOG_TRACE, "origin", origin, crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone); crm_time_log(LOG_TRACE, "now", now, crm_time_log_date | crm_time_log_timeofday | crm_time_log_with_timezone); crm_time_log(LOG_TRACE, "delay", delay, crm_time_log_duration); delay_s = crm_time_get_seconds(delay); CRM_CHECK(delay_s >= 0, delay_s = 0); start_delay = delay_s * 1000; if (xml_obj) { crm_info("Calculated a start delay of %llds for %s", delay_s, ID(xml_obj)); } if (meta) { g_hash_table_replace(meta, strdup(XML_OP_ATTR_START_DELAY), crm_itoa(start_delay)); } crm_time_free(origin); crm_time_free(delay); } else if (!origin && xml_obj) { crm_config_err("Operation %s contained an invalid " XML_OP_ATTR_ORIGIN ": %s", ID(xml_obj), value); } } return start_delay; } static int unpack_timeout(const char *value, action_t *action, xmlNode *xml_obj, unsigned long long interval, GHashTable *config_hash) { int timeout = 0; if (value == NULL && xml_obj == NULL && action && safe_str_eq(action->task, RSC_STATUS) && interval == 0) { xmlNode *min_interval_mon = find_min_interval_mon(action->rsc, FALSE); if (min_interval_mon) { value = crm_element_value(min_interval_mon, XML_ATTR_TIMEOUT); pe_rsc_trace(action->rsc, "\t%s uses the timeout value '%s' from the minimum interval monitor", action->uuid, value); } } if (value == NULL && config_hash) { value = pe_pref(config_hash, "default-action-timeout"); } timeout = crm_get_msec(value); if (timeout < 0) { timeout = 0; } return timeout; } static void unpack_versioned_meta(xmlNode *versioned_meta, xmlNode *xml_obj, unsigned long long interval, crm_time_t *now) { xmlNode *attrs = NULL; xmlNode *attr = NULL; for (attrs = __xml_first_child(versioned_meta); attrs != NULL; attrs = __xml_next_element(attrs)) { for (attr = __xml_first_child(attrs); attr != NULL; attr = __xml_next_element(attr)) { const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME); const char *value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE); if (safe_str_eq(name, XML_OP_ATTR_START_DELAY)) { int start_delay = unpack_start_delay(value, NULL); crm_xml_add_int(attr, XML_NVPAIR_ATTR_VALUE, start_delay); } else if (safe_str_eq(name, XML_OP_ATTR_ORIGIN)) { int start_delay = unpack_interval_origin(value, NULL, xml_obj, interval, now); crm_xml_add(attr, XML_NVPAIR_ATTR_NAME, XML_OP_ATTR_START_DELAY); crm_xml_add_int(attr, XML_NVPAIR_ATTR_VALUE, start_delay); } else if (safe_str_eq(name, XML_ATTR_TIMEOUT)) { int timeout = unpack_timeout(value, NULL, NULL, 0, NULL); crm_xml_add_int(attr, XML_NVPAIR_ATTR_VALUE, timeout); } } } } void unpack_operation(action_t * action, xmlNode * xml_obj, resource_t * container, pe_working_set_t * data_set) { unsigned long long interval = 0; int timeout = 0; char *value_ms = NULL; const char *value = NULL; const char *field = NULL; CRM_CHECK(action->rsc != NULL, return); unpack_instance_attributes(data_set->input, data_set->op_defaults, XML_TAG_META_SETS, NULL, action->meta, NULL, FALSE, data_set->now); if (xml_obj) { xmlAttrPtr xIter = NULL; for (xIter = xml_obj->properties; xIter; xIter = xIter->next) { const char *prop_name = (const char *)xIter->name; const char *prop_value = crm_element_value(xml_obj, prop_name); g_hash_table_replace(action->meta, strdup(prop_name), strdup(prop_value)); } } unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_META_SETS, NULL, action->meta, NULL, FALSE, data_set->now); unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_ATTR_SETS, NULL, action->meta, NULL, FALSE, data_set->now); pe_unpack_versioned_attributes(data_set->input, xml_obj, XML_TAG_ATTR_SETS, NULL, action->versioned_parameters, data_set->now); pe_unpack_versioned_attributes(data_set->input, xml_obj, XML_TAG_META_SETS, NULL, action->versioned_meta, data_set->now); g_hash_table_remove(action->meta, "id"); field = XML_LRM_ATTR_INTERVAL; value = g_hash_table_lookup(action->meta, field); if (value != NULL) { interval = crm_get_interval(value); if (interval > 0) { value_ms = crm_itoa(interval); g_hash_table_replace(action->meta, strdup(field), value_ms); } else { g_hash_table_remove(action->meta, field); } } - /* Begin compatibility code ("requires" set on start action not resource) */ + /* @COMPAT data sets < 1.1.10 ("requires" on start action not resource) */ value = g_hash_table_lookup(action->meta, "requires"); if (safe_str_neq(action->task, RSC_START) && safe_str_neq(action->task, RSC_PROMOTE)) { action->needs = rsc_req_nothing; value = "nothing (not start/promote)"; } else if (safe_str_eq(value, "nothing")) { action->needs = rsc_req_nothing; } else if (safe_str_eq(value, "quorum")) { action->needs = rsc_req_quorum; } else if (safe_str_eq(value, "unfencing")) { action->needs = rsc_req_stonith; set_bit(action->rsc->flags, pe_rsc_needs_unfencing); if (is_not_set(data_set->flags, pe_flag_stonith_enabled)) { crm_notice("%s requires unfencing but fencing is disabled", action->rsc->id); } } else if (is_set(data_set->flags, pe_flag_stonith_enabled) && safe_str_eq(value, "fencing")) { action->needs = rsc_req_stonith; if (is_not_set(data_set->flags, pe_flag_stonith_enabled)) { crm_notice("%s requires fencing but fencing is disabled", action->rsc->id); } - /* End compatibility code */ + /* @COMPAT end compatibility code */ } else if (is_set(action->rsc->flags, pe_rsc_needs_fencing)) { action->needs = rsc_req_stonith; value = "fencing (resource)"; } else if (is_set(action->rsc->flags, pe_rsc_needs_quorum)) { action->needs = rsc_req_quorum; value = "quorum (resource)"; } else { action->needs = rsc_req_nothing; value = "nothing (resource)"; } pe_rsc_trace(action->rsc, "\tAction %s requires: %s", action->task, value); value = unpack_operation_on_fail(action); if (value == NULL) { } else if (safe_str_eq(value, "block")) { action->on_fail = action_fail_block; g_hash_table_insert(action->meta, strdup(XML_OP_ATTR_ON_FAIL), strdup("block")); } else if (safe_str_eq(value, "fence")) { action->on_fail = action_fail_fence; value = "node fencing"; if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) { crm_config_err("Specifying on_fail=fence and" " stonith-enabled=false makes no sense"); action->on_fail = action_fail_stop; action->fail_role = RSC_ROLE_STOPPED; value = "stop resource"; } } else if (safe_str_eq(value, "standby")) { action->on_fail = action_fail_standby; value = "node standby"; } else if (safe_str_eq(value, "ignore") || safe_str_eq(value, "nothing")) { action->on_fail = action_fail_ignore; value = "ignore"; } else if (safe_str_eq(value, "migrate")) { action->on_fail = action_fail_migrate; value = "force migration"; } else if (safe_str_eq(value, "stop")) { action->on_fail = action_fail_stop; action->fail_role = RSC_ROLE_STOPPED; value = "stop resource"; } else if (safe_str_eq(value, "restart")) { action->on_fail = action_fail_recover; value = "restart (and possibly migrate)"; } else if (safe_str_eq(value, "restart-container")) { if (container) { action->on_fail = action_fail_restart_container; value = "restart container (and possibly migrate)"; } else { value = NULL; } } else { pe_err("Resource %s: Unknown failure type (%s)", action->rsc->id, value); value = NULL; } /* defaults */ if (value == NULL && container) { action->on_fail = action_fail_restart_container; value = "restart container (and possibly migrate) (default)"; /* for baremetal remote nodes, ensure that any failure that results in * dropping an active connection to a remote node results in fencing of * the remote node. * * There are only two action failures that don't result in fencing. * 1. probes - probe failures are expected. * 2. start - a start failure indicates that an active connection does not already * exist. The user can set op on-fail=fence if they really want to fence start * failures. */ } else if (((value == NULL) || !is_set(action->rsc->flags, pe_rsc_managed)) && (is_rsc_baremetal_remote_node(action->rsc, data_set) && !(safe_str_eq(action->task, CRMD_ACTION_STATUS) && interval == 0) && (safe_str_neq(action->task, CRMD_ACTION_START)))) { if (!is_set(action->rsc->flags, pe_rsc_managed)) { action->on_fail = action_fail_stop; action->fail_role = RSC_ROLE_STOPPED; value = "stop unmanaged baremetal remote node (enforcing default)"; } else { if (is_set(data_set->flags, pe_flag_stonith_enabled)) { value = "fence baremetal remote node (default)"; } else { value = "recover baremetal remote node connection (default)"; } if (action->rsc->remote_reconnect_interval) { action->fail_role = RSC_ROLE_STOPPED; } action->on_fail = action_fail_reset_remote; } } else if (value == NULL && safe_str_eq(action->task, CRMD_ACTION_STOP)) { if (is_set(data_set->flags, pe_flag_stonith_enabled)) { action->on_fail = action_fail_fence; value = "resource fence (default)"; } else { action->on_fail = action_fail_block; value = "resource block (default)"; } } else if (value == NULL) { action->on_fail = action_fail_recover; value = "restart (and possibly migrate) (default)"; } pe_rsc_trace(action->rsc, "\t%s failure handling: %s", action->task, value); value = NULL; if (xml_obj != NULL) { value = g_hash_table_lookup(action->meta, "role_after_failure"); } if (value != NULL && action->fail_role == RSC_ROLE_UNKNOWN) { action->fail_role = text2role(value); } /* defaults */ if (action->fail_role == RSC_ROLE_UNKNOWN) { if (safe_str_eq(action->task, CRMD_ACTION_PROMOTE)) { action->fail_role = RSC_ROLE_SLAVE; } else { action->fail_role = RSC_ROLE_STARTED; } } pe_rsc_trace(action->rsc, "\t%s failure results in: %s", action->task, role2text(action->fail_role)); field = XML_OP_ATTR_START_DELAY; value = g_hash_table_lookup(action->meta, XML_OP_ATTR_START_DELAY); if (value) { unpack_start_delay(value, action->meta); } else { value = g_hash_table_lookup(action->meta, XML_OP_ATTR_ORIGIN); unpack_interval_origin(value, action->meta, xml_obj, interval, data_set->now); } field = XML_ATTR_TIMEOUT; value = g_hash_table_lookup(action->meta, field); timeout = unpack_timeout(value, action, xml_obj, interval, data_set->config_hash); g_hash_table_replace(action->meta, strdup(XML_ATTR_TIMEOUT), crm_itoa(timeout)); unpack_versioned_meta(action->versioned_meta, xml_obj, interval, data_set->now); } static xmlNode * find_rsc_op_entry_helper(resource_t * rsc, const char *key, gboolean include_disabled) { unsigned long long number = 0; gboolean do_retry = TRUE; char *local_key = NULL; const char *name = NULL; const char *value = NULL; const char *interval = NULL; char *match_key = NULL; xmlNode *op = NULL; xmlNode *operation = NULL; retry: for (operation = __xml_first_child(rsc->ops_xml); operation != NULL; operation = __xml_next_element(operation)) { if (crm_str_eq((const char *)operation->name, "op", TRUE)) { name = crm_element_value(operation, "name"); interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL); value = crm_element_value(operation, "enabled"); if (!include_disabled && value && crm_is_true(value) == FALSE) { continue; } number = crm_get_interval(interval); match_key = generate_op_key(rsc->id, name, number); if (safe_str_eq(key, match_key)) { op = operation; } free(match_key); if (rsc->clone_name) { match_key = generate_op_key(rsc->clone_name, name, number); if (safe_str_eq(key, match_key)) { op = operation; } free(match_key); } if (op != NULL) { free(local_key); return op; } } } free(local_key); if (do_retry == FALSE) { return NULL; } do_retry = FALSE; if (strstr(key, CRMD_ACTION_MIGRATE) || strstr(key, CRMD_ACTION_MIGRATED)) { local_key = generate_op_key(rsc->id, "migrate", 0); key = local_key; goto retry; } else if (strstr(key, "_notify_")) { local_key = generate_op_key(rsc->id, "notify", 0); key = local_key; goto retry; } return NULL; } xmlNode * find_rsc_op_entry(resource_t * rsc, const char *key) { return find_rsc_op_entry_helper(rsc, key, FALSE); } void print_node(const char *pre_text, node_t * node, gboolean details) { if (node == NULL) { crm_trace("%s%s: ", pre_text == NULL ? "" : pre_text, pre_text == NULL ? "" : ": "); return; } CRM_ASSERT(node->details); crm_trace("%s%s%sNode %s: (weight=%d, fixed=%s)", pre_text == NULL ? "" : pre_text, pre_text == NULL ? "" : ": ", node->details->online ? "" : "Unavailable/Unclean ", node->details->uname, node->weight, node->fixed ? "True" : "False"); if (details) { char *pe_mutable = strdup("\t\t"); GListPtr gIter = node->details->running_rsc; crm_trace("\t\t===Node Attributes"); g_hash_table_foreach(node->details->attrs, print_str_str, pe_mutable); free(pe_mutable); crm_trace("\t\t=== Resources"); for (; gIter != NULL; gIter = gIter->next) { resource_t *rsc = (resource_t *) gIter->data; print_resource(LOG_DEBUG_4, "\t\t", rsc, FALSE); } } } /* * Used by the HashTable for-loop */ void print_str_str(gpointer key, gpointer value, gpointer user_data) { crm_trace("%s%s %s ==> %s", user_data == NULL ? "" : (char *)user_data, user_data == NULL ? "" : ": ", (char *)key, (char *)value); } void print_resource(int log_level, const char *pre_text, resource_t * rsc, gboolean details) { long options = pe_print_log | pe_print_pending; if (rsc == NULL) { do_crm_log(log_level - 1, "%s%s: ", pre_text == NULL ? "" : pre_text, pre_text == NULL ? "" : ": "); return; } if (details) { options |= pe_print_details; } rsc->fns->print(rsc, pre_text, options, &log_level); } void pe_free_action(action_t * action) { if (action == NULL) { return; } g_list_free_full(action->actions_before, free); /* action_wrapper_t* */ g_list_free_full(action->actions_after, free); /* action_wrapper_t* */ if (action->extra) { g_hash_table_destroy(action->extra); } if (action->meta) { g_hash_table_destroy(action->meta); } if (action->versioned_parameters) { free_xml(action->versioned_parameters); } if (action->versioned_meta) { free_xml(action->versioned_meta); } free(action->cancel_task); free(action->task); free(action->uuid); free(action->node); free(action); } GListPtr find_recurring_actions(GListPtr input, node_t * not_on_node) { const char *value = NULL; GListPtr result = NULL; GListPtr gIter = input; CRM_CHECK(input != NULL, return NULL); for (; gIter != NULL; gIter = gIter->next) { action_t *action = (action_t *) gIter->data; value = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL); if (value == NULL) { /* skip */ } else if (safe_str_eq(value, "0")) { /* skip */ } else if (safe_str_eq(CRMD_ACTION_CANCEL, action->task)) { /* skip */ } else if (not_on_node == NULL) { crm_trace("(null) Found: %s", action->uuid); result = g_list_prepend(result, action); } else if (action->node == NULL) { /* skip */ } else if (action->node->details != not_on_node->details) { crm_trace("Found: %s", action->uuid); result = g_list_prepend(result, action); } } return result; } enum action_tasks get_complex_task(resource_t * rsc, const char *name, gboolean allow_non_atomic) { enum action_tasks task = text2task(name); if (rsc == NULL) { return task; } else if (allow_non_atomic == FALSE || rsc->variant == pe_native) { switch (task) { case stopped_rsc: case started_rsc: case action_demoted: case action_promoted: crm_trace("Folding %s back into its atomic counterpart for %s", name, rsc->id); return task - 1; break; default: break; } } return task; } action_t * find_first_action(GListPtr input, const char *uuid, const char *task, node_t * on_node) { GListPtr gIter = NULL; CRM_CHECK(uuid || task, return NULL); for (gIter = input; gIter != NULL; gIter = gIter->next) { action_t *action = (action_t *) gIter->data; if (uuid != NULL && safe_str_neq(uuid, action->uuid)) { continue; } else if (task != NULL && safe_str_neq(task, action->task)) { continue; } else if (on_node == NULL) { return action; } else if (action->node == NULL) { continue; } else if (on_node->details == action->node->details) { return action; } } return NULL; } GListPtr find_actions(GListPtr input, const char *key, const node_t *on_node) { GListPtr gIter = input; GListPtr result = NULL; CRM_CHECK(key != NULL, return NULL); for (; gIter != NULL; gIter = gIter->next) { action_t *action = (action_t *) gIter->data; if (safe_str_neq(key, action->uuid)) { crm_trace("%s does not match action %s", key, action->uuid); continue; } else if (on_node == NULL) { crm_trace("Action %s matches (ignoring node)", key); result = g_list_prepend(result, action); } else if (action->node == NULL) { crm_trace("Action %s matches (unallocated, assigning to %s)", key, on_node->details->uname); action->node = node_copy(on_node); result = g_list_prepend(result, action); } else if (on_node->details == action->node->details) { crm_trace("Action %s on %s matches", key, on_node->details->uname); result = g_list_prepend(result, action); } else { crm_trace("Action %s on node %s does not match requested node %s", key, action->node->details->uname, on_node->details->uname); } } return result; } GListPtr find_actions_exact(GListPtr input, const char *key, node_t * on_node) { GListPtr gIter = input; GListPtr result = NULL; CRM_CHECK(key != NULL, return NULL); for (; gIter != NULL; gIter = gIter->next) { action_t *action = (action_t *) gIter->data; crm_trace("Matching %s against %s", key, action->uuid); if (safe_str_neq(key, action->uuid)) { crm_trace("Key mismatch: %s vs. %s", key, action->uuid); continue; } else if (on_node == NULL || action->node == NULL) { crm_trace("on_node=%p, action->node=%p", on_node, action->node); continue; } else if (safe_str_eq(on_node->details->id, action->node->details->id)) { result = g_list_prepend(result, action); } crm_trace("Node mismatch: %s vs. %s", on_node->details->id, action->node->details->id); } return result; } static void resource_node_score(resource_t * rsc, node_t * node, int score, const char *tag) { node_t *match = NULL; if (rsc->children) { GListPtr gIter = rsc->children; for (; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; resource_node_score(child_rsc, node, score, tag); } } pe_rsc_trace(rsc, "Setting %s for %s on %s: %d", tag, rsc->id, node->details->uname, score); match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id); if (match == NULL) { match = node_copy(node); g_hash_table_insert(rsc->allowed_nodes, (gpointer) match->details->id, match); } match->weight = merge_weights(match->weight, score); } void resource_location(resource_t * rsc, node_t * node, int score, const char *tag, pe_working_set_t * data_set) { if (node != NULL) { resource_node_score(rsc, node, score, tag); } else if (data_set != NULL) { GListPtr gIter = data_set->nodes; for (; gIter != NULL; gIter = gIter->next) { node_t *node_iter = (node_t *) gIter->data; resource_node_score(rsc, node_iter, score, tag); } } else { GHashTableIter iter; node_t *node_iter = NULL; g_hash_table_iter_init(&iter, rsc->allowed_nodes); while (g_hash_table_iter_next(&iter, NULL, (void **)&node_iter)) { resource_node_score(rsc, node_iter, score, tag); } } if (node == NULL && score == -INFINITY) { if (rsc->allocated_to) { crm_info("Deallocating %s from %s", rsc->id, rsc->allocated_to->details->uname); free(rsc->allocated_to); rsc->allocated_to = NULL; } } } #define sort_return(an_int, why) do { \ free(a_uuid); \ free(b_uuid); \ crm_trace("%s (%d) %c %s (%d) : %s", \ a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=', \ b_xml_id, b_call_id, why); \ return an_int; \ } while(0) gint sort_op_by_callid(gconstpointer a, gconstpointer b) { int a_call_id = -1; int b_call_id = -1; char *a_uuid = NULL; char *b_uuid = NULL; const xmlNode *xml_a = a; const xmlNode *xml_b = b; const char *a_xml_id = crm_element_value_const(xml_a, XML_ATTR_ID); const char *b_xml_id = crm_element_value_const(xml_b, XML_ATTR_ID); if (safe_str_eq(a_xml_id, b_xml_id)) { /* We have duplicate lrm_rsc_op entries in the status * section which is unliklely to be a good thing * - we can handle it easily enough, but we need to get * to the bottom of why it's happening. */ pe_err("Duplicate lrm_rsc_op entries named %s", a_xml_id); sort_return(0, "duplicate"); } crm_element_value_const_int(xml_a, XML_LRM_ATTR_CALLID, &a_call_id); crm_element_value_const_int(xml_b, XML_LRM_ATTR_CALLID, &b_call_id); if (a_call_id == -1 && b_call_id == -1) { /* both are pending ops so it doesn't matter since * stops are never pending */ sort_return(0, "pending"); } else if (a_call_id >= 0 && a_call_id < b_call_id) { sort_return(-1, "call id"); } else if (b_call_id >= 0 && a_call_id > b_call_id) { sort_return(1, "call id"); } else if (b_call_id >= 0 && a_call_id == b_call_id) { /* * The op and last_failed_op are the same * Order on last-rc-change */ int last_a = -1; int last_b = -1; crm_element_value_const_int(xml_a, XML_RSC_OP_LAST_CHANGE, &last_a); crm_element_value_const_int(xml_b, XML_RSC_OP_LAST_CHANGE, &last_b); crm_trace("rc-change: %d vs %d", last_a, last_b); if (last_a >= 0 && last_a < last_b) { sort_return(-1, "rc-change"); } else if (last_b >= 0 && last_a > last_b) { sort_return(1, "rc-change"); } sort_return(0, "rc-change"); } else { /* One of the inputs is a pending operation * Attempt to use XML_ATTR_TRANSITION_MAGIC to determine its age relative to the other */ int a_id = -1; int b_id = -1; int dummy = -1; const char *a_magic = crm_element_value_const(xml_a, XML_ATTR_TRANSITION_MAGIC); const char *b_magic = crm_element_value_const(xml_b, XML_ATTR_TRANSITION_MAGIC); CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic")); if(!decode_transition_magic(a_magic, &a_uuid, &a_id, &dummy, &dummy, &dummy, &dummy)) { sort_return(0, "bad magic a"); } if(!decode_transition_magic(b_magic, &b_uuid, &b_id, &dummy, &dummy, &dummy, &dummy)) { sort_return(0, "bad magic b"); } /* try to determine the relative age of the operation... * some pending operations (ie. a start) may have been superseded * by a subsequent stop * * [a|b]_id == -1 means it's a shutdown operation and _always_ comes last */ if (safe_str_neq(a_uuid, b_uuid) || a_id == b_id) { /* * some of the logic in here may be redundant... * * if the UUID from the TE doesn't match then one better * be a pending operation. * pending operations don't survive between elections and joins * because we query the LRM directly */ if (b_call_id == -1) { sort_return(-1, "transition + call"); } else if (a_call_id == -1) { sort_return(1, "transition + call"); } } else if ((a_id >= 0 && a_id < b_id) || b_id == -1) { sort_return(-1, "transition"); } else if ((b_id >= 0 && a_id > b_id) || a_id == -1) { sort_return(1, "transition"); } } /* we should never end up here */ CRM_CHECK(FALSE, sort_return(0, "default")); } time_t get_effective_time(pe_working_set_t * data_set) { if(data_set) { if (data_set->now == NULL) { crm_trace("Recording a new 'now'"); data_set->now = crm_time_new(NULL); } return crm_time_get_seconds_since_epoch(data_set->now); } crm_trace("Defaulting to 'now'"); return time(NULL); } gboolean get_target_role(resource_t * rsc, enum rsc_role_e * role) { enum rsc_role_e local_role = RSC_ROLE_UNKNOWN; const char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET_ROLE); CRM_CHECK(role != NULL, return FALSE); if (value == NULL || safe_str_eq("started", value) || safe_str_eq("default", value)) { return FALSE; } local_role = text2role(value); if (local_role == RSC_ROLE_UNKNOWN) { crm_config_err("%s: Unknown value for %s: %s", rsc->id, XML_RSC_ATTR_TARGET_ROLE, value); return FALSE; } else if (local_role > RSC_ROLE_STARTED) { if (uber_parent(rsc)->variant == pe_master) { if (local_role > RSC_ROLE_SLAVE) { /* This is what we'd do anyway, just leave the default to avoid messing up the placement algorithm */ return FALSE; } } else { crm_config_err("%s is not part of a master/slave resource, a %s of '%s' makes no sense", rsc->id, XML_RSC_ATTR_TARGET_ROLE, value); return FALSE; } } *role = local_role; return TRUE; } gboolean order_actions(action_t * lh_action, action_t * rh_action, enum pe_ordering order) { GListPtr gIter = NULL; action_wrapper_t *wrapper = NULL; GListPtr list = NULL; if (order == pe_order_none) { return FALSE; } if (lh_action == NULL || rh_action == NULL) { return FALSE; } crm_trace("Ordering Action %s before %s", lh_action->uuid, rh_action->uuid); /* Ensure we never create a dependency on ourselves... it's happened */ CRM_ASSERT(lh_action != rh_action); /* Filter dups, otherwise update_action_states() has too much work to do */ gIter = lh_action->actions_after; for (; gIter != NULL; gIter = gIter->next) { action_wrapper_t *after = (action_wrapper_t *) gIter->data; if (after->action == rh_action && (after->type & order)) { return FALSE; } } wrapper = calloc(1, sizeof(action_wrapper_t)); wrapper->action = rh_action; wrapper->type = order; list = lh_action->actions_after; list = g_list_prepend(list, wrapper); lh_action->actions_after = list; wrapper = NULL; /* order |= pe_order_implies_then; */ /* order ^= pe_order_implies_then; */ wrapper = calloc(1, sizeof(action_wrapper_t)); wrapper->action = lh_action; wrapper->type = order; list = rh_action->actions_before; list = g_list_prepend(list, wrapper); rh_action->actions_before = list; return TRUE; } action_t * get_pseudo_op(const char *name, pe_working_set_t * data_set) { action_t *op = NULL; if(data_set->singletons) { op = g_hash_table_lookup(data_set->singletons, name); } if (op == NULL) { op = custom_action(NULL, strdup(name), name, NULL, TRUE, TRUE, data_set); set_bit(op->flags, pe_action_pseudo); set_bit(op->flags, pe_action_runnable); } return op; } void destroy_ticket(gpointer data) { ticket_t *ticket = data; if (ticket->state) { g_hash_table_destroy(ticket->state); } free(ticket->id); free(ticket); } ticket_t * ticket_new(const char *ticket_id, pe_working_set_t * data_set) { ticket_t *ticket = NULL; if (ticket_id == NULL || strlen(ticket_id) == 0) { return NULL; } if (data_set->tickets == NULL) { data_set->tickets = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, destroy_ticket); } ticket = g_hash_table_lookup(data_set->tickets, ticket_id); if (ticket == NULL) { ticket = calloc(1, sizeof(ticket_t)); if (ticket == NULL) { crm_err("Cannot allocate ticket '%s'", ticket_id); return NULL; } crm_trace("Creaing ticket entry for %s", ticket_id); ticket->id = strdup(ticket_id); ticket->granted = FALSE; ticket->last_granted = -1; ticket->standby = FALSE; ticket->state = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); g_hash_table_insert(data_set->tickets, strdup(ticket->id), ticket); } return ticket; } static void filter_parameters(xmlNode * param_set, const char *param_string, bool need_present) { int len = 0; char *name = NULL; char *match = NULL; if (param_set == NULL) { return; } if (param_set) { xmlAttrPtr xIter = param_set->properties; while (xIter) { const char *prop_name = (const char *)xIter->name; xIter = xIter->next; name = NULL; len = strlen(prop_name) + 3; name = malloc(len); if(name) { sprintf(name, " %s ", prop_name); name[len - 1] = 0; match = strstr(param_string, name); } if (need_present && match == NULL) { crm_trace("%s not found in %s", prop_name, param_string); xml_remove_prop(param_set, prop_name); } else if (need_present == FALSE && match) { crm_trace("%s found in %s", prop_name, param_string); xml_remove_prop(param_set, prop_name); } free(name); } } } bool fix_remote_addr(resource_t * rsc) { const char *name; const char *value; const char *attr_list[] = { XML_ATTR_TYPE, XML_AGENT_ATTR_CLASS, XML_AGENT_ATTR_PROVIDER }; const char *value_list[] = { "remote", "ocf", "pacemaker" }; if(rsc == NULL) { return FALSE; } name = "addr"; value = g_hash_table_lookup(rsc->parameters, name); if (safe_str_eq(value, "#uname") == FALSE) { return FALSE; } for (int lpc = 0; lpc < DIMOF(attr_list); lpc++) { name = attr_list[lpc]; value = crm_element_value(rsc->xml, attr_list[lpc]); if (safe_str_eq(value, value_list[lpc]) == FALSE) { return FALSE; } } return TRUE; } static void append_versioned_params(xmlNode *versioned_params, const char *ra_version, xmlNode *params) { GHashTable *hash = pe_unpack_versioned_parameters(versioned_params, ra_version); char *key = NULL; char *value = NULL; GHashTableIter iter; g_hash_table_iter_init(&iter, hash); while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) { crm_xml_add(params, key, value); } g_hash_table_destroy(hash); } op_digest_cache_t * rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node, pe_working_set_t * data_set) { op_digest_cache_t *data = NULL; GHashTable *local_rsc_params = NULL; xmlNode *local_versioned_params = NULL; action_t *action = NULL; char *key = NULL; int interval = 0; const char *op_id = ID(xml_op); const char *interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL); const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK); const char *digest_all; const char *digest_restart; const char *secure_list; const char *restart_list; const char *op_version; const char *ra_version; CRM_ASSERT(node != NULL); data = g_hash_table_lookup(node->details->digest_cache, op_id); if (data) { return data; } data = calloc(1, sizeof(op_digest_cache_t)); CRM_ASSERT(data != NULL); digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST); digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST); secure_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_SECURE); restart_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_RESTART); op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION); ra_version = crm_element_value(xml_op, XML_ATTR_RA_VERSION); /* key is freed in custom_action */ interval = crm_parse_int(interval_s, "0"); key = generate_op_key(rsc->id, task, interval); action = custom_action(rsc, key, task, node, TRUE, FALSE, data_set); key = NULL; local_rsc_params = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); get_rsc_attributes(local_rsc_params, rsc, node, data_set); local_versioned_params = create_xml_node(NULL, XML_TAG_RSC_VER_ATTRS); pe_get_versioned_attributes(local_versioned_params, rsc, node, data_set); data->params_all = create_xml_node(NULL, XML_TAG_PARAMS); if (fix_remote_addr(rsc)) { // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside crm_xml_add(data->params_all, "addr", node->details->uname); crm_trace("Fixing addr for %s on %s", rsc->id, node->details->uname); } g_hash_table_foreach(local_rsc_params, hash2field, data->params_all); g_hash_table_foreach(action->extra, hash2field, data->params_all); g_hash_table_foreach(rsc->parameters, hash2field, data->params_all); g_hash_table_foreach(action->meta, hash2metafield, data->params_all); append_versioned_params(local_versioned_params, ra_version, data->params_all); append_versioned_params(rsc->versioned_parameters, ra_version, data->params_all); append_versioned_params(action->versioned_parameters, ra_version, data->params_all); filter_action_parameters(data->params_all, op_version); data->digest_all_calc = calculate_operation_digest(data->params_all, op_version); if (secure_list && is_set(data_set->flags, pe_flag_sanitized)) { data->params_secure = copy_xml(data->params_all); if (secure_list) { filter_parameters(data->params_secure, secure_list, FALSE); } data->digest_secure_calc = calculate_operation_digest(data->params_secure, op_version); } if (digest_restart) { data->params_restart = copy_xml(data->params_all); if (restart_list) { filter_parameters(data->params_restart, restart_list, TRUE); } data->digest_restart_calc = calculate_operation_digest(data->params_restart, op_version); } data->rc = RSC_DIGEST_MATCH; if (digest_restart && strcmp(data->digest_restart_calc, digest_restart) != 0) { data->rc = RSC_DIGEST_RESTART; } else if (digest_all == NULL) { /* it is unknown what the previous op digest was */ data->rc = RSC_DIGEST_UNKNOWN; } else if (strcmp(digest_all, data->digest_all_calc) != 0) { data->rc = RSC_DIGEST_ALL; } g_hash_table_insert(node->details->digest_cache, strdup(op_id), data); g_hash_table_destroy(local_rsc_params); free_xml(local_versioned_params); pe_free_action(action); return data; } const char *rsc_printable_id(resource_t *rsc) { if (is_not_set(rsc->flags, pe_rsc_unique)) { return ID(rsc->xml); } return rsc->id; } void clear_bit_recursive(resource_t * rsc, unsigned long long flag) { GListPtr gIter = rsc->children; clear_bit(rsc->flags, flag); for (; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; clear_bit_recursive(child_rsc, flag); } } void set_bit_recursive(resource_t * rsc, unsigned long long flag) { GListPtr gIter = rsc->children; set_bit(rsc->flags, flag); for (; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; set_bit_recursive(child_rsc, flag); } } action_t * pe_fence_op(node_t * node, const char *op, bool optional, pe_working_set_t * data_set) { char *key = NULL; action_t *stonith_op = NULL; if(op == NULL) { op = data_set->stonith_action; } key = crm_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op); if(data_set->singletons) { stonith_op = g_hash_table_lookup(data_set->singletons, key); } if(stonith_op == NULL) { stonith_op = custom_action(NULL, key, CRM_OP_FENCE, node, optional, TRUE, data_set); add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname); add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id); add_hash_param(stonith_op->meta, "stonith_action", op); } else { free(key); } if(optional == FALSE) { crm_trace("%s is no longer optional", stonith_op->uuid); pe_clear_action_bit(stonith_op, pe_action_optional); } return stonith_op; } void trigger_unfencing( resource_t * rsc, node_t *node, const char *reason, action_t *dependency, pe_working_set_t * data_set) { if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) { /* No resources require it */ return; } else if (rsc != NULL && is_not_set(rsc->flags, pe_rsc_fence_device)) { /* Wasn't a stonith device */ return; } else if(node && node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) { action_t *unfence = pe_fence_op(node, "on", FALSE, data_set); crm_notice("Unfencing %s: %s", node->details->uname, reason); if(dependency) { order_actions(unfence, dependency, pe_order_optional); } } else if(rsc) { GHashTableIter iter; g_hash_table_iter_init(&iter, rsc->allowed_nodes); while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) { trigger_unfencing(rsc, node, reason, dependency, data_set); } } } } gboolean add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref) { tag_t *tag = NULL; GListPtr gIter = NULL; gboolean is_existing = FALSE; CRM_CHECK(tags && tag_name && obj_ref, return FALSE); tag = g_hash_table_lookup(tags, tag_name); if (tag == NULL) { tag = calloc(1, sizeof(tag_t)); if (tag == NULL) { return FALSE; } tag->id = strdup(tag_name); tag->refs = NULL; g_hash_table_insert(tags, strdup(tag_name), tag); } for (gIter = tag->refs; gIter != NULL; gIter = gIter->next) { const char *existing_ref = (const char *) gIter->data; if (crm_str_eq(existing_ref, obj_ref, TRUE)){ is_existing = TRUE; break; } } if (is_existing == FALSE) { tag->refs = g_list_append(tag->refs, strdup(obj_ref)); crm_trace("Added: tag=%s ref=%s", tag->id, obj_ref); } return TRUE; } diff --git a/pengine/test10/remote-fence-unclean-3.dot b/pengine/test10/remote-fence-unclean-3.dot index 11d1208148..b32b77e3a7 100644 --- a/pengine/test10/remote-fence-unclean-3.dot +++ b/pengine/test10/remote-fence-unclean-3.dot @@ -1,27 +1,19 @@ digraph "g" { -"all_stopped" -> "fence1_start_0 overcloud-controller-1" [ style = bold] +"all_stopped" -> "fence1_start_0 overcloud-controller-0" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] -"fence1_monitor_0 overcloud-controller-0" -> "fence1_start_0 overcloud-controller-1" [ style = bold] +"fence1_monitor_0 overcloud-controller-0" -> "fence1_start_0 overcloud-controller-0" [ style = bold] "fence1_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] -"fence1_monitor_0 overcloud-controller-1" -> "fence1_start_0 overcloud-controller-1" [ style = bold] +"fence1_monitor_0 overcloud-controller-1" -> "fence1_start_0 overcloud-controller-0" [ style = bold] "fence1_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] -"fence1_monitor_0 overcloud-controller-2" -> "fence1_start_0 overcloud-controller-1" [ style = bold] +"fence1_monitor_0 overcloud-controller-2" -> "fence1_start_0 overcloud-controller-0" [ style = bold] "fence1_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] -"fence1_monitor_60000 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] -"fence1_start_0 overcloud-controller-1" -> "fence1_monitor_60000 overcloud-controller-1" [ style = bold] -"fence1_start_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] -"foo_monitor_0 overcloud-controller-0" -> "foo_start_0 overcloud-controller-0" [ style = bold] -"foo_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] -"foo_monitor_0 overcloud-controller-1" -> "foo_start_0 overcloud-controller-0" [ style = bold] -"foo_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] -"foo_monitor_0 overcloud-controller-2" -> "foo_start_0 overcloud-controller-0" [ style = bold] -"foo_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] -"foo_start_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] +"fence1_monitor_60000 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] +"fence1_start_0 overcloud-controller-0" -> "fence1_monitor_60000 overcloud-controller-0" [ style = bold] +"fence1_start_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] "overcloud-novacompute-0_stop_0 overcloud-controller-0" -> "all_stopped" [ style = bold] "overcloud-novacompute-0_stop_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' overcloud-novacompute-0" -> "stonith_complete" [ style = bold] "stonith 'reboot' overcloud-novacompute-0" [ style=bold color="green" fontcolor="black"] "stonith_complete" -> "all_stopped" [ style = bold] -"stonith_complete" -> "foo_start_0 overcloud-controller-0" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/remote-fence-unclean-3.exp b/pengine/test10/remote-fence-unclean-3.exp index 14e1b0b423..2e341bdc36 100644 --- a/pengine/test10/remote-fence-unclean-3.exp +++ b/pengine/test10/remote-fence-unclean-3.exp @@ -1,163 +1,114 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/pengine/test10/remote-fence-unclean-3.scores b/pengine/test10/remote-fence-unclean-3.scores index 0284218a05..24bdcf9caa 100644 --- a/pengine/test10/remote-fence-unclean-3.scores +++ b/pengine/test10/remote-fence-unclean-3.scores @@ -1,1298 +1,1294 @@ Allocation scores: clone_color: galera-bundle-master allocation score on galera-bundle-0: 0 clone_color: galera-bundle-master allocation score on galera-bundle-1: 0 clone_color: galera-bundle-master allocation score on galera-bundle-2: 0 clone_color: galera-bundle-master allocation score on overcloud-controller-0: -INFINITY clone_color: galera-bundle-master allocation score on overcloud-controller-1: -INFINITY clone_color: galera-bundle-master allocation score on overcloud-controller-2: -INFINITY clone_color: galera-bundle-master allocation score on overcloud-novacompute-0: -INFINITY clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY clone_color: galera-bundle-master allocation score on rabbitmq-bundle-1: -INFINITY clone_color: galera-bundle-master allocation score on rabbitmq-bundle-2: -INFINITY clone_color: galera:0 allocation score on galera-bundle-0: INFINITY clone_color: galera:0 allocation score on galera-bundle-1: -INFINITY clone_color: galera:0 allocation score on galera-bundle-2: -INFINITY clone_color: galera:0 allocation score on overcloud-controller-0: -INFINITY clone_color: galera:0 allocation score on overcloud-controller-1: -INFINITY clone_color: galera:0 allocation score on overcloud-controller-2: -INFINITY clone_color: galera:0 allocation score on overcloud-novacompute-0: -INFINITY clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY clone_color: galera:0 allocation score on rabbitmq-bundle-1: -INFINITY clone_color: galera:0 allocation score on rabbitmq-bundle-2: -INFINITY clone_color: galera:1 allocation score on galera-bundle-0: -INFINITY clone_color: galera:1 allocation score on galera-bundle-1: INFINITY clone_color: galera:1 allocation score on galera-bundle-2: -INFINITY clone_color: galera:1 allocation score on overcloud-controller-0: -INFINITY clone_color: galera:1 allocation score on overcloud-controller-1: -INFINITY clone_color: galera:1 allocation score on overcloud-controller-2: -INFINITY clone_color: galera:1 allocation score on overcloud-novacompute-0: -INFINITY clone_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY clone_color: galera:1 allocation score on rabbitmq-bundle-1: -INFINITY clone_color: galera:1 allocation score on rabbitmq-bundle-2: -INFINITY clone_color: galera:2 allocation score on galera-bundle-0: -INFINITY clone_color: galera:2 allocation score on galera-bundle-1: -INFINITY clone_color: galera:2 allocation score on galera-bundle-2: INFINITY clone_color: galera:2 allocation score on overcloud-controller-0: -INFINITY clone_color: galera:2 allocation score on overcloud-controller-1: -INFINITY clone_color: galera:2 allocation score on overcloud-controller-2: -INFINITY clone_color: galera:2 allocation score on overcloud-novacompute-0: -INFINITY clone_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY clone_color: galera:2 allocation score on rabbitmq-bundle-1: -INFINITY clone_color: galera:2 allocation score on rabbitmq-bundle-2: -INFINITY clone_color: rabbitmq-bundle-clone allocation score on overcloud-controller-0: -INFINITY clone_color: rabbitmq-bundle-clone allocation score on overcloud-controller-1: -INFINITY clone_color: rabbitmq-bundle-clone allocation score on overcloud-controller-2: -INFINITY clone_color: rabbitmq-bundle-clone allocation score on overcloud-novacompute-0: -INFINITY clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-1: 0 clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-2: 0 clone_color: rabbitmq:0 allocation score on overcloud-controller-0: -INFINITY clone_color: rabbitmq:0 allocation score on overcloud-controller-1: -INFINITY clone_color: rabbitmq:0 allocation score on overcloud-controller-2: -INFINITY clone_color: rabbitmq:0 allocation score on overcloud-novacompute-0: -INFINITY clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-1: -INFINITY clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-2: -INFINITY clone_color: rabbitmq:1 allocation score on overcloud-controller-0: -INFINITY clone_color: rabbitmq:1 allocation score on overcloud-controller-1: -INFINITY clone_color: rabbitmq:1 allocation score on overcloud-controller-2: -INFINITY clone_color: rabbitmq:1 allocation score on overcloud-novacompute-0: -INFINITY clone_color: rabbitmq:1 allocation score on rabbitmq-bundle-0: -INFINITY clone_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY clone_color: rabbitmq:1 allocation score on rabbitmq-bundle-2: -INFINITY clone_color: rabbitmq:2 allocation score on overcloud-controller-0: -INFINITY clone_color: rabbitmq:2 allocation score on overcloud-controller-1: -INFINITY clone_color: rabbitmq:2 allocation score on overcloud-controller-2: -INFINITY clone_color: rabbitmq:2 allocation score on overcloud-novacompute-0: -INFINITY clone_color: rabbitmq:2 allocation score on rabbitmq-bundle-0: -INFINITY clone_color: rabbitmq:2 allocation score on rabbitmq-bundle-1: -INFINITY clone_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY clone_color: redis-bundle-master allocation score on galera-bundle-1: -INFINITY clone_color: redis-bundle-master allocation score on galera-bundle-2: -INFINITY clone_color: redis-bundle-master allocation score on overcloud-controller-0: -INFINITY clone_color: redis-bundle-master allocation score on overcloud-controller-1: -INFINITY clone_color: redis-bundle-master allocation score on overcloud-controller-2: -INFINITY clone_color: redis-bundle-master allocation score on overcloud-novacompute-0: -INFINITY clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY clone_color: redis-bundle-master allocation score on rabbitmq-bundle-1: -INFINITY clone_color: redis-bundle-master allocation score on rabbitmq-bundle-2: -INFINITY clone_color: redis-bundle-master allocation score on redis-bundle-0: 0 clone_color: redis-bundle-master allocation score on redis-bundle-1: 0 clone_color: redis-bundle-master allocation score on redis-bundle-2: 0 clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY clone_color: redis:0 allocation score on galera-bundle-1: -INFINITY clone_color: redis:0 allocation score on galera-bundle-2: -INFINITY clone_color: redis:0 allocation score on overcloud-controller-0: -INFINITY clone_color: redis:0 allocation score on overcloud-controller-1: -INFINITY clone_color: redis:0 allocation score on overcloud-controller-2: -INFINITY clone_color: redis:0 allocation score on overcloud-novacompute-0: -INFINITY clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY clone_color: redis:0 allocation score on rabbitmq-bundle-1: -INFINITY clone_color: redis:0 allocation score on rabbitmq-bundle-2: -INFINITY clone_color: redis:0 allocation score on redis-bundle-0: INFINITY clone_color: redis:0 allocation score on redis-bundle-1: -INFINITY clone_color: redis:0 allocation score on redis-bundle-2: -INFINITY clone_color: redis:1 allocation score on galera-bundle-0: -INFINITY clone_color: redis:1 allocation score on galera-bundle-1: -INFINITY clone_color: redis:1 allocation score on galera-bundle-2: -INFINITY clone_color: redis:1 allocation score on overcloud-controller-0: -INFINITY clone_color: redis:1 allocation score on overcloud-controller-1: -INFINITY clone_color: redis:1 allocation score on overcloud-controller-2: -INFINITY clone_color: redis:1 allocation score on overcloud-novacompute-0: -INFINITY clone_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY clone_color: redis:1 allocation score on rabbitmq-bundle-1: -INFINITY clone_color: redis:1 allocation score on rabbitmq-bundle-2: -INFINITY clone_color: redis:1 allocation score on redis-bundle-0: -INFINITY clone_color: redis:1 allocation score on redis-bundle-1: INFINITY clone_color: redis:1 allocation score on redis-bundle-2: -INFINITY clone_color: redis:2 allocation score on galera-bundle-0: -INFINITY clone_color: redis:2 allocation score on galera-bundle-1: -INFINITY clone_color: redis:2 allocation score on galera-bundle-2: -INFINITY clone_color: redis:2 allocation score on overcloud-controller-0: -INFINITY clone_color: redis:2 allocation score on overcloud-controller-1: -INFINITY clone_color: redis:2 allocation score on overcloud-controller-2: -INFINITY clone_color: redis:2 allocation score on overcloud-novacompute-0: -INFINITY clone_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY clone_color: redis:2 allocation score on rabbitmq-bundle-1: -INFINITY clone_color: redis:2 allocation score on rabbitmq-bundle-2: -INFINITY clone_color: redis:2 allocation score on redis-bundle-0: -INFINITY clone_color: redis:2 allocation score on redis-bundle-1: -INFINITY clone_color: redis:2 allocation score on redis-bundle-2: INFINITY container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY container_color: galera-bundle allocation score on galera-bundle-1: -INFINITY container_color: galera-bundle allocation score on galera-bundle-2: -INFINITY container_color: galera-bundle allocation score on overcloud-controller-0: 0 container_color: galera-bundle allocation score on overcloud-controller-1: 0 container_color: galera-bundle allocation score on overcloud-controller-2: 0 container_color: galera-bundle allocation score on overcloud-novacompute-0: -INFINITY container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY container_color: galera-bundle allocation score on rabbitmq-bundle-1: -INFINITY container_color: galera-bundle allocation score on rabbitmq-bundle-2: -INFINITY container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY container_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY container_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY container_color: galera-bundle-0 allocation score on overcloud-controller-0: INFINITY container_color: galera-bundle-0 allocation score on overcloud-controller-1: 0 container_color: galera-bundle-0 allocation score on overcloud-controller-2: 0 container_color: galera-bundle-0 allocation score on overcloud-novacompute-0: -INFINITY container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: galera-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: galera-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: galera-bundle-1 allocation score on galera-bundle-0: -INFINITY container_color: galera-bundle-1 allocation score on galera-bundle-1: -INFINITY container_color: galera-bundle-1 allocation score on galera-bundle-2: -INFINITY container_color: galera-bundle-1 allocation score on overcloud-controller-0: 0 container_color: galera-bundle-1 allocation score on overcloud-controller-1: INFINITY container_color: galera-bundle-1 allocation score on overcloud-controller-2: 0 container_color: galera-bundle-1 allocation score on overcloud-novacompute-0: -INFINITY container_color: galera-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY container_color: galera-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY container_color: galera-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY container_color: galera-bundle-2 allocation score on galera-bundle-0: -INFINITY container_color: galera-bundle-2 allocation score on galera-bundle-1: -INFINITY container_color: galera-bundle-2 allocation score on galera-bundle-2: -INFINITY container_color: galera-bundle-2 allocation score on overcloud-controller-0: 0 container_color: galera-bundle-2 allocation score on overcloud-controller-1: 0 container_color: galera-bundle-2 allocation score on overcloud-controller-2: INFINITY container_color: galera-bundle-2 allocation score on overcloud-novacompute-0: -INFINITY container_color: galera-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY container_color: galera-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY container_color: galera-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY container_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY container_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY container_color: galera-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY container_color: galera-bundle-docker-0 allocation score on overcloud-controller-1: 0 container_color: galera-bundle-docker-0 allocation score on overcloud-controller-2: 0 container_color: galera-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: galera-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY container_color: galera-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY container_color: galera-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY container_color: galera-bundle-docker-1 allocation score on overcloud-controller-0: 0 container_color: galera-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY container_color: galera-bundle-docker-1 allocation score on overcloud-controller-2: 0 container_color: galera-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY container_color: galera-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY container_color: galera-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY container_color: galera-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY container_color: galera-bundle-docker-2 allocation score on overcloud-controller-0: 0 container_color: galera-bundle-docker-2 allocation score on overcloud-controller-1: 0 container_color: galera-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY container_color: galera-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY container_color: galera-bundle-master allocation score on galera-bundle-1: -INFINITY container_color: galera-bundle-master allocation score on galera-bundle-2: -INFINITY container_color: galera-bundle-master allocation score on overcloud-controller-0: 0 container_color: galera-bundle-master allocation score on overcloud-controller-1: 0 container_color: galera-bundle-master allocation score on overcloud-controller-2: 0 container_color: galera-bundle-master allocation score on overcloud-novacompute-0: 0 container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY container_color: galera-bundle-master allocation score on rabbitmq-bundle-1: -INFINITY container_color: galera-bundle-master allocation score on rabbitmq-bundle-2: -INFINITY container_color: galera:0 allocation score on galera-bundle-0: -INFINITY container_color: galera:0 allocation score on galera-bundle-1: -INFINITY container_color: galera:0 allocation score on galera-bundle-2: -INFINITY container_color: galera:0 allocation score on overcloud-controller-0: 0 container_color: galera:0 allocation score on overcloud-controller-1: 0 container_color: galera:0 allocation score on overcloud-controller-2: 0 container_color: galera:0 allocation score on overcloud-novacompute-0: 0 container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: galera:0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: galera:0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: galera:1 allocation score on galera-bundle-0: -INFINITY container_color: galera:1 allocation score on galera-bundle-1: -INFINITY container_color: galera:1 allocation score on galera-bundle-2: -INFINITY container_color: galera:1 allocation score on overcloud-controller-0: 0 container_color: galera:1 allocation score on overcloud-controller-1: 0 container_color: galera:1 allocation score on overcloud-controller-2: 0 container_color: galera:1 allocation score on overcloud-novacompute-0: 0 container_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY container_color: galera:1 allocation score on rabbitmq-bundle-1: -INFINITY container_color: galera:1 allocation score on rabbitmq-bundle-2: -INFINITY container_color: galera:2 allocation score on galera-bundle-0: -INFINITY container_color: galera:2 allocation score on galera-bundle-1: -INFINITY container_color: galera:2 allocation score on galera-bundle-2: -INFINITY container_color: galera:2 allocation score on overcloud-controller-0: 0 container_color: galera:2 allocation score on overcloud-controller-1: 0 container_color: galera:2 allocation score on overcloud-controller-2: 0 container_color: galera:2 allocation score on overcloud-novacompute-0: 0 container_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY container_color: galera:2 allocation score on rabbitmq-bundle-1: -INFINITY container_color: galera:2 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on overcloud-controller-0: 0 container_color: haproxy-bundle allocation score on overcloud-controller-0: 0 container_color: haproxy-bundle allocation score on overcloud-controller-0: 0 container_color: haproxy-bundle allocation score on overcloud-controller-0: 0 container_color: haproxy-bundle allocation score on overcloud-controller-0: 0 container_color: haproxy-bundle allocation score on overcloud-controller-0: 0 container_color: haproxy-bundle allocation score on overcloud-controller-0: 0 container_color: haproxy-bundle allocation score on overcloud-controller-1: 0 container_color: haproxy-bundle allocation score on overcloud-controller-1: 0 container_color: haproxy-bundle allocation score on overcloud-controller-1: 0 container_color: haproxy-bundle allocation score on overcloud-controller-1: 0 container_color: haproxy-bundle allocation score on overcloud-controller-1: 0 container_color: haproxy-bundle allocation score on overcloud-controller-1: 0 container_color: haproxy-bundle allocation score on overcloud-controller-1: 0 container_color: haproxy-bundle allocation score on overcloud-controller-2: 0 container_color: haproxy-bundle allocation score on overcloud-controller-2: 0 container_color: haproxy-bundle allocation score on overcloud-controller-2: 0 container_color: haproxy-bundle allocation score on overcloud-controller-2: 0 container_color: haproxy-bundle allocation score on overcloud-controller-2: 0 container_color: haproxy-bundle allocation score on overcloud-controller-2: 0 container_color: haproxy-bundle allocation score on overcloud-controller-2: 0 container_color: haproxy-bundle allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: 0 container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: 0 container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: 0 container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: 0 container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: 0 container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: 0 container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY container_color: openstack-cinder-backup allocation score on galera-bundle-0: -INFINITY container_color: openstack-cinder-backup allocation score on galera-bundle-1: -INFINITY container_color: openstack-cinder-backup allocation score on galera-bundle-2: -INFINITY container_color: openstack-cinder-backup allocation score on overcloud-controller-0: 0 container_color: openstack-cinder-backup allocation score on overcloud-controller-1: 0 container_color: openstack-cinder-backup allocation score on overcloud-controller-2: 0 container_color: openstack-cinder-backup allocation score on overcloud-novacompute-0: -INFINITY container_color: openstack-cinder-backup allocation score on rabbitmq-bundle-0: -INFINITY container_color: openstack-cinder-backup allocation score on rabbitmq-bundle-1: -INFINITY container_color: openstack-cinder-backup allocation score on rabbitmq-bundle-2: -INFINITY container_color: openstack-cinder-backup allocation score on redis-bundle-0: -INFINITY container_color: openstack-cinder-backup allocation score on redis-bundle-1: -INFINITY container_color: openstack-cinder-backup allocation score on redis-bundle-2: -INFINITY container_color: openstack-cinder-backup-docker-0 allocation score on galera-bundle-0: -INFINITY container_color: openstack-cinder-backup-docker-0 allocation score on galera-bundle-1: -INFINITY container_color: openstack-cinder-backup-docker-0 allocation score on galera-bundle-2: -INFINITY container_color: openstack-cinder-backup-docker-0 allocation score on overcloud-controller-0: 0 container_color: openstack-cinder-backup-docker-0 allocation score on overcloud-controller-1: INFINITY container_color: openstack-cinder-backup-docker-0 allocation score on overcloud-controller-2: 0 container_color: openstack-cinder-backup-docker-0 allocation score on overcloud-novacompute-0: -INFINITY container_color: openstack-cinder-backup-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: openstack-cinder-backup-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: openstack-cinder-backup-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: openstack-cinder-backup-docker-0 allocation score on redis-bundle-0: -INFINITY container_color: openstack-cinder-backup-docker-0 allocation score on redis-bundle-1: -INFINITY container_color: openstack-cinder-backup-docker-0 allocation score on redis-bundle-2: -INFINITY container_color: openstack-cinder-volume allocation score on galera-bundle-0: -INFINITY container_color: openstack-cinder-volume allocation score on galera-bundle-1: -INFINITY container_color: openstack-cinder-volume allocation score on galera-bundle-2: -INFINITY container_color: openstack-cinder-volume allocation score on overcloud-controller-0: 0 container_color: openstack-cinder-volume allocation score on overcloud-controller-1: 0 container_color: openstack-cinder-volume allocation score on overcloud-controller-2: 0 container_color: openstack-cinder-volume allocation score on overcloud-novacompute-0: -INFINITY container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-0: -INFINITY container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-1: -INFINITY container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-2: -INFINITY container_color: openstack-cinder-volume allocation score on redis-bundle-0: -INFINITY container_color: openstack-cinder-volume allocation score on redis-bundle-1: -INFINITY container_color: openstack-cinder-volume allocation score on redis-bundle-2: -INFINITY container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-1: -INFINITY container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-2: -INFINITY container_color: openstack-cinder-volume-docker-0 allocation score on overcloud-controller-0: INFINITY container_color: openstack-cinder-volume-docker-0 allocation score on overcloud-controller-1: 0 container_color: openstack-cinder-volume-docker-0 allocation score on overcloud-controller-2: 0 container_color: openstack-cinder-volume-docker-0 allocation score on overcloud-novacompute-0: -INFINITY container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-1: -INFINITY container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-2: -INFINITY container_color: rabbitmq-bundle allocation score on overcloud-controller-0: 0 container_color: rabbitmq-bundle allocation score on overcloud-controller-1: 0 container_color: rabbitmq-bundle allocation score on overcloud-controller-2: 0 container_color: rabbitmq-bundle allocation score on overcloud-novacompute-0: -INFINITY container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-1: -INFINITY container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-2: -INFINITY container_color: rabbitmq-bundle-0 allocation score on overcloud-controller-0: INFINITY container_color: rabbitmq-bundle-0 allocation score on overcloud-controller-1: 0 container_color: rabbitmq-bundle-0 allocation score on overcloud-controller-2: 0 container_color: rabbitmq-bundle-0 allocation score on overcloud-novacompute-0: -INFINITY container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: rabbitmq-bundle-1 allocation score on overcloud-controller-0: 0 container_color: rabbitmq-bundle-1 allocation score on overcloud-controller-1: INFINITY container_color: rabbitmq-bundle-1 allocation score on overcloud-controller-2: 0 container_color: rabbitmq-bundle-1 allocation score on overcloud-novacompute-0: -INFINITY container_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY container_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY container_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY container_color: rabbitmq-bundle-2 allocation score on overcloud-controller-0: 0 container_color: rabbitmq-bundle-2 allocation score on overcloud-controller-1: 0 container_color: rabbitmq-bundle-2 allocation score on overcloud-controller-2: INFINITY container_color: rabbitmq-bundle-2 allocation score on overcloud-novacompute-0: -INFINITY container_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY container_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY container_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY container_color: rabbitmq-bundle-clone allocation score on overcloud-controller-0: 0 container_color: rabbitmq-bundle-clone allocation score on overcloud-controller-1: 0 container_color: rabbitmq-bundle-clone allocation score on overcloud-controller-2: 0 container_color: rabbitmq-bundle-clone allocation score on overcloud-novacompute-0: 0 container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-1: 0 container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-2: 0 container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-1: 0 container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-2: 0 container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-0: 0 container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-2: 0 container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY container_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY container_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY container_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-0: 0 container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-1: 0 container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY container_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY container_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY container_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY container_color: rabbitmq:0 allocation score on overcloud-controller-0: 0 container_color: rabbitmq:0 allocation score on overcloud-controller-1: 0 container_color: rabbitmq:0 allocation score on overcloud-controller-2: 0 container_color: rabbitmq:0 allocation score on overcloud-novacompute-0: 0 container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY container_color: rabbitmq:0 allocation score on rabbitmq-bundle-1: 0 container_color: rabbitmq:0 allocation score on rabbitmq-bundle-2: 0 container_color: rabbitmq:1 allocation score on overcloud-controller-0: 0 container_color: rabbitmq:1 allocation score on overcloud-controller-1: 0 container_color: rabbitmq:1 allocation score on overcloud-controller-2: 0 container_color: rabbitmq:1 allocation score on overcloud-novacompute-0: 0 container_color: rabbitmq:1 allocation score on rabbitmq-bundle-0: 0 container_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY container_color: rabbitmq:1 allocation score on rabbitmq-bundle-2: 0 container_color: rabbitmq:2 allocation score on overcloud-controller-0: 0 container_color: rabbitmq:2 allocation score on overcloud-controller-1: 0 container_color: rabbitmq:2 allocation score on overcloud-controller-2: 0 container_color: rabbitmq:2 allocation score on overcloud-novacompute-0: 0 container_color: rabbitmq:2 allocation score on rabbitmq-bundle-0: 0 container_color: rabbitmq:2 allocation score on rabbitmq-bundle-1: 0 container_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY container_color: redis-bundle allocation score on galera-bundle-1: -INFINITY container_color: redis-bundle allocation score on galera-bundle-2: -INFINITY container_color: redis-bundle allocation score on overcloud-controller-0: 0 container_color: redis-bundle allocation score on overcloud-controller-1: 0 container_color: redis-bundle allocation score on overcloud-controller-2: 0 container_color: redis-bundle allocation score on overcloud-novacompute-0: -INFINITY container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY container_color: redis-bundle allocation score on rabbitmq-bundle-1: -INFINITY container_color: redis-bundle allocation score on rabbitmq-bundle-2: -INFINITY container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY container_color: redis-bundle allocation score on redis-bundle-1: -INFINITY container_color: redis-bundle allocation score on redis-bundle-2: -INFINITY container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY container_color: redis-bundle-0 allocation score on galera-bundle-1: -INFINITY container_color: redis-bundle-0 allocation score on galera-bundle-2: -INFINITY container_color: redis-bundle-0 allocation score on overcloud-controller-0: INFINITY container_color: redis-bundle-0 allocation score on overcloud-controller-1: 0 container_color: redis-bundle-0 allocation score on overcloud-controller-2: 0 container_color: redis-bundle-0 allocation score on overcloud-novacompute-0: -INFINITY container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: redis-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: redis-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY container_color: redis-bundle-0 allocation score on redis-bundle-1: -INFINITY container_color: redis-bundle-0 allocation score on redis-bundle-2: -INFINITY container_color: redis-bundle-1 allocation score on galera-bundle-0: -INFINITY container_color: redis-bundle-1 allocation score on galera-bundle-1: -INFINITY container_color: redis-bundle-1 allocation score on galera-bundle-2: -INFINITY container_color: redis-bundle-1 allocation score on overcloud-controller-0: 0 container_color: redis-bundle-1 allocation score on overcloud-controller-1: INFINITY container_color: redis-bundle-1 allocation score on overcloud-controller-2: 0 container_color: redis-bundle-1 allocation score on overcloud-novacompute-0: -INFINITY container_color: redis-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY container_color: redis-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY container_color: redis-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY container_color: redis-bundle-1 allocation score on redis-bundle-0: -INFINITY container_color: redis-bundle-1 allocation score on redis-bundle-1: -INFINITY container_color: redis-bundle-1 allocation score on redis-bundle-2: -INFINITY container_color: redis-bundle-2 allocation score on galera-bundle-0: -INFINITY container_color: redis-bundle-2 allocation score on galera-bundle-1: -INFINITY container_color: redis-bundle-2 allocation score on galera-bundle-2: -INFINITY container_color: redis-bundle-2 allocation score on overcloud-controller-0: 0 container_color: redis-bundle-2 allocation score on overcloud-controller-1: 0 container_color: redis-bundle-2 allocation score on overcloud-controller-2: INFINITY container_color: redis-bundle-2 allocation score on overcloud-novacompute-0: -INFINITY container_color: redis-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY container_color: redis-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY container_color: redis-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY container_color: redis-bundle-2 allocation score on redis-bundle-0: -INFINITY container_color: redis-bundle-2 allocation score on redis-bundle-1: -INFINITY container_color: redis-bundle-2 allocation score on redis-bundle-2: -INFINITY container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY container_color: redis-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY container_color: redis-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY container_color: redis-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY container_color: redis-bundle-docker-0 allocation score on overcloud-controller-1: 0 container_color: redis-bundle-docker-0 allocation score on overcloud-controller-2: 0 container_color: redis-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY container_color: redis-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY container_color: redis-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY container_color: redis-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY container_color: redis-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY container_color: redis-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY container_color: redis-bundle-docker-1 allocation score on overcloud-controller-0: 0 container_color: redis-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY container_color: redis-bundle-docker-1 allocation score on overcloud-controller-2: 0 container_color: redis-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY container_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY container_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY container_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY container_color: redis-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY container_color: redis-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY container_color: redis-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY container_color: redis-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY container_color: redis-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY container_color: redis-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY container_color: redis-bundle-docker-2 allocation score on overcloud-controller-0: 0 container_color: redis-bundle-docker-2 allocation score on overcloud-controller-1: 0 container_color: redis-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY container_color: redis-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY container_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY container_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY container_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY container_color: redis-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY container_color: redis-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY container_color: redis-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY container_color: redis-bundle-master allocation score on galera-bundle-1: -INFINITY container_color: redis-bundle-master allocation score on galera-bundle-2: -INFINITY container_color: redis-bundle-master allocation score on overcloud-controller-0: 0 container_color: redis-bundle-master allocation score on overcloud-controller-1: 0 container_color: redis-bundle-master allocation score on overcloud-controller-2: 0 container_color: redis-bundle-master allocation score on overcloud-novacompute-0: 0 container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY container_color: redis-bundle-master allocation score on rabbitmq-bundle-1: -INFINITY container_color: redis-bundle-master allocation score on rabbitmq-bundle-2: -INFINITY container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY container_color: redis-bundle-master allocation score on redis-bundle-1: -INFINITY container_color: redis-bundle-master allocation score on redis-bundle-2: -INFINITY container_color: redis:0 allocation score on galera-bundle-0: -INFINITY container_color: redis:0 allocation score on galera-bundle-1: -INFINITY container_color: redis:0 allocation score on galera-bundle-2: -INFINITY container_color: redis:0 allocation score on overcloud-controller-0: 0 container_color: redis:0 allocation score on overcloud-controller-1: 0 container_color: redis:0 allocation score on overcloud-controller-2: 0 container_color: redis:0 allocation score on overcloud-novacompute-0: 0 container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY container_color: redis:0 allocation score on rabbitmq-bundle-1: -INFINITY container_color: redis:0 allocation score on rabbitmq-bundle-2: -INFINITY container_color: redis:0 allocation score on redis-bundle-0: -INFINITY container_color: redis:0 allocation score on redis-bundle-1: -INFINITY container_color: redis:0 allocation score on redis-bundle-2: -INFINITY container_color: redis:1 allocation score on galera-bundle-0: -INFINITY container_color: redis:1 allocation score on galera-bundle-1: -INFINITY container_color: redis:1 allocation score on galera-bundle-2: -INFINITY container_color: redis:1 allocation score on overcloud-controller-0: 0 container_color: redis:1 allocation score on overcloud-controller-1: 0 container_color: redis:1 allocation score on overcloud-controller-2: 0 container_color: redis:1 allocation score on overcloud-novacompute-0: 0 container_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY container_color: redis:1 allocation score on rabbitmq-bundle-1: -INFINITY container_color: redis:1 allocation score on rabbitmq-bundle-2: -INFINITY container_color: redis:1 allocation score on redis-bundle-0: -INFINITY container_color: redis:1 allocation score on redis-bundle-1: -INFINITY container_color: redis:1 allocation score on redis-bundle-2: -INFINITY container_color: redis:2 allocation score on galera-bundle-0: -INFINITY container_color: redis:2 allocation score on galera-bundle-1: -INFINITY container_color: redis:2 allocation score on galera-bundle-2: -INFINITY container_color: redis:2 allocation score on overcloud-controller-0: 0 container_color: redis:2 allocation score on overcloud-controller-1: 0 container_color: redis:2 allocation score on overcloud-controller-2: 0 container_color: redis:2 allocation score on overcloud-novacompute-0: 0 container_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY container_color: redis:2 allocation score on rabbitmq-bundle-1: -INFINITY container_color: redis:2 allocation score on rabbitmq-bundle-2: -INFINITY container_color: redis:2 allocation score on redis-bundle-0: -INFINITY container_color: redis:2 allocation score on redis-bundle-1: -INFINITY container_color: redis:2 allocation score on redis-bundle-2: -INFINITY galera:0 promotion score on galera-bundle-0: 100 galera:1 promotion score on galera-bundle-1: 100 galera:2 promotion score on galera-bundle-2: 100 native_color: fence1 allocation score on overcloud-controller-0: 0 native_color: fence1 allocation score on overcloud-controller-1: 0 native_color: fence1 allocation score on overcloud-controller-2: 0 native_color: fence1 allocation score on overcloud-novacompute-0: -INFINITY -native_color: foo allocation score on overcloud-controller-0: 0 -native_color: foo allocation score on overcloud-controller-1: 0 -native_color: foo allocation score on overcloud-controller-2: 0 -native_color: foo allocation score on overcloud-novacompute-0: 0 native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY native_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY native_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY native_color: galera-bundle-0 allocation score on overcloud-controller-0: INFINITY native_color: galera-bundle-0 allocation score on overcloud-controller-1: 0 native_color: galera-bundle-0 allocation score on overcloud-controller-2: 0 native_color: galera-bundle-0 allocation score on overcloud-novacompute-0: -INFINITY native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY native_color: galera-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY native_color: galera-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY native_color: galera-bundle-1 allocation score on galera-bundle-0: -INFINITY native_color: galera-bundle-1 allocation score on galera-bundle-1: -INFINITY native_color: galera-bundle-1 allocation score on galera-bundle-2: -INFINITY native_color: galera-bundle-1 allocation score on overcloud-controller-0: 0 native_color: galera-bundle-1 allocation score on overcloud-controller-1: INFINITY native_color: galera-bundle-1 allocation score on overcloud-controller-2: 0 native_color: galera-bundle-1 allocation score on overcloud-novacompute-0: -INFINITY native_color: galera-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY native_color: galera-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY native_color: galera-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY native_color: galera-bundle-2 allocation score on galera-bundle-0: -INFINITY native_color: galera-bundle-2 allocation score on galera-bundle-1: -INFINITY native_color: galera-bundle-2 allocation score on galera-bundle-2: -INFINITY native_color: galera-bundle-2 allocation score on overcloud-controller-0: 0 native_color: galera-bundle-2 allocation score on overcloud-controller-1: 0 native_color: galera-bundle-2 allocation score on overcloud-controller-2: INFINITY native_color: galera-bundle-2 allocation score on overcloud-novacompute-0: -INFINITY native_color: galera-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY native_color: galera-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY native_color: galera-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY native_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY native_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY native_color: galera-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY native_color: galera-bundle-docker-0 allocation score on overcloud-controller-1: 0 native_color: galera-bundle-docker-0 allocation score on overcloud-controller-2: 0 native_color: galera-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY native_color: galera-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY native_color: galera-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY native_color: galera-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY native_color: galera-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY native_color: galera-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY native_color: galera-bundle-docker-1 allocation score on overcloud-controller-2: 0 native_color: galera-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY native_color: galera-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY native_color: galera-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY native_color: galera-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY native_color: galera-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY native_color: galera-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY native_color: galera-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY native_color: galera-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY native_color: galera:0 allocation score on galera-bundle-0: INFINITY native_color: galera:0 allocation score on galera-bundle-1: -INFINITY native_color: galera:0 allocation score on galera-bundle-2: -INFINITY native_color: galera:0 allocation score on overcloud-controller-0: -INFINITY native_color: galera:0 allocation score on overcloud-controller-1: -INFINITY native_color: galera:0 allocation score on overcloud-controller-2: -INFINITY native_color: galera:0 allocation score on overcloud-novacompute-0: -INFINITY native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY native_color: galera:0 allocation score on rabbitmq-bundle-1: -INFINITY native_color: galera:0 allocation score on rabbitmq-bundle-2: -INFINITY native_color: galera:1 allocation score on galera-bundle-0: -INFINITY native_color: galera:1 allocation score on galera-bundle-1: INFINITY native_color: galera:1 allocation score on galera-bundle-2: -INFINITY native_color: galera:1 allocation score on overcloud-controller-0: -INFINITY native_color: galera:1 allocation score on overcloud-controller-1: -INFINITY native_color: galera:1 allocation score on overcloud-controller-2: -INFINITY native_color: galera:1 allocation score on overcloud-novacompute-0: -INFINITY native_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY native_color: galera:1 allocation score on rabbitmq-bundle-1: -INFINITY native_color: galera:1 allocation score on rabbitmq-bundle-2: -INFINITY native_color: galera:2 allocation score on galera-bundle-0: -INFINITY native_color: galera:2 allocation score on galera-bundle-1: -INFINITY native_color: galera:2 allocation score on galera-bundle-2: INFINITY native_color: galera:2 allocation score on overcloud-controller-0: -INFINITY native_color: galera:2 allocation score on overcloud-controller-1: -INFINITY native_color: galera:2 allocation score on overcloud-controller-2: -INFINITY native_color: galera:2 allocation score on overcloud-novacompute-0: -INFINITY native_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY native_color: galera:2 allocation score on rabbitmq-bundle-1: -INFINITY native_color: galera:2 allocation score on rabbitmq-bundle-2: -INFINITY native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY native_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY native_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: INFINITY native_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: INFINITY native_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY native_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY native_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY native_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: INFINITY native_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY native_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY native_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY native_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY native_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY native_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY native_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY native_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY native_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY native_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY native_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY native_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY native_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY native_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY native_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY native_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY native_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY native_color: ip-10.0.0.7 allocation score on galera-bundle-0: -INFINITY native_color: ip-10.0.0.7 allocation score on galera-bundle-1: -INFINITY native_color: ip-10.0.0.7 allocation score on galera-bundle-2: -INFINITY native_color: ip-10.0.0.7 allocation score on overcloud-controller-0: 0 native_color: ip-10.0.0.7 allocation score on overcloud-controller-1: INFINITY native_color: ip-10.0.0.7 allocation score on overcloud-controller-2: 0 native_color: ip-10.0.0.7 allocation score on overcloud-novacompute-0: -INFINITY native_color: ip-10.0.0.7 allocation score on rabbitmq-bundle-0: -INFINITY native_color: ip-10.0.0.7 allocation score on rabbitmq-bundle-1: -INFINITY native_color: ip-10.0.0.7 allocation score on rabbitmq-bundle-2: -INFINITY native_color: ip-10.0.0.7 allocation score on redis-bundle-0: -INFINITY native_color: ip-10.0.0.7 allocation score on redis-bundle-1: -INFINITY native_color: ip-10.0.0.7 allocation score on redis-bundle-2: -INFINITY native_color: ip-172.16.1.9 allocation score on galera-bundle-0: -INFINITY native_color: ip-172.16.1.9 allocation score on galera-bundle-1: -INFINITY native_color: ip-172.16.1.9 allocation score on galera-bundle-2: -INFINITY native_color: ip-172.16.1.9 allocation score on overcloud-controller-0: 0 native_color: ip-172.16.1.9 allocation score on overcloud-controller-1: INFINITY native_color: ip-172.16.1.9 allocation score on overcloud-controller-2: 0 native_color: ip-172.16.1.9 allocation score on overcloud-novacompute-0: -INFINITY native_color: ip-172.16.1.9 allocation score on rabbitmq-bundle-0: -INFINITY native_color: ip-172.16.1.9 allocation score on rabbitmq-bundle-1: -INFINITY native_color: ip-172.16.1.9 allocation score on rabbitmq-bundle-2: -INFINITY native_color: ip-172.16.1.9 allocation score on redis-bundle-0: -INFINITY native_color: ip-172.16.1.9 allocation score on redis-bundle-1: -INFINITY native_color: ip-172.16.1.9 allocation score on redis-bundle-2: -INFINITY native_color: ip-172.16.2.4 allocation score on galera-bundle-0: -INFINITY native_color: ip-172.16.2.4 allocation score on galera-bundle-1: -INFINITY native_color: ip-172.16.2.4 allocation score on galera-bundle-2: -INFINITY native_color: ip-172.16.2.4 allocation score on overcloud-controller-0: 0 native_color: ip-172.16.2.4 allocation score on overcloud-controller-1: 0 native_color: ip-172.16.2.4 allocation score on overcloud-controller-2: INFINITY native_color: ip-172.16.2.4 allocation score on overcloud-novacompute-0: -INFINITY native_color: ip-172.16.2.4 allocation score on rabbitmq-bundle-0: -INFINITY native_color: ip-172.16.2.4 allocation score on rabbitmq-bundle-1: -INFINITY native_color: ip-172.16.2.4 allocation score on rabbitmq-bundle-2: -INFINITY native_color: ip-172.16.2.4 allocation score on redis-bundle-0: -INFINITY native_color: ip-172.16.2.4 allocation score on redis-bundle-1: -INFINITY native_color: ip-172.16.2.4 allocation score on redis-bundle-2: -INFINITY native_color: ip-172.16.2.8 allocation score on galera-bundle-0: -INFINITY native_color: ip-172.16.2.8 allocation score on galera-bundle-1: -INFINITY native_color: ip-172.16.2.8 allocation score on galera-bundle-2: -INFINITY native_color: ip-172.16.2.8 allocation score on overcloud-controller-0: INFINITY native_color: ip-172.16.2.8 allocation score on overcloud-controller-1: 0 native_color: ip-172.16.2.8 allocation score on overcloud-controller-2: 0 native_color: ip-172.16.2.8 allocation score on overcloud-novacompute-0: -INFINITY native_color: ip-172.16.2.8 allocation score on rabbitmq-bundle-0: -INFINITY native_color: ip-172.16.2.8 allocation score on rabbitmq-bundle-1: -INFINITY native_color: ip-172.16.2.8 allocation score on rabbitmq-bundle-2: -INFINITY native_color: ip-172.16.2.8 allocation score on redis-bundle-0: -INFINITY native_color: ip-172.16.2.8 allocation score on redis-bundle-1: -INFINITY native_color: ip-172.16.2.8 allocation score on redis-bundle-2: -INFINITY native_color: ip-172.16.3.9 allocation score on galera-bundle-0: -INFINITY native_color: ip-172.16.3.9 allocation score on galera-bundle-1: -INFINITY native_color: ip-172.16.3.9 allocation score on galera-bundle-2: -INFINITY native_color: ip-172.16.3.9 allocation score on overcloud-controller-0: 0 native_color: ip-172.16.3.9 allocation score on overcloud-controller-1: 0 native_color: ip-172.16.3.9 allocation score on overcloud-controller-2: INFINITY native_color: ip-172.16.3.9 allocation score on overcloud-novacompute-0: -INFINITY native_color: ip-172.16.3.9 allocation score on rabbitmq-bundle-0: -INFINITY native_color: ip-172.16.3.9 allocation score on rabbitmq-bundle-1: -INFINITY native_color: ip-172.16.3.9 allocation score on rabbitmq-bundle-2: -INFINITY native_color: ip-172.16.3.9 allocation score on redis-bundle-0: -INFINITY native_color: ip-172.16.3.9 allocation score on redis-bundle-1: -INFINITY native_color: ip-172.16.3.9 allocation score on redis-bundle-2: -INFINITY native_color: ip-192.168.24.9 allocation score on galera-bundle-0: -INFINITY native_color: ip-192.168.24.9 allocation score on galera-bundle-1: -INFINITY native_color: ip-192.168.24.9 allocation score on galera-bundle-2: -INFINITY native_color: ip-192.168.24.9 allocation score on overcloud-controller-0: INFINITY native_color: ip-192.168.24.9 allocation score on overcloud-controller-1: 0 native_color: ip-192.168.24.9 allocation score on overcloud-controller-2: 0 native_color: ip-192.168.24.9 allocation score on overcloud-novacompute-0: -INFINITY native_color: ip-192.168.24.9 allocation score on rabbitmq-bundle-0: -INFINITY native_color: ip-192.168.24.9 allocation score on rabbitmq-bundle-1: -INFINITY native_color: ip-192.168.24.9 allocation score on rabbitmq-bundle-2: -INFINITY native_color: ip-192.168.24.9 allocation score on redis-bundle-0: -INFINITY native_color: ip-192.168.24.9 allocation score on redis-bundle-1: -INFINITY native_color: ip-192.168.24.9 allocation score on redis-bundle-2: -INFINITY native_color: openstack-cinder-backup-docker-0 allocation score on galera-bundle-0: -INFINITY native_color: openstack-cinder-backup-docker-0 allocation score on galera-bundle-1: -INFINITY native_color: openstack-cinder-backup-docker-0 allocation score on galera-bundle-2: -INFINITY native_color: openstack-cinder-backup-docker-0 allocation score on overcloud-controller-0: 0 native_color: openstack-cinder-backup-docker-0 allocation score on overcloud-controller-1: INFINITY native_color: openstack-cinder-backup-docker-0 allocation score on overcloud-controller-2: 0 native_color: openstack-cinder-backup-docker-0 allocation score on overcloud-novacompute-0: -INFINITY native_color: openstack-cinder-backup-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY native_color: openstack-cinder-backup-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY native_color: openstack-cinder-backup-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY native_color: openstack-cinder-backup-docker-0 allocation score on redis-bundle-0: -INFINITY native_color: openstack-cinder-backup-docker-0 allocation score on redis-bundle-1: -INFINITY native_color: openstack-cinder-backup-docker-0 allocation score on redis-bundle-2: -INFINITY native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-1: -INFINITY native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-2: -INFINITY native_color: openstack-cinder-volume-docker-0 allocation score on overcloud-controller-0: INFINITY native_color: openstack-cinder-volume-docker-0 allocation score on overcloud-controller-1: 0 native_color: openstack-cinder-volume-docker-0 allocation score on overcloud-controller-2: 0 native_color: openstack-cinder-volume-docker-0 allocation score on overcloud-novacompute-0: -INFINITY native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-1: -INFINITY native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-2: -INFINITY native_color: overcloud-novacompute-0 allocation score on overcloud-controller-0: -INFINITY native_color: overcloud-novacompute-0 allocation score on overcloud-controller-1: -INFINITY native_color: overcloud-novacompute-0 allocation score on overcloud-controller-2: -INFINITY native_color: overcloud-novacompute-0 allocation score on overcloud-novacompute-0: -INFINITY native_color: rabbitmq-bundle-0 allocation score on overcloud-controller-0: INFINITY native_color: rabbitmq-bundle-0 allocation score on overcloud-controller-1: 0 native_color: rabbitmq-bundle-0 allocation score on overcloud-controller-2: 0 native_color: rabbitmq-bundle-0 allocation score on overcloud-novacompute-0: -INFINITY native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY native_color: rabbitmq-bundle-1 allocation score on overcloud-controller-0: 0 native_color: rabbitmq-bundle-1 allocation score on overcloud-controller-1: INFINITY native_color: rabbitmq-bundle-1 allocation score on overcloud-controller-2: 0 native_color: rabbitmq-bundle-1 allocation score on overcloud-novacompute-0: -INFINITY native_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY native_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY native_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY native_color: rabbitmq-bundle-2 allocation score on overcloud-controller-0: 0 native_color: rabbitmq-bundle-2 allocation score on overcloud-controller-1: 0 native_color: rabbitmq-bundle-2 allocation score on overcloud-controller-2: INFINITY native_color: rabbitmq-bundle-2 allocation score on overcloud-novacompute-0: -INFINITY native_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY native_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY native_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-1: 0 native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-2: 0 native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-2: 0 native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY native_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY native_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY native_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY native_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY native_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY native_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY native_color: rabbitmq:0 allocation score on overcloud-controller-0: -INFINITY native_color: rabbitmq:0 allocation score on overcloud-controller-1: -INFINITY native_color: rabbitmq:0 allocation score on overcloud-controller-2: -INFINITY native_color: rabbitmq:0 allocation score on overcloud-novacompute-0: -INFINITY native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY native_color: rabbitmq:0 allocation score on rabbitmq-bundle-1: -INFINITY native_color: rabbitmq:0 allocation score on rabbitmq-bundle-2: -INFINITY native_color: rabbitmq:1 allocation score on overcloud-controller-0: -INFINITY native_color: rabbitmq:1 allocation score on overcloud-controller-1: -INFINITY native_color: rabbitmq:1 allocation score on overcloud-controller-2: -INFINITY native_color: rabbitmq:1 allocation score on overcloud-novacompute-0: -INFINITY native_color: rabbitmq:1 allocation score on rabbitmq-bundle-0: -INFINITY native_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY native_color: rabbitmq:1 allocation score on rabbitmq-bundle-2: -INFINITY native_color: rabbitmq:2 allocation score on overcloud-controller-0: -INFINITY native_color: rabbitmq:2 allocation score on overcloud-controller-1: -INFINITY native_color: rabbitmq:2 allocation score on overcloud-controller-2: -INFINITY native_color: rabbitmq:2 allocation score on overcloud-novacompute-0: -INFINITY native_color: rabbitmq:2 allocation score on rabbitmq-bundle-0: -INFINITY native_color: rabbitmq:2 allocation score on rabbitmq-bundle-1: -INFINITY native_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY native_color: redis-bundle-0 allocation score on galera-bundle-1: -INFINITY native_color: redis-bundle-0 allocation score on galera-bundle-2: -INFINITY native_color: redis-bundle-0 allocation score on overcloud-controller-0: INFINITY native_color: redis-bundle-0 allocation score on overcloud-controller-1: 0 native_color: redis-bundle-0 allocation score on overcloud-controller-2: 0 native_color: redis-bundle-0 allocation score on overcloud-novacompute-0: -INFINITY native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY native_color: redis-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY native_color: redis-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY native_color: redis-bundle-0 allocation score on redis-bundle-1: -INFINITY native_color: redis-bundle-0 allocation score on redis-bundle-2: -INFINITY native_color: redis-bundle-1 allocation score on galera-bundle-0: -INFINITY native_color: redis-bundle-1 allocation score on galera-bundle-1: -INFINITY native_color: redis-bundle-1 allocation score on galera-bundle-2: -INFINITY native_color: redis-bundle-1 allocation score on overcloud-controller-0: 0 native_color: redis-bundle-1 allocation score on overcloud-controller-1: INFINITY native_color: redis-bundle-1 allocation score on overcloud-controller-2: 0 native_color: redis-bundle-1 allocation score on overcloud-novacompute-0: -INFINITY native_color: redis-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY native_color: redis-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY native_color: redis-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY native_color: redis-bundle-1 allocation score on redis-bundle-0: -INFINITY native_color: redis-bundle-1 allocation score on redis-bundle-1: -INFINITY native_color: redis-bundle-1 allocation score on redis-bundle-2: -INFINITY native_color: redis-bundle-2 allocation score on galera-bundle-0: -INFINITY native_color: redis-bundle-2 allocation score on galera-bundle-1: -INFINITY native_color: redis-bundle-2 allocation score on galera-bundle-2: -INFINITY native_color: redis-bundle-2 allocation score on overcloud-controller-0: 0 native_color: redis-bundle-2 allocation score on overcloud-controller-1: 0 native_color: redis-bundle-2 allocation score on overcloud-controller-2: INFINITY native_color: redis-bundle-2 allocation score on overcloud-novacompute-0: -INFINITY native_color: redis-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY native_color: redis-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY native_color: redis-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY native_color: redis-bundle-2 allocation score on redis-bundle-0: -INFINITY native_color: redis-bundle-2 allocation score on redis-bundle-1: -INFINITY native_color: redis-bundle-2 allocation score on redis-bundle-2: -INFINITY native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY native_color: redis-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY native_color: redis-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY native_color: redis-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY native_color: redis-bundle-docker-0 allocation score on overcloud-controller-1: 0 native_color: redis-bundle-docker-0 allocation score on overcloud-controller-2: 0 native_color: redis-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY native_color: redis-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY native_color: redis-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY native_color: redis-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY native_color: redis-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY native_color: redis-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY native_color: redis-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY native_color: redis-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY native_color: redis-bundle-docker-1 allocation score on overcloud-controller-2: 0 native_color: redis-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY native_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY native_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY native_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY native_color: redis-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY native_color: redis-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY native_color: redis-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY native_color: redis-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY native_color: redis-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY native_color: redis-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY native_color: redis-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY native_color: redis-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY native_color: redis-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY native_color: redis-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY native_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY native_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY native_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY native_color: redis-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY native_color: redis-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY native_color: redis-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY native_color: redis:0 allocation score on galera-bundle-0: -INFINITY native_color: redis:0 allocation score on galera-bundle-1: -INFINITY native_color: redis:0 allocation score on galera-bundle-2: -INFINITY native_color: redis:0 allocation score on overcloud-controller-0: -INFINITY native_color: redis:0 allocation score on overcloud-controller-1: -INFINITY native_color: redis:0 allocation score on overcloud-controller-2: -INFINITY native_color: redis:0 allocation score on overcloud-novacompute-0: -INFINITY native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY native_color: redis:0 allocation score on rabbitmq-bundle-1: -INFINITY native_color: redis:0 allocation score on rabbitmq-bundle-2: -INFINITY native_color: redis:0 allocation score on redis-bundle-0: INFINITY native_color: redis:0 allocation score on redis-bundle-1: -INFINITY native_color: redis:0 allocation score on redis-bundle-2: -INFINITY native_color: redis:1 allocation score on galera-bundle-0: -INFINITY native_color: redis:1 allocation score on galera-bundle-1: -INFINITY native_color: redis:1 allocation score on galera-bundle-2: -INFINITY native_color: redis:1 allocation score on overcloud-controller-0: -INFINITY native_color: redis:1 allocation score on overcloud-controller-1: -INFINITY native_color: redis:1 allocation score on overcloud-controller-2: -INFINITY native_color: redis:1 allocation score on overcloud-novacompute-0: -INFINITY native_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY native_color: redis:1 allocation score on rabbitmq-bundle-1: -INFINITY native_color: redis:1 allocation score on rabbitmq-bundle-2: -INFINITY native_color: redis:1 allocation score on redis-bundle-0: -INFINITY native_color: redis:1 allocation score on redis-bundle-1: INFINITY native_color: redis:1 allocation score on redis-bundle-2: -INFINITY native_color: redis:2 allocation score on galera-bundle-0: -INFINITY native_color: redis:2 allocation score on galera-bundle-1: -INFINITY native_color: redis:2 allocation score on galera-bundle-2: -INFINITY native_color: redis:2 allocation score on overcloud-controller-0: -INFINITY native_color: redis:2 allocation score on overcloud-controller-1: -INFINITY native_color: redis:2 allocation score on overcloud-controller-2: -INFINITY native_color: redis:2 allocation score on overcloud-novacompute-0: -INFINITY native_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY native_color: redis:2 allocation score on rabbitmq-bundle-1: -INFINITY native_color: redis:2 allocation score on rabbitmq-bundle-2: -INFINITY native_color: redis:2 allocation score on redis-bundle-0: -INFINITY native_color: redis:2 allocation score on redis-bundle-1: -INFINITY native_color: redis:2 allocation score on redis-bundle-2: INFINITY redis:0 promotion score on redis-bundle-0: 1 redis:1 promotion score on redis-bundle-1: 1 redis:2 promotion score on redis-bundle-2: 1 diff --git a/pengine/test10/remote-fence-unclean-3.summary b/pengine/test10/remote-fence-unclean-3.summary index cc60ebed21..ec24500635 100644 --- a/pengine/test10/remote-fence-unclean-3.summary +++ b/pengine/test10/remote-fence-unclean-3.summary @@ -1,92 +1,85 @@ Current cluster status: Online: [ overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 ] RemoteOFFLINE: [ overcloud-novacompute-0 ] Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ] - foo (ocf::heartbeat:Dummy): Stopped fence1 (stonith:fence_xvm): Stopped overcloud-novacompute-0 (ocf::pacemaker:remote): FAILED overcloud-controller-0 Docker container set: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-controller-0 rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-controller-1 rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-controller-2 Docker container set: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] galera-bundle-0 (ocf::heartbeat:galera): Master overcloud-controller-0 galera-bundle-1 (ocf::heartbeat:galera): Master overcloud-controller-1 galera-bundle-2 (ocf::heartbeat:galera): Master overcloud-controller-2 Docker container set: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] redis-bundle-0 (ocf::heartbeat:redis): Master overcloud-controller-0 redis-bundle-1 (ocf::heartbeat:redis): Slave overcloud-controller-1 redis-bundle-2 (ocf::heartbeat:redis): Slave overcloud-controller-2 ip-192.168.24.9 (ocf::heartbeat:IPaddr2): Started overcloud-controller-0 ip-10.0.0.7 (ocf::heartbeat:IPaddr2): Started overcloud-controller-1 ip-172.16.2.4 (ocf::heartbeat:IPaddr2): Started overcloud-controller-2 ip-172.16.2.8 (ocf::heartbeat:IPaddr2): Started overcloud-controller-0 ip-172.16.1.9 (ocf::heartbeat:IPaddr2): Started overcloud-controller-1 ip-172.16.3.9 (ocf::heartbeat:IPaddr2): Started overcloud-controller-2 Docker container set: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-0 haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started overcloud-controller-1 haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started overcloud-controller-2 Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-0 Docker container: openstack-cinder-backup [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-backup:latest] openstack-cinder-backup-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-1 Transition Summary: * Fence overcloud-novacompute-0 - * Start foo (overcloud-controller-0) - * Start fence1 (overcloud-controller-1) + * Start fence1 (overcloud-controller-0) * Stop overcloud-novacompute-0 (overcloud-controller-0) Executing cluster transition: - * Resource action: foo monitor on overcloud-controller-2 - * Resource action: foo monitor on overcloud-controller-1 - * Resource action: foo monitor on overcloud-controller-0 * Resource action: fence1 monitor on overcloud-controller-2 * Resource action: fence1 monitor on overcloud-controller-1 * Resource action: fence1 monitor on overcloud-controller-0 * Resource action: overcloud-novacompute-0 stop on overcloud-controller-0 * Fencing overcloud-novacompute-0 (reboot) * Pseudo action: stonith_complete * Pseudo action: all_stopped - * Resource action: foo start on overcloud-controller-0 - * Resource action: fence1 start on overcloud-controller-1 - * Resource action: fence1 monitor=60000 on overcloud-controller-1 + * Resource action: fence1 start on overcloud-controller-0 + * Resource action: fence1 monitor=60000 on overcloud-controller-0 Revised cluster status: Online: [ overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 ] RemoteOFFLINE: [ overcloud-novacompute-0 ] Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ] - foo (ocf::heartbeat:Dummy): Started overcloud-controller-0 - fence1 (stonith:fence_xvm): Started overcloud-controller-1 + fence1 (stonith:fence_xvm): Started overcloud-controller-0 overcloud-novacompute-0 (ocf::pacemaker:remote): Stopped Docker container set: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-controller-0 rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-controller-1 rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-controller-2 Docker container set: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] galera-bundle-0 (ocf::heartbeat:galera): Master overcloud-controller-0 galera-bundle-1 (ocf::heartbeat:galera): Master overcloud-controller-1 galera-bundle-2 (ocf::heartbeat:galera): Master overcloud-controller-2 Docker container set: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] redis-bundle-0 (ocf::heartbeat:redis): Master overcloud-controller-0 redis-bundle-1 (ocf::heartbeat:redis): Slave overcloud-controller-1 redis-bundle-2 (ocf::heartbeat:redis): Slave overcloud-controller-2 ip-192.168.24.9 (ocf::heartbeat:IPaddr2): Started overcloud-controller-0 ip-10.0.0.7 (ocf::heartbeat:IPaddr2): Started overcloud-controller-1 ip-172.16.2.4 (ocf::heartbeat:IPaddr2): Started overcloud-controller-2 ip-172.16.2.8 (ocf::heartbeat:IPaddr2): Started overcloud-controller-0 ip-172.16.1.9 (ocf::heartbeat:IPaddr2): Started overcloud-controller-1 ip-172.16.3.9 (ocf::heartbeat:IPaddr2): Started overcloud-controller-2 Docker container set: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-0 haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started overcloud-controller-1 haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started overcloud-controller-2 Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-0 Docker container: openstack-cinder-backup [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-backup:latest] openstack-cinder-backup-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-1