diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c index 3e0abedad4..d58d6beb8f 100644 --- a/lib/pengine/complex.c +++ b/lib/pengine/complex.c @@ -1,975 +1,981 @@ /* * Copyright (C) 2004 Andrew Beekhof * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include #include #include #include #include void populate_hash(xmlNode * nvpair_list, GHashTable * hash, const char **attrs, int attrs_length); resource_object_functions_t resource_class_functions[] = { { native_unpack, native_find_rsc, native_parameter, native_print, native_active, native_resource_state, native_location, native_free }, { group_unpack, native_find_rsc, native_parameter, group_print, group_active, group_resource_state, native_location, group_free }, { clone_unpack, native_find_rsc, native_parameter, clone_print, clone_active, clone_resource_state, native_location, clone_free }, { master_unpack, native_find_rsc, native_parameter, clone_print, clone_active, clone_resource_state, native_location, clone_free }, { container_unpack, native_find_rsc, native_parameter, container_print, container_active, container_resource_state, native_location, container_free } }; enum pe_obj_types get_resource_type(const char *name) { if (safe_str_eq(name, XML_CIB_TAG_RESOURCE)) { return pe_native; } else if (safe_str_eq(name, XML_CIB_TAG_GROUP)) { return pe_group; } else if (safe_str_eq(name, XML_CIB_TAG_INCARNATION)) { return pe_clone; } else if (safe_str_eq(name, XML_CIB_TAG_MASTER)) { return pe_master; } else if (safe_str_eq(name, XML_CIB_TAG_CONTAINER)) { return pe_container; } return pe_unknown; } const char * get_resource_typename(enum pe_obj_types type) { switch (type) { case pe_native: return XML_CIB_TAG_RESOURCE; case pe_group: return XML_CIB_TAG_GROUP; case pe_clone: return XML_CIB_TAG_INCARNATION; case pe_master: return XML_CIB_TAG_MASTER; case pe_container: return XML_CIB_TAG_CONTAINER; case pe_unknown: return "unknown"; } return ""; } static void dup_attr(gpointer key, gpointer value, gpointer user_data) { add_hash_param(user_data, key, value); } void get_meta_attributes(GHashTable * meta_hash, resource_t * rsc, node_t * node, pe_working_set_t * data_set) { GHashTable *node_hash = NULL; const char *version = crm_element_value(data_set->input, XML_ATTR_CRM_VERSION); if (node) { node_hash = node->details->attrs; } if (rsc->xml) { xmlAttrPtr xIter = NULL; for (xIter = rsc->xml->properties; xIter; xIter = xIter->next) { const char *prop_name = (const char *)xIter->name; const char *prop_value = crm_element_value(rsc->xml, prop_name); add_hash_param(meta_hash, prop_name, prop_value); } } unpack_instance_attributes(data_set->input, rsc->xml, XML_TAG_META_SETS, node_hash, meta_hash, NULL, FALSE, data_set->now); if(version == NULL || compare_version(version, "3.0.9") < 0) { /* populate from the regular attributes until the GUI can create * meta attributes */ unpack_instance_attributes(data_set->input, rsc->xml, XML_TAG_ATTR_SETS, node_hash, meta_hash, NULL, FALSE, data_set->now); } /* set anything else based on the parent */ if (rsc->parent != NULL) { g_hash_table_foreach(rsc->parent->meta, dup_attr, meta_hash); } /* and finally check the defaults */ unpack_instance_attributes(data_set->input, data_set->rsc_defaults, XML_TAG_META_SETS, node_hash, meta_hash, NULL, FALSE, data_set->now); } void get_rsc_attributes(GHashTable * meta_hash, resource_t * rsc, node_t * node, pe_working_set_t * data_set) { GHashTable *node_hash = NULL; if (node) { node_hash = node->details->attrs; } unpack_instance_attributes(data_set->input, rsc->xml, XML_TAG_ATTR_SETS, node_hash, meta_hash, NULL, FALSE, data_set->now); /* set anything else based on the parent */ if (rsc->parent != NULL) { get_rsc_attributes(meta_hash, rsc->parent, node, data_set); } else { /* and finally check the defaults */ unpack_instance_attributes(data_set->input, data_set->rsc_defaults, XML_TAG_ATTR_SETS, node_hash, meta_hash, NULL, FALSE, data_set->now); } } #if ENABLE_VERSIONED_ATTRS void pe_get_versioned_attributes(xmlNode * meta_hash, resource_t * rsc, node_t * node, pe_working_set_t * data_set) { GHashTable *node_hash = NULL; if (node) { node_hash = node->details->attrs; } pe_unpack_versioned_attributes(data_set->input, rsc->xml, XML_TAG_ATTR_SETS, node_hash, meta_hash, data_set->now); /* set anything else based on the parent */ if (rsc->parent != NULL) { pe_get_versioned_attributes(meta_hash, rsc->parent, node, data_set); } else { /* and finally check the defaults */ pe_unpack_versioned_attributes(data_set->input, data_set->rsc_defaults, XML_TAG_ATTR_SETS, node_hash, meta_hash, data_set->now); } } #endif static char * template_op_key(xmlNode * op) { const char *name = crm_element_value(op, "name"); const char *role = crm_element_value(op, "role"); char *key = NULL; if (role == NULL || crm_str_eq(role, RSC_ROLE_STARTED_S, TRUE) || crm_str_eq(role, RSC_ROLE_SLAVE_S, TRUE)) { role = RSC_ROLE_UNKNOWN_S; } key = crm_concat(name, role, '-'); return key; } static gboolean unpack_template(xmlNode * xml_obj, xmlNode ** expanded_xml, pe_working_set_t * data_set) { xmlNode *cib_resources = NULL; xmlNode *template = NULL; xmlNode *new_xml = NULL; xmlNode *child_xml = NULL; xmlNode *rsc_ops = NULL; xmlNode *template_ops = NULL; const char *template_ref = NULL; const char *clone = NULL; const char *id = NULL; if (xml_obj == NULL) { pe_err("No resource object for template unpacking"); return FALSE; } template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE); if (template_ref == NULL) { return TRUE; } id = ID(xml_obj); if (id == NULL) { pe_err("'%s' object must have a id", crm_element_name(xml_obj)); return FALSE; } if (crm_str_eq(template_ref, id, TRUE)) { pe_err("The resource object '%s' should not reference itself", id); return FALSE; } cib_resources = get_xpath_object("//"XML_CIB_TAG_RESOURCES, data_set->input, LOG_TRACE); if (cib_resources == NULL) { pe_err("No resources configured"); return FALSE; } template = find_entity(cib_resources, XML_CIB_TAG_RSC_TEMPLATE, template_ref); if (template == NULL) { pe_err("No template named '%s'", template_ref); return FALSE; } new_xml = copy_xml(template); xmlNodeSetName(new_xml, xml_obj->name); crm_xml_replace(new_xml, XML_ATTR_ID, id); clone = crm_element_value(xml_obj, XML_RSC_ATTR_INCARNATION); if(clone) { crm_xml_add(new_xml, XML_RSC_ATTR_INCARNATION, clone); } template_ops = find_xml_node(new_xml, "operations", FALSE); for (child_xml = __xml_first_child(xml_obj); child_xml != NULL; child_xml = __xml_next_element(child_xml)) { xmlNode *new_child = NULL; new_child = add_node_copy(new_xml, child_xml); if (crm_str_eq((const char *)new_child->name, "operations", TRUE)) { rsc_ops = new_child; } } if (template_ops && rsc_ops) { xmlNode *op = NULL; GHashTable *rsc_ops_hash = g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, NULL); for (op = __xml_first_child(rsc_ops); op != NULL; op = __xml_next_element(op)) { char *key = template_op_key(op); g_hash_table_insert(rsc_ops_hash, key, op); } for (op = __xml_first_child(template_ops); op != NULL; op = __xml_next_element(op)) { char *key = template_op_key(op); if (g_hash_table_lookup(rsc_ops_hash, key) == NULL) { add_node_copy(rsc_ops, op); } free(key); } if (rsc_ops_hash) { g_hash_table_destroy(rsc_ops_hash); } free_xml(template_ops); } /*free_xml(*expanded_xml); */ *expanded_xml = new_xml; /* Disable multi-level templates for now */ /*if(unpack_template(new_xml, expanded_xml, data_set) == FALSE) { free_xml(*expanded_xml); *expanded_xml = NULL; return FALSE; } */ return TRUE; } static gboolean add_template_rsc(xmlNode * xml_obj, pe_working_set_t * data_set) { const char *template_ref = NULL; const char *id = NULL; if (xml_obj == NULL) { pe_err("No resource object for processing resource list of template"); return FALSE; } template_ref = crm_element_value(xml_obj, XML_CIB_TAG_RSC_TEMPLATE); if (template_ref == NULL) { return TRUE; } id = ID(xml_obj); if (id == NULL) { pe_err("'%s' object must have a id", crm_element_name(xml_obj)); return FALSE; } if (crm_str_eq(template_ref, id, TRUE)) { pe_err("The resource object '%s' should not reference itself", id); return FALSE; } if (add_tag_ref(data_set->template_rsc_sets, template_ref, id) == FALSE) { return FALSE; } return TRUE; } static void handle_rsc_isolation(resource_t *rsc) { resource_t *top = uber_parent(rsc); resource_t *iso = rsc; const char *wrapper = NULL; const char *value; /* check for isolation wrapper mapping if the parent doesn't have one set * isolation mapping is enabled by default. For safety, we are allowing isolation * to be disabled by setting the meta attr, isolation=false. */ value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_ISOLATION); if (top->isolation_wrapper == NULL && (value == NULL || crm_is_true(value))) { if (g_hash_table_lookup(rsc->parameters, "pcmk_docker_image")) { wrapper = "docker-wrapper"; } /* add more isolation technologies here as we expand */ } else if (top->isolation_wrapper) { goto set_rsc_opts; } if (wrapper == NULL) { return; } /* if this is a cloned primitive/group, go head and set the isolation wrapper at * at the clone level. this is really the only sane thing to do in this situation. * This allows someone to clone an isolated resource without having to shuffle * around the isolation attributes to the clone parent */ if (top == rsc->parent && pe_rsc_is_clone(top)) { iso = top; } iso->isolation_wrapper = wrapper; set_bit(top->flags, pe_rsc_unique); set_rsc_opts: pe_warn_once(pe_wo_isolation, "Support for 'isolation' resource meta-attribute" " is deprecated and will be removed in a future release" " (use bundle syntax instead)"); clear_bit(rsc->flags, pe_rsc_allow_migrate); set_bit(rsc->flags, pe_rsc_unique); if (pe_rsc_is_clone(top)) { add_hash_param(rsc->meta, XML_RSC_ATTR_UNIQUE, XML_BOOLEAN_TRUE); } } static void check_deprecated_stonith(resource_t *rsc) { GHashTableIter iter; char *key; g_hash_table_iter_init(&iter, rsc->parameters); while (g_hash_table_iter_next(&iter, (gpointer *) &key, NULL)) { if (crm_starts_with(key, "pcmk_")) { char *cmp = key + 5; // the part after "pcmk_" if (!strcmp(cmp, "poweroff_action")) { pe_warn_once(pe_wo_poweroff, "Support for the 'pcmk_poweroff_action' stonith resource parameter" " is deprecated and will be removed in a future version" " (use 'pcmk_off_action' instead)"); } else if (!strcmp(cmp, "arg_map")) { pe_warn_once(pe_wo_arg_map, "Support for the 'pcmk_arg_map' stonith resource parameter" " is deprecated and will be removed in a future version" " (use 'pcmk_host_argument' instead)"); } else if (crm_ends_with(cmp, "_cmd")) { pe_warn_once(pe_wo_stonith_cmd, "Support for the 'pcmk_*_cmd' stonith resource parameters" " is deprecated and will be removed in a future version" " (use 'pcmk_*_action' instead)"); } } } } gboolean common_unpack(xmlNode * xml_obj, resource_t ** rsc, resource_t * parent, pe_working_set_t * data_set) { bool isdefault = FALSE; xmlNode *expanded_xml = NULL; xmlNode *ops = NULL; resource_t *top = NULL; const char *value = NULL; const char *rclass = NULL; /* Look for this after any templates have been expanded */ const char *id = crm_element_value(xml_obj, XML_ATTR_ID); int container_remote_node = 0; int baremetal_remote_node = 0; bool has_versioned_params = FALSE; crm_log_xml_trace(xml_obj, "Processing resource input..."); if (id == NULL) { pe_err("Must specify id tag in "); return FALSE; } else if (rsc == NULL) { pe_err("Nowhere to unpack resource into"); return FALSE; } if (unpack_template(xml_obj, &expanded_xml, data_set) == FALSE) { return FALSE; } *rsc = calloc(1, sizeof(resource_t)); (*rsc)->cluster = data_set; if (expanded_xml) { crm_log_xml_trace(expanded_xml, "Expanded resource..."); (*rsc)->xml = expanded_xml; (*rsc)->orig_xml = xml_obj; } else { (*rsc)->xml = xml_obj; (*rsc)->orig_xml = NULL; } /* Do not use xml_obj from here on, use (*rsc)->xml in case templates are involved */ rclass = crm_element_value((*rsc)->xml, XML_AGENT_ATTR_CLASS); (*rsc)->parent = parent; ops = find_xml_node((*rsc)->xml, "operations", FALSE); (*rsc)->ops_xml = expand_idref(ops, data_set->input); (*rsc)->variant = get_resource_type(crm_element_name((*rsc)->xml)); if ((*rsc)->variant == pe_unknown) { pe_err("Unknown resource type: %s", crm_element_name((*rsc)->xml)); free(*rsc); return FALSE; } (*rsc)->parameters = crm_str_table_new(); #if ENABLE_VERSIONED_ATTRS (*rsc)->versioned_parameters = create_xml_node(NULL, XML_TAG_RSC_VER_ATTRS); #endif (*rsc)->meta = crm_str_table_new(); (*rsc)->allowed_nodes = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, g_hash_destroy_str); (*rsc)->known_on = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, g_hash_destroy_str); value = crm_element_value((*rsc)->xml, XML_RSC_ATTR_INCARNATION); if (value) { (*rsc)->id = crm_concat(id, value, ':'); add_hash_param((*rsc)->meta, XML_RSC_ATTR_INCARNATION, value); } else { (*rsc)->id = strdup(id); } (*rsc)->fns = &resource_class_functions[(*rsc)->variant]; pe_rsc_trace((*rsc), "Unpacking resource..."); get_meta_attributes((*rsc)->meta, *rsc, NULL, data_set); get_rsc_attributes((*rsc)->parameters, *rsc, NULL, data_set); #if ENABLE_VERSIONED_ATTRS pe_get_versioned_attributes((*rsc)->versioned_parameters, *rsc, NULL, data_set); #endif (*rsc)->flags = 0; set_bit((*rsc)->flags, pe_rsc_runnable); set_bit((*rsc)->flags, pe_rsc_provisional); if (is_set(data_set->flags, pe_flag_is_managed_default)) { set_bit((*rsc)->flags, pe_rsc_managed); } (*rsc)->rsc_cons = NULL; (*rsc)->rsc_tickets = NULL; (*rsc)->actions = NULL; (*rsc)->role = RSC_ROLE_STOPPED; (*rsc)->next_role = RSC_ROLE_UNKNOWN; (*rsc)->recovery_type = recovery_stop_start; (*rsc)->stickiness = data_set->default_resource_stickiness; (*rsc)->migration_threshold = INFINITY; (*rsc)->failure_timeout = 0; value = g_hash_table_lookup((*rsc)->meta, XML_CIB_ATTR_PRIORITY); (*rsc)->priority = crm_parse_int(value, "0"); (*rsc)->effective_priority = (*rsc)->priority; value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_NOTIFY); if (crm_is_true(value)) { set_bit((*rsc)->flags, pe_rsc_notify); } if (xml_contains_remote_node((*rsc)->xml)) { (*rsc)->is_remote_node = TRUE; if (g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_CONTAINER)) { container_remote_node = 1; } else { baremetal_remote_node = 1; } } value = g_hash_table_lookup((*rsc)->meta, XML_OP_ATTR_ALLOW_MIGRATE); #if ENABLE_VERSIONED_ATTRS has_versioned_params = xml_has_children((*rsc)->versioned_parameters); #endif if (crm_is_true(value) && has_versioned_params) { pe_rsc_trace((*rsc), "Migration is disabled for resources with versioned parameters"); } else if (crm_is_true(value)) { set_bit((*rsc)->flags, pe_rsc_allow_migrate); } else if ((value == NULL) && baremetal_remote_node && !has_versioned_params) { /* by default, we want baremetal remote-nodes to be able * to float around the cluster without having to stop all the * resources within the remote-node before moving. Allowing * migration support enables this feature. If this ever causes * problems, migration support can be explicitly turned off with * allow-migrate=false. * We don't support migration for versioned resources, though. */ set_bit((*rsc)->flags, pe_rsc_allow_migrate); } value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MANAGED); if (value != NULL && safe_str_neq("default", value)) { gboolean bool_value = TRUE; crm_str_to_boolean(value, &bool_value); if (bool_value == FALSE) { clear_bit((*rsc)->flags, pe_rsc_managed); } else { set_bit((*rsc)->flags, pe_rsc_managed); } } value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MAINTENANCE); if (value != NULL && safe_str_neq("default", value)) { gboolean bool_value = FALSE; crm_str_to_boolean(value, &bool_value); if (bool_value == TRUE) { clear_bit((*rsc)->flags, pe_rsc_managed); set_bit((*rsc)->flags, pe_rsc_maintenance); } } else if (is_set(data_set->flags, pe_flag_maintenance_mode)) { clear_bit((*rsc)->flags, pe_rsc_managed); set_bit((*rsc)->flags, pe_rsc_maintenance); } pe_rsc_trace((*rsc), "Options for %s", (*rsc)->id); handle_rsc_isolation(*rsc); top = uber_parent(*rsc); value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_UNIQUE); if (crm_is_true(value) || pe_rsc_is_clone(top) == FALSE) { set_bit((*rsc)->flags, pe_rsc_unique); } value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_RESTART); if (safe_str_eq(value, "restart")) { (*rsc)->restart_type = pe_restart_restart; pe_rsc_trace((*rsc), "\tDependency restart handling: restart"); } else { (*rsc)->restart_type = pe_restart_ignore; pe_rsc_trace((*rsc), "\tDependency restart handling: ignore"); } value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_MULTIPLE); if (safe_str_eq(value, "stop_only")) { (*rsc)->recovery_type = recovery_stop_only; pe_rsc_trace((*rsc), "\tMultiple running resource recovery: stop only"); } else if (safe_str_eq(value, "block")) { (*rsc)->recovery_type = recovery_block; pe_rsc_trace((*rsc), "\tMultiple running resource recovery: block"); } else { (*rsc)->recovery_type = recovery_stop_start; pe_rsc_trace((*rsc), "\tMultiple running resource recovery: stop/start"); } value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_STICKINESS); if (value != NULL && safe_str_neq("default", value)) { (*rsc)->stickiness = char2score(value); } value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_FAIL_STICKINESS); if (value != NULL && safe_str_neq("default", value)) { (*rsc)->migration_threshold = char2score(value); } else if (value == NULL) { /* Make a best-effort guess at a migration threshold for people with 0.6 configs * try with underscores and hyphens, from both the resource and global defaults section */ const char *legacy = NULL; legacy = g_hash_table_lookup((*rsc)->meta, "resource-failure-stickiness"); if (legacy == NULL) { legacy = g_hash_table_lookup((*rsc)->meta, "resource_failure_stickiness"); } if (legacy) { value = legacy; pe_warn_once(pe_wo_rsc_failstick, "Support for 'resource-failure-stickiness' resource meta-attribute" " is deprecated and will be removed in a future release" " (use 'migration-threshold' resource meta-attribute instead)"); } legacy = g_hash_table_lookup(data_set->config_hash, "default-resource-failure-stickiness"); if (legacy == NULL) { legacy = g_hash_table_lookup(data_set->config_hash, "default_resource_failure_stickiness"); } if (legacy) { if (value == NULL) { value = legacy; } pe_warn_once(pe_wo_default_rscfs, "Support for 'default-resource-failure-stickiness' cluster option" " is deprecated and will be removed in a future release" " (use 'migration-threshold' in rsc_defaults instead)"); } if (value) { int fail_sticky = char2score(value); if (fail_sticky == -INFINITY) { (*rsc)->migration_threshold = 1; pe_rsc_info((*rsc), "Set a migration threshold of %d for %s based on a failure-stickiness of %s", (*rsc)->migration_threshold, (*rsc)->id, value); } else if ((*rsc)->stickiness != 0 && fail_sticky != 0) { (*rsc)->migration_threshold = (*rsc)->stickiness / fail_sticky; if ((*rsc)->migration_threshold < 0) { /* Make sure it's positive */ (*rsc)->migration_threshold = 0 - (*rsc)->migration_threshold; } (*rsc)->migration_threshold += 1; pe_rsc_info((*rsc), "Calculated a migration threshold for %s of %d based on a stickiness of %d/%s", (*rsc)->id, (*rsc)->migration_threshold, (*rsc)->stickiness, value); } } } if (safe_str_eq(rclass, PCMK_RESOURCE_CLASS_STONITH)) { set_bit(data_set->flags, pe_flag_have_stonith_resource); set_bit((*rsc)->flags, pe_rsc_fence_device); check_deprecated_stonith(*rsc); } value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_REQUIRES); handle_requires_pref: if (safe_str_eq(value, "nothing")) { } else if (safe_str_eq(value, "quorum")) { set_bit((*rsc)->flags, pe_rsc_needs_quorum); } else if (safe_str_eq(value, "unfencing")) { if (is_set((*rsc)->flags, pe_rsc_fence_device)) { crm_config_warn("%s is a fencing device but requires (un)fencing", (*rsc)->id); value = "quorum"; isdefault = TRUE; goto handle_requires_pref; } else if (is_not_set(data_set->flags, pe_flag_stonith_enabled)) { crm_config_warn("%s requires (un)fencing but fencing is disabled", (*rsc)->id); value = "quorum"; isdefault = TRUE; goto handle_requires_pref; } else { set_bit((*rsc)->flags, pe_rsc_needs_fencing); set_bit((*rsc)->flags, pe_rsc_needs_unfencing); } } else if (safe_str_eq(value, "fencing")) { set_bit((*rsc)->flags, pe_rsc_needs_fencing); if (is_not_set(data_set->flags, pe_flag_stonith_enabled)) { crm_config_warn("%s requires fencing but fencing is disabled", (*rsc)->id); } } else { if (value) { crm_config_err("Invalid value for %s->requires: %s%s", (*rsc)->id, value, is_set(data_set->flags, pe_flag_stonith_enabled) ? "" : " (stonith-enabled=false)"); } isdefault = TRUE; if(is_set((*rsc)->flags, pe_rsc_fence_device)) { value = "quorum"; + } else if (safe_str_eq(crm_element_value((*rsc)->xml, XML_AGENT_ATTR_CLASS), "ocf") + && safe_str_eq(crm_element_value((*rsc)->xml, XML_AGENT_ATTR_PROVIDER), "pacemaker") + && safe_str_eq(crm_element_value((*rsc)->xml, XML_ATTR_TYPE), "remote") + ) { + value = "quorum"; + } else if (is_set(data_set->flags, pe_flag_enable_unfencing)) { value = "unfencing"; } else if (is_set(data_set->flags, pe_flag_stonith_enabled)) { value = "fencing"; } else if (data_set->no_quorum_policy == no_quorum_ignore) { value = "nothing"; } else { value = "quorum"; } goto handle_requires_pref; } pe_rsc_trace((*rsc), "\tRequired to start: %s%s", value, isdefault?" (default)":""); value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_FAIL_TIMEOUT); if (value != NULL) { /* call crm_get_msec() and convert back to seconds */ (*rsc)->failure_timeout = (crm_get_msec(value) / 1000); } if (baremetal_remote_node) { value = g_hash_table_lookup((*rsc)->parameters, XML_REMOTE_ATTR_RECONNECT_INTERVAL); if (value) { /* reconnect delay works by setting failure_timeout and preventing the * connection from starting until the failure is cleared. */ (*rsc)->remote_reconnect_interval = (crm_get_msec(value) / 1000); /* we want to override any default failure_timeout in use when remote * reconnect_interval is in use. */ (*rsc)->failure_timeout = (*rsc)->remote_reconnect_interval; } } get_target_role(*rsc, &((*rsc)->next_role)); pe_rsc_trace((*rsc), "\tDesired next state: %s", (*rsc)->next_role != RSC_ROLE_UNKNOWN ? role2text((*rsc)->next_role) : "default"); if ((*rsc)->fns->unpack(*rsc, data_set) == FALSE) { return FALSE; } if (is_set(data_set->flags, pe_flag_symmetric_cluster)) { // This tag must stay exactly the same because it is tested elsewhere resource_location(*rsc, NULL, 0, "symmetric_default", data_set); } else if (container_remote_node) { /* remote resources tied to a container resource must always be allowed * to opt-in to the cluster. Whether the connection resource is actually * allowed to be placed on a node is dependent on the container resource */ resource_location(*rsc, NULL, 0, "remote_connection_default", data_set); } pe_rsc_trace((*rsc), "\tAction notification: %s", is_set((*rsc)->flags, pe_rsc_notify) ? "required" : "not required"); (*rsc)->utilization = crm_str_table_new(); unpack_instance_attributes(data_set->input, (*rsc)->xml, XML_TAG_UTILIZATION, NULL, (*rsc)->utilization, NULL, FALSE, data_set->now); /* data_set->resources = g_list_append(data_set->resources, (*rsc)); */ if (expanded_xml) { if (add_template_rsc(xml_obj, data_set) == FALSE) { return FALSE; } } return TRUE; } void common_update_score(resource_t * rsc, const char *id, int score) { node_t *node = NULL; node = pe_hash_table_lookup(rsc->allowed_nodes, id); if (node != NULL) { pe_rsc_trace(rsc, "Updating score for %s on %s: %d + %d", rsc->id, id, node->weight, score); node->weight = merge_weights(node->weight, score); } if (rsc->children) { GListPtr gIter = rsc->children; for (; gIter != NULL; gIter = gIter->next) { resource_t *child_rsc = (resource_t *) gIter->data; common_update_score(child_rsc, id, score); } } } gboolean is_parent(resource_t *child, resource_t *rsc) { resource_t *parent = child; if (parent == NULL || rsc == NULL) { return FALSE; } while (parent->parent != NULL) { if (parent->parent == rsc) { return TRUE; } parent = parent->parent; } return FALSE; } resource_t * uber_parent(resource_t * rsc) { resource_t *parent = rsc; if (parent == NULL) { return NULL; } while (parent->parent != NULL && parent->parent->variant != pe_container) { parent = parent->parent; } return parent; } void common_free(resource_t * rsc) { if (rsc == NULL) { return; } pe_rsc_trace(rsc, "Freeing %s %d", rsc->id, rsc->variant); g_list_free(rsc->rsc_cons); g_list_free(rsc->rsc_cons_lhs); g_list_free(rsc->rsc_tickets); g_list_free(rsc->dangling_migrations); if (rsc->parameters != NULL) { g_hash_table_destroy(rsc->parameters); } #if ENABLE_VERSIONED_ATTRS if (rsc->versioned_parameters != NULL) { free_xml(rsc->versioned_parameters); } #endif if (rsc->meta != NULL) { g_hash_table_destroy(rsc->meta); } if (rsc->utilization != NULL) { g_hash_table_destroy(rsc->utilization); } if (rsc->parent == NULL && is_set(rsc->flags, pe_rsc_orphan)) { free_xml(rsc->xml); rsc->xml = NULL; free_xml(rsc->orig_xml); rsc->orig_xml = NULL; /* if rsc->orig_xml, then rsc->xml is an expanded xml from a template */ } else if (rsc->orig_xml) { free_xml(rsc->xml); rsc->xml = NULL; } if (rsc->running_on) { g_list_free(rsc->running_on); rsc->running_on = NULL; } if (rsc->known_on) { g_hash_table_destroy(rsc->known_on); rsc->known_on = NULL; } if (rsc->actions) { g_list_free(rsc->actions); rsc->actions = NULL; } if (rsc->allowed_nodes) { g_hash_table_destroy(rsc->allowed_nodes); rsc->allowed_nodes = NULL; } g_list_free(rsc->fillers); g_list_free(rsc->rsc_location); pe_rsc_trace(rsc, "Resource freed"); free(rsc->id); free(rsc->clone_name); free(rsc->allocated_to); free(rsc->variant_opaque); free(rsc->pending_task); free(rsc); } diff --git a/pengine/test10/bug-rh-1097457.dot b/pengine/test10/bug-rh-1097457.dot index 5984811766..94ffe136a5 100644 --- a/pengine/test10/bug-rh-1097457.dot +++ b/pengine/test10/bug-rh-1097457.dot @@ -1,126 +1,126 @@ digraph "g" { "FAKE1-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FAKE1_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FAKE2-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FAKE2_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FAKE3-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FAKE3_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FAKE4-IP_monitor_0 lamaVM1" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] "FAKE4-IP_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] "FAKE4-IP_monitor_0 lamaVM3" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] "FAKE4-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FAKE4-IP_monitor_30000 lamaVM2" [ style=bold color="green" fontcolor="black"] "FAKE4-IP_start_0 lamaVM2" -> "FAKE4-IP_monitor_30000 lamaVM2" [ style = bold] "FAKE4-IP_start_0 lamaVM2" -> "lamaVM2-G4_running_0" [ style = bold] "FAKE4-IP_start_0 lamaVM2" [ style=bold color="green" fontcolor="black"] "FAKE4-IP_stop_0 lamaVM2" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] "FAKE4-IP_stop_0 lamaVM2" -> "FAKE4_stop_0 lamaVM2" [ style = bold] "FAKE4-IP_stop_0 lamaVM2" -> "all_stopped" [ style = bold] "FAKE4-IP_stop_0 lamaVM2" -> "lamaVM2-G4_stopped_0" [ style = bold] "FAKE4-IP_stop_0 lamaVM2" [ style=bold color="green" fontcolor="orange"] "FAKE4_monitor_0 lamaVM1" -> "FAKE4_start_0 lamaVM2" [ style = bold] "FAKE4_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] "FAKE4_monitor_0 lamaVM3" -> "FAKE4_start_0 lamaVM2" [ style = bold] "FAKE4_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FAKE4_monitor_30000 lamaVM2" [ style=bold color="green" fontcolor="black"] "FAKE4_start_0 lamaVM2" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] "FAKE4_start_0 lamaVM2" -> "FAKE4_monitor_30000 lamaVM2" [ style = bold] "FAKE4_start_0 lamaVM2" -> "lamaVM2-G4_running_0" [ style = bold] "FAKE4_start_0 lamaVM2" [ style=bold color="green" fontcolor="black"] "FAKE4_stop_0 lamaVM2" -> "FAKE4_start_0 lamaVM2" [ style = bold] "FAKE4_stop_0 lamaVM2" -> "all_stopped" [ style = bold] "FAKE4_stop_0 lamaVM2" -> "lamaVM2-G4_stopped_0" [ style = bold] "FAKE4_stop_0 lamaVM2" [ style=bold color="green" fontcolor="orange"] "FAKE5-IP_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] "FAKE5-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FAKE5_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] "FAKE6-IP_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] "FAKE6-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FAKE6-clone_running_0" [ style=bold color="green" fontcolor="orange"] "FAKE6-clone_start_0" -> "FAKE6-clone_running_0" [ style = bold] "FAKE6-clone_start_0" -> "FAKE6_start_0 lamaVM2" [ style = bold] "FAKE6-clone_start_0" [ style=bold color="green" fontcolor="orange"] "FAKE6-clone_stop_0" -> "FAKE6-clone_stopped_0" [ style = bold] "FAKE6-clone_stop_0" -> "FAKE6_stop_0 lamaVM2" [ style = bold] "FAKE6-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "FAKE6-clone_stopped_0" -> "FAKE6-clone_start_0" [ style = bold] "FAKE6-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "FAKE6_monitor_30000 lamaVM2" [ style=bold color="green" fontcolor="black"] "FAKE6_start_0 lamaVM2" -> "FAKE6-clone_running_0" [ style = bold] "FAKE6_start_0 lamaVM2" -> "FAKE6_monitor_30000 lamaVM2" [ style = bold] "FAKE6_start_0 lamaVM2" [ style=bold color="green" fontcolor="black"] "FAKE6_stop_0 lamaVM2" -> "FAKE6-clone_stopped_0" [ style = bold] "FAKE6_stop_0 lamaVM2" -> "FAKE6_start_0 lamaVM2" [ style = bold] "FAKE6_stop_0 lamaVM2" -> "all_stopped" [ style = bold] "FAKE6_stop_0 lamaVM2" [ style=bold color="green" fontcolor="orange"] "FSlun1_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FSlun2_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FSlun3_monitor_0 lamaVM1" -> "FSlun3_start_0 lama2" [ style = bold] "FSlun3_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] "FSlun3_monitor_0 lamaVM3" -> "FSlun3_start_0 lama2" [ style = bold] "FSlun3_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] "FSlun3_monitor_10000 lama2" [ style=bold color="green" fontcolor="black"] "FSlun3_monitor_10000 lamaVM2" [ style=bold color="green" fontcolor="black"] "FSlun3_start_0 lama2" -> "FSlun3_monitor_10000 lama2" [ style = bold] "FSlun3_start_0 lama2" -> "lamaVM2-G4_start_0" [ style = bold] "FSlun3_start_0 lama2" [ style=bold color="green" fontcolor="black"] "FSlun3_stop_0 lamaVM2" -> "FSlun3_start_0 lama2" [ style = bold] "FSlun3_stop_0 lamaVM2" -> "all_stopped" [ style = bold] "FSlun3_stop_0 lamaVM2" [ style=bold color="green" fontcolor="orange"] "FSlun4_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] "VM2_monitor_10000 lama3" [ style=bold color="green" fontcolor="black"] "VM2_start_0 lama3" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] "VM2_start_0 lama3" -> "FAKE4_start_0 lamaVM2" [ style = bold] "VM2_start_0 lama3" -> "FAKE6_start_0 lamaVM2" [ style = bold] "VM2_start_0 lama3" -> "FSlun3_start_0 lama2" [ style = bold] "VM2_start_0 lama3" -> "VM2_monitor_10000 lama3" [ style = bold] "VM2_start_0 lama3" -> "lamaVM2_start_0 lama3" [ style = bold] "VM2_start_0 lama3" [ style=bold color="green" fontcolor="black"] "VM2_stop_0 lama3" -> "VM2_start_0 lama3" [ style = bold] "VM2_stop_0 lama3" -> "all_stopped" [ style = bold] "VM2_stop_0 lama3" -> "stonith 'reboot' lamaVM2" [ style = bold] "VM2_stop_0 lama3" [ style=bold color="green" fontcolor="black"] +"all_stopped" -> "lamaVM2_start_0 lama3" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "lamaVM2-G4_running_0" [ style=bold color="green" fontcolor="orange"] "lamaVM2-G4_start_0" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] "lamaVM2-G4_start_0" -> "FAKE4_start_0 lamaVM2" [ style = bold] "lamaVM2-G4_start_0" -> "lamaVM2-G4_running_0" [ style = bold] "lamaVM2-G4_start_0" [ style=bold color="green" fontcolor="orange"] "lamaVM2-G4_stop_0" -> "FAKE4-IP_stop_0 lamaVM2" [ style = bold] "lamaVM2-G4_stop_0" -> "FAKE4_stop_0 lamaVM2" [ style = bold] "lamaVM2-G4_stop_0" -> "lamaVM2-G4_stopped_0" [ style = bold] "lamaVM2-G4_stop_0" [ style=bold color="green" fontcolor="orange"] "lamaVM2-G4_stopped_0" -> "FSlun3_stop_0 lamaVM2" [ style = bold] "lamaVM2-G4_stopped_0" -> "lamaVM2-G4_start_0" [ style = bold] "lamaVM2-G4_stopped_0" [ style=bold color="green" fontcolor="orange"] "lamaVM2_monitor_30000 lama3" [ style=bold color="green" fontcolor="black"] "lamaVM2_start_0 lama3" -> "FAKE4-IP_monitor_30000 lamaVM2" [ style = bold] "lamaVM2_start_0 lama3" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] "lamaVM2_start_0 lama3" -> "FAKE4_monitor_30000 lamaVM2" [ style = bold] "lamaVM2_start_0 lama3" -> "FAKE4_start_0 lamaVM2" [ style = bold] "lamaVM2_start_0 lama3" -> "FAKE6_monitor_30000 lamaVM2" [ style = bold] "lamaVM2_start_0 lama3" -> "FAKE6_start_0 lamaVM2" [ style = bold] "lamaVM2_start_0 lama3" -> "FSlun3_monitor_10000 lamaVM2" [ style = bold] "lamaVM2_start_0 lama3" -> "lamaVM2_monitor_30000 lama3" [ style = bold] "lamaVM2_start_0 lama3" [ style=bold color="green" fontcolor="black"] "lamaVM2_stop_0 lama3" -> "VM2_stop_0 lama3" [ style = bold] "lamaVM2_stop_0 lama3" -> "all_stopped" [ style = bold] "lamaVM2_stop_0 lama3" -> "lamaVM2_start_0 lama3" [ style = bold] "lamaVM2_stop_0 lama3" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' lamaVM2" -> "FAKE4-IP_stop_0 lamaVM2" [ style = bold] "stonith 'reboot' lamaVM2" -> "FAKE4_stop_0 lamaVM2" [ style = bold] "stonith 'reboot' lamaVM2" -> "FAKE6-clone_stop_0" [ style = bold] "stonith 'reboot' lamaVM2" -> "FAKE6_stop_0 lamaVM2" [ style = bold] "stonith 'reboot' lamaVM2" -> "FSlun3_stop_0 lamaVM2" [ style = bold] "stonith 'reboot' lamaVM2" -> "lamaVM2-G4_stop_0" [ style = bold] "stonith 'reboot' lamaVM2" -> "stonith_complete" [ style = bold] "stonith 'reboot' lamaVM2" [ style=bold color="green" fontcolor="orange"] "stonith_complete" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] "stonith_complete" -> "FAKE4_start_0 lamaVM2" [ style = bold] "stonith_complete" -> "FAKE6_start_0 lamaVM2" [ style = bold] "stonith_complete" -> "FSlun3_start_0 lama2" [ style = bold] "stonith_complete" -> "VM2_start_0 lama3" [ style = bold] "stonith_complete" -> "all_stopped" [ style = bold] -"stonith_complete" -> "lamaVM2_start_0 lama3" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/bug-rh-1097457.exp b/pengine/test10/bug-rh-1097457.exp index 4eedd9123e..f1451b5a2b 100644 --- a/pengine/test10/bug-rh-1097457.exp +++ b/pengine/test10/bug-rh-1097457.exp @@ -1,694 +1,694 @@ - + - + - + diff --git a/pengine/test10/bug-rh-1097457.summary b/pengine/test10/bug-rh-1097457.summary index e23c6adc25..0e7d2e0b9a 100644 --- a/pengine/test10/bug-rh-1097457.summary +++ b/pengine/test10/bug-rh-1097457.summary @@ -1,125 +1,125 @@ 2 of 26 resources DISABLED and 0 BLOCKED from being started due to failures Current cluster status: Online: [ lama2 lama3 ] Containers: [ lamaVM1:VM1 lamaVM2:VM2 lamaVM3:VM3 ] restofencelama2 (stonith:fence_ipmilan): Started lama3 restofencelama3 (stonith:fence_ipmilan): Started lama2 VM1 (ocf::heartbeat:VirtualDomain): Started lama2 FSlun1 (ocf::heartbeat:Filesystem): Started lamaVM1 FSlun2 (ocf::heartbeat:Filesystem): Started lamaVM1 VM2 (ocf::heartbeat:VirtualDomain): FAILED lama3 VM3 (ocf::heartbeat:VirtualDomain): Started lama3 FSlun3 (ocf::heartbeat:Filesystem): FAILED lamaVM2 FSlun4 (ocf::heartbeat:Filesystem): Started lamaVM3 FAKE5-IP (ocf::heartbeat:IPaddr2): Stopped ( disabled ) FAKE6-IP (ocf::heartbeat:IPaddr2): Stopped ( disabled ) FAKE5 (ocf::heartbeat:Dummy): Started lamaVM3 Resource Group: lamaVM1-G1 FAKE1 (ocf::heartbeat:Dummy): Started lamaVM1 FAKE1-IP (ocf::heartbeat:IPaddr2): Started lamaVM1 Resource Group: lamaVM1-G2 FAKE2 (ocf::heartbeat:Dummy): Started lamaVM1 FAKE2-IP (ocf::heartbeat:IPaddr2): Started lamaVM1 Resource Group: lamaVM1-G3 FAKE3 (ocf::heartbeat:Dummy): Started lamaVM1 FAKE3-IP (ocf::heartbeat:IPaddr2): Started lamaVM1 Resource Group: lamaVM2-G4 FAKE4 (ocf::heartbeat:Dummy): Started lamaVM2 FAKE4-IP (ocf::heartbeat:IPaddr2): Started lamaVM2 Clone Set: FAKE6-clone [FAKE6] Started: [ lamaVM1 lamaVM2 lamaVM3 ] Transition Summary: * Fence (reboot) lamaVM2 (resource: VM2) 'guest is unclean' * Recover VM2 ( lama3 ) * Recover FSlun3 ( lamaVM2 -> lama2 ) * Restart FAKE4 ( lamaVM2 ) due to required VM2 start * Restart FAKE4-IP ( lamaVM2 ) due to required VM2 start * Restart FAKE6:2 ( lamaVM2 ) due to required VM2 start * Restart lamaVM2 ( lama3 ) due to required VM2 start Executing cluster transition: * Resource action: FSlun1 monitor on lamaVM3 * Resource action: FSlun2 monitor on lamaVM3 * Resource action: FSlun3 monitor on lamaVM3 * Resource action: FSlun3 monitor on lamaVM1 * Resource action: FSlun4 monitor on lamaVM1 * Resource action: FAKE5-IP monitor on lamaVM3 * Resource action: FAKE5-IP monitor on lamaVM1 * Resource action: FAKE6-IP monitor on lamaVM3 * Resource action: FAKE6-IP monitor on lamaVM1 * Resource action: FAKE5 monitor on lamaVM1 * Resource action: FAKE1 monitor on lamaVM3 * Resource action: FAKE1-IP monitor on lamaVM3 * Resource action: FAKE2 monitor on lamaVM3 * Resource action: FAKE2-IP monitor on lamaVM3 * Resource action: FAKE3 monitor on lamaVM3 * Resource action: FAKE3-IP monitor on lamaVM3 * Resource action: FAKE4 monitor on lamaVM3 * Resource action: FAKE4 monitor on lamaVM1 * Resource action: FAKE4-IP monitor on lamaVM3 * Resource action: FAKE4-IP monitor on lamaVM1 * Resource action: lamaVM2 stop on lama3 * Resource action: VM2 stop on lama3 * Pseudo action: stonith-lamaVM2-reboot on lamaVM2 * Pseudo action: stonith_complete * Resource action: VM2 start on lama3 * Resource action: VM2 monitor=10000 on lama3 * Pseudo action: lamaVM2-G4_stop_0 * Pseudo action: FAKE4-IP_stop_0 * Pseudo action: FAKE6-clone_stop_0 - * Resource action: lamaVM2 start on lama3 - * Resource action: lamaVM2 monitor=30000 on lama3 - * Resource action: FSlun3 monitor=10000 on lamaVM2 * Pseudo action: FAKE4_stop_0 * Pseudo action: FAKE6_stop_0 * Pseudo action: FAKE6-clone_stopped_0 * Pseudo action: FAKE6-clone_start_0 * Pseudo action: lamaVM2-G4_stopped_0 - * Resource action: FAKE6 start on lamaVM2 - * Resource action: FAKE6 monitor=30000 on lamaVM2 - * Pseudo action: FAKE6-clone_running_0 * Pseudo action: FSlun3_stop_0 * Pseudo action: all_stopped * Resource action: FSlun3 start on lama2 * Pseudo action: lamaVM2-G4_start_0 + * Resource action: lamaVM2 start on lama3 + * Resource action: lamaVM2 monitor=30000 on lama3 + * Resource action: FSlun3 monitor=10000 on lama2 + * Resource action: FSlun3 monitor=10000 on lamaVM2 * Resource action: FAKE4 start on lamaVM2 * Resource action: FAKE4 monitor=30000 on lamaVM2 * Resource action: FAKE4-IP start on lamaVM2 * Resource action: FAKE4-IP monitor=30000 on lamaVM2 - * Resource action: FSlun3 monitor=10000 on lama2 + * Resource action: FAKE6 start on lamaVM2 + * Resource action: FAKE6 monitor=30000 on lamaVM2 + * Pseudo action: FAKE6-clone_running_0 * Pseudo action: lamaVM2-G4_running_0 Revised cluster status: Online: [ lama2 lama3 ] Containers: [ lamaVM1:VM1 lamaVM2:VM2 lamaVM3:VM3 ] restofencelama2 (stonith:fence_ipmilan): Started lama3 restofencelama3 (stonith:fence_ipmilan): Started lama2 VM1 (ocf::heartbeat:VirtualDomain): Started lama2 FSlun1 (ocf::heartbeat:Filesystem): Started lamaVM1 FSlun2 (ocf::heartbeat:Filesystem): Started lamaVM1 VM2 (ocf::heartbeat:VirtualDomain): FAILED lama3 VM3 (ocf::heartbeat:VirtualDomain): Started lama3 FSlun3 (ocf::heartbeat:Filesystem): FAILED [ lama2 lamaVM2 ] FSlun4 (ocf::heartbeat:Filesystem): Started lamaVM3 FAKE5-IP (ocf::heartbeat:IPaddr2): Stopped ( disabled ) FAKE6-IP (ocf::heartbeat:IPaddr2): Stopped ( disabled ) FAKE5 (ocf::heartbeat:Dummy): Started lamaVM3 Resource Group: lamaVM1-G1 FAKE1 (ocf::heartbeat:Dummy): Started lamaVM1 FAKE1-IP (ocf::heartbeat:IPaddr2): Started lamaVM1 Resource Group: lamaVM1-G2 FAKE2 (ocf::heartbeat:Dummy): Started lamaVM1 FAKE2-IP (ocf::heartbeat:IPaddr2): Started lamaVM1 Resource Group: lamaVM1-G3 FAKE3 (ocf::heartbeat:Dummy): Started lamaVM1 FAKE3-IP (ocf::heartbeat:IPaddr2): Started lamaVM1 Resource Group: lamaVM2-G4 FAKE4 (ocf::heartbeat:Dummy): Started lamaVM2 FAKE4-IP (ocf::heartbeat:IPaddr2): Started lamaVM2 Clone Set: FAKE6-clone [FAKE6] Started: [ lamaVM1 lamaVM2 lamaVM3 ] diff --git a/pengine/test10/bundle-order-fencing.dot b/pengine/test10/bundle-order-fencing.dot index d653250798..980bab4b23 100644 --- a/pengine/test10/bundle-order-fencing.dot +++ b/pengine/test10/bundle-order-fencing.dot @@ -1,453 +1,447 @@ digraph "g" { "Cancel redis_monitor_45000 redis-bundle-1" -> "redis_promote_0 redis-bundle-1" [ style = bold] "Cancel redis_monitor_45000 redis-bundle-1" [ style=bold color="green" fontcolor="black"] "Cancel redis_monitor_60000 redis-bundle-1" -> "redis_promote_0 redis-bundle-1" [ style = bold] "Cancel redis_monitor_60000 redis-bundle-1" [ style=bold color="green" fontcolor="black"] "all_stopped" [ style=bold color="green" fontcolor="orange"] "galera-bundle-0_monitor_0 controller-1" -> "galera-bundle-0_start_0 controller-2" [ style = dashed] "galera-bundle-0_monitor_0 controller-1" [ style=bold color="green" fontcolor="black"] "galera-bundle-0_monitor_0 controller-2" -> "galera-bundle-0_start_0 controller-2" [ style = dashed] "galera-bundle-0_monitor_0 controller-2" [ style=bold color="green" fontcolor="black"] "galera-bundle-0_monitor_60000 controller-2" [ style=dashed color="red" fontcolor="black"] "galera-bundle-0_start_0 controller-2" -> "galera-bundle-0_monitor_60000 controller-2" [ style = dashed] "galera-bundle-0_start_0 controller-2" -> "galera_monitor_20000 galera-bundle-0" [ style = dashed] "galera-bundle-0_start_0 controller-2" -> "galera_monitor_30000 galera-bundle-0" [ style = dashed] "galera-bundle-0_start_0 controller-2" -> "galera_start_0 galera-bundle-0" [ style = dashed] "galera-bundle-0_start_0 controller-2" [ style=dashed color="red" fontcolor="black"] "galera-bundle-0_stop_0 controller-0" -> "all_stopped" [ style = bold] "galera-bundle-0_stop_0 controller-0" -> "galera-bundle-0_start_0 controller-2" [ style = dashed] "galera-bundle-0_stop_0 controller-0" -> "galera-bundle-docker-0_stop_0 controller-0" [ style = bold] "galera-bundle-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] "galera-bundle-1_monitor_0 controller-2" [ style=bold color="green" fontcolor="black"] "galera-bundle-2_monitor_0 controller-1" [ style=bold color="green" fontcolor="black"] "galera-bundle-docker-0_stop_0 controller-0" -> "all_stopped" [ style = bold] "galera-bundle-docker-0_stop_0 controller-0" -> "galera-bundle_stopped_0" [ style = bold] "galera-bundle-docker-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] "galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold] "galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold] "galera-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle-master_demoted_0" -> "galera-bundle-master_start_0" [ style = bold] "galera-bundle-master_demoted_0" -> "galera-bundle-master_stop_0" [ style = bold] "galera-bundle-master_demoted_0" -> "galera-bundle_demoted_0" [ style = bold] "galera-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold] "galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold] "galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-0" [ style = dashed] "galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle-master_stop_0" -> "galera-bundle-master_stopped_0" [ style = bold] "galera-bundle-master_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] "galera-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle-master_stopped_0" -> "galera-bundle-master_start_0" [ style = bold] "galera-bundle-master_stopped_0" -> "galera-bundle_stopped_0" [ style = bold] "galera-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle_demote_0" -> "galera-bundle-master_demote_0" [ style = bold] "galera-bundle_demote_0" -> "galera-bundle_demoted_0" [ style = bold] "galera-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle_demoted_0" -> "galera-bundle_start_0" [ style = bold] "galera-bundle_demoted_0" -> "galera-bundle_stop_0" [ style = bold] "galera-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold] "galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle_stop_0" -> "galera-bundle-docker-0_stop_0 controller-0" [ style = bold] "galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold] "galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] "galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] "galera-bundle_stopped_0" -> "galera-bundle_start_0" [ style = bold] "galera-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] "galera_demote_0 galera-bundle-0" -> "galera-bundle-master_demoted_0" [ style = bold] "galera_demote_0 galera-bundle-0" -> "galera_monitor_20000 galera-bundle-0" [ style = dashed] "galera_demote_0 galera-bundle-0" -> "galera_monitor_30000 galera-bundle-0" [ style = dashed] "galera_demote_0 galera-bundle-0" -> "galera_stop_0 galera-bundle-0" [ style = bold] "galera_demote_0 galera-bundle-0" [ style=bold color="green" fontcolor="orange"] "galera_monitor_20000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] "galera_monitor_30000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] "galera_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = dashed] "galera_start_0 galera-bundle-0" -> "galera_monitor_20000 galera-bundle-0" [ style = dashed] "galera_start_0 galera-bundle-0" -> "galera_monitor_30000 galera-bundle-0" [ style = dashed] "galera_start_0 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] "galera_stop_0 galera-bundle-0" -> "all_stopped" [ style = bold] "galera_stop_0 galera-bundle-0" -> "galera-bundle-master_stopped_0" [ style = bold] "galera_stop_0 galera-bundle-0" -> "galera_start_0 galera-bundle-0" [ style = dashed] "galera_stop_0 galera-bundle-0" [ style=bold color="green" fontcolor="orange"] "haproxy-bundle-docker-0_stop_0 controller-0" -> "all_stopped" [ style = bold] "haproxy-bundle-docker-0_stop_0 controller-0" -> "haproxy-bundle_stopped_0" [ style = bold] "haproxy-bundle-docker-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] "haproxy-bundle_stop_0" -> "haproxy-bundle-docker-0_stop_0 controller-0" [ style = bold] "haproxy-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] "haproxy-bundle_stopped_0" -> "ip-10.0.0.109_stop_0 controller-0" [ style = bold] "haproxy-bundle_stopped_0" -> "ip-172.17.4.11_stop_0 controller-0" [ style = bold] "haproxy-bundle_stopped_0" -> "ip-192.168.24.7_stop_0 controller-0" [ style = bold] "haproxy-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] "ip-10.0.0.109_monitor_10000 controller-1" [ style=bold color="green" fontcolor="black"] "ip-10.0.0.109_start_0 controller-1" -> "ip-10.0.0.109_monitor_10000 controller-1" [ style = bold] "ip-10.0.0.109_start_0 controller-1" [ style=bold color="green" fontcolor="black"] "ip-10.0.0.109_stop_0 controller-0" -> "all_stopped" [ style = bold] "ip-10.0.0.109_stop_0 controller-0" -> "ip-10.0.0.109_start_0 controller-1" [ style = bold] "ip-10.0.0.109_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] "ip-172.17.4.11_monitor_10000 controller-1" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_start_0 controller-1" -> "ip-172.17.4.11_monitor_10000 controller-1" [ style = bold] "ip-172.17.4.11_start_0 controller-1" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_stop_0 controller-0" -> "all_stopped" [ style = bold] "ip-172.17.4.11_stop_0 controller-0" -> "ip-172.17.4.11_start_0 controller-1" [ style = bold] "ip-172.17.4.11_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] "ip-192.168.24.7_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-192.168.24.7_start_0 controller-2" -> "ip-192.168.24.7_monitor_10000 controller-2" [ style = bold] "ip-192.168.24.7_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-192.168.24.7_stop_0 controller-0" -> "all_stopped" [ style = bold] "ip-192.168.24.7_stop_0 controller-0" -> "ip-192.168.24.7_start_0 controller-2" [ style = bold] "ip-192.168.24.7_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-0_monitor_0 controller-1" -> "rabbitmq-bundle-0_start_0 controller-1" [ style = dashed] "rabbitmq-bundle-0_monitor_0 controller-1" [ style=bold color="green" fontcolor="black"] "rabbitmq-bundle-0_monitor_0 controller-2" -> "rabbitmq-bundle-0_start_0 controller-1" [ style = dashed] "rabbitmq-bundle-0_monitor_0 controller-2" [ style=bold color="green" fontcolor="black"] "rabbitmq-bundle-0_monitor_60000 controller-1" [ style=dashed color="red" fontcolor="black"] "rabbitmq-bundle-0_start_0 controller-1" -> "rabbitmq-bundle-0_monitor_60000 controller-1" [ style = dashed] "rabbitmq-bundle-0_start_0 controller-1" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] "rabbitmq-bundle-0_start_0 controller-1" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] "rabbitmq-bundle-0_start_0 controller-1" [ style=dashed color="red" fontcolor="black"] "rabbitmq-bundle-0_stop_0 controller-0" -> "all_stopped" [ style = bold] "rabbitmq-bundle-0_stop_0 controller-0" -> "rabbitmq-bundle-0_start_0 controller-1" [ style = dashed] "rabbitmq-bundle-0_stop_0 controller-0" -> "rabbitmq-bundle-docker-0_stop_0 controller-0" [ style = bold] "rabbitmq-bundle-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-1_monitor_0 controller-2" [ style=bold color="green" fontcolor="black"] "rabbitmq-bundle-2_monitor_0 controller-1" [ style=bold color="green" fontcolor="black"] "rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] "rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold] "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "rabbitmq-bundle-clone_pre_notify_start_0" [ style = bold] "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "rabbitmq-bundle_stopped_0" [ style = bold] "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-bundle-clone_post_notify_stopped_0" [ style = bold] "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] "rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq_post_notify_running_0 rabbitmq-bundle-0" [ style = bold] "rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq_post_notify_running_0 rabbitmq-bundle-1" [ style = bold] "rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq_post_notify_running_0 rabbitmq-bundle-2" [ style = bold] "rabbitmq-bundle-clone_post_notify_running_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_post_notify_stopped_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style = bold] "rabbitmq-bundle-clone_post_notify_stopped_0" -> "rabbitmq_post_notify_stonith_0 rabbitmq-bundle-1" [ style = bold] "rabbitmq-bundle-clone_post_notify_stopped_0" -> "rabbitmq_post_notify_stonith_0 rabbitmq-bundle-2" [ style = bold] "rabbitmq-bundle-clone_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = bold] "rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq_pre_notify_start_0 rabbitmq-bundle-1" [ style = bold] "rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq_pre_notify_start_0 rabbitmq-bundle-2" [ style = bold] "rabbitmq-bundle-clone_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold] "rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-1" [ style = bold] "rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-2" [ style = bold] "rabbitmq-bundle-clone_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] "rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] "rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] "rabbitmq-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_stop_0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold] "rabbitmq-bundle-clone_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] "rabbitmq-bundle-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_post_notify_stopped_0" [ style = bold] "rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] "rabbitmq-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle-docker-0_stop_0 controller-0" -> "all_stopped" [ style = bold] "rabbitmq-bundle-docker-0_stop_0 controller-0" -> "rabbitmq-bundle_stopped_0" [ style = bold] "rabbitmq-bundle-docker-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] "rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] "rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-docker-0_stop_0 controller-0" [ style = bold] "rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] "rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_confirmed-post_notify_stonith_0" -> "all_stopped" [ style = bold] "rabbitmq_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] "rabbitmq_post_notify_running_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] "rabbitmq_post_notify_running_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_running_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] "rabbitmq_post_notify_running_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_running_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] "rabbitmq_post_notify_running_0 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_stonith_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style = bold] "rabbitmq_post_notify_stonith_0 rabbitmq-bundle-1" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_stonith_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style = bold] "rabbitmq_post_notify_stonith_0 rabbitmq-bundle-2" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 rabbitmq-bundle-1" [ style = bold] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 rabbitmq-bundle-2" [ style = bold] "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_pre_notify_start_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = bold] "rabbitmq_pre_notify_start_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] "rabbitmq_pre_notify_start_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = bold] "rabbitmq_pre_notify_start_0 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold] "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold] "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] "rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] "rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] "rabbitmq_start_0 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] "rabbitmq_stop_0 rabbitmq-bundle-0" -> "all_stopped" [ style = bold] "rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold] "rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] "rabbitmq_stop_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-0_monitor_0 controller-1" -> "redis-bundle-0_start_0 controller-1" [ style = dashed] "redis-bundle-0_monitor_0 controller-1" [ style=bold color="green" fontcolor="black"] "redis-bundle-0_monitor_0 controller-2" -> "redis-bundle-0_start_0 controller-1" [ style = dashed] "redis-bundle-0_monitor_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis-bundle-0_monitor_60000 controller-1" [ style=dashed color="red" fontcolor="black"] "redis-bundle-0_start_0 controller-1" -> "redis-bundle-0_monitor_60000 controller-1" [ style = dashed] "redis-bundle-0_start_0 controller-1" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] "redis-bundle-0_start_0 controller-1" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] "redis-bundle-0_start_0 controller-1" -> "redis_start_0 redis-bundle-0" [ style = dashed] "redis-bundle-0_start_0 controller-1" [ style=dashed color="red" fontcolor="black"] "redis-bundle-0_stop_0 controller-0" -> "all_stopped" [ style = bold] "redis-bundle-0_stop_0 controller-0" -> "redis-bundle-0_start_0 controller-1" [ style = dashed] "redis-bundle-0_stop_0 controller-0" -> "redis-bundle-docker-0_stop_0 controller-0" [ style = bold] "redis-bundle-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-1_monitor_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis-bundle-2_monitor_0 controller-1" [ style=bold color="green" fontcolor="black"] "redis-bundle-docker-0_stop_0 controller-0" -> "all_stopped" [ style = bold] "redis-bundle-docker-0_stop_0 controller-0" -> "redis-bundle_stopped_0" [ style = bold] "redis-bundle-docker-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis-bundle-master_pre_notify_promote_0" [ style = bold] "redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis-bundle-master_pre_notify_start_0" [ style = bold] "redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis-bundle-master_pre_notify_stop_0" [ style = bold] "redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis-bundle_demoted_0" [ style = bold] "redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis_monitor_20000 redis-bundle-1" [ style = bold] "redis-bundle-master_confirmed-post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis-bundle_promoted_0" [ style = bold] "redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_20000 redis-bundle-1" [ style = bold] "redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] "redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] "redis-bundle-master_confirmed-post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle-master_pre_notify_promote_0" [ style = bold] "redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle_running_0" [ style = bold] "redis-bundle-master_confirmed-post_notify_running_0" -> "redis_monitor_20000 redis-bundle-1" [ style = bold] "redis-bundle-master_confirmed-post_notify_running_0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] "redis-bundle-master_confirmed-post_notify_running_0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] "redis-bundle-master_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold] "redis-bundle-master_confirmed-post_notify_stopped_0" -> "redis-bundle-master_pre_notify_promote_0" [ style = bold] "redis-bundle-master_confirmed-post_notify_stopped_0" -> "redis-bundle-master_pre_notify_start_0" [ style = bold] "redis-bundle-master_confirmed-post_notify_stopped_0" -> "redis-bundle_stopped_0" [ style = bold] "redis-bundle-master_confirmed-post_notify_stopped_0" -> "redis_monitor_20000 redis-bundle-1" [ style = bold] "redis-bundle-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_confirmed-pre_notify_demote_0" -> "redis-bundle-master_demote_0" [ style = bold] "redis-bundle-master_confirmed-pre_notify_demote_0" -> "redis-bundle-master_post_notify_demoted_0" [ style = bold] "redis-bundle-master_confirmed-pre_notify_demote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold] "redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] "redis-bundle-master_confirmed-pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_post_notify_running_0" [ style = bold] "redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_start_0" [ style = bold] "redis-bundle-master_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_confirmed-pre_notify_stop_0" -> "redis-bundle-master_post_notify_stopped_0" [ style = bold] "redis-bundle-master_confirmed-pre_notify_stop_0" -> "redis-bundle-master_stop_0" [ style = bold] "redis-bundle-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold] "redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold] "redis-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_demoted_0" -> "redis-bundle-master_post_notify_demoted_0" [ style = bold] "redis-bundle-master_demoted_0" -> "redis-bundle-master_promote_0" [ style = bold] "redis-bundle-master_demoted_0" -> "redis-bundle-master_start_0" [ style = bold] "redis-bundle-master_demoted_0" -> "redis-bundle-master_stop_0" [ style = bold] "redis-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_post_notify_demoted_0" -> "redis-bundle-master_confirmed-post_notify_demoted_0" [ style = bold] "redis-bundle-master_post_notify_demoted_0" -> "redis_post_notify_demoted_0 redis-bundle-1" [ style = bold] "redis-bundle-master_post_notify_demoted_0" -> "redis_post_notify_demoted_0 redis-bundle-2" [ style = bold] "redis-bundle-master_post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_post_notify_promoted_0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] "redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-0" [ style = bold] "redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-1" [ style = bold] "redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-2" [ style = bold] "redis-bundle-master_post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_post_notify_running_0" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] "redis-bundle-master_post_notify_running_0" -> "redis_post_notify_running_0 redis-bundle-0" [ style = bold] "redis-bundle-master_post_notify_running_0" -> "redis_post_notify_running_0 redis-bundle-1" [ style = bold] "redis-bundle-master_post_notify_running_0" -> "redis_post_notify_running_0 redis-bundle-2" [ style = bold] "redis-bundle-master_post_notify_running_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_post_notify_stopped_0" -> "redis-bundle-master_confirmed-post_notify_stopped_0" [ style = bold] "redis-bundle-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 redis-bundle-1" [ style = bold] "redis-bundle-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 redis-bundle-2" [ style = bold] "redis-bundle-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_pre_notify_demote_0" -> "redis-bundle-master_confirmed-pre_notify_demote_0" [ style = bold] "redis-bundle-master_pre_notify_demote_0" -> "redis_pre_notify_demote_0 redis-bundle-1" [ style = bold] "redis-bundle-master_pre_notify_demote_0" -> "redis_pre_notify_demote_0 redis-bundle-2" [ style = bold] "redis-bundle-master_pre_notify_demote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_pre_notify_promote_0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] "redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-0" [ style = bold] "redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-1" [ style = bold] "redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-2" [ style = bold] "redis-bundle-master_pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_pre_notify_start_0" -> "redis-bundle-master_confirmed-pre_notify_start_0" [ style = bold] "redis-bundle-master_pre_notify_start_0" -> "redis_pre_notify_start_0 redis-bundle-1" [ style = bold] "redis-bundle-master_pre_notify_start_0" -> "redis_pre_notify_start_0 redis-bundle-2" [ style = bold] "redis-bundle-master_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_pre_notify_stop_0" -> "redis-bundle-master_confirmed-pre_notify_stop_0" [ style = bold] "redis-bundle-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 redis-bundle-1" [ style = bold] "redis-bundle-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 redis-bundle-2" [ style = bold] "redis-bundle-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_promote_0" -> "redis_promote_0 redis-bundle-1" [ style = bold] "redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_promoted_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold] "redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_running_0" -> "redis-bundle-master_post_notify_running_0" [ style = bold] "redis-bundle-master_running_0" -> "redis-bundle-master_promote_0" [ style = bold] "redis-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = bold] "redis-bundle-master_start_0" -> "redis_start_0 redis-bundle-0" [ style = dashed] "redis-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_stop_0" -> "redis-bundle-master_stopped_0" [ style = bold] "redis-bundle-master_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold] "redis-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle-master_stopped_0" -> "redis-bundle-master_post_notify_stopped_0" [ style = bold] "redis-bundle-master_stopped_0" -> "redis-bundle-master_promote_0" [ style = bold] "redis-bundle-master_stopped_0" -> "redis-bundle-master_start_0" [ style = bold] "redis-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle_demote_0" -> "redis-bundle-master_demote_0" [ style = bold] "redis-bundle_demote_0" -> "redis-bundle_demoted_0" [ style = bold] "redis-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle_demoted_0" -> "redis-bundle_promote_0" [ style = bold] "redis-bundle_demoted_0" -> "redis-bundle_start_0" [ style = bold] "redis-bundle_demoted_0" -> "redis-bundle_stop_0" [ style = bold] "redis-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] "redis-bundle_promote_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle_running_0" -> "redis-bundle_promote_0" [ style = bold] "redis-bundle_running_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = bold] "redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle_stop_0" -> "redis-bundle-docker-0_stop_0 controller-0" [ style = bold] "redis-bundle_stop_0" -> "redis-bundle-master_stop_0" [ style = bold] "redis-bundle_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold] "redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-bundle_stopped_0" -> "redis-bundle_promote_0" [ style = bold] "redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = bold] "redis-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis_confirmed-post_notify_stonith_0" -> "all_stopped" [ style = bold] "redis_confirmed-post_notify_stonith_0" -> "redis_monitor_20000 redis-bundle-1" [ style = bold] "redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_demote_0 redis-bundle-0" -> "redis-bundle-master_demoted_0" [ style = bold] "redis_demote_0 redis-bundle-0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] "redis_demote_0 redis-bundle-0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] "redis_demote_0 redis-bundle-0" -> "redis_stop_0 redis-bundle-0" [ style = bold] "redis_demote_0 redis-bundle-0" [ style=bold color="green" fontcolor="orange"] "redis_monitor_20000 redis-bundle-1" [ style=bold color="green" fontcolor="black"] "redis_monitor_45000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] "redis_monitor_60000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] "redis_post_notify_demoted_0 redis-bundle-1" -> "redis-bundle-master_confirmed-post_notify_demoted_0" [ style = bold] "redis_post_notify_demoted_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] "redis_post_notify_demoted_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_demoted_0" [ style = bold] "redis_post_notify_demoted_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] "redis_post_notify_promoted_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] "redis_post_notify_promoted_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] "redis_post_notify_promoted_0 redis-bundle-1" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] "redis_post_notify_promoted_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] "redis_post_notify_promoted_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] "redis_post_notify_promoted_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] "redis_post_notify_running_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] "redis_post_notify_running_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] "redis_post_notify_running_0 redis-bundle-1" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] "redis_post_notify_running_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] "redis_post_notify_running_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] "redis_post_notify_running_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0 redis-bundle-1" -> "redis-bundle-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 redis-bundle-1" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 redis-bundle-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 redis-bundle-1" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 redis-bundle-2" [ style = bold] "redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_pre_notify_demote_0 redis-bundle-1" -> "redis-bundle-master_confirmed-pre_notify_demote_0" [ style = bold] "redis_pre_notify_demote_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_demote_0 redis-bundle-2" -> "redis-bundle-master_confirmed-pre_notify_demote_0" [ style = bold] "redis_pre_notify_demote_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_promote_0 redis-bundle-0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] "redis_pre_notify_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_promote_0 redis-bundle-1" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] "redis_pre_notify_promote_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_promote_0 redis-bundle-2" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] "redis_pre_notify_promote_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_start_0 redis-bundle-1" -> "redis-bundle-master_confirmed-pre_notify_start_0" [ style = bold] "redis_pre_notify_start_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_start_0 redis-bundle-2" -> "redis-bundle-master_confirmed-pre_notify_start_0" [ style = bold] "redis_pre_notify_start_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_stop_0 redis-bundle-1" -> "redis-bundle-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_stop_0 redis-bundle-2" -> "redis-bundle-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] "redis_promote_0 redis-bundle-1" -> "redis-bundle-master_promoted_0" [ style = bold] "redis_promote_0 redis-bundle-1" -> "redis_monitor_20000 redis-bundle-1" [ style = bold] "redis_promote_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] "redis_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = dashed] "redis_start_0 redis-bundle-0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] "redis_start_0 redis-bundle-0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] "redis_start_0 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] "redis_stop_0 redis-bundle-0" -> "all_stopped" [ style = bold] "redis_stop_0 redis-bundle-0" -> "redis-bundle-master_stopped_0" [ style = bold] "redis_stop_0 redis-bundle-0" -> "redis_start_0 redis-bundle-0" [ style = dashed] "redis_stop_0 redis-bundle-0" [ style=bold color="green" fontcolor="orange"] "stonith 'off' galera-bundle-0" -> "galera-bundle-master_stop_0" [ style = bold] "stonith 'off' galera-bundle-0" -> "stonith_complete" [ style = bold] "stonith 'off' galera-bundle-0" [ style=bold color="green" fontcolor="orange"] "stonith 'off' rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] "stonith 'off' rabbitmq-bundle-0" -> "rabbitmq_post_notify_stonith_0" [ style = bold] "stonith 'off' rabbitmq-bundle-0" -> "stonith_complete" [ style = bold] "stonith 'off' rabbitmq-bundle-0" [ style=bold color="green" fontcolor="orange"] "stonith 'off' redis-bundle-0" -> "redis-bundle-master_stop_0" [ style = bold] "stonith 'off' redis-bundle-0" -> "redis_post_notify_stonith_0" [ style = bold] "stonith 'off' redis-bundle-0" -> "stonith_complete" [ style = bold] "stonith 'off' redis-bundle-0" [ style=bold color="green" fontcolor="orange"] -"stonith 'reboot' controller-0" -> "galera-bundle-0_stop_0 controller-0" [ style = bold] "stonith 'reboot' controller-0" -> "galera-bundle-docker-0_stop_0 controller-0" [ style = bold] "stonith 'reboot' controller-0" -> "haproxy-bundle-docker-0_stop_0 controller-0" [ style = bold] "stonith 'reboot' controller-0" -> "ip-10.0.0.109_stop_0 controller-0" [ style = bold] "stonith 'reboot' controller-0" -> "ip-172.17.4.11_stop_0 controller-0" [ style = bold] "stonith 'reboot' controller-0" -> "ip-192.168.24.7_stop_0 controller-0" [ style = bold] -"stonith 'reboot' controller-0" -> "rabbitmq-bundle-0_stop_0 controller-0" [ style = bold] "stonith 'reboot' controller-0" -> "rabbitmq-bundle-docker-0_stop_0 controller-0" [ style = bold] -"stonith 'reboot' controller-0" -> "redis-bundle-0_stop_0 controller-0" [ style = bold] "stonith 'reboot' controller-0" -> "redis-bundle-docker-0_stop_0 controller-0" [ style = bold] "stonith 'reboot' controller-0" -> "stonith 'off' galera-bundle-0" [ style = bold] "stonith 'reboot' controller-0" -> "stonith 'off' rabbitmq-bundle-0" [ style = bold] "stonith 'reboot' controller-0" -> "stonith 'off' redis-bundle-0" [ style = bold] "stonith 'reboot' controller-0" -> "stonith_complete" [ style = bold] "stonith 'reboot' controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254000dcb3f_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254000dcb3f_start_0 controller-2" -> "stonith-fence_ipmilan-5254000dcb3f_monitor_60000 controller-2" [ style = bold] "stonith-fence_ipmilan-5254000dcb3f_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254000dcb3f_stop_0 controller-0" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-5254000dcb3f_stop_0 controller-0" -> "stonith-fence_ipmilan-5254000dcb3f_start_0 controller-2" [ style = bold] "stonith-fence_ipmilan-5254000dcb3f_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] "stonith-fence_ipmilan-5254003e8e97_monitor_60000 controller-1" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254003e8e97_start_0 controller-1" -> "stonith-fence_ipmilan-5254003e8e97_monitor_60000 controller-1" [ style = bold] "stonith-fence_ipmilan-5254003e8e97_start_0 controller-1" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254003e8e97_stop_0 controller-0" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-5254003e8e97_stop_0 controller-0" -> "stonith-fence_ipmilan-5254003e8e97_start_0 controller-1" [ style = bold] "stonith-fence_ipmilan-5254003e8e97_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] "stonith_complete" -> "all_stopped" [ style = bold] -"stonith_complete" -> "galera-bundle-0_start_0 controller-2" [ style = dashed] "stonith_complete" -> "galera_start_0 galera-bundle-0" [ style = dashed] "stonith_complete" -> "ip-10.0.0.109_start_0 controller-1" [ style = bold] "stonith_complete" -> "ip-172.17.4.11_start_0 controller-1" [ style = bold] "stonith_complete" -> "ip-192.168.24.7_start_0 controller-2" [ style = bold] -"stonith_complete" -> "rabbitmq-bundle-0_start_0 controller-1" [ style = dashed] "stonith_complete" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] -"stonith_complete" -> "redis-bundle-0_start_0 controller-1" [ style = dashed] "stonith_complete" -> "redis_promote_0 redis-bundle-1" [ style = bold] "stonith_complete" -> "redis_start_0 redis-bundle-0" [ style = dashed] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/bundle-order-fencing.exp b/pengine/test10/bundle-order-fencing.exp index 708815fef1..dc4c5c9910 100644 --- a/pengine/test10/bundle-order-fencing.exp +++ b/pengine/test10/bundle-order-fencing.exp @@ -1,2069 +1,2057 @@ - - - - - + - - - - - + - - - - - + diff --git a/pengine/test10/bundle-order-fencing.summary b/pengine/test10/bundle-order-fencing.summary index ee2c361240..0457f833ba 100644 --- a/pengine/test10/bundle-order-fencing.summary +++ b/pengine/test10/bundle-order-fencing.summary @@ -1,234 +1,234 @@ Using the original execution date of: 2017-09-12 10:51:59Z Current cluster status: Node controller-0 (1): UNCLEAN (offline) Online: [ controller-1 controller-2 ] Containers: [ galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ] Docker container set: rabbitmq-bundle [192.168.24.1:8787/rhosp12/openstack-rabbitmq-docker:pcmklatest] rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): FAILED controller-0 (UNCLEAN) rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Started controller-1 rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Started controller-2 Docker container set: galera-bundle [192.168.24.1:8787/rhosp12/openstack-mariadb-docker:pcmklatest] galera-bundle-0 (ocf::heartbeat:galera): FAILED Master controller-0 (UNCLEAN) galera-bundle-1 (ocf::heartbeat:galera): Master controller-1 galera-bundle-2 (ocf::heartbeat:galera): Master controller-2 Docker container set: redis-bundle [192.168.24.1:8787/rhosp12/openstack-redis-docker:pcmklatest] redis-bundle-0 (ocf::heartbeat:redis): FAILED Master controller-0 (UNCLEAN) redis-bundle-1 (ocf::heartbeat:redis): Slave controller-1 redis-bundle-2 (ocf::heartbeat:redis): Slave controller-2 ip-192.168.24.7 (ocf::heartbeat:IPaddr2): Started controller-0 (UNCLEAN) ip-10.0.0.109 (ocf::heartbeat:IPaddr2): Started controller-0 (UNCLEAN) ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.1.19 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.3.19 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-0 (UNCLEAN) Docker container set: haproxy-bundle [192.168.24.1:8787/rhosp12/openstack-haproxy-docker:pcmklatest] haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started controller-0 (UNCLEAN) haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started controller-2 haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started controller-1 openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-2 stonith-fence_ipmilan-525400efba5c (stonith:fence_ipmilan): Started controller-2 stonith-fence_ipmilan-5254003e8e97 (stonith:fence_ipmilan): Started controller-0 (UNCLEAN) stonith-fence_ipmilan-5254000dcb3f (stonith:fence_ipmilan): Started controller-0 (UNCLEAN) Transition Summary: * Fence (off) redis-bundle-0 (resource: redis-bundle-docker-0) 'guest is unclean' * Fence (off) rabbitmq-bundle-0 (resource: rabbitmq-bundle-docker-0) 'guest is unclean' * Fence (off) galera-bundle-0 (resource: galera-bundle-docker-0) 'guest is unclean' * Fence (reboot) controller-0 'peer is no longer part of the cluster' * Stop rabbitmq-bundle-docker-0 ( controller-0 ) due to node availability * Stop rabbitmq-bundle-0 ( controller-0 ) due to unrunnable rabbitmq-bundle-docker-0 start * Stop rabbitmq:0 ( rabbitmq-bundle-0 ) due to unrunnable rabbitmq-bundle-docker-0 start * Stop galera-bundle-docker-0 ( controller-0 ) due to node availability * Stop galera-bundle-0 ( controller-0 ) due to unrunnable galera-bundle-docker-0 start * Stop galera:0 ( Master galera-bundle-0 ) due to unrunnable galera-bundle-docker-0 start * Stop redis-bundle-docker-0 ( controller-0 ) due to node availability * Stop redis-bundle-0 ( controller-0 ) due to unrunnable redis-bundle-docker-0 start * Stop redis:0 ( Master redis-bundle-0 ) due to unrunnable redis-bundle-docker-0 start * Promote redis:1 ( Slave -> Master redis-bundle-1 ) * Move ip-192.168.24.7 ( controller-0 -> controller-2 ) * Move ip-10.0.0.109 ( controller-0 -> controller-1 ) * Move ip-172.17.4.11 ( controller-0 -> controller-1 ) * Stop haproxy-bundle-docker-0 ( controller-0 ) due to node availability * Move stonith-fence_ipmilan-5254003e8e97 ( controller-0 -> controller-1 ) * Move stonith-fence_ipmilan-5254000dcb3f ( controller-0 -> controller-2 ) Executing cluster transition: * Pseudo action: rabbitmq-bundle-clone_pre_notify_stop_0 + * Pseudo action: rabbitmq-bundle-0_stop_0 * Resource action: rabbitmq-bundle-0 monitor on controller-2 * Resource action: rabbitmq-bundle-0 monitor on controller-1 * Resource action: rabbitmq-bundle-1 monitor on controller-2 * Resource action: rabbitmq-bundle-2 monitor on controller-1 + * Pseudo action: galera-bundle-0_stop_0 * Resource action: galera-bundle-0 monitor on controller-2 * Resource action: galera-bundle-0 monitor on controller-1 * Resource action: galera-bundle-1 monitor on controller-2 * Resource action: galera-bundle-2 monitor on controller-1 * Resource action: redis cancel=45000 on redis-bundle-1 * Resource action: redis cancel=60000 on redis-bundle-1 * Pseudo action: redis-bundle-master_pre_notify_demote_0 + * Pseudo action: redis-bundle-0_stop_0 * Resource action: redis-bundle-0 monitor on controller-2 * Resource action: redis-bundle-0 monitor on controller-1 * Resource action: redis-bundle-1 monitor on controller-2 * Resource action: redis-bundle-2 monitor on controller-1 * Pseudo action: stonith-fence_ipmilan-5254003e8e97_stop_0 * Pseudo action: stonith-fence_ipmilan-5254000dcb3f_stop_0 * Pseudo action: haproxy-bundle_stop_0 * Pseudo action: redis-bundle_demote_0 * Pseudo action: galera-bundle_demote_0 * Pseudo action: rabbitmq-bundle_stop_0 * Pseudo action: rabbitmq-bundle_start_0 * Fencing controller-0 (reboot) * Resource action: rabbitmq notify on rabbitmq-bundle-1 * Resource action: rabbitmq notify on rabbitmq-bundle-2 * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_stop_0 - * Pseudo action: rabbitmq-bundle-0_stop_0 + * Pseudo action: rabbitmq-bundle-docker-0_stop_0 * Pseudo action: galera-bundle-master_demote_0 - * Pseudo action: galera-bundle-0_stop_0 * Resource action: redis notify on redis-bundle-1 * Resource action: redis notify on redis-bundle-2 * Pseudo action: redis-bundle-master_confirmed-pre_notify_demote_0 * Pseudo action: redis-bundle-master_demote_0 - * Pseudo action: redis-bundle-0_stop_0 * Pseudo action: haproxy-bundle-docker-0_stop_0 * Resource action: stonith-fence_ipmilan-5254003e8e97 start on controller-1 * Resource action: stonith-fence_ipmilan-5254000dcb3f start on controller-2 * Pseudo action: stonith-redis-bundle-0-off on redis-bundle-0 * Pseudo action: stonith-rabbitmq-bundle-0-off on rabbitmq-bundle-0 * Pseudo action: stonith-galera-bundle-0-off on galera-bundle-0 * Pseudo action: stonith_complete * Pseudo action: haproxy-bundle_stopped_0 * Pseudo action: rabbitmq_post_notify_stop_0 * Pseudo action: rabbitmq-bundle-clone_stop_0 - * Pseudo action: rabbitmq-bundle-docker-0_stop_0 * Pseudo action: galera_demote_0 * Pseudo action: galera-bundle-master_demoted_0 * Pseudo action: redis_post_notify_stop_0 * Pseudo action: redis_demote_0 * Pseudo action: redis-bundle-master_demoted_0 * Pseudo action: ip-192.168.24.7_stop_0 * Pseudo action: ip-10.0.0.109_stop_0 * Pseudo action: ip-172.17.4.11_stop_0 * Resource action: stonith-fence_ipmilan-5254003e8e97 monitor=60000 on controller-1 * Resource action: stonith-fence_ipmilan-5254000dcb3f monitor=60000 on controller-2 * Pseudo action: galera-bundle_demoted_0 * Pseudo action: galera-bundle_stop_0 * Pseudo action: rabbitmq_stop_0 * Pseudo action: rabbitmq-bundle-clone_stopped_0 * Pseudo action: galera-bundle-master_stop_0 * Pseudo action: galera-bundle-docker-0_stop_0 * Pseudo action: redis-bundle-master_post_notify_demoted_0 * Resource action: ip-192.168.24.7 start on controller-2 * Resource action: ip-10.0.0.109 start on controller-1 * Resource action: ip-172.17.4.11 start on controller-1 * Pseudo action: rabbitmq-bundle-clone_post_notify_stopped_0 * Pseudo action: galera_stop_0 * Pseudo action: galera-bundle-master_stopped_0 * Resource action: redis notify on redis-bundle-1 * Resource action: redis notify on redis-bundle-2 * Pseudo action: redis-bundle-master_confirmed-post_notify_demoted_0 * Pseudo action: redis-bundle-master_pre_notify_stop_0 * Resource action: ip-192.168.24.7 monitor=10000 on controller-2 * Resource action: ip-10.0.0.109 monitor=10000 on controller-1 * Resource action: ip-172.17.4.11 monitor=10000 on controller-1 * Pseudo action: redis-bundle_demoted_0 * Pseudo action: redis-bundle_stop_0 * Pseudo action: galera-bundle_stopped_0 * Pseudo action: galera-bundle_start_0 * Resource action: rabbitmq notify on rabbitmq-bundle-1 * Resource action: rabbitmq notify on rabbitmq-bundle-2 * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_stopped_0 * Pseudo action: rabbitmq-bundle-clone_pre_notify_start_0 * Pseudo action: galera-bundle-master_start_0 * Resource action: redis notify on redis-bundle-1 * Resource action: redis notify on redis-bundle-2 * Pseudo action: redis-bundle-master_confirmed-pre_notify_stop_0 * Pseudo action: redis-bundle-master_stop_0 * Pseudo action: redis-bundle-docker-0_stop_0 * Pseudo action: rabbitmq-bundle_stopped_0 * Pseudo action: rabbitmq_notified_0 * Resource action: rabbitmq notify on rabbitmq-bundle-1 * Resource action: rabbitmq notify on rabbitmq-bundle-2 * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_start_0 * Pseudo action: rabbitmq-bundle-clone_start_0 * Pseudo action: galera-bundle-master_running_0 * Pseudo action: redis_stop_0 * Pseudo action: redis-bundle-master_stopped_0 * Pseudo action: galera-bundle_running_0 * Pseudo action: rabbitmq-bundle-clone_running_0 * Pseudo action: redis-bundle-master_post_notify_stopped_0 * Pseudo action: rabbitmq-bundle-clone_post_notify_running_0 * Resource action: redis notify on redis-bundle-1 * Resource action: redis notify on redis-bundle-2 * Pseudo action: redis-bundle-master_confirmed-post_notify_stopped_0 * Pseudo action: redis-bundle-master_pre_notify_start_0 * Pseudo action: redis-bundle_stopped_0 * Pseudo action: redis-bundle_start_0 * Resource action: rabbitmq notify on rabbitmq-bundle-0 * Resource action: rabbitmq notify on rabbitmq-bundle-1 * Resource action: rabbitmq notify on rabbitmq-bundle-2 * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_running_0 * Pseudo action: redis_notified_0 * Resource action: redis notify on redis-bundle-1 * Resource action: redis notify on redis-bundle-2 * Pseudo action: redis-bundle-master_confirmed-pre_notify_start_0 * Pseudo action: redis-bundle-master_start_0 * Pseudo action: rabbitmq-bundle_running_0 * Pseudo action: all_stopped * Pseudo action: redis-bundle-master_running_0 * Pseudo action: redis-bundle-master_post_notify_running_0 * Resource action: redis notify on redis-bundle-0 * Resource action: redis notify on redis-bundle-1 * Resource action: redis notify on redis-bundle-2 * Pseudo action: redis-bundle-master_confirmed-post_notify_running_0 * Pseudo action: redis-bundle_running_0 * Pseudo action: redis-bundle-master_pre_notify_promote_0 * Pseudo action: redis-bundle_promote_0 * Resource action: redis notify on redis-bundle-0 * Resource action: redis notify on redis-bundle-1 * Resource action: redis notify on redis-bundle-2 * Pseudo action: redis-bundle-master_confirmed-pre_notify_promote_0 * Pseudo action: redis-bundle-master_promote_0 * Resource action: redis promote on redis-bundle-1 * Pseudo action: redis-bundle-master_promoted_0 * Pseudo action: redis-bundle-master_post_notify_promoted_0 * Resource action: redis notify on redis-bundle-0 * Resource action: redis notify on redis-bundle-1 * Resource action: redis notify on redis-bundle-2 * Pseudo action: redis-bundle-master_confirmed-post_notify_promoted_0 * Pseudo action: redis-bundle_promoted_0 * Resource action: redis monitor=20000 on redis-bundle-1 Using the original execution date of: 2017-09-12 10:51:59Z Revised cluster status: Online: [ controller-1 controller-2 ] OFFLINE: [ controller-0 ] Containers: [ galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ] Docker container set: rabbitmq-bundle [192.168.24.1:8787/rhosp12/openstack-rabbitmq-docker:pcmklatest] rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): FAILED rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Started controller-1 rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Started controller-2 Docker container set: galera-bundle [192.168.24.1:8787/rhosp12/openstack-mariadb-docker:pcmklatest] galera-bundle-0 (ocf::heartbeat:galera): FAILED Master galera-bundle-1 (ocf::heartbeat:galera): Master controller-1 galera-bundle-2 (ocf::heartbeat:galera): Master controller-2 Docker container set: redis-bundle [192.168.24.1:8787/rhosp12/openstack-redis-docker:pcmklatest] redis-bundle-0 (ocf::heartbeat:redis): FAILED Master redis-bundle-1 (ocf::heartbeat:redis): Master controller-1 redis-bundle-2 (ocf::heartbeat:redis): Slave controller-2 ip-192.168.24.7 (ocf::heartbeat:IPaddr2): Started controller-2 ip-10.0.0.109 (ocf::heartbeat:IPaddr2): Started controller-1 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.1.19 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.3.19 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-1 Docker container set: haproxy-bundle [192.168.24.1:8787/rhosp12/openstack-haproxy-docker:pcmklatest] haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started controller-2 haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started controller-1 openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-2 stonith-fence_ipmilan-525400efba5c (stonith:fence_ipmilan): Started controller-2 stonith-fence_ipmilan-5254003e8e97 (stonith:fence_ipmilan): Started controller-1 stonith-fence_ipmilan-5254000dcb3f (stonith:fence_ipmilan): Started controller-2 diff --git a/pengine/test10/guest-node-host-dies.dot b/pengine/test10/guest-node-host-dies.dot index a85250df74..c50e07127c 100644 --- a/pengine/test10/guest-node-host-dies.dot +++ b/pengine/test10/guest-node-host-dies.dot @@ -1,133 +1,131 @@ digraph "g" { "Fencing_monitor_120000 rhel7-4" [ style=bold color="green" fontcolor="black"] "Fencing_start_0 rhel7-4" -> "Fencing_monitor_120000 rhel7-4" [ style = bold] "Fencing_start_0 rhel7-4" [ style=bold color="green" fontcolor="black"] "Fencing_stop_0 rhel7-4" -> "Fencing_start_0 rhel7-4" [ style = bold] "Fencing_stop_0 rhel7-4" -> "all_stopped" [ style = bold] "Fencing_stop_0 rhel7-4" [ style=bold color="green" fontcolor="black"] "all_stopped" -> "Fencing_start_0 rhel7-4" [ style = bold] +"all_stopped" -> "lxc1_start_0 rhel7-2" [ style = bold] +"all_stopped" -> "lxc2_start_0 rhel7-3" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "container1_start_0 rhel7-2" -> "lxc-ms_promote_0 lxc1" [ style = bold] "container1_start_0 rhel7-2" -> "lxc-ms_start_0 lxc1" [ style = bold] "container1_start_0 rhel7-2" -> "lxc1_start_0 rhel7-2" [ style = bold] "container1_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"] "container1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] "container1_stop_0 rhel7-1" -> "container1_start_0 rhel7-2" [ style = bold] "container1_stop_0 rhel7-1" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] "container1_stop_0 rhel7-1" -> "stonith 'reboot' lxc1" [ style = bold] "container1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] "container2_start_0 rhel7-3" -> "lxc-ms_start_0 lxc2" [ style = bold] "container2_start_0 rhel7-3" -> "lxc2_start_0 rhel7-3" [ style = bold] "container2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"] "container2_stop_0 rhel7-1" -> "all_stopped" [ style = bold] "container2_stop_0 rhel7-1" -> "container2_start_0 rhel7-3" [ style = bold] "container2_stop_0 rhel7-1" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] "container2_stop_0 rhel7-1" -> "stonith 'reboot' lxc2" [ style = bold] "container2_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold] "lxc-ms-master_demote_0" -> "lxc-ms_demote_0 lxc1" [ style = bold] "lxc-ms-master_demote_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_demoted_0" -> "lxc-ms-master_promote_0" [ style = bold] "lxc-ms-master_demoted_0" -> "lxc-ms-master_start_0" [ style = bold] "lxc-ms-master_demoted_0" -> "lxc-ms-master_stop_0" [ style = bold] "lxc-ms-master_demoted_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_promote_0" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc-ms-master_promote_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_promoted_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_running_0" -> "lxc-ms-master_promote_0" [ style = bold] "lxc-ms-master_running_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_start_0" -> "lxc-ms-master_running_0" [ style = bold] "lxc-ms-master_start_0" -> "lxc-ms_start_0 lxc1" [ style = bold] "lxc-ms-master_start_0" -> "lxc-ms_start_0 lxc2" [ style = bold] "lxc-ms-master_start_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_stop_0" -> "lxc-ms-master_stopped_0" [ style = bold] "lxc-ms-master_stop_0" -> "lxc-ms_stop_0 lxc1" [ style = bold] "lxc-ms-master_stop_0" -> "lxc-ms_stop_0 lxc2" [ style = bold] "lxc-ms-master_stop_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_stopped_0" -> "lxc-ms-master_promote_0" [ style = bold] "lxc-ms-master_stopped_0" -> "lxc-ms-master_start_0" [ style = bold] "lxc-ms-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms_demote_0 lxc1" -> "lxc-ms-master_demoted_0" [ style = bold] "lxc-ms_demote_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc-ms_demote_0 lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold] "lxc-ms_demote_0 lxc1" [ style=bold color="green" fontcolor="orange"] "lxc-ms_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"] "lxc-ms_promote_0 lxc1" -> "lxc-ms-master_promoted_0" [ style = bold] "lxc-ms_promote_0 lxc1" [ style=bold color="green" fontcolor="black"] "lxc-ms_start_0 lxc1" -> "lxc-ms-master_running_0" [ style = bold] "lxc-ms_start_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc-ms_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "lxc-ms_start_0 lxc2" -> "lxc-ms-master_running_0" [ style = bold] "lxc-ms_start_0 lxc2" -> "lxc-ms_monitor_10000 lxc2" [ style = bold] "lxc-ms_start_0 lxc2" [ style=bold color="green" fontcolor="black"] "lxc-ms_stop_0 lxc1" -> "all_stopped" [ style = bold] "lxc-ms_stop_0 lxc1" -> "lxc-ms-master_stopped_0" [ style = bold] "lxc-ms_stop_0 lxc1" -> "lxc-ms_start_0 lxc1" [ style = bold] "lxc-ms_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"] "lxc-ms_stop_0 lxc2" -> "all_stopped" [ style = bold] "lxc-ms_stop_0 lxc2" -> "lxc-ms-master_stopped_0" [ style = bold] "lxc-ms_stop_0 lxc2" -> "lxc-ms_start_0 lxc2" [ style = bold] "lxc-ms_stop_0 lxc2" [ style=bold color="green" fontcolor="orange"] "lxc1_monitor_0 rhel7-3" -> "lxc1_start_0 rhel7-2" [ style = bold] "lxc1_monitor_0 rhel7-3" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_0 rhel7-4" -> "lxc1_start_0 rhel7-2" [ style = bold] "lxc1_monitor_0 rhel7-4" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_0 rhel7-5" -> "lxc1_start_0 rhel7-2" [ style = bold] "lxc1_monitor_0 rhel7-5" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_30000 rhel7-2" [ style=bold color="green" fontcolor="black"] "lxc1_start_0 rhel7-2" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc1_start_0 rhel7-2" -> "lxc-ms_start_0 lxc1" [ style = bold] "lxc1_start_0 rhel7-2" -> "lxc1_monitor_30000 rhel7-2" [ style = bold] "lxc1_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"] "lxc1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] "lxc1_stop_0 rhel7-1" -> "container1_stop_0 rhel7-1" [ style = bold] "lxc1_stop_0 rhel7-1" -> "lxc1_start_0 rhel7-2" [ style = bold] "lxc1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] "lxc2_monitor_0 rhel7-2" -> "lxc2_start_0 rhel7-3" [ style = bold] "lxc2_monitor_0 rhel7-2" [ style=bold color="green" fontcolor="black"] "lxc2_monitor_0 rhel7-4" -> "lxc2_start_0 rhel7-3" [ style = bold] "lxc2_monitor_0 rhel7-4" [ style=bold color="green" fontcolor="black"] "lxc2_monitor_0 rhel7-5" -> "lxc2_start_0 rhel7-3" [ style = bold] "lxc2_monitor_0 rhel7-5" [ style=bold color="green" fontcolor="black"] "lxc2_monitor_30000 rhel7-3" [ style=bold color="green" fontcolor="black"] "lxc2_start_0 rhel7-3" -> "lxc-ms_monitor_10000 lxc2" [ style = bold] "lxc2_start_0 rhel7-3" -> "lxc-ms_start_0 lxc2" [ style = bold] "lxc2_start_0 rhel7-3" -> "lxc2_monitor_30000 rhel7-3" [ style = bold] "lxc2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"] "lxc2_stop_0 rhel7-1" -> "all_stopped" [ style = bold] "lxc2_stop_0 rhel7-1" -> "container2_stop_0 rhel7-1" [ style = bold] "lxc2_stop_0 rhel7-1" -> "lxc2_start_0 rhel7-3" [ style = bold] "lxc2_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] "rsc_rhel7-1_monitor_5000 rhel7-5" [ style=bold color="green" fontcolor="black"] "rsc_rhel7-1_start_0 rhel7-5" -> "rsc_rhel7-1_monitor_5000 rhel7-5" [ style = bold] "rsc_rhel7-1_start_0 rhel7-5" [ style=bold color="green" fontcolor="black"] "rsc_rhel7-1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] "rsc_rhel7-1_stop_0 rhel7-1" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] "rsc_rhel7-1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' lxc1" -> "lxc-ms-master_stop_0" [ style = bold] "stonith 'reboot' lxc1" -> "lxc-ms_demote_0 lxc1" [ style = bold] "stonith 'reboot' lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold] "stonith 'reboot' lxc1" -> "stonith_complete" [ style = bold] "stonith 'reboot' lxc1" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' lxc2" -> "lxc-ms-master_stop_0" [ style = bold] "stonith 'reboot' lxc2" -> "lxc-ms_stop_0 lxc2" [ style = bold] "stonith 'reboot' lxc2" -> "stonith_complete" [ style = bold] "stonith 'reboot' lxc2" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' rhel7-1" -> "container1_stop_0 rhel7-1" [ style = bold] "stonith 'reboot' rhel7-1" -> "container2_stop_0 rhel7-1" [ style = bold] -"stonith 'reboot' rhel7-1" -> "lxc1_stop_0 rhel7-1" [ style = bold] -"stonith 'reboot' rhel7-1" -> "lxc2_stop_0 rhel7-1" [ style = bold] "stonith 'reboot' rhel7-1" -> "rsc_rhel7-1_stop_0 rhel7-1" [ style = bold] "stonith 'reboot' rhel7-1" -> "stonith_complete" [ style = bold] "stonith 'reboot' rhel7-1" [ style=bold color="green" fontcolor="black"] "stonith_complete" -> "all_stopped" [ style = bold] "stonith_complete" -> "container1_start_0 rhel7-2" [ style = bold] "stonith_complete" -> "container2_start_0 rhel7-3" [ style = bold] "stonith_complete" -> "lxc-ms_promote_0 lxc1" [ style = bold] "stonith_complete" -> "lxc-ms_start_0 lxc1" [ style = bold] "stonith_complete" -> "lxc-ms_start_0 lxc2" [ style = bold] -"stonith_complete" -> "lxc1_start_0 rhel7-2" [ style = bold] -"stonith_complete" -> "lxc2_start_0 rhel7-3" [ style = bold] "stonith_complete" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/guest-node-host-dies.exp b/pengine/test10/guest-node-host-dies.exp index 8dbadde32b..b5a34ea171 100644 --- a/pengine/test10/guest-node-host-dies.exp +++ b/pengine/test10/guest-node-host-dies.exp @@ -1,670 +1,662 @@ + + + - - - - - - - - + + + + - - - - - - - - + diff --git a/pengine/test10/guest-node-host-dies.summary b/pengine/test10/guest-node-host-dies.summary index 4feee8892f..9813d2b97d 100644 --- a/pengine/test10/guest-node-host-dies.summary +++ b/pengine/test10/guest-node-host-dies.summary @@ -1,82 +1,82 @@ Current cluster status: Node rhel7-1 (1): UNCLEAN (offline) Online: [ rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] Fencing (stonith:fence_xvm): Started rhel7-4 rsc_rhel7-1 (ocf::heartbeat:IPaddr2): Started rhel7-1 ( UNCLEAN ) container1 (ocf::heartbeat:VirtualDomain): FAILED rhel7-1 (UNCLEAN) container2 (ocf::heartbeat:VirtualDomain): FAILED rhel7-1 (UNCLEAN) Master/Slave Set: lxc-ms-master [lxc-ms] Stopped: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] Transition Summary: * Fence (reboot) lxc2 (resource: container2) 'guest is unclean' * Fence (reboot) lxc1 (resource: container1) 'guest is unclean' * Fence (reboot) rhel7-1 'rsc_rhel7-1 is thought to be active there' * Restart Fencing ( rhel7-4 ) due to resource definition change * Move rsc_rhel7-1 ( rhel7-1 -> rhel7-5 ) * Recover container1 ( rhel7-1 -> rhel7-2 ) * Recover container2 ( rhel7-1 -> rhel7-3 ) * Recover lxc-ms:0 (Master lxc1) * Recover lxc-ms:1 (Slave lxc2) * Move lxc1 ( rhel7-1 -> rhel7-2 ) * Move lxc2 ( rhel7-1 -> rhel7-3 ) Executing cluster transition: * Resource action: Fencing stop on rhel7-4 * Pseudo action: lxc-ms-master_demote_0 + * Pseudo action: lxc1_stop_0 * Resource action: lxc1 monitor on rhel7-5 * Resource action: lxc1 monitor on rhel7-4 * Resource action: lxc1 monitor on rhel7-3 + * Pseudo action: lxc2_stop_0 * Resource action: lxc2 monitor on rhel7-5 * Resource action: lxc2 monitor on rhel7-4 * Resource action: lxc2 monitor on rhel7-2 * Fencing rhel7-1 (reboot) * Pseudo action: rsc_rhel7-1_stop_0 - * Pseudo action: lxc1_stop_0 - * Pseudo action: lxc2_stop_0 * Pseudo action: container1_stop_0 * Pseudo action: container2_stop_0 * Pseudo action: stonith-lxc2-reboot on lxc2 * Pseudo action: stonith-lxc1-reboot on lxc1 * Pseudo action: stonith_complete * Resource action: rsc_rhel7-1 start on rhel7-5 * Resource action: container1 start on rhel7-2 * Resource action: container2 start on rhel7-3 * Pseudo action: lxc-ms_demote_0 * Pseudo action: lxc-ms-master_demoted_0 * Pseudo action: lxc-ms-master_stop_0 - * Resource action: lxc1 start on rhel7-2 - * Resource action: lxc2 start on rhel7-3 * Resource action: rsc_rhel7-1 monitor=5000 on rhel7-5 * Pseudo action: lxc-ms_stop_0 * Pseudo action: lxc-ms_stop_0 * Pseudo action: lxc-ms-master_stopped_0 * Pseudo action: lxc-ms-master_start_0 - * Resource action: lxc1 monitor=30000 on rhel7-2 - * Resource action: lxc2 monitor=30000 on rhel7-3 * Pseudo action: all_stopped * Resource action: Fencing start on rhel7-4 * Resource action: Fencing monitor=120000 on rhel7-4 + * Resource action: lxc1 start on rhel7-2 + * Resource action: lxc2 start on rhel7-3 * Resource action: lxc-ms start on lxc1 * Resource action: lxc-ms start on lxc2 * Pseudo action: lxc-ms-master_running_0 + * Resource action: lxc1 monitor=30000 on rhel7-2 + * Resource action: lxc2 monitor=30000 on rhel7-3 * Resource action: lxc-ms monitor=10000 on lxc2 * Pseudo action: lxc-ms-master_promote_0 * Resource action: lxc-ms promote on lxc1 * Pseudo action: lxc-ms-master_promoted_0 Revised cluster status: Online: [ rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] OFFLINE: [ rhel7-1 ] Containers: [ lxc1:container1 lxc2:container2 ] Fencing (stonith:fence_xvm): Started rhel7-4 rsc_rhel7-1 (ocf::heartbeat:IPaddr2): Started rhel7-5 container1 (ocf::heartbeat:VirtualDomain): Started rhel7-2 container2 (ocf::heartbeat:VirtualDomain): Started rhel7-3 Master/Slave Set: lxc-ms-master [lxc-ms] Masters: [ lxc1 ] Slaves: [ lxc2 ] diff --git a/pengine/test10/remote-fence-unclean.dot b/pengine/test10/remote-fence-unclean.dot index b2829a7c85..76a676d322 100644 --- a/pengine/test10/remote-fence-unclean.dot +++ b/pengine/test10/remote-fence-unclean.dot @@ -1,37 +1,37 @@ digraph "g" { "FAKE2_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"] "FAKE2_start_0 18builder" -> "FAKE2_monitor_60000 18builder" [ style = bold] "FAKE2_start_0 18builder" [ style=bold color="green" fontcolor="black"] "FAKE2_stop_0 remote1" -> "FAKE2_start_0 18builder" [ style = bold] "FAKE2_stop_0 remote1" -> "all_stopped" [ style = bold] "FAKE2_stop_0 remote1" -> "remote1_stop_0 18node1" [ style = bold] "FAKE2_stop_0 remote1" [ style=bold color="green" fontcolor="orange"] "FAKE3_monitor_60000 18node1" [ style=bold color="green" fontcolor="black"] "FAKE3_start_0 18node1" -> "FAKE3_monitor_60000 18node1" [ style = bold] "FAKE3_start_0 18node1" [ style=bold color="green" fontcolor="black"] "FAKE3_stop_0 18builder" -> "FAKE3_start_0 18node1" [ style = bold] "FAKE3_stop_0 18builder" -> "all_stopped" [ style = bold] "FAKE3_stop_0 18builder" [ style=bold color="green" fontcolor="black"] "FAKE4_monitor_60000 18node2" [ style=bold color="green" fontcolor="black"] "FAKE4_start_0 18node2" -> "FAKE4_monitor_60000 18node2" [ style = bold] "FAKE4_start_0 18node2" [ style=bold color="green" fontcolor="black"] "FAKE4_stop_0 18node1" -> "FAKE4_start_0 18node2" [ style = bold] "FAKE4_stop_0 18node1" -> "all_stopped" [ style = bold] "FAKE4_stop_0 18node1" [ style=bold color="green" fontcolor="black"] +"all_stopped" -> "remote1_start_0 18node1" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "remote1_monitor_60000 18node1" [ style=bold color="green" fontcolor="black"] "remote1_start_0 18node1" -> "remote1_monitor_60000 18node1" [ style = bold] "remote1_start_0 18node1" [ style=bold color="green" fontcolor="black"] "remote1_stop_0 18node1" -> "all_stopped" [ style = bold] "remote1_stop_0 18node1" -> "remote1_start_0 18node1" [ style = bold] "remote1_stop_0 18node1" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' remote1" -> "FAKE2_stop_0 remote1" [ style = bold] "stonith 'reboot' remote1" -> "stonith_complete" [ style = bold] "stonith 'reboot' remote1" [ style=bold color="green" fontcolor="black"] "stonith_complete" -> "FAKE2_start_0 18builder" [ style = bold] "stonith_complete" -> "FAKE3_start_0 18node1" [ style = bold] "stonith_complete" -> "FAKE4_start_0 18node2" [ style = bold] "stonith_complete" -> "all_stopped" [ style = bold] -"stonith_complete" -> "remote1_start_0 18node1" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/remote-fence-unclean.exp b/pengine/test10/remote-fence-unclean.exp index 3a07384982..f77d7f611c 100644 --- a/pengine/test10/remote-fence-unclean.exp +++ b/pengine/test10/remote-fence-unclean.exp @@ -1,211 +1,211 @@ - + diff --git a/pengine/test10/remote-partial-migrate2.dot b/pengine/test10/remote-partial-migrate2.dot index a8bf29bf9c..17c8bf3f51 100644 --- a/pengine/test10/remote-partial-migrate2.dot +++ b/pengine/test10/remote-partial-migrate2.dot @@ -1,159 +1,155 @@ digraph "g" { "FAKE12_monitor_10000 pcmk2" [ style=bold color="green" fontcolor="black"] "FAKE12_start_0 pcmk2" -> "FAKE12_monitor_10000 pcmk2" [ style = bold] "FAKE12_start_0 pcmk2" [ style=bold color="green" fontcolor="black"] "FAKE12_stop_0 pcmk1" -> "FAKE12_start_0 pcmk2" [ style = bold] "FAKE12_stop_0 pcmk1" -> "all_stopped" [ style = bold] "FAKE12_stop_0 pcmk1" [ style=bold color="green" fontcolor="black"] "FAKE14_monitor_10000 pcmk_remote1" [ style=bold color="green" fontcolor="black"] "FAKE14_start_0 pcmk_remote1" -> "FAKE14_monitor_10000 pcmk_remote1" [ style = bold] "FAKE14_start_0 pcmk_remote1" [ style=bold color="green" fontcolor="black"] "FAKE14_stop_0 pcmk2" -> "FAKE14_start_0 pcmk_remote1" [ style = bold] "FAKE14_stop_0 pcmk2" -> "all_stopped" [ style = bold] "FAKE14_stop_0 pcmk2" [ style=bold color="green" fontcolor="black"] "FAKE17_monitor_10000 pcmk_remote4" [ style=bold color="green" fontcolor="black"] "FAKE17_start_0 pcmk_remote4" -> "FAKE17_monitor_10000 pcmk_remote4" [ style = bold] "FAKE17_start_0 pcmk_remote4" [ style=bold color="green" fontcolor="black"] "FAKE17_stop_0 pcmk_remote1" -> "FAKE17_start_0 pcmk_remote4" [ style = bold] "FAKE17_stop_0 pcmk_remote1" -> "all_stopped" [ style = bold] "FAKE17_stop_0 pcmk_remote1" [ style=bold color="green" fontcolor="black"] "FAKE25_monitor_10000 pcmk_remote4" [ style=bold color="green" fontcolor="black"] "FAKE25_start_0 pcmk_remote4" -> "FAKE25_monitor_10000 pcmk_remote4" [ style = bold] "FAKE25_start_0 pcmk_remote4" [ style=bold color="green" fontcolor="black"] "FAKE25_stop_0 pcmk_remote1" -> "FAKE25_start_0 pcmk_remote4" [ style = bold] "FAKE25_stop_0 pcmk_remote1" -> "all_stopped" [ style = bold] "FAKE25_stop_0 pcmk_remote1" [ style=bold color="green" fontcolor="black"] "FAKE28_monitor_10000 pcmk1" [ style=bold color="green" fontcolor="black"] "FAKE28_start_0 pcmk1" -> "FAKE28_monitor_10000 pcmk1" [ style = bold] "FAKE28_start_0 pcmk1" [ style=bold color="green" fontcolor="black"] "FAKE28_stop_0 pcmk3" -> "FAKE28_start_0 pcmk1" [ style = bold] "FAKE28_stop_0 pcmk3" -> "all_stopped" [ style = bold] "FAKE28_stop_0 pcmk3" [ style=bold color="green" fontcolor="black"] "FAKE30_monitor_10000 pcmk_remote1" [ style=bold color="green" fontcolor="black"] "FAKE30_start_0 pcmk_remote1" -> "FAKE30_monitor_10000 pcmk_remote1" [ style = bold] "FAKE30_start_0 pcmk_remote1" [ style=bold color="green" fontcolor="black"] "FAKE30_stop_0 pcmk1" -> "FAKE30_start_0 pcmk_remote1" [ style = bold] "FAKE30_stop_0 pcmk1" -> "all_stopped" [ style = bold] "FAKE30_stop_0 pcmk1" [ style=bold color="green" fontcolor="black"] "FAKE33_monitor_10000 pcmk_remote4" [ style=bold color="green" fontcolor="black"] "FAKE33_start_0 pcmk_remote4" -> "FAKE33_monitor_10000 pcmk_remote4" [ style = bold] "FAKE33_start_0 pcmk_remote4" [ style=bold color="green" fontcolor="black"] "FAKE33_stop_0 pcmk_remote1" -> "FAKE33_start_0 pcmk_remote4" [ style = bold] "FAKE33_stop_0 pcmk_remote1" -> "all_stopped" [ style = bold] "FAKE33_stop_0 pcmk_remote1" [ style=bold color="green" fontcolor="black"] "FAKE38_monitor_10000 pcmk_remote1" [ style=bold color="green" fontcolor="black"] "FAKE38_start_0 pcmk_remote1" -> "FAKE38_monitor_10000 pcmk_remote1" [ style = bold] "FAKE38_start_0 pcmk_remote1" [ style=bold color="green" fontcolor="black"] "FAKE38_stop_0 pcmk2" -> "FAKE38_start_0 pcmk_remote1" [ style = bold] "FAKE38_stop_0 pcmk2" -> "all_stopped" [ style = bold] "FAKE38_stop_0 pcmk2" [ style=bold color="green" fontcolor="black"] "FAKE39_monitor_10000 pcmk_remote2" [ style=bold color="green" fontcolor="black"] "FAKE39_start_0 pcmk_remote2" -> "FAKE39_monitor_10000 pcmk_remote2" [ style = bold] "FAKE39_start_0 pcmk_remote2" [ style=bold color="green" fontcolor="black"] "FAKE39_stop_0 pcmk1" -> "FAKE39_start_0 pcmk_remote2" [ style = bold] "FAKE39_stop_0 pcmk1" -> "all_stopped" [ style = bold] "FAKE39_stop_0 pcmk1" [ style=bold color="green" fontcolor="black"] "FAKE41_monitor_10000 pcmk_remote4" [ style=bold color="green" fontcolor="black"] "FAKE41_start_0 pcmk_remote4" -> "FAKE41_monitor_10000 pcmk_remote4" [ style = bold] "FAKE41_start_0 pcmk_remote4" [ style=bold color="green" fontcolor="black"] "FAKE41_stop_0 pcmk_remote2" -> "FAKE41_start_0 pcmk_remote4" [ style = bold] "FAKE41_stop_0 pcmk_remote2" -> "all_stopped" [ style = bold] "FAKE41_stop_0 pcmk_remote2" [ style=bold color="green" fontcolor="black"] "FAKE47_monitor_10000 pcmk_remote2" [ style=bold color="green" fontcolor="black"] "FAKE47_start_0 pcmk_remote2" -> "FAKE47_monitor_10000 pcmk_remote2" [ style = bold] "FAKE47_start_0 pcmk_remote2" [ style=bold color="green" fontcolor="black"] "FAKE47_stop_0 pcmk_remote1" -> "FAKE47_start_0 pcmk_remote2" [ style = bold] "FAKE47_stop_0 pcmk_remote1" -> "all_stopped" [ style = bold] "FAKE47_stop_0 pcmk_remote1" [ style=bold color="green" fontcolor="black"] "FAKE48_monitor_10000 pcmk_remote3" [ style=bold color="green" fontcolor="black"] "FAKE48_start_0 pcmk_remote3" -> "FAKE48_monitor_10000 pcmk_remote3" [ style = bold] "FAKE48_start_0 pcmk_remote3" [ style=bold color="green" fontcolor="black"] "FAKE48_stop_0 pcmk1" -> "FAKE48_start_0 pcmk_remote3" [ style = bold] "FAKE48_stop_0 pcmk1" -> "all_stopped" [ style = bold] "FAKE48_stop_0 pcmk1" [ style=bold color="green" fontcolor="black"] "FAKE49_monitor_10000 pcmk_remote4" [ style=bold color="green" fontcolor="black"] "FAKE49_start_0 pcmk_remote4" -> "FAKE49_monitor_10000 pcmk_remote4" [ style = bold] "FAKE49_start_0 pcmk_remote4" [ style=bold color="green" fontcolor="black"] "FAKE49_stop_0 pcmk_remote3" -> "FAKE49_start_0 pcmk_remote4" [ style = bold] "FAKE49_stop_0 pcmk_remote3" -> "all_stopped" [ style = bold] "FAKE49_stop_0 pcmk_remote3" [ style=bold color="green" fontcolor="black"] "FAKE5_monitor_10000 pcmk_remote4" [ style=bold color="green" fontcolor="black"] "FAKE5_start_0 pcmk_remote4" -> "FAKE5_monitor_10000 pcmk_remote4" [ style = bold] "FAKE5_start_0 pcmk_remote4" [ style=bold color="green" fontcolor="black"] "FAKE5_stop_0 pcmk1" -> "FAKE5_start_0 pcmk_remote4" [ style = bold] "FAKE5_stop_0 pcmk1" -> "all_stopped" [ style = bold] "FAKE5_stop_0 pcmk1" [ style=bold color="green" fontcolor="black"] "FAKE9_monitor_10000 pcmk_remote4" [ style=bold color="green" fontcolor="black"] "FAKE9_start_0 pcmk_remote4" -> "FAKE9_monitor_10000 pcmk_remote4" [ style = bold] "FAKE9_start_0 pcmk_remote4" [ style=bold color="green" fontcolor="black"] "FAKE9_stop_0 pcmk2" -> "FAKE9_start_0 pcmk_remote4" [ style = bold] "FAKE9_stop_0 pcmk2" -> "all_stopped" [ style = bold] "FAKE9_stop_0 pcmk2" [ style=bold color="green" fontcolor="black"] +"all_stopped" -> "pcmk_remote5_start_0 pcmk2" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "pcmk_remote2_migrate_from_0 pcmk1" -> "pcmk_remote2_start_0 pcmk1" [ style = bold] "pcmk_remote2_migrate_from_0 pcmk1" -> "pcmk_remote2_stop_0 pcmk3" [ style = bold] "pcmk_remote2_migrate_from_0 pcmk1" [ style=bold color="green" fontcolor="black"] "pcmk_remote2_monitor_60000 pcmk1" [ style=bold color="green" fontcolor="black"] "pcmk_remote2_start_0 pcmk1" -> "FAKE39_monitor_10000 pcmk_remote2" [ style = bold] "pcmk_remote2_start_0 pcmk1" -> "FAKE39_start_0 pcmk_remote2" [ style = bold] "pcmk_remote2_start_0 pcmk1" -> "FAKE41_stop_0 pcmk_remote2" [ style = bold] "pcmk_remote2_start_0 pcmk1" -> "FAKE47_monitor_10000 pcmk_remote2" [ style = bold] "pcmk_remote2_start_0 pcmk1" -> "FAKE47_start_0 pcmk_remote2" [ style = bold] "pcmk_remote2_start_0 pcmk1" -> "pcmk_remote2_monitor_60000 pcmk1" [ style = bold] "pcmk_remote2_start_0 pcmk1" [ style=bold color="green" fontcolor="orange"] "pcmk_remote2_stop_0 pcmk3" -> "all_stopped" [ style = bold] "pcmk_remote2_stop_0 pcmk3" -> "pcmk_remote2_start_0 pcmk1" [ style = bold] "pcmk_remote2_stop_0 pcmk3" [ style=bold color="green" fontcolor="black"] "pcmk_remote4_monitor_60000 pcmk2" [ style=bold color="green" fontcolor="black"] "pcmk_remote4_start_0 pcmk2" -> "FAKE17_monitor_10000 pcmk_remote4" [ style = bold] "pcmk_remote4_start_0 pcmk2" -> "FAKE17_start_0 pcmk_remote4" [ style = bold] "pcmk_remote4_start_0 pcmk2" -> "FAKE25_monitor_10000 pcmk_remote4" [ style = bold] "pcmk_remote4_start_0 pcmk2" -> "FAKE25_start_0 pcmk_remote4" [ style = bold] "pcmk_remote4_start_0 pcmk2" -> "FAKE33_monitor_10000 pcmk_remote4" [ style = bold] "pcmk_remote4_start_0 pcmk2" -> "FAKE33_start_0 pcmk_remote4" [ style = bold] "pcmk_remote4_start_0 pcmk2" -> "FAKE41_monitor_10000 pcmk_remote4" [ style = bold] "pcmk_remote4_start_0 pcmk2" -> "FAKE41_start_0 pcmk_remote4" [ style = bold] "pcmk_remote4_start_0 pcmk2" -> "FAKE49_monitor_10000 pcmk_remote4" [ style = bold] "pcmk_remote4_start_0 pcmk2" -> "FAKE49_start_0 pcmk_remote4" [ style = bold] "pcmk_remote4_start_0 pcmk2" -> "FAKE5_monitor_10000 pcmk_remote4" [ style = bold] "pcmk_remote4_start_0 pcmk2" -> "FAKE5_start_0 pcmk_remote4" [ style = bold] "pcmk_remote4_start_0 pcmk2" -> "FAKE9_monitor_10000 pcmk_remote4" [ style = bold] "pcmk_remote4_start_0 pcmk2" -> "FAKE9_start_0 pcmk_remote4" [ style = bold] "pcmk_remote4_start_0 pcmk2" -> "pcmk_remote4_monitor_60000 pcmk2" [ style = bold] "pcmk_remote4_start_0 pcmk2" [ style=bold color="green" fontcolor="black"] "pcmk_remote5_migrate_from_0 pcmk2" -> "pcmk_remote5_start_0 pcmk2" [ style = bold] "pcmk_remote5_migrate_from_0 pcmk2" -> "pcmk_remote5_stop_0 pcmk1" [ style = bold] "pcmk_remote5_migrate_from_0 pcmk2" [ style=bold color="green" fontcolor="black"] "pcmk_remote5_migrate_to_0 pcmk1" -> "pcmk_remote5_migrate_from_0 pcmk2" [ style = bold] "pcmk_remote5_migrate_to_0 pcmk1" [ style=bold color="green" fontcolor="black"] "pcmk_remote5_monitor_60000 pcmk2" [ style=bold color="green" fontcolor="black"] "pcmk_remote5_start_0 pcmk2" -> "pcmk_remote5_monitor_60000 pcmk2" [ style = bold] "pcmk_remote5_start_0 pcmk2" [ style=bold color="green" fontcolor="orange"] "pcmk_remote5_stop_0 pcmk1" -> "all_stopped" [ style = bold] "pcmk_remote5_stop_0 pcmk1" -> "pcmk_remote5_start_0 pcmk2" [ style = bold] "pcmk_remote5_stop_0 pcmk1" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' pcmk4" -> "stonith_complete" [ style = bold] "stonith 'reboot' pcmk4" [ style=bold color="green" fontcolor="black"] "stonith_complete" -> "FAKE12_start_0 pcmk2" [ style = bold] "stonith_complete" -> "FAKE14_start_0 pcmk_remote1" [ style = bold] "stonith_complete" -> "FAKE17_start_0 pcmk_remote4" [ style = bold] "stonith_complete" -> "FAKE25_start_0 pcmk_remote4" [ style = bold] "stonith_complete" -> "FAKE28_start_0 pcmk1" [ style = bold] "stonith_complete" -> "FAKE30_start_0 pcmk_remote1" [ style = bold] "stonith_complete" -> "FAKE33_start_0 pcmk_remote4" [ style = bold] "stonith_complete" -> "FAKE38_start_0 pcmk_remote1" [ style = bold] "stonith_complete" -> "FAKE39_start_0 pcmk_remote2" [ style = bold] "stonith_complete" -> "FAKE41_start_0 pcmk_remote4" [ style = bold] "stonith_complete" -> "FAKE47_start_0 pcmk_remote2" [ style = bold] "stonith_complete" -> "FAKE48_start_0 pcmk_remote3" [ style = bold] "stonith_complete" -> "FAKE49_start_0 pcmk_remote4" [ style = bold] "stonith_complete" -> "FAKE5_start_0 pcmk_remote4" [ style = bold] "stonith_complete" -> "FAKE9_start_0 pcmk_remote4" [ style = bold] "stonith_complete" -> "all_stopped" [ style = bold] -"stonith_complete" -> "pcmk_remote2_migrate_from_0 pcmk1" [ style = bold] -"stonith_complete" -> "pcmk_remote2_start_0 pcmk1" [ style = bold] -"stonith_complete" -> "pcmk_remote4_start_0 pcmk2" [ style = bold] -"stonith_complete" -> "pcmk_remote5_migrate_to_0 pcmk1" [ style = bold] -"stonith_complete" -> "pcmk_remote5_start_0 pcmk2" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/remote-partial-migrate2.exp b/pengine/test10/remote-partial-migrate2.exp index abf281f612..bae190cf70 100644 --- a/pengine/test10/remote-partial-migrate2.exp +++ b/pengine/test10/remote-partial-migrate2.exp @@ -1,869 +1,854 @@ - - - - - + - - - - - - - - + - - - - - + - + - + - + diff --git a/pengine/test10/remote-partial-migrate2.summary b/pengine/test10/remote-partial-migrate2.summary index 2a242bdfe9..6b6428de75 100644 --- a/pengine/test10/remote-partial-migrate2.summary +++ b/pengine/test10/remote-partial-migrate2.summary @@ -1,208 +1,208 @@ Current cluster status: Node pcmk4 (4): UNCLEAN (offline) Online: [ pcmk1 pcmk2 pcmk3 ] RemoteOnline: [ pcmk_remote1 pcmk_remote2 pcmk_remote3 pcmk_remote5 ] RemoteOFFLINE: [ pcmk_remote4 ] shooter (stonith:fence_docker_cts): Started pcmk3 pcmk_remote1 (ocf::pacemaker:remote): Started pcmk1 pcmk_remote2 (ocf::pacemaker:remote): Started [ pcmk1 pcmk3 ] pcmk_remote3 (ocf::pacemaker:remote): Started pcmk3 pcmk_remote4 (ocf::pacemaker:remote): Stopped pcmk_remote5 (ocf::pacemaker:remote): Started pcmk1 FAKE1 (ocf::heartbeat:Dummy): Started pcmk_remote2 FAKE2 (ocf::heartbeat:Dummy): Started pcmk_remote3 FAKE3 (ocf::heartbeat:Dummy): Started pcmk_remote1 FAKE4 (ocf::heartbeat:Dummy): Started pcmk_remote5 FAKE5 (ocf::heartbeat:Dummy): Started pcmk1 FAKE6 (ocf::heartbeat:Dummy): Started pcmk_remote1 FAKE7 (ocf::heartbeat:Dummy): Started pcmk_remote2 FAKE8 (ocf::heartbeat:Dummy): Started pcmk_remote3 FAKE9 (ocf::heartbeat:Dummy): Started pcmk2 FAKE10 (ocf::heartbeat:Dummy): Started pcmk_remote5 FAKE11 (ocf::heartbeat:Dummy): Started pcmk1 FAKE12 (ocf::heartbeat:Dummy): Started pcmk1 FAKE13 (ocf::heartbeat:Dummy): Started pcmk3 FAKE14 (ocf::heartbeat:Dummy): Started pcmk2 FAKE15 (ocf::heartbeat:Dummy): Started pcmk_remote2 FAKE16 (ocf::heartbeat:Dummy): Started pcmk_remote3 FAKE17 (ocf::heartbeat:Dummy): Started pcmk_remote1 FAKE18 (ocf::heartbeat:Dummy): Started pcmk_remote5 FAKE19 (ocf::heartbeat:Dummy): Started pcmk3 FAKE20 (ocf::heartbeat:Dummy): Started pcmk2 FAKE21 (ocf::heartbeat:Dummy): Started pcmk1 FAKE22 (ocf::heartbeat:Dummy): Started pcmk_remote1 FAKE23 (ocf::heartbeat:Dummy): Started pcmk_remote2 FAKE24 (ocf::heartbeat:Dummy): Started pcmk_remote3 FAKE25 (ocf::heartbeat:Dummy): Started pcmk_remote1 FAKE26 (ocf::heartbeat:Dummy): Started pcmk_remote5 FAKE27 (ocf::heartbeat:Dummy): Started pcmk3 FAKE28 (ocf::heartbeat:Dummy): Started pcmk3 FAKE29 (ocf::heartbeat:Dummy): Started pcmk2 FAKE30 (ocf::heartbeat:Dummy): Started pcmk1 FAKE31 (ocf::heartbeat:Dummy): Started pcmk_remote2 FAKE32 (ocf::heartbeat:Dummy): Started pcmk_remote3 FAKE33 (ocf::heartbeat:Dummy): Started pcmk_remote1 FAKE34 (ocf::heartbeat:Dummy): Started pcmk_remote5 FAKE35 (ocf::heartbeat:Dummy): Started pcmk1 FAKE36 (ocf::heartbeat:Dummy): Started pcmk3 FAKE37 (ocf::heartbeat:Dummy): Started pcmk2 FAKE38 (ocf::heartbeat:Dummy): Started pcmk2 FAKE39 (ocf::heartbeat:Dummy): Started pcmk1 FAKE40 (ocf::heartbeat:Dummy): Started pcmk_remote3 FAKE41 (ocf::heartbeat:Dummy): Started pcmk_remote2 FAKE42 (ocf::heartbeat:Dummy): Started pcmk_remote5 FAKE43 (ocf::heartbeat:Dummy): Started pcmk_remote1 FAKE44 (ocf::heartbeat:Dummy): Started pcmk2 FAKE45 (ocf::heartbeat:Dummy): Started pcmk3 FAKE46 (ocf::heartbeat:Dummy): Started pcmk1 FAKE47 (ocf::heartbeat:Dummy): Started pcmk_remote1 FAKE48 (ocf::heartbeat:Dummy): Started pcmk1 FAKE49 (ocf::heartbeat:Dummy): Started pcmk_remote3 FAKE50 (ocf::heartbeat:Dummy): Started pcmk_remote5 Transition Summary: * Fence (reboot) pcmk4 'peer is no longer part of the cluster' * Migrate pcmk_remote2 ( pcmk3 -> pcmk1 ) * Start pcmk_remote4 (pcmk2) * Migrate pcmk_remote5 ( pcmk1 -> pcmk2 ) * Move FAKE5 ( pcmk1 -> pcmk_remote4 ) * Move FAKE9 ( pcmk2 -> pcmk_remote4 ) * Move FAKE12 ( pcmk1 -> pcmk2 ) * Move FAKE14 ( pcmk2 -> pcmk_remote1 ) * Move FAKE17 ( pcmk_remote1 -> pcmk_remote4 ) * Move FAKE25 ( pcmk_remote1 -> pcmk_remote4 ) * Move FAKE28 ( pcmk3 -> pcmk1 ) * Move FAKE30 ( pcmk1 -> pcmk_remote1 ) * Move FAKE33 ( pcmk_remote1 -> pcmk_remote4 ) * Move FAKE38 ( pcmk2 -> pcmk_remote1 ) * Move FAKE39 ( pcmk1 -> pcmk_remote2 ) * Move FAKE41 ( pcmk_remote2 -> pcmk_remote4 ) * Move FAKE47 ( pcmk_remote1 -> pcmk_remote2 ) * Move FAKE48 ( pcmk1 -> pcmk_remote3 ) * Move FAKE49 ( pcmk_remote3 -> pcmk_remote4 ) Executing cluster transition: + * Resource action: pcmk_remote2 migrate_from on pcmk1 + * Resource action: pcmk_remote2 stop on pcmk3 + * Resource action: pcmk_remote4 start on pcmk2 + * Resource action: pcmk_remote5 migrate_to on pcmk1 * Resource action: FAKE5 stop on pcmk1 * Resource action: FAKE9 stop on pcmk2 * Resource action: FAKE12 stop on pcmk1 * Resource action: FAKE14 stop on pcmk2 * Resource action: FAKE17 stop on pcmk_remote1 * Resource action: FAKE25 stop on pcmk_remote1 * Resource action: FAKE28 stop on pcmk3 * Resource action: FAKE30 stop on pcmk1 * Resource action: FAKE33 stop on pcmk_remote1 * Resource action: FAKE38 stop on pcmk2 * Resource action: FAKE39 stop on pcmk1 * Resource action: FAKE47 stop on pcmk_remote1 * Resource action: FAKE48 stop on pcmk1 * Resource action: FAKE49 stop on pcmk_remote3 * Fencing pcmk4 (reboot) + * Pseudo action: pcmk_remote2_start_0 + * Resource action: pcmk_remote4 monitor=60000 on pcmk2 + * Resource action: pcmk_remote5 migrate_from on pcmk2 + * Resource action: pcmk_remote5 stop on pcmk1 + * Resource action: FAKE41 stop on pcmk_remote2 * Pseudo action: stonith_complete - * Resource action: pcmk_remote2 migrate_from on pcmk1 - * Resource action: pcmk_remote2 stop on pcmk3 - * Resource action: pcmk_remote4 start on pcmk2 - * Resource action: pcmk_remote5 migrate_to on pcmk1 + * Pseudo action: all_stopped + * Resource action: pcmk_remote2 monitor=60000 on pcmk1 + * Pseudo action: pcmk_remote5_start_0 * Resource action: FAKE5 start on pcmk_remote4 * Resource action: FAKE9 start on pcmk_remote4 * Resource action: FAKE12 start on pcmk2 * Resource action: FAKE14 start on pcmk_remote1 * Resource action: FAKE17 start on pcmk_remote4 * Resource action: FAKE25 start on pcmk_remote4 * Resource action: FAKE28 start on pcmk1 * Resource action: FAKE30 start on pcmk_remote1 * Resource action: FAKE33 start on pcmk_remote4 * Resource action: FAKE38 start on pcmk_remote1 + * Resource action: FAKE39 start on pcmk_remote2 + * Resource action: FAKE41 start on pcmk_remote4 + * Resource action: FAKE47 start on pcmk_remote2 * Resource action: FAKE48 start on pcmk_remote3 * Resource action: FAKE49 start on pcmk_remote4 - * Pseudo action: pcmk_remote2_start_0 - * Resource action: pcmk_remote4 monitor=60000 on pcmk2 - * Resource action: pcmk_remote5 migrate_from on pcmk2 - * Resource action: pcmk_remote5 stop on pcmk1 + * Resource action: pcmk_remote5 monitor=60000 on pcmk2 * Resource action: FAKE5 monitor=10000 on pcmk_remote4 * Resource action: FAKE9 monitor=10000 on pcmk_remote4 * Resource action: FAKE12 monitor=10000 on pcmk2 * Resource action: FAKE14 monitor=10000 on pcmk_remote1 * Resource action: FAKE17 monitor=10000 on pcmk_remote4 * Resource action: FAKE25 monitor=10000 on pcmk_remote4 * Resource action: FAKE28 monitor=10000 on pcmk1 * Resource action: FAKE30 monitor=10000 on pcmk_remote1 * Resource action: FAKE33 monitor=10000 on pcmk_remote4 * Resource action: FAKE38 monitor=10000 on pcmk_remote1 - * Resource action: FAKE39 start on pcmk_remote2 - * Resource action: FAKE41 stop on pcmk_remote2 - * Resource action: FAKE47 start on pcmk_remote2 - * Resource action: FAKE48 monitor=10000 on pcmk_remote3 - * Resource action: FAKE49 monitor=10000 on pcmk_remote4 - * Pseudo action: all_stopped - * Resource action: pcmk_remote2 monitor=60000 on pcmk1 - * Pseudo action: pcmk_remote5_start_0 * Resource action: FAKE39 monitor=10000 on pcmk_remote2 - * Resource action: FAKE41 start on pcmk_remote4 - * Resource action: FAKE47 monitor=10000 on pcmk_remote2 - * Resource action: pcmk_remote5 monitor=60000 on pcmk2 * Resource action: FAKE41 monitor=10000 on pcmk_remote4 + * Resource action: FAKE47 monitor=10000 on pcmk_remote2 + * Resource action: FAKE48 monitor=10000 on pcmk_remote3 + * Resource action: FAKE49 monitor=10000 on pcmk_remote4 Revised cluster status: Online: [ pcmk1 pcmk2 pcmk3 ] OFFLINE: [ pcmk4 ] RemoteOnline: [ pcmk_remote1 pcmk_remote2 pcmk_remote3 pcmk_remote4 pcmk_remote5 ] shooter (stonith:fence_docker_cts): Started pcmk3 pcmk_remote1 (ocf::pacemaker:remote): Started pcmk1 pcmk_remote2 (ocf::pacemaker:remote): Started pcmk1 pcmk_remote3 (ocf::pacemaker:remote): Started pcmk3 pcmk_remote4 (ocf::pacemaker:remote): Started pcmk2 pcmk_remote5 (ocf::pacemaker:remote): Started pcmk2 FAKE1 (ocf::heartbeat:Dummy): Started pcmk_remote2 FAKE2 (ocf::heartbeat:Dummy): Started pcmk_remote3 FAKE3 (ocf::heartbeat:Dummy): Started pcmk_remote1 FAKE4 (ocf::heartbeat:Dummy): Started pcmk_remote5 FAKE5 (ocf::heartbeat:Dummy): Started pcmk_remote4 FAKE6 (ocf::heartbeat:Dummy): Started pcmk_remote1 FAKE7 (ocf::heartbeat:Dummy): Started pcmk_remote2 FAKE8 (ocf::heartbeat:Dummy): Started pcmk_remote3 FAKE9 (ocf::heartbeat:Dummy): Started pcmk_remote4 FAKE10 (ocf::heartbeat:Dummy): Started pcmk_remote5 FAKE11 (ocf::heartbeat:Dummy): Started pcmk1 FAKE12 (ocf::heartbeat:Dummy): Started pcmk2 FAKE13 (ocf::heartbeat:Dummy): Started pcmk3 FAKE14 (ocf::heartbeat:Dummy): Started pcmk_remote1 FAKE15 (ocf::heartbeat:Dummy): Started pcmk_remote2 FAKE16 (ocf::heartbeat:Dummy): Started pcmk_remote3 FAKE17 (ocf::heartbeat:Dummy): Started pcmk_remote4 FAKE18 (ocf::heartbeat:Dummy): Started pcmk_remote5 FAKE19 (ocf::heartbeat:Dummy): Started pcmk3 FAKE20 (ocf::heartbeat:Dummy): Started pcmk2 FAKE21 (ocf::heartbeat:Dummy): Started pcmk1 FAKE22 (ocf::heartbeat:Dummy): Started pcmk_remote1 FAKE23 (ocf::heartbeat:Dummy): Started pcmk_remote2 FAKE24 (ocf::heartbeat:Dummy): Started pcmk_remote3 FAKE25 (ocf::heartbeat:Dummy): Started pcmk_remote4 FAKE26 (ocf::heartbeat:Dummy): Started pcmk_remote5 FAKE27 (ocf::heartbeat:Dummy): Started pcmk3 FAKE28 (ocf::heartbeat:Dummy): Started pcmk1 FAKE29 (ocf::heartbeat:Dummy): Started pcmk2 FAKE30 (ocf::heartbeat:Dummy): Started pcmk_remote1 FAKE31 (ocf::heartbeat:Dummy): Started pcmk_remote2 FAKE32 (ocf::heartbeat:Dummy): Started pcmk_remote3 FAKE33 (ocf::heartbeat:Dummy): Started pcmk_remote4 FAKE34 (ocf::heartbeat:Dummy): Started pcmk_remote5 FAKE35 (ocf::heartbeat:Dummy): Started pcmk1 FAKE36 (ocf::heartbeat:Dummy): Started pcmk3 FAKE37 (ocf::heartbeat:Dummy): Started pcmk2 FAKE38 (ocf::heartbeat:Dummy): Started pcmk_remote1 FAKE39 (ocf::heartbeat:Dummy): Started pcmk_remote2 FAKE40 (ocf::heartbeat:Dummy): Started pcmk_remote3 FAKE41 (ocf::heartbeat:Dummy): Started pcmk_remote4 FAKE42 (ocf::heartbeat:Dummy): Started pcmk_remote5 FAKE43 (ocf::heartbeat:Dummy): Started pcmk_remote1 FAKE44 (ocf::heartbeat:Dummy): Started pcmk2 FAKE45 (ocf::heartbeat:Dummy): Started pcmk3 FAKE46 (ocf::heartbeat:Dummy): Started pcmk1 FAKE47 (ocf::heartbeat:Dummy): Started pcmk_remote2 FAKE48 (ocf::heartbeat:Dummy): Started pcmk_remote3 FAKE49 (ocf::heartbeat:Dummy): Started pcmk_remote4 FAKE50 (ocf::heartbeat:Dummy): Started pcmk_remote5 diff --git a/pengine/test10/remote-recover-all.dot b/pengine/test10/remote-recover-all.dot index ad421e6cbc..5b79602f03 100644 --- a/pengine/test10/remote-recover-all.dot +++ b/pengine/test10/remote-recover-all.dot @@ -1,164 +1,163 @@ digraph "g" { +"all_stopped" -> "galera-0_start_0 controller-2" [ style = bold] "all_stopped" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "all_stopped" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "all_stopped" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "galera-0_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_start_0 controller-2" -> "galera-0_monitor_20000 controller-2" [ style = bold] "galera-0_start_0 controller-2" -> "galera_monitor_10000 galera-0" [ style = bold] "galera-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_stop_0 controller-1" -> "all_stopped" [ style = bold] "galera-0_stop_0 controller-1" -> "galera-0_start_0 controller-2" [ style = bold] "galera-0_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera-2_stop_0 controller-1" -> "all_stopped" [ style = bold] "galera-2_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera-master_demote_0" -> "galera-master_demoted_0" [ style = bold] "galera-master_demote_0" -> "galera_demote_0 galera-2" [ style = bold] "galera-master_demote_0" [ style=bold color="green" fontcolor="orange"] "galera-master_demoted_0" -> "galera-master_stop_0" [ style = bold] "galera-master_demoted_0" [ style=bold color="green" fontcolor="orange"] "galera-master_stop_0" -> "galera-master_stopped_0" [ style = bold] "galera-master_stop_0" -> "galera_stop_0 galera-2" [ style = bold] "galera-master_stop_0" [ style=bold color="green" fontcolor="orange"] "galera-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "galera_demote_0 galera-2" -> "galera-master_demoted_0" [ style = bold] "galera_demote_0 galera-2" -> "galera_stop_0 galera-2" [ style = bold] "galera_demote_0 galera-2" [ style=bold color="green" fontcolor="orange"] "galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"] "galera_stop_0 galera-2" -> "all_stopped" [ style = bold] "galera_stop_0 galera-2" -> "galera-2_stop_0 controller-1" [ style = bold] "galera_stop_0 galera-2" -> "galera-master_stopped_0" [ style = bold] "galera_stop_0 galera-2" [ style=bold color="green" fontcolor="orange"] "haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy-clone_stop_0" -> "haproxy_stop_0 controller-1" [ style = bold] "haproxy-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "haproxy-clone_stopped_0" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "haproxy_stop_0 controller-1" -> "all_stopped" [ style = bold] "haproxy_stop_0 controller-1" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.14_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_start_0 controller-2" -> "ip-172.17.1.14_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.14_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_stop_0 controller-1" -> "all_stopped" [ style = bold] "ip-172.17.1.14_stop_0 controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "ip-172.17.1.14_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.17_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_start_0 controller-2" -> "ip-172.17.1.17_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.17_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_stop_0 controller-1" -> "all_stopped" [ style = bold] "ip-172.17.1.17_stop_0 controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "ip-172.17.1.17_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.4.11_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_start_0 controller-2" -> "ip-172.17.4.11_monitor_10000 controller-2" [ style = bold] "ip-172.17.4.11_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_stop_0 controller-1" -> "all_stopped" [ style = bold] "ip-172.17.4.11_stop_0 controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "ip-172.17.4.11_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "messaging-1_stop_0 controller-1" -> "all_stopped" [ style = bold] "messaging-1_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "rabbitmq-clone_stop_0" -> "rabbitmq-clone_stopped_0" [ style = bold] "rabbitmq-clone_stop_0" -> "rabbitmq_stop_0 messaging-1" [ style = bold] "rabbitmq-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_confirmed-post_notify_stonith_0" -> "all_stopped" [ style = bold] "rabbitmq_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_post_notify_stonith_0 messaging-0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0 messaging-0" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_stonith_0 messaging-2" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0 messaging-2" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_stop_0 messaging-1" -> "all_stopped" [ style = bold] "rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] "redis-master_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold] "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_stop_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_post_notify_stopped_0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_pre_notify_stop_0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-2" [ style = bold] "redis-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stop_0" -> "redis-master_stopped_0" [ style = bold] "redis-master_stop_0" -> "redis_stop_0 controller-1" [ style = bold] "redis-master_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stopped_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis_confirmed-post_notify_stonith_0" -> "all_stopped" [ style = bold] "redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_post_notify_stonith_0 controller-0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0 controller-2" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_pre_notify_stop_0 controller-0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_stop_0 controller-2" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_stop_0 controller-1" -> "all_stopped" [ style = bold] "redis_stop_0 controller-1" -> "redis-master_stopped_0" [ style = bold] "redis_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] -"stonith 'reboot' controller-1" -> "galera-0_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "galera-2_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "haproxy-clone_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "haproxy_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "messaging-1_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "redis-master_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_post_notify_stonith_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "stonith 'reboot' galera-2" [ style = bold] "stonith 'reboot' controller-1" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' galera-2" -> "galera-master_stop_0" [ style = bold] "stonith 'reboot' galera-2" -> "galera_demote_0 galera-2" [ style = bold] "stonith 'reboot' galera-2" -> "galera_stop_0 galera-2" [ style = bold] "stonith 'reboot' galera-2" -> "stonith 'reboot' messaging-1" [ style = bold] "stonith 'reboot' galera-2" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' messaging-1" -> "rabbitmq-clone_stop_0" [ style = bold] "stonith 'reboot' messaging-1" -> "rabbitmq_post_notify_stonith_0" [ style = bold] "stonith 'reboot' messaging-1" -> "rabbitmq_stop_0 messaging-1" [ style = bold] "stonith 'reboot' messaging-1" -> "stonith_complete" [ style = bold] "stonith 'reboot' messaging-1" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" -> "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith_complete" -> "all_stopped" [ style = bold] -"stonith_complete" -> "galera-0_start_0 controller-2" [ style = bold] "stonith_complete" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "stonith_complete" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "stonith_complete" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/remote-recover-all.exp b/pengine/test10/remote-recover-all.exp index b0af5c4378..556ccfd7d6 100644 --- a/pengine/test10/remote-recover-all.exp +++ b/pengine/test10/remote-recover-all.exp @@ -1,862 +1,858 @@ - + - + - - - - - + diff --git a/pengine/test10/remote-recover-all.summary b/pengine/test10/remote-recover-all.summary index 6c9f0586b2..ba074e5082 100644 --- a/pengine/test10/remote-recover-all.summary +++ b/pengine/test10/remote-recover-all.summary @@ -1,154 +1,154 @@ Using the original execution date of: 2017-05-03 13:33:24Z Current cluster status: Node controller-1 (2): UNCLEAN (offline) Online: [ controller-0 controller-2 ] RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-1 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ] Master/Slave Set: galera-master [galera] Masters: [ galera-0 galera-1 galera-2 ] Stopped: [ controller-0 controller-1 controller-2 messaging-0 messaging-1 messaging-2 ] Master/Slave Set: redis-master [redis] redis (ocf::heartbeat:redis): Slave controller-1 (UNCLEAN) Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) Clone Set: haproxy-clone [haproxy] haproxy (systemd:haproxy): Started controller-1 (UNCLEAN) Started: [ controller-0 controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) Transition Summary: * Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable' * Fence (reboot) galera-2 'resources are active and the connection is unrecoverable' * Fence (reboot) controller-1 'peer is no longer part of the cluster' * Stop messaging-1 (controller-1) due to node availability * Move galera-0 ( controller-1 -> controller-2 ) * Stop galera-2 (controller-1) due to node availability * Stop rabbitmq:2 (messaging-1) due to node availability * Stop galera:1 ( Master galera-2 ) due to node availability * Stop redis:0 ( Slave controller-1 ) due to node availability * Move ip-172.17.1.14 ( controller-1 -> controller-2 ) * Move ip-172.17.1.17 ( controller-1 -> controller-2 ) * Move ip-172.17.4.11 ( controller-1 -> controller-2 ) * Stop haproxy:0 (controller-1) due to node availability * Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) due to resource definition change * Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) due to resource definition change * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) Executing cluster transition: + * Pseudo action: galera-0_stop_0 * Pseudo action: galera-master_demote_0 * Pseudo action: redis-master_pre_notify_stop_0 * Resource action: stonith-fence_ipmilan-525400bbf613 stop on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0 * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 * Fencing controller-1 (reboot) - * Pseudo action: galera-0_stop_0 * Pseudo action: redis_post_notify_stop_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-pre_notify_stop_0 * Pseudo action: redis-master_stop_0 * Pseudo action: haproxy-clone_stop_0 * Fencing galera-2 (reboot) * Pseudo action: galera_demote_0 * Pseudo action: galera-master_demoted_0 * Pseudo action: galera-master_stop_0 * Pseudo action: redis_stop_0 * Pseudo action: redis-master_stopped_0 * Pseudo action: haproxy_stop_0 * Pseudo action: haproxy-clone_stopped_0 * Fencing messaging-1 (reboot) * Pseudo action: stonith_complete - * Resource action: galera-0 start on controller-2 * Pseudo action: rabbitmq_post_notify_stop_0 * Pseudo action: rabbitmq-clone_stop_0 * Pseudo action: galera_stop_0 - * Resource action: galera monitor=10000 on galera-0 * Pseudo action: galera-master_stopped_0 * Pseudo action: redis-master_post_notify_stopped_0 * Pseudo action: ip-172.17.1.14_stop_0 * Pseudo action: ip-172.17.1.17_stop_0 * Pseudo action: ip-172.17.4.11_stop_0 - * Resource action: galera-0 monitor=20000 on controller-2 * Pseudo action: galera-2_stop_0 * Resource action: rabbitmq notify on messaging-2 * Resource action: rabbitmq notify on messaging-0 * Pseudo action: rabbitmq_notified_0 * Pseudo action: rabbitmq_stop_0 * Pseudo action: rabbitmq-clone_stopped_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-post_notify_stopped_0 * Resource action: ip-172.17.1.14 start on controller-2 * Resource action: ip-172.17.1.17 start on controller-2 * Resource action: ip-172.17.4.11 start on controller-2 * Pseudo action: messaging-1_stop_0 * Pseudo action: redis_notified_0 * Resource action: ip-172.17.1.14 monitor=10000 on controller-2 * Resource action: ip-172.17.1.17 monitor=10000 on controller-2 * Resource action: ip-172.17.4.11 monitor=10000 on controller-2 * Pseudo action: all_stopped + * Resource action: galera-0 start on controller-2 + * Resource action: galera monitor=10000 on galera-0 * Resource action: stonith-fence_ipmilan-525400bbf613 start on controller-0 * Resource action: stonith-fence_ipmilan-525400bbf613 monitor=60000 on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd start on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0 * Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2 + * Resource action: galera-0 monitor=20000 on controller-2 * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 Using the original execution date of: 2017-05-03 13:33:24Z Revised cluster status: Online: [ controller-0 controller-2 ] OFFLINE: [ controller-1 ] RemoteOnline: [ galera-0 galera-1 messaging-0 messaging-2 ] RemoteOFFLINE: [ galera-2 messaging-1 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Stopped messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-2 galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Stopped Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-1 ] Master/Slave Set: galera-master [galera] Masters: [ galera-0 galera-1 ] Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ] Master/Slave Set: redis-master [redis] Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-2 Clone Set: haproxy-clone [haproxy] Started: [ controller-0 controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-2 diff --git a/pengine/test10/remote-recover-connection.dot b/pengine/test10/remote-recover-connection.dot index d6fdefede2..6cd342fa27 100644 --- a/pengine/test10/remote-recover-connection.dot +++ b/pengine/test10/remote-recover-connection.dot @@ -1,131 +1,125 @@ digraph "g" { "all_stopped" [ style=bold color="green" fontcolor="orange"] "galera-0_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_start_0 controller-2" -> "galera-0_monitor_20000 controller-2" [ style = bold] "galera-0_start_0 controller-2" -> "galera_monitor_10000 galera-0" [ style = bold] "galera-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_stop_0 controller-1" -> "all_stopped" [ style = bold] "galera-0_stop_0 controller-1" -> "galera-0_start_0 controller-2" [ style = bold] "galera-0_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera-2_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"] "galera-2_start_0 controller-2" -> "galera-2_monitor_20000 controller-2" [ style = bold] "galera-2_start_0 controller-2" -> "galera_monitor_10000 galera-2" [ style = bold] "galera-2_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "galera-2_stop_0 controller-1" -> "all_stopped" [ style = bold] "galera-2_stop_0 controller-1" -> "galera-2_start_0 controller-2" [ style = bold] "galera-2_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"] "galera_monitor_10000 galera-2" [ style=bold color="green" fontcolor="black"] "haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy-clone_stop_0" -> "haproxy_stop_0 controller-1" [ style = bold] "haproxy-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "haproxy-clone_stopped_0" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "haproxy_stop_0 controller-1" -> "all_stopped" [ style = bold] "haproxy_stop_0 controller-1" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.14_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_start_0 controller-2" -> "ip-172.17.1.14_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.14_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_stop_0 controller-1" -> "all_stopped" [ style = bold] "ip-172.17.1.14_stop_0 controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "ip-172.17.1.14_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.17_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_start_0 controller-2" -> "ip-172.17.1.17_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.17_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_stop_0 controller-1" -> "all_stopped" [ style = bold] "ip-172.17.1.17_stop_0 controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "ip-172.17.1.17_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.4.11_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_start_0 controller-2" -> "ip-172.17.4.11_monitor_10000 controller-2" [ style = bold] "ip-172.17.4.11_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_stop_0 controller-1" -> "all_stopped" [ style = bold] "ip-172.17.4.11_stop_0 controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "ip-172.17.4.11_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "messaging-1_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"] "messaging-1_start_0 controller-2" -> "messaging-1_monitor_20000 controller-2" [ style = bold] "messaging-1_start_0 controller-2" -> "rabbitmq_monitor_10000 messaging-1" [ style = bold] "messaging-1_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "messaging-1_stop_0 controller-1" -> "all_stopped" [ style = bold] "messaging-1_stop_0 controller-1" -> "messaging-1_start_0 controller-2" [ style = bold] "messaging-1_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "rabbitmq_monitor_10000 messaging-1" [ style=bold color="green" fontcolor="black"] "redis-master_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold] "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_stop_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_post_notify_stopped_0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_pre_notify_stop_0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-2" [ style = bold] "redis-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stop_0" -> "redis-master_stopped_0" [ style = bold] "redis-master_stop_0" -> "redis_stop_0 controller-1" [ style = bold] "redis-master_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stopped_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis_confirmed-post_notify_stonith_0" -> "all_stopped" [ style = bold] "redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_post_notify_stonith_0 controller-0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0 controller-2" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_pre_notify_stop_0 controller-0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_stop_0 controller-2" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_stop_0 controller-1" -> "all_stopped" [ style = bold] "redis_stop_0 controller-1" -> "redis-master_stopped_0" [ style = bold] "redis_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] -"stonith 'reboot' controller-1" -> "galera-0_stop_0 controller-1" [ style = bold] -"stonith 'reboot' controller-1" -> "galera-2_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "haproxy-clone_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "haproxy_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] -"stonith 'reboot' controller-1" -> "messaging-1_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "redis-master_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_post_notify_stonith_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "stonith_complete" [ style = bold] "stonith 'reboot' controller-1" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" -> "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith_complete" -> "all_stopped" [ style = bold] -"stonith_complete" -> "galera-0_start_0 controller-2" [ style = bold] -"stonith_complete" -> "galera-2_start_0 controller-2" [ style = bold] "stonith_complete" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "stonith_complete" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "stonith_complete" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] -"stonith_complete" -> "messaging-1_start_0 controller-2" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/remote-recover-connection.exp b/pengine/test10/remote-recover-connection.exp index cf74efb0f9..40338b406b 100644 --- a/pengine/test10/remote-recover-connection.exp +++ b/pengine/test10/remote-recover-connection.exp @@ -1,708 +1,687 @@ - - - - - - - - + - - - - - - - - + - - - - - - - - + diff --git a/pengine/test10/remote-recover-connection.summary b/pengine/test10/remote-recover-connection.summary index b0433fed01..8246cd958d 100644 --- a/pengine/test10/remote-recover-connection.summary +++ b/pengine/test10/remote-recover-connection.summary @@ -1,140 +1,140 @@ Using the original execution date of: 2017-05-03 13:33:24Z Current cluster status: Node controller-1 (2): UNCLEAN (offline) Online: [ controller-0 controller-2 ] RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-1 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ] Master/Slave Set: galera-master [galera] Masters: [ galera-0 galera-1 galera-2 ] Stopped: [ controller-0 controller-1 controller-2 messaging-0 messaging-1 messaging-2 ] Master/Slave Set: redis-master [redis] redis (ocf::heartbeat:redis): Slave controller-1 (UNCLEAN) Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) Clone Set: haproxy-clone [haproxy] haproxy (systemd:haproxy): Started controller-1 (UNCLEAN) Started: [ controller-0 controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) Transition Summary: * Fence (reboot) controller-1 'peer is no longer part of the cluster' * Move messaging-1 ( controller-1 -> controller-2 ) * Move galera-0 ( controller-1 -> controller-2 ) * Move galera-2 ( controller-1 -> controller-2 ) * Stop redis:0 ( Slave controller-1 ) due to node availability * Move ip-172.17.1.14 ( controller-1 -> controller-2 ) * Move ip-172.17.1.17 ( controller-1 -> controller-2 ) * Move ip-172.17.4.11 ( controller-1 -> controller-2 ) * Stop haproxy:0 (controller-1) due to node availability * Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) due to resource definition change * Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) due to resource definition change * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) Executing cluster transition: + * Pseudo action: messaging-1_stop_0 + * Pseudo action: galera-0_stop_0 + * Pseudo action: galera-2_stop_0 * Pseudo action: redis-master_pre_notify_stop_0 * Resource action: stonith-fence_ipmilan-525400bbf613 stop on controller-0 * Resource action: stonith-fence_ipmilan-525400bbf613 start on controller-0 * Resource action: stonith-fence_ipmilan-525400bbf613 monitor=60000 on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd start on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0 * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 * Fencing controller-1 (reboot) - * Pseudo action: messaging-1_stop_0 - * Pseudo action: galera-0_stop_0 - * Pseudo action: galera-2_stop_0 + * Resource action: messaging-1 start on controller-2 + * Resource action: galera-0 start on controller-2 + * Resource action: galera-2 start on controller-2 + * Resource action: rabbitmq monitor=10000 on messaging-1 + * Resource action: galera monitor=10000 on galera-2 + * Resource action: galera monitor=10000 on galera-0 * Pseudo action: redis_post_notify_stop_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-pre_notify_stop_0 * Pseudo action: redis-master_stop_0 * Pseudo action: haproxy-clone_stop_0 * Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2 * Pseudo action: stonith_complete - * Resource action: messaging-1 start on controller-2 - * Resource action: galera-0 start on controller-2 - * Resource action: galera-2 start on controller-2 - * Resource action: rabbitmq monitor=10000 on messaging-1 - * Resource action: galera monitor=10000 on galera-2 - * Resource action: galera monitor=10000 on galera-0 + * Resource action: messaging-1 monitor=20000 on controller-2 + * Resource action: galera-0 monitor=20000 on controller-2 + * Resource action: galera-2 monitor=20000 on controller-2 * Pseudo action: redis_stop_0 * Pseudo action: redis-master_stopped_0 * Pseudo action: haproxy_stop_0 * Pseudo action: haproxy-clone_stopped_0 * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 - * Resource action: messaging-1 monitor=20000 on controller-2 - * Resource action: galera-0 monitor=20000 on controller-2 - * Resource action: galera-2 monitor=20000 on controller-2 * Pseudo action: redis-master_post_notify_stopped_0 * Pseudo action: ip-172.17.1.14_stop_0 * Pseudo action: ip-172.17.1.17_stop_0 * Pseudo action: ip-172.17.4.11_stop_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-post_notify_stopped_0 * Resource action: ip-172.17.1.14 start on controller-2 * Resource action: ip-172.17.1.17 start on controller-2 * Resource action: ip-172.17.4.11 start on controller-2 * Pseudo action: redis_notified_0 * Resource action: ip-172.17.1.14 monitor=10000 on controller-2 * Resource action: ip-172.17.1.17 monitor=10000 on controller-2 * Resource action: ip-172.17.4.11 monitor=10000 on controller-2 * Pseudo action: all_stopped Using the original execution date of: 2017-05-03 13:33:24Z Revised cluster status: Online: [ controller-0 controller-2 ] OFFLINE: [ controller-1 ] RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Started controller-2 messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-2 galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Started controller-2 Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-1 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ] Master/Slave Set: galera-master [galera] Masters: [ galera-0 galera-1 galera-2 ] Stopped: [ controller-0 controller-1 controller-2 messaging-0 messaging-1 messaging-2 ] Master/Slave Set: redis-master [redis] Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-2 Clone Set: haproxy-clone [haproxy] Started: [ controller-0 controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-2 diff --git a/pengine/test10/remote-recover-fail.dot b/pengine/test10/remote-recover-fail.dot index 7b6edaa413..3375687f91 100644 --- a/pengine/test10/remote-recover-fail.dot +++ b/pengine/test10/remote-recover-fail.dot @@ -1,38 +1,38 @@ digraph "g" { "FAKE1_monitor_10000 rhel7-auto2" [ style=bold color="green" fontcolor="black"] "FAKE1_start_0 rhel7-auto2" -> "FAKE1_monitor_10000 rhel7-auto2" [ style = bold] "FAKE1_start_0 rhel7-auto2" [ style=bold color="green" fontcolor="black"] "FAKE2_monitor_10000 rhel7-auto3" [ style=bold color="green" fontcolor="black"] "FAKE2_start_0 rhel7-auto3" -> "FAKE2_monitor_10000 rhel7-auto3" [ style = bold] "FAKE2_start_0 rhel7-auto3" [ style=bold color="green" fontcolor="black"] "FAKE2_stop_0 rhel7-auto4" -> "FAKE2_start_0 rhel7-auto3" [ style = bold] "FAKE2_stop_0 rhel7-auto4" -> "all_stopped" [ style = bold] "FAKE2_stop_0 rhel7-auto4" -> "rhel7-auto4_stop_0 rhel7-auto2" [ style = bold] "FAKE2_stop_0 rhel7-auto4" [ style=bold color="green" fontcolor="orange"] "FAKE3_monitor_10000 rhel7-auto2" [ style=bold color="green" fontcolor="black"] "FAKE4_monitor_10000 rhel7-auto3" [ style=bold color="green" fontcolor="black"] "FAKE6_monitor_10000 rhel7-auto2" [ style=bold color="green" fontcolor="black"] "FAKE6_start_0 rhel7-auto2" -> "FAKE6_monitor_10000 rhel7-auto2" [ style = bold] "FAKE6_start_0 rhel7-auto2" [ style=bold color="green" fontcolor="black"] "FAKE6_stop_0 rhel7-auto4" -> "FAKE6_start_0 rhel7-auto2" [ style = bold] "FAKE6_stop_0 rhel7-auto4" -> "all_stopped" [ style = bold] "FAKE6_stop_0 rhel7-auto4" -> "rhel7-auto4_stop_0 rhel7-auto2" [ style = bold] "FAKE6_stop_0 rhel7-auto4" [ style=bold color="green" fontcolor="orange"] +"all_stopped" -> "rhel7-auto4_start_0 rhel7-auto2" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "rhel7-auto4_monitor_60000 rhel7-auto2" [ style=bold color="green" fontcolor="black"] "rhel7-auto4_start_0 rhel7-auto2" -> "rhel7-auto4_monitor_60000 rhel7-auto2" [ style = bold] "rhel7-auto4_start_0 rhel7-auto2" [ style=bold color="green" fontcolor="black"] "rhel7-auto4_stop_0 rhel7-auto2" -> "all_stopped" [ style = bold] "rhel7-auto4_stop_0 rhel7-auto2" -> "rhel7-auto4_start_0 rhel7-auto2" [ style = bold] "rhel7-auto4_stop_0 rhel7-auto2" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' rhel7-auto4" -> "FAKE2_stop_0 rhel7-auto4" [ style = bold] "stonith 'reboot' rhel7-auto4" -> "FAKE6_stop_0 rhel7-auto4" [ style = bold] "stonith 'reboot' rhel7-auto4" -> "stonith_complete" [ style = bold] "stonith 'reboot' rhel7-auto4" [ style=bold color="green" fontcolor="black"] "stonith_complete" -> "FAKE1_start_0 rhel7-auto2" [ style = bold] "stonith_complete" -> "FAKE2_start_0 rhel7-auto3" [ style = bold] "stonith_complete" -> "FAKE6_start_0 rhel7-auto2" [ style = bold] "stonith_complete" -> "all_stopped" [ style = bold] -"stonith_complete" -> "rhel7-auto4_start_0 rhel7-auto2" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/remote-recover-fail.exp b/pengine/test10/remote-recover-fail.exp index bd014ae66c..f9085666ca 100644 --- a/pengine/test10/remote-recover-fail.exp +++ b/pengine/test10/remote-recover-fail.exp @@ -1,220 +1,220 @@ - + diff --git a/pengine/test10/remote-recover-no-resources.dot b/pengine/test10/remote-recover-no-resources.dot index 1e16221065..8c2f783154 100644 --- a/pengine/test10/remote-recover-no-resources.dot +++ b/pengine/test10/remote-recover-no-resources.dot @@ -1,143 +1,142 @@ digraph "g" { +"all_stopped" -> "galera-0_start_0 controller-2" [ style = bold] "all_stopped" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "all_stopped" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "all_stopped" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "galera-0_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_start_0 controller-2" -> "galera-0_monitor_20000 controller-2" [ style = bold] "galera-0_start_0 controller-2" -> "galera_monitor_10000 galera-0" [ style = bold] "galera-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_stop_0 controller-1" -> "all_stopped" [ style = bold] "galera-0_stop_0 controller-1" -> "galera-0_start_0 controller-2" [ style = bold] "galera-0_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera-2_stop_0 controller-1" -> "all_stopped" [ style = bold] "galera-2_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"] "haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy-clone_stop_0" -> "haproxy_stop_0 controller-1" [ style = bold] "haproxy-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "haproxy-clone_stopped_0" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "haproxy_stop_0 controller-1" -> "all_stopped" [ style = bold] "haproxy_stop_0 controller-1" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.14_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_start_0 controller-2" -> "ip-172.17.1.14_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.14_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_stop_0 controller-1" -> "all_stopped" [ style = bold] "ip-172.17.1.14_stop_0 controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "ip-172.17.1.14_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.17_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_start_0 controller-2" -> "ip-172.17.1.17_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.17_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_stop_0 controller-1" -> "all_stopped" [ style = bold] "ip-172.17.1.17_stop_0 controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "ip-172.17.1.17_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.4.11_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_start_0 controller-2" -> "ip-172.17.4.11_monitor_10000 controller-2" [ style = bold] "ip-172.17.4.11_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_stop_0 controller-1" -> "all_stopped" [ style = bold] "ip-172.17.4.11_stop_0 controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "ip-172.17.4.11_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "messaging-1_stop_0 controller-1" -> "all_stopped" [ style = bold] "messaging-1_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "rabbitmq-clone_stop_0" -> "rabbitmq-clone_stopped_0" [ style = bold] "rabbitmq-clone_stop_0" -> "rabbitmq_stop_0 messaging-1" [ style = bold] "rabbitmq-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_confirmed-post_notify_stonith_0" -> "all_stopped" [ style = bold] "rabbitmq_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_post_notify_stonith_0 messaging-0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0 messaging-0" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_stonith_0 messaging-2" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0 messaging-2" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_stop_0 messaging-1" -> "all_stopped" [ style = bold] "rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] "redis-master_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold] "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_stop_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_post_notify_stopped_0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_pre_notify_stop_0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-2" [ style = bold] "redis-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stop_0" -> "redis-master_stopped_0" [ style = bold] "redis-master_stop_0" -> "redis_stop_0 controller-1" [ style = bold] "redis-master_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stopped_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis_confirmed-post_notify_stonith_0" -> "all_stopped" [ style = bold] "redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_post_notify_stonith_0 controller-0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0 controller-2" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_pre_notify_stop_0 controller-0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_stop_0 controller-2" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_stop_0 controller-1" -> "all_stopped" [ style = bold] "redis_stop_0 controller-1" -> "redis-master_stopped_0" [ style = bold] "redis_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] -"stonith 'reboot' controller-1" -> "galera-0_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "galera-2_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "haproxy-clone_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "haproxy_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "messaging-1_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "redis-master_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_post_notify_stonith_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "stonith 'reboot' messaging-1" [ style = bold] "stonith 'reboot' controller-1" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' messaging-1" -> "rabbitmq-clone_stop_0" [ style = bold] "stonith 'reboot' messaging-1" -> "rabbitmq_post_notify_stonith_0" [ style = bold] "stonith 'reboot' messaging-1" -> "rabbitmq_stop_0 messaging-1" [ style = bold] "stonith 'reboot' messaging-1" -> "stonith_complete" [ style = bold] "stonith 'reboot' messaging-1" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" -> "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith_complete" -> "all_stopped" [ style = bold] -"stonith_complete" -> "galera-0_start_0 controller-2" [ style = bold] "stonith_complete" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "stonith_complete" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "stonith_complete" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/remote-recover-no-resources.exp b/pengine/test10/remote-recover-no-resources.exp index 987acfdd9a..0a57e2737a 100644 --- a/pengine/test10/remote-recover-no-resources.exp +++ b/pengine/test10/remote-recover-no-resources.exp @@ -1,755 +1,751 @@ - + - + - - - - - + diff --git a/pengine/test10/remote-recover-no-resources.summary b/pengine/test10/remote-recover-no-resources.summary index b682e5fdc5..bed02d0452 100644 --- a/pengine/test10/remote-recover-no-resources.summary +++ b/pengine/test10/remote-recover-no-resources.summary @@ -1,145 +1,145 @@ Using the original execution date of: 2017-05-03 13:33:24Z Current cluster status: Node controller-1 (2): UNCLEAN (offline) Online: [ controller-0 controller-2 ] RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-1 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ] Master/Slave Set: galera-master [galera] Masters: [ galera-0 galera-1 ] Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ] Master/Slave Set: redis-master [redis] redis (ocf::heartbeat:redis): Slave controller-1 (UNCLEAN) Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) Clone Set: haproxy-clone [haproxy] haproxy (systemd:haproxy): Started controller-1 (UNCLEAN) Started: [ controller-0 controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) Transition Summary: * Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable' * Fence (reboot) controller-1 'peer is no longer part of the cluster' * Stop messaging-1 (controller-1) due to node availability * Move galera-0 ( controller-1 -> controller-2 ) * Stop galera-2 ( controller-1 ) due to node availability * Stop rabbitmq:2 (messaging-1) due to node availability * Stop redis:0 ( Slave controller-1 ) due to node availability * Move ip-172.17.1.14 ( controller-1 -> controller-2 ) * Move ip-172.17.1.17 ( controller-1 -> controller-2 ) * Move ip-172.17.4.11 ( controller-1 -> controller-2 ) * Stop haproxy:0 (controller-1) due to node availability * Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) due to resource definition change * Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) due to resource definition change * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) Executing cluster transition: + * Pseudo action: galera-0_stop_0 * Pseudo action: redis-master_pre_notify_stop_0 * Resource action: stonith-fence_ipmilan-525400bbf613 stop on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0 * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 * Fencing controller-1 (reboot) - * Pseudo action: galera-0_stop_0 * Pseudo action: galera-2_stop_0 * Pseudo action: redis_post_notify_stop_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-pre_notify_stop_0 * Pseudo action: redis-master_stop_0 * Pseudo action: haproxy-clone_stop_0 * Fencing messaging-1 (reboot) * Pseudo action: stonith_complete - * Resource action: galera-0 start on controller-2 * Pseudo action: rabbitmq_post_notify_stop_0 * Pseudo action: rabbitmq-clone_stop_0 - * Resource action: galera monitor=10000 on galera-0 * Pseudo action: redis_stop_0 * Pseudo action: redis-master_stopped_0 * Pseudo action: haproxy_stop_0 * Pseudo action: haproxy-clone_stopped_0 - * Resource action: galera-0 monitor=20000 on controller-2 * Resource action: rabbitmq notify on messaging-2 * Resource action: rabbitmq notify on messaging-0 * Pseudo action: rabbitmq_notified_0 * Pseudo action: rabbitmq_stop_0 * Pseudo action: rabbitmq-clone_stopped_0 * Pseudo action: redis-master_post_notify_stopped_0 * Pseudo action: ip-172.17.1.14_stop_0 * Pseudo action: ip-172.17.1.17_stop_0 * Pseudo action: ip-172.17.4.11_stop_0 * Pseudo action: messaging-1_stop_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-post_notify_stopped_0 * Resource action: ip-172.17.1.14 start on controller-2 * Resource action: ip-172.17.1.17 start on controller-2 * Resource action: ip-172.17.4.11 start on controller-2 * Pseudo action: redis_notified_0 * Resource action: ip-172.17.1.14 monitor=10000 on controller-2 * Resource action: ip-172.17.1.17 monitor=10000 on controller-2 * Resource action: ip-172.17.4.11 monitor=10000 on controller-2 * Pseudo action: all_stopped + * Resource action: galera-0 start on controller-2 + * Resource action: galera monitor=10000 on galera-0 * Resource action: stonith-fence_ipmilan-525400bbf613 start on controller-0 * Resource action: stonith-fence_ipmilan-525400bbf613 monitor=60000 on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd start on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0 * Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2 + * Resource action: galera-0 monitor=20000 on controller-2 * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 Using the original execution date of: 2017-05-03 13:33:24Z Revised cluster status: Online: [ controller-0 controller-2 ] OFFLINE: [ controller-1 ] RemoteOnline: [ galera-0 galera-1 messaging-0 messaging-2 ] RemoteOFFLINE: [ galera-2 messaging-1 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Stopped messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-2 galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Stopped Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-1 ] Master/Slave Set: galera-master [galera] Masters: [ galera-0 galera-1 ] Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ] Master/Slave Set: redis-master [redis] Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-2 Clone Set: haproxy-clone [haproxy] Started: [ controller-0 controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-2 diff --git a/pengine/test10/remote-recover-unknown.dot b/pengine/test10/remote-recover-unknown.dot index a8b4e18ae8..8ce59b4443 100644 --- a/pengine/test10/remote-recover-unknown.dot +++ b/pengine/test10/remote-recover-unknown.dot @@ -1,146 +1,145 @@ digraph "g" { +"all_stopped" -> "galera-0_start_0 controller-2" [ style = bold] "all_stopped" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "all_stopped" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "all_stopped" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "galera-0_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_start_0 controller-2" -> "galera-0_monitor_20000 controller-2" [ style = bold] "galera-0_start_0 controller-2" -> "galera_monitor_10000 galera-0" [ style = bold] "galera-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_stop_0 controller-1" -> "all_stopped" [ style = bold] "galera-0_stop_0 controller-1" -> "galera-0_start_0 controller-2" [ style = bold] "galera-0_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera-2_stop_0 controller-1" -> "all_stopped" [ style = bold] "galera-2_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera_monitor_0 galera-2" [ style=dashed color="red" fontcolor="black"] "galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"] "haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy-clone_stop_0" -> "haproxy_stop_0 controller-1" [ style = bold] "haproxy-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "haproxy-clone_stopped_0" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "haproxy_stop_0 controller-1" -> "all_stopped" [ style = bold] "haproxy_stop_0 controller-1" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.14_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_start_0 controller-2" -> "ip-172.17.1.14_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.14_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_stop_0 controller-1" -> "all_stopped" [ style = bold] "ip-172.17.1.14_stop_0 controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "ip-172.17.1.14_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.17_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_start_0 controller-2" -> "ip-172.17.1.17_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.17_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_stop_0 controller-1" -> "all_stopped" [ style = bold] "ip-172.17.1.17_stop_0 controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "ip-172.17.1.17_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.4.11_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_start_0 controller-2" -> "ip-172.17.4.11_monitor_10000 controller-2" [ style = bold] "ip-172.17.4.11_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_stop_0 controller-1" -> "all_stopped" [ style = bold] "ip-172.17.4.11_stop_0 controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "ip-172.17.4.11_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "messaging-1_stop_0 controller-1" -> "all_stopped" [ style = bold] "messaging-1_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "rabbitmq-clone_stop_0" -> "rabbitmq-clone_stopped_0" [ style = bold] "rabbitmq-clone_stop_0" -> "rabbitmq_stop_0 messaging-1" [ style = bold] "rabbitmq-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_confirmed-post_notify_stonith_0" -> "all_stopped" [ style = bold] "rabbitmq_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_post_notify_stonith_0 messaging-0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0 messaging-0" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_stonith_0 messaging-2" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0 messaging-2" [ style=bold color="green" fontcolor="black"] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-0" [ style = bold] "rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 messaging-2" [ style = bold] "rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "rabbitmq_stop_0 messaging-1" -> "all_stopped" [ style = bold] "rabbitmq_stop_0 messaging-1" -> "messaging-1_stop_0 controller-1" [ style = bold] "rabbitmq_stop_0 messaging-1" -> "rabbitmq-clone_stopped_0" [ style = bold] "rabbitmq_stop_0 messaging-1" [ style=bold color="green" fontcolor="orange"] "redis-master_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold] "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_stop_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_post_notify_stopped_0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_pre_notify_stop_0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-2" [ style = bold] "redis-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stop_0" -> "redis-master_stopped_0" [ style = bold] "redis-master_stop_0" -> "redis_stop_0 controller-1" [ style = bold] "redis-master_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stopped_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis_confirmed-post_notify_stonith_0" -> "all_stopped" [ style = bold] "redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_post_notify_stonith_0 controller-0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0 controller-2" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_pre_notify_stop_0 controller-0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_stop_0 controller-2" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_stop_0 controller-1" -> "all_stopped" [ style = bold] "redis_stop_0 controller-1" -> "redis-master_stopped_0" [ style = bold] "redis_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] -"stonith 'reboot' controller-1" -> "galera-0_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "galera-2_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "haproxy-clone_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "haproxy_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "messaging-1_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "redis-master_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_post_notify_stonith_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "stonith 'reboot' galera-2" [ style = bold] "stonith 'reboot' controller-1" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' galera-2" -> "stonith 'reboot' messaging-1" [ style = bold] "stonith 'reboot' galera-2" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' messaging-1" -> "rabbitmq-clone_stop_0" [ style = bold] "stonith 'reboot' messaging-1" -> "rabbitmq_post_notify_stonith_0" [ style = bold] "stonith 'reboot' messaging-1" -> "rabbitmq_stop_0 messaging-1" [ style = bold] "stonith 'reboot' messaging-1" -> "stonith_complete" [ style = bold] "stonith 'reboot' messaging-1" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" -> "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith_complete" -> "all_stopped" [ style = bold] -"stonith_complete" -> "galera-0_start_0 controller-2" [ style = bold] "stonith_complete" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "stonith_complete" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "stonith_complete" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/remote-recover-unknown.exp b/pengine/test10/remote-recover-unknown.exp index b8d51be76d..0d7b318d09 100644 --- a/pengine/test10/remote-recover-unknown.exp +++ b/pengine/test10/remote-recover-unknown.exp @@ -1,770 +1,766 @@ - + - + - - - - - + diff --git a/pengine/test10/remote-recover-unknown.summary b/pengine/test10/remote-recover-unknown.summary index 09f10d84ae..d47f174fb1 100644 --- a/pengine/test10/remote-recover-unknown.summary +++ b/pengine/test10/remote-recover-unknown.summary @@ -1,147 +1,147 @@ Using the original execution date of: 2017-05-03 13:33:24Z Current cluster status: Node controller-1 (2): UNCLEAN (offline) Online: [ controller-0 controller-2 ] RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-1 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ] Master/Slave Set: galera-master [galera] Masters: [ galera-0 galera-1 ] Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ] Master/Slave Set: redis-master [redis] redis (ocf::heartbeat:redis): Slave controller-1 (UNCLEAN) Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) Clone Set: haproxy-clone [haproxy] haproxy (systemd:haproxy): Started controller-1 (UNCLEAN) Started: [ controller-0 controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) Transition Summary: * Fence (reboot) galera-2 'resources are in an unknown state and the connection is unrecoverable' * Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable' * Fence (reboot) controller-1 'peer is no longer part of the cluster' * Stop messaging-1 (controller-1) due to node availability * Move galera-0 ( controller-1 -> controller-2 ) * Stop galera-2 (controller-1) due to node availability * Stop rabbitmq:2 (messaging-1) due to node availability * Stop redis:0 ( Slave controller-1 ) due to node availability * Move ip-172.17.1.14 ( controller-1 -> controller-2 ) * Move ip-172.17.1.17 ( controller-1 -> controller-2 ) * Move ip-172.17.4.11 ( controller-1 -> controller-2 ) * Stop haproxy:0 (controller-1) due to node availability * Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) due to resource definition change * Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) due to resource definition change * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) Executing cluster transition: + * Pseudo action: galera-0_stop_0 * Pseudo action: redis-master_pre_notify_stop_0 * Resource action: stonith-fence_ipmilan-525400bbf613 stop on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0 * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 * Fencing controller-1 (reboot) - * Pseudo action: galera-0_stop_0 * Pseudo action: galera-2_stop_0 * Pseudo action: redis_post_notify_stop_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-pre_notify_stop_0 * Pseudo action: redis-master_stop_0 * Pseudo action: haproxy-clone_stop_0 * Fencing galera-2 (reboot) * Fencing messaging-1 (reboot) * Pseudo action: stonith_complete - * Resource action: galera-0 start on controller-2 * Pseudo action: rabbitmq_post_notify_stop_0 * Pseudo action: rabbitmq-clone_stop_0 - * Resource action: galera monitor=10000 on galera-0 * Pseudo action: redis_stop_0 * Pseudo action: redis-master_stopped_0 * Pseudo action: haproxy_stop_0 * Pseudo action: haproxy-clone_stopped_0 - * Resource action: galera-0 monitor=20000 on controller-2 * Resource action: rabbitmq notify on messaging-2 * Resource action: rabbitmq notify on messaging-0 * Pseudo action: rabbitmq_notified_0 * Pseudo action: rabbitmq_stop_0 * Pseudo action: rabbitmq-clone_stopped_0 * Pseudo action: redis-master_post_notify_stopped_0 * Pseudo action: ip-172.17.1.14_stop_0 * Pseudo action: ip-172.17.1.17_stop_0 * Pseudo action: ip-172.17.4.11_stop_0 * Pseudo action: messaging-1_stop_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-post_notify_stopped_0 * Resource action: ip-172.17.1.14 start on controller-2 * Resource action: ip-172.17.1.17 start on controller-2 * Resource action: ip-172.17.4.11 start on controller-2 * Pseudo action: redis_notified_0 * Resource action: ip-172.17.1.14 monitor=10000 on controller-2 * Resource action: ip-172.17.1.17 monitor=10000 on controller-2 * Resource action: ip-172.17.4.11 monitor=10000 on controller-2 * Pseudo action: all_stopped + * Resource action: galera-0 start on controller-2 + * Resource action: galera monitor=10000 on galera-0 * Resource action: stonith-fence_ipmilan-525400bbf613 start on controller-0 * Resource action: stonith-fence_ipmilan-525400bbf613 monitor=60000 on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd start on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0 * Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2 + * Resource action: galera-0 monitor=20000 on controller-2 * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 Using the original execution date of: 2017-05-03 13:33:24Z Revised cluster status: Online: [ controller-0 controller-2 ] OFFLINE: [ controller-1 ] RemoteOnline: [ galera-0 galera-1 messaging-0 messaging-2 ] RemoteOFFLINE: [ galera-2 messaging-1 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Stopped messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-2 galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Stopped Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 messaging-1 ] Master/Slave Set: galera-master [galera] Masters: [ galera-0 galera-1 ] Stopped: [ controller-0 controller-1 controller-2 galera-2 messaging-0 messaging-1 messaging-2 ] Master/Slave Set: redis-master [redis] Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-2 Clone Set: haproxy-clone [haproxy] Started: [ controller-0 controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-2 diff --git a/pengine/test10/remote-recovery.dot b/pengine/test10/remote-recovery.dot index d6fdefede2..6cd342fa27 100644 --- a/pengine/test10/remote-recovery.dot +++ b/pengine/test10/remote-recovery.dot @@ -1,131 +1,125 @@ digraph "g" { "all_stopped" [ style=bold color="green" fontcolor="orange"] "galera-0_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_start_0 controller-2" -> "galera-0_monitor_20000 controller-2" [ style = bold] "galera-0_start_0 controller-2" -> "galera_monitor_10000 galera-0" [ style = bold] "galera-0_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "galera-0_stop_0 controller-1" -> "all_stopped" [ style = bold] "galera-0_stop_0 controller-1" -> "galera-0_start_0 controller-2" [ style = bold] "galera-0_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera-2_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"] "galera-2_start_0 controller-2" -> "galera-2_monitor_20000 controller-2" [ style = bold] "galera-2_start_0 controller-2" -> "galera_monitor_10000 galera-2" [ style = bold] "galera-2_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "galera-2_stop_0 controller-1" -> "all_stopped" [ style = bold] "galera-2_stop_0 controller-1" -> "galera-2_start_0 controller-2" [ style = bold] "galera-2_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "galera_monitor_10000 galera-0" [ style=bold color="green" fontcolor="black"] "galera_monitor_10000 galera-2" [ style=bold color="green" fontcolor="black"] "haproxy-clone_stop_0" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy-clone_stop_0" -> "haproxy_stop_0 controller-1" [ style = bold] "haproxy-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "haproxy-clone_stopped_0" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] "haproxy-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "haproxy_stop_0 controller-1" -> "all_stopped" [ style = bold] "haproxy_stop_0 controller-1" -> "haproxy-clone_stopped_0" [ style = bold] "haproxy_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.14_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_start_0 controller-2" -> "ip-172.17.1.14_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.14_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.14_stop_0 controller-1" -> "all_stopped" [ style = bold] "ip-172.17.1.14_stop_0 controller-1" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "ip-172.17.1.14_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.1.17_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_start_0 controller-2" -> "ip-172.17.1.17_monitor_10000 controller-2" [ style = bold] "ip-172.17.1.17_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.1.17_stop_0 controller-1" -> "all_stopped" [ style = bold] "ip-172.17.1.17_stop_0 controller-1" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "ip-172.17.1.17_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "ip-172.17.4.11_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_start_0 controller-2" -> "ip-172.17.4.11_monitor_10000 controller-2" [ style = bold] "ip-172.17.4.11_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "ip-172.17.4.11_stop_0 controller-1" -> "all_stopped" [ style = bold] "ip-172.17.4.11_stop_0 controller-1" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] "ip-172.17.4.11_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "messaging-1_monitor_20000 controller-2" [ style=bold color="green" fontcolor="black"] "messaging-1_start_0 controller-2" -> "messaging-1_monitor_20000 controller-2" [ style = bold] "messaging-1_start_0 controller-2" -> "rabbitmq_monitor_10000 messaging-1" [ style = bold] "messaging-1_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "messaging-1_stop_0 controller-1" -> "all_stopped" [ style = bold] "messaging-1_stop_0 controller-1" -> "messaging-1_start_0 controller-2" [ style = bold] "messaging-1_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "rabbitmq_monitor_10000 messaging-1" [ style=bold color="green" fontcolor="black"] "redis-master_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold] "redis-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" -> "redis-master_stop_0" [ style = bold] "redis-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_post_notify_stopped_0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis-master_pre_notify_stop_0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-0" [ style = bold] "redis-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 controller-2" [ style = bold] "redis-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stop_0" -> "redis-master_stopped_0" [ style = bold] "redis-master_stop_0" -> "redis_stop_0 controller-1" [ style = bold] "redis-master_stop_0" [ style=bold color="green" fontcolor="orange"] "redis-master_stopped_0" -> "redis-master_post_notify_stopped_0" [ style = bold] "redis-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "redis_confirmed-post_notify_stonith_0" -> "all_stopped" [ style = bold] "redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_post_notify_stonith_0 controller-0" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0 controller-2" -> "redis-master_confirmed-post_notify_stopped_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-0" [ style = bold] "redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 controller-2" [ style = bold] "redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] "redis_pre_notify_stop_0 controller-0" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "redis_pre_notify_stop_0 controller-2" -> "redis-master_confirmed-pre_notify_stop_0" [ style = bold] "redis_pre_notify_stop_0 controller-2" [ style=bold color="green" fontcolor="black"] "redis_stop_0 controller-1" -> "all_stopped" [ style = bold] "redis_stop_0 controller-1" -> "redis-master_stopped_0" [ style = bold] "redis_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] -"stonith 'reboot' controller-1" -> "galera-0_stop_0 controller-1" [ style = bold] -"stonith 'reboot' controller-1" -> "galera-2_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "haproxy-clone_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "haproxy_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.14_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.1.17_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "ip-172.17.4.11_stop_0 controller-1" [ style = bold] -"stonith 'reboot' controller-1" -> "messaging-1_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "redis-master_stop_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_post_notify_stonith_0" [ style = bold] "stonith 'reboot' controller-1" -> "redis_stop_0 controller-1" [ style = bold] "stonith 'reboot' controller-1" -> "stonith_complete" [ style = bold] "stonith 'reboot' controller-1" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" -> "stonith-fence_ipmilan-5254005bdbb5_monitor_60000 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" -> "stonith-fence_ipmilan-5254005bdbb5_start_0 controller-2" [ style = bold] "stonith-fence_ipmilan-5254005bdbb5_stop_0 controller-1" [ style=bold color="green" fontcolor="orange"] "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" -> "stonith-fence_ipmilan-525400b4f6bd_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400b4f6bd_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_monitor_60000 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "all_stopped" [ style = bold] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" -> "stonith-fence_ipmilan-525400bbf613_start_0 controller-0" [ style = bold] "stonith-fence_ipmilan-525400bbf613_stop_0 controller-0" [ style=bold color="green" fontcolor="black"] "stonith_complete" -> "all_stopped" [ style = bold] -"stonith_complete" -> "galera-0_start_0 controller-2" [ style = bold] -"stonith_complete" -> "galera-2_start_0 controller-2" [ style = bold] "stonith_complete" -> "ip-172.17.1.14_start_0 controller-2" [ style = bold] "stonith_complete" -> "ip-172.17.1.17_start_0 controller-2" [ style = bold] "stonith_complete" -> "ip-172.17.4.11_start_0 controller-2" [ style = bold] -"stonith_complete" -> "messaging-1_start_0 controller-2" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/remote-recovery.exp b/pengine/test10/remote-recovery.exp index cf74efb0f9..40338b406b 100644 --- a/pengine/test10/remote-recovery.exp +++ b/pengine/test10/remote-recovery.exp @@ -1,708 +1,687 @@ - - - - - - - - + - - - - - - - - + - - - - - - - - + diff --git a/pengine/test10/remote-recovery.summary b/pengine/test10/remote-recovery.summary index b0433fed01..8246cd958d 100644 --- a/pengine/test10/remote-recovery.summary +++ b/pengine/test10/remote-recovery.summary @@ -1,140 +1,140 @@ Using the original execution date of: 2017-05-03 13:33:24Z Current cluster status: Node controller-1 (2): UNCLEAN (offline) Online: [ controller-0 controller-2 ] RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Started controller-1 (UNCLEAN) Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-1 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ] Master/Slave Set: galera-master [galera] Masters: [ galera-0 galera-1 galera-2 ] Stopped: [ controller-0 controller-1 controller-2 messaging-0 messaging-1 messaging-2 ] Master/Slave Set: redis-master [redis] redis (ocf::heartbeat:redis): Slave controller-1 (UNCLEAN) Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-1 (UNCLEAN) Clone Set: haproxy-clone [haproxy] haproxy (systemd:haproxy): Started controller-1 (UNCLEAN) Started: [ controller-0 controller-2 ] Stopped: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) Transition Summary: * Fence (reboot) controller-1 'peer is no longer part of the cluster' * Move messaging-1 ( controller-1 -> controller-2 ) * Move galera-0 ( controller-1 -> controller-2 ) * Move galera-2 ( controller-1 -> controller-2 ) * Stop redis:0 ( Slave controller-1 ) due to node availability * Move ip-172.17.1.14 ( controller-1 -> controller-2 ) * Move ip-172.17.1.17 ( controller-1 -> controller-2 ) * Move ip-172.17.4.11 ( controller-1 -> controller-2 ) * Stop haproxy:0 (controller-1) due to node availability * Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) due to resource definition change * Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) due to resource definition change * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) Executing cluster transition: + * Pseudo action: messaging-1_stop_0 + * Pseudo action: galera-0_stop_0 + * Pseudo action: galera-2_stop_0 * Pseudo action: redis-master_pre_notify_stop_0 * Resource action: stonith-fence_ipmilan-525400bbf613 stop on controller-0 * Resource action: stonith-fence_ipmilan-525400bbf613 start on controller-0 * Resource action: stonith-fence_ipmilan-525400bbf613 monitor=60000 on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd stop on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd start on controller-0 * Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0 * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 * Fencing controller-1 (reboot) - * Pseudo action: messaging-1_stop_0 - * Pseudo action: galera-0_stop_0 - * Pseudo action: galera-2_stop_0 + * Resource action: messaging-1 start on controller-2 + * Resource action: galera-0 start on controller-2 + * Resource action: galera-2 start on controller-2 + * Resource action: rabbitmq monitor=10000 on messaging-1 + * Resource action: galera monitor=10000 on galera-2 + * Resource action: galera monitor=10000 on galera-0 * Pseudo action: redis_post_notify_stop_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-pre_notify_stop_0 * Pseudo action: redis-master_stop_0 * Pseudo action: haproxy-clone_stop_0 * Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2 * Pseudo action: stonith_complete - * Resource action: messaging-1 start on controller-2 - * Resource action: galera-0 start on controller-2 - * Resource action: galera-2 start on controller-2 - * Resource action: rabbitmq monitor=10000 on messaging-1 - * Resource action: galera monitor=10000 on galera-2 - * Resource action: galera monitor=10000 on galera-0 + * Resource action: messaging-1 monitor=20000 on controller-2 + * Resource action: galera-0 monitor=20000 on controller-2 + * Resource action: galera-2 monitor=20000 on controller-2 * Pseudo action: redis_stop_0 * Pseudo action: redis-master_stopped_0 * Pseudo action: haproxy_stop_0 * Pseudo action: haproxy-clone_stopped_0 * Resource action: stonith-fence_ipmilan-5254005bdbb5 monitor=60000 on controller-2 - * Resource action: messaging-1 monitor=20000 on controller-2 - * Resource action: galera-0 monitor=20000 on controller-2 - * Resource action: galera-2 monitor=20000 on controller-2 * Pseudo action: redis-master_post_notify_stopped_0 * Pseudo action: ip-172.17.1.14_stop_0 * Pseudo action: ip-172.17.1.17_stop_0 * Pseudo action: ip-172.17.4.11_stop_0 * Resource action: redis notify on controller-0 * Resource action: redis notify on controller-2 * Pseudo action: redis-master_confirmed-post_notify_stopped_0 * Resource action: ip-172.17.1.14 start on controller-2 * Resource action: ip-172.17.1.17 start on controller-2 * Resource action: ip-172.17.4.11 start on controller-2 * Pseudo action: redis_notified_0 * Resource action: ip-172.17.1.14 monitor=10000 on controller-2 * Resource action: ip-172.17.1.17 monitor=10000 on controller-2 * Resource action: ip-172.17.4.11 monitor=10000 on controller-2 * Pseudo action: all_stopped Using the original execution date of: 2017-05-03 13:33:24Z Revised cluster status: Online: [ controller-0 controller-2 ] OFFLINE: [ controller-1 ] RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] messaging-0 (ocf::pacemaker:remote): Started controller-0 messaging-1 (ocf::pacemaker:remote): Started controller-2 messaging-2 (ocf::pacemaker:remote): Started controller-0 galera-0 (ocf::pacemaker:remote): Started controller-2 galera-1 (ocf::pacemaker:remote): Started controller-0 galera-2 (ocf::pacemaker:remote): Started controller-2 Clone Set: rabbitmq-clone [rabbitmq] Started: [ messaging-0 messaging-1 messaging-2 ] Stopped: [ controller-0 controller-1 controller-2 galera-0 galera-1 galera-2 ] Master/Slave Set: galera-master [galera] Masters: [ galera-0 galera-1 galera-2 ] Stopped: [ controller-0 controller-1 controller-2 messaging-0 messaging-1 messaging-2 ] Master/Slave Set: redis-master [redis] Masters: [ controller-0 ] Slaves: [ controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] ip-192.168.24.6 (ocf::heartbeat:IPaddr2): Started controller-0 ip-10.0.0.102 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.1.17 (ocf::heartbeat:IPaddr2): Started controller-2 ip-172.17.3.15 (ocf::heartbeat:IPaddr2): Started controller-0 ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-2 Clone Set: haproxy-clone [haproxy] Started: [ controller-0 controller-2 ] Stopped: [ controller-1 galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-0 stonith-fence_ipmilan-525400bbf613 (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-2 diff --git a/pengine/test10/remote-unclean2.dot b/pengine/test10/remote-unclean2.dot index 3f8981b88b..2311a72964 100644 --- a/pengine/test10/remote-unclean2.dot +++ b/pengine/test10/remote-unclean2.dot @@ -1,14 +1,14 @@ digraph "g" { +"all_stopped" -> "rhel7-auto4_start_0 rhel7-auto1" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "rhel7-auto4_monitor_60000 rhel7-auto1" [ style=bold color="green" fontcolor="black"] "rhel7-auto4_start_0 rhel7-auto1" -> "rhel7-auto4_monitor_60000 rhel7-auto1" [ style = bold] "rhel7-auto4_start_0 rhel7-auto1" [ style=bold color="green" fontcolor="black"] "rhel7-auto4_stop_0 rhel7-auto1" -> "all_stopped" [ style = bold] "rhel7-auto4_stop_0 rhel7-auto1" -> "rhel7-auto4_start_0 rhel7-auto1" [ style = bold] "rhel7-auto4_stop_0 rhel7-auto1" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' rhel7-auto4" -> "stonith_complete" [ style = bold] "stonith 'reboot' rhel7-auto4" [ style=bold color="green" fontcolor="black"] "stonith_complete" -> "all_stopped" [ style = bold] -"stonith_complete" -> "rhel7-auto4_start_0 rhel7-auto1" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/remote-unclean2.exp b/pengine/test10/remote-unclean2.exp index ca0b3ba692..2c73d82aa8 100644 --- a/pengine/test10/remote-unclean2.exp +++ b/pengine/test10/remote-unclean2.exp @@ -1,81 +1,81 @@ - + diff --git a/pengine/test10/whitebox-fail1.dot b/pengine/test10/whitebox-fail1.dot index bfff4bfb99..9b755f981c 100644 --- a/pengine/test10/whitebox-fail1.dot +++ b/pengine/test10/whitebox-fail1.dot @@ -1,60 +1,60 @@ digraph "g" { "A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "B_monitor_0 lxc2" -> "B_start_0 lxc1" [ style = bold] "B_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "B_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] "B_start_0 lxc1" -> "B_monitor_10000 lxc1" [ style = bold] "B_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "B_stop_0 lxc1" -> "B_start_0 lxc1" [ style = bold] "B_stop_0 lxc1" -> "all_stopped" [ style = bold] "B_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"] "D_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "M-clone_running_0" [ style=bold color="green" fontcolor="orange"] "M-clone_start_0" -> "M-clone_running_0" [ style = bold] "M-clone_start_0" -> "M_start_0 lxc1" [ style = bold] "M-clone_start_0" [ style=bold color="green" fontcolor="orange"] "M-clone_stop_0" -> "M-clone_stopped_0" [ style = bold] "M-clone_stop_0" -> "M_stop_0 lxc1" [ style = bold] "M-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "M-clone_stopped_0" -> "M-clone_start_0" [ style = bold] "M-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "M_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] "M_start_0 lxc1" -> "M-clone_running_0" [ style = bold] "M_start_0 lxc1" -> "M_monitor_10000 lxc1" [ style = bold] "M_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "M_stop_0 lxc1" -> "M-clone_stopped_0" [ style = bold] "M_stop_0 lxc1" -> "M_start_0 lxc1" [ style = bold] "M_stop_0 lxc1" -> "all_stopped" [ style = bold] "M_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"] +"all_stopped" -> "lxc1_start_0 18node2" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "container1_start_0 18node2" -> "B_start_0 lxc1" [ style = bold] "container1_start_0 18node2" -> "M_start_0 lxc1" [ style = bold] "container1_start_0 18node2" -> "lxc1_start_0 18node2" [ style = bold] "container1_start_0 18node2" [ style=bold color="green" fontcolor="black"] "container1_stop_0 18node2" -> "all_stopped" [ style = bold] "container1_stop_0 18node2" -> "container1_start_0 18node2" [ style = bold] "container1_stop_0 18node2" -> "stonith 'reboot' lxc1" [ style = bold] "container1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_30000 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_start_0 18node2" -> "B_monitor_10000 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "B_start_0 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "M_monitor_10000 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "M_start_0 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "lxc1_monitor_30000 18node2" [ style = bold] "lxc1_start_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_stop_0 18node2" -> "all_stopped" [ style = bold] "lxc1_stop_0 18node2" -> "container1_stop_0 18node2" [ style = bold] "lxc1_stop_0 18node2" -> "lxc1_start_0 18node2" [ style = bold] "lxc1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' lxc1" -> "B_stop_0 lxc1" [ style = bold] "stonith 'reboot' lxc1" -> "M-clone_stop_0" [ style = bold] "stonith 'reboot' lxc1" -> "M_stop_0 lxc1" [ style = bold] "stonith 'reboot' lxc1" -> "stonith_complete" [ style = bold] "stonith 'reboot' lxc1" [ style=bold color="green" fontcolor="orange"] "stonith_complete" -> "B_start_0 lxc1" [ style = bold] "stonith_complete" -> "M_start_0 lxc1" [ style = bold] "stonith_complete" -> "all_stopped" [ style = bold] "stonith_complete" -> "container1_start_0 18node2" [ style = bold] -"stonith_complete" -> "lxc1_start_0 18node2" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/whitebox-fail1.exp b/pengine/test10/whitebox-fail1.exp index 901a1e3023..1532c6eaa4 100644 --- a/pengine/test10/whitebox-fail1.exp +++ b/pengine/test10/whitebox-fail1.exp @@ -1,317 +1,317 @@ - + - + - + diff --git a/pengine/test10/whitebox-fail1.summary b/pengine/test10/whitebox-fail1.summary index d1f34804e4..a5b85dd1ac 100644 --- a/pengine/test10/whitebox-fail1.summary +++ b/pengine/test10/whitebox-fail1.summary @@ -1,59 +1,59 @@ Current cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc2:container2 ] container1 (ocf::heartbeat:VirtualDomain): FAILED 18node2 container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] Started: [ 18node1 18node2 18node3 lxc2 ] A (ocf::pacemaker:Dummy): Started 18node1 B (ocf::pacemaker:Dummy): FAILED lxc1 C (ocf::pacemaker:Dummy): Started lxc2 D (ocf::pacemaker:Dummy): Started 18node1 Transition Summary: * Fence (reboot) lxc1 (resource: container1) 'guest is unclean' * Recover container1 ( 18node2 ) * Recover M:4 ( lxc1 ) * Recover B ( lxc1 ) * Restart lxc1 ( 18node2 ) due to required container1 start Executing cluster transition: * Resource action: A monitor on lxc2 * Resource action: B monitor on lxc2 * Resource action: D monitor on lxc2 * Resource action: lxc1 stop on 18node2 * Resource action: container1 stop on 18node2 * Pseudo action: stonith-lxc1-reboot on lxc1 * Pseudo action: stonith_complete * Resource action: container1 start on 18node2 * Pseudo action: M-clone_stop_0 * Pseudo action: B_stop_0 - * Resource action: lxc1 start on 18node2 - * Resource action: lxc1 monitor=30000 on 18node2 * Pseudo action: M_stop_0 * Pseudo action: M-clone_stopped_0 * Pseudo action: M-clone_start_0 - * Resource action: B start on lxc1 * Pseudo action: all_stopped + * Resource action: lxc1 start on 18node2 + * Resource action: lxc1 monitor=30000 on 18node2 * Resource action: M start on lxc1 * Pseudo action: M-clone_running_0 - * Resource action: B monitor=10000 on lxc1 + * Resource action: B start on lxc1 * Resource action: M monitor=10000 on lxc1 + * Resource action: B monitor=10000 on lxc1 Revised cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc1:container1 lxc2:container2 ] container1 (ocf::heartbeat:VirtualDomain): Started 18node2 container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] Started: [ 18node1 18node2 18node3 lxc1 lxc2 ] A (ocf::pacemaker:Dummy): Started 18node1 B (ocf::pacemaker:Dummy): Started lxc1 C (ocf::pacemaker:Dummy): Started lxc2 D (ocf::pacemaker:Dummy): Started 18node1 diff --git a/pengine/test10/whitebox-fail2.dot b/pengine/test10/whitebox-fail2.dot index bfff4bfb99..9b755f981c 100644 --- a/pengine/test10/whitebox-fail2.dot +++ b/pengine/test10/whitebox-fail2.dot @@ -1,60 +1,60 @@ digraph "g" { "A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "B_monitor_0 lxc2" -> "B_start_0 lxc1" [ style = bold] "B_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "B_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] "B_start_0 lxc1" -> "B_monitor_10000 lxc1" [ style = bold] "B_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "B_stop_0 lxc1" -> "B_start_0 lxc1" [ style = bold] "B_stop_0 lxc1" -> "all_stopped" [ style = bold] "B_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"] "D_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] "M-clone_running_0" [ style=bold color="green" fontcolor="orange"] "M-clone_start_0" -> "M-clone_running_0" [ style = bold] "M-clone_start_0" -> "M_start_0 lxc1" [ style = bold] "M-clone_start_0" [ style=bold color="green" fontcolor="orange"] "M-clone_stop_0" -> "M-clone_stopped_0" [ style = bold] "M-clone_stop_0" -> "M_stop_0 lxc1" [ style = bold] "M-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "M-clone_stopped_0" -> "M-clone_start_0" [ style = bold] "M-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "M_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] "M_start_0 lxc1" -> "M-clone_running_0" [ style = bold] "M_start_0 lxc1" -> "M_monitor_10000 lxc1" [ style = bold] "M_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "M_stop_0 lxc1" -> "M-clone_stopped_0" [ style = bold] "M_stop_0 lxc1" -> "M_start_0 lxc1" [ style = bold] "M_stop_0 lxc1" -> "all_stopped" [ style = bold] "M_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"] +"all_stopped" -> "lxc1_start_0 18node2" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "container1_start_0 18node2" -> "B_start_0 lxc1" [ style = bold] "container1_start_0 18node2" -> "M_start_0 lxc1" [ style = bold] "container1_start_0 18node2" -> "lxc1_start_0 18node2" [ style = bold] "container1_start_0 18node2" [ style=bold color="green" fontcolor="black"] "container1_stop_0 18node2" -> "all_stopped" [ style = bold] "container1_stop_0 18node2" -> "container1_start_0 18node2" [ style = bold] "container1_stop_0 18node2" -> "stonith 'reboot' lxc1" [ style = bold] "container1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_30000 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_start_0 18node2" -> "B_monitor_10000 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "B_start_0 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "M_monitor_10000 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "M_start_0 lxc1" [ style = bold] "lxc1_start_0 18node2" -> "lxc1_monitor_30000 18node2" [ style = bold] "lxc1_start_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_stop_0 18node2" -> "all_stopped" [ style = bold] "lxc1_stop_0 18node2" -> "container1_stop_0 18node2" [ style = bold] "lxc1_stop_0 18node2" -> "lxc1_start_0 18node2" [ style = bold] "lxc1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' lxc1" -> "B_stop_0 lxc1" [ style = bold] "stonith 'reboot' lxc1" -> "M-clone_stop_0" [ style = bold] "stonith 'reboot' lxc1" -> "M_stop_0 lxc1" [ style = bold] "stonith 'reboot' lxc1" -> "stonith_complete" [ style = bold] "stonith 'reboot' lxc1" [ style=bold color="green" fontcolor="orange"] "stonith_complete" -> "B_start_0 lxc1" [ style = bold] "stonith_complete" -> "M_start_0 lxc1" [ style = bold] "stonith_complete" -> "all_stopped" [ style = bold] "stonith_complete" -> "container1_start_0 18node2" [ style = bold] -"stonith_complete" -> "lxc1_start_0 18node2" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/whitebox-fail2.exp b/pengine/test10/whitebox-fail2.exp index 901a1e3023..1532c6eaa4 100644 --- a/pengine/test10/whitebox-fail2.exp +++ b/pengine/test10/whitebox-fail2.exp @@ -1,317 +1,317 @@ - + - + - + diff --git a/pengine/test10/whitebox-fail2.summary b/pengine/test10/whitebox-fail2.summary index ebf6c515e3..afee2618ba 100644 --- a/pengine/test10/whitebox-fail2.summary +++ b/pengine/test10/whitebox-fail2.summary @@ -1,59 +1,59 @@ Current cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc2:container2 ] container1 (ocf::heartbeat:VirtualDomain): FAILED 18node2 container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] Started: [ 18node1 18node2 18node3 lxc2 ] A (ocf::pacemaker:Dummy): Started 18node1 B (ocf::pacemaker:Dummy): FAILED lxc1 C (ocf::pacemaker:Dummy): Started lxc2 D (ocf::pacemaker:Dummy): Started 18node1 Transition Summary: * Fence (reboot) lxc1 (resource: container1) 'guest is unclean' * Recover container1 ( 18node2 ) * Recover M:4 ( lxc1 ) * Recover B ( lxc1 ) * Recover lxc1 ( 18node2 ) Executing cluster transition: * Resource action: A monitor on lxc2 * Resource action: B monitor on lxc2 * Resource action: D monitor on lxc2 * Resource action: lxc1 stop on 18node2 * Resource action: container1 stop on 18node2 * Pseudo action: stonith-lxc1-reboot on lxc1 * Pseudo action: stonith_complete * Resource action: container1 start on 18node2 * Pseudo action: M-clone_stop_0 * Pseudo action: B_stop_0 - * Resource action: lxc1 start on 18node2 - * Resource action: lxc1 monitor=30000 on 18node2 * Pseudo action: M_stop_0 * Pseudo action: M-clone_stopped_0 * Pseudo action: M-clone_start_0 - * Resource action: B start on lxc1 * Pseudo action: all_stopped + * Resource action: lxc1 start on 18node2 + * Resource action: lxc1 monitor=30000 on 18node2 * Resource action: M start on lxc1 * Pseudo action: M-clone_running_0 - * Resource action: B monitor=10000 on lxc1 + * Resource action: B start on lxc1 * Resource action: M monitor=10000 on lxc1 + * Resource action: B monitor=10000 on lxc1 Revised cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc1:container1 lxc2:container2 ] container1 (ocf::heartbeat:VirtualDomain): Started 18node2 container2 (ocf::heartbeat:VirtualDomain): Started 18node2 shoot1 (stonith:fence_xvm): Started 18node3 Clone Set: M-clone [M] Started: [ 18node1 18node2 18node3 lxc1 lxc2 ] A (ocf::pacemaker:Dummy): Started 18node1 B (ocf::pacemaker:Dummy): Started lxc1 C (ocf::pacemaker:Dummy): Started lxc2 D (ocf::pacemaker:Dummy): Started 18node1 diff --git a/pengine/test10/whitebox-imply-stop-on-fence.dot b/pengine/test10/whitebox-imply-stop-on-fence.dot index 7b536ea55a..1ef3cba3d1 100644 --- a/pengine/test10/whitebox-imply-stop-on-fence.dot +++ b/pengine/test10/whitebox-imply-stop-on-fence.dot @@ -1,111 +1,109 @@ digraph "g" { "R-lxc-01_kiff-01_monitor_10000 kiff-02" [ style=bold color="green" fontcolor="black"] "R-lxc-01_kiff-01_start_0 kiff-02" -> "R-lxc-01_kiff-01_monitor_10000 kiff-02" [ style = bold] "R-lxc-01_kiff-01_start_0 kiff-02" -> "lxc-01_kiff-01_start_0 kiff-02" [ style = bold] "R-lxc-01_kiff-01_start_0 kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] "R-lxc-01_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] "R-lxc-01_kiff-01_stop_0 kiff-01" -> "R-lxc-01_kiff-01_start_0 kiff-02" [ style = bold] "R-lxc-01_kiff-01_stop_0 kiff-01" -> "all_stopped" [ style = bold] "R-lxc-01_kiff-01_stop_0 kiff-01" -> "shared0-clone_stop_0" [ style = bold] "R-lxc-01_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "R-lxc-02_kiff-01_monitor_10000 kiff-02" [ style=bold color="green" fontcolor="black"] "R-lxc-02_kiff-01_start_0 kiff-02" -> "R-lxc-02_kiff-01_monitor_10000 kiff-02" [ style = bold] "R-lxc-02_kiff-01_start_0 kiff-02" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] "R-lxc-02_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] "R-lxc-02_kiff-01_stop_0 kiff-01" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] "R-lxc-02_kiff-01_stop_0 kiff-01" -> "all_stopped" [ style = bold] "R-lxc-02_kiff-01_stop_0 kiff-01" -> "shared0-clone_stop_0" [ style = bold] "R-lxc-02_kiff-01_stop_0 kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] "R-lxc-02_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "all_stopped" -> "fence-kiff-02_start_0 kiff-02" [ style = bold] +"all_stopped" -> "lxc-01_kiff-01_start_0 kiff-02" [ style = bold] +"all_stopped" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "clvmd-clone_stop_0" -> "clvmd-clone_stopped_0" [ style = bold] "clvmd-clone_stop_0" -> "clvmd_stop_0 kiff-01" [ style = bold] "clvmd-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "clvmd-clone_stopped_0" -> "dlm-clone_stop_0" [ style = bold] "clvmd-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "clvmd_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] "clvmd_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] "clvmd_stop_0 kiff-01" -> "all_stopped" [ style = bold] "clvmd_stop_0 kiff-01" -> "clvmd-clone_stopped_0" [ style = bold] "clvmd_stop_0 kiff-01" -> "dlm_stop_0 kiff-01" [ style = bold] "clvmd_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "dlm-clone_stop_0" -> "dlm-clone_stopped_0" [ style = bold] "dlm-clone_stop_0" -> "dlm_stop_0 kiff-01" [ style = bold] "dlm-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "dlm-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "dlm_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] "dlm_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] "dlm_stop_0 kiff-01" -> "all_stopped" [ style = bold] "dlm_stop_0 kiff-01" -> "dlm-clone_stopped_0" [ style = bold] "dlm_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "fence-kiff-02_monitor_60000 kiff-02" [ style=bold color="green" fontcolor="black"] "fence-kiff-02_start_0 kiff-02" -> "fence-kiff-02_monitor_60000 kiff-02" [ style = bold] "fence-kiff-02_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] "fence-kiff-02_stop_0 kiff-01" -> "all_stopped" [ style = bold] "fence-kiff-02_stop_0 kiff-01" -> "fence-kiff-02_start_0 kiff-02" [ style = bold] "fence-kiff-02_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "lxc-01_kiff-01_monitor_30000 kiff-02" [ style=bold color="green" fontcolor="black"] "lxc-01_kiff-01_start_0 kiff-02" -> "lxc-01_kiff-01_monitor_30000 kiff-02" [ style = bold] "lxc-01_kiff-01_start_0 kiff-02" -> "vm-fs_monitor_20000 lxc-01_kiff-01" [ style = bold] "lxc-01_kiff-01_start_0 kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] "lxc-01_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] "lxc-01_kiff-01_stop_0 kiff-01" -> "R-lxc-01_kiff-01_stop_0 kiff-01" [ style = bold] "lxc-01_kiff-01_stop_0 kiff-01" -> "all_stopped" [ style = bold] "lxc-01_kiff-01_stop_0 kiff-01" -> "lxc-01_kiff-01_start_0 kiff-02" [ style = bold] "lxc-01_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "lxc-02_kiff-01_monitor_30000 kiff-02" [ style=bold color="green" fontcolor="black"] "lxc-02_kiff-01_start_0 kiff-02" -> "lxc-02_kiff-01_monitor_30000 kiff-02" [ style = bold] "lxc-02_kiff-01_start_0 kiff-02" [ style=bold color="green" fontcolor="black"] "lxc-02_kiff-01_stop_0 kiff-01" -> "R-lxc-02_kiff-01_stop_0 kiff-01" [ style = bold] "lxc-02_kiff-01_stop_0 kiff-01" -> "all_stopped" [ style = bold] "lxc-02_kiff-01_stop_0 kiff-01" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] "lxc-02_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "shared0-clone_stop_0" -> "shared0-clone_stopped_0" [ style = bold] "shared0-clone_stop_0" -> "shared0_stop_0 kiff-01" [ style = bold] "shared0-clone_stop_0" [ style=bold color="green" fontcolor="orange"] "shared0-clone_stopped_0" -> "clvmd-clone_stop_0" [ style = bold] "shared0-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] "shared0_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] "shared0_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] "shared0_stop_0 kiff-01" -> "all_stopped" [ style = bold] "shared0_stop_0 kiff-01" -> "clvmd_stop_0 kiff-01" [ style = bold] "shared0_stop_0 kiff-01" -> "shared0-clone_stopped_0" [ style = bold] "shared0_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' kiff-01" -> "R-lxc-01_kiff-01_stop_0 kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "R-lxc-02_kiff-01_stop_0 kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "clvmd-clone_stop_0" [ style = bold] "stonith 'reboot' kiff-01" -> "clvmd_stop_0 kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "dlm-clone_stop_0" [ style = bold] "stonith 'reboot' kiff-01" -> "dlm_stop_0 kiff-01" [ style = bold] -"stonith 'reboot' kiff-01" -> "lxc-01_kiff-01_stop_0 kiff-01" [ style = bold] -"stonith 'reboot' kiff-01" -> "lxc-02_kiff-01_stop_0 kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "shared0-clone_stop_0" [ style = bold] "stonith 'reboot' kiff-01" -> "shared0_stop_0 kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "stonith 'reboot' lxc-01_kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "stonith 'reboot' lxc-02_kiff-01" [ style = bold] "stonith 'reboot' kiff-01" -> "stonith_complete" [ style = bold] "stonith 'reboot' kiff-01" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' lxc-01_kiff-01" -> "stonith_complete" [ style = bold] "stonith 'reboot' lxc-01_kiff-01" -> "vm-fs_stop_0 lxc-01_kiff-01" [ style = bold] "stonith 'reboot' lxc-01_kiff-01" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' lxc-02_kiff-01" -> "stonith_complete" [ style = bold] "stonith 'reboot' lxc-02_kiff-01" [ style=bold color="green" fontcolor="orange"] "stonith_complete" -> "R-lxc-01_kiff-01_start_0 kiff-02" [ style = bold] "stonith_complete" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] "stonith_complete" -> "all_stopped" [ style = bold] -"stonith_complete" -> "lxc-01_kiff-01_start_0 kiff-02" [ style = bold] -"stonith_complete" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] "stonith_complete" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] "vm-fs_monitor_0 lxc-01_kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] "vm-fs_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] "vm-fs_monitor_0 lxc-02_kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] "vm-fs_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] "vm-fs_monitor_20000 lxc-01_kiff-01" [ style=bold color="green" fontcolor="black"] "vm-fs_start_0 lxc-01_kiff-01" -> "vm-fs_monitor_20000 lxc-01_kiff-01" [ style = bold] "vm-fs_start_0 lxc-01_kiff-01" [ style=bold color="green" fontcolor="black"] "vm-fs_stop_0 lxc-01_kiff-01" -> "all_stopped" [ style = bold] "vm-fs_stop_0 lxc-01_kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] "vm-fs_stop_0 lxc-01_kiff-01" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/whitebox-imply-stop-on-fence.exp b/pengine/test10/whitebox-imply-stop-on-fence.exp index f80dde1ba4..c73d1d291a 100644 --- a/pengine/test10/whitebox-imply-stop-on-fence.exp +++ b/pengine/test10/whitebox-imply-stop-on-fence.exp @@ -1,590 +1,582 @@ - + - + - + - - - - - + - + - + - + - - - - - + diff --git a/pengine/test10/whitebox-imply-stop-on-fence.summary b/pengine/test10/whitebox-imply-stop-on-fence.summary index d272b25d21..5ce580e4a2 100644 --- a/pengine/test10/whitebox-imply-stop-on-fence.summary +++ b/pengine/test10/whitebox-imply-stop-on-fence.summary @@ -1,104 +1,104 @@ Current cluster status: Node kiff-01 (1): UNCLEAN (offline) Online: [ kiff-02 ] Containers: [ lxc-01_kiff-02:R-lxc-01_kiff-02 lxc-02_kiff-02:R-lxc-02_kiff-02 ] fence-kiff-01 (stonith:fence_ipmilan): Started kiff-02 fence-kiff-02 (stonith:fence_ipmilan): Started kiff-01 (UNCLEAN) Clone Set: dlm-clone [dlm] dlm (ocf::pacemaker:controld): Started kiff-01 (UNCLEAN) Started: [ kiff-02 ] Stopped: [ lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] Clone Set: clvmd-clone [clvmd] clvmd (ocf::heartbeat:clvm): Started kiff-01 (UNCLEAN) Started: [ kiff-02 ] Stopped: [ lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] Clone Set: shared0-clone [shared0] shared0 (ocf::heartbeat:Filesystem): Started kiff-01 (UNCLEAN) Started: [ kiff-02 ] Stopped: [ lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] R-lxc-01_kiff-01 (ocf::heartbeat:VirtualDomain): FAILED kiff-01 (UNCLEAN) R-lxc-02_kiff-01 (ocf::heartbeat:VirtualDomain): Started kiff-01 (UNCLEAN) R-lxc-01_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 R-lxc-02_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 vm-fs (ocf::heartbeat:Filesystem): FAILED lxc-01_kiff-01 Transition Summary: * Fence (reboot) lxc-02_kiff-01 (resource: R-lxc-02_kiff-01) 'guest is unclean' * Fence (reboot) lxc-01_kiff-01 (resource: R-lxc-01_kiff-01) 'guest is unclean' * Fence (reboot) kiff-01 'peer is no longer part of the cluster' * Move fence-kiff-02 ( kiff-01 -> kiff-02 ) * Stop dlm:0 (kiff-01) due to node availability * Stop clvmd:0 (kiff-01) due to node availability * Stop shared0:0 (kiff-01) due to node availability * Recover R-lxc-01_kiff-01 ( kiff-01 -> kiff-02 ) * Move R-lxc-02_kiff-01 ( kiff-01 -> kiff-02 ) * Recover vm-fs ( lxc-01_kiff-01 ) * Move lxc-01_kiff-01 ( kiff-01 -> kiff-02 ) * Move lxc-02_kiff-01 ( kiff-01 -> kiff-02 ) Executing cluster transition: * Pseudo action: fence-kiff-02_stop_0 * Resource action: dlm monitor on lxc-02_kiff-02 * Resource action: dlm monitor on lxc-01_kiff-02 * Resource action: clvmd monitor on lxc-02_kiff-02 * Resource action: clvmd monitor on lxc-01_kiff-02 * Resource action: shared0 monitor on lxc-02_kiff-02 * Resource action: shared0 monitor on lxc-01_kiff-02 * Resource action: vm-fs monitor on lxc-02_kiff-02 * Resource action: vm-fs monitor on lxc-01_kiff-02 - * Fencing kiff-01 (reboot) * Pseudo action: lxc-01_kiff-01_stop_0 * Pseudo action: lxc-02_kiff-01_stop_0 + * Fencing kiff-01 (reboot) + * Pseudo action: R-lxc-01_kiff-01_stop_0 + * Pseudo action: R-lxc-02_kiff-01_stop_0 * Pseudo action: stonith-lxc-02_kiff-01-reboot on lxc-02_kiff-01 * Pseudo action: stonith-lxc-01_kiff-01-reboot on lxc-01_kiff-01 * Pseudo action: stonith_complete - * Pseudo action: R-lxc-01_kiff-01_stop_0 - * Pseudo action: R-lxc-02_kiff-01_stop_0 - * Pseudo action: vm-fs_stop_0 * Pseudo action: shared0-clone_stop_0 * Resource action: R-lxc-01_kiff-01 start on kiff-02 * Resource action: R-lxc-02_kiff-01 start on kiff-02 - * Resource action: lxc-01_kiff-01 start on kiff-02 - * Resource action: lxc-02_kiff-01 start on kiff-02 + * Pseudo action: vm-fs_stop_0 * Pseudo action: shared0_stop_0 * Pseudo action: shared0-clone_stopped_0 * Resource action: R-lxc-01_kiff-01 monitor=10000 on kiff-02 * Resource action: R-lxc-02_kiff-01 monitor=10000 on kiff-02 - * Resource action: vm-fs start on lxc-01_kiff-01 - * Resource action: lxc-01_kiff-01 monitor=30000 on kiff-02 - * Resource action: lxc-02_kiff-01 monitor=30000 on kiff-02 * Pseudo action: clvmd-clone_stop_0 - * Resource action: vm-fs monitor=20000 on lxc-01_kiff-01 * Pseudo action: clvmd_stop_0 * Pseudo action: clvmd-clone_stopped_0 * Pseudo action: dlm-clone_stop_0 * Pseudo action: dlm_stop_0 * Pseudo action: dlm-clone_stopped_0 * Pseudo action: all_stopped * Resource action: fence-kiff-02 start on kiff-02 + * Resource action: lxc-01_kiff-01 start on kiff-02 + * Resource action: lxc-02_kiff-01 start on kiff-02 * Resource action: fence-kiff-02 monitor=60000 on kiff-02 + * Resource action: vm-fs start on lxc-01_kiff-01 + * Resource action: lxc-01_kiff-01 monitor=30000 on kiff-02 + * Resource action: lxc-02_kiff-01 monitor=30000 on kiff-02 + * Resource action: vm-fs monitor=20000 on lxc-01_kiff-01 Revised cluster status: Online: [ kiff-02 ] OFFLINE: [ kiff-01 ] Containers: [ lxc-01_kiff-01:R-lxc-01_kiff-01 lxc-01_kiff-02:R-lxc-01_kiff-02 lxc-02_kiff-01:R-lxc-02_kiff-01 lxc-02_kiff-02:R-lxc-02_kiff-02 ] fence-kiff-01 (stonith:fence_ipmilan): Started kiff-02 fence-kiff-02 (stonith:fence_ipmilan): Started kiff-02 Clone Set: dlm-clone [dlm] Started: [ kiff-02 ] Stopped: [ kiff-01 lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] Clone Set: clvmd-clone [clvmd] Started: [ kiff-02 ] Stopped: [ kiff-01 lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] Clone Set: shared0-clone [shared0] Started: [ kiff-02 ] Stopped: [ kiff-01 lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] R-lxc-01_kiff-01 (ocf::heartbeat:VirtualDomain): Started kiff-02 R-lxc-02_kiff-01 (ocf::heartbeat:VirtualDomain): Started kiff-02 R-lxc-01_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 R-lxc-02_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 vm-fs (ocf::heartbeat:Filesystem): Started lxc-01_kiff-01 diff --git a/pengine/test10/whitebox-ms-ordering.dot b/pengine/test10/whitebox-ms-ordering.dot index 1f4d95b1ad..d5112b92de 100644 --- a/pengine/test10/whitebox-ms-ordering.dot +++ b/pengine/test10/whitebox-ms-ordering.dot @@ -1,112 +1,112 @@ digraph "g" { +"all_stopped" -> "lxc1_start_0 18node1" [ style = bold] +"all_stopped" -> "lxc2_start_0 18node1" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "container1_monitor_0 18node1" -> "container1_start_0 18node1" [ style = bold] "container1_monitor_0 18node1" [ style=bold color="green" fontcolor="black"] "container1_monitor_0 18node2" -> "container1_start_0 18node1" [ style = bold] "container1_monitor_0 18node2" [ style=bold color="green" fontcolor="black"] "container1_monitor_0 18node3" -> "container1_start_0 18node1" [ style = bold] "container1_monitor_0 18node3" [ style=bold color="green" fontcolor="black"] "container1_start_0 18node1" -> "lxc-ms_promote_0 lxc1" [ style = bold] "container1_start_0 18node1" -> "lxc-ms_start_0 lxc1" [ style = bold] "container1_start_0 18node1" -> "lxc1_start_0 18node1" [ style = bold] "container1_start_0 18node1" [ style=bold color="green" fontcolor="black"] "container2_monitor_0 18node1" -> "container2_start_0 18node1" [ style = bold] "container2_monitor_0 18node1" [ style=bold color="green" fontcolor="black"] "container2_monitor_0 18node2" -> "container2_start_0 18node1" [ style = bold] "container2_monitor_0 18node2" [ style=bold color="green" fontcolor="black"] "container2_monitor_0 18node3" -> "container2_start_0 18node1" [ style = bold] "container2_monitor_0 18node3" [ style=bold color="green" fontcolor="black"] "container2_start_0 18node1" -> "lxc-ms_start_0 lxc2" [ style = bold] "container2_start_0 18node1" -> "lxc2_start_0 18node1" [ style = bold] "container2_start_0 18node1" [ style=bold color="green" fontcolor="black"] "lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold] "lxc-ms-master_demote_0" -> "lxc-ms_demote_0 lxc1" [ style = bold] "lxc-ms-master_demote_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_demoted_0" -> "lxc-ms-master_promote_0" [ style = bold] "lxc-ms-master_demoted_0" -> "lxc-ms-master_start_0" [ style = bold] "lxc-ms-master_demoted_0" -> "lxc-ms-master_stop_0" [ style = bold] "lxc-ms-master_demoted_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_promote_0" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc-ms-master_promote_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_promoted_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_running_0" -> "lxc-ms-master_promote_0" [ style = bold] "lxc-ms-master_running_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_start_0" -> "lxc-ms-master_running_0" [ style = bold] "lxc-ms-master_start_0" -> "lxc-ms_start_0 lxc1" [ style = bold] "lxc-ms-master_start_0" -> "lxc-ms_start_0 lxc2" [ style = bold] "lxc-ms-master_start_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_stop_0" -> "lxc-ms-master_stopped_0" [ style = bold] "lxc-ms-master_stop_0" -> "lxc-ms_stop_0 lxc1" [ style = bold] "lxc-ms-master_stop_0" -> "lxc-ms_stop_0 lxc2" [ style = bold] "lxc-ms-master_stop_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms-master_stopped_0" -> "lxc-ms-master_promote_0" [ style = bold] "lxc-ms-master_stopped_0" -> "lxc-ms-master_start_0" [ style = bold] "lxc-ms-master_stopped_0" [ style=bold color="green" fontcolor="orange"] "lxc-ms_demote_0 lxc1" -> "lxc-ms-master_demoted_0" [ style = bold] "lxc-ms_demote_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc-ms_demote_0 lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold] "lxc-ms_demote_0 lxc1" [ style=bold color="green" fontcolor="orange"] "lxc-ms_monitor_0 18node1" -> "lxc-ms-master_start_0" [ style = bold] "lxc-ms_monitor_0 18node1" [ style=bold color="green" fontcolor="black"] "lxc-ms_monitor_0 18node2" -> "lxc-ms-master_start_0" [ style = bold] "lxc-ms_monitor_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc-ms_monitor_0 18node3" -> "lxc-ms-master_start_0" [ style = bold] "lxc-ms_monitor_0 18node3" [ style=bold color="green" fontcolor="black"] "lxc-ms_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"] "lxc-ms_promote_0 lxc1" -> "lxc-ms-master_promoted_0" [ style = bold] "lxc-ms_promote_0 lxc1" [ style=bold color="green" fontcolor="black"] "lxc-ms_start_0 lxc1" -> "lxc-ms-master_running_0" [ style = bold] "lxc-ms_start_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc-ms_start_0 lxc1" [ style=bold color="green" fontcolor="black"] "lxc-ms_start_0 lxc2" -> "lxc-ms-master_running_0" [ style = bold] "lxc-ms_start_0 lxc2" -> "lxc-ms_monitor_10000 lxc2" [ style = bold] "lxc-ms_start_0 lxc2" [ style=bold color="green" fontcolor="black"] "lxc-ms_stop_0 lxc1" -> "all_stopped" [ style = bold] "lxc-ms_stop_0 lxc1" -> "lxc-ms-master_stopped_0" [ style = bold] "lxc-ms_stop_0 lxc1" -> "lxc-ms_start_0 lxc1" [ style = bold] "lxc-ms_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"] "lxc-ms_stop_0 lxc2" -> "all_stopped" [ style = bold] "lxc-ms_stop_0 lxc2" -> "lxc-ms-master_stopped_0" [ style = bold] "lxc-ms_stop_0 lxc2" -> "lxc-ms_start_0 lxc2" [ style = bold] "lxc-ms_stop_0 lxc2" [ style=bold color="green" fontcolor="orange"] "lxc1_monitor_0 18node1" -> "lxc1_start_0 18node1" [ style = bold] "lxc1_monitor_0 18node1" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_0 18node2" -> "lxc1_start_0 18node1" [ style = bold] "lxc1_monitor_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_0 18node3" -> "lxc1_start_0 18node1" [ style = bold] "lxc1_monitor_0 18node3" [ style=bold color="green" fontcolor="black"] "lxc1_monitor_30000 18node1" [ style=bold color="green" fontcolor="black"] "lxc1_start_0 18node1" -> "lxc-ms_promote_0 lxc1" [ style = bold] "lxc1_start_0 18node1" -> "lxc-ms_start_0 lxc1" [ style = bold] "lxc1_start_0 18node1" -> "lxc1_monitor_30000 18node1" [ style = bold] "lxc1_start_0 18node1" [ style=bold color="green" fontcolor="black"] "lxc2_monitor_0 18node1" -> "lxc2_start_0 18node1" [ style = bold] "lxc2_monitor_0 18node1" [ style=bold color="green" fontcolor="black"] "lxc2_monitor_0 18node2" -> "lxc2_start_0 18node1" [ style = bold] "lxc2_monitor_0 18node2" [ style=bold color="green" fontcolor="black"] "lxc2_monitor_0 18node3" -> "lxc2_start_0 18node1" [ style = bold] "lxc2_monitor_0 18node3" [ style=bold color="green" fontcolor="black"] "lxc2_monitor_30000 18node1" [ style=bold color="green" fontcolor="black"] "lxc2_start_0 18node1" -> "lxc-ms_monitor_10000 lxc2" [ style = bold] "lxc2_start_0 18node1" -> "lxc-ms_start_0 lxc2" [ style = bold] "lxc2_start_0 18node1" -> "lxc2_monitor_30000 18node1" [ style = bold] "lxc2_start_0 18node1" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' lxc1" -> "lxc-ms-master_stop_0" [ style = bold] "stonith 'reboot' lxc1" -> "lxc-ms_demote_0 lxc1" [ style = bold] "stonith 'reboot' lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold] "stonith 'reboot' lxc1" -> "stonith_complete" [ style = bold] "stonith 'reboot' lxc1" [ style=bold color="green" fontcolor="orange"] "stonith 'reboot' lxc2" -> "lxc-ms-master_stop_0" [ style = bold] "stonith 'reboot' lxc2" -> "lxc-ms_stop_0 lxc2" [ style = bold] "stonith 'reboot' lxc2" -> "stonith_complete" [ style = bold] "stonith 'reboot' lxc2" [ style=bold color="green" fontcolor="orange"] "stonith_complete" -> "all_stopped" [ style = bold] "stonith_complete" -> "container1_start_0 18node1" [ style = bold] "stonith_complete" -> "container2_start_0 18node1" [ style = bold] "stonith_complete" -> "lxc-ms_promote_0 lxc1" [ style = bold] "stonith_complete" -> "lxc-ms_start_0 lxc1" [ style = bold] "stonith_complete" -> "lxc-ms_start_0 lxc2" [ style = bold] -"stonith_complete" -> "lxc1_start_0 18node1" [ style = bold] -"stonith_complete" -> "lxc2_start_0 18node1" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/whitebox-ms-ordering.exp b/pengine/test10/whitebox-ms-ordering.exp index c8fee5e6c0..d5608e4456 100644 --- a/pengine/test10/whitebox-ms-ordering.exp +++ b/pengine/test10/whitebox-ms-ordering.exp @@ -1,585 +1,585 @@ + + + - - - + + + - - - diff --git a/pengine/test10/whitebox-ms-ordering.summary b/pengine/test10/whitebox-ms-ordering.summary index d8ff62c505..46fe9d1bb2 100644 --- a/pengine/test10/whitebox-ms-ordering.summary +++ b/pengine/test10/whitebox-ms-ordering.summary @@ -1,73 +1,73 @@ Current cluster status: Online: [ 18node1 18node2 18node3 ] shooter (stonith:fence_xvm): Started 18node2 container1 (ocf::heartbeat:VirtualDomain): FAILED container2 (ocf::heartbeat:VirtualDomain): FAILED Master/Slave Set: lxc-ms-master [lxc-ms] Stopped: [ 18node1 18node2 18node3 ] Transition Summary: * Fence (reboot) lxc2 (resource: container2) 'guest is unclean' * Fence (reboot) lxc1 (resource: container1) 'guest is unclean' * Start container1 (18node1) * Start container2 (18node1) * Recover lxc-ms:0 (Master lxc1) * Recover lxc-ms:1 (Slave lxc2) * Start lxc1 (18node1) * Start lxc2 (18node1) Executing cluster transition: * Resource action: container1 monitor on 18node3 * Resource action: container1 monitor on 18node2 * Resource action: container1 monitor on 18node1 * Resource action: container2 monitor on 18node3 * Resource action: container2 monitor on 18node2 * Resource action: container2 monitor on 18node1 * Resource action: lxc-ms monitor on 18node3 * Resource action: lxc-ms monitor on 18node2 * Resource action: lxc-ms monitor on 18node1 * Pseudo action: lxc-ms-master_demote_0 * Resource action: lxc1 monitor on 18node3 * Resource action: lxc1 monitor on 18node2 * Resource action: lxc1 monitor on 18node1 * Resource action: lxc2 monitor on 18node3 * Resource action: lxc2 monitor on 18node2 * Resource action: lxc2 monitor on 18node1 * Pseudo action: stonith-lxc2-reboot on lxc2 * Pseudo action: stonith-lxc1-reboot on lxc1 * Pseudo action: stonith_complete * Resource action: container1 start on 18node1 * Resource action: container2 start on 18node1 * Pseudo action: lxc-ms_demote_0 * Pseudo action: lxc-ms-master_demoted_0 * Pseudo action: lxc-ms-master_stop_0 - * Resource action: lxc1 start on 18node1 - * Resource action: lxc2 start on 18node1 * Pseudo action: lxc-ms_stop_0 * Pseudo action: lxc-ms_stop_0 * Pseudo action: lxc-ms-master_stopped_0 * Pseudo action: lxc-ms-master_start_0 - * Resource action: lxc1 monitor=30000 on 18node1 - * Resource action: lxc2 monitor=30000 on 18node1 * Pseudo action: all_stopped + * Resource action: lxc1 start on 18node1 + * Resource action: lxc2 start on 18node1 * Resource action: lxc-ms start on lxc1 * Resource action: lxc-ms start on lxc2 * Pseudo action: lxc-ms-master_running_0 + * Resource action: lxc1 monitor=30000 on 18node1 + * Resource action: lxc2 monitor=30000 on 18node1 * Resource action: lxc-ms monitor=10000 on lxc2 * Pseudo action: lxc-ms-master_promote_0 * Resource action: lxc-ms promote on lxc1 * Pseudo action: lxc-ms-master_promoted_0 Revised cluster status: Online: [ 18node1 18node2 18node3 ] Containers: [ lxc1:container1 lxc2:container2 ] shooter (stonith:fence_xvm): Started 18node2 container1 (ocf::heartbeat:VirtualDomain): Started 18node1 container2 (ocf::heartbeat:VirtualDomain): Started 18node1 Master/Slave Set: lxc-ms-master [lxc-ms] Masters: [ lxc1 ] Slaves: [ lxc2 ] diff --git a/pengine/test10/whitebox-unexpectedly-running.dot b/pengine/test10/whitebox-unexpectedly-running.dot index fa1171ec83..f16e705eb2 100644 --- a/pengine/test10/whitebox-unexpectedly-running.dot +++ b/pengine/test10/whitebox-unexpectedly-running.dot @@ -1,26 +1,28 @@ digraph "g" { "FAKE-crashed_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"] "FAKE-crashed_start_0 18builder" -> "FAKE-crashed_monitor_60000 18builder" [ style = bold] "FAKE-crashed_start_0 18builder" -> "remote2_start_0 18builder" [ style = bold] "FAKE-crashed_start_0 18builder" [ style=bold color="green" fontcolor="black"] "FAKE-crashed_stop_0 18builder" -> "FAKE-crashed_start_0 18builder" [ style = bold] "FAKE-crashed_stop_0 18builder" -> "all_stopped" [ style = bold] "FAKE-crashed_stop_0 18builder" -> "stonith 'reboot' remote2" [ style = bold] "FAKE-crashed_stop_0 18builder" [ style=bold color="green" fontcolor="black"] "FAKE_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"] +"all_stopped" -> "remote1_start_0 18builder" [ style = bold] +"all_stopped" -> "remote2_start_0 18builder" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "remote1_monitor_0 18builder" -> "remote1_start_0 18builder" [ style = bold] "remote1_monitor_0 18builder" [ style=bold color="green" fontcolor="black"] "remote1_monitor_30000 18builder" [ style=bold color="green" fontcolor="black"] "remote1_start_0 18builder" -> "remote1_monitor_30000 18builder" [ style = bold] "remote1_start_0 18builder" [ style=bold color="green" fontcolor="black"] "remote2_monitor_0 18builder" -> "remote2_start_0 18builder" [ style = bold] "remote2_monitor_0 18builder" [ style=bold color="green" fontcolor="black"] "remote2_monitor_30000 18builder" [ style=bold color="green" fontcolor="black"] "remote2_start_0 18builder" -> "remote2_monitor_30000 18builder" [ style = bold] "remote2_start_0 18builder" [ style=bold color="green" fontcolor="black"] "stonith 'reboot' remote2" -> "stonith_complete" [ style = bold] "stonith 'reboot' remote2" [ style=bold color="green" fontcolor="orange"] "stonith_complete" -> "all_stopped" [ style = bold] "stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/whitebox-unexpectedly-running.exp b/pengine/test10/whitebox-unexpectedly-running.exp index c4e13b93c0..46376a5b9e 100644 --- a/pengine/test10/whitebox-unexpectedly-running.exp +++ b/pengine/test10/whitebox-unexpectedly-running.exp @@ -1,161 +1,167 @@ + + + + + +